Designing a slowly changing dimension type 2 script with postgresql

user_3749836 0
Viewed 7626

Let's say I have the following target table:

CREATE TABLE DimCustomer (
CustomerKey serial PRIMARY KEY,
    CustomerNum int NOT NULL,
    CustomerName varchar(25) NOT NULL,
    Planet varchar(25) NOT NULL,
    RowIsCurrent char(1) NOT NULL DEFAULT 'Y',
    RowStartDate date NOT NULL DEFAULT CURRENT_TIMESTAMP,
    RowEndDate date NOT NULL DEFAULT '12/31/9999'
);

INSERT INTO DimCustomer
(CustomerNum, CustomerName, Planet,  RowStartDate) 
VALUES (101,'Anakin Skywalker', 'Tatooine',   CURRENT_TIMESTAMP - INTERVAL '101 days'),
       (102,'Yoda', 'Coruscant',  CURRENT_TIMESTAMP - INTERVAL '100 days'),
       (103,'Obi-Wan Kenobi', 'Coruscant',  CURRENT_TIMESTAMP - INTERVAL '100 days')

And I have a following staging table:

CREATE TABLE Staging_DimCustomer
(
    CustomerNum int NOT NULL,
    CustomerName varchar(25) NOT NULL,
    Planet varchar(25) NOT NULL,
    ChangeDate date NOT NULL DEFAULT CURRENT_TIMESTAMP,
    RankNo int NOT NULL DEFAULT 1
)
INSERT INTO Staging_DimCustomer(CustomerNum, CustomerName, Planet, ChangeDate)
VALUES
(103,'Ben Kenobi', 'Coruscant',   CURRENT_TIMESTAMP - INTERVAL '99 days')

In the staging table, it looks like 'Obi-Wan Kenobi'(customernum 103) changed his name to 'Ben Kenobi'. I want to create a script that implements scd type 2 and produces the following result(slowly changing dimension type 2):

enter image description here

Following is my attempt:

INSERT INTO DimCustomer (
  CustomerNum, CustomerName, Planet, RowIsCurrent, RowStartDate, RowEndDate
  ) 
 select CustomerNum, CustomerName, Planet, 'Y', ChangeDate, '12/31/9999'
 from Staging_DimCustomer 

 ON CONFLICT (CustomerNum) and RowIsCurrent = 'Y'
  DO UPDATE SET
    CustomerName = EXCLUDED.CustomerName,
    Planet = EXCLUDED.Planet,
    RowIsCurrent = 'N',
    RowEndDate = EXCLUDED.ChangeDate

I dont know how to look for the values that changed, update the existing rows to retire it and then insert the new rows with rowiscurrent = 'Y' flag. I am trying to model my solution based on this sql server article http://www.made2mentor.com/2013/08/how-to-load-slowly-changing-dimensions-using-t-sql-merge/.

dimensional-modeling postgresql sql
2 Answers

Assuming the changes are all on the most current row, then you can update the current row and then insert:

with u as (
      update dimCustomer c
          set RowIsCurrent = 'N',
              RowEndDate = sc.ChangeDate
      from Staging_DimCustomer sc
      where sc.CustomerNum = c.CustomerNum and
            c.RowIsCurrent = 'Y'
     )
insert into dimCustomer (CustomerNum, CustomerName, Planet, RowIsCurrent, RowStartDate, RowEndDate
                         ) 
     select CustomerNum, CustomerName, Planet, 'Y', ChangeDate, '9999-12-31'::date
     from Staging_DimCustomer sc;

This assumes that the changes take place on the most current record. It is rather trickier to implement historic changes, and I'm guessing that is not necessary.

Note that you might want an additional check that the row being inserted is actually different from the current row.

EDIT:

If you want to avoid changes for rows that already exist, you can do:

with sc as (
      select *
      from Staging_DimCustomer
      where not exists (select 1
                        from DimCustomer c
                        where c.CustomerNum = sc.CustomerNum and
                              c.CustomerName = sc.CustomerName and
                              . . .  -- whatever other columns you want to check
                      )
     ),
     u as (
      update dimCustomer c
          set RowIsCurrent = 'N',
              RowEndDate = sc.ChangeDate
      from sc
      where sc.CustomerNum = c.CustomerNum and
            c.RowIsCurrent = 'Y'
     )
insert into dimCustomer (CustomerNum, CustomerName, Planet, RowIsCurrent, RowStartDate, RowEndDate
                         ) 
     select CustomerNum, CustomerName, Planet, 'Y', ChangeDate, '9999-12-31'::date
     from sc;
user_11440350

I think this should work fine , not updating or inserting the already existing records:

with us as (
  update dimCustomer c
      set RowIsCurrent = 'N',
          RowEndDate = sc.ChangeDate
  from Staging_DimCustomer sc
  where sc.CustomerNum = c.CustomerNum and
        c.RowIsCurrent = 'Y' and 
        sc.customername <> c.customername
 ),
 u as (
 select stg.customernum,stg.customername,stg.planet ,stg.changedate from Staging_DimCustomer  stg
 Inner join  DimCustomer dim on dim.customernum=stg.customernum and dim.rowiscurrent='Y'
 and (dim.customername <> stg.customername
      or dim.planet <> stg.planet
      )
 UNION
    select stg.customernum,stg.customername,stg.planet ,stg.changedate from Staging_DimCustomer  stg
 where  stg.customernum not IN(select dim.customernum  from DimCustomer dim where dim.rowiscurrent='Y')
 )
insert into dimCustomer (CustomerNum, CustomerName, Planet, RowIsCurrent, RowStartDate, RowEndDate
                     ) 
select CustomerNum, CustomerName, Planet, 'Y', ChangeDate, '9999-12-31'::date
 from  u ;
user_142058680
Related