Horrible performance and millions of rows to compress.

Question

Hello. I wrote a script to compress a table that's readings from a computer terminal. I read the original table then insert a row in a new table for the beginning of a new reading and then update a column for the and end time and amount of the duplicate readings. The compression is working fine but as I'm testing and increasing the rows the longer it is taking to run. There are 11,000,000 rows to compress and it's hanging after I try and do anything over 20,000. I don't know if it's hanging or taking a long time to start. I put in checks to see when it's running and committing and when I have a small amount of records it takes a minute then starts and run through fast. When I add a more records it just site there.
Any suggestions?
set serveroutput on;
set serveroutput on size 1000000;
truncate table dms_am_nrt_test;
declare
rel_counter number := 1;
row_counter number := 0;
curr_station number;
curr_rel_conc number := 0;
begin_date date;
station number;
test varchar2(5);
curr_index number;
trip number := 5000;
counter number := 1;
cursor c1 is Select *
     From
          gilbert_r.loop_test;
begin
select station_num into station from gilbert_r.loop_test
where rownum = 1;
curr_station := station;
for c1_rec in c1 loop
select station_num into station from gilbert_r.loop_test
where nrt_temp_id = c1_rec.nrt_temp_id;
if curr_station != c1_rec.station_num
then
row_counter := 0;
curr_station := station;
end if;
if c1_rec.rel_conc > 0 or row_counter = 0 or (curr_rel_conc > 0 and c1_rec.rel_conc = 0)
then
curr_rel_conc := c1_rec.rel_conc;
rel_counter := 1;
begin_date := c1_rec.begin_datetime;
select dms_am_nrt_seq.nextval into curr_index from dual;
Insert into site.dms_am_nrt_test
               (nrt_id,
                struc_record_id,
                begin_datetime,
                end_date_time,
                num_readings,
                station_num,
                port_num,
                port_loc,
                agent,
                abs_conc,
                rel_conc,
                units,
                height,
                area,
                rt,
                peak_width,
                 station_status,
                 alarm,
                error,
                error_code1,
                error_code2,
                error_code3,
                error_code4,
                error_code5,
                flow_rate)
               values
               (curr_index,
                c1_rec.Struc_Record_ID,
                begin_date,
                begin_date,
                 rel_counter,
                 c1_rec.Station_Num,
                 c1_rec.Port_Num,
                 c1_rec.Port_Loc,
            c1_rec.Agent,
                 c1_rec.Abs_Conc,
                 c1_rec.Rel_Conc,
                 c1_rec.Units,
                 c1_rec.Height,
                 c1_rec.Area,
                 c1_rec.RT,
                 c1_rec.Peak_Width,
                 trim(c1_rec.Station_Status),
                 c1_rec.Alarm,
                 trim(c1_rec.Error),
                 trim(c1_rec.Error_Code1),
                 trim(c1_rec.Error_Code2),
                 trim(c1_rec.Error_Code3),
                 trim(c1_rec.Error_Code4),
                 trim(c1_rec.Error_Code5),
                  c1_rec.Flow_Rate);
else if
c1_rec.rel_conc = 0 and curr_rel_conc = 0
then
rel_counter := rel_counter + 1;
curr_rel_conc := c1_rec.rel_conc;
begin_date := c1_rec.begin_datetime;
Update site.dms_am_nrt_test
               set
                    end_date_time = begin_date,
                    num_readings = rel_counter
               where
                    nrt_id = curr_index;
end if;
end if;
if counter = trip
then
commit;
trip := trip + 5000;
dbms_output.put_line('Commit');
end if;
row_counter := row_counter + 1;
counter := counter + 1;
end loop;
commit;
end;Message was edited by:
SightSeeker1
Message was edited by:
SightSeeker1
Message was edited by:
SightSeeker1

Answer 1

Hello
Well, after a bit of playing around, and looking at this ask tom article, I've come up with this:
create table dt_test_loop (station number, begin_time date, rel_conc number(3,2))
insert into dt_test_loop values(1, to_date('12:00','hh24:mi'), 0 );
insert into dt_test_loop values(1, to_date('12:01','hh24:mi'), 0);
insert into dt_test_loop values(1, to_date('12:02','hh24:mi'), 0);
insert into dt_test_loop values(1, to_date('12:03','hh24:mi'), 3.3);
insert into dt_test_loop values(1, to_date('12:04','hh24:mi'), 0);
insert into dt_test_loop values(2, to_date('12:00','hh24:mi'), 0 );
insert into dt_test_loop values(2, to_date('12:01','hh24:mi'), 0);
insert into dt_test_loop values(2, to_date('12:02','hh24:mi'), 0);
insert into dt_test_loop values(2, to_date('12:03','hh24:mi'), 4.2);
insert into dt_test_loop values(1, to_date('12:04','hh24:mi'), 0);
insert into dt_test_loop values(1, to_date('12:05','hh24:mi'), 0);
select
     station,
     begin_time,
     end_time,
     rel_conc,
     num_readings
FROM
     SELECT
          station,
          min(begin_time) over(partition by max_rn order by max_rn) begin_time,
          max(begin_time) over(partition by max_rn order by max_rn) end_time,
          rel_conc,
          count(*) over(partition by max_rn order by max_rn) num_readings,
          row_number() over(partition by max_rn order by max_rn) rn,
          max_rn
     FROM
          SELECT
               station,
               rel_conc,
               begin_time,
               max(rn) over(order by station,begin_time) max_rn
          FROM
               (select
                    rel_conc,
                    station,
                    begin_time,
                    case
                         when      rel_conc <> lag(rel_conc) over (order by station,begin_time) OR
                              station <> lag(station ) over (order by station,begin_time) then
                              row_number() over (order by station,begin_time)
                         when row_number() over (order by station,begin_time) = 1 then 1
                    else
                              null
                    end rn
               from
                    dt_test_loop
WHERE
     rn = 1Which gives:
  STATION BEGIN END_T  REL_CONC NUM_READINGS
        1 12:00 12:02         0            3
        1 12:03 12:03       3.3            1
        1 12:04 12:05         0            3
        2 12:00 12:02         0            3
        2 12:03 12:03       4.2            1As you can see, it's slightly wrong in that the readings for station 1 have all been grouped together from 12:04->12:05. The reason for this is the
over (order by station,begin_time)
part. What you really need is another column (which hopefully you have) that records the sequence of readings i.e. an insert timestamp or sequence number. If you can use that instead of station adn begin time, you are rocking! :-)...
create table dt_test_loop (station number, begin_time date, rel_conc number(3,2), ins_seq number)
insert into dt_test_loop values(1, to_date('12:00','hh24:mi'), 0 ,1);
insert into dt_test_loop values(1, to_date('12:01','hh24:mi'), 0,2);
insert into dt_test_loop values(1, to_date('12:02','hh24:mi'), 0,3);
insert into dt_test_loop values(1, to_date('12:03','hh24:mi'), 3.3,4);
insert into dt_test_loop values(1, to_date('12:04','hh24:mi'), 0,5);
insert into dt_test_loop values(2, to_date('12:00','hh24:mi'), 0,6 );
insert into dt_test_loop values(2, to_date('12:01','hh24:mi'), 0,7);
insert into dt_test_loop values(2, to_date('12:02','hh24:mi'), 0,8);
insert into dt_test_loop values(2, to_date('12:03','hh24:mi'), 4.2,9);
insert into dt_test_loop values(1, to_date('12:04','hh24:mi'), 0,10);
insert into dt_test_loop values(1, to_date('12:05','hh24:mi'), 0,11);
select
     station,
     begin_time,
     end_time,
     rel_conc,
     num_readings
FROM
     SELECT
          station,
          min(begin_time) over(partition by max_rn order by max_rn) begin_time,
          max(begin_time) over(partition by max_rn order by max_rn) end_time,
          rel_conc,
          count(*) over(partition by max_rn order by max_rn) num_readings,
          row_number() over(partition by max_rn order by max_rn) rn,
          max_rn
     FROM
          SELECT
               station,
               rel_conc,
               begin_time,
               max(rn) over(order by ins_seq ) max_rn
          FROM
               (select
                    rel_conc,
                    station,
                    begin_time,
                    ins_seq,
                    case
                         when      rel_conc <> lag(rel_conc) over (order by ins_seq ) OR
                              station <> lag(station ) over (order by ins_seq ) then
                              row_number() over (order by ins_seq )
                         when row_number() over (order by ins_seq ) = 1 then 1
                    else
                              null
                    end rn
               from
                    dt_test_loop
WHERE
     rn = 1Which gives:
  STATION BEGIN END_T  REL_CONC NUM_READINGS
        1 12:00 12:02         0            3
        1 12:03 12:03       3.3            1
        1 12:04 12:04         0            1
        2 12:00 12:02         0            3
        2 12:03 12:03       4.2            1
        1 12:04 12:05         0            2Also, I'm sure it can be simplified a bit more, but that's what I got....:-)
HTH
David