Parallel Execution in Oracle

Question

Hi,
I have source data which keeps on populating.I want to process the records parallely using Oracle Stored Procedure.I should not reprocess the records that are already processed.Please suggest an logic.
Right now i have only option of calling the procedure multiple times in unix environment.
Regards,
Venkat

Answer 1

An example of how to create a worloads table that enables target table (to process) to be chunked, and processed.
SQL> --// we have a table with a PK that is not a gap-free
SQL> --// sequential number
SQL> create table sample_data(
  2          id              primary key,
  3          last_update,
  4          name
  5  ) organization index
  6  nologging as
  7  select
  8          object_id,
  9          sysdate,
10          object_name
11  from       all_objects
12  /
Table created.
SQL>
SQL> --// first couple of rows - as can be seen, we cannot
SQL> --// use id column ranges as the 1-1000 range could be
SQL> --// 10 rows, the 1001-2000 range 1 row, and the 2001-3000
SQL> --// range 900 rows.
SQL> select * from(
  2          select * from sample_data order by 1
  3  ) where rownum < 11;
        ID LAST_UPDATE         NAME
       100 2013/07/08 09:10:01 ORA$BASE
       116 2013/07/08 09:10:01 DUAL
       117 2013/07/08 09:10:01 DUAL
       280 2013/07/08 09:10:01 MAP_OBJECT
       365 2013/07/08 09:10:01 SYSTEM_PRIVILEGE_MAP
       367 2013/07/08 09:10:01 SYSTEM_PRIVILEGE_MAP
       368 2013/07/08 09:10:01 TABLE_PRIVILEGE_MAP
       370 2013/07/08 09:10:01 TABLE_PRIVILEGE_MAP
       371 2013/07/08 09:10:01 STMT_AUDIT_OPTION_MAP
       373 2013/07/08 09:10:01 STMT_AUDIT_OPTION_MAP
10 rows selected.
SQL>
SQL> --// we create a workloads table - we'll use this to
SQL> --// hand out work to a parallel process
SQL> create table workloads(
  2          workload_name   varchar2(30),
  3          workload_id     number,
  4          workload_data   number,
  5          constraint pk_workloads primary key
  6          ( workload_name, workload_id )
  7  ) organization index
  8  /
Table created.
SQL>
SQL> --// we create the workloads (1 workload per rows in
SQL> --// this example) for processing the sample_data table
SQL> insert into workloads
  2  select
  3          'SampleData1',
  4          rownum,
  5          id
  6  from       sample_data;
57365 rows created.
SQL> commit;
Commit complete.
SQL>
SQL> --// we can now chunk the SampleData1 workload using the
SQL> --// workload_id as it is a sequential gap free number to
SQL> --// use for even distribution of work
SQL> col VALUE format 999,999,999
SQL> select 'Start at ' as "LABEL", min(workload_id) as  "VALUE" from workloads where workload_name = 'SampleData1'
  2  union all
  3  select 'End at ', max(workload_id) from workloads where workload_name = 'SampleData1'
  4  union all
  5  select 'For rowcount ', count(*) from workloads where workload_name = 'SampleData1'
  6  /
LABEL                VALUE
Start at                 1
End at              57,365
For rowcount        57,365
SQL>
SQL> --// for example, we want to create 10 workload buckets and
SQL> --// fill each with a range of workload identifiers
SQL> with workload_totals( start_id, end_id ) as(
  2          select min(workload_id), max(workload_id) from workloads where workload_name = 'SampleData1'
  3  ),
  4  buckets_needed( b ) as(
  5          select ceil(end_id/10) from workload_totals
  6  ),
  7  buckets( workload_id, bucket) as (
  8          select workload_id, ceil(workload_id/b) from workloads, buckets_needed where workload_name = 'SampleData1'
  9          order by 1
10  )
11  select
12          row_number() over(order by min(workload_id))    as BUCKET,
13          min(workload_id)                                as START_WORLOAD_ID,
14          max(workload_id)                                as END_WORKLOAD_ID
15  from       buckets
16  group by bucket
17  order by 1
18  /
    BUCKET START_WORLOAD_ID END_WORKLOAD_ID
         1                1            5737
         2             5738           11474
         3            11475           17211
         4            17212           22948
         5            22949           28685
         6            28686           34422
         7            34423           40159
         8            40160           45896
         9            45897           51633
        10            51634           57365
10 rows selected.
SQL>
SQL> --// we need now a procedure that will work as a thread and be called, in
SQL> --// parallel, to process a workload bucket
SQL> create or replace procedure ProcessWorkload(
  2          workloadName    varchar2,
  3          startID         number,
  4          endID           number
  5  ) is
  6  begin
  7          --// we make this simple - we read the workload data from
  8          --// workloads table and do an update of sample data
  9          update(
10                  select
11                          s.*
12                  from    sample_data s
13                  where   s.id in(
14                          select
15                                  w.workload_data
16                          from    workloads w
17                          where   w.workload_name = 'SampleData1'
18                          and     w.workload_id between startID and endID
19                  )
20          )
21          set     name = lower(name),
22                  last_update = sysdate;
23  end;
24  /
Procedure created.
SQL>
SQL> --// process sample_data in parallel via 10 workload buckets, using 2
SQL> --// parallel processes
SQL> var c refcursor
SQL> declare
  2          taskName        varchar2(30);
  3  begin
  4          taskName := 'Parallel-PQ-Process1';
  5          DBMS_PARALLEL_EXECUTE.create_task( taskName );
  6
  7          --// we use our SQL above to create 10 buckets, with each bucket
  8          --// (or chunk) specifying the start and end workload id's to
  9          --// process
10          DBMS_PARALLEL_EXECUTE.create_chunks_by_sql(
11                  task_name => taskName,
12                  sql_stmt =>
13                          'with workload_totals( start_id, end_id ) as(
14                                  select min(workload_id), max(workload_id) from workloads where workload_name = ''SampleData1''
15                          ),
16                          buckets_needed( b ) as(
17                                  select ceil(end_id/10) from workload_totals
18                          ),
19                          buckets( workload_id, bucket) as (
20                                  select workload_id, ceil(workload_id/b) from workloads, buckets_needed where workload_name = ''SampleData1''
21                                  order by 1
22                          )
23                          select
24                                  min(workload_id),
25                                  max(workload_id)
26                          from    buckets
27                          group by bucket ',
28                  by_rowid => false
29          );
30
31          --// next we process the 10 buckets/chunks, two at a time
32          DBMS_PARALLEL_EXECUTE.Run_Task(
33                  task_name => taskName,
34                  sql_stmt => 'begin ProcessWorkload(''SampleData1'',:start_id,:end_id); end;',
35                  language_flag => DBMS_SQL.NATIVE,
36                  parallel_level => 2
37          );
38
39          --// wait for it to complete
40          while DBMS_PARALLEL_EXECUTE.task_status( taskName ) != DBMS_PARALLEL_EXECUTE.Finished loop
41                  DBMS_LOCK.Sleep(10);
42          end loop;
43
44          --// stats cursor
45          open :c for
46          select
47                  task_name,
48                  job_name,
49                  chunk_id,
50                  status,
51                  start_id,
52                  end_id,
53                  end_id-start_id                         as UPDATES_DONE,
54                  to_char(start_ts,'hh24:mi:ss.ff')       as START_TS,
55                  to_char(end_ts,'hh24:mi:ss.ff')         as END_TS,
56                  end_ts-start_ts                         as DURATION
57          from    user_parallel_execute_chunks
58          where   task_name = taskName
59          order by 1,2;
60
61          --// remove task
62          DBMS_PARALLEL_EXECUTE.drop_task( taskName );
63  end;
64  /
PL/SQL procedure successfully completed.
SQL>
SQL> col TASK_NAME format a20
SQL> col JOB_NAME format a20
SQL> col START_TS format a15
SQL> col END_TS format a15
SQL> col DURATION format a28
SQL> print c
TASK_NAME            JOB_NAME               CHUNK_ID STATUS       START_ID     END_ID UPDATES_DONE START_TS        END_TS          DURATION
Parallel-PQ-Process1 TASK$_15266_1              5821 PROCESSED           1       5737         5736 09:10:19.066131 09:10:19.313965 +000000000 00:00:00.247834
Parallel-PQ-Process1 TASK$_15266_1              5822 PROCESSED       28686      34422         5736 09:10:19.315984 09:10:19.682781 +000000000 00:00:00.366797
Parallel-PQ-Process1 TASK$_15266_1              5823 PROCESSED        5738      11474         5736 09:10:19.684925 09:10:19.818145 +000000000 00:00:00.133220
Parallel-PQ-Process1 TASK$_15266_1              5824 PROCESSED       17212      22948         5736 09:10:19.818694 09:10:19.950409 +000000000 00:00:00.131715
Parallel-PQ-Process1 TASK$_15266_1              5830 PROCESSED       51634      57365         5731 09:10:20.605421 09:10:20.721599 +000000000 00:00:00.116178
Parallel-PQ-Process1 TASK$_15266_1              5826 PROCESSED       40160      45896         5736 09:10:20.080270 09:10:20.214443 +000000000 00:00:00.134173
Parallel-PQ-Process1 TASK$_15266_1              5827 PROCESSED       11475      17211         5736 09:10:20.217662 09:10:20.339758 +000000000 00:00:00.122096
Parallel-PQ-Process1 TASK$_15266_1              5828 PROCESSED       34423      40159         5736 09:10:20.342242 09:10:20.453802 +000000000 00:00:00.111560
Parallel-PQ-Process1 TASK$_15266_1              5829 PROCESSED       45897      51633         5736 09:10:20.454376 09:10:20.603116 +000000000 00:00:00.148740
Parallel-PQ-Process1 TASK$_15266_1              5825 PROCESSED       22949      28685         5736 09:10:19.950975 09:10:20.079311 +000000000 00:00:00.128336
10 rows selected.
SQL>
SQL> --// updated sample data
SQL> select * from(
  2          select * from sample_data order by 1
  3  ) where rownum < 11;
        ID LAST_UPDATE         NAME
       100 2013/07/08 09:10:19 ora$base
       116 2013/07/08 09:10:19 dual
       117 2013/07/08 09:10:19 dual
       280 2013/07/08 09:10:19 map_object
       365 2013/07/08 09:10:19 system_privilege_map
       367 2013/07/08 09:10:19 system_privilege_map
       368 2013/07/08 09:10:19 table_privilege_map
       370 2013/07/08 09:10:19 table_privilege_map
       371 2013/07/08 09:10:19 stmt_audit_option_map
       373 2013/07/08 09:10:19 stmt_audit_option_map
10 rows selected.
SQL>
To eliminate the workload table approach, you need to find an alternative method for chunking the target table. Whether such a method is available depends entirely on the structure and nature of your target table  (and is the preferred approach over using a secondary worktable to drive parallel processing).