PARALLEL_ENABLED with SQL subquery input.

Question

Oracle Database 11g Enterprise Edition Release 11.2.0.3.0 - 64bit
PL/SQL Release 11.2.0.3.0
IBM/AIX RISC System
Hello Experts,
I tried to emulate William Robertson's "Parallel PL/SQL launcher Program" to accept a select query as input. The source of the code is from http://www.williamrobertson.net/ (on the Left hand side select 'Oracle' >> 'Oracle Code and Scripts'. On the Right Hand list select the 2nd link 'Parallel PL/SQL launcher update').
Mr. Robertson uses a funtion with "dbms_lock.sleep" and sends that as an input to a PARALLEL_ENABLED function. However, I tried create another Pipeline Funtion with a select query as input parameter. Can the experts kindly suggest the approach on how I can use a select query as input to a PARALLEL_ENABLED Function. My purpose is not to have a PARALLEL hint in a select query but to have a function parallelly execute multiple select queries at the same time and PIPE out the result.
DROP TABLE pq_driver PURGE;
DROP TABLE log_times PURGE;
DROP  TYPE varchar2_tt;
CREATE TYPE varchar2_tt AS TABLE OF VARCHAR2(32000);
GRANT EXECUTE ON varchar2_tt TO PUBLIC;
CREATE TABLE pq_driver
( thread_id NUMBER(1) NOT NULL PRIMARY KEY )
PARTITION BY LIST(thread_id)
( PARTITION p1 VALUES(1)
, PARTITION p2 VALUES(2)
, PARTITION p3 VALUES(3)
, PARTITION p4 VALUES(4) )
PARALLEL 4
INSERT ALL
INTO pq_driver VALUES (1)
INTO pq_driver VALUES (2)
INTO pq_driver VALUES (3)
INTO pq_driver VALUES (4)
SELECT * FROM dual;
COMMENT ON TABLE pq_driver IS 'Control table for generating parallel ref cursors with package parallel_launch';
EXEC dbms_stats.lock_table_stats('SYSADM','PQ_DRIVER');
-- PQ selection is sensitive to table stats. Analyzing table with
-- exec DBMS_STATS.GATHER_TABLE_STATS(user,'pq_driver')
-- causes serialisation.
-- Will automatic stats gathering have the same effect?
CREATE TABLE log_times
( thread_id   INTEGER NOT NULL CONSTRAINT log_times_pk PRIMARY KEY
, what        VARCHAR2(4000)
, sid         INTEGER
, serial#     INTEGER
, start_time  TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL
, end_time    TIMESTAMP
, label       VARCHAR2(4000)
, errors      VARCHAR2(4000) );
-- Note: The trigger can be ignored if it fails.
drop trigger log_times_defaults_trg;
CREATE or replace TRIGGER log_times_defaults_trg
BEFORE INSERT ON log_times
FOR EACH ROW
BEGIN
    SELECT sid, serial#
    INTO   :new.sid, :new.serial#
    FROM  sys.v_$session
    WHERE  sid IN
           ( SELECT sid FROM sys.v_$mystat );
END;
show errors
CREATE OR REPLACE PACKAGE parallel_launch
AS
    TYPE rc_pq_driver IS REF CURSOR RETURN pq_driver%ROWTYPE;
    TYPE inrec_type is record (table_name VARCHAR2 (32000));
    TYPE inrec IS TABLE OF inrec_type;
    -- PQ launch vehicle:
    -- Must be strongly typed ref cursor to allow partition by range or hash
    -- Must be in package spec to be visible to SQL
    -- Must be an autonomous transaction if we are doing any DML in the submitted procedures
    FUNCTION pq_submit
        ( p_job_list  VARCHAR2_TT
        , p_pq_refcur rc_pq_driver )
        RETURN varchar2_tt
        PARALLEL_ENABLE(PARTITION p_pq_refcur BY ANY)
        PIPELINED;
    PROCEDURE submit
        ( p_job1 VARCHAR2
        , p_job2 VARCHAR2
        , p_job3 VARCHAR2
        , p_job4 VARCHAR2 );
    -- Convenient test procedure - calls 'parallel_launch.slow_proc()' 4 times to see whether they all run at once:
    PROCEDURE test;
    -- Dummy procedure for testing - calls dbms_lock.sleep(2)
    PROCEDURE slow_proc( p_id INTEGER );
  FUNCTION test_rs (p_sql IN SYS_REFCURSOR)
  RETURN inrec
  PIPELINED;
END parallel_launch;
show errors
CREATE OR REPLACE PACKAGE BODY parallel_launch
AS
    TYPE stats_rec IS RECORD
    ( thread_id            log_times.thread_id%TYPE
    , what                 log_times.what%TYPE
    , start_timestr        VARCHAR2(8)
    , sid                  sys.v_$mystat.sid%TYPE
    , serial#              v$px_session.serial#%TYPE
    , pq_actual_degree     NUMBER
    , pq_requested_degree  NUMBER
    , rowcount             PLS_INTEGER := 0 );
    g_clear_stats_rec stats_rec;
    PROCEDURE log_start
        ( p_thread_id  log_times.thread_id%TYPE
        , p_what       log_times.what%TYPE )
    IS
        PRAGMA AUTONOMOUS_TRANSACTION;
    BEGIN
        DELETE log_times WHERE thread_id = p_thread_id;
        INSERT INTO log_times
        ( thread_id, what )
        VALUES
        ( p_thread_id, p_what );
        COMMIT;
    END log_start;
    PROCEDURE log_end
        ( p_thread_id  log_times.thread_id%TYPE
        , p_errors     log_times.errors%TYPE DEFAULT NULL )
    IS
        PRAGMA AUTONOMOUS_TRANSACTION;
    BEGIN
        UPDATE log_times
        SET    end_time = SYSTIMESTAMP
             , errors = p_errors
        WHERE  thread_id = p_thread_id;
        COMMIT;
    END log_end;
    PROCEDURE execute_command
        ( p_what log_times.what%TYPE )
    IS
        PRAGMA AUTONOMOUS_TRANSACTION;
    BEGIN
        EXECUTE IMMEDIATE p_what;
        COMMIT;
    END execute_command;
    FUNCTION pq_submit
        ( p_job_list  VARCHAR2_TT
        , p_pq_refcur rc_pq_driver )
        RETURN varchar2_tt
        PARALLEL_ENABLE(PARTITION p_pq_refcur BY ANY)
        PIPELINED
    IS
        v_error_text VARCHAR2(32000);
        r pq_driver%ROWTYPE;
        r_row_stats stats_rec;
    BEGIN
        LOOP
            FETCH p_pq_refcur INTO r;
            EXIT WHEN p_pq_refcur%NOTFOUND;
            SELECT TO_CHAR(SYSDATE,'HH24:MI:SS')
                 , s.sid
                 , pqs.serial#
                 , pqs.degree
                 , pqs.req_degree
            INTO   r_row_stats.start_timestr
                 , r_row_stats.sid
                 , r_row_stats.serial#
                 , r_row_stats.pq_actual_degree
                 , r_row_stats.pq_requested_degree
            FROM   ( SELECT sid FROM sys.v_$mystat WHERE rownum = 1 ) s
                   LEFT JOIN sys.v_$px_session pqs ON pqs.sid = s.sid;
            r_row_stats.thread_id := r.thread_id;
            r_row_stats.rowcount := p_pq_refcur%ROWCOUNT;
            r_row_stats.what := 'BEGIN ' || RTRIM(p_job_list(r.thread_id),';') || '; END;';
            DBMS_OUTPUT.PUT_LINE( r_row_stats.thread_id);
            BEGIN
                log_start(r.thread_id, r_row_stats.what);
                execute_command(r_row_stats.what);
                log_end(r.thread_id);
                PIPE ROW
                ( RPAD('sid=' || r_row_stats.sid || ' serial#=' || r_row_stats.serial# || ':',25) ||
                  'Degree of parallelism: requested '  || r_row_stats.pq_requested_degree ||
                  ', actual ' || r_row_stats.pq_actual_degree || ': ' ||
                  r_row_stats.start_timestr || ' - ' || TO_CHAR(SYSDATE,'HH24:MI:SS') );
            EXCEPTION
                WHEN OTHERS THEN
                    log_end(r_row_stats.thread_id, SQLERRM);
                    PIPE ROW('sid=' || r_row_stats.sid || ' serial#=' || r_row_stats.serial# || ': ' || SQLERRM);
            END;
        END LOOP;
--        IF r_row_stats.rowcount = 0 THEN
--            RAISE_APPLICATION_ERROR
--            ( -20000
--            , 'Cursor returned no rows' );
--        END IF;
        RETURN;
    END pq_submit;
     PROCEDURE submit
        ( p_job1 VARCHAR2
        , p_job2 VARCHAR2
        , p_job3 VARCHAR2
        , p_job4 VARCHAR2 )
    IS
        v_results VARCHAR2_TT;
    BEGIN
        SELECT /*+ PARALLEL(4) */ column_value
        BULK COLLECT INTO v_results
        FROM TABLE(
             parallel_launch.pq_submit
             ( VARCHAR2_TT(p_job1,p_job2,p_job3,p_job4)
             , CURSOR(SELECT thread_id FROM pq_driver)
        IF v_results.COUNT = 0 THEN
            v_results.EXTEND;
            v_results(1) := 'parallel_launch.SUBMIT: No rows returned from table function PQ_SUBMIT';
        END IF;
        FOR i IN v_results.FIRST..v_results.LAST LOOP
            DBMS_OUTPUT.PUT_LINE(v_results(i));
        END LOOP;
    END submit;
PROCEDURE test
    IS
    BEGIN
        submit
        ( 'parallel_launch.test_rs (CURSOR
  (SELECT /*+ parallel (ALLT,4) */ table_name
   from all_tables ALLT))'
        , 'parallel_launch.test_rs (CURSOR
  (SELECT /*+ parallel (ALLT,4) */ table_name
   from all_tab_columns ALLT))'
        ,'parallel_launch.test_rs (CURSOR
  (SELECT /*+ parallel (ALLT,4) */ table_name
   from all_tables ALLT))'
         , 'parallel_launch.test_rs (CURSOR
  (SELECT /*+ parallel (ALLT,4) */ table_name
   from all_tab_columns ALLT))' );
    END test;
    PROCEDURE slow_proc
        ( p_id INTEGER )
    IS
    BEGIN
        dbms_lock.sleep(2);
    END slow_proc;
FUNCTION test_rs (p_sql IN SYS_REFCURSOR)
  RETURN inrec
  PIPELINED
IS
v_inrec inrec_type;
BEGIN
  LOOP
   FETCH p_sql INTO v_inrec;
   EXIT WHEN p_sql%NOTFOUND;
  END LOOP;
  CLOSE p_sql;
  pipe row (v_inrec);
  RETURN;
END;
END parallel_launch;
show errorsThank you all,
Aj

Answer 1

Hi, Due to proprietary reasons I couldn’t expose the code earlier. However, below is the example.
As I already mentioned, the front end calls the following select query 3 times for three diff years (in sequence), each table has 6 mill rows and performing a count takes 40 sec for one select to execute.
Now, the report shows 3 graphs on one page. So it calls 3 selects hence the total is 40+40+40 = 120 sec.
As per our new requirement we need to show 2009 and 2008 years data as well. So two more new tables have been created. Calling 5 select queries sequentially is taking 200 sec. Hence the challenge, how do I call 5 Pipe function at the same time and get the total result back in 40 sec. (We don’t have a Java layer) Please let me know if I can provide any further information.
Thank you,
Aj
SELECT * FROM TABLE (db_ATG_yr_pkg.get_ATG_records ('JCITY','HOMEATG', '201210'  ) );  
SELECT * FROM TABLE (db_ATG_yr_pkg.get_ATG_records ('ACITY','HOMEATG', '201110'  ) );  
SELECT * FROM TABLE (db_ATG_yr_pkg.get_ATG_records ('BCITY','HOMEATG', '201010'  ) );  
--After new requirement we are adding
SELECT * FROM TABLE (db_ATG_yr_pkg.get_ATG_records ('ACITY','HOMEATG', '200910'  ) );  
SELECT * FROM TABLE (db_ATG_yr_pkg.get_ATG_records ('BCITY','HOMEATG', '200810'  ) );  
CREATE OR REPLACE PACKAGE db_ATG_yr_pkg
IS
TYPE temprec_typ IS RECORD (gender_group VARCHAR2 (3), et_group VARCHAR2 (3)
                            ,period_year VARCHAR2 (10), PERIOD_MONTH VARCHAR2 (10)
                            ,emp_count NUMBER, total_temps NUMBER);
TYPE temprecset IS TABLE OF temprec_typ;
FUNCTION get_ATG_records (
  p_city_str IN VARCHAR2 DEFAULT 'JCITY',
  p_tab_name IN VARCHAR2 DEFAULT 'HOMEATG',
   p_period IN VARCHAR2 DEFAULT '201210')
  RETURN totalrecset
  PIPELINED
END db_ATG_yr_pkg;
CREATE OR REPLACE PACKAGE BODY db_ATG_yr_pkg
IS
FUNCTION get_ATG_records (
  p_city_str IN VARCHAR2 DEFAULT 'JCITY',
  p_tab_name IN VARCHAR2 DEFAULT 'HOMEATG',
   p_period IN VARCHAR2 DEFAULT '201210')
  RETURN totalrecset
  PIPELINED
IS
  PRAGMA AUTONOMOUS_TRANSACTION;
  v_select_qry    LONG;
  v_city_str      VARCHAR2 (1000);
  p_rpt_year      VARCHAR2 (10);
  cursor_rowcount NUMBER;
  CurSOR1         SYS_REFCURSOR;
BEGIN
v_city_str :=   p_city_str;
  -- selecting only the year
  p_rpt_year      :=
   SUBSTR (p_period ,1,4);
  v_hdcnt_str := ' count(tot_count) ';
  v_totaltemps_str := ' count(temp_count) ';
  -- builing select query
  v_select_qry := 'Select  A.atg_GENDER_GROUP ,
                     A.atg_et_group,
           SUBSTR(a.atg_period,1,4) AS PERIOD_YEAR,
           SUBSTR( a.atg_period,5) AS PERIOD_MONTH,';
  v_select_qry := v_select_qry || v_hdcnt_str || ' as emp_count, ';
  v_select_qry := v_select_qry || v_totaltemps_str || ' as total_temps ';
--determining table based on year 
v_select_qry      := 
     v_select_qry || ' FROM YR_TABLE' || p_tab_name || 'db_YR' || p_rpt_year || '  A  where atg_city = ' || CHR (39) || v_city_str ||CHR(39);
-- grouping data
  v_select_qry      :=
   v_select_qry || ' group by  A.atg_GENDER_GROUP ,
                     A.atg_et_group,
           SUBSTR(a.atg_period,1,4) AS PERIOD_YEAR,
           SUBSTR( a.atg_period,5) AS PERIOD_MONTH,';
   -- opening cursor       
  OPEN CurSOR1 FOR v_select_qry;
  LOOP
   FETCH CurSOR1 INTO temp_rec;
   cursor_rowcount := CurSOR1%ROWCOUNT;
   EXIT WHEN CurSOR1%NOTFOUND;
   bo_data.EXTEND;
   bo_data (bo_data.COUNT)      :=
    ATGdb_obj (temp_rec.gender_group, temp_rec.et_group, temp_rec.PERIOD_MONTH
              ,temp_rec.period_year, temp_rec.emp_count, temp_rec.total_temps);
   -- piping data
   PIPE ROW (temp_rec);
  END LOOP;
EXCEPTION
  WHEN OTHERS THEN
   v_error := 'ERROR ' || SQLERRM;
   temp_rec.gender_group := v_error;
   PIPE ROW (temp_rec);
END get_ATG_records;
END db_ATG_yr_pkg;
SHO ERR