Problem while having a large set of data to work on!

Question

Hi,
I am facing great problem with processing large set of data. I have a requirement in which i'm supposed to generate a report.
I have a table and a MView, which i have joined to reduce the number of records to process. The MView holds 200,00,000 records while the table 18,00,000. Based on join conditions and where clause i'm able to break down the useful data to approx 4,50,000 and i'm getting 8 of my report columns from this join. I'm dumping these records into the table from where i'll be generating the report by spooling.
Below is the block which takes 12mins to insert into the report table MY_ACCOUNT_PHOTON_DUMP:
begin
dbms_output.put_line(to_char(sysdate,'hh24:mi:ss'));
insert into MY_ACCOUNT_PHOTON_DUMP --- Report table
(SUBSCR_NO, ACCOUNT_NO, AREA_CODE, DEL_NO, CIRCLE, REGISTRATION_DT, EMAIL_ID, ALT_CNTCT_NO)
select crm.SUBSCR_NO, crm.ACCOUNT_NO, crm.AREA_CODE, crm.DEL_NO, crm.CIRCLE_ID,
aa.CREATED_DATE, aa.EMAIL_ID, aa.ALTERNATE_CONTACT
from MV_CRM_SUBS_DTLS crm, --- MView
(select /*+ ALL_ROWS */ A.ALTERNATE_CONTACT, A.CREATED_DATE, A.EMAIL_ID, B.SUBSCR_NO
from MCCI_PROFILE_DTLS a, MCCI_PROFILE_SUBSCR_DTLS b
where A.PROFILE_ID = B.PROFILE_ID
and B.ACE_STATUS = 'N'
) aa --- Join of two tables giviing me 18,00,000 recs
where crm.SUBSCR_NO = aa.SUBSCR_NO
and crm.SRVC_TYPE_ID = '125'
and crm.END_DT IS NULL;
INTERNET_METER_TABLE_PROC_1('MCCIPRD','MY_ACCOUNT_PHOTON_DUMP'); --- calling procedure to analyze the report table
COMMIT;
dbms_output.put_line(to_char(sysdate,'hh24:mi:ss'));
end; --- 12 min 04 secFor the rest of the 13 columns required i am running a block which has a FOR UPDATE cursor on the report table:
declare
cursor cur is
select SUBSCR_NO, ACCOUNT_NO, AREA_CODE, DEL_NO,
CIRCLE, REGISTRATION_DT, EMAIL_ID, ALT_CNTCT_NO
from MCCIPRD.MY_ACCOUNT_PHOTON_DUMP --where ACCOUNT_NO = 901237064
for update of
MRKT_SEGMNT, AON, ONLINE_PAY, PAID_AMNT, E_BILL, ECS, BILLED_AMNT,
SRVC_TAX, BILL_PLAN, USAGE_IN_MB, USAGE_IN_MIN, NO_OF_LOGIN, PHOTON_TYPE;
v_aon VARCHAR2(10) := NULL;
v_online_pay VARCHAR2(10) := NULL;
v_ebill VARCHAR2(10) := NULL;
v_mkt_sgmnt VARCHAR2(50) := NULL;
v_phtn_type VARCHAR2(50) := NULL;
v_login NUMBER(10) := 0;
v_paid_amnt VARCHAR2(50) := NULL;
v_ecs VARCHAR2(10) := NULL;
v_bill_plan VARCHAR2(100):= NULL;
v_billed_amnt VARCHAR2(10) := NULL;
v_srvc_tx_amnt VARCHAR2(10) := NULL;
v_usg_mb NUMBER(10) := NULL;
v_usg_min NUMBER(10) := NULL;
begin
dbms_output.put_line(to_char(sysdate,'hh24:mi:ss'));
for rec in cur loop
begin
select apps.TTL_GET_DEL_AON@MCCI_TO_PRD591(rec.ACCOUNT_NO, rec.DEL_NO, rec.CIRCLE)
into v_aon from dual;
exception
when others then
v_aon := 'NA';
end;
SELECT DECODE(COUNT(*),0,'NO','YES') into v_online_pay
FROM TTL_DESCRIPTIONS@MCCI_TO_PRD591
WHERE DESCRIPTION_CODE IN(SELECT DESCRIPTION_CODE FROM TTL_BMF_TRANS_DESCR@MCCI_TO_PRD591
WHERE BMF_TRANS_TYPE
IN (SELECT BMF_TRANS_TYPE FROM
TTL_BMF@MCCI_TO_PRD591 WHERE ACCOUNT_NO = rec.ACCOUNT_NO
AND POST_DATE BETWEEN
TO_DATE('01-'||TO_CHAR(SYSDATE,'MM-YYYY'),'DD-MM-YYYY') AND SYSDATE
AND DESCRIPTION_TEXT IN (select DESCRIPTION from fnd_lookup_values@MCCI_TO_PRD591 where
LOOKUP_TYPE='TTL_ONLINE_PAYMENT');
SELECT decode(count( *),0,'NO','YES') into v_ebill
FROM TTL_CUST_ADD_DTLS@MCCI_TO_PRD591
WHERE CUST_ACCT_NBR = rec.ACCOUNT_NO
AND UPPER(CUSTOMER_PREF_MODE) ='EMAIL';
begin
select ACC_SUB_CAT_DESC into v_mkt_sgmnt
from ttl_cust_dtls@MCCI_TO_PRD591 a, TTL_ACCOUNT_CATEGORIES@MCCI_TO_PRD591 b
where a.CUST_ACCT_NBR = rec.ACCOUNT_NO
and a.market_code = b.ACC_SUB_CAT;
exception
when others then
v_mkt_sgmnt := 'NA';
end;
begin
select nvl(sum(TRANS_AMOUNT),0) into v_paid_amnt
from ttl_bmf@MCCI_TO_PRD591
where account_no = rec.ACCOUNT_NO
AND POST_DATE
BETWEEN TO_DATE('01-'||TO_CHAR(SYSDATE,'MM-YYYY'),'DD-MM-YYYY')
AND SYSDATE;
exception
when others then
v_paid_amnt := 'NA';
end;
SELECT decode(count(1),0,'NO','YES') into v_ecs
from ts.Billdesk_Registration_MV@MCCI_TO_PRD591 where ACCOUNT_NO = rec.ACCOUNT_NO
and UPPER(REGISTRATION_TYPE ) = 'ECS';
SELECT decode(COUNT(*),0,'PHOTON WHIZ','PHOTON PLUS') into v_phtn_type
FROM ts.ttl_cust_ord_prdt_dtls@MCCI_TO_PRD591 A, ttl_product_mstr@MCCI_TO_PRD591 b
WHERE A.SUBSCRIBER_NBR = rec.SUBSCR_NO
and (A.prdt_disconnection_date IS NULL OR A.prdt_disconnection_date > SYSDATE )
AND A.prdt_disc_flag = 'N'
AND A.prdt_nbr = b.product_number
AND A.prdt_type_id = b.prouduct_type_id
AND b.first_level LIKE 'Feature%'
AND UPPER (b.product_desc) LIKE '%HSIA%';
SELECT count(1) into v_login
FROM MCCIPRD.MYACCOUNT_SESSION_INFO a
WHERE (A.DEL_NO = rec.DEL_NO or A.DEL_NO = ltrim(rec.AREA_CODE,'0')||rec.DEL_NO)
AND to_char(A.LOGIN_TIME,'Mon-YYYY') = to_char(sysdate-5,'Mon-YYYY');
begin
select PACKAGE_NAME, BILLED_AMOUNT, SERVICE_TAX_AMOUNT, USAGE_IN_MB, USAGE_IN_MIN
into v_bill_plan, v_billed_amnt, v_srvc_tx_amnt, v_usg_mb, v_usg_min from
(select rank() over(order by STATEMENT_DATE desc) rk,
PACKAGE_NAME, USAGE_IN_MB, USAGE_IN_MIN
nvl(BILLED_AMOUNT,'0') BILLED_AMOUNT, NVL(SRVC_TAX_AMNT,'0') SERVICE_TAX_AMOUNT
from MCCIPRD.MCCI_IM_BILLED_DATA
where (DEL_NUM = rec.DEL_NO or DEL_NUM = ltrim(rec.AREA_CODE,'0')||rec.DEL_NO)
and STATEMENT_DATE like '%'||to_char(SYSDATE,'Mon-YY')||'%')
where rk = 1;
exception
when others then
v_bill_plan := 'NA';
v_billed_amnt := '0';
v_srvc_tx_amnt := '0';
v_usg_mb := 0;
v_usg_min := 0;
end;
-- UPDATE THE DUMP TABLE --
update MCCIPRD.MY_ACCOUNT_PHOTON_DUMP
set MRKT_SEGMNT = v_mkt_sgmnt, AON = v_aon, ONLINE_PAY = v_online_pay, PAID_AMNT = v_paid_amnt,
E_BILL = v_ebill, ECS = v_ecs, BILLED_AMNT = v_billed_amnt, SRVC_TAX = v_srvc_tx_amnt,
BILL_PLAN = v_bill_plan, USAGE_IN_MB = v_usg_mb, USAGE_IN_MIN = v_usg_min, NO_OF_LOGIN = v_login,
PHOTON_TYPE = v_phtn_type
where current of cur;
end loop;
COMMIT;
dbms_output.put_line(to_char(sysdate,'hh24:mi:ss'));
exception when others then
dbms_output.put_line(SQLCODE||'::'||SQLERRM);
end;The report takes >6hrs. I know that most of the SELECT queries have ACCOUNT_NO as WHERE clause and can be joined, but when i joining few of these blocks with the initial INSERT query it was no better.
The individual queries within the cursor loop dont take more then 0.3 sec to execute.
I'm using the FOR UPDATE as i know that the report table is being used solely for this purpose.
Can somebody plz help me with this, i'm in desperate need of good advice here.
Thanks!!
Edited by: user11089213 on Aug 30, 2011 12:01 AM

Answer 1

Hi,
Below is the explain plan for the original query:
select /*+ ALL_ROWS */  crm.SUBSCR_NO, crm.ACCOUNT_NO, ltrim(crm.AREA_CODE,'0'), crm.DEL_NO, >crm.CIRCLE_ID
from MV_CRM_SUBS_DTLS crm,
        (select /*+ ALL_ROWS */  A.ALTERNATE_CONTACT, A.CREATED_DATE, A.EMAIL_ID, B.SUBSCR_NO
        from MCCIPRD.MCCI_PROFILE_DTLS a, MCCIPRD.MCCI_PROFILE_SUBSCR_DTLS b
        where A.PROFILE_ID = B.PROFILE_ID
        and   B.ACE_STATUS = 'N'
        ) aa
where crm.SUBSCR_NO    = aa.SUBSCR_NO
and   crm.SRVC_TYPE_ID = '125'
and   crm.END_DT IS NULL
| Id  | Operation              | Name                     | Rows  | Bytes |TempSpc| Cost (%CPU)| Time     |
|   0 | SELECT STATEMENT       |                          |  1481K|   100M|       |   245K  (5)| 00:49:09 |
|*  1 |  HASH JOIN             |                          |  1481K|   100M|    46M|   245K  (5)| 00:49:09 |
|*  2 |   HASH JOIN            |                          |  1480K|    29M|    38M| 13884   (9)| 00:02:47 |
|*  3 |    TABLE ACCESS FULL   | MCCI_PROFILE_SUBSCR_DTLS |  1480K|    21M|       |  3383  (13)| 00:00:41 |
|   4 |    INDEX FAST FULL SCAN| SYS_C002680              |  2513K|    14M|       |  6024   (5)| 00:01:13 |
|*  5 |   MAT_VIEW ACCESS FULL | MV_CRM_SUBS_DTLS_08AUG   |  1740K|    82M|       |   224K  (5)| 00:44:49 |
Predicate Information (identified by operation id):
   1 - access("CRM"."SUBSCR_NO"="B"."SUBSCR_NO")
   2 - access("A"."PROFILE_ID"="B"."PROFILE_ID")
   3 - filter("B"."ACE_STATUS"='N')
   5 - filter("CRM"."END_DT" IS NULL AND "CRM"."SRVC_TYPE_ID"='125')Whereas for the modified MView query, the plane remains the same:
select /*+ ALL_ROWS */ crm.SUBSCR_NO, crm.ACCOUNT_NO, ltrim(crm.AREA_CODE,'0'), crm.DEL_NO, >crm.CIRCLE_ID
from    (select * from MV_CRM_SUBS_DTLS
         where SRVC_TYPE_ID = '125'
         and   END_DT IS NULL) crm,
        (select /*+ ALL_ROWS */  A.ALTERNATE_CONTACT, A.CREATED_DATE, A.EMAIL_ID, B.SUBSCR_NO
        from MCCIPRD.MCCI_PROFILE_DTLS a, MCCIPRD.MCCI_PROFILE_SUBSCR_DTLS b
        where A.PROFILE_ID = B.PROFILE_ID
        and   B.ACE_STATUS = 'N'
        ) aa
where crm.SUBSCR_NO  = aa.SUBSCR_NO
| Id  | Operation              | Name                     | Rows  | Bytes |TempSpc| Cost (%CPU)| Time     |
|   0 | SELECT STATEMENT       |                          |  1481K|   100M|       |   245K  (5)| 00:49:09 |
|*  1 |  HASH JOIN             |                          |  1481K|   100M|    46M|   245K  (5)| 00:49:09 |
|*  2 |   HASH JOIN            |                          |  1480K|    29M|    38M| 13884   (9)| 00:02:47 |
|*  3 |    TABLE ACCESS FULL   | MCCI_PROFILE_SUBSCR_DTLS |  1480K|    21M|       |  3383  (13)| 00:00:41 |
|   4 |    INDEX FAST FULL SCAN| SYS_C002680              |  2513K|    14M|       |  6024   (5)| 00:01:13 |
|*  5 |   MAT_VIEW ACCESS FULL | MV_CRM_SUBS_DTLS_08AUG   |  1740K|    82M|       |   224K  (5)| 00:44:49 |
Predicate Information (identified by operation id):
   1 - access("CRM"."SUBSCR_NO"="B"."SUBSCR_NO")
   2 - access("A"."PROFILE_ID"="B"."PROFILE_ID")
   3 - filter("B"."ACE_STATUS"='N')
   5 - filter("CRM"."END_DT" IS NULL AND "CRM"."SRVC_TYPE_ID"='125')Also took your advice and tried to merge all the queries into single INSERT SQL, will be posting the results shortly.
Edited by: BluShadow on 30-Aug-2011 10:21
added {noformat}{noformat} tags.  Please read {message:id=9360002} to learn to do this yourself