Response time of query utterly upside down because of small where clause change

Question

Hello,
I'm wondering why a small change on a where clause in a query has a dramatic impact on its response time.
Here is the query, with its plan and a few details:
select * from (
SELECT xyz_id, time_oper, ...
     FROM (SELECT 
                    d.xyz_id xyz_id,
                    TO_CHAR (di.time_operation, 'DD/MM/YYYY') time_oper,
                    di.time_operation time_operation,
                    UPPER (d.delivery_name || ' ' || d.delivery_firstname) custname,
                    d.ticket_language ticket_language, d.payed,
                    dsum.delivery_mode delivery_mode,
                    d.station_delivery station_delivery,
                    d.total_price total_price, d.crm_cust_id custid,
                    d.bene_cust_id person_id, d.xyz_num, dpe.ers_pnr ers_pnr,
                    d.delivery_name,
                    TO_CHAR (dsum.first_travel_date, 'DD/MM/YYYY') first_traveldate,
                    d.crm_company custtype, UPPER (d.client_name) partyname,
                    getremark(d.xyz_num) remark,
                    d.client_app, di.work_unit, di.account_unit,
                    di.distrib_code,
                    UPPER (d.crm_name || ' ' || d.crm_firstname) crm_custname,
                   getspecialproduct(di.xyz_id) specialproduct
               FROM xyz d, xyz_info di, xyz_pnr_ers dpe, xyz_summary dsum
              WHERE d.cancel_state = 'N'
             -- AND d.payed = 'N'
                AND dsum.delivery_mode NOT IN ('DD')
                AND dsum.payment_method NOT IN ('AC', 'AG')
                AND d.xyz_blocked IS NULL
                AND di.xyz_id = d.xyz_id
                AND di.operation = 'CREATE'
                AND dpe.xyz_id(+) = d.xyz_id
                AND EXISTS (SELECT 1
                              FROM xyz_ticket dt
                             WHERE dt.xyz_id = d.xyz_id)
                AND dsum.xyz_id = di.xyz_id
           ORDER BY di.time_operation DESC)
    WHERE ROWNUM < 1002
) view
WHERE view.DISTRIB_CODE in ('NS') AND view.TIME_OPERATION > TO_DATE('20/5/2013', 'dd/MM/yyyy')
plan with "d.payed = 'N'" (no rows, *extremely* slow):
| Id  | Operation                          | Name             | Rows  | Bytes | Cost (%CPU)| Time     |
|   0 | SELECT STATEMENT                   |                  |  1001 |  4166K| 39354   (1)| 00:02:59 |
|*  1 |  VIEW                              |                  |  1001 |  4166K| 39354   (1)| 00:02:59 |
|*  2 |   COUNT STOPKEY                    |                  |       |       |            |          |
|   3 |    VIEW                            |                  |  1001 |  4166K| 39354   (1)| 00:02:59 |
|   4 |     NESTED LOOPS OUTER             |                  |  1001 |   130K| 39354   (1)| 00:02:59 |
|   5 |      NESTED LOOPS SEMI             |                  |   970 |   111K| 36747   (1)| 00:02:47 |
|   6 |       NESTED LOOPS                 |                  |   970 |   104K| 34803   (1)| 00:02:39 |
|   7 |        NESTED LOOPS                |                  |   970 | 54320 | 32857   (1)| 00:02:30 |
|*  8 |         TABLE ACCESS BY INDEX ROWID| XYZ_INFO         |    19M|   704M| 28886   (1)| 00:02:12 |
|   9 |          INDEX FULL SCAN DESCENDING| DNIN_IDX_NI5     | 36967 |       |   296   (2)| 00:00:02 |
|* 10 |         TABLE ACCESS BY INDEX ROWID| XYZ_SUMMARY      |     1 |    19 |     2   (0)| 00:00:01 |
|* 11 |          INDEX UNIQUE SCAN         | SB11_DSMM_XYZ_UK |     1 |       |     1   (0)| 00:00:01 |
|* 12 |        TABLE ACCESS BY INDEX ROWID | XYZ              |     1 |    54 |     2   (0)| 00:00:01 |
|* 13 |         INDEX UNIQUE SCAN          | XYZ_PK           |     1 |       |     1   (0)| 00:00:01 |
|* 14 |       INDEX RANGE SCAN             | DNTI_NI1         |    32M|   249M|     2   (0)| 00:00:01 |
|  15 |      TABLE ACCESS BY INDEX ROWID   | XYZ_PNR_ERS      |     1 |    15 |     4   (0)| 00:00:01 |
|* 16 |       INDEX RANGE SCAN             | DNPE_XYZ         |     1 |       |     2   (0)| 00:00:01 |
Predicate Information (identified by operation id):
  1 - filter("DISTRIB_CODE"='NS' AND "TIME_OPERATION">TO_DATE(' 2013-05-20', 'syyyy-mm-dd'))
  2 - filter(ROWNUM<1002)
  8 - filter("DI"."OPERATION"='CREATE')
10 - filter("DSUM"."DELIVERY_MODE"<>'DD' AND "DSUM"."PAYMENT_METHOD"<>'AC' AND "DSUM"."PAYMENT_METHOD"<>'AG')
11 - access("DSUM"."XYZ_ID"="DI"."XYZ_ID")
12 - filter("D"."PAYED"='N' AND "D"."XYZ_BLOCKED" IS NULL AND "D"."CANCEL_STATE"='N')
              ^^^^^^^^^^^^^^
13 - access("DI"."XYZ_ID"="D"."XYZ_ID")
14 - access("DT"."XYZ_ID"="D"."XYZ_ID")
16 - access("DPE"."XYZ_ID"(+)="D"."XYZ_ID")
plan with "d.payed = 'N'" (+/- 450 rows, less than two minutes):
| Id  | Operation                          | Name             | Rows  | Bytes | Cost (%CPU)| Time     |
|   0 | SELECT STATEMENT                   |                  |  1001 |  4166K| 58604   (1)| 00:04:27 |
|*  1 |  VIEW                              |                  |  1001 |  4166K| 58604   (1)| 00:04:27 |
|*  2 |   COUNT STOPKEY                    |                  |       |       |            |          |
|   3 |    VIEW                            |                  |  1002 |  4170K| 58604   (1)| 00:04:27 |
|   4 |     NESTED LOOPS OUTER             |                  |  1002 |   130K| 58604   (1)| 00:04:27 |
|   5 |      NESTED LOOPS SEMI             |                  |  1002 |   115K| 55911   (1)| 00:04:14 |
|   6 |       NESTED LOOPS                 |                  |  1476 |   158K| 52952   (1)| 00:04:01 |
|   7 |        NESTED LOOPS                |                  |  1476 | 82656 | 49992   (1)| 00:03:48 |
|*  8 |         TABLE ACCESS BY INDEX ROWID| XYZ_INFO         |    19M|   704M| 43948   (1)| 00:03:20 |
|   9 |          INDEX FULL SCAN DESCENDING| DNIN_IDX_NI5     | 56244 |       |   449   (1)| 00:00:03 |
|* 10 |         TABLE ACCESS BY INDEX ROWID| XYZ_SUMMARY      |     1 |    19 |     2   (0)| 00:00:01 |
|* 11 |          INDEX UNIQUE SCAN         | AAAA_DSMM_XYZ_UK |     1 |       |     1   (0)| 00:00:01 |
|* 12 |        TABLE ACCESS BY INDEX ROWID | XYZ              |     1 |    54 |     2   (0)| 00:00:01 |
|* 13 |         INDEX UNIQUE SCAN          | XYZ_PK           |     1 |       |     1   (0)| 00:00:01 |
|* 14 |       INDEX RANGE SCAN             | DNTI_NI1         |    22M|   168M|     2   (0)| 00:00:01 |
|  15 |      TABLE ACCESS BY INDEX ROWID   | XYZ_PNR_ERS      |     1 |    15 |     4   (0)| 00:00:01 |
|* 16 |       INDEX RANGE SCAN             | DNPE_XYZ         |     1 |       |     2   (0)| 00:00:01 |
Predicate Information (identified by operation id):
   1 - filter("DISTRIB_CODE"='NS' AND "TIME_OPERATION">TO_DATE(' 2013-05-20', 'syyyy-mm-dd'))
   2 - filter(ROWNUM<1002)
   8 - filter("DI"."OPERATION"='CREATE')
  10 - filter("DSUM"."DELIVERY_MODE"<>'DD' AND "DSUM"."PAYMENT_METHOD"<>'AC' AND "DSUM"."PAYMENT_METHOD"<>'AG')
  11 - access("DSUM"."XYZ_ID"="DI"."XYZ_ID")
  12 - filter("D"."XYZ_BLOCKED" IS NULL AND "D"."CANCEL_STATE"='N')
  13 - access("DI"."XYZ_ID"="D"."XYZ_ID")
  14 - access("DT"."XYZ_ID"="D"."XYZ_ID")
  16 - access("DPE"."XYZ_ID"(+)="D"."XYZ_ID")
XYZ.PAYED values breakdown:
P   COUNT(1)
Y   12202716
N    9430207
tables nb of records:
TABLE_NAME           NUM_ROWS
XYZ                  21606776
XYZ_INFO            186301951
XYZ_PNR_ERS           9716471
XYZ_SUMMARY          21616607
Everything that comes inside the "select * from(...) view" parentheses is defined in a view. We've noticed that the line "AND d.payed = 'N'" (commented above) is the guilty clause: the query takes one or two seconds to return between 400 and 500 rows if this line is removed, when included in the query, the response time then switches to *hours* -sic !- but then the result set is empty (no rows returned). The plan is exactly the same whether this "d.payed = 'N'" is added or removed, I mean the nb of steps, access paths, join order etc., only the rows/bytes/cost columns values change, as you can see.
We've found no other way of solving this perf issue but by taking out this "d.payed = 'N'" condition and setting it outside the view along with view.DISTRIB_CODE and view.TIME_OPERATION.
But we would like to understand why such a small change on the XYZ.PAYED column turns everything upside down that much, and we'd like to be able to tell the optimizer to perform this check on payed = 'N' by itself in the end, just like we did, through the use of a hint if possible...
Anybody ever encountered such a behaviour before ? Do you have any advice regarding the use of a hint to reach the same response time as that we've got by setting the payed = N condition outside of the view definition ??
Thanks a lot in advance.
Regards,
Seb

Answer 1

I am really sorry I couldn't get back earlier to this forum...
Thanks to you all for your answers.
First I'd just like to correct a small mistake I made, when writing
"the query takes one or two seconds": I meant one or 2 *minutes*. Sorry.
> What table/columns are indexed by "DNTI_NI1"?
aaaa.dnti_ni1 is an index ON aaaa.xyz_ticket(xyz_id, ticket_status)
> And what are the indexes on xyz table?
Too many:
XYZ_ARCHIV_STATE_IND           ARCHIVE_STATE
XYZ_BENE_CUST_ID_IND           BENE_CUST_ID
XYZ_BENE_TTL_IND               BENE_TTL
XYZ_CANCEL_STATE_IND           CANCEL_STATE
XYZ_CLIENT_APP_NI              CLIENT_APP
XYZ_CRM_CUST_ID_IND            CRM_CUST_ID
XYZ_DELIVE_MODE_IND            DELIVERY_MODE
XYZ_DELIV_BLOCK_IND            DELIVERY_BLOCKED
XYZ_DELIV_STATE_IND            DELIVERY_STATE
XYZ_XYZ_BLOCKED                XYZ_BLOCKED
XYZ_FIRST_TRAVELDATE_IND       FIRST_TRAVELDATE
XYZ_MASTER_XYZ_IND             MASTER_XYZ_ID
XYZ_ORG_ID_NI                  ORG_ID
XYZ_PAYMT_STATE_IND            PAYMENT_STATE
XYZ_PK                         XYZ_ID
XYZ_TO_PO_IDX                  TO_PO
XYZ_UK                         XYZ_NUM
For ex. XYZ_CANCEL_STATE_IND on CANCEL_STATE seems superfluous to me, as the column may only contain Y or N (or be null)...
> Have you traced both cases to compare statistics? What differences did it reveal?
Yes but it only shows more of *everything* (more tables blocks accessed, the same
for indexes blocks, for almost all objects involved) for the slowest query !
Greping WAIT on the two trc files made for every statement and counting the
object IDs access show that the quicker query requires much less I/Os; the
slowest one overall needs much more blocks to be read (except for the indexes
DNSG_NI1 or DNPE_XYZ for example). Below I replaced obj# with the table/index
name, the first column is the figure showing how many times the object was
accessed in the 10053 file (I ctrl-C'ed my second execution ofr course, the
figures should be much higher !!):
[login.hostname] ? grep WAIT OM-quick.trc|...|sort|uniq -c
    335 XYZ_SUMMARY
  20816 AAAA_DSMM_XYZ_UK (index on xyz_summary.xyz_id)
    192 XYZ
   4804 XYZ_INFO
    246 XYZ_SEGMENT
      6 XYZ_REMARKS
     63 XYZ_PNR_ERS
    719 XYZ_PK           (index on xyz.xyz_id)
   2182 DNIN_IDX_NI5     (index on xyz.xyz_id)
    877 DNSG_NI1         (index on xyz_segment.xyz_id, segment_status)
    980 DNTI_NI1         (index on xyz_ticket.xyz_id, ticket_status)
    850 DNPE_XYZ         (index on xyz_pnr_ers.xyz_id)
[login.hostname] ? grep WAIT OM-slow.trc|...|sort|uniq -c
   1733 XYZ_SUMMARY
  38225 AAAA_DSMM_XYZ_UK  (index on xyz_summary.xyz_id)
   4359 XYZ
  12536 XYZ_INFO
     65 XYZ_SEGMENT
     17 XYZ_REMARKS
     20 XYZ_PNR_ERS
   8598 XYZ_PK
   7406 DNIN_IDX_NI5
     29 DNSG_NI1
   2475 DNTI_NI1
     27 DNPE_XYZ
The overwhelmingly dominant wait event is by far 'db file sequential read':
[login.hostname] ? grep WAIT OM-*elect.txt|cut -d"'" -f2|sort |uniq -c
     36 SQL*Net message from client
     38 SQL*Net message to client
107647 db file sequential read
      1 latch free
      1 latch: object queue header operation
      3 latch: session allocation
> It will be worth knowing the estimations...
It show the same plan with a higher cost when PAYED = N is added:
SQL> select * from sb11.dnr d
  2* where d.dnr_blocked IS NULL and d.cancel_state = 'N'
SQL> /
| Id  | Operation                   | Name                 | Rows  | Bytes | Cost (%CPU)| Time     |
|   0 | SELECT STATEMENT            |                      |  1002 |   166K|    40   (3)| 00:00:01 |
|*  1 |  TABLE ACCESS BY INDEX ROWID| XYZ                  |  1002 |   166K|    40   (3)| 00:00:01 |
|*  2 |   INDEX RANGE SCAN          | XYZ_CANCEL_STATE_IND |       |       |     8   (0)| 00:00:01 |
Predicate Information (identified by operation id):
   1 - filter("D"."XYZ_BLOCKED" IS NULL)
   2 - access("D"."CANCEL_STATE"='N')
SQL> select * from sb11.dnr d
  2  where d.dnr_blocked IS NULL and d.cancel_state = 'N'
  3* and d.payed = 'N'
SQL> /
Execution Plan
Plan hash value: 1292668880
| Id  | Operation                   | Name                 | Rows  | Bytes | Cost (%CPU)| Time     |
|   0 | SELECT STATEMENT            |                      |  1001 |   166K|    89   (3)| 00:00:01 |
|*  1 |  TABLE ACCESS BY INDEX ROWID| XYZ                  |  1001 |   166K|    89   (3)| 00:00:01 |
|*  2 |   INDEX RANGE SCAN          | XYZ_CANCEL_STATE_IND |       |       |    15   (0)| 00:00:01 |