How to execute exact match & contains text search simultaneoulsy in Oracle 10g

Question

Hi,
We have scenario where there are more than 50 million rows in a table with description column length as 1000 character. We have a web interface from where we generate a rule of comma separated keywords like
"Standard", Single, Cancel, "deal"    & so on. The words in quotes needs to be checked for exact match & the one without quotes will be searched using contains.
The problem is that we can have a rule of such a combination as large as 4000 characters of inclusion & 2000 such characters for exclusion (not to consider the description under exclusion) and this search when run on the table with millions of rows does not work using oracle regular expression, but works with smaller no. of search keywords.
Is there a better way to do such a kind of search in Oracle or if not then outside oracle using any other tool
Thanks,
AP

Answer 1

Please find below the table script, few insert statements along with the SP & function. Please help.
-- Create Table
CREATE TABLE Roomdescriptionmaster
ID long,
ROOMDESCRIPTION nvarchar2, --- max 1000 charaters
Createddate datetime
----- Insert statements
INSERT INTO ROOMDESCRIPTION (ID, ROOMDESCRIPTION, Createddate ) VALUES (1, 'Double Room (2 Adults + 2 Children) | FREE cancellation before Mar 16, 2014 PAY LATER All-Inclusive [ Included:10 % VAT] Meals:All meals and select beverages are included in the room rate.Cancellation:If canceled or modified up to 2 days before date of arrival,no fee will be charged.If canceled or modified later,100 percent of the first two nights will be charged.In case of no-show, the total price of the reservation will be charged. Prepayment:No deposit will be charged', sysdate)
INSERT INTO ROOMDESCRIPTION (ID, ROOMDESCRIPTION, Createddate ) VALUES (1, 'Double or Twin Room | FREE cancellation before Feb 1, 2014 PAY LATER All-Inclusive [ Included:10 % VAT] Meals:All meals and select beverages are included in the room rate.Cancellation:If canceled or modified up to 2 days before date of arrival,no fee will be charged.If canceled or modified later,100 percent of the first two nights will be charged.In case of no-show, the total price of the reservation will be charged. Prepayment:No deposit will be charged', sysdate)
INSERT INTO ROOMDESCRIPTION (ID, ROOMDESCRIPTION, Createddate ) VALUES (1, 'Quadruple Room (3 Adults + 1 Child) | FREE cancellation before Mar 16, 2014 PAY LATER Full board included [ Included:10 % VAT] Meals:Breakfast, lunch & dinner included.Cancellation:If canceled or modified up to 2 days before date of arrival,no fee will be charged.If canceled or modified later,100 percent of the first two nights will be charged.In case of no-show, the total price of the reservation will be charged. Prepayment:No deposit will be charged', sysdate)
INSERT INTO ROOMDESCRIPTION (ID, ROOMDESCRIPTION, Createddate ) VALUES (1, 'Triple Room with Lateral Sea View (2 Adults + 1 Child) | FREE cancellation before Dec 6, 2013 PAY LATER All-Inclusive [ Included:10 % VAT] Meals:All meals and select beverages are included in the room rate.Cancellation:If canceled or modified up to 2 days before date of arrival,no fee will be charged.If canceled or modified later,100 percent of the first two nights will be charged.In case of no-show, the total price of the reservation will be charged. Prepayment:No deposit will be charged', sysdate)
INSERT INTO ROOMDESCRIPTION (ID, ROOMDESCRIPTION, Createddate ) VALUES (1, 'Single Room with Lateral Sea View | FREE cancellation before Dec 6, 2013 PAY LATER All-Inclusive [ Included:10 % VAT] Meals:All meals and select beverages are included in the room rate.Cancellation:If canceled or modified up to 2 days before date of arrival,no fee will be charged.If canceled or modified later,100 percent of the first two nights will be charged.In case of no-show, the total price of the reservation will be charged. Prepayment:No deposit will be charged', sysdate)
--SP
CREATE OR REPLACE PROCEDURE
SP_PGHGETROOMDESCRIPTION(v_BId                number,
                           v_DaysOfData         integer,
                           v_Incl1              nvarchar2,
                           v_Incl2              nvarchar2,
                           v_Incl3              nvarchar2,
                           v_Excl1              nvarchar2,
                           v_CurrentIndex       integer,
                           v_RecordPerPage      integer,
                           v_IndexMultiplier    integer,
                           ref_recordset        out sys_refcursor
                           ) as
start_index        integer;
end_index          integer;
Incl1              nvarchar2(2000);
Incl2              nvarchar2(2000);
Incl3              nvarchar2(2000);
Excl1              nvarchar2(2000);
v_desc_utf_value   VARCHAR2(10);
begin
v_desc_utf_value:= 'utf8';
  if v_incl1 is null or trim(v_incl1) = '' then
     --dbms_output.put_line('include 1 is null or blank');
     Incl1 := '';
  else
     Incl1 := lower(v_Incl1);
  end if;
  if v_incl2 is null or trim(v_incl2) = '' then
     --dbms_output.put_line('include 2 is null or blank');
     Incl2 := '';
  else
     Incl2 := lower(v_Incl2);
  end if;
  if v_incl3 is null or trim(v_incl3) = '' then
     --dbms_output.put_line('include 3 is null or blank');
     Incl3 := '';
  else
     Incl3 := lower(v_Incl3);
  end if;
  if v_Excl1 is null or trim(v_Excl1) = '' then
     --dbms_output.put_line('Exclude 1 is null or blank');
     Excl1 := '';
  else
     Excl1 := lower(v_Excl1);
  end if;
   -- Old code
   --     and regexp_like(lower(ROOMDESCRIPTION), Incl1, 'i')
   --    and regexp_like(lower(ROOMDESCRIPTION), Incl2, 'i')
   --     and regexp_like(lower(ROOMDESCRIPTION), Incl3, 'i')
   --     and not regexp_like(lower(ROOMDESCRIPTION), Excl1, 'i')
--- First call to SP
  if v_CurrentIndex = 1 then
      start_index := v_RecordPerPage * v_IndexMultiplier;
      end_index   := (v_CurrentIndex - 1 + v_IndexMultiplier) * v_RecordPerPage;
      open ref_recordset for
     select * from (
      select ROOMDESCRIPTION, Createddate,  rownum as rn
      from roomdescriptionmaster
      where BID = v_BId
      and TO_NUMBER(trunc(sysdate) - to_date(to_char(createddate, 'yyyy-mm-dd'),'yyyy-mm-dd')) <= v_DaysOfData
      and length(FN_GET_RESTRICTION(lower(ROOMDESCRIPTION),Incl1,Incl2,Incl3,Excl1,v_desc_utf_value)) > 0
      and row_num <= v_RecordPerPage * v_IndexMultiplier
      order by row_num;
  else
  --- Subsequent calls to SP using paging from UI
      start_index := (v_CurrentIndex - 1) * v_RecordPerPage + 1;
      end_index   := (v_CurrentIndex - 1 + v_IndexMultiplier) * v_RecordPerPage;
      open ref_recordset for
      select * from (
      select ROOMDESCRIPTION, Createddate,  rownum as rn
      from roomdescriptionmaster
      where BID = v_BId
      and TO_NUMBER(trunc(sysdate) - to_date(to_char(createddate, 'yyyy-mm-dd'),'yyyy-mm-dd')) <= v_DaysOfData
      and length(FN_GET_RESTRICTION(lower(ROOMDESCRIPTION),Incl1,Incl2,Incl3,Excl1,v_desc_utf_value)) > 0
      order by roomdescriptionmasterid desc
      where rn >= start_index
      and rn <= end_index
      order by rn;
  end if;
  commit;
end SP_PGHGETROOMDESCRIPTION;
--Function
CREATE OR REPLACE FUNCTION FN_GET_RESTRICTION(
v_rate_description IN NVARCHAR2,
v_include_1 IN NVARCHAR2,
v_include_2 IN NVARCHAR2,
v_include_3 IN NVARCHAR2,
v_exclude IN NVARCHAR2, v_desc_utf_value IN VARCHAR2)
  RETURN NVARCHAR2
IS
CURSOR include_1_cur IS
select regexp_substr(str, '[^,]+', 1, level) str
from (select v_include_1 str from dual)
connect by level <= length(str)-length(replace(str,','))+1;
CURSOR include_2_cur IS
select regexp_substr(str, '[^,]+', 1, level) str
from (select v_include_2 str from dual)
connect by level <= length(str)-length(replace(str,','))+1;
CURSOR include_3_cur IS
select regexp_substr(str, '[^,]+', 1, level) str
from (select v_include_3 str from dual)
connect by level <= length(str)-length(replace(str,','))+1;
CURSOR exclude_cur IS
select regexp_substr(str, '[^,]+', 1, level) str
from (select v_exclude str from dual)
connect by level <= length(str)-length(replace(str,','))+1;
include_1_rec include_1_cur%rowtype;
include_2_rec include_2_cur%rowtype;
include_3_rec include_3_cur%rowtype;
exclude_rec exclude_cur%rowtype;
tmp_var NVARCHAR2(200);
tmp_var_int NUMBER;
tmp_flag_int NUMBER;
return_str NVARCHAR2(200);
tmp_length NUMBER;
tmp_length_include_1 NUMBER;
tmp_length_include_2 NUMBER;
tmp_length_include_3 NUMBER;
tmp_length_exclude NUMBER;
tmp_regex_pattern VARCHAR2(1000);
flag_include_1_match INTEGER;
flag_include_2_match INTEGER;
flag_include_3_match INTEGER;
flag_exclude_match INTEGER;
BEGIN
  tmp_length_include_1 := nvl(length(v_include_1),0);
  tmp_length_include_2 := nvl(length(v_include_2),0);
  tmp_length_include_3 := nvl(length(v_include_3),0);
  tmp_length_exclude := nvl(length(v_exclude),0);
  flag_include_1_match := 0;
flag_include_2_match := 0;
flag_include_3_match := 0;
flag_exclude_match := 0;
  IF tmp_length_include_1>0 OR tmp_length_include_2 >0
  OR tmp_length_include_3 >0 OR tmp_length_exclude >0 THEN
  IF v_desc_utf_value ='utf8' THEN
    ----------------------------------------------------- UTF 8 STARTED --------------
-----------------------------------------   INCLUDE 1
  tmp_length := tmp_length_include_1;
  IF tmp_length > 0 THEN
  tmp_flag_int :=0;
FOR include_1_rec in include_1_cur
LOOP
  tmp_var := trim('' || include_1_rec.str);
  --dbms_output.put_line(tmp_var);
  tmp_regex_pattern := '[^[:alnum:]]'||tmp_var||'[^[:alnum:]]|^'||tmp_var||'$|^'||tmp_var||'[^[:alnum:]]|[^[:alnum:]]'||tmp_var||'$';
  tmp_var_int := nvl(regexp_instr(v_rate_description,tmp_regex_pattern,1,1),0);
  IF (tmp_var_int <> 0) THEN
  tmp_flag_int :=1;
  flag_include_1_match := 1;
  EXIT;
  END IF;
END LOOP;
ELSE
  flag_include_1_match := 1;
END IF;
--------------------------------------------  INCLUDE 2
tmp_length := tmp_length_include_2;
  IF tmp_length > 0 THEN
    tmp_flag_int :=0;
IF flag_include_1_match =1 THEN
  FOR include_2_rec in include_2_cur
LOOP
  tmp_var := trim('' || include_2_rec.str);
tmp_regex_pattern := '[^[:alnum:]]'||tmp_var||'[^[:alnum:]]|^'||tmp_var||'$|^'||tmp_var||'[^[:alnum:]]|[^[:alnum:]]'||tmp_var||'$';
  tmp_var_int := nvl(regexp_instr(v_rate_description,tmp_regex_pattern,1,1),0);
  IF (tmp_var_int <> 0) THEN
  tmp_flag_int :=1;
  flag_include_2_match := 1;
  EXIT;
  END IF;
END LOOP;
END IF;
ELSE
  flag_include_2_match := 1;
END IF;
-------------------------------------------- INCLUDE 3
tmp_length := tmp_length_include_3;
  IF tmp_length > 0 THEN
  tmp_flag_int :=0;
   IF flag_include_2_match =1 THEN
  FOR include_3_rec in include_3_cur
LOOP
  tmp_var := trim('' || include_3_rec.str);
tmp_regex_pattern := '[^[:alnum:]]'||tmp_var||'[^[:alnum:]]|^'||tmp_var||'$|^'||tmp_var||'[^[:alnum:]]|[^[:alnum:]]'||tmp_var||'$';
  tmp_var_int := nvl(regexp_instr(v_rate_description,tmp_regex_pattern,1,1),0);
  IF (tmp_var_int <> 0) THEN
  tmp_flag_int :=1;
  flag_include_3_match := 1;
  EXIT;
  END IF;
END LOOP;
END IF;
ELSE
  flag_include_3_match := 1;
END IF;
-------------------------------------------- EXCLUDE
tmp_length := tmp_length_exclude;
IF tmp_length > 0 and flag_include_3_match =1 THEN
  FOR exclude_rec in exclude_cur
LOOP
  tmp_var := trim('' || exclude_rec.str);
tmp_regex_pattern := '[^[:alnum:]]'||tmp_var||'[^[:alnum:]]|^'||tmp_var||'$|^'||tmp_var||'[^[:alnum:]]|[^[:alnum:]]'||tmp_var||'$';
  tmp_var_int := nvl(regexp_instr(v_rate_description,tmp_regex_pattern,1,1),0);
  IF (tmp_var_int <> 0) THEN
  tmp_flag_int := -1;
  return_str := '';
  EXIT;
  END IF;
END LOOP;
END IF;
  ELSE
  ----------------------------------------------------- UTF 16 STARTED --------------
  -----------------------------------------   INCLUDE 1
  tmp_length := tmp_length_include_1;
  IF tmp_length > 0 THEN
  tmp_flag_int :=0;
  FOR include_1_rec in include_1_cur
LOOP
  tmp_var := trim('' || include_1_rec.str);
  --dbms_output.put_line(tmp_var);
tmp_var_int := nvl(INSTR(v_rate_description,tmp_var,1,1),0);
  IF (tmp_var_int <> 0) THEN
  tmp_flag_int :=1;
  flag_include_1_match := 1;
  EXIT;
  END IF;
END LOOP;
ELSE
  flag_include_1_match := 1;
END IF;
--------------------------------------------  INCLUDE 2
tmp_length := tmp_length_include_2;
  IF tmp_length > 0 THEN
    tmp_flag_int :=0;
IF flag_include_1_match =1 THEN
  FOR include_2_rec in include_2_cur
LOOP
  tmp_var := trim('' || include_2_rec.str);
tmp_var_int := nvl(INSTR(v_rate_description,tmp_var,1,1),0);
  IF (tmp_var_int <> 0) THEN
  tmp_flag_int :=1;
  flag_include_2_match := 1;
  EXIT;
  END IF;
END LOOP;
END IF;
ELSE
  flag_include_2_match := 1;
END IF;
-------------------------------------------- INCLUDE 3
tmp_length := tmp_length_include_3;
  IF tmp_length > 0 THEN
  tmp_flag_int :=0;
   IF flag_include_2_match =1 THEN
  FOR include_3_rec in include_3_cur
LOOP
  tmp_var := trim('' || include_3_rec.str);
tmp_var_int := nvl(INSTR(v_rate_description,tmp_var,1,1),0);
  IF (tmp_var_int <> 0) THEN
  tmp_flag_int :=1;
  flag_include_3_match := 1;
  EXIT;
  END IF;
END LOOP;
END IF;
ELSE
  flag_include_3_match := 1;
END IF;
-------------------------------------------- EXCLUDE
tmp_length := tmp_length_exclude;
IF tmp_length > 0 and flag_include_3_match =1 THEN
  FOR exclude_rec in exclude_cur
LOOP
  tmp_var := trim('' || exclude_rec.str);
  tmp_var_int := nvl(INSTR(v_rate_description,tmp_var,1,1),0);
  IF (tmp_var_int <> 0) THEN
  tmp_flag_int := -1;
  return_str := '';
  EXIT;
  END IF;
END LOOP;
END IF;
END IF;
IF tmp_flag_int = 1 THEN
return_str := 'truely matched';
ELSE
return_str := '';
END IF;
ELSE
return_str := '';
END IF;
return return_str;
EXCEPTION
  WHEN OTHERS THEN
  --dbms_output.put_line('Exception');
    RAISE;
END FN_GET_RESTRICTION;