Oracle Text - CTX Context Index Soundex Problem

Question

Hi,
I'm running into a problem with Oracle Text when searching using the ! (soundex) option. I've created a simple test example to highlight the issue.
Oracle Database 10g Enterprise Edition Release 10.2.0.4.0 - 64bit
Windows 2008 Server 64-bit
create table test_tab (test_col  varchar2(200));
insert all
  into test_tab (test_col) values ('ab-tönes')
  into test_tab (test_col) values ('ab-tones')
  into test_tab (test_col) values ('abtones')
  into test_tab (test_col) values ('ab tones')
  into test_tab (test_col) values ('ab-tanes')
  select * from dual
select * from test_tab
begin
      ctx_ddl.create_preference ('test_lex1', 'basic_lexer');
      ctx_ddl.set_attribute ('test_lex1', 'whitespace', '/\|-_+&''');
      ctx_ddl.set_attribute('test_lex1','base_letter','YES');
      -- ctx_ddl.set_attribute('test_lex1','skipjoins','-');
end;
create index test_idx on test_tab (test_col)
  indextype is ctxsys.context
    parameters
      ('lexer        test_lex1'     
select token_text from dr$test_idx$i;
TOKEN_TEXT
AB
ABTONES
TANES
TONES
select * from test_tab where contains (test_col, '!ab tones') > 0;
TEST_COL
ab-tönes
ab-tones
ab tones
select * from test_tab where soundex(test_col) = soundex('ab tones');
TEST_COL
ab-tönes
ab-tones
abtones
ab tones
ab-tanes
So my question is, can anyone suggest an approach whereby I can get the Oracle Text Context index (or CTXCAT index if it's more appropriate) to return all 5 rows like the simple Soundex is doing?
I can't really use soundex as this search query will form part of a search screen for a multi-language application. Soundex is limited to English sounding words, so I need the solution to be able to compare strings that may not "sound" English.
It must be an attribute of the BASIC_LEXER, and I've tried skipjoins, start/end-joins, stop lists, but I just cannot get the Soundex feature of Oracle Text to function like the SOUNDEX() function!
Looking at how the tokens are stored dr$test_idx$i I need Oracle Text to almost concat 'AB' and 'TONES' to search as a single string.
Any help greatly appreciated.
Thanks,

Answer 1

I am not getting the same problem that you are getting with the umlat, but I don't see what is different.  Please post the result of:
select ctx_report.create_index_script ('test_idx') from dual;
Here are the results on my system.  Perhaps you can spot the difference.  I added an empty_stoplist, so that it won't print out a long list of stopwords.
SCOTT@orcl12c> create table test_tab (test_col    varchar2(200))
  2  /
Table created.
SCOTT@orcl12c> insert all
  2    into test_tab (test_col) values ('ab-tönes')
  3    into test_tab (test_col) values ('ab-tones')
  4    into test_tab (test_col) values ('abtones')
  5    into test_tab (test_col) values ('ab tones')
  6    into test_tab (test_col) values ('ab-tanes')
  7  select * from dual
  8  /
5 rows created.
SCOTT@orcl12c> select * from test_tab
  2  /
TEST_COL
ab-tönes
ab-tones
abtones
ab tones
ab-tanes
5 rows selected.
SCOTT@orcl12c> begin
  2    ctx_ddl.create_preference ('test_lex1', 'basic_lexer');
  3    ctx_ddl.set_attribute('test_lex1','base_letter','YES');
  4  end;
  5  /
PL/SQL procedure successfully completed.
SCOTT@orcl12c> create or replace procedure test_proc
  2    (p_rowid in          rowid,
  3      p_clob    in out nocopy clob)
  4  as
  5  begin
  6    select replace (translate (test_col, '/\|-_+&''', '      '), ' ', '')
  7    into   p_clob
  8    from   test_tab
  9    where  rowid = p_rowid;
10  end test_proc;
11  /
Procedure created.
SCOTT@orcl12c> show errors
No errors.
SCOTT@orcl12c> begin
  2    ctx_ddl.create_preference ('test_ds', 'user_datastore');
  3    ctx_ddl.set_attribute ('test_ds', 'procedure', 'test_proc');
  4  end;
  5  /
PL/SQL procedure successfully completed.
SCOTT@orcl12c> create index test_idx on test_tab (test_col)
  2    indextype is ctxsys.context
  3    parameters
  4       ('lexer    test_lex1
  5         datastore    test_ds
  6         stoplist    ctxsys.empty_stoplist')
  7  /
Index created.
SCOTT@orcl12c> select token_text from dr$test_idx$i
  2  /
TOKEN_TEXT
ABTANES
ABTONES
2 rows selected.
SCOTT@orcl12c> variable search_string varchar2(100)
SCOTT@orcl12c> exec :search_string := 'ab tones'
PL/SQL procedure successfully completed.
SCOTT@orcl12c> select * from test_tab
  2  where  contains
  3            (test_col,
  4             '!' || replace (:search_string, ' ', ' !') ||
  5             ' or !' || replace (:search_string, ' ', '')) > 0
  6  /
TEST_COL
ab-tönes
ab-tones
abtones
ab tones
ab-tanes
5 rows selected.
SCOTT@orcl12c> exec :search_string := 'abtones'
PL/SQL procedure successfully completed.
SCOTT@orcl12c> /
TEST_COL
ab-tönes
ab-tones
abtones
ab tones
ab-tanes
5 rows selected.
SCOTT@orcl12c> exec :search_string := 'ab tönes'
PL/SQL procedure successfully completed.
SCOTT@orcl12c> /
TEST_COL
ab-tönes
ab-tones
abtones
ab tones
ab-tanes
5 rows selected.
SCOTT@orcl12c> select ctx_report.create_index_script ('test_idx') from dual
  2  /
CTX_REPORT.CREATE_INDEX_SCRIPT('TEST_IDX')
begin
  ctx_ddl.create_preference('"TEST_IDX_DST"','USER_DATASTORE');
  ctx_ddl.set_attribute('"TEST_IDX_DST"','PROCEDURE','"SCOTT"."TEST_PROC"');
end;
begin
  ctx_ddl.create_preference('"TEST_IDX_FIL"','NULL_FILTER');
end;
begin
  ctx_ddl.create_section_group('"TEST_IDX_SGP"','NULL_SECTION_GROUP');
end;
begin
  ctx_ddl.create_preference('"TEST_IDX_LEX"','BASIC_LEXER');
  ctx_ddl.set_attribute('"TEST_IDX_LEX"','BASE_LETTER','YES');
end;
begin
  ctx_ddl.create_preference('"TEST_IDX_WDL"','BASIC_WORDLIST');
  ctx_ddl.set_attribute('"TEST_IDX_WDL"','STEMMER','ENGLISH');
  ctx_ddl.set_attribute('"TEST_IDX_WDL"','FUZZY_MATCH','GENERIC');
end;
begin
  ctx_ddl.create_stoplist('"TEST_IDX_SPL"','BASIC_STOPLIST');
end;
begin
  ctx_ddl.create_preference('"TEST_IDX_STO"','BASIC_STORAGE');
  ctx_ddl.set_attribute('"TEST_IDX_STO"','R_TABLE_CLAUSE','lob (data) store as (
cache)');
  ctx_ddl.set_attribute('"TEST_IDX_STO"','I_INDEX_CLAUSE','compress 2');
end;
begin
  ctx_output.start_log('TEST_IDX_LOG');
end;
create index "SCOTT"."TEST_IDX"
  on "SCOTT"."TEST_TAB"
      ("TEST_COL")
  indextype is ctxsys.context
  parameters('
    datastore       "TEST_IDX_DST"
    filter          "TEST_IDX_FIL"
    section group   "TEST_IDX_SGP"
    lexer           "TEST_IDX_LEX"
    wordlist        "TEST_IDX_WDL"
    stoplist        "TEST_IDX_SPL"
    storage         "TEST_IDX_STO"
begin
  ctx_output.end_log;
end;
1 row selected.