MULTI_STOPLIST

Question

I've created a custom STOPLIST with ...
begin
ctx_ddl.create_stoplist('multistop', 'MULTI_STOPLIST');
ctx_ddl.add_stopword('multistop', 'ab','german');
ctx_ddl.add_stopword('multistop', 'Ab','german');
ctx_ddl.add_stopword('multistop', 'AB','german');
ctx_ddl.add_stopword('multistop', 'dann','german');
ctx_ddl.add_stopword('multistop', 'Dann','german');
ctx_ddl.add_stopword('multistop', 'THIS','english');
ctx_ddl.add_stopword('multistop', 'whose','english');
ctx_ddl.add_stopword('multistop', 'Whose','english');
ctx_ddl.add_stopword('multistop', 'WHOSE','english');
ctx_ddl.add_stopword('multistop', 'and','english');
ctx_ddl.add_stopword('multistop', 'And','english');
ctx_ddl.add_stopword('multistop', 'AND','english');
end;
exec ctx_ddl.create_preference('english_lexer','basic_lexer');
exec ctx_ddl.create_preference('german_lexer','basic_lexer');
exec ctx_ddl.set_attribute('german_lexer','composite','german');
exec ctx_ddl.set_attribute('german_lexer','mixed_case','yes');
exec ctx_ddl.set_attribute('german_lexer','index_themes','yes');
exec ctx_ddl.set_attribute('german_lexer','theme_language','german');
exec ctx_ddl.set_attribute('german_lexer','alternate_spelling','german');
exec ctx_ddl.create_preference('global_lexer', 'multi_lexer');
exec ctx_ddl.add_sub_lexer('global_lexer','default','german_lexer');
exec ctx_ddl.add_sub_lexer('global_lexer','english','english_lexer','eng');
create index doc_idx on documents(text_file) indextype is ctxsys.context
parameters ('DATASTORE CTXSYS.DIRECT_DATASTORE lexer global_lexer language column sprache_id STOPLIST multistop filter ctxsys.auto_filter');
after setting it to the PARAMETER STOPLIST for my index and creating the index I use the procedure CTX_DOC.TOKENS() to show the tokens of a document. And there are also terms of the STOPLIST included. WHATS WRONG ? Seems to be the STOPLIST isn't in use.

Answer 1

Although the documentation says that ctx_doc.tokens does not return stopwords, that seems to be false. In the example below, although there are stopwords from ctx_doc.tokens, they are not in the token_text column of the dr$doc_idx$i table.
SCOTT@orcl_11g>
SCOTT@orcl_11g> begin
  2    ctx_ddl.create_stoplist('multistop', 'MULTI_STOPLIST');
  3    ctx_ddl.add_stopword('multistop', 'ab','german');
  4    ctx_ddl.add_stopword('multistop', 'Ab','german');
  5    ctx_ddl.add_stopword('multistop', 'AB','german');
  6    ctx_ddl.add_stopword('multistop', 'dann','german');
  7    ctx_ddl.add_stopword('multistop', 'Dann','german');
  8    ctx_ddl.add_stopword('multistop', 'this','english');
  9    ctx_ddl.add_stopword('multistop', 'This','english');
10    ctx_ddl.add_stopword('multistop', 'THIS','english');
11    ctx_ddl.add_stopword('multistop', 'whose','english');
12    ctx_ddl.add_stopword('multistop', 'Whose','english');
13    ctx_ddl.add_stopword('multistop', 'WHOSE','english');
14    ctx_ddl.add_stopword('multistop', 'and','english');
15    ctx_ddl.add_stopword('multistop', 'And','english');
16    ctx_ddl.add_stopword('multistop', 'AND','english');
17  end;
18  /
PL/SQL procedure successfully completed.
SCOTT@orcl_11g> begin
  2    ctx_ddl.create_preference('english_lexer','basic_lexer');
  3    ctx_ddl.create_preference('german_lexer','basic_lexer');
  4    ctx_ddl.set_attribute('german_lexer','composite','german');
  5    ctx_ddl.set_attribute('german_lexer','mixed_case','yes');
  6    ctx_ddl.set_attribute('german_lexer','alternate_spelling','german');
  7    ctx_ddl.create_preference('global_lexer', 'multi_lexer');
  8    ctx_ddl.add_sub_lexer('global_lexer','default','german_lexer' );
  9    ctx_ddl.add_sub_lexer('global_lexer','english','english_lexer', 'eng');
10    ctx_ddl.add_sub_lexer('global_lexer','german','german_lexer', 'ger');
11  end;
12  /
PL/SQL procedure successfully completed.
SCOTT@orcl_11g> create table documents (id number primary key, text_file varchar2 (60), sprache_id varchar2 (3))
  2  /
Table created.
SCOTT@orcl_11g> insert into documents values (1, 'testing this whose and whatever', 'eng')
  2  /
1 row created.
SCOTT@orcl_11g> insert into documents values (2, 'Ich habe nichts zu sagen ab dann.', 'ger')
  2  /
1 row created.
SCOTT@orcl_11g> create index doc_idx on documents(text_file) indextype is ctxsys.context
  2  parameters
  3    ('DATASTORE       CTXSYS.DIRECT_DATASTORE
  4        lexer            global_lexer
  5        language column  sprache_id
  6        STOPLIST       multistop
  7        filter        ctxsys.auto_filter')
  8  /
Index created.
SCOTT@orcl_11g> declare
  2    the_tokens ctx_doc.token_tab;
  3  begin
  4    ctx_doc.set_key_type ('PRIMARY_KEY');
  5    ctx_doc.tokens('doc_idx','1',the_tokens);
  6    for i in 1..the_tokens.count loop
  7        dbms_output.put_line(the_tokens(i).token);
  8    end loop;
  9    ctx_doc.tokens('doc_idx','2',the_tokens);
10    for i in 1..the_tokens.count loop
11        dbms_output.put_line(the_tokens(i).token);
12    end loop;
13  end;
14  /
TESTING
THIS
WHOSE
AND
WHATEVER
Ich
habe
nichts
zu
sagen
ab
dann
PL/SQL procedure successfully completed.
SCOTT@orcl_11g> select token_text from dr$doc_idx$i
  2  /
TOKEN_TEXT
Ich
TESTING
WHATEVER
habe
nichts
sagen
zu
7 rows selected.
SCOTT@orcl_11g>