Issues with language-specific characters and Multi Lexer

Question

I want to create a text index with global lexer and different languages. But how to create the index to satisfy all languages?
Oracle EE 10.2.0.4 (UTF8) on Solaris 10
1.) Create global lexer with german as default and czech, turkish as additional languages.
begin
     ctx_ddl.drop_preference('global_lexer');
     ctx_ddl.drop_preference('german_lexer');
     ctx_ddl.drop_preference('turkish_lexer');
     ctx_ddl.drop_preference('czech_lexer');
end;
begin
     ctx_ddl.create_preference('german_lexer','basic_lexer');
     ctx_ddl.create_preference('turkish_lexer','basic_lexer');
     ctx_ddl.create_preference('czech_lexer','basic_lexer');
     ctx_ddl.create_preference('global_lexer', 'multi_lexer');
end;
begin
     ctx_ddl.set_attribute('german_lexer','composite','german');
     ctx_ddl.set_attribute('german_lexer','mixed_case','no');
     ctx_ddl.set_attribute('german_lexer','alternate_spelling','german');
     ctx_ddl.set_attribute('german_lexer','base_letter','yes');
     ctx_ddl.set_attribute('german_lexer','base_letter_type','specific');
     ctx_ddl.set_attribute('german_lexer','printjoins','_');
     ctx_ddl.set_attribute('czech_lexer','mixed_case','no');
     ctx_ddl.set_attribute('czech_lexer','base_letter','yes');
     ctx_ddl.set_attribute('czech_lexer','base_letter_type','specific');
     ctx_ddl.set_attribute('czech_lexer','printjoins','_');
     ctx_ddl.set_attribute('turkish_lexer','mixed_case','no');
     ctx_ddl.set_attribute('turkish_lexer','base_letter','yes');
     ctx_ddl.set_attribute('turkish_lexer','base_letter_type','specific');
     ctx_ddl.set_attribute('turkish_lexer','printjoins','_');
     ctx_ddl.add_sub_lexer('global_lexer', 'default', 'german_lexer');
     ctx_ddl.add_sub_lexer('global_lexer', 'czech',   'czech_lexer',   'CZH');
     ctx_ddl.add_sub_lexer('global_lexer', 'turkish', 'turkish_lexer', 'TRH');
end;
/2.) Create table and insert data
drop table text_search;
create table text_search (
     lang   varchar2(5)
   , name   varchar2(100)
insert into text_search(lang, name) values ('DEH', 'Strauß');
insert into text_search(lang, name) values ('DEH', 'Möllbäck');
insert into text_search(lang, name) values ('TRH', 'Öğem');
insert into text_search(lang, name) values ('TRH', 'Öger');
insert into text_search(lang, name) values ('CZH', 'Tomáš');
insert into text_search(lang, name) values ('CZH', 'Černínová');
commit;3.) The index creation now produces different results depending on the language settings:
-- *Option A)*
alter session set nls_language=german;
drop index i_text_search;
create index i_text_search on text_search (name)
   indextype is ctxsys.context
        parameters ('
                section group CTXSYS.AUTO_SECTION_GROUP
                lexer global_lexer language column lang
                memory 300000000'
select * from dr$i_text_search$I;
-- *Option B)*
alter session set nls_language=turkish;
drop index i_text_search;
create index i_text_search on text_search (name)
   indextype is ctxsys.context
        parameters ('
                section group CTXSYS.AUTO_SECTION_GROUP
                lexer global_lexer language column lang
                memory 300000000'
select * from dr$i_text_search$I;
-- *Option C)*
alter session set nls_language=czech;
drop index i_text_search;
create index i_text_search on text_search (name)
   indextype is ctxsys.context
        parameters ('
                section group CTXSYS.AUTO_SECTION_GROUP
                lexer global_lexer language column lang
                memory 300000000'
select * from dr$i_text_search$I;And now I get different:
Option A)
dr$i_text_search$I with nls_language=german:
STRAUß
STRAUSS
MOLLBACK
OĞEM
OGER
TOMAŠ
ČERNINOVA
Problems, e.g.:
A turkish client now does not find his data (the select returns 0 rows)
alter session set nls_language=turkish;
select * from text_search
where contains (name, 'Öğem') > 0;
Option B)
dr$i_text_search$I with nls_language=turkish:
STRAUß
STRAUSS
MÖLLBACK
ÖĞEM
ÖGER
TOMAŠ
ČERNINOVA
Problems, e.g.:
A czech client now does not find his data (the select returns 0 rows)
alter session set nls_language=czech;
select * from text_search
where contains (name, 'Černínová') > 0;
Option C)
dr$i_text_search$I with nls_language=czech:
STRAUß
STRAUSS
MOLLBACK
OĞEM
OGER
TOMAS
CERNINOVA
Problems, e.g.:
A turkish client now does not find his data (the select returns 0 rows)
alter session set nls_language=turkish;
select * from text_search
where contains (name, 'Öğem') > 0;
----> How can these problems be avoided? What am I doing wrong?

Answer 1

You need to change your base_letter_type from specific to generic. Also, if you are going to use both alternate_spelling and base_letter in your german_lexer, then you might want to set override_base_letter to true. Please see the run of your code below, with those changes applied. The special characters got mangled in my spool file, but hopefully you get the idea.
SCOTT@orcl_11gR2> begin
  2            ctx_ddl.drop_preference('global_lexer');
  3            ctx_ddl.drop_preference('german_lexer');
  4            ctx_ddl.drop_preference('turkish_lexer');
  5            ctx_ddl.drop_preference('czech_lexer');
  6  end;
  7  /
PL/SQL procedure successfully completed.
SCOTT@orcl_11gR2> begin
  2            ctx_ddl.create_preference('german_lexer','basic_lexer');
  3            ctx_ddl.create_preference('turkish_lexer','basic_lexer');
  4            ctx_ddl.create_preference('czech_lexer','basic_lexer');
  5            ctx_ddl.create_preference('global_lexer', 'multi_lexer');
  6  end;
  7  /
PL/SQL procedure successfully completed.
SCOTT@orcl_11gR2> begin
  2            ctx_ddl.set_attribute('german_lexer','composite','german');
  3            ctx_ddl.set_attribute('german_lexer','mixed_case','no');
  4            ctx_ddl.set_attribute('german_lexer','alternate_spelling','german');
  5            ctx_ddl.set_attribute('german_lexer','base_letter','yes');
  6            ctx_ddl.set_attribute('german_lexer','base_letter_type','generic');
  7            ctx_ddl.set_attribute('german_lexer','override_base_letter', 'true');
  8            ctx_ddl.set_attribute('german_lexer','printjoins','_');
  9 
10            ctx_ddl.set_attribute('czech_lexer','mixed_case','no');
11            ctx_ddl.set_attribute('czech_lexer','base_letter','yes');
12            ctx_ddl.set_attribute('czech_lexer','base_letter_type','generic');
13            ctx_ddl.set_attribute('czech_lexer','printjoins','_');
14 
15            ctx_ddl.set_attribute('turkish_lexer','mixed_case','no');
16            ctx_ddl.set_attribute('turkish_lexer','base_letter','yes');
17            ctx_ddl.set_attribute('turkish_lexer','base_letter_type','generic');
18            ctx_ddl.set_attribute('turkish_lexer','printjoins','_');
19 
20            ctx_ddl.add_sub_lexer('global_lexer', 'default', 'german_lexer');
21            ctx_ddl.add_sub_lexer('global_lexer', 'czech',   'czech_lexer',   'CZH');
22            ctx_ddl.add_sub_lexer('global_lexer', 'turkish', 'turkish_lexer', 'TRH');
23  end;
24  /
PL/SQL procedure successfully completed.
SCOTT@orcl_11gR2> drop table text_search;
Table dropped.
SCOTT@orcl_11gR2> create table text_search (
  2         lang      varchar2(5)
  3       , name      varchar2(100)
  4  );
Table created.
SCOTT@orcl_11gR2> insert into text_search(lang, name) values ('DEH', 'Strauß');
1 row created.
SCOTT@orcl_11gR2> insert into text_search(lang, name) values ('DEH', 'Möllbäck');
1 row created.
SCOTT@orcl_11gR2> insert into text_search(lang, name) values ('TRH', 'Öğem');
1 row created.
SCOTT@orcl_11gR2> insert into text_search(lang, name) values ('TRH', 'Öger');
1 row created.
SCOTT@orcl_11gR2> insert into text_search(lang, name) values ('CZH', 'Tomáš');
1 row created.
SCOTT@orcl_11gR2> insert into text_search(lang, name) values ('CZH', 'ÄŒernÃnová');
1 row created.
SCOTT@orcl_11gR2> commit;
Commit complete.
SCOTT@orcl_11gR2>
SCOTT@orcl_11gR2> -- *Option A)*
SCOTT@orcl_11gR2> alter session set nls_language=german;
Session altered.
SCOTT@orcl_11gR2> drop index i_text_search;
drop index i_text_search
ERROR at line 1:
ORA-01418: Angegebener Index ist nicht vorhanden
SCOTT@orcl_11gR2> create index i_text_search on text_search (name)
  2       indextype is ctxsys.context
  3            parameters ('
  4                 section group CTXSYS.AUTO_SECTION_GROUP
  5                 lexer global_lexer language column lang
  6                 memory 300000000'
  7            );
Index created.
SCOTT@orcl_11gR2> select token_text from dr$i_text_search$I;
TOKEN_TEXT
AYEM
AŒERNA
CK
GER
LLBA
MA
NOVA
STRAUAY
TOMA
9 rows selected.
SCOTT@orcl_11gR2> alter session set nls_language=turkish;
Session altered.
SCOTT@orcl_11gR2> select * from text_search
  2  where contains (name, 'Öğem') > 0;
LANG
NAME
TRH
Öğem
1 row selected.
SCOTT@orcl_11gR2>
SCOTT@orcl_11gR2> -- *Option B)*
SCOTT@orcl_11gR2> alter session set nls_language=turkish;
Session altered.
SCOTT@orcl_11gR2> drop index i_text_search;
Index dropped.
SCOTT@orcl_11gR2> create index i_text_search on text_search (name)
  2       indextype is ctxsys.context
  3            parameters ('
  4                 section group CTXSYS.AUTO_SECTION_GROUP
  5                 lexer global_lexer language column lang
  6                 memory 300000000'
  7            );
Index created.
SCOTT@orcl_11gR2> select token_text from dr$i_text_search$I;
TOKEN_TEXT
AYEM
AŒERNA
CK
GER
LLBA
MA
NOVA
STRAUAY
TOMA
9 rows selected.
SCOTT@orcl_11gR2> alter session set nls_language=czech;
Session altered.
SCOTT@orcl_11gR2> select * from text_search
  2  where contains (name, 'ÄŒernÃnová') > 0;
LANG
NAME
CZH
ÄŒernÃnová
1 row selected.
SCOTT@orcl_11gR2>
SCOTT@orcl_11gR2> -- *Option C)*
SCOTT@orcl_11gR2> alter session set nls_language=czech;
Session altered.
SCOTT@orcl_11gR2> drop index i_text_search;
Index dropped.
SCOTT@orcl_11gR2> create index i_text_search on text_search (name)
  2       indextype is ctxsys.context
  3            parameters ('
  4                 section group CTXSYS.AUTO_SECTION_GROUP
  5                 lexer global_lexer language column lang
  6                 memory 300000000'
  7            );
Index created.
SCOTT@orcl_11gR2> select token_text from dr$i_text_search$I;
TOKEN_TEXT
AYEM
AŒERNA
CK
GER
LLBA
MA
NOVA
STRAUAY
TOMA
9 rows selected.
SCOTT@orcl_11gR2> alter session set nls_language=turkish;
Session altered.
SCOTT@orcl_11gR2> select * from text_search
  2  where contains (name, 'Öğem') > 0;
LANG
NAME
TRH
Öğem
1 row selected.
SCOTT@orcl_11gR2>