Problem in XML Parsing via oracle procedure...

Question

Hi,
I wrote one oracle procedure for xml parsing.
I have one valid xml file which has "encode UTF-8". The XML file contains some comments also. While we are parsing the xml file at that time it is not parse successfully and also it is not giving any error. After the following line it is skip rest of the codes(lines).
dbms_xmlparser.parseclob(l_parser, l_clob);
At the end of the xml file there are some comments which is like "<!-- abc --> ".
When I am changing the "encode UTF-8 to ISO-88596-1" & removing the comments which wrote on bottom of the file then its working fine, but the files which we are getting from the system is contains the encode UTF-8 and we don't want to preprocess on that xml files. Even if we will do that via shell script or perl script then it will be overhead to the system and in a single stroke our system will parse more than 5k xml files, so if we will do some preprocess on it, it will take some more time approx 1-2 minutes extra.
So, If someone knows any solution of this problem, then please guide & help me on this.
My xml file structure is as follows:-
<?xml version="1.0" encoding="UTF-8"?>
<mcd xmlns:HTML="http://www.w3.org/TR/REC-xml">
     <child>
          <child1>32.401 V5.5</child1>
          <child2>ZoneGate</child2>
     </child>
     <mc>
          <newid>
               <id>12</id>
          </newid>
          <mindex>
               <date>20111102180000</date>
               <mt>abc1</mt>
               <mt>abc2</mt>
               <mvalue>
                    <r>val_1</r>
                    <r>val_2</r>
               </mvalue>
          </mindex>
     </mc>
</mcd>
<!--
ALARM STATUS
morning 10
afternoon 14
evening 18
night 22
-->
<!--
PARAM:EID = 1
PARAM:GId = 3
PARAM:GSId = 0
--!>
And my oracle procedure is as follows:-
create or replace procedure loadXMLtotable(dir_name IN varchar2, xmlfile IN varchar2) AS
-- Defining the variables
ecode               NUMBER;
emesg           VARCHAR2(200);
l_bfile      BFILE;
l_clob      CLOB;
l_dest_offset      INTEGER:=1;
l_src_offset      INTEGER:=1;
l_Char_set_id      NUMBER := NLS_CHARSET_ID('UTF8');
l_lang_context      INTEGER := dbms_lob.default_lang_ctx;
l_warning           INTEGER;
l_parser dbms_xmlparser.Parser;
l_doc dbms_xmldom.DOMDocument;
l_nl1 dbms_xmldom.DOMNodeList;
l_nl2 dbms_xmldom.DOMNodeList;
l_n dbms_xmldom.DOMNode;
node1 dbms_xmldom.DOMNode;
colid integer ; -- column id used for identifying which column it belongs.
l_xmltype XMLTYPE;
sub_xmltype XMLTYPE;
num_nodes number;
l_index PLS_INTEGER;
l_subIndex           PLS_INTEGER;
starttime Date;
     temp_datatime VARCHAR(25);
columnname varchar2(300);
columnvalue varchar2(300);
-- creating a Type which is a type of "test_hem" table RowType, which I created in SVN server
TYPE tab_type IS TABLE OF test_hem%ROWTYPE;
t_tab tab_type := tab_type();
BEGIN
-- Passing the xmlfile and virtual directory name which we gave at the time of directory creation
l_bfile := BFileName('MY_FILES', xmlfile);
dbms_lob.createtemporary(l_clob, cache=>FALSE);
dbms_lob.open(l_bfile, dbms_lob.lob_readonly);
--dbms_lob.loadFromFile(dest_lob => l_clob,
-- src_lob => l_bfile,
-- amount => dbms_lob.getLength(l_bfile));
dbms_lob.loadclobfromfile(l_clob, l_bfile, dbms_lob.getlength(l_bfile),
l_dest_offset, l_src_offset, l_Char_set_id, l_lang_context, l_warning);
dbms_lob.close(l_bfile);
-- make sure implicit date conversions are performed correctly
dbms_session.set_nls('NLS_DATE_FORMAT','''YYYY-MON-DD HH24:MI:SS''');
dbms_output.put_line('Date format set');
-- Create a parser.
l_parser := dbms_xmlparser.newParser;
dbms_output.put_line('output 1');
-- Parse the document and create a new DOM document.
dbms_xmlparser.parseclob(l_parser, l_clob);
dbms_output.put_line(' passed parsing');
l_doc := dbms_xmlparser.getDocument(l_parser);
dbms_output.put_line(' passed getdocument');
-- Free resources associated with the CLOB and Parser now they are no longer needed.
dbms_lob.freetemporary(l_clob);
dbms_xmlparser.freeParser(l_parser);
-- Get a list of all the EMP nodes in the document using the XPATH syntax.
l_nl1 := dbms_xslprocessor.selectNodes(dbms_xmldom.makeNode(l_doc),'//mcd/child');
-- Loop through the list and create a new record in a tble collection
FOR cur_sel IN 0 .. dbms_xmldom.getLength(l_nl1) - 1 LOOP
l_n := dbms_xmldom.item(l_nl1, cur_sel);
t_tab.extend;
-- Use XPATH syntax to assign values to he elements of the collection.
dbms_xslprocessor.valueOf(l_n,'child1/text()',t_tab(t_tab.last).country);
-- putting the state and vendorname into the table rowtype
dbms_xslprocessor.valueOf(l_n,'child2/text()',t_tab(t_tab.last).state);
END LOOP;
-- getting the version and putting into the table rowtype
l_n := dbms_xslprocessor.selectSingleNode(dbms_xmldom.makeNode(l_doc),'//mcd/mc/newid/id');
dbms_xslprocessor.valueOf(l_n,'id/text()',t_tab(t_tab.last).id);
-- selecting the nodes whose starting tag is "mindex"
l_nl1 := dbms_xslprocessor.selectNodes(dbms_xmldom.makeNode(l_doc),'//mcd/mc/mindex');
-- checking the total number of nodes whose starting through "mi"
num_nodes := dbms_xmldom.getLength(l_nl1);
l_index := 1;
-- For loop to iterate the nodes.
FOR cur_sel IN 0 .. dbms_xmldom.getLength(l_nl1) - 1 LOOP
-- whole current node is selected and storing into the node1 variable
node1 := dbms_xmldom.item(l_nl1, cur_sel);
-- setting the xmltype as AL32UTF8
l_xmltype := xmltype(l_bfile, nls_charset_id('AL32UTF8'));
-- if selecting parent node containing the mt child node then only proceed else skip that parent node.
IF (l_xmltype.Existsnode('//mcd/mc/mindex[' || l_index || ']/mt') > 0 and l_xmltype.Existsnode('//mcd/mc/mindex[' || l_index || ']/mvalue/r') > 0) Then
-- fetch the datatime, convert it into to_date format and store it into table rowtype
temp_datatime := dbms_xslprocessor.valueOf(node1, 'date/text()');
t_tab(t_tab.last).data_time := to_char(to_date(temp_datatime, 'YYYYMmcDHH24MISS'));
l_subIndex := 1;
                              while (l_xmltype.Existsnode('//mcd/mc/mindex[' || l_index || ']/mt[' || l_subIndex || ']') > 0 and l_xmltype.Existsnode('//mcd/mc/mindex[' || l_index || ']/mvalue/r['|| l_subIndex || ']') > 0 ) LOOP
                              -- getting mt and corresponging mvalue/r values
dbms_xslprocessor.valueOf(node1,'mt[' || l_subIndex || ']/text()',columnname);
dbms_xslprocessor.valueOf(node1,'mvalue/r[' || l_subIndex || ']/text()',columnvalue);
l_subIndex := l_subIndex + 1;
-- getting the column to which this mapping belongs.
select columnid into colid from abc_table where columnname=name;
CASE colid
WHEN 1 THEN t_tab(t_tab.last).col1 := columnvalue;
                         WHEN 2 THEN t_tab(t_tab.last).col2 := columnvalue;
                         WHEN 3 THEN t_tab(t_tab.last).col3 := columnvalue;
ELSE dbms_output.put_line('No column mapping for counter ' || columnname) ;
END CASE; -- end of case statement.
END LOOP;
-- Insert data into the real table from the table collection.
FORALL i IN t_tab.first .. t_tab.last
INSERT INTO test_hem VALUES t_tab(i);
END IF;
l_index := l_index + 1;
COMMIT;
END LOOP;
commit;
EXCEPTION
WHEN OTHERS THEN
ecode := SQLCODE;
emesg := SQLERRM;
dbms_output.put_line(TO_CHAR(ecode) || '-' || emesg);
     dbms_lob.freetemporary(l_clob);
     dbms_xmlparser.freeParser(l_parser);
     dbms_xmldom.freeDocument(l_doc);
END;

Answer 1

Sorry Odie,
I am new to this site as well as PL/SQL. I am giving additional details which you had mentioned in your last comments.
our Oracle Database version is "10.2.0.4.0"
The structure of target table Instrument_Details is as follows:
Create table Instrument_Details (
instrument_id          Integer  Primary Key,
provider_name          Varchar2(32),
version_number          Varchar2(32),
location_id                  Integer,
installation_date             Date,
granularity                  Integer,
time_out                  Integer );
Note:- Here test_hem is alias of Instrument_details.
Here instrument_id is a primary key.
provider_name contains the child2 xml tag value.
version_number contains the child1 xml tag value.
location_id contains the newid/id value which is map to other table which fetching the location name corresponding to the location_id.
installation_date contains the date xml tag value.
Now we have created one mapping tables where we mapped the xml tag values "mt" with table column name means "abc1 = granularity", "abc2 = time_out" in that table.
these table column value are written under mvalue xml tag.
_Our Database Character set is_:-
NLS_CHARACTERSET WE8ISO8859P1
Now as you suggest me to format your code. I am writing the xml code and procedure code again.
My xml file structure is as follows:-
<?xml version="1.0" encoding="UTF-8"?>
<mcd xmlns:HTML="http://www.w3.org/TR/REC-xml">
  <child>
      <child1>32.401 V5.5</child1>
      <child2>ZoneGate</child2>
  </child>
  <mc>
      <newid>
           <id>12</id>
      </newid>
  <mindex>
       <date>20111102180000</date>
       <mt>abc1</mt>
       <mt>abc2</mt>
       <mvalue>
             <r>val_1</r>   -- here val_1 and val_2 are numeric values
             <r>val_2</r>
        </mvalue>
  </mindex>
  </mc>
</mcd>
<!--
ALARM STATUS
morning 10
afternoon 14
evening 18
night 22
-->
<!--
PARAM:EID = 1
PARAM:GId = 3
PARAM:GSId = 0
--!> And my oracle procedure is as follows:-
create or replace procedure loadXMLtotable(dir_name IN varchar2, xmlfile IN varchar2) AS
-- Defining the variables
ecode NUMBER;
emesg VARCHAR2(200);
l_bfile BFILE;
l_clob CLOB;
l_dest_offset INTEGER:=1;
l_src_offset INTEGER:=1;
l_Char_set_id NUMBER := NLS_CHARSET_ID('UTF8');
l_lang_context INTEGER := dbms_lob.default_lang_ctx;
l_warning INTEGER;
l_parser dbms_xmlparser.Parser;
l_doc dbms_xmldom.DOMDocument;
l_nl1 dbms_xmldom.DOMNodeList;
l_nl2 dbms_xmldom.DOMNodeList;
l_n dbms_xmldom.DOMNode;
node1 dbms_xmldom.DOMNode;
colid integer ; -- column id used for identifying which column it belongs.
l_xmltype XMLTYPE;
sub_xmltype XMLTYPE;
num_nodes number;
l_index PLS_INTEGER;
l_subIndex PLS_INTEGER;
starttime Date;
temp_datatime VARCHAR(25);
columnname varchar2(300);
columnvalue varchar2(300);
-- creating a Type which is a type of "Instrument_Details" table RowType, which I created in SVN server
TYPE tab_type IS TABLE OF Instrument_Details%ROWTYPE;
t_tab tab_type := tab_type();
BEGIN
-- Passing the xmlfile and virtual directory name which we gave at the time of directory creation
l_bfile := BFileName('MY_FILES', xmlfile);
dbms_lob.createtemporary(l_clob, cache=>FALSE);
dbms_lob.open(l_bfile, dbms_lob.lob_readonly);
--dbms_lob.loadFromFile(dest_lob => l_clob,
-- src_lob => l_bfile,
-- amount => dbms_lob.getLength(l_bfile));
dbms_lob.loadclobfromfile(l_clob, l_bfile, dbms_lob.getlength(l_bfile),
l_dest_offset, l_src_offset, l_Char_set_id, l_lang_context, l_warning);
dbms_lob.close(l_bfile);
-- make sure implicit date conversions are performed correctly
dbms_session.set_nls('NLS_DATE_FORMAT','''YYYY-MON-DD HH24:MI:SS''');
dbms_output.put_line('Date format set');
-- Create a parser.
l_parser := dbms_xmlparser.newParser;
dbms_output.put_line('output 1');
-- Parse the document and create a new DOM document.
dbms_xmlparser.parseclob(l_parser, l_clob);
*-- Below lines are skipping....*
dbms_output.put_line(' passed parsing');
l_doc := dbms_xmlparser.getDocument(l_parser);
dbms_output.put_line(' passed getdocument');
-- Free resources associated with the CLOB and Parser now they are no longer needed.
dbms_lob.freetemporary(l_clob);
dbms_xmlparser.freeParser(l_parser);
-- Get a list of all the EMP nodes in the document using the XPATH syntax.
l_nl1 := dbms_xslprocessor.selectNodes(dbms_xmldom.makeNode(l_doc),'//mcd/child');
-- Loop through the list and create a new record in a tble collection
FOR cur_sel IN 0 .. dbms_xmldom.getLength(l_nl1) - 1 LOOP
l_n := dbms_xmldom.item(l_nl1, cur_sel);
t_tab.extend;
-- Use XPATH syntax to assign values to he elements of the collection.
dbms_xslprocessor.valueOf(l_n,'child1/text()',t_tab(t_tab.last).country);
-- putting the state and vendorname into the table rowtype
   dbms_xslprocessor.valueOf(l_n,'child2/text()',t_tab(t_tab.last).state);
END LOOP;
-- getting the version and putting into the table rowtype
   l_n := dbms_xslprocessor.selectSingleNode(dbms_xmldom.makeNode(l_doc),'//mcd/mc/newid/id');
  dbms_xslprocessor.valueOf(l_n,'id/text()',t_tab(t_tab.last).id);
-- selecting the nodes whose starting tag is "mindex"
  l_nl1 := dbms_xslprocessor.selectNodes(dbms_xmldom.makeNode(l_doc),'//mcd/mc/mindex');
-- checking the total number of nodes whose starting through "mi"
  num_nodes := dbms_xmldom.getLength(l_nl1);
l_index := 1;
  -- For loop to iterate the nodes.
  FOR cur_sel IN 0 .. dbms_xmldom.getLength(l_nl1) - 1 LOOP
  -- whole current node is selected and storing into the node1 variable
  node1 := dbms_xmldom.item(l_nl1, cur_sel);
  -- setting the xmltype as AL32UTF8
   l_xmltype := xmltype(l_bfile, nls_charset_id('AL32UTF8'));
  -- if selecting parent node containing the mt child node then only proceed else skip that parent node.
    IF (l_xmltype.Existsnode('//mcd/mc/mindex[' || l_index || ']/mt') > 0 and l_xmltype.Existsnode('//mcd/mc/mindex[' || l_index || ']/mvalue/r') > 0) Then
  -- fetch the datatime, convert it into to_date format and store it into table rowtype
    temp_datatime := dbms_xslprocessor.valueOf(node1, 'date/text()');
    t_tab(t_tab.last).data_time := to_char(to_date(temp_datatime, 'YYYYMmcDHH24MISS'));
    l_subIndex := 1;
   while (l_xmltype.Existsnode('//mcd/mc/mindex[' || l_index || ']/mt[' || l_subIndex || ']') > 0 and l_xmltype.Existsnode('//mcd/mc/mindex[' || l_index || ']/mvalue/r['|| l_subIndex || ']') > 0 ) LOOP
-- getting mt and corresponging mvalue/r values
   dbms_xslprocessor.valueOf(node1,'mt[' || l_subIndex || ']/text()',columnname);
  dbms_xslprocessor.valueOf(node1,'mvalue/r[' || l_subIndex || ']/text()',columnvalue);
  l_subIndex := l_subIndex + 1;
  -- getting the column to which this mapping belongs.
  select columnid into colid from abc_table where columnname=name;
  CASE colid
  WHEN 1 THEN t_tab(t_tab.last).col1 := columnvalue;
  WHEN 2 THEN t_tab(t_tab.last).col2 := columnvalue;
  WHEN 3 THEN t_tab(t_tab.last).col3 := columnvalue;
      ELSE dbms_output.put_line('No column mapping for counter ' || columnname) ;
  END CASE; -- end of case statement.
  END LOOP;
-- Insert data into the real table from the table collection.
  FORALL i IN t_tab.first .. t_tab.last
    INSERT INTO test_hem VALUES t_tab(i);
  END IF;
  l_index := l_index + 1;
COMMIT;
END LOOP;
commit;
EXCEPTION
  WHEN OTHERS THEN
  ecode := SQLCODE;
  emesg := SQLERRM;
  dbms_output.put_line(TO_CHAR(ecode) || '-' || emesg);
  dbms_lob.freetemporary(l_clob);
  dbms_xmlparser.freeParser(l_parser);
  dbms_xmldom.freeDocument(l_doc);
END;Thanks in advance for your help...