Getting virtual memory error when fetching records from database

Question

HI,
I am using Oracle as Database and the Oracle JDBC driver. I have simple code which is a select statement but the problem is the resultset dies while fetching the data i.e. 5,50,000. And it gives me the memory error as its storing all in the memory. One of the way which i have serched in the old threads is using the batch method fetching rows at a time but can you tell me how to implement in my code. I am pasting my code.
The overall functionality of my code is that it's reterving data from database and generating an XML file that would be validated with a DTD.
//My Code
public class Invoicef3 implements ExtractF3 {
     final String queryString = "select * from hsbc_f3_statement
order by bill_no, duplicate,
invoice_address1,
invoice_address2,
invoice_address3,
invoice_address4,
invoice_address5,
invoice_address6,
main_section, order_1, page,
section, product_category,
sub_sect_1, order_2,
sub_sect_2, child_product,
sub_sect_3, account,
line,entry_date, currency, tier";
     public ArrayList process() {
          Connection con = null;
          Statement stmt = null;
          ResultSet rset = null;
          ArrayList arr1 = null;
          try {
               con =
ConnectionManager.getConnection();
               stmt = con.createStatement();
          rset = stmt.executeQuery(queryString);
               arr1 = new ArrayList();
               while (rset.next()) {
                    arr1.add(
                         new F3StatementExtract(
                              rset.getString(1),
                              rset.getInt(2),
                              rset.getString(3),
                              rset.getInt(4),
                              rset.getInt(5),
                              rset.getString(6),
                              rset.getInt(7),
                              rset.getString(8),
                              rset.getInt(9),
                              rset.getString(10),
                              rset.getInt(11)));
               rset.close();
               stmt.close();
          } catch (SQLException e) {
               e.printStackTrace();
          } finally {
               ConnectionManager.close(rset);
               ConnectionManager.close(stmt);
               ConnectionManager.close(con);
          return arr1;
}

Answer 1

The problem is that you are fetching and processing all the rows for the query, which the VM cannot handle given the heap space available. The points you could think over are:
* Allocate more heap memory (this would help only to a limited extent)
* Try to process only a few records at a time instead of all of them (there is actually no need to process all the records at a time. Try processing records in lots of say 1000)
* Avoid selecting all the columns [SELECT *] from the table, if all of them are not going to be used.
There is a slight change i have done in the code is that i am using two quereies now one is fetching all the Bills and the secondquery is fetching all the data for the relevant BILL.
//My Code
public class Invoicef3 implements ExtractF3 {
     /*Query to get distinct bill numbers*/
     final String queryString1 =
          "select distinct(bill_no) from hsbc_print_bills";
     /*Query to get distinct bill numbers statement details*/
     final String queryString =
          "select * from hsbc_f3_statement where bill_no='";
     public ArrayList process() {
          Connection con = null;
          Statement stmt = null;
          ResultSet rset = null;
          ArrayList arr1 = null;
          ArrayList arr2 = null;
          try {
               con = ConnectionManager.getConnection();
               stmt = con.createStatement();
               rset = stmt.executeQuery(queryString1);
               arr1 = new ArrayList();
               while (rset.next()) {
                    arr1.add(new F3BillExtract(rset.getString(1))); //generating the Bill_No's
               System.out.print(arr1.size());
               rset.close();
               stmt.close();
               for (int i = 0; i < arr1.size(); i++) {
                    stmt = con.createStatement();
                    rset =
                         stmt.executeQuery(
                              queryString
                                   + (((F3BillExtract) arr1.get(i)).getBill_No())
                                   + "'");
                    arr2 = new ArrayList();
                    /*Fetching the statement Details of the particular Bill_No*/
                    while (rset.next()) {
                         arr2.add(
                              new F3StatementExtract(
                                   rset.getString(1),
                                   rset.getInt(2),
                                   rset.getString(3),
                                   rset.getInt(4),
                                   rset.getInt(5),
                                   rset.getString(6),
                                   rset.getInt(7),
                                   rset.getString(8),
                                   rset.getInt(9),
                                   rset.getString(10),
                                   rset.getInt(11),
                                   rset.getString(12),
                                   rset.getFloat(13),
                                   rset.getDate(14),
                                   rset.getString(15),
                                   rset.getInt(16),
                                   rset.getString(17),
                                   rset.getString(18),
                                   rset.getString(19),
                                   rset.getString(20),
                                   rset.getString(21),
                                   rset.getString(22),
                                   rset.getString(23),
                                   rset.getString(24),
                                   rset.getString(25),
                                   rset.getString(26),
                                   rset.getString(27),
                                   rset.getString(28),
                                   rset.getString(29),
                                   rset.getString(30),
                                   rset.getDate(31),
                                   rset.getDate(32),
                                   rset.getDate(33),
                                   rset.getDate(34),
                                   rset.getString(35),
                                   rset.getString(36),
                                   rset.getString(37),
                                   rset.getString(38),
                                   rset.getString(39),
                                   rset.getString(40),
                                   rset.getFloat(41),
                                   rset.getFloat(42),
                                   rset.getFloat(43),
                                   rset.getInt(44),
                                   rset.getFloat(45),
                                   rset.getString(46),
                                   rset.getString(47)));
                    rset.close();
                    stmt.close();
                    ((F3BillExtract) arr1.get(i)).setArr(arr2);
          } catch (SQLException e) {
               e.printStackTrace();
          } finally {
               ConnectionManager.close(rset);
               ConnectionManager.close(stmt);
               ConnectionManager.close(con);
          return arr1;
}