Info Retrieval term frequency (td.idf) calculation problem

Question

anyone who's done any IR work like search engines should be familiar with the td.idf forumula i've tried to use. problem is, i've got it wrong somewhere. The results come back the same for each document and sometimes i get negative results.
<%@ page language="java" contentType="text/html"
    errorPage="errorpage.jsp" import="java.util.*,java.sql.*,java.lang.Math.*" %>
<%
    String strUser= "cooks";
    String strPass= "d885764c";
    String strConnectURL= ("jdbc:mysql://localhost/cooks");
    Class.forName("com.mysql.jdbc.Driver");
    Connection conn = DriverManager.getConnection(strConnectURL,strUser,strPass);
    Statement stmt = conn.createStatement();
    ResultSet rs;
    String sql;
    String searchTerm = request.getParameter("search");
    ArrayList score = new ArrayList();
    ArrayList product = new ArrayList();     
    final int recall = 20;
    ArrayList splitTerms = new ArrayList();
    StringTokenizer st = new StringTokenizer(searchTerm);
    while(st.hasMoreTokens())
        splitTerms.add(st.nextToken());
    String tempSearchTerm = ""+splitTerms.get(0);
    if(splitTerms.size() == 1 && tempSearchTerm.length() == 5)
        try
            int productCode = Integer.parseInt(""+splitTerms.get(0));
            sql = "SELECT id FROM product WHERE id = "+productCode+";";
            rs = stmt.executeQuery(sql);
            while(rs.next())
                    response.sendRedirect("product.jsp?productID="+rs.getString(1));
        catch(NumberFormatException e){}
    ArrayList docBaseN = new ArrayList();
    ArrayList docBaseD = new ArrayList();
    ArrayList docBaseNid = new ArrayList();
    ArrayList docBaseDid = new ArrayList();
    // ------------------------ Get items that have part of searchTerm in name -------------------------- //
    sql = "SELECT id, name FROM product WHERE UPPER(name) LIKE UPPER('%";
    for(int i=0; i<splitTerms.size(); i++)
        sql += splitTerms.get(i) + "%')";
        if(i < splitTerms.size()-1)
            sql += " OR description LIKE UPPER('%";
    sql += ";";
    rs = stmt.executeQuery(sql);
    while(rs.next())
        docBaseNid.add(rs.getInt(1));
        docBaseN.add(rs.getString(2));
    out.print("<br><hr><strong>GET RELEVANT NAMED ITEMS: </strong>\t"+sql+"<br>");
    // --------------------- Get items that have part of searchTerm in description ---------------------- //
    sql = "SELECT id, description FROM product WHERE UPPER(description) LIKE UPPER('%";
    for(int i=0; i<splitTerms.size(); i++)
        sql += splitTerms.get(i) + "%')";
        if(i < splitTerms.size()-1)
            sql += " OR description LIKE UPPER('%";
    sql += ";";
    rs = stmt.executeQuery(sql);
    while(rs.next())
        docBaseDid.add(rs.getInt(1));
        docBaseD.add(rs.getString(2));
    out.print("<strong>GET RELEVANT DESCRIBED ITEMS: </strong>\t"+sql+"<hr>");
    rs.close();
    stmt.close();
    conn.close();
    // ---------------------------------- Calculate TD.IDF for n ---------------------------------------- //
    double TDn = 0.0;
    double[] TDIDFn = new double[docBaseN.size()];
    double Nn = 0;
    int sumN = 0;
    for(int i=0; i<docBaseN.size(); i++)
        st = new StringTokenizer(""+docBaseN.get(i));
        sumN += st.countTokens();
        while(st.hasMoreTokens())
            String baseTerm = st.nextToken();
            for(int j=0; j<splitTerms.size(); j++)
                session.setAttribute("debug",136);
                if(baseTerm.equalsIgnoreCase(""+splitTerms.get(j)))
                    Nn ++;
    try
        out.print("Nn = "+Nn+"\tsumN = "+sumN+" ");
        TDn = Nn/sumN;
        out.print(TDn);
    catch(Exception e)
        TDn = 0;
    double[] IDFn = new double[docBaseN.size()];
    int Dn = docBaseN.size();
    int[] docsWithTn = new int[docBaseN.size()];
    for(int i=0; i<docBaseN.size(); i++)
        IDFn[i] = 0.00;
        TDIDFn[i] = 0.00;
        docsWithTn[i] = 0;
    for(int i=0; i<splitTerms.size(); i++)
        for(int j=0; j<docBaseN.size(); j++)
            st = new StringTokenizer(""+docBaseN.get(j));
            out.print("tokens = "+st.countTokens()+"<p>");
            compare:
                while(st.hasMoreTokens())
                    String tempStrA = ""+st.nextToken();
                    String tempStrB = ""+splitTerms.get(i);
                    out.print(tempStrA+" "+tempStrB+"<br>");
                    if(tempStrA.equalsIgnoreCase(tempStrB))
                        out.print("found match <p>");
                        docsWithTn[j] ++;
                        out.print(docsWithTn[j]+"<p>");
                        break compare;
    for(int i=0; i<docsWithTn.length; i++)
        try
            IDFn[i] += 1-(Math.log(Dn/docsWithTn));
out.print("Dn = "+Dn+"<br>");
out.print("docsWithTn["+i+"] = "+docsWithTn[i]+"<br>");
out.print("IDFn["+i+"] = "+IDFn[i]+"<br>");
catch(Exception e)
IDFn[i] = 0;
out.print("IDFn["+i+"] = "+IDFn[i]+"<br>");
for(int i=0; i<IDFn.length; i++)
TDIDFn[i] += TDn*(IDFn[i]);
out.print("<br><strong>id = "+docBaseNid.get(i)+" \tTDIDFn["+i+"] = "+TDIDFn[i]+"</strong>");
out.print("<p>");
// ---------------------------------- Calculate TD.IDF for d ---------------------------------------- //
double TDd = 0.0;
double[] TDIDFd = new double[docBaseD.size()];
double Nd = 0;
int sumD = 0;
for(int i=0; i<docBaseD.size(); i++)
st = new StringTokenizer(""+docBaseD.get(i));
sumD += st.countTokens();
while(st.hasMoreTokens())
String baseTerm = st.nextToken();
for(int j=0; j<splitTerms.size(); j++)
if(baseTerm.equalsIgnoreCase(""+splitTerms.get(j)))
Nd ++;
try
out.print("Nd = "+Nd+"\tsumD = "+sumD+" ");
TDd = Nd/sumD;
out.print(TDd);
catch(Exception e)
TDd = 0;
double[] IDFd = new double[docBaseD.size()];
int Dd = docBaseD.size();
int[] docsWithTd = new int[docBaseD.size()];
for(int i=0; i<docBaseD.size(); i++)
IDFd[i] = 0.00;
TDIDFd[i] = 0.00;
docsWithTd[i] = 0;
for(int i=0; i<splitTerms.size(); i++)
for(int j=0; j<docBaseD.size(); j++)
st = new StringTokenizer(""+docBaseD.get(j));
out.print("tokens = "+st.countTokens()+"<p>");
compare:
while(st.hasMoreTokens())
String tempStrA = ""+st.nextToken();
String tempStrB = ""+splitTerms.get(i);
out.print(tempStrA+" "+tempStrB+"<br>");
if(tempStrA.equalsIgnoreCase(tempStrB))
out.print("found match <p>");
docsWithTd[j] ++;
out.print(docsWithTd[j]+"<p>");
break compare;
for(int i=0; i<docsWithTd.length; i++)
try
IDFd[i] += 1-(Math.log(Dd/docsWithTd[i]));
out.print("Dd = "+Dd+"<br>");
out.print("docsWithTd["+i+"] = "+docsWithTd[i]+"<br>");
out.print("IDFd["+i+"] = "+IDFd[i]+"<br>");
catch(Exception e)
IDFd[i] = 0;
out.print("IDFd["+i+"] = "+IDFd[i]+"<br>");
for(int i=0; i<IDFd.length; i++)
TDIDFd[i] += TDd*(IDFd[i]);
out.print("<br><strong>id = "+docBaseDid.get(i)+" \tTDIDFd["+i+"] = "+TDIDFd[i]+"</strong>");
%>

Answer 1

ricky171
Welcome to the forum. Please don't post in old threads that are long dead. When you have a question, please start a topic of your own. Feel free to provide a link to an old thread if relevant.
Also, please don't solicit off-forum communication by email. This not only goes against the spirit of the forum, which is to share problems and their solutions, but will also fetch you plenty of spam, but rarely any help.
I'm locking this thread now. It's more than 4½ years old.
Also blocking both the posts soliciting off-forum communication.
db