Crawler Help: Object reference not set to an instance of an object

Question

I'm trying to write a custom crawler and having some difficulties.  I'm getting the document information from a database.  I'm trying to have the ClickThroughURL be a web URL and the IndexingURL be a UNC path to the file on a back-end file share.  Also, I'm not using DocFetch.  The problem I'm having is that when the crawler runs I get the following error for every card:
"4/19/05 13:43:30- (940) Aborted Card creation for document: TestDoc1.  Import error: IDispatch error #19876 (0x80044fa4): [Error Importing Card.
Error writing Indexing File.
SOAP fault: faultcode='soap:Server' faultstring='Server was unable to process request. --> Object reference not set to an instance of an object.']"
Has anyone seen this before?  Any help you can provide would be greatly appreciated.  I have included the code from my document.vb in case that helps.
Thanks,
Jerry
DOCUMENT.VB
Imports System
Imports Plumtree.Remote.Util
Imports Plumtree.Remote.Crawler
Imports System.Resources
Imports System.Globalization
Imports System.Threading
Imports System.IO
Imports System.Data.SqlClient
Imports System.Text
Namespace Plumtree.Remote.CWS.MoFoDocsOpen
    Public Class Document
        Implements IDocument
        Private m_logger As ICrawlerLog
        Private DocumentLocation As String
        Private d_DocumentNumber As Integer
        Private d_Library As String
        Private d_Name As String
        Private d_Author As String
        Private d_AuthorID As String
        Private d_Category As String
        Private d_ClientName As String
        Private d_ClientNumber As String
        Private d_DateCreated As DateTime
        Private d_DocumentName As String
        Private d_DocumentType As String
        Private d_EnteredBy As String
        Private d_EnteredByID As String
        Private d_FolderID As String
        Private d_KEFlag As String
        Private d_LastEdit As DateTime
        Private d_LastEditBy As String
        Private d_LastEditByID As String
        Private d_Maintainer As String
        Private d_MaintainerID As String
        Private d_MatterName As String
        Private d_MatterNumber As String
        Private d_Practice As String
        Private d_Description As String
        Private d_Version As Integer
        Private d_Path As String
        Private d_FileName As String
        Public Sub New(ByVal provider As DocumentProvider, ByVal documentLocation As String, ByVal signature As String)
            Dim location() As String = DocumentLocation.Split("||")
            Me.DocumentLocation = DocumentLocation
            Me.d_DocumentNumber = location(0)
            Me.d_Library = location(2)
            Dim objConn As New SqlConnection
            Dim objCmd As New SqlCommand
            Dim objRec As SqlDataReader
            objConn.ConnectionString = "Server=sad2525;Database=PortalDocs;Uid=sa;Pwd=;"
            objConn.Open()
            objCmd.CommandText = "SELECT * FROM DocsOpenAggregate WHERE Library = '" & Me.d_Library & "' AND DocumentNumber = " & Me.d_DocumentNumber
            objCmd.Connection = objConn
            objRec = objCmd.ExecuteReader()
            Do While objRec.Read() = True
                Me.d_Name = objRec("Name")
                Me.d_Author = objRec("Author")
                Me.d_AuthorID = objRec("AuthorID")
                Me.d_Category = objRec("Category")
                Me.d_ClientName = objRec("ClientName")
                Me.d_ClientNumber = objRec("ClientNumber")
                Me.d_DateCreated = objRec("DateCreated")
                Me.d_DocumentName = objRec("DocumentName")
                Me.d_DocumentType = objRec("DocumentType")
                Me.d_EnteredBy = objRec("EnteredBy")
                Me.d_EnteredByID = objRec("EnteredByID")
                Me.d_FolderID = objRec("FolderID")
                Me.d_KEFlag = objRec("KEFlag")
                Me.d_LastEdit = objRec("LastEdit")
                Me.d_LastEditBy = objRec("LastEditBy")
                Me.d_LastEditByID = objRec("LastEditByID")
                Me.d_Maintainer = objRec("Maintainer")
                Me.d_MaintainerID = objRec("MaintainerID")
                Me.d_MatterName = objRec("MatterName")
                Me.d_MatterNumber = objRec("MatterNumber")
                Me.d_Practice = objRec("Practice")
                Me.d_Description = objRec("Description")
                Me.d_Version = objRec("Version")
                Me.d_Path = objRec("Path")
                Me.d_FileName = objRec("FileName")
            Loop
            objCmd = Nothing
            If objRec.IsClosed = False Then objRec.Close()
            objRec = Nothing
            If objConn.State <> ConnectionState.Closed Then objConn.Close()
            objConn = Nothing
        End Sub
        'If using DocFetch, this method returns a file path to the document in the backend repository.
        Public Function GetDocument() As String Implements IDocument.GetDocument
            m_logger.Log(&#034;Document.GetDocument called for &#034; & Me.DocumentLocation)
            Return Me.d_Path
        End Function
        'Returns the metadata information about this document.
        Public Function GetMetaData(ByVal aFilter() As String) As DocumentMetaData Implements IDocument.GetMetaData
            m_logger.Log(&#034;Document.GetMetaData called for &#034; & DocumentLocation)
            Dim DOnvp(23) As NamedValue
            DOnvp(0) = New NamedValue(&#034;Author&#034;, Me.d_Author)
            DOnvp(1) = New NamedValue(&#034;AuthorID&#034;, Me.d_AuthorID)
            DOnvp(2) = New NamedValue(&#034;Category&#034;, Me.d_Category)
            DOnvp(3) = New NamedValue(&#034;ClientName&#034;, Me.d_ClientName)
            DOnvp(4) = New NamedValue(&#034;ClientNumber&#034;, Me.d_ClientNumber)
            DOnvp(5) = New NamedValue(&#034;DateCreated&#034;, Me.d_DateCreated)
            DOnvp(6) = New NamedValue(&#034;DocumentName&#034;, Me.d_DocumentName)
            DOnvp(7) = New NamedValue(&#034;DocumentType&#034;, Me.d_DocumentType)
            DOnvp(8) = New NamedValue(&#034;EnteredBy&#034;, Me.d_EnteredBy)
            DOnvp(9) = New NamedValue(&#034;EnteredByID&#034;, Me.d_EnteredByID)
            DOnvp(10) = New NamedValue(&#034;FolderID&#034;, Me.d_FolderID)
            DOnvp(11) = New NamedValue(&#034;KEFlag&#034;, Me.d_KEFlag)
            DOnvp(12) = New NamedValue(&#034;LastEdit&#034;, Me.d_LastEdit)
            DOnvp(13) = New NamedValue(&#034;LastEditBy&#034;, Me.d_LastEditBy)
            DOnvp(14) = New NamedValue(&#034;LastEditByID&#034;, Me.d_LastEditByID)
            DOnvp(15) = New NamedValue(&#034;Maintainer&#034;, Me.d_Maintainer)
            DOnvp(16) = New NamedValue(&#034;MaintainerID&#034;, Me.d_MaintainerID)
            DOnvp(17) = New NamedValue(&#034;MatterName&#034;, Me.d_MatterName)
            DOnvp(18) = New NamedValue(&#034;MatterNumber&#034;, Me.d_MatterNumber)
            DOnvp(19) = New NamedValue(&#034;Practice&#034;, Me.d_Practice)
            DOnvp(20) = New NamedValue(&#034;Description&#034;, Me.d_Description)
            DOnvp(21) = New NamedValue(&#034;Version&#034;, Me.d_Version)
            DOnvp(22) = New NamedValue(&#034;Path&#034;, Me.d_Path)
            DOnvp(23) = New NamedValue(&#034;FileName&#034;, Me.d_FileName)
            Dim metaData As New DocumentMetaData(DOnvp)
            Dim strExt As String = Right(Me.d_FileName, Len(Me.d_FileName) - InStrRev(Me.d_FileName, &#034;.&#034;))
            Select Case LCase(strExt)
                Case &#034;xml&#034;
                    metaData.ContentType = &#034;text/xml&#034;
                    metaData.ImageUUID = &#034;{F8F6B82F-53C6-11D2-88B7-006008168DE5}&#034;
                Case &#034;vsd&#034;
                    metaData.ContentType = &#034;application/vnd.visio&#034;
                    metaData.ImageUUID = &#034;{2CEEC472-7CF0-11d3-BB3A-00105ACE365C}&#034;
                Case &#034;mpp&#034;
                    metaData.ContentType = &#034;application/vnd.ms-project&#034;
                    metaData.ImageUUID = &#034;{8D6D9F50-D512-11d3-8DB0-00C04FF44474}&#034;
                Case &#034;pdf&#034;
                    metaData.ContentType = &#034;application/pdf&#034;
                    metaData.ImageUUID = &#034;{64FED895-D031-11D2-8909-006008168DE5}&#034;
                Case &#034;doc&#034;
                    metaData.ContentType = &#034;application/msword&#034;
                    metaData.ImageUUID = &#034;{0C35DD71-6453-11D2-88C3-006008168DE5}&#034;
                Case &#034;dot&#034;
                    metaData.ContentType = &#034;application/msword&#034;
                    metaData.ImageUUID = &#034;{0C35DD71-6453-11D2-88C3-006008168DE5}&#034;
                Case &#034;rtf&#034;
                    metaData.ContentType = &#034;text/richtext&#034;
                    metaData.ImageUUID = &#034;{F8F6B82F-53C6-11D2-88B7-006008168DE5}&#034;
                Case &#034;xls&#034;
                    metaData.ContentType = &#034;application/vnd.ms-excel&#034;
                    metaData.ImageUUID = &#034;{0C35DD72-6453-11D2-88C3-006008168DE5}&#034;
                Case &#034;xlt&#034;
                    metaData.ContentType = &#034;application/vnd.ms-excel&#034;
                    metaData.ImageUUID = &#034;{0C35DD72-6453-11D2-88C3-006008168DE5}&#034;
                Case &#034;pps&#034;
                    metaData.ContentType = &#034;application/vnd.ms-powerpoint&#034;
                    metaData.ImageUUID = &#034;{0C35DD73-6453-11D2-88C3-006008168DE5}&#034;
                Case &#034;ppt&#034;
                    metaData.ContentType = &#034;application/vnd.ms-powerpoint&#034;
                    metaData.ImageUUID = &#034;{0C35DD73-6453-11D2-88C3-006008168DE5}&#034;
                Case &#034;htm&#034;
                    metaData.ContentType = &#034;text/html&#034;
                    metaData.ImageUUID = &#034;{D2E2D5E0-84C9-11D2-A0C5-0060979C42D8}&#034;
                Case &#034;html&#034;
                    metaData.ContentType = &#034;text/html&#034;
                    metaData.ImageUUID = &#034;{D2E2D5E0-84C9-11D2-A0C5-0060979C42D8}&#034;
                Case &#034;asp&#034;
                    metaData.ContentType = &#034;text/plain&#034;
                    metaData.ImageUUID = &#034;{F8F6B82F-53C6-11D2-88B7-006008168DE5}&#034;
                Case &#034;idq&#034;
                    metaData.ContentType = &#034;text/plain&#034;
                    metaData.ImageUUID = &#034;{F8F6B82F-53C6-11D2-88B7-006008168DE5}&#034;
                Case &#034;txt&#034;
                    metaData.ContentType = &#034;text/plain&#034;
                    metaData.ImageUUID = &#034;{F8F6B82F-53C6-11D2-88B7-006008168DE5}&#034;
                Case &#034;log&#034;
                    metaData.ContentType = &#034;text/plain&#034;
                    metaData.ImageUUID = &#034;{F8F6B82F-53C6-11D2-88B7-006008168DE5}&#034;
                Case &#034;sql&#034;
                    metaData.ContentType = &#034;text/plain&#034;
                    metaData.ImageUUID = &#034;{F8F6B82F-53C6-11D2-88B7-006008168DE5}&#034;
                Case Else
                    metaData.ContentType = &#034;application/octet-stream&#034;
                    metaData.ImageUUID = &#034;{F8F6B82F-53C6-11D2-88B7-006008168DE5}&#034;
            End Select
            metaData.Name = Me.d_Name
            metaData.Description = Me.d_Description
            metaData.FileName = Me.d_FileName ' This is a file name - for example &#034;2jd005_.DOC&#034;
            metaData.IndexingURL = Me.d_Path ' This is a file path - for example &#034;\\fileserver01\docsd$\SF01\DOCS\MLS1\NONE\2jd005_.DOC&#034;
            metaData.ClickThroughURL = &#034;http://mofoweb/docsopen.asp?Unique=&#034; & HttpUtility.HtmlEncode(Me.DocumentLocation)
            metaData.UseDocFetch = False
            Return metaData
        End Function
        'Returns the signature or last-modified-date of this document that indicates to the portal whether the document needs refreshing.
        Public Function GetDocumentSignature() As String Implements IDocument.GetDocumentSignature
            Dim SigString As New StringBuilder
            Dim SigEncode As String
            SigString.Append(Me.d_DocumentNumber & &#034;||&#034;)
            SigString.Append(Me.d_Library & &#034;||&#034;)
            SigString.Append(Me.d_Name & &#034;||&#034;)
            SigString.Append(Me.d_Author & &#034;||&#034;)
            SigString.Append(Me.d_AuthorID & &#034;||&#034;)
            SigString.Append(Me.d_Category & &#034;||&#034;)
            SigString.Append(Me.d_ClientName & &#034;||&#034;)
            SigString.Append(Me.d_ClientNumber & &#034;||&#034;)
            SigString.Append(Me.d_DateCreated & &#034;||&#034;)
            SigString.Append(Me.d_DocumentName & &#034;||&#034;)
            SigString.Append(Me.d_DocumentType & &#034;||&#034;)
            SigString.Append(Me.d_EnteredBy & &#034;||&#034;)
            SigString.Append(Me.d_EnteredByID & &#034;||&#034;)
            SigString.Append(Me.d_FolderID & &#034;||&#034;)
            SigString.Append(Me.d_KEFlag & &#034;||&#034;)
            SigString.Append(Me.d_LastEdit & &#034;||&#034;)
            SigString.Append(Me.d_LastEditBy & &#034;||&#034;)
            SigString.Append(Me.d_LastEditByID & &#034;||&#034;)
            SigString.Append(Me.d_Maintainer & &#034;||&#034;)
            SigString.Append(Me.d_MaintainerID & &#034;||&#034;)
            SigString.Append(Me.d_MatterName & &#034;||&#034;)
            SigString.Append(Me.d_MatterNumber & &#034;||&#034;)
            SigString.Append(Me.d_Practice & &#034;||&#034;)
            SigString.Append(Me.d_Description & &#034;||&#034;)
            SigString.Append(Me.d_Version & &#034;||&#034;)
            SigString.Append(Me.d_Path & &#034;||&#034;)
            SigString.Append(Me.d_FileName & &#034;||&#034;)
            Dim encoding As New UTF8Encoding
            Dim byteArray As Byte() = encoding.GetBytes(SigString.ToString())
            SigEncode = System.Convert.ToBase64String(byteArray, 0, byteArray.Length)
            Return SigEncode
        End Function
        'Returns an array of the users with access to this document.
        Public Function GetUsers() As ACLEntry() Implements IDocument.GetUsers
            'no acl info retrieved
            Dim aclArray(-1) As ACLEntry
            Return aclArray
        End Function
        'Returns an array of the groups with access to this document.
        Public Function GetGroups() As ACLEntry() Implements IDocument.GetGroups
            'no acl info retrieved
            Dim aclArray(-1) As ACLEntry
            Return aclArray
        End Function
    End Class
End Namespace

Answer 1

1. I don't think you can just set the index url to a unc path.
2. Try creating an index aspx page. In your MetaData.IndexURL set it to the index aspx page, and include query string params for the encoded unc path as well as the content type.
3. In the index servlet, get the content type and path from the query string
4. Get the filename from the file path
5. Set the headers for content-type and Content-Disposition, e.g.
Response.ContentType="application/msword"
Response.AddHeader("Content-Disposition", "inline; filename'" + filename)
6. Stream out the file:
FileStream fs = new FileStream(path, FileMode.Open)
byte[] buffer = new byte[40000]
int result
System.IO.Stream output = Response.OutputStream
do
result = fs.Read(buffer, 0, 40000)
output.Write(buffer, 0, result)
while (result == 40000)
put the above in a try-catch, and then delete the temp file in the finally block.
If this does not help, set a breakpoint in the code to find the error. Also use Log4Net to log any errors.