Python segfault

Question

Hello,
Follows is a test program I have written. I am attempting to do multi-process inserts. Sometimes the program appears to deadlock and every other time, when it completes, it segfaults at the end complaining that a database handle is still in use. I am opening and deleting the container in the parent process and using the multiprocess module to handle the forking and IPC for me.
from bsddb3.db import *
from dbxml import *
import time
from multiprocessing import Process, Pool, Queue
numberOfItems = 100000
xml = """<item><type/></item>"""
def strAsDocument(mgr, str):
doc = mgr.createDocument()
doc.setContent(str)
return doc
def insertDoc(container, environment, mgr, number):
xtxn = mgr.createTransaction()
uc = mgr.createUpdateContext()
names = [];
print "inserting " + str(number) + " records"
for i in xrange(number):
name = container.putDocument(xtxn, 'item', xml, uc, DBXML_GEN_NAME)
names.append(name)
xtxn.commit()
print "done";
del uc
del xtxn
def go():
environment = DBEnv()
environment.set_cachesize(0, 25 * 1024 * 1024)
environment.open("env", DB_CREATE|DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_RECOVER|DB_THREAD, 0)
try:
config = XmlContainerConfig()
config.setAllowCreate(True)
config.setTransactional(True)
mgr = XmlManager(environment, 0)
uc = mgr.createUpdateContext()
try:
mgr.removeContainer("test.dbxml")
except:
pass
container = mgr.openContainer("test.dbxml", config)
container.setAutoIndexing(False, uc)
before = time.time()
ps = []
for i in range(5):
p = Process(target=insertDoc, args=(container, environment, mgr, 2000))
p.start()
ps.append(p)
for p in ps:
p.join()
print time.time() - before
del container
del mgr
del uc
except XmlException, inst:
print "XmlException (", inst.exceptionCode,"): ", inst.what
if inst.exceptionCode == DATABASE_ERROR:
print "Database error code:",inst.dbError
environment.close(0)
for i in range(5):
go()
gives me:
[root@vladivar python]# python test.py
inserting 2000 records
inserting 2000 records
inserting 2000 records
inserting 2000 records
inserting 2000 records
done
done
done
done
done
2.8827149868
Traceback (most recent call last):
File "test.py", line 72, in <module>
go()
File "test.py", line 69, in go
environment.close(0)
bsddb3.db.DBInvalidArgError: (22, 'Invalid argument -- Open database handle: test.dbxml/secondary_configuration')
Segmentation fault
Thanks.

Answer 1

This is very helpful, thanks.
w.r.t the GIL it was this posting that put me off: http://www.dabeaz.com/blog/2010/01/python-gil-visualized.html. But on reflection I think you're right; it's probably not an issue because the application will most likely be IO-bound. I will try the threading module on your recommendation (I much prefer threading anyway). I wasn't worried about the expense of forking as I would do this rarely (when a worker was started, which would service multiple requests) and it'll never be running on Windows. It appears though that controlling the database handles through a fork() is harder than I expected.
Considering I am going to now try the threading module this might seem a mute point but I did rewrite my test case to open the environment in each process instead of before fork()ing and I had deadlock issues, I'd like to understand why. Do I understand you correctly that I need to serialize opening of the environment and containers? I understand that creation needs to be serialized but opening too? In case you are interested here is my deadlocking (non-segfaulting) test case:
from bsddb3.db import *
from dbxml import *
import time
from multiprocessing import Process
xml = """<item><type/></item>"""
class DBTest:
    def insertDoc(self, number):
        uc = self.mgr.createUpdateContext()
        try:
            names = [];
            print "inserting " + str(number) + " records"
            for i in xrange(number):
                name = self.container.putDocument('item', xml, uc, DBXML_GEN_NAME)
                names.append(name)
            print "done";
        finally:
            del uc
    def joinEnvironment(self):
        self.environment = DBEnv()
        self.environment.open("env", DB_JOINENV|DB_THREAD)
    @staticmethod
    def createEnvironment():
        environment = DBEnv()
        environment.set_cachesize(0, 25 * 1024 * 1024)
        environment.open("env", DB_CREATE|
                         DB_INIT_LOCK|
                         DB_INIT_MPOOL|
                         DB_THREAD, 0)
        environment.close(0)
    def createContainers(self):
        mgr = XmlManager(self.environment, 0)
        uc = mgr.createUpdateContext()
        config = XmlContainerConfig()
        config.setAllowCreate(True)
        config.setThreaded(True)
        try:
            mgr.removeContainer("test.dbxml")
        except:
            pass
        container = mgr.openContainer("test.dbxml", config)
        container.setAutoIndexing(False, uc)
        del container
    def openContainers(self):
        config = XmlContainerConfig()
        config.setAllowCreate(False)
        config.setThreaded(True)
        self.mgr = XmlManager(self.environment, 0)
        self.container = self.mgr.openContainer("test.dbxml", config)
    def cleanup(self):
        if hasattr(self, 'container'):
            del self.container
        if hasattr(self, 'mgr'):
            del self.mgr
        if hasattr(self, 'environment'):
            self.environment.close(0)
            del self.environment
# called by fork()ed process
def doProcess(num):
    test = DBTest()
    try:
        test.joinEnvironment()
        test.openContainers()
        test.insertDoc(num)
    except XmlException, inst:
        print "XmlException (", inst.exceptionCode,"): ", inst.what
        if inst.exceptionCode == DATABASE_ERROR:
            print "Database error code:",inst.dbError
    finally:
        test.cleanup()
# main
DBTest.createEnvironment()
test = DBTest()
test.joinEnvironment()
test.createContainers()
test.cleanup()
ps = []
for i in range(3):
    p = Process(target=doProcess, args=(5000,))
    p.start()
    ps.append(p)
for p in ps:
   p.join()