Reclaiming memory when using concurrent data store

Question

Hello,
I use the concurrent data store in my python application and I'm noticing that
the system memory usage increases and is never freed when the application
is done. The following python code is a unit test that simulates my app's workload:
##########BEGIN PYTHON CODE##################
"""TestCases for multi-threaded access to a DB.
#import gc
#gc.enable()
#gc.set_debug(gc.DEBUG_LEAK)
import os
import sys
import time
import errno
import shutil
import tempfile
from pprint import pprint
from random import random
try:
    True, False
except NameError:
    True = 1
    False = 0
DASH = '-'
try:
    from threading import Thread, currentThread
    have_threads = True
except ImportError:
    have_threads = False
import unittest
verbose = 1
from bsddb import db, dbutils
class BaseThreadedTestCase(unittest.TestCase):
    dbtype       = db.DB_UNKNOWN  # must be set in derived class
    dbopenflags  = 0
    dbsetflags   = 0
    envflags     = 0
    def setUp(self):
        if verbose:
            dbutils._deadlock_VerboseFile = sys.stdout
        homeDir = os.path.join(os.path.dirname(sys.argv[0]), 'db_home')
        self.homeDir = homeDir
        try:
            os.mkdir(homeDir)
        except OSError, e:
            if e.errno <> errno.EEXIST: raise
        self.env = db.DBEnv()
        self.setEnvOpts()
        self.env.open(homeDir, self.envflags | db.DB_CREATE)
        self.filename = self.__class__.__name__ + '.db'
        self.d = db.DB(self.env)
        if self.dbsetflags:
            self.d.set_flags(self.dbsetflags)
        self.d.open(self.filename, self.dbtype, self.dbopenflags|db.DB_CREATE)
    def tearDown(self):
        self.d.close()
        self.env.close()
        del self.d
        del self.env
        #shutil.rmtree(self.homeDir)
        #print "\nGARBAGE:"
        #gc.collect()
        #print "\nGARBAGE OBJECTS:"
        #for x in gc.garbage:
        #    s = str(x)
        #    print type(x),"\n ", s
    def setEnvOpts(self):
        pass
    def makeData(self, key):
        return DASH.join([key] * 5)
class ConcurrentDataStoreBase(BaseThreadedTestCase):
    dbopenflags = db.DB_THREAD
    envflags    = db.DB_THREAD | db.DB_INIT_CDB | db.DB_INIT_MPOOL
    readers     = 0 # derived class should set
    writers     = 0
    records     = 1000
    def test01_1WriterMultiReaders(self):
        if verbose:
            print '\n', '-=' * 30
            print "Running %s.test01_1WriterMultiReaders..." % \
                  self.__class__.__name__
        threads = []
        for x in range(self.writers):
            wt = Thread(target = self.writerThread,
                        args = (self.d, self.records, x),
                        name = 'writer %d' % x,
                        )#verbose = verbose)
            threads.append(wt)
        for x in range(self.readers):
            rt = Thread(target = self.readerThread,
                        args = (self.d, x),
                        name = 'reader %d' % x,
                        )#verbose = verbose)
            threads.append(rt)
        for t in threads:
            t.start()
        for t in threads:
            t.join()
    def writerThread(self, d, howMany, writerNum):
        #time.sleep(0.01 * writerNum + 0.01)
        name = currentThread().getName()
        start = howMany * writerNum
        stop = howMany * (writerNum + 1) - 1
        if verbose:
            print "%s: creating records %d - %d" % (name, start, stop)
        for x in range(start, stop):
            key = '%04d' % x
            #dbutils.DeadlockWrap(d.put, key, self.makeData(key),
            #                     max_retries=12)
            d.put(key, self.makeData(key))
            if verbose and x % 100 == 0:
                print "%s: records %d - %d finished" % (name, start, x)
        if verbose:
            print "%s: finished creating records" % name
##         # Each write-cursor will be exclusive, the only one that can update the DB...
##         if verbose: print "%s: deleting a few records" % name
##         c = d.cursor(flags = db.DB_WRITECURSOR)
##         for x in range(10):
##             key = int(random() * howMany) + start
##             key = '%04d' % key
##             if d.has_key(key):
##                 c.set(key)
##                 c.delete()
##         c.close()
        if verbose:
            print "%s: thread finished" % name
        d.sync()
        del d
    def readerThread(self, d, readerNum):
        time.sleep(0.01 * readerNum)
        name = currentThread().getName()
        for loop in range(5):
            c = d.cursor()
            count = 0
            rec = c.first()
            while rec:
                count += 1
                key, data = rec
                self.assertEqual(self.makeData(key), data)
                rec = c.next()
            if verbose:
                print "%s: found %d records" % (name, count)
            c.close()
            time.sleep(0.05)
        if verbose:
            print "%s: thread finished" % name
        del d
    def setEnvOpts(self):
        #print "Setting cache size:", self.env.set_cachesize(0, 2000)
        pass
class BTreeConcurrentDataStore(ConcurrentDataStoreBase):
    dbtype  = db.DB_BTREE
    writers = 10
    readers = 100
    records = 100000
def test_suite():
    suite = unittest.TestSuite()
    if have_threads:
        suite.addTest(unittest.makeSuite(BTreeConcurrentDataStore))
    else:
        print "Threads not available, skipping thread tests."
    return suite
if __name__ == '__main__':
    unittest.main(defaultTest='test_suite')
    #print "\nGARBAGE:"
    #gc.collect()
    #print "\nGARBAGE OBJECTS:"
    #for x in gc.garbage:
    #    s = str(x)
    #    print type(x),"\n ", s
##########END PYTHON CODE##################Using the linux command 'top' prior to and during the execution of
the python script above, I noticed that a considerable amount of memory
is used up and never reclaimed when it ends.If you delete the db_home,
however, the memory is reclaimed.
Am I conjuring up the bsddb concurrent db store incorrectly somehow?
I'm using python 2.5.1 and the builtin bsddb module.
Thanks,
Gerald
Message was edited by:
user590005
Message was edited by:
user590005

Answer 1

I think I am seeing what you are reporing, but I need to check further into
the reason for this.
Running your program and monitoring with Top/vmstat before/after the test, and
after deleting db_home is:
BEFORE RUNNING PYTHON TEST:
++++++++++++++++++++++++++
top - 17:00:17 up 7:00, 6 users, load average: 0.07, 0.38, 0.45
Tasks: 111 total, 1 running, 109 sleeping, 0 stopped, 1 zombie
Cpu(s): 3.6% us, 0.7% sy, 0.0% ni, 95.5% id, 0.0% wa, 0.2% hi, 0.0% si
Mem: 1545196k total, 1407100k used, 138096k freeTerminal, 20700k buffers
Swap: 2040212k total, 168k used, 2040044k free, 935936k cached
[swhitman@swhitman-lnx python]$ vmstat
procs -----------memory---------- ---swap-- -----io---- system ----cpu----
r b swpd free buff cache si so bi bo in cs us sy id wa
1 0 160 247096 20860 833604 0 0 31 22 527 675 7 1 91 1
AFTER RUNNING PYTHON TEST:
++++++++++++++++++++++++++
top - 17:02:00 up 7:02, 6 users, load average: 2.58, 1.36, 0.80
Tasks: 111 total, 1 running, 109 sleeping, 0 stopped, 1 zombie
Cpu(s): 3.7% us, 0.5% sy, 0.0% ni, 95.8% id, 0.0% wa, 0.0% hi, 0.0% si
Mem: 1545196k total, 1508156k used, 37040k free, 20948k buffers
Swap: 2040212k total, 168k used, 2040044k free, 1035788k cached
[swhitman@swhitman-lnx python]$ vmstat
procs -----------memory---------- ---swap-- -----io---- system ----cpu----
r b swpd free buff cache si so bi bo in cs us sy id wa
0 0 160 143312 21120 935784 0 0 31 25 527 719 7 1 91 1
AFTER RUNNING PYTHON TEST & DB_HOME IS DELETED:
++++++++++++++++++++++++++++++++++++++++++++++
But I think DB_ENV->close
top - 17:02:48 up 7:02, 6 users, load average: 1.22, 1.17, 0.76
Tasks: 111 total, 1 running, 109 sleeping, 0 stopped, 1 zombie
Cpu(s): 8.8% us, 0.5% sy, 0.0% ni, 90.5% id, 0.0% wa, 0.2% hi, 0.0% si
Mem: 1545196k total, 1405236k used, 139960k free, 21044k buffers
Swap: 2040212k total, 168k used, 2040044k free, 934032k cached
[swhitman@swhitman-lnx python]$ vmstat
procs -----------memory---------- ---swap-- -----io---- system ----cpu----
r b swpd free buff cache si so bi bo in cs us sy id wa
1 0 160 246208 21132 833852 0 0 31 25 527 719 7 1 91 1
So the Top/vmstat Memory Usage Summary is:
before test after test after rm db_home/*
Top 1407100k 1508156k 1405236k
mem used
vmstat
free/cache 247096/833604 143312/935784 246208/833852