BDB Lib Crash (4.6.21)

Question

I got core dump again in my application when calling db->put()
I'm opening more than 1000 files it seemed to crashed after 1000 files opened. I know there was a limitation of 1024 file handles per process so that I had already enlarged that by calling setrlimit:
limit.rlim_cur = 10000;
limit.rlim_max = 10000;
n = setrlimit(RLIMIT_NOFILE, &limit); // n == 0 after returned.I'm using 4.6.21, Ubuntu 8.10, 64 bits CPU, multi-thread accessing but I added mutex to make one DB file was accessing by one thread a time, with DB Environment:
    ret = db_env_create(&envp, 0);
    if (ret != 0) {
        ERR("StorageRoot: Error creating env handle: %s\n", db_strerror(ret));
        envp = NULL;
        return NULL;
    /* we can optimize this parameter in DB_CONFIG file */
    ret = envp->set_cachesize(envp, 0, _1M * 64, 0);
    if (ret != 0) {
        ERR("StorageRoot: DB->set_cachesize failed: %s.\n", db_strerror(ret));
        return NULL;
    /* we can optimize this parameter in DB_CONFIG file */
    ret = envp->mutex_set_max(envp, 100000);
    if (ret != 0) {
        ERR("StorageRoot: DB->mutex_set_max failed: %s.\n", db_strerror(ret));
        return NULL;
    envp->set_errpfx  (envp, "store");
    envp->set_errcall (envp, bdb_err_callback);
    envp->set_msgcall (envp, bdb_msg_callback);
    envp->set_data_dir(envp, sr->path);
    /* Open the environment. */
    env_flags = DB_CREATE | DB_RECOVER | DB_INIT_TXN | DB_THREAD | DB_INIT_MPOOL;
    ret = envp->open(envp,
                     env_home,        /* env home directory */
                     env_flags,       /* Open flags */
                     0);              /* File mode (default) */
    if (ret != 0) {
        ERR("StorageRoot: Environment open failed: %s", db_strerror(ret));
        return NULL;
{code}
{code:title=Create DB}
    ret = db_create(&dbp, envp, 0);
    if (ret != 0) {
        ERR("Store: store_open -- %s\n", db_strerror(ret));
        store_destroy(handle);
        return NULL;
    /* Point to the memory malloc'd by db_create() */
    handle->dbp = dbp;
    /* Set the open flags */
    open_flags = DB_CREATE | DB_THREAD;     /* Allow database creation */
    /* Set the page size to 8K */
    dbp->set_pagesize(dbp, _8K);
    /* Now open the database */
    ret = dbp->open(dbp,        /* Pointer to the database */
                    NULL,       /* Txn pointer */
                    handle->db_name, /* File name */
                    NULL,       /* Logical db name */
                    DB_HASH,    /* Database type (using hash) */
                    open_flags, /* Open flags */
                    0);         /* File mode. Using defaults */
{code}
{code:title=Crash Callstack}
#11 0x000000000043e4d2 in sig_handler (sig=11) at main.cpp:278                      <<-- SIGSERV
#12 <signal handler called>
#13 0x00007f363e7a27e1 in __memp_fget (dbmfp=0x7f36241dd440, pgnoaddr=0x7f36241dd9b4, txn=0x0, flags=1, addrp=0x7f36241dd9a8)
    at ../dist/../mp/mp_fget.c:243
#14 0x00007f363e6df62e in __ham_get_cpage (dbc=0x7f36241dd7a0, mode=DB_LOCK_WRITE) at ../dist/../hash/hash_page.c:2356
#15 0x00007f363e6cabbc in __ham_lookup (dbc=0x7f36241dd7a0, key=0x7f3638f6adf0, sought=37, mode=DB_LOCK_WRITE,
    pgnop=0x7f3638f6ac9c) at ../dist/../hash/hash.c:1747
#16 0x00007f363e6c8cd6 in __hamc_put (dbc=0x7f36241dd7a0, key=0x7f3638f6adf0, data=0x7f3638f6adc0, flags=20, pgnop=0x7f3638f6ac9c)
    at ../dist/../hash/hash.c:1014
#17 0x00007f363e74b59e in __dbc_put (dbc_arg=0x7f36241dd7a0, key=0x7f3638f6adf0, data=0x7f3638f6adc0, flags=20)
    at ../dist/../db/db_cam.c:1455
#18 0x00007f363e740578 in __db_put (dbp=0x7f36241dce90, txn=0x0, key=0x7f3638f6adf0, data=0x7f3638f6adc0, flags=20)
    at ../dist/../db/db_am.c:399
#19 0x00007f363e758c9b in __db_put_pp (dbp=0x7f36241dce90, txn=0x0, key=0x7f3638f6adf0, data=0x7f3638f6adc0, flags=20)
    at ../dist/../db/db_iface.c:1562
#20 0x000000000049033d in store_put (handle=0x7f36241dcd80, k=0x116e4e2 "?206pyq\016(1\020c\202o?[", b=0x116e4f6)
    at store.c:453
#21 0x000000000049857b in part_put (self=0x10f4560, dest=0x7f3638f6b010, key=0x116e4e2 "?206pyq\016(1\020c\202o?[",
    block=0x116e4f6) at partition.c:123
#22 0x000000000049681a in onmessage_sync_put (_piranha=0x10f2570, from=0x116e4b0, keep=0x7f3638f6b09c, msgbuf=0x116e410)
    at syncput.c:609
#23 0x00000000004a15d9 in __network_onreceived (self=0x1105c10, from=0x116e4b0, buf=0x116e410) at src/upcall.c:107
   +--../dist/../mp/mp_fget.c--------------------------------------------------------------+
   |229              * Determine the cache and hash bucket where this page lives and get   |
   |230              * local pointers to them.  Reset on each pass through this code, the  |
   |231              * page number can change.                                             |
   |232              */                                                                    |
   |233             MP_GET_BUCKET(dbmfp, *pgnoaddr, &infop, hp, ret);                      |
   |234             if (ret != 0)                                                          |
   |235                     return (ret);                                                  |
   |236             c_mp = infop->primary;                                                 |
   |237                                                                                    |
   |238             /* Search the hash chain for the page. */                              |
   |239             st_hsearch = 0;                                                        |
   |240             b_locked = 1;                                                          |
   |241             SH_TAILQ_FOREACH(bhp, &hp->hash_bucket, hq, __bh) {                    |
   |242                     ++st_hsearch;                                                  |
=*|243                     if (bhp->pgno != *pgnoaddr || bhp->mf_offset != mf_offset)     |
   |244                             continue;                                              |
   |245                                                                                    |
   |246                     /* Snapshot reads -- get the version visible at read_lsn. */   |
   |247                     if (mvcc && !edit && read_lsnp != NULL) {                      |
   |248                             while (bhp != NULL &&                                  |
   |249                                 !BH_OWNED_BY(dbenv, bhp, txn) &&                   |
   |250                                 !BH_VISIBLE(dbenv, bhp, read_lsnp, vlsn))          |
   |251                                     bhp = SH_CHAIN_PREV(bhp, vc, __bh);            |
   |252                                                                                    |
   |253                             DB_ASSERT(dbenv, bhp != NULL);                         |
   |254                     }                                                              |
   |255                                                                                    |
   |256                     makecopy = mvcc && dirty && !BH_OWNED_BY(dbenv, bhp, txn);     |
   |257                                                                                    |
   |258                     if (F_ISSET(bhp, BH_FROZEN) && !F_ISSET(bhp, BH_FREED)) {      |
   |259                             DB_ASSERT(dbenv, frozen_bhp == NULL);                  |
   +---------------------------------------------------------------------------------------+
core process 2549 In: __memp_fget                        Line: 243  PC: 0x7f363e7a27e1
(gdb) f 13
#13 0x00007f363e7a27e1 in __memp_fget (dbmfp=0x7f36241dd440, pgnoaddr=0x7f36241dd9b4, txn=0x0, flags=1, addrp=0x7f36241dd9a8)
    at ../dist/../mp/mp_fget.c:243
(gdb) p bhp
$2 = (BH *) 0x7a1e922e2668c362
(gdb) p bhp->mf_offset
Cannot access memory at address 0x7a1e922e2668c38a
(gdb) p *bhp
Cannot access memory at address 0x7a1e922e2668c362
(gdb)
{code}
On the same environment, I changed to build and link with Lib 4.8.24. It was working fine. Does it a bug of 4.6.21?
I'm curious that the application was working fine at other machines with same 4.6.21 installed with Ubuntu 8.10, which libdb4.6 was install by "apt-get" not by make install, 64 bits CPU. Why it works fine?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               

Answer 1

Hello,
The thread you mentioned was followed up again in:
Segmentation Faults on AIX with 4.7.25
At this time the problem was reproduced in 4.7.25. The resolution for the problems reported were:
1. Applying patch, patch.4.7.25.16300.dif, on top of
Berkeley DB 4.7.25. In the code that manipulates
db file names the patch will correctly hold the mutex
on the mpool region when freeing memory.
2. If the patch is not applied, a workaround is to set the
EnvironmentConfig setPrivate method to true.
The fix for the issue reported in #16300 is included in Berkeley DB 4.8.
Thanks,
Sandra