DB_DBT_MALLOC was set when secondary db's callback function was called!

Question

I have a secondary db associated with a primary db,when i insert some test records into primary db
,i saw my sdb_callback was called twice!the first call with the correct data,but the seocond call
was with wrong data,data.size was wrong ,with a big value and flags was set to DB_DBT_MALLOC ,
what should i do when the callback was called twice?just return DB_DONOTINDEX?
i modified the example code from source code/example_c/bench_001.c,make the bulk_fill the DB to
have a secondary as following
int sdb_callback(DB sdbp,          / secondary db handle */
                              const DBT pkey,   / primary db record's key */
                              const DBT pdata,  / primary db record's data */
                              DBT skey)         / secondary db record's key */
     if(pdata->flags & DB_DBT_MALLOC)
          fprintf(stderr,"DB_DBT_MALLOC fired size:%d ksize:%d \r\n",pdata->size,pkey->size);
          return DB_DONOTINDEX;
     }else
          return DB_DONOTINDEX;
FOR work WITH secondary db,i comment following out
     //if ((ret = dbp->set_flags(dbp, DB_DUP)) != 0) {
     //     dbp->err(dbp, ret, "set_flags");
     //     goto err;
     char sdb_filename[256];
     sprintf(sdb_filename, "s_%s",DATABASE );
     if ((ret = db_create(&sdbp, dbenv, 0)) != 0) {
          dbp->err(dbp, ret, "crate");
          exit(EXIT_FAILURE);
     /* set page size */
     if((ret = sdbp->set_pagesize(sdbp, pagesize)) != 0){
          dbp->err(dbp, ret, "crate");
          exit(EXIT_FAILURE);
     /* allow sorted duplicates. */
     /* try to open db*/
     ret = sdbp->open(sdbp, NULL, sdb_filename, NULL, DB_BTREE, DB_CREATE, 0664);
     if(ret != 0)
          dbp->err(dbp, ret, "%s: sdb open", DATABASE);
     dbp->associate(dbp, NULL, sdbp, sdb_callback, 0);
then i run bulk_fill test,DB_DBT_MALLOC was OFTEN fired,but not every put!
Edited by: feiman on May 23, 2010 4:35 AM

Answer 1

following are codes from example_c bench_001.c
i run ./a.out -B
then it appears that when one record input to main db,secondary db's callback was called twice,except the FIRST record!!!
#include <sys/types.h>
#include <sys/time.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#ifdef _WIN32
extern int getopt(int, char * const *, const char *);
#else
#include <unistd.h>
#endif
#include <db.h>
#define     DATABASE     "bench_001.db"
int     bulk_fill(DB_ENV *, DB *, int, u_int, int, int, int *, int *);
int     compare_int(DB *, const DBT *, const DBT *);
DB_ENV db_init(char , char *, u_int, int);
int     main(int, char *[]);
void     usage(void);
const char
     progname = "bench_001";          / Program name. */
int
sdb_callback(DB sdbp,          / secondary db handle */
                              const DBT pkey,   / primary db record's key */
                              const DBT pdata,  / primary db record's data */
                              DBT skey)         / secondary db record's key */
     skey->data = pkey->data;
     skey->size = pkey->size;
fprintf(stderr,"callback called size:%d ksize:%d \r\n",pdata->size,pkey->size);
     return 0;
* db_init --
*     Initialize the environment.
DB_ENV *
     db_init(home, prefix, cachesize, txn)
     char home, prefix;
u_int cachesize;
int txn;
     DB_ENV *dbenv;
     u_int32_t flags;
     int ret;
     if ((ret = db_env_create(&dbenv, 0)) != 0) {
          dbenv->err(dbenv, ret, "db_env_create");
          return (NULL);
     dbenv->set_errfile(dbenv, stderr);
     dbenv->set_errpfx(dbenv, prefix);
     (void)dbenv->set_cachesize(dbenv, 0,
          cachesize == 0 ? 50 * 1024 * 1024 : (u_int32_t)cachesize, 0);
     flags = DB_CREATE | DB_INIT_MPOOL;
     if (txn)
          flags |= DB_INIT_TXN | DB_INIT_LOCK;
     if ((ret = dbenv->open(dbenv, home, flags, 0)) != 0) {
          dbenv->err(dbenv, ret, "DB_ENV->open: %s", home);
          (void)dbenv->close(dbenv, 0);
          return (NULL);
     return (dbenv);
* bulk_fill - bulk_fill a db
*     Since we open/created the db with transactions (potentially),
* we need to populate it with transactions. We'll bundle the puts
* UPDATES_PER_BULK_PUT to a transaction.
#define     UPDATES_PER_BULK_PUT     3
int
     bulk_fill(dbenv, dbp, txn, datalen, num, dups, countp, iterp)
     DB_ENV *dbenv;
DB *dbp;
u_int datalen;
int txn, num, dups;
int countp, iterp;
     DBT key, data;
     u_int32_t flag = 0;
     DB_TXN *txnp;
     struct data {
          int id;
          char str[1];
     } *data_val;
     int count, i, iter, ret;
     void ptrk, ptrd;
     int iCnt = 0;
     time_t now,then;
     time(&now);
     time(&then);
     * Insert records into the database, where the key is the user
     * input and the data is the user input in reverse order.
     txnp = NULL;
     ret = 0;
     count = 0;
     iter = 0;
     ptrk = ptrd = NULL;
     data_val = malloc(datalen);
     memcpy(data_val->str, "0123456789012345678901234567890123456789",
          datalen - sizeof(data_val->id));
     memset(&key, 0, sizeof(DBT));
     memset(&data, 0, sizeof(DBT));
     * Need to account for proper buffer size,
     * The buffer must be at least as large as the page size of
     * the underlying database, aligned for unsigned integer
     * access, and be a multiple of 1024 bytes in size.
     key.ulen = (u_int32_t)UPDATES_PER_BULK_PUT *
          sizeof(u_int32_t) * datalen * 1024;
     key.flags = DB_DBT_USERMEM | DB_DBT_BULK;
     key.data = malloc(key.ulen);
     data.ulen = (u_int32_t)UPDATES_PER_BULK_PUT *
          (u_int32_t)datalen * 1024;
     data.flags = DB_DBT_USERMEM | DB_DBT_BULK;
     data.data = malloc(data.ulen);
     if (dups)
          flag |= DB_MULTIPLE;
     else
          flag |= DB_MULTIPLE_KEY;
     DB_MULTIPLE_WRITE_INIT(ptrk, &key);
     if (dups)
          DB_MULTIPLE_WRITE_INIT(ptrd, &data);
     for (i = 0; i < num; i++) {
          if (txn != 0 && (i+1) % UPDATES_PER_BULK_PUT == 0) {
               if (txnp != NULL) {
                    ret = txnp->commit(txnp, 0);
                    txnp = NULL;
                    if (ret != 0)
                         goto err;
               if ((ret =
                    dbenv->txn_begin(dbenv, NULL, &txnp, 0)) != 0)
                    goto err;
          data_val->id = 0;
          do {
               if (dups) {
                    DB_MULTIPLE_WRITE_NEXT(ptrk, &key,
                         &i, sizeof(i));
                    assert(ptrk != NULL);
                    DB_MULTIPLE_WRITE_NEXT(ptrd, &data,
                         data_val, datalen);
                    assert(ptrd != NULL);
               else {
                    DB_MULTIPLE_KEY_WRITE_NEXT(ptrk,
                         &key, &i, sizeof(i), data_val, datalen);
                    assert(ptrk != NULL);
               if ((i+1) % UPDATES_PER_BULK_PUT == 0) {
                    switch (ret = dbp->put(dbp, txnp, &key, &data, flag)) {
                    case 0:
                         iter++;
                         free (key.data);
                         free (data.data);
                         memset(&key, 0, sizeof(DBT));
                         memset(&data, 0, sizeof(DBT));
                         key.ulen =
                              (u_int32_t)UPDATES_PER_BULK_PUT *
                              sizeof(u_int32_t) * datalen * 1024;
                         key.flags = DB_DBT_USERMEM|DB_DBT_BULK;
                         key.data = malloc(key.ulen);
                         data.ulen =
                              (u_int32_t)UPDATES_PER_BULK_PUT *
                              (u_int32_t)datalen * 1024;
                         data.flags =
                              DB_DBT_USERMEM|DB_DBT_BULK;
                         data.data = malloc(data.ulen);
                         DB_MULTIPLE_WRITE_INIT(ptrk, &key);
                         if (dups)
                              DB_MULTIPLE_WRITE_INIT(ptrd, &data);
                         break;
                    default:
                         dbp->err(dbp, ret, "Bulk DB->put");
                         free (key.data);
                         free (data.data);
                         goto err;
               count++;
          } while (++data_val->id < dups);
     if ((num % UPDATES_PER_BULK_PUT) != 0) {
          switch (ret = dbp->put(dbp, txnp, &key, &data, flag)) {
          case 0:
               iter++;
               free (key.data);
               free (data.data);
               break;
          default:
               free (key.data);
               free (data.data);
               dbp->err(dbp, ret, "Bulk DB->put");
               goto err;
     if (txnp != NULL)
          ret = txnp->commit(txnp, 0);
     printf("%d\n", count);
     *countp = count;
     *iterp = iter;
     return (ret);
err:     if (txnp != NULL)
               (void)txnp->abort(txnp);
          return (ret);
int
     main(argc, argv)
     int argc;
char *argv[];
     extern char *optarg;
     extern int optind;
     DB dbp,sdbp;
     DB_ENV *dbenv;
     DB_TXN *txnp;
     struct timeval start_time, end_time;
     double secs;
     u_int cache, datalen, pagesize;
     int biter, ch, count, dups, env, init, iter, num;
     int ret, rflag, txn, bulk, delete;
     txnp = NULL;
     datalen = 20;
     iter = num = 100;
     env = 1;
     dups = init = rflag = txn = bulk = delete = 0;
     pagesize = 65536;
     cache = 1000 * pagesize;
     while ((ch = getopt(argc, argv, "c:d:EIi:l:n:p:RTBD")) != EOF)
          switch (ch) {
          case 'c':
               cache = (u_int)atoi(optarg);
               break;
          case 'd':
               dups = atoi(optarg);
               break;
          case 'E':
               env = 0;
               break;
          case 'I':
               init = 1;
               break;
          case 'i':
               iter = atoi(optarg);
               break;
          case 'l':
               datalen = (u_int)atoi(optarg);
               break;
          case 'n':
               num = atoi(optarg);
               break;
          case 'p':
               pagesize = (u_int)atoi(optarg);
               break;
          case 'R':
               rflag = 1;
               break;
          case 'T':
               txn = 1;
               break;
          case 'B':
               bulk = 1;
               break;
          case 'D':
               delete = 1;
               break;
          case '?':
          default:
               usage();
     argc -= optind;
     argv += optind;
     /* Remove the previous database. */
     if (!rflag) {
          if (env)
               (void)system("rm -rf BENCH_001; mkdir BENCH_001");
          else
               (void)unlink(DATABASE);
     dbenv = NULL;
     if (env == 1 &&
          (dbenv = db_init("BENCH_001", "bench_001", cache, txn)) == NULL)
          return (-1);
     if (init)
          exit(0);
     /* Create and initialize database object, open the database. */
     if ((ret = db_create(&dbp, dbenv, 0)) != 0) {
          fprintf(stderr,
               "%s: db_create: %s\n", progname, db_strerror(ret));
          exit(EXIT_FAILURE);
     dbp->set_errfile(dbp, stderr);
     dbp->set_errpfx(dbp, progname);
     if ((ret = dbp->set_bt_compare(dbp, compare_int)) != 0) {
          dbp->err(dbp, ret, "set_bt_compare");
          goto err;
     if ((ret = dbp->set_pagesize(dbp, pagesize)) != 0) {
          dbp->err(dbp, ret, "set_pagesize");
          goto err;
     if (dups && (ret = dbp->set_flags(dbp, DB_DUP)) != 0) {
          dbp->err(dbp, ret, "set_flags");
     goto err;
     if (env == 0 && (ret = dbp->set_cachesize(dbp, 0, cache, 0)) != 0) {
          dbp->err(dbp, ret, "set_cachesize");
          goto err;
     //if ((ret = dbp->set_flags(dbp, DB_DUP)) != 0) {
     //     dbp->err(dbp, ret, "set_flags");
     //     goto err;
     if (txn != 0)
          if ((ret = dbenv->txn_begin(dbenv, NULL, &txnp, 0)) != 0)
               goto err;
     if ((ret = dbp->open(
          dbp, txnp, DATABASE, NULL, DB_BTREE, DB_CREATE, 0664)) != 0) {
               dbp->err(dbp, ret, "%s: open", DATABASE);
               if (txnp != NULL)
                    (void)txnp->abort(txnp);
               goto err;
     char sdb_filename[256];
     sprintf(sdb_filename, "s_%s",DATABASE );
     if ((ret = db_create(&sdbp, dbenv, 0)) != 0) {
          dbp->err(dbp, ret, "crate");
          exit(EXIT_FAILURE);
     /* set page size */
     if((ret = sdbp->set_pagesize(sdbp, pagesize)) != 0){
          dbp->err(dbp, ret, "crate");
          exit(EXIT_FAILURE);
     /* allow sorted duplicates. */
     /* try to open db*/
     ret = sdbp->open(sdbp, NULL, sdb_filename, NULL, DB_BTREE, DB_CREATE, 0664);
     if(ret != 0)
          dbp->err(dbp, ret, "%s: sdb open", DATABASE);
     dbp->associate(dbp, NULL, sdbp, sdb_callback, 0);
     if (txnp != NULL)
          ret = txnp->commit(txnp, 0);
     txnp = NULL;
     if (ret != 0)
          goto err;
          if (bulk) {
               /* Time the get loop. */
               (void)gettimeofday(&start_time, NULL);
               if ((ret = bulk_fill(dbenv, dbp, txn, datalen,
                    num, dups, &count, &biter)) != 0)
                    goto err;
               (void)gettimeofday(&end_time, NULL);
               secs =
                    (((double)end_time.tv_sec * 1000000 +
                    end_time.tv_usec) -
                    ((double)start_time.tv_sec * 1000000 +
                    start_time.tv_usec))
                    / 1000000;
               printf("%d records put using %d batches",
                    count, biter);
               printf(" in %.2f seconds: ", secs);
               printf("%.0f records/second\n", (double)count / secs);
     /* Close everything down. */
     if ((ret = dbp->close(dbp, rflag ? DB_NOSYNC : 0)) != 0) {
          fprintf(stderr,
               "%s: DB->close: %s\n", progname, db_strerror(ret));
          return (1);
     return (ret);
err:     (void)dbp->close(dbp, 0);
     return (1);
int
     compare_int(dbp, a, b)
     DB *dbp;
const DBT a, b;
     int ai, bi;
     dbp = dbp;                    /* Lint. */
     * Returns:
     *     < 0 if a < b
     *     = 0 if a = b
     *     > 0 if a > b
     memcpy(&ai, a->data, sizeof(int));
     memcpy(&bi, b->data, sizeof(int));
     return (ai - bi);
void
     usage()
     (void)fprintf(stderr, "usage: %s %s\n\t%s\n",
          progname, "[-EIRTBE] [-c cachesize] [-d dups]",
          "[-i iterations] [-l datalen] [-n keys] [-p pagesize]");
     exit(EXIT_FAILURE);
}