Java API, problem with secondary keys using multi key creator

Question

Hi,
I'm developing an application that inserts a few 100k or so records into a Queue DB, then access them using one of four Hash SecondaryDatabases. Three of these secondary dbs use a SecondaryMultiKeyCreator to generate the keys, and one uses a SecondaryKeyCreator. As a test, I'm trying to iterate through each secondary key. When trying to iterate through the keys of any of the secondary databases that use a SecondaryMultiKeyCreator, I have problems. I'm attempting to iterate through the keys by:
1: creating a StoredMap of the SecondaryDatabase
2: getting a StoredKeySet from said map
3: getting a StoredIterator on said StoredKeySet
4: iterate
The first call to StoredIterator.next() fails at my key binding (TupleBinding) because it is only receiving 2 bytes of data for the key, when it should be getting more, so an Exception is thrown. I suspected a problem with my SecondaryMultiKeyCreator, so I added some debug code to check the size of the DatabaseEntries it creates immediately before adding them to the results key Set. It checks out right. I also checked my key binding like so:
1: use binding to convert the key object to a DatabaseEntry
2: use binding to convert the created DatabaseEntry back to a key object
3: check to see if the old object contains the same data as the new one
Everything checked out ok.
What it boils down to is this: my key creator is adding DatabaseEntries of the correct size to the results set, but when the keys are being read back later on, my key binding is only receiving 2 bytes of data. For the one SecondaryDatabase that doesn't use a SecondaryMultiKeyCreator, but just a SecondaryKeyCreator, there are no issues and I am able to iterate through its secondary keys as expected.
EDIT: New discovery: if I only add ONE DatabaseEntry to the results set in my SecondaryMultiKeyCreator, I am able to iterate through the keys as I would like to.
Any ideas or suggestions?
Thank you for your attention,
-Justin
Message was edited by:
Steamroller

Answer 1

Hi Justin,
Sorry about the delayed response here. I have created a patch that resolves the problem.
If you apply the patch to your 4.6.21 source tree, and then rebuild Berkeley DB, the improper behavior should be resolved.
Regards,
Alex
diff -rc db/db_am.c db/db_am.c
*** db/db_am.c     Thu Jun 14 04:21:30 2007
--- db/db_am.c     Fri Jun 13 11:20:28 2008
*** 331,338 ****
       F_SET(dbc, DBC_TRANSIENT);
!      switch (flags) {
!      case DB_APPEND:
             * If there is an append callback, the value stored in
             * data->data may be replaced and then freed.  To avoid
--- 331,337 ----
       F_SET(dbc, DBC_TRANSIENT);
!       if (flags == DB_APPEND && LIST_FIRST(&dbp->s_secondaries) == NULL) {
             * If there is an append callback, the value stored in
             * data->data may be replaced and then freed.  To avoid
*** 367,388 ****
-            * Secondary indices:  since we've returned zero from an append
-            * function, we've just put a record, and done so outside
-            * __dbc_put.  We know we're not a secondary-- the interface
-            * prevents puts on them--but we may be a primary.  If so,
-            * update our secondary indices appropriately.
-            * If the application is managing this key's data, we need a
-            * copy of it here.  It will be freed in __db_put_pp.
-           DB_ASSERT(dbenv, !F_ISSET(dbp, DB_AM_SECONDARY));
-           if (LIST_FIRST(&dbp->s_secondaries) != NULL &&
-               (ret = __dbt_usercopy(dbenv, key)) == 0)
-                ret = __db_append_primary(dbc, key, &tdata);
             * The append callback, if one exists, may have allocated
             * a new tdata.data buffer.  If so, free it.
--- 366,371 ----
*** 390,401 ****
            /* No need for a cursor put;  we're done. */
            goto done;
!      default:
!           /* Fall through to normal cursor put. */
!           break;
!      if (ret == 0)
            ret = __dbc_put(dbc,
                key, data, flags == 0 ? DB_KEYLAST : flags);
--- 373,379 ----
            /* No need for a cursor put;  we're done. */
            goto done;
!      } else
            ret = __dbc_put(dbc,
                key, data, flags == 0 ? DB_KEYLAST : flags);
diff -rc db/db_cam.c db/db_cam.c
*** db/db_cam.c     Tue Jun  5 21:46:24 2007
--- db/db_cam.c     Thu Jun 12 16:41:29 2008
*** 899,905 ****
       DB_ENV *dbenv;
       DB dbp, sdbp;
       DBC dbc_n, oldopd, opd, sdbc, *pdbc;
!      DBT olddata, oldpkey, newdata, pkey, temppkey, tempskey;
       DBT all_skeys, skeyp, *tskeyp;
       db_pgno_t pgno;
       int cmp, have_oldrec, ispartial, nodel, re_pad, ret, s_count, t_ret;
--- 899,905 ----
       DB_ENV *dbenv;
       DB dbp, sdbp;
       DBC dbc_n, oldopd, opd, sdbc, *pdbc;
!      DBT olddata, oldpkey, newdata, pkey, temppkey, tempskey, tdata;
       DBT all_skeys, skeyp, *tskeyp;
       db_pgno_t pgno;
       int cmp, have_oldrec, ispartial, nodel, re_pad, ret, s_count, t_ret;
*** 1019,1026 ****
        * should have been caught by the checking routine, but
        * add a sprinkling of paranoia.
!      DB_ASSERT(dbenv, flags == DB_CURRENT || flags == DB_KEYFIRST ||
!            flags == DB_KEYLAST || flags == DB_NOOVERWRITE);
        * We'll want to use DB_RMW in a few places, but it's only legal
--- 1019,1027 ----
        * should have been caught by the checking routine, but
        * add a sprinkling of paranoia.
!       DB_ASSERT(dbenv, flags == DB_APPEND || flags == DB_CURRENT ||
!             flags == DB_KEYFIRST || flags == DB_KEYLAST ||
!             flags == DB_NOOVERWRITE);
        * We'll want to use DB_RMW in a few places, but it's only legal
*** 1048,1053 ****
--- 1049,1107 ----
                 goto err;
            have_oldrec = 1; /* We've looked for the old record. */
+      } else if (flags == DB_APPEND) {
+           /*
+            * With DB_APPEND, we need to do the insert to populate the
+            * key value. So we swap the 'normal' order of updating
+            * secondary / verifying foreign databases and inserting.
+            *
+            * If there is an append callback, the value stored in
+            * data->data may be replaced and then freed.  To avoid
+            * passing a freed pointer back to the user, just operate
+            * on a copy of the data DBT.
+            */
+           tdata = *data;
+           /*
+            * If this cursor is going to be closed immediately, we don't
+            * need to take precautions to clean it up on error.
+            */
+           if (F_ISSET(dbc_arg, DBC_TRANSIENT))
+                dbc_n = dbc_arg;
+           else if ((ret = __dbc_idup(dbc_arg, &dbc_n, 0)) != 0)
+                goto err;
+
+           pgno = PGNO_INVALID;
+
+           /*
+            * Append isn't a normal put operation;  call the appropriate
+            * access method's append function.
+            */
+           switch (dbp->type) {
+           case DB_QUEUE:
+                if ((ret = __qam_append(dbc_n, key, &tdata)) != 0)
+                     goto err;
+                break;
+           case DB_RECNO:
+                if ((ret = __ram_append(dbc_n, key, &tdata)) != 0)
+                     goto err;
+                break;
+           default:
+                /* The interface should prevent this. */
+                DB_ASSERT(dbenv,
+                    dbp->type == DB_QUEUE || dbp->type == DB_RECNO);
+
+                ret = __db_ferr(dbenv, "DBC->put", 0);
+                goto err;
+           }
+           /*
+            * The append callback, if one exists, may have allocated
+            * a new tdata.data buffer.  If so, free it.
+            */
+           FREE_IF_NEEDED(dbenv, &tdata);
+           pkey.data = key->data;
+           pkey.size = key->size;
+           /* An append cannot be replacing an existing item. */
+           nodel = 1;
       } else {
            /* Set pkey so we can use &pkey everywhere instead of key.  */
            pkey.data = key->data;
*** 1400,1405 ****
--- 1454,1465 ----
  skip_s_update:
+       * If this is an append operation, the insert was done prior to the
+       * secondary updates, so we are finished.
+       */
+      if (flags == DB_APPEND)
+           goto done;
+      /*
        * If we have an off-page duplicates cursor, and the operation applies
        * to it, perform the operation.  Duplicate the cursor and call the
        * underlying function.