Some questions & Performance probleme

Question

Hello all,
I'm currently trying to make an application using berkeley DB.
But I came accross really Ugly performance (439 insert / sec) and I also try to switch to 'DB_QUEUE' but I've got a
'illegal record number size' when putting record into my db.
# time ./prog 10000
done in 22766.2 ms (439.248 updates/sec)
real 0m22.807s
user 0m0.320s
sys 0m0.580s
And when I call 'db->set_re_len(1024)' and change 'DB_TREE' to 'DB_QUEUE' (at line db->open),
I got this:
illegal record number size
terminate called after throwing an instance of 'DbException'
what(): Db::put: Invalid argument
Aborted (core dumped)
Could someone explain me what I'm doing wrong here ?
Thanks you very much for any help
Laurent
Here the program:
========================================================
#include <db_cxx.h>
#include <sys/time.h>
char buffer[1024];
#define get_timeval(timeval) gettimeofday(&(timeval), 0)
struct timeval start,end;
void
usage()
std::cerr << "You should give a number as ARG1 !\n";
exit(1);
int
main(int argc, char** argv)
if ( argc!=2 )
usage();
DbEnv* env;
Db* db;
DbSequence* seq;
Db* dbSeq;
Dbt seqKey;
Dbt key;
Dbt data;
env = new DbEnv(0);
env->set_flags( DB_AUTO_COMMIT , 1 );
env->set_flags( DB_TXN_WRITE_NOSYNC , 1 );
key.set_data( new db_seq_t );
key.set_size( sizeof(db_seq_t) );
data.set_data( new char[1024] );
data.set_size( 1024 );
u_int32_t flags = DB_CREATE
| DB_PRIVATE
| DB_INIT_LOCK
| DB_INIT_LOG
| DB_INIT_TXN
| DB_INIT_MPOOL;
env->open("/files/lauma/test/PONG",flags,0);
dbSeq = new Db(env,0);
dbSeq->open(0,"sequences.db",0,DB_BTREE,DB_CREATE,0640);
db = new Db( env , 0 );
db->set_pagesize(65536);
//db->set_re_len( 1024 );
db->open( 0 , "mytables.db" , NULL , DB_BTREE , DB_CREATE , 0640 );
seq = new DbSequence( dbSeq , 0 );
seqKey.set_data(const_cast<char*>("mytables.db"));
seqKey.set_size(strlen("mytables.db")+1);
seq->open(0 , &key , DB_CREATE );
int count = atoi(argv[1]);
get_timeval(start);
for(int i=0;i<count;++i)
if ( (i%1000) == 0 ) std::cout << i << " saves ...\n";
DbTxn* txn;
env->txn_begin(0,&txn,0);
db_seq_t dbid;
seq->get(txn,1,&dbid,0);
memmove( key.get_data() , &dbid , sizeof(db_seq_t) );
memmove( data.get_data() , buffer , 1024 );
db->put(txn,&key,&data,0);
txn->commit(0);
get_timeval(end);
double d = ((end.tv_sec*1000.0 + end.tv_usec/1000.0) - (start.tv_sec*1000.0 + start.tv_usec/1000.0));
std::cout << "done in " << d << " ms (" << (d?(count*1000.0/d):-1) << " updates/sec)" << std::endl;
seq->close(0);
dbSeq->close(0);
db->close(0);
env->close(0);
delete seq;
delete dbSeq;
delete db;
delete env;
}

Answer 1

Thanks for your answer.
But it's seems the probleme is elsewhere.
The probleme occurs when I use a 'db_seq_t' as the key value like in:
for( ... )
db_seq_t dbid = ... get sequence ...
Dbt key( &dbid , sizeof(db_seq_t) );
db->put(txn,&key,&data,0);
But if I do the following:
for( ... )
db_seq_t dbid = ... get sequence ...
char buffer[10];
sprintf( buffer , "%010lld" , dbid );
Dbt key( buffer , 10 );
db->put(txn,&key,&data,0);
The performance are ok !
Do you have any idea ??
Thank you very much
Laurent Marzullo
I paste the new code here:
=============================================
#include <db_cxx.h>
#include <sys/time.h>
char buffer[1024];
#define get_timeval(timeval) gettimeofday(&(timeval), 0)
struct timeval start,end;
void
usage()
std::cerr << "prog <count> <char | int>\n";
exit(1);
u_int32_t     pagesize = 32 * 1024;
u_int          bulkbufsize = 4 * 1024 * 1024;
u_int logbufsize = 8 * 1024 * 1024;
u_int cachesize = 32 * 1024 * 1024;
u_int          datasize = 255;
u_int          keysize = 8;
u_int numitems = 0;
int
main(int argc, char** argv)
if ( argc!=3 )
usage();
bool kchar = false;
if ( strcmp( argv[2] , "char" ) == 0 )
kchar = true;
else if ( strcmp( argv[2] , "int" ) == 0 )
kchar = false;
else
usage();
DbEnv* env;
Db* db;
DbSequence* seq;
Db* dbSeq;
Dbt seqKey;
Dbt keychar;
Dbt keyint;
Dbt data;
env = new DbEnv(0);
env->set_flags( DB_AUTO_COMMIT , 1 );
env->set_flags( DB_TXN_WRITE_NOSYNC , 1 );
env->set_lg_bsize( logbufsize );
env->log_set_config( DB_LOG_IN_MEMORY | DB_LOG_DSYNC , 0 );
keychar.set_flags( DB_DBT_USERMEM );
keyint.set_flags( DB_DBT_USERMEM );
data.set_flags( DB_DBT_USERMEM );
data.set_data( new char[datasize] );
data.set_size( datasize );
data.set_ulen( datasize );
u_int32_t flags = DB_CREATE
| DB_PRIVATE
| DB_INIT_LOCK
| DB_INIT_LOG
| DB_INIT_TXN
| DB_INIT_MPOOL;
//env->log_set_config( DB_LOG_IN_MEMORY , 1 );
//env->set_cache_max(0,8388608);
env->set_cachesize(1,8388608,1);
env->open("/files/lauma/test/PONG",flags,0);
dbSeq = new Db(env,0);
dbSeq->open(0,"sequences.db",0,DB_BTREE,DB_CREATE,0640);
db = new Db( env , 0 );
db->set_pagesize(65536);
//db->set_re_len( 1024 );
db->open( 0 , "mytables.db" , NULL , DB_BTREE , DB_CREATE , 0666 );
seq = new DbSequence( dbSeq , 0 );
seqKey.set_data(const_cast<char*>("mytables.db"));
seqKey.set_size(strlen("mytables.db")+1);
seq->open(0 , &seqKey , DB_CREATE );
int count = atoi(argv[1]);
get_timeval(start);
for(int i=0;i<count;++i)
if ( (i%1000) == 0 ) std::cout << i << " saves ...\n";
DbTxn* txn;
env->txn_begin(0,&txn,0);
db_seq_t dbid;
seq->get(txn,1,&dbid,0);
if ( kchar )
char buffer[10];
sprintf( buffer , "%010lld" , dbid );
keychar.set_data( buffer );
keychar.set_size( 10 );
keychar.set_ulen( 10 );
else
keyint.set_data(&dbid);
keyint.set_size(sizeof(db_seq_t));
keyint.set_ulen(sizeof(db_seq_t));
data.set_data( buffer );
data.set_size( datasize );
memset( buffer , 'A' , datasize );
buffer[1023] = '\0';
if ( kchar )
db->put(txn,&keychar,&data,0);
else
db->put(txn,&keyint,&data,0);
txn->commit(0);
get_timeval(end);
double d = ((end.tv_sec*1000.0 + end.tv_usec/1000.0) - (start.tv_sec*1000.0 + start.tv_usec/1000.0));
std::cout << "done in " << d << " ms (" << (d?(count*1000.0/d):-1) << " updates/sec)" << std::endl;
seq->close(0);
dbSeq->close(0);
db->close(0);
env->close(0);
delete seq;
delete dbSeq;
delete db;
delete env;
}