Segmentation fault when master dies

Question

I am running my 'pure' master/slave setup now with 1 master and 2 slaves. When I kill the master, the first slave segfaults the second one keeps running and recovers when the master is started again.
This is the gdb backtrace:
#0  0xb7eace64 in __env_alloc_free () from /usr/local/lib/libdb-4.6.so
#1  0xb7e5a3a9 in __rep_grow_sites () from /usr/local/lib/libdb-4.6.so
#2  0xb7e4c271 in __rep_vote1 () from /usr/local/lib/libdb-4.6.so
#3  0xb7e5647c in __rep_process_message () from /usr/local/lib/libdb-4.6.so
#4  0xb7e5e66e in __repmgr_msg_thread () from /usr/local/lib/libdb-4.6.so
#5  0xb7dcb342 in start_thread () from /lib/tls/i686/nosegneg/libpthread.so.0
#6  0xb7c3555e in clone () from /lib/tls/i686/nosegneg/libc.so.6I'm not sure what to do with this.

Answer 1

I have found and fixed the root cause of this issue. Here is a patch that solves
the problem. This patch is for the 4.7.25 release.
*** rep/rep_elect.c.orig        Mon Feb  9 11:27:59 2009
--- rep/rep_elect.c     Mon Feb  9 11:28:09 2009
*** 32,38 ****
               __P((ENV *, u_int32_t, u_int32_t, int *, u_int32_t *));
  static int __rep_fire_elected __P((ENV *, REP *, u_int32_t));
  static void __rep_elect_master __P((ENV *, REP *));
! static int __rep_tally __P((ENV *, REP *, int, u_int32_t *, u_int32_t, roff_t)
  static int __rep_wait __P((ENV *, db_timeout_t *, int *, int, u_int32_t));
--- 32,38 ----
               __P((ENV *, u_int32_t, u_int32_t, int *, u_int32_t *));
  static int __rep_fire_elected __P((ENV *, REP *, u_int32_t));
  static void __rep_elect_master __P((ENV *, REP *));
! static int __rep_tally __P((ENV *, REP *, int, u_int32_t *, u_int32_t, int));
  static int __rep_wait __P((ENV *, db_timeout_t *, int *, int, u_int32_t));
*** 265,271 ****
        /* Tally our own vote */
        if (__rep_tally(env, rep, rep->eid, &rep->sites, rep->egen,
!           rep->tally_off) != 0) {
                ret = EINVAL;
                goto lockdone;
--- 265,271 ----
        /* Tally our own vote */
        if (__rep_tally(env, rep, rep->eid, &rep->sites, rep->egen,
!           1) != 0) {
                ret = EINVAL;
                goto lockdone;
*** 346,352 ****
                if (rep->winner == rep->eid) {
                        (void)__rep_tally(env, rep, rep->eid, &rep->votes,
!                           egen, rep->v2tally_off);
                        RPRINT(env, DB_VERB_REP_ELECT, (env,
                            "Counted my vote %d", rep->votes));
--- 346,352 ----
                if (rep->winner == rep->eid) {
                        (void)__rep_tally(env, rep, rep->eid, &rep->votes,
!                           egen, 2);
                        RPRINT(env, DB_VERB_REP_ELECT, (env,
                            "Counted my vote %d", rep->votes));
*** 611,623 ****
        if (rep->sites + 1 > rep->nsites)
                rep->nsites = rep->sites + 1;
-       if (rep->nsites > rep->asites &&
-           (ret = __rep_grow_sites(env, rep->nsites)) != 0) {
-               RPRINT(env, DB_VERB_REP_ELECT, (env,
-                   "Grow sites returned error %d", ret));
-               goto err;
         * Ignore vote1's if we're in phase 2.
--- 611,616 ----
*** 632,638 ****
         * ignore the vote.
        if ((ret = __rep_tally(env, rep, eid, &rep->sites,
!           vi->egen, rep->tally_off)) != 0) {
                RPRINT(env, DB_VERB_REP_ELECT,
                    (env, "Tally returned %d, sites %d", ret, rep->sites));
                ret = 0;
--- 625,631 ----
         * ignore the vote.
        if ((ret = __rep_tally(env, rep, eid, &rep->sites,
!           vi->egen, 1)) != 0) {
                RPRINT(env, DB_VERB_REP_ELECT,
                    (env, "Tally returned %d, sites %d", ret, rep->sites));
                ret = 0;
*** 677,683 ****
                F_CLR(rep, REP_F_EPHASE1);
                if (master == rep->eid) {
                        (void)__rep_tally(env, rep, rep->eid,
!                           &rep->votes, egen, rep->v2tally_off);
                        RPRINT(env, DB_VERB_REP_ELECT, (env,
                            "After phase 1 done: counted vote %d of %d",
                            rep->votes, rep->nvotes));
--- 670,676 ----
                F_CLR(rep, REP_F_EPHASE1);
                if (master == rep->eid) {
                        (void)__rep_tally(env, rep, rep->eid,
!                           &rep->votes, egen, 2);
                        RPRINT(env, DB_VERB_REP_ELECT, (env,
                            "After phase 1 done: counted vote %d of %d",
                            rep->votes, rep->nvotes));
*** 812,818 ****
         * __rep_tally takes care of cases 2, 3 and 4.
        if ((ret = __rep_tally(env, rep, eid, &rep->votes,
!           vi->egen, rep->v2tally_off)) != 0) {
                ret = 0;
                goto err;
--- 805,811 ----
         * __rep_tally takes care of cases 2, 3 and 4.
        if ((ret = __rep_tally(env, rep, eid, &rep->votes,
!           vi->egen, 2)) != 0) {
                ret = 0;
                goto err;
*** 838,855 ****
   *    caller passed in.
  static int
! __rep_tally(env, rep, eid, countp, egen, vtoff)
        ENV *env;
        REP *rep;
        int eid;
        u_int32_t *countp;
        u_int32_t egen;
!       roff_t vtoff;
        REP_VTALLY *tally, *vtp;
!       u_int32_t i;
!       tally = R_ADDR(env->reginfo, vtoff);
        vtp = &tally[0];
        for (i = 0; i < *countp;) {
--- 831,865 ----
   *    caller passed in.
  static int
! __rep_tally(env, rep, eid, countp, egen, phase)
        ENV *env;
        REP *rep;
        int eid;
        u_int32_t *countp;
        u_int32_t egen;
!       int phase;
        REP_VTALLY *tally, *vtp;
!       u_int32_t i, max_sites;
!       int ret;
!        * The counts are indices, and therefore 0-based.
!       if ((*countp + 1) > rep->nsites)
!               max_sites = (*countp + 1);
!       else
!               max_sites = rep->nsites;
!       if (max_sites > rep->asites &&
!           (ret = __rep_grow_sites(env, max_sites)) != 0) {
!               RPRINT(env, DB_VERB_REP_ELECT, (env,
!                   "Grow sites returned error %d", ret));
!               return (ret);
!       if (phase == 1)
!               tally = R_ADDR(env->reginfo, rep->tally_off);
!       else
!               tally = R_ADDR(env->reginfo, rep->v2tally_off);
        vtp = &tally[0];
        for (i = 0; i < *countp;) {
*** 882,889 ****
         * If we get here, we have a new voter we haven't seen before.  Tally
         * this vote.
!       RPRINT(env, DB_VERB_REP_ELECT, (env, "Tallying VOTE%c[%d] (%d, %lu)",
!           vtoff == rep->tally_off ? '1' : '2', i, eid, (u_long)egen));
        vtp->eid = eid;
        vtp->egen = egen;
--- 892,899 ----
         * If we get here, we have a new voter we haven't seen before.  Tally
         * this vote.
!       RPRINT(env, DB_VERB_REP_ELECT, (env, "Tallying VOTE%d[%d] (%d, %lu)",
!           phase, i, eid, (u_long)egen));
        vtp->eid = eid;
        vtp->egen = egen;