Very subtle DB corruption ?

Question

Hello all,
    I'm experiencing what I call a very subtle bug. Subtle because there are no exceptions, no memory leak, not even an error message.
    My application uses BDB XML and it inserts, updates, removes and searches documents - pretty basic functionality. But after some time of use, my users are calling me saying the searches never return any results. I mean, they send a request, then it's sent to BDB, but it doesnt return.
    First I thought it was another memory leak, but memory cosumption is high as always, but it is stable at some point. Just for checking, I restarted the server and tried searching again - dog slow, no return, even with 600MB free RAM.
    The only solution was to run recovery, which I chose to do manually at this time (and I dont do it automatically yet). Started the server, and then it returns to its normal state.
     So, I can just wonder what can possibility cause this? The containers are getting corrupted (although no corruption messages appear)? My Environment is managed by this class:
package br.gov.al.delegaciainterativa.controles;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import javax.naming.LimitExceededException;
import br.gov.al.delegaciainterativa.utils.Dir;
import com.sleepycat.db.DatabaseException;
import com.sleepycat.db.Environment;
import com.sleepycat.db.EnvironmentConfig;
import com.sleepycat.dbxml.XmlContainer;
import com.sleepycat.dbxml.XmlContainerConfig;
import com.sleepycat.dbxml.XmlException;
import com.sleepycat.dbxml.XmlManager;
import com.sleepycat.dbxml.XmlManagerConfig;
public class EnvironmentInit {
    private Environment myEnv; // objeto ambiente
    private XmlManager myManager;
    private XmlManagerConfig managerConfig;
    private File envPath;
    private boolean ismyEnvOpen = false;
    private String nomeAmbiente;
    private String separador;
    private String envHome;
    private ArrayList containersAbertos;
    public EnvironmentInit(String envHome, String nomeAmbiente)
            throws Throwable {
        this.nomeAmbiente = nomeAmbiente;
        containersAbertos = new ArrayList();
        separador = System.getProperty("file.separator"); // pega o separador
        this.envHome = envHome + separador + nomeAmbiente;
         System.out.println(".EnvironmentInit:iniciar");       
        iniciar();
    public EnvironmentInit(String envHome, String nomeAmbiente, XmlManager mgr)
    throws Throwable {
        this.nomeAmbiente = nomeAmbiente;
        this.myManager = mgr;
        containersAbertos = new ArrayList();
        separador = System.getProperty("file.separator"); // pega o separador
        this.envHome = envHome + separador + nomeAmbiente;
        iniciar();
    private void iniciar() throws Exception {
        Dir.criarDiretorio(this.envHome); // cria o diretorio
        envPath = new File(this.envHome); // converte o caminho de String p/ File, requerido pelo construtor do Environment
        //System.out.println(envPath);
        if (!envPath.isDirectory()) {
//            System.out.println("Criando dir.. " + envHome);
//            Dir.criarDiretorio(this.envHome);
            throw new Exception(envPath.getPath()
                    + " does not exist or is not a directory.");
        try {
            EnvironmentConfig envConf = new EnvironmentConfig();
            //envConf.setCacheSize(50 * 1024 * 1024); //let default cache size.
            envConf.setAllowCreate(true); // If the environment does not
                                          // exits,create it.
            envConf.setInitializeCache(true); // Turn on the shared memory
                                              // region.
            envConf.setInitializeLocking(true); // Turn on the locking
                                                 // subsystem
            envConf.setInitializeLogging(true); // Turn on the logging subsystem
            envConf.setTransactional(true); // Turn on the transactional
                                            // subsystem - Passo 1 para setar transa??????es.
            //envConf.setLogInMemory(true);
            //envConf.setLogBufferSize(10 * 1024 * 1024); //100 Mbs de log. Logs são usados para recuperação do banco em caso de corrupção.
            envConf.setErrorStream(System.err);
            //envConf.setRunRecovery(true); // Roda o recovery automaticamente
            myEnv = new Environment(envPath, envConf);
            managerConfig = new XmlManagerConfig();
            managerConfig.setAdoptEnvironment(true); // autoriza ao XmlManager, quando for fechado, fechar tamb???m o ambiente
            //  managerConfig.setAllowAutoOpen(true); // autoriza a abrir um
                                                  // container automaticamente
            //managerConfig.setAllowExternalAccess(true); // acesso externo
            myManager = new XmlManager(myEnv, managerConfig);
            myManager.setDefaultContainerType(XmlContainer.WholedocContainer);
            ismyEnvOpen = true;
        } catch (DatabaseException de) {
            // Exception handling goes here
            System.err.println("[erro] EnvironmentInit:iniciar - erro no banco");
        } catch (FileNotFoundException fnfe) {
            // Exception handling goes here
            System.err.println("[erro] EnvironmentInit:iniciar - Config faltando");
        } catch (Exception e) {
            // Exception handling goes here
            System.err.println("[erro] EnvironmentInit:iniciar  \n" + e.toString());
    //Returns the path to the database environment
    public File getDbEnvPath() {
        return envPath;
    //Returns the database environment encapsulated by this class.
    public Environment getEnvironment() {
        return myEnv;
    //Returns the XmlManager encapsulated by this class.
    public XmlManager getManager() {
        return myManager;
     * Reabre o container com as configura??????es j??? preparadas. ??? necess???rio
     * re-abrir posteriormente os containers
    public void reabrir() {
        if (ismyEnvOpen == false) {
            try {
                iniciar();
                //System.out.println(".EnvironmentInit: reabrir()");
            } catch (Exception e) {
                System.err.println("[erro] EnvironmentInit:iniciar - Erro ao tentar reabrir ambiente!");
                //e.printStackTrace();
    public String getName() {
        return nomeAmbiente;
    public void cleanup() throws DatabaseException {
        ismyEnvOpen = false;
        fechaContainersAbertos();
        try {
            if (myManager != null) {
                // myEnv.close(); // fechado automaticamente pelo myManager
                //myManager.close();
                myManager.delete(); //trocado pelo close() pois é mais seguro.
                ismyEnvOpen = false;
        } catch (Exception de) {
            // Exception handling goes here
            System.err.println("[erro] EnvironmentInit:iniciar - N�o conseguiu fechar o banco");
        System.out.println(".EnvironmentInit:cleanup");
    private void registraContainerAberto(XmlContainer container) {
        containersAbertos.add(container);
    private void fechaContainersAbertos() {
        for (int i = 0; i < containersAbertos.size(); i++) {
            try {
                System.out.println(".EnvironmentInit:fechaContainer:" + ((XmlContainer) containersAbertos.get(i)).getName());
                ((XmlContainer) containersAbertos.get(i)).closeContainer();
                ((XmlContainer) containersAbertos.get(i)).delete(); //realmente garante o fechamento, destruindo o objeto associado.
            } catch (XmlException e) {
                System.err
                        .println("[erro] EnvironmentInit:iniciar - Erro ao tentar fechar container");
                e.printStackTrace();
        containersAbertos.clear();
    public XmlManagerConfig getXmlManagerConfig() {
        return managerConfig;
    public XmlContainer abrirContainer(String nome) {
        int ind = indiceContainerRegistrado(nome);
        XmlContainer container = null;
        if (ind >= 0) {
            //System.out.println(".EnvironmentInit:abrirContainer (j??? aberto) : " + nome + " - ambiente aberto!");
            return (XmlContainer) containersAbertos.get(ind);
        boolean existe = Dir
                .existeArquivo(envPath.toString() + separador, nome);
        try {
            if (!existe) {
                container = myManager.createContainer(nome);               
                System.out.println(".EnvironmentInit:abrirContainer (create) : " + nome + " - criando container");
                return null;
            } else {
                 System.out.println(".EnvironmentInit:abrirContainer - nome = '" nome "'");
                XmlContainerConfig conf = new XmlContainerConfig();
                conf.setAllowCreate(true);
                conf.setTransactional(true);
                 container = myManager.openContainer(nome, conf);
                //System.out.println(".EnvironmentInit:abrirContainer (open) : " + nome + " - ambiente aberto!");
            registraContainerAberto(container); // registra que o container foi
            // aberto
            return container;
        } catch (XmlException e) {
            e.printStackTrace();
        return null;
    public boolean removeContainer(XmlContainer c) {
        String path = null;
        try {
            path = envPath.toString() + separador + c.getName();
            //.closeContainer();
            c.delete(); //trocado pelo close acima por ser mais seguro.
            myManager.removeContainer(path);
            System.out.println(".EnvironmentInit:removeContainer : " + path);
            return true;
        } catch (XmlException e) {
            System.err.println("[erro] EnvironmentInit:iniciar - Erro ao tentar remover container ");
            //e.printStackTrace();
        return false;
    public int indiceContainerRegistrado(String nome) {
        try {
            for (int i = 0; i < containersAbertos.size(); i++) {
                XmlContainer cont = (XmlContainer)containersAbertos.get(i);
                if (cont.getName().compareTo(nome) == 0)
                    return i;
        } catch (Exception e) {
        return -1;
    Is there anything wrong with it, that could be causing this? Any help would be much appreciated.
thanks,
-- Breno Costa

Answer 1

Hello all,
    I'm experiencing what I call a very subtle bug. Subtle because there are no exceptions, no memory leak, not even an error message.
    My application uses BDB XML and it inserts, updates, removes and searches documents - pretty basic functionality. But after some time of use, my users are calling me saying the searches never return any results. I mean, they send a request, then it's sent to BDB, but it doesnt return.
    First I thought it was another memory leak, but memory cosumption is high as always, but it is stable at some point. Just for checking, I restarted the server and tried searching again - dog slow, no return, even with 600MB free RAM.
    The only solution was to run recovery, which I chose to do manually at this time (and I dont do it automatically yet). Started the server, and then it returns to its normal state.
     So, I can just wonder what can possibility cause this? The containers are getting corrupted (although no corruption messages appear)? My Environment is managed by this class:
package br.gov.al.delegaciainterativa.controles;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import javax.naming.LimitExceededException;
import br.gov.al.delegaciainterativa.utils.Dir;
import com.sleepycat.db.DatabaseException;
import com.sleepycat.db.Environment;
import com.sleepycat.db.EnvironmentConfig;
import com.sleepycat.dbxml.XmlContainer;
import com.sleepycat.dbxml.XmlContainerConfig;
import com.sleepycat.dbxml.XmlException;
import com.sleepycat.dbxml.XmlManager;
import com.sleepycat.dbxml.XmlManagerConfig;
public class EnvironmentInit {
    private Environment myEnv; // objeto ambiente
    private XmlManager myManager;
    private XmlManagerConfig managerConfig;
    private File envPath;
    private boolean ismyEnvOpen = false;
    private String nomeAmbiente;
    private String separador;
    private String envHome;
    private ArrayList containersAbertos;
    public EnvironmentInit(String envHome, String nomeAmbiente)
            throws Throwable {
        this.nomeAmbiente = nomeAmbiente;
        containersAbertos = new ArrayList();
        separador = System.getProperty("file.separator"); // pega o separador
        this.envHome = envHome + separador + nomeAmbiente;
         System.out.println(".EnvironmentInit:iniciar");       
        iniciar();
    public EnvironmentInit(String envHome, String nomeAmbiente, XmlManager mgr)
    throws Throwable {
        this.nomeAmbiente = nomeAmbiente;
        this.myManager = mgr;
        containersAbertos = new ArrayList();
        separador = System.getProperty("file.separator"); // pega o separador
        this.envHome = envHome + separador + nomeAmbiente;
        iniciar();
    private void iniciar() throws Exception {
        Dir.criarDiretorio(this.envHome); // cria o diretorio
        envPath = new File(this.envHome); // converte o caminho de String p/ File, requerido pelo construtor do Environment
        //System.out.println(envPath);
        if (!envPath.isDirectory()) {
//            System.out.println("Criando dir.. " + envHome);
//            Dir.criarDiretorio(this.envHome);
            throw new Exception(envPath.getPath()
                    + " does not exist or is not a directory.");
        try {
            EnvironmentConfig envConf = new EnvironmentConfig();
            //envConf.setCacheSize(50 * 1024 * 1024); //let default cache size.
            envConf.setAllowCreate(true); // If the environment does not
                                          // exits,create it.
            envConf.setInitializeCache(true); // Turn on the shared memory
                                              // region.
            envConf.setInitializeLocking(true); // Turn on the locking
                                                 // subsystem
            envConf.setInitializeLogging(true); // Turn on the logging subsystem
            envConf.setTransactional(true); // Turn on the transactional
                                            // subsystem - Passo 1 para setar transa??????es.
            //envConf.setLogInMemory(true);
            //envConf.setLogBufferSize(10 * 1024 * 1024); //100 Mbs de log. Logs são usados para recuperação do banco em caso de corrupção.
            envConf.setErrorStream(System.err);
            //envConf.setRunRecovery(true); // Roda o recovery automaticamente
            myEnv = new Environment(envPath, envConf);
            managerConfig = new XmlManagerConfig();
            managerConfig.setAdoptEnvironment(true); // autoriza ao XmlManager, quando for fechado, fechar tamb???m o ambiente
            //  managerConfig.setAllowAutoOpen(true); // autoriza a abrir um
                                                  // container automaticamente
            //managerConfig.setAllowExternalAccess(true); // acesso externo
            myManager = new XmlManager(myEnv, managerConfig);
            myManager.setDefaultContainerType(XmlContainer.WholedocContainer);
            ismyEnvOpen = true;
        } catch (DatabaseException de) {
            // Exception handling goes here
            System.err.println("[erro] EnvironmentInit:iniciar - erro no banco");
        } catch (FileNotFoundException fnfe) {
            // Exception handling goes here
            System.err.println("[erro] EnvironmentInit:iniciar - Config faltando");
        } catch (Exception e) {
            // Exception handling goes here
            System.err.println("[erro] EnvironmentInit:iniciar  \n" + e.toString());
    //Returns the path to the database environment
    public File getDbEnvPath() {
        return envPath;
    //Returns the database environment encapsulated by this class.
    public Environment getEnvironment() {
        return myEnv;
    //Returns the XmlManager encapsulated by this class.
    public XmlManager getManager() {
        return myManager;
     * Reabre o container com as configura??????es j??? preparadas. ??? necess???rio
     * re-abrir posteriormente os containers
    public void reabrir() {
        if (ismyEnvOpen == false) {
            try {
                iniciar();
                //System.out.println(".EnvironmentInit: reabrir()");
            } catch (Exception e) {
                System.err.println("[erro] EnvironmentInit:iniciar - Erro ao tentar reabrir ambiente!");
                //e.printStackTrace();
    public String getName() {
        return nomeAmbiente;
    public void cleanup() throws DatabaseException {
        ismyEnvOpen = false;
        fechaContainersAbertos();
        try {
            if (myManager != null) {
                // myEnv.close(); // fechado automaticamente pelo myManager
                //myManager.close();
                myManager.delete(); //trocado pelo close() pois é mais seguro.
                ismyEnvOpen = false;
        } catch (Exception de) {
            // Exception handling goes here
            System.err.println("[erro] EnvironmentInit:iniciar - N�o conseguiu fechar o banco");
        System.out.println(".EnvironmentInit:cleanup");
    private void registraContainerAberto(XmlContainer container) {
        containersAbertos.add(container);
    private void fechaContainersAbertos() {
        for (int i = 0; i < containersAbertos.size(); i++) {
            try {
                System.out.println(".EnvironmentInit:fechaContainer:" + ((XmlContainer) containersAbertos.get(i)).getName());
                ((XmlContainer) containersAbertos.get(i)).closeContainer();
                ((XmlContainer) containersAbertos.get(i)).delete(); //realmente garante o fechamento, destruindo o objeto associado.
            } catch (XmlException e) {
                System.err
                        .println("[erro] EnvironmentInit:iniciar - Erro ao tentar fechar container");
                e.printStackTrace();
        containersAbertos.clear();
    public XmlManagerConfig getXmlManagerConfig() {
        return managerConfig;
    public XmlContainer abrirContainer(String nome) {
        int ind = indiceContainerRegistrado(nome);
        XmlContainer container = null;
        if (ind >= 0) {
            //System.out.println(".EnvironmentInit:abrirContainer (j??? aberto) : " + nome + " - ambiente aberto!");
            return (XmlContainer) containersAbertos.get(ind);
        boolean existe = Dir
                .existeArquivo(envPath.toString() + separador, nome);
        try {
            if (!existe) {
                container = myManager.createContainer(nome);               
                System.out.println(".EnvironmentInit:abrirContainer (create) : " + nome + " - criando container");
                return null;
            } else {
                 System.out.println(".EnvironmentInit:abrirContainer - nome = '" nome "'");
                XmlContainerConfig conf = new XmlContainerConfig();
                conf.setAllowCreate(true);
                conf.setTransactional(true);
                 container = myManager.openContainer(nome, conf);
                //System.out.println(".EnvironmentInit:abrirContainer (open) : " + nome + " - ambiente aberto!");
            registraContainerAberto(container); // registra que o container foi
            // aberto
            return container;
        } catch (XmlException e) {
            e.printStackTrace();
        return null;
    public boolean removeContainer(XmlContainer c) {
        String path = null;
        try {
            path = envPath.toString() + separador + c.getName();
            //.closeContainer();
            c.delete(); //trocado pelo close acima por ser mais seguro.
            myManager.removeContainer(path);
            System.out.println(".EnvironmentInit:removeContainer : " + path);
            return true;
        } catch (XmlException e) {
            System.err.println("[erro] EnvironmentInit:iniciar - Erro ao tentar remover container ");
            //e.printStackTrace();
        return false;
    public int indiceContainerRegistrado(String nome) {
        try {
            for (int i = 0; i < containersAbertos.size(); i++) {
                XmlContainer cont = (XmlContainer)containersAbertos.get(i);
                if (cont.getName().compareTo(nome) == 0)
                    return i;
        } catch (Exception e) {
        return -1;
    Is there anything wrong with it, that could be causing this? Any help would be much appreciated.
thanks,
-- Breno Costa