TTL specified in put operation doesn't always work when using write-behind

Question

I'm using a distributed cache with a write-behind cache store (see the config below). I found that when I do something like myCache.put(key, value, ttl), the entry survives the specified ttl. I tried doing the same with a distributed cache with a write-through cachestore and there everything does happen correctly.
Is this sort of operation not permitted in caches containing a write-behind cachestore? If not wouldn't it be better to throw an UnsupportedOperationException.
I created a small test to simulate this. I added values to the cache with a TTL of 1 to 10 seconds and found that the 10 second entries stayed in the cache.
Configuration used:
<?xml version="1.0"?>
<!DOCTYPE cache-config SYSTEM "cache-config.dtd">
<cache-config>
     <caching-scheme-mapping>
          <cache-mapping>
               <cache-name>TTL_TEST</cache-name>
               <scheme-name>testScheme</scheme-name>
          </cache-mapping>
     </caching-scheme-mapping>
     <caching-schemes>
          <distributed-scheme>
               <scheme-name>testScheme</scheme-name>
               <service-name>testService</service-name>
               <backing-map-scheme>
                    <read-write-backing-map-scheme>
                         <internal-cache-scheme>
                              <local-scheme>
                                   <service-name>testBackLocalService</service-name>
                              </local-scheme>
                         </internal-cache-scheme>
                         <cachestore-scheme>
                              <class-scheme>
                                   <scheme-name>testBackStore</scheme-name>
                                   <class-name>TTLTestServer$TestCacheStore</class-name>
                              </class-scheme>
                         </cachestore-scheme>
                         <write-delay>3s</write-delay>
                    </read-write-backing-map-scheme>
               </backing-map-scheme>
               <local-storage>true</local-storage>
               <autostart>true</autostart>
          </distributed-scheme>
     </caching-schemes>
</cache-config>Code of test:
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StopWatch;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import com.google.common.collect.Lists;
import com.tangosol.net.CacheFactory;
import com.tangosol.net.NamedCache;
import com.tangosol.net.cache.CacheStore;
@Test
public class TTLTestServer
     private static final int RETRIES = 5;
     private static final Logger logger = LoggerFactory.getLogger( TTLTestServer.class );
     private NamedCache m_cache;
      * List of Time-To-Lives in seconds to check
     private final List<Integer> m_listOfTTLs = Lists.newArrayList(1, 3, 5, 10);
      * Test is done in separate threads to speed up the test
     private final  ExecutorService m_executorService = Executors.newCachedThreadPool();
     @BeforeClass
     public void setup()
          logger.info("Getting the cache");
          m_cache =  CacheFactory.getCache("TTL_TEST");
     public static class TestCacheStore implements CacheStore
          public void erase(Object arg0)
          public void eraseAll(Collection arg0)
          public void store(Object arg0, Object arg1)
          public void storeAll(Map arg0)
          public Object load(Object arg0)
          {return null;}
          public Map loadAll(Collection arg0)
          {return null;}
     public void testTTL() throws InterruptedException, ExecutionException
          logger.info("Starting TTL test");
          List<Future<StopWatch>> futures = Lists.newArrayList();
          for (final Integer ttl : m_listOfTTLs)
               futures.add(m_executorService.submit(new Callable()
                    public Object call() throws Exception
                         StopWatch stopWatch= new StopWatch("TTL=" + ttl);
                         for (int retry = 0; retry < RETRIES; retry++)
                              logger.info("Adding a value in cache for TTL={} in try={}", ttl, retry+1);
                              stopWatch.start("Retry="+retry);
                              m_cache.put(ttl, null, ttl*1000);
                              waitUntilNotInCacheAnymore(ttl, retry);
                              stopWatch.stop();
                         return stopWatch;
                    private void waitUntilNotInCacheAnymore(final Integer ttl, final int currentTry) throws InterruptedException
                         DateTime startTime = new DateTime();
                         long maxMillisToWait = ttl*2*1000;     //wait max 2 times the time of the ttl
                         while(m_cache.containsKey(ttl) )
                              Duration timeTaken = new Duration(startTime, new DateTime());
                              if(timeTaken.getMillis() > maxMillisToWait)
                                   throw new RuntimeException("Already waiting " + timeTaken + " for ttl=" + ttl + " and retry=" +  currentTry);
                              Thread.sleep(1000);
          logger.info("Waiting until all futures are finished");
          m_executorService.shutdown();
          logger.info("Getting results from futures");
          for (Future<StopWatch> future : futures)
               StopWatch sw = future.get();
               logger.info(sw.prettyPrint());
}Failure message:
FAILED: testTTL
java.util.concurrent.ExecutionException: java.lang.RuntimeException: Already waiting PT20.031S for ttl=10 and retry=0
     at java.util.concurrent.FutureTask$Sync.innerGet(Unknown Source)
     at java.util.concurrent.FutureTask.get(Unknown Source)
     at TTLTestServer.testTTL(TTLTestServer.java:159)
Caused by: java.lang.RuntimeException: Already waiting PT20.031S for ttl=10 and retry=0
     at TTLTestServer$1.waitUntilNotInCacheAnymore(TTLTestServer.java:139)
     at TTLTestServer$1.call(TTLTestServer.java:122)
     at java.util.concurrent.FutureTask$Sync.innerRun(Unknown Source)
     at java.util.concurrent.FutureTask.run(Unknown Source)
     at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source)
     at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
     at java.lang.Thread.run(Unknown Source)I'm using Coherence 3.4.2.
Best regards
Jan

Answer 1

Hi, still no luck. However, I noticed that setting the write-delay value of the write-behind store to 0s or 1s, solved the problem. It only starts to given me "the node has already been removed" excpetions once the write-delay value is 2s or higher.
You can find the coherence-cache-config.xml below:
<?xml version="1.0"?>
<!DOCTYPE cache-config SYSTEM "cache-config.dtd">
<cache-config>
     <caching-scheme-mapping>
          <cache-mapping>
               <cache-name>TTL_TEST</cache-name>
               <scheme-name>testScheme</scheme-name>
          </cache-mapping>
     </caching-scheme-mapping>
     <caching-schemes>
          <distributed-scheme>
               <scheme-name>testScheme</scheme-name>
               <service-name>testService</service-name>
               <backing-map-scheme>
                    <read-write-backing-map-scheme>
                         <internal-cache-scheme>
                              <local-scheme>
                                   <service-name>testBackLocalService</service-name>
                              </local-scheme>
                         </internal-cache-scheme>
                         <cachestore-scheme>
                              <class-scheme>
                                   <scheme-name>testBackStore</scheme-name>
                                   <class-name>TTLTestServer$TestCacheStore</class-name>
                              </class-scheme>
                         </cachestore-scheme>
                         <write-delay>2s</write-delay>
                    </read-write-backing-map-scheme>
               </backing-map-scheme>
               <local-storage>true</local-storage>
               <autostart>true</autostart>
          </distributed-scheme>
     </caching-schemes>
</cache-config>You can find the test program below:
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.springframework.util.StopWatch;
import com.tangosol.net.CacheFactory;
import com.tangosol.net.NamedCache;
import com.tangosol.net.cache.CacheStore;
public class TTLTestServer
     private static final int RETRIES = 5;
     private NamedCache m_cache;
      * List of Time-To-Lives in seconds to check
     private final List<Integer> m_listOfTTLs = new ArrayList<Integer>();
      * @param args
      * @throws Exception
     public static void main( String[] args ) throws Exception
          new TTLTestServer().test();
      * Empty CacheStore
      * @author jbe
     public static class TestCacheStore implements CacheStore
          public void erase(Object arg0)
          @SuppressWarnings ( "unchecked" )
          public void eraseAll(Collection arg0)
          public void store(Object arg0, Object arg1)
          @SuppressWarnings ( "unchecked" )
          public void storeAll(Map arg0)
          public Object load(Object arg0)
          {return null;}
          @SuppressWarnings ( "unchecked" )
          public Map loadAll(Collection arg0)
          {return null;}
      * Sets up and executes the test setting values in a cache with a given time-to-live value and waiting for the value to disappear.
      * @throws Exception
     private void test() throws Exception
          System.out.println(new DateTime() + " - Setting up TTL test");
          m_cache =  CacheFactory.getCache("TTL_TEST");
          m_listOfTTLs.add( 1 );
          m_listOfTTLs.add( 3 );
          m_listOfTTLs.add( 5 );
          m_listOfTTLs.add( 10);
          System.out.println(new DateTime() + " - Starting TTL test");
          for (final Integer ttl : m_listOfTTLs)
               StopWatch sw = doTest(ttl);
               System.out.println(sw.prettyPrint());
      * Adds a value to the cache with the time-to-live as given by the ttl parameter and waits until it's removed from the cache.
      * Repeats this {@link #RETRIES} times
      * @param ttl
      * @return
      * @throws Exception
     private StopWatch doTest(Integer ttl) throws Exception
          StopWatch stopWatch= new StopWatch("TTL=" + ttl);
          for (int retry = 0; retry < RETRIES; retry++)
               System.out.println(new DateTime() + " - Adding a value in cache for TTL=" + ttl + " in try= " + (retry+1));
               stopWatch.start("Retry="+retry);
               m_cache.put(ttl, null, ttl*1000);
               waitUntilNotInCacheAnymore(ttl, retry);
               stopWatch.stop();
          return stopWatch;
      * Wait until the value for the given ttl is not in the cache anymore
      * @param ttl
      * @param currentTry
      * @throws InterruptedException
     private void waitUntilNotInCacheAnymore(final Integer ttl, final int currentTry) throws InterruptedException
          DateTime startTime = new DateTime();
          long maxMillisToWait = ttl*2*1000;     //wait max 2 times the time of the ttl
          while(m_cache.containsKey(ttl) )
               Duration timeTaken = new Duration(startTime, new DateTime());
               if(timeTaken.getMillis() > maxMillisToWait)
                    throw new RuntimeException("Already waiting " + timeTaken + " for ttl=" + ttl + " and retry=" +  currentTry);
               Thread.sleep(1000);
}You can find the output below:
2009-12-03T11:50:04.584+01:00 - Setting up TTL test
2009-12-03 11:50:04.803/0.250 Oracle Coherence 3.5.2/463p2 <Info> (thread=main, member=n/a): Loaded operational configuration from resource "jar:file:/C:/Temp/coherence3.5.2/coherence-java-v3.5.2b463-p1_2/coherence/lib/coherence.jar!/tangosol-coherence.xml"
2009-12-03 11:50:04.803/0.250 Oracle Coherence 3.5.2/463p2 <Info> (thread=main, member=n/a): Loaded operational overrides from resource "jar:file:/C:/Temp/coherence3.5.2/coherence-java-v3.5.2b463-p1_2/coherence/lib/coherence.jar!/tangosol-coherence-override-dev.xml"
2009-12-03 11:50:04.803/0.250 Oracle Coherence 3.5.2/463p2 <D5> (thread=main, member=n/a): Optional configuration override "/tangosol-coherence-override.xml" is not specified
2009-12-03 11:50:04.803/0.250 Oracle Coherence 3.5.2/463p2 <D5> (thread=main, member=n/a): Optional configuration override "/custom-mbeans.xml" is not specified
Oracle Coherence Version 3.5.2/463p2
Grid Edition: Development mode
Copyright (c) 2000, 2009, Oracle and/or its affiliates. All rights reserved.
2009-12-03 11:50:04.943/0.390 Oracle Coherence GE 3.5.2/463p2 <Info> (thread=main, member=n/a): Loaded cache configuration from "file:/C:/jb/workspace3.5/TTLTest/target/classes/coherence-cache-config.xml"
2009-12-03 11:50:05.318/0.765 Oracle Coherence GE 3.5.2/463p2 <D5> (thread=Cluster, member=n/a): Service Cluster joined the cluster with senior service member n/a
2009-12-03 11:50:08.568/4.015 Oracle Coherence GE 3.5.2/463p2 <Info> (thread=Cluster, member=n/a): Created a new cluster "cluster:0xD3FB" with Member(Id=1, Timestamp=2009-12-03 11:50:05.193, Address=172.16.44.32:8088, MachineId=36896, Location=process:11848, Role=TTLTestServerTTLTestServer, Edition=Grid Edition, Mode=Development, CpuCount=2, SocketCount=2) UID=0xAC102C20000001255429380990201F98
2009-12-03 11:50:08.584/4.031 Oracle Coherence GE 3.5.2/463p2 <D5> (thread=Invocation:Management, member=1): Service Management joined the cluster with senior service member 1
2009-12-03 11:50:08.756/4.203 Oracle Coherence GE 3.5.2/463p2 <D5> (thread=DistributedCache:testService, member=1): Service testService joined the cluster with senior service member 1
2009-12-03T11:50:08.803+01:00 - Starting TTL test
2009-12-03T11:50:08.818+01:00 - Adding a value in cache for TTL=1 in try= 1
2009-12-03T11:50:09.818+01:00 - Adding a value in cache for TTL=1 in try= 2
Exception in thread "main" (Wrapped: Failed request execution for testService service on Member(Id=1, Timestamp=2009-12-03 11:50:05.193, Address=172.16.44.32:8088, MachineId=36896, Location=process:11848, Role=TTLTestServerTTLTestServer)) java.lang.IllegalStateException: the node has already been removed
     at com.tangosol.util.Base.ensureRuntimeException(Base.java:293)
     at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.tagException(Grid.CDB:36)
     at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.DistributedCache.onContainsKeyRequest(DistributedCache.CDB:41)
     at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.DistributedCache$ContainsKeyRequest.run(DistributedCache.CDB:1)
     at com.tangosol.coherence.component.net.message.requestMessage.DistributedCacheKeyRequest.onReceived(DistributedCacheKeyRequest.CDB:12)
     at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.onMessage(Grid.CDB:9)
     at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.onNotify(Grid.CDB:136)
     at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.DistributedCache.onNotify(DistributedCache.CDB:3)
     at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:42)
     at java.lang.Thread.run(Thread.java:619)
Caused by: java.lang.IllegalStateException: the node has already been removed
     at com.tangosol.util.AbstractSparseArray$Crawler.remove(AbstractSparseArray.java:1274)
     at com.tangosol.net.cache.OldCache.evict(OldCache.java:580)
     at com.tangosol.net.cache.OldCache.containsKey(OldCache.java:171)
     at com.tangosol.net.cache.ReadWriteBackingMap.containsKey(ReadWriteBackingMap.java:597)
     at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.DistributedCache.onContainsKeyRequest(DistributedCache.CDB:25)
     ... 7 more
2009-12-03 11:50:10.834/6.281 Oracle Coherence GE 3.5.2/463p2 <D4> (thread=ShutdownHook, member=1): ShutdownHook: stopping cluster node
2009-12-03 11:50:10.834/6.281 Oracle Coherence GE 3.5.2/463p2 <D5> (thread=Cluster, member=1): Service Cluster left the clusterBest regards
Jan