How to properly implement custom ServiceFailurePolicy

Question

I'd like to use Coherence service guardian to interrupt & cancel hung cacheloader threads, which are going into infinite loop due to database connection became unresponsive.
For that purpose I've tried first to create working testcase in order to learn how service guardian works.
I've created a basic distributed cache with fake cache loader to imitate a delay in response, then an infinite loop, and custom test service guardian policy.
I do see cache loader being interrupted by a call to onGuardableRecovery, but then strange thing happens: despite cacheloader thread being successfully released, service guardian still thinks that the service is stuck and tries several times to recover by calling onGuardableRecovery and finally to terminate service with the call to onGuardableTerminate.
What I'm missing here or doing wrong ?
Below are testcase classes & cache config:
package test.guardian;
import com.tangosol.net.CacheFactory;
import com.tangosol.net.cache.CacheStore;
import com.tangosol.util.Base;
import java.util.Collection;
import java.util.Map;
public class testCacheStore extends Base implements CacheStore {
private String datasource_name = null;
public static volatile boolean loop_thread = true;
public testCacheStore(String p_datasource_name) {
CacheFactory.log("inside cacheloader constructor, p_datasource_name="+p_datasource_name,CacheFactory.LOG_DEBUG);
this.datasource_name = p_datasource_name;
public Object load(Object oKey) {
CacheFactory.log("inside cacheloader",CacheFactory.LOG_DEBUG);
try {
Thread.sleep(120*1000);
catch (InterruptedException e) {
CacheFactory.log("loader thread was interrupted"+"\n"+CacheFactory.getStackTrace(e),CacheFactory.LOG_DEBUG);
while (testCacheStore.loop_thread) {String n = "1"+"2";}
CacheFactory.log("loader stop_thread flag was set to stop thread",CacheFactory.LOG_DEBUG);
CacheFactory.log("return value from the store",CacheFactory.LOG_DEBUG);
return ((String) oKey)+"-value";
public Map loadAll(Collection colKeys) {
throw new UnsupportedOperationException();
public void store(Object oKey, Object oValue) {
public void storeAll(Map mapEntries) {
throw new UnsupportedOperationException();
public void erase(Object oKey) {
public void eraseAll(Collection colKeys) {
CacheFactory.log("inside eraseAll method", CacheFactory.LOG_DEBUG);
throw new UnsupportedOperationException();
package test.guardian;
import com.tangosol.net.CacheFactory;
import com.tangosol.net.Guardable;
import com.tangosol.net.ServiceFailurePolicy;
import com.tangosol.net.Service;
import com.tangosol.net.Cluster;
public class mycustomGPolicy implements ServiceFailurePolicy {   
public static mycustomGPolicy getInstance() {
return new mycustomGPolicy();
public mycustomGPolicy(String p_info) {
CacheFactory.log("parametrized policy class instance created",CacheFactory.LOG_DEBUG);
CacheFactory.log("p_info="+p_info,CacheFactory.LOG_DEBUG);
public mycustomGPolicy() {
CacheFactory.log("policy class instance created",CacheFactory.LOG_DEBUG);
//CacheFactory.log("p_info="+p_info,CacheFactory.LOG_DEBUG);
public void onGuardableRecovery(Guardable p_guardable, Service p_service) {
CacheFactory.log("inside onGuardableRecovery",CacheFactory.LOG_DEBUG);
CacheFactory.log("p_guardable="+p_guardable.toString(),CacheFactory.LOG_DEBUG);
CacheFactory.log("p_guardable.getClass().getCanonicalName()="+p_guardable.getClass().getCanonicalName(),CacheFactory.LOG_DEBUG);
testCacheStore.loop_thread = false;
p_guardable.recover();
p_guardable.getContext().heartbeat();
public void onGuardableTerminate(Guardable p_guardable, Service p_service) {
CacheFactory.log("inside onGuardableTerminate",CacheFactory.LOG_DEBUG);
CacheFactory.log("p_guardable="+p_guardable.toString(),CacheFactory.LOG_DEBUG);
CacheFactory.log("p_guardable.getClass().getCanonicalName()="+p_guardable.getClass().getCanonicalName(),CacheFactory.LOG_DEBUG);
testCacheStore.loop_thread = false;
p_guardable.terminate();
public void onServiceFailed(Cluster p_cluster) {
CacheFactory.log("inside onServiceFailed",CacheFactory.LOG_DEBUG);
package test.guardian;
import com.tangosol.net.CacheFactory;
import com.tangosol.net.NamedCache;
public class testGuardianCache {
public static void main(String[] args) {
try {
log("building cache instance");
NamedCache cache = CacheFactory.getCache("test-guardable");
log("requesting cache data");
String resp4 = (String)cache.get("test1223g");
log("response:");
System.out.println(resp4+"\n");
} catch (com.tangosol.io.pof.PortableException e) {
Throwable e_src = e;
while (e_src.getCause() != null ) {
e_src = e_src.getCause();
e_src.printStackTrace();
public static void log(String info) {
System.out.println(info);
<?xml version="1.0" encoding="UTF-8"?>
<cache-config>
     <caching-scheme-mapping>
          <cache-mapping>
               <cache-name>test-guardable</cache-name>
               <scheme-name>test-guardable-schema</scheme-name>
          </cache-mapping>
     </caching-scheme-mapping>
     <caching-schemes>
          <distributed-scheme>
               <scheme-name>test-guardable-schema</scheme-name>
               <service-name>test-guardable-service</service-name>
               <backing-map-scheme>
                    <read-write-backing-map-scheme>
                         <cachestore-timeout>0</cachestore-timeout>
                         <internal-cache-scheme>
                              <external-scheme>
                              <scheme-name>LocalNioLtd</scheme-name>
                              <high-units>10000</high-units>
                              <unit-calculator>FIXED</unit-calculator>
                              <nio-memory-manager>
                              <initial-size>10M</initial-size>
                              <maximum-size>50M</maximum-size>
                              </nio-memory-manager>
                              </external-scheme>
                         </internal-cache-scheme>
                         <cachestore-scheme>
                              <class-scheme>
                                   <class-name>test.guardian.testCacheStore</class-name>
                                   <init-params>
                                   <init-param>
                                   <param-type>java.lang.String</param-type>
                                   <param-value>jdbc-coherence/MY-CACHE-DS</param-value>
                                   </init-param>
                                   </init-params>
                              </class-scheme>
                         </cachestore-scheme>
                         <rollback-cachestore-failures>true</rollback-cachestore-failures>
                         <read-only>true</read-only>
                         <write-delay-seconds>0</write-delay-seconds>
                    </read-write-backing-map-scheme>
               </backing-map-scheme>
               <listener/>
               <autostart>true</autostart>
               <thread-count>10</thread-count>
               <guardian-timeout>10000</guardian-timeout>
               <service-failure-policy>
          <class-name>test.guardian.mycustomGPolicy</class-name>
                                   <init-params>
                                   <init-param>
                                   <param-name>datasource</param-name>
                                   <param-type>java.lang.String</param-type>
                                   <param-value>jdbc-coherence/MY-CACHE-DS</param-value>
                                   </init-param>
                                   </init-params>
          </service-failure-policy>
          </distributed-scheme>
     </caching-schemes>
</cache-config>
<?xml version="1.0"?>
<coherence>
     <services>
          <service>
               <service-type>DistributedCache</service-type>
               <service-component>DistributedCache</service-component>
               <init-params>
                    <init-param id="4">
                         <param-name>local-storage</param-name>
                         <param-value system-property="tangosol.coherence.distributed.localstorage">false</param-value>
                    </init-param>
               </init-params>
          </service>
     </services>
     <cluster-config>
<member-identity>
<cluster-name>WORKSTATION</cluster-name>
</member-identity>
          <unicast-listener>
               <address system-property="tangosol.coherence.localhost">localhost</address>
               <port system-property="tangosol.coherence.localport">7070</port>
               <port-auto-adjust system-property="tangosol.coherence.localport.adjust">true</port-auto-adjust>
               <well-known-addresses>
<socket-address id="1"><address>localhost</address><port>7070</port></socket-address>
</well-known-addresses>
          </unicast-listener>
          <authorized-hosts/>
          <logging-config>
               <severity-level system-property="tangosol.coherence.log.level">5</severity-level>
               <character-limit system-property="tangosol.coherence.log.limit">0</character-limit>
          </logging-config>
     </cluster-config>
     <logging-config>
          <severity-level>5</severity-level>
          <message-format>{date} &lt;{level}&gt; (thread={thread}, member={member}): {text}</message-format>
          <character-limit>8192</character-limit>
     </logging-config>
</coherence>
setlocal
set COHERENCE_HOME=c:\coherence36
set PROG_HOME=C:\coherence36\deploy
set COH_OPTS=-cp %COHERENCE_HOME%\lib\coherence.jar;%PROG_HOME%\testguardable.jar
set JMX_OPTS=-Dcom.sun.management.jmxremote -Dtangosol.coherence.management=all -Dtangosol.coherence.management.remote=true -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.ssl=false
set OVERRIDE_OPTS=-Dtangosol.coherence.override=%PROG_HOME%\tangosol-coherence-override.xml
rem
rem
start "1testguardable" /MAX C:\jdev10g\jdk\jre\bin\java.exe %COH_OPTS% %OVERRIDE_OPTS% -Xms256m -Xmx256m -Xloggc: -Dtangosol.coherence.distributed.localstorage=true -Dtangosol.coherence.cacheconfig=%PROG_HOME%\testguardable-cache-config.xml com.tangosol.net.DefaultCacheServer

Answer 1

INFO - 2010-07-23 23:43:07.906 <Info> (thread=main, member=n/a): Loaded operational configuration from "jar:file:/C:/coherence36/lib/coherence.jar!/tangosol-coherence.xml"
INFO - 2010-07-23 23:43:07.921 <Info> (thread=main, member=n/a): Loaded operational overrides from "file:/C:/-PRG/coherence_ws/Guardian/deploy/tangosol-coherence-override.xml"
DEBUG - 2010-07-23 23:43:07.937 <D5> (thread=main, member=n/a): Optional configuration override "/custom-mbeans.xml" is not specified
DEBUG -
Oracle Coherence Version 3.6.0.0 Build 17229
Grid Edition: Development mode
Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
INFO - 2010-07-23 23:43:08.531 <Info> (thread=main, member=n/a): Loaded cache configuration from "file:/C:/-PRG/coherence_ws/Guardian/deploy/testguardable-cache-config.xml"
DEBUG - 2010-07-23 23:43:09.515 <D4> (thread=main, member=n/a): TCMP bound to /192.168.1.10:7070 using SystemSocketProvider
INFO - 2010-07-23 23:43:41.000 <Info> (thread=Cluster, member=n/a): Created a new cluster "WORKSTATION" with Member(Id=1, Timestamp=2010-07-23 23:43:09.531, Address=192.168.1.10:7070, MachineId=26890, Location=site:local,machine:WS,process:5880, Role=CoherenceServer, Edition=Grid Edition, Mode=Development, CpuCount=2, SocketCount=1) UID=0xC0A8010A0000012A01427FDB690A1B9E
INFO - 2010-07-23 23:43:41.015 <Info> (thread=main, member=n/a): Started cluster Name=WORKSTATION
WellKnownAddressList(Size=1,
WKA{Address=192.168.1.10, Port=7070}
MasterMemberSet
ThisMember=Member(Id=1, Timestamp=2010-07-23 23:43:09.531, Address=192.168.1.10:7070, MachineId=26890, Location=site:local,machine:WS,process:5880, Role=CoherenceServer)
OldestMember=Member(Id=1, Timestamp=2010-07-23 23:43:09.531, Address=192.168.1.10:7070, MachineId=26890, Location=site:local,machine:WS,process:5880, Role=CoherenceServer)
ActualMemberSet=MemberSet(Size=1, BitSetCount=2
Member(Id=1, Timestamp=2010-07-23 23:43:09.531, Address=192.168.1.10:7070, MachineId=26890, Location=site:local,machine:WS,process:5880, Role=CoherenceServer)
RecycleMillis=1200000
RecycleSet=MemberSet(Size=0, BitSetCount=0
TcpRing{Connections=[]}
IpMonitor{AddressListSize=0}
DEBUG - 2010-07-23 23:43:41.093 <D5> (thread=Invocation:Management, member=1): Service Management joined the cluster with senior service member 1
DEBUG - 2010-07-23 23:43:41.718 <D5> (thread=main, member=1): parametrized policy class instance created
DEBUG - 2010-07-23 23:43:41.718 <D5> (thread=main, member=1): p_info=jdbc-coherence/MY-CACHE-DS
DEBUG - 2010-07-23 23:43:41.750 <D5> (thread=DistributedCache:test-guardable-service, member=1): Service test-guardable-service joined the cluster with senior service member 1
INFO - 2010-07-23 23:43:41.890 <Info> (thread=main, member=1):
Services
ClusterService{Name=Cluster, State=(SERVICE_STARTED, STATE_JOINED), Id=0, Version=3.6, OldestMemberId=1}
InvocationService{Name=Management, State=(SERVICE_STARTED), Id=1, Version=3.1, OldestMemberId=1}
PartitionedCache{Name=test-guardable-service, State=(SERVICE_STARTED), LocalStorage=enabled, PartitionCount=257, BackupCount=1, AssignedPartitions=257, BackupPartitions=0}
Started DefaultCacheServer...
DEBUG - 2010-07-23 23:44:50.250 <D5> (thread=Cluster, member=1): Member(Id=2, Timestamp=2010-07-23 23:44:50.258, Address=192.168.1.10:7072, MachineId=26890, Location=site:local,machine:WS,process:3376, Role=TestGuardianTestGuardianCache) joined Cluster with senior member 1
DEBUG - 2010-07-23 23:44:50.390 <D5> (thread=Cluster, member=1): Member 2 joined Service Management with senior member 1
DEBUG - 2010-07-23 23:44:51.093 <D5> (thread=Cluster, member=1): Member 2 joined Service test-guardable-service with senior member 1
DEBUG - 2010-07-23 23:44:51.218 <D5> (thread=DistributedCache:test-guardable-service, member=1): inside cacheloader constructor, p_datasource_name=jdbc-coherence/MY-CACHE-DS
DEBUG - 2010-07-23 23:44:51.359 <D5> (thread=test-guardable-serviceWorker:4, member=1): inside cacheloader
DEBUG - 2010-07-23 23:45:00.375 <D5> (thread=Recovery Thread, member=1): inside onGuardableRecovery
DEBUG - 2010-07-23 23:45:00.375 <D5> (thread=Recovery Thread, member=1): p_guardable=Guard{Daemon=test-guardable-serviceWorker:4}
DEBUG - 2010-07-23 23:45:00.375 <D5> (thread=Recovery Thread, member=1): p_guardable.getClass().getCanonicalName()=com.tangosol.coherence.component.util.daemon.queueProcessor.Service$DaemonPool$Daemon$Guard
ERROR - 2010-07-23 23:45:00.375 <Error> (thread=DistributedCache:test-guardable-service, member=1): Attempting recovery (due to soft timeout) of {WrapperGuardable Guard{Daemon=test-guardable-serviceWorker:4} Service=PartitionedCache{Name=test-guardable-service, State=(SERVICE_STARTED), LocalStorage=enabled, PartitionCount=257, BackupCount=1, AssignedPartitions=257, BackupPartitions=0}}
WARN - 2010-07-23 23:45:00.375 <Warning> (thread=Recovery Thread, member=1): A worker thread has been executing task: Message "GetRequest"
FromMember=Member(Id=2, Timestamp=2010-07-23 23:44:50.258, Address=192.168.1.10:7072, MachineId=26890, Location=site:local,machine:WS,process:3376, Role=TestGuardianTestGuardianCache)
FromMessageId=29
Internal=false
MessagePartCount=1
PendingCount=0
MessageType=59
ToPollId=0
Poll=null
Packets
[000]=Directed{PacketType=0x0DDF00D5, ToId=1, FromId=2, Direction=Incoming, ReceivedMillis=23:44:51.343, ToMemberSet=null, ServiceId=2, MessageType=59, FromMessageId=29, ToMessageId=27, MessagePartCount=1, MessagePartIndex=0, NackInProgress=false, ResendScheduled=none, Timeout=none, PendingResendSkips=0, DeliveryState=unsent, Body=0}
Service=PartitionedCache{Name=test-guardable-service, State=(SERVICE_STARTED), LocalStorage=enabled, PartitionCount=257, BackupCount=1, AssignedPartitions=257, BackupPartitions=0}
ToMemberSet=MemberSet(Size=1, BitSetCount=2
Member(Id=1, Timestamp=2010-07-23 23:43:09.531, Address=192.168.1.10:7070, MachineId=26890, Location=site:local,machine:WS,process:5880, Role=CoherenceServer)
NotifySent=false
} for 9016ms and appears to be stuck; attempting to interrupt: test-guardable-serviceWorker:4
DEBUG - 2010-07-23 23:45:00.375 <D5> (thread=test-guardable-serviceWorker:4, member=1): loader thread was interrupted
     at java.lang.Thread.sleep(Native Method)
     at test.guardian.testCacheStore.load(testCacheStore.java:26)
     at com.tangosol.net.cache.ReadWriteBackingMap$CacheStoreWrapper.load(ReadWriteBackingMap.java:4759)
     at com.tangosol.net.cache.ReadWriteBackingMap$CacheStoreWrapper.loadInternal(ReadWriteBackingMap.java:4344)
     at com.tangosol.net.cache.ReadWriteBackingMap.get(ReadWriteBackingMap.java:807)
     at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache.onGetRequest(PartitionedCache.CDB:22)
     at com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$GetRequest.run(PartitionedCache.CDB:1)
     at com.tangosol.coherence.component.util.DaemonPool$WrapperTask.run(DaemonPool.CDB:1)
     at com.tangosol.coherence.component.util.DaemonPool$WrapperTask.run(DaemonPool.CDB:32)
     at com.tangosol.coherence.component.util.DaemonPool$Daemon.onNotify(DaemonPool.CDB:63)
     at com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:42)
     at java.lang.Thread.run(Thread.java:595)
DEBUG - 2010-07-23 23:45:00.375 <D5> (thread=test-guardable-serviceWorker:4, member=1): loader stop_thread flag was set to stop thread
DEBUG - 2010-07-23 23:45:00.375 <D5> (thread=test-guardable-serviceWorker:4, member=1): return value from the store
DEBUG - 2010-07-23 23:45:00.390 <D5> (thread=Cluster, member=1): TcpRing disconnected from Member(Id=2, Timestamp=2010-07-23 23:44:50.258, Address=192.168.1.10:7072, MachineId=26890, Location=site:local,machine:WS,process:3376, Role=TestGuardianTestGuardianCache) due to a peer departure; removing the member.
DEBUG - 2010-07-23 23:45:00.390 <D5> (thread=Cluster, member=1): Member 2 left service Management with senior member 1
DEBUG - 2010-07-23 23:45:00.390 <D5> (thread=Cluster, member=1): Member 2 left service test-guardable-service with senior member 1
DEBUG - 2010-07-23 23:45:00.390 <D5> (thread=Cluster, member=1): Member(Id=2, Timestamp=2010-07-23 23:45:00.39, Address=192.168.1.10:7072, MachineId=26890, Location=site:local,machine:WS,process:3376, Role=TestGuardianTestGuardianCache) left Cluster with senior member 1
ERROR - 2010-07-23 23:48:16.250 <Error> (thread=Cluster, member=1): Attempting recovery (due to soft timeout) of {WrapperGuardable Guard{Daemon=DistributedCache:test-guardable-service} Service=PartitionedCache{Name=test-guardable-service, State=(SERVICE_STARTED), LocalStorage=enabled, PartitionCount=257, BackupCount=1, AssignedPartitions=257, BackupPartitions=0}}
DEBUG - 2010-07-23 23:48:16.250 <D5> (thread=Recovery Thread, member=1): inside onGuardableRecovery
DEBUG - 2010-07-23 23:48:16.250 <D5> (thread=Recovery Thread, member=1): p_guardable=Guard{Daemon=DistributedCache:test-guardable-service}
DEBUG - 2010-07-23 23:48:16.250 <D5> (thread=Recovery Thread, member=1): p_guardable.getClass().getCanonicalName()=com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$Guard
DEBUG - Interrupted PartitionedCache, Thread[DistributedCache:test-guardable-service,5,Cluster]
ERROR - 2010-07-23 23:48:46.765 <Error> (thread=Cluster, member=1): Terminating guarded execution (due to hard timeout) of {WrapperGuardable Guard{Daemon=DistributedCache:test-guardable-service} Service=PartitionedCache{Name=test-guardable-service, State=(SERVICE_STARTED), LocalStorage=enabled, PartitionCount=257, BackupCount=1, AssignedPartitions=257, BackupPartitions=0}}
DEBUG - 2010-07-23 23:48:46.765 <D5> (thread=Termination Thread, member=1): inside onGuardableTerminate
DEBUG - 2010-07-23 23:48:46.765 <D5> (thread=Termination Thread, member=1): p_guardable=Guard{Daemon=DistributedCache:test-guardable-service}
DEBUG - 2010-07-23 23:48:46.765 <D5> (thread=Termination Thread, member=1): p_guardable.getClass().getCanonicalName()=com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$Guard
DEBUG - 2010-07-23 23:48:46.765 <D5> (thread=DistributedCache:test-guardable-service, member=1): Service test-guardable-service left the cluster
INFO - 2010-07-23 23:48:46.890 <Info> (thread=main, member=1): Restarting Service: test-guardable-service
DEBUG - 2010-07-23 23:48:46.906 <D5> (thread=main, member=1): parametrized policy class instance created
DEBUG - 2010-07-23 23:48:46.906 <D5> (thread=main, member=1): p_info=jdbc-coherence/MY-CACHE-DS
DEBUG - 2010-07-23 23:48:46.906 <D5> (thread=DistributedCache:test-guardable-service, member=1): Service test-guardable-service joined the cluster with senior service member 1
ERROR - 2010-07-23 23:48:55.265 <Error> (thread=Cluster, member=1): Attempting recovery (due to soft timeout) of {WrapperGuardable Guard{Daemon=DistributedCache:test-guardable-service} Service=PartitionedCache{Name=test-guardable-service, State=(SERVICE_STOPPED), Not initialized}}
DEBUG - 2010-07-23 23:48:55.265 <D5> (thread=Recovery Thread, member=1): inside onGuardableRecovery
DEBUG - 2010-07-23 23:48:55.265 <D5> (thread=Recovery Thread, member=1): p_guardable=Guard{Daemon=DistributedCache:test-guardable-service}
DEBUG - 2010-07-23 23:48:55.265 <D5> (thread=Recovery Thread, member=1): p_guardable.getClass().getCanonicalName()=com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$Guard
ERROR - 2010-07-23 23:48:56.281 <Error> (thread=Cluster, member=1): Terminating guarded execution (due to hard timeout) of {WrapperGuardable Guard{Daemon=DistributedCache:test-guardable-service} Service=PartitionedCache{Name=test-guardable-service, State=(SERVICE_STOPPED), Not initialized}}
DEBUG - 2010-07-23 23:48:56.281 <D5> (thread=Termination Thread, member=1): inside onGuardableTerminate
DEBUG - 2010-07-23 23:48:56.281 <D5> (thread=Termination Thread, member=1): p_guardable=Guard{Daemon=DistributedCache:test-guardable-service}
DEBUG - 2010-07-23 23:48:56.281 <D5> (thread=Termination Thread, member=1): p_guardable.getClass().getCanonicalName()=com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$Guard