Classloader leak because of DFC

Nothing to add to Being without dfc.properties, just a solution:

import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.security.AccessControlContext;
import java.security.Policy;
import java.security.ProtectionDomain;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;

import org.springframework.util.ReflectionUtils;

import com.documentum.fc.common.DfLogger;
/**
 * @author Andrey B. Panfilov <andrew@panfilov.tel>
 */
@SuppressWarnings("PMD")
public class DfcCleanupListener implements ServletContextListener {

    public DfcCleanupListener() {
        super();
    }

    @Override
    public void contextInitialized(ServletContextEvent servletContextEvent) {
        // do nothing
    }

    @Override
    public void contextDestroyed(ServletContextEvent servletContextEvent) {
        launchHooks();
        cleanupThreads();
        cleanupSecurityPolicy();
        cleanupLog4j();
    }

    private void cleanupLog4j() {
        Object loggers = getValue("s_loggers", DfLogger.class);
        if (loggers != null) {
            loggers = getValue("m_loggersMap", loggers);
        }
        Object prefixes = getValue("s_prefixes", DfLogger.class);
        Object counters = getValue("s_muteCounter", DfLogger.class);
        for (Thread thread : Thread.getAllStackTraces().keySet()) {
            Object threadLocals = getValue("threadLocals", thread);
            if (threadLocals == null) {
                continue;
            }
            if (loggers != null) {
                invoke("remove", threadLocals, loggers);
            }
            if (prefixes != null) {
                invoke("remove", threadLocals, prefixes);
            }
            if (counters != null) {
                invoke("remove", threadLocals, counters);
            }
        }
    }

    private void cleanupSecurityPolicy() {
        Policy.setPolicy(null);
    }

    @SuppressWarnings("unchecked")
    private void launchHooks() {
        try {
            Class clazz = Class.forName("java.lang.ApplicationShutdownHooks");
            Object hooks = getValue("hooks", clazz);
            if (hooks == null) {
                return;
            }
            List<Thread> threads = new ArrayList<Thread>();
            for (Thread thread : ((Map<Thread, Thread>) hooks).keySet()) {
                if (isInSameClassLoader(thread, getClass().getClassLoader())) {
                    threads.add(thread);
                }
            }
            for (Thread thread : threads) {
                Runtime.getRuntime().removeShutdownHook(thread);
            }
            for (Thread hook : threads) {
                hook.start();
            }
            for (Thread hook : threads) {
                try {
                    hook.join();
                } catch (InterruptedException ex) {
                    Logger.error(ex.getMessage(), ex);
                }
            }
        } catch (ClassNotFoundException ex) {
            Logger.error(ex.getMessage(), ex);
        }
    }

    private void cleanupThreads() {
        for (Thread thread : getThreadsWithSameClassLoader(getClass()
                .getClassLoader())) {
            Logger.error("Stopping thread {0}", null, thread.getName());
            stopThread(thread);
        }
    }

    private void stopThread(Thread thread) {
        if ("java.util.TimerThread".equals(thread.getClass().getName())) {
            cancelTimerThread(thread);
        }
        if (thread.isAlive()) {
            thread.interrupt();
        }
    }

    private void cancelTimerThread(Thread thread) {
        Object queue = getValue("queue", thread);
        if (queue == null) {
            return;
        }
        synchronized (queue) {
            setField("newTasksMayBeScheduled", thread, false);
            invoke("clear", queue);
            invoke("notify", queue);
        }
    }

    private static <T> T invoke(String method, Object object, Object... args) {
        try {
            if (object == null) {
                return null;
            }
            Method mtd = ReflectionUtils.findMethod(object.getClass(), method);
            if (mtd == null) {
                return null;
            }
            return invoke(mtd, object, args);
        } catch (InvocationTargetException | IllegalAccessException e) {
            Logger.error(e.getMessage(), e);
        }
        return null;
    }

    @SuppressWarnings("unchecked")
    private static <T> T invoke(Method method, Object object, Object... args)
        throws IllegalAccessException, InvocationTargetException {
        ReflectionUtils.makeAccessible(method);
        return (T) method.invoke(object, args);
    }

    private static void setField(String field, Object object, Object value) {
        try {
            if (object == null) {
                return;
            }
            Field fld = ReflectionUtils.findField(object.getClass(), field);
            if (fld == null) {
                return;
            }
            setField(fld, object, value);
        } catch (InvocationTargetException | IllegalAccessException e) {
            Logger.error(e.getMessage(), e);
        }
    }

    private static void setField(Field field, Object object, Object value)
        throws IllegalAccessException, InvocationTargetException {
        ReflectionUtils.makeAccessible(field);
        field.set(object, value);
    }

    private static Object getValue(String field, Object object) {
        try {
            if (object == null) {
                return null;
            }
            Field fld = ReflectionUtils.findField(object.getClass(), field);
            if (fld == null) {
                return null;
            }
            return getValue(fld, object);
        } catch (InvocationTargetException | IllegalAccessException e) {
            Logger.error(e.getMessage(), e);
        }
        return null;
    }

    private static Object getValue(String field, Class clazz) {
        try {
            if (clazz == null) {
                return null;
            }
            Field fld = ReflectionUtils.findField(clazz, field);
            if (fld == null) {
                return null;
            }
            return getValue(fld, null);
        } catch (InvocationTargetException | IllegalAccessException e) {
            Logger.error(e.getMessage(), e);
        }
        return null;
    }

    private static Object getValue(Field field, Object object)
        throws IllegalAccessException, InvocationTargetException {
        ReflectionUtils.makeAccessible(field);
        return field.get(object);
    }

    public static List<Thread> getThreadsWithSameClassLoader(
            ClassLoader classLoader) {
        List<Thread> result = new ArrayList<Thread>();
        for (Thread thread : Thread.getAllStackTraces().keySet()) {
            if (isInSameClassLoader(thread, classLoader)) {
                result.add(thread);
            }
        }
        return result;
    }

    private static boolean isInSameClassLoader(Thread thread,
            ClassLoader classLoader) {
        if (classLoader == null) {
            return false;
        }
        Object inheritedAccessControlContext = getValue(
                "inheritedAccessControlContext", thread);
        if (!(inheritedAccessControlContext instanceof AccessControlContext)) {
            return false;
        }
        Object contexts = getValue("context", inheritedAccessControlContext);
        if (!(contexts instanceof ProtectionDomain[])) {
            return false;
        }
        for (ProtectionDomain domain : (ProtectionDomain[]) contexts) {
            if (domain == null) {
                continue;
            }
            if (classLoader.equals(domain.getClassLoader())) {
                return true;
            }
        }
        return false;
    }

}

6 thoughts on “Classloader leak because of DFC

  1. How this code solves common leak like this:

    The web application [context-name] created a ThreadLocal with key of type [com.documentum.fc.client.impl.bof.cache.ClassCacheManager$2]?

    Like

  2. That’s a good question.

    com.documentum.fc.client.impl.bof.cache.ClassCacheManager$2 is originated from anonymous class in ClassCacheManager:

    ThreadLocal<Boolean> m_downloading = new ThreadLocal<Boolean>() {
    	protected synchronized Boolean initialValue() {
    		return Boolean.FALSE;
    	}
    };
    

    and I would say that whether it is a leak or not depends on Application Classloader, for example, TomCat cleanups (or at least tries) such references (WebappClassLoaderBase.java#checkThreadLocalsForLeaks), one the other hand, I specially added cleanup code for log4j because in my environment cleaning up log4j references takes significant time (about 30 minutes). So, the question is how to help GC and make it happy, for example, in case of ClassCacheManager it is possible to write something like:

    private void cleanupThreadLocals() {
        List<Object> leaks = getThreadLocalLeaks();
        for (Thread thread : Thread.getAllStackTraces().keySet()) {
            Object threadLocals = getValue("threadLocals", thread);
            if (threadLocals == null) {
                continue;
            }
            for (Object object : leaks) {
                if (object == null) {
                    continue;
                }
                invoke("remove", threadLocals, object);
            }
        }
    }
    
    protected List<Object> getThreadLocalLeaks() {
        List<Object> result = new ArrayList<>();
    
        // DfLogger leaks
        Object loggers = getValue("s_loggers", DfLogger.class);
        if (loggers != null) {
            result.add(getValue("m_loggersMap", loggers));
        }
        result.add(getValue("s_prefixes", DfLogger.class));
        result.add(getValue("s_muteCounter", DfLogger.class));
    
        // ClassCacheManager leaks
        result.add(getValue("m_downloading", ClassCacheManager.getInstance()));
        return result;
    }
    

    But the problem is that EMC coders put ThreadLocals into code without thinking about consequences, and if the goal is to remove all potential leaks it is required to analyze all cases, for example, I have found following ThreadLocals in DFC:

    com.documentum.dmcl.impl.ApiContext#m_lastSessionContext
    com.documentum.dmcl.impl.ApiContext#m_persistentObjectAccessTracking
    com.documentum.dmcl.impl.ApiSessionContextForApiSession#m_exceptionChain
    com.documentum.fc.client.DfClient.ClientImpl#m_ignoreFilePathLengthErrors
    com.documentum.fc.client.DfPersistentObject#m_refreshingData
    com.documentum.fc.client.acs.impl.DfAcsClient#m_requestExpirationInterval
    com.documentum.fc.client.fulltext.impl.DfFtUtils#s_dateFormat
    com.documentum.fc.client.impl.bof.cache.ClassCacheManager#m_downloading
    com.documentum.fc.client.impl.bof.classmgmt.DelayedDelegationClassLoader#m_findingClass
    com.documentum.fc.client.impl.bof.classmgmt.DelayedDelegationClassLoader#m_definingPackage
    com.documentum.fc.client.impl.bof.security.RoleRequestManager#m_roleRequests
    com.documentum.fc.client.impl.objectprotocol.AbstractObjectProtocol#s_dateFormat
    com.documentum.fc.client.impl.objectprotocol.ObjectProtocolV1#s_UTCDateFormat
    com.documentum.fc.client.impl.util.LenientDmclDateFormat#m_candidateFormats
    com.documentum.fc.client.transaction.impl.TransactionManager#m_activeTransaction
    com.documentum.fc.common.DfLogger#s_prefixes
    com.documentum.fc.common.DfLogger#s_muteCounter
    com.documentum.fc.common.DfLogger.Loggers#m_loggersMap
    com.documentum.fc.expr.impl.lang.docbasic.runtime.DfDbDateVariant#t_dateFormats
    com.documentum.fc.tracing.impl.TracerLayout#m_callDepthHistory
    com.documentum.registry.DfRegistrySnapshot#m_currentKey
    com.documentum.tracing.tracer.DfTracer#s_threadContext

    And the problem is that some ThreadLocals are static fields, some ThreadLocals are singleton’s fields (ClassCacheManager for example) and some are object’s fields – in this case it is required to use TomCat’s approach.

    Like

  3. Seems to me issue with threadlocals was non-blocking.

    Something else in our application are leaking. Maybe DFC (we are using DFC 7.1 together with xCP 2.1) or not. Don’t have enough of time to investigate.

    But thx for sharing code. It very useful.

    Like

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s