This is a object containing several threadpools. Each pool has a name. There is a executorMap of <name, Executor> inside ExecutorService recording the mapping from name to executor , which contains a threadpool. User calls instance.startExecutorService(ExecutorType, maxThreads) to create the Executors. The ExecutorType in HBase usage is a enum, and we can treat it as a String, and maxThreads is the size of threadPool.
Then,
user call submit EventHandler to add executable handlers for the
Executor (threadpool). Each EventHandler has a EventType. In HBase
usage, EventType is also a enum. The important trick is that, there is
inner mapping from EventType to ExecutorType, so that each EventHandler
is associated with a Executor. Here we use a Map to record the mapping
relationship, and use registerEventToExecutor(EventType, ExecutorType)
to relate them.
For EventHandler, it is an abstract class, and
user can define his own concrete implementations for function
process(). What's more user can use setListener(EventHandlerListener) to
register a coprocessor for each EventHandler. The EventHandlerListerner
is also an interface, and user can define his own function for
beforeProcess(EventHandler) and afterProcess(EventHandler). When
EventHandler is executed, the order of calling is beforeProcess ->
process -> afterProcess.
When done call shutdown().
UML of ExecutorService as belows. Note: ExecutorType and EventType are inner mapped
We extract the ExecutorService from HBase and use it for our own logic, and there are the source codes of the example.
package ExecutorServiceStudy; import java.io.IOException; import java.io.Writer; import java.lang.management.ThreadInfo; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import ExecutorServiceStudy.EventHandler.EventHandlerListener; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; /** * This is a generic executor service. This component abstracts a threadpool, a * queue to which {@link EventHandler.EventType}s can be submitted, and a * <code>Runnable</code> that handles the object that is added to the queue. * * <p> * In order to create a new service, create an instance of this class and then * do: <code>instance.startExecutorService("myService");</code>. When done call * {@link #shutdown()}. * * <p> * In order to use the service created above, call {@link #submit(EventHandler)} * . Register pre- and post- processing listeners by registering your * implementation of {@link EventHandler.EventHandlerListener} with * {@link #registerListener(EventHandler.EventType, EventHandler.EventHandlerListener)} * . Be sure to deregister your listener when done via * {@link #unregisterListener(EventHandler.EventType)}. */ public class ExecutorService { private static final Log LOG = LogFactory.getLog(ExecutorService.class); // hold the all the executors created in a map addressable by their names private final ConcurrentHashMap<String, Executor> executorMap = new ConcurrentHashMap<String, Executor>(); // listeners that are called before and after an event is processed private ConcurrentHashMap<EventHandler.EventType, EventHandlerListener> eventHandlerListeners = new ConcurrentHashMap<EventHandler.EventType, EventHandlerListener>(); // Name of the server hosting this executor service. private final String servername; // a map from EventHandler.EventType to ExecutorType private Map<EventHandler.EventType, ExecutorType> eventType2ExecutorType = Maps .newHashMap(); // user register the mapping from event type to executor type public boolean registerEventToExecutor(EventHandler.EventType eventtype, ExecutorType executortype) { if (eventType2ExecutorType.containsKey(eventtype)) { return false; } else { eventType2ExecutorType.put(eventtype, executortype); return true; } } public String getInnerExecutors() { StringBuffer sb = new StringBuffer(); for (EventHandler.EventType et : eventType2ExecutorType.keySet()) { sb.append(et.eventType).append(": ") .append(eventType2ExecutorType.get(et).executorType) .append("\n"); } return sb.toString(); } /** * The following is a list of all executor types, both those that run in the * master and those that run in the regionserver. */ static public class ExecutorType { public String executorType; public ExecutorType(String value) { this.executorType = value; } /** * @param serverName * @return Conflation of the executor type and the passed servername. */ String getExecutorName(String serverName) { return this.toString() + "-" + serverName; } } /** * Returns the executor service type (the thread pool instance) for the * passed event handler type. * * @param type * EventHandler type. */ public ExecutorType getExecutorServiceType(final EventHandler.EventType type) { return eventType2ExecutorType.get(type); } /** * Default constructor. * * @param servername * Name of the hosting server. */ public ExecutorService(final String servername) { super(); this.servername = servername; } /** * Start an executor service with a given name. If there was a service * already started with the same name, this throws a RuntimeException. * * @param name * Name of the service to start. */ void startExecutorService(String name, int maxThreads) { if (this.executorMap.get(name) != null) { throw new RuntimeException("An executor service with the name " + name + " is already running!"); } Executor hbes = new Executor(name, maxThreads, this.eventHandlerListeners); if (this.executorMap.putIfAbsent(name, hbes) != null) { throw new RuntimeException("An executor service with the name " + name + " is already running (2)!"); } LOG.debug("Starting executor service name=" + name + ", corePoolSize=" + hbes.threadPoolExecutor.getCorePoolSize() + ", maxPoolSize=" + hbes.threadPoolExecutor.getMaximumPoolSize()); } boolean isExecutorServiceRunning(String name) { return this.executorMap.containsKey(name); } public void shutdown() { for (Entry<String, Executor> entry : this.executorMap.entrySet()) { List<Runnable> wasRunning = entry.getValue().threadPoolExecutor .shutdownNow(); if (!wasRunning.isEmpty()) { LOG.info(entry.getValue() + " had " + wasRunning + " on shutdown"); } } this.executorMap.clear(); } Executor getExecutor(final ExecutorType type) { return getExecutor(type.getExecutorName(this.servername)); } Executor getExecutor(String name) { Executor executor = this.executorMap.get(name); return executor; } public void startExecutorService(final ExecutorType type, final int maxThreads) { String name = type.getExecutorName(this.servername); if (isExecutorServiceRunning(name)) { LOG.debug("Executor service " + toString() + " already running on " + this.servername); return; } startExecutorService(name, maxThreads); } public void submit(final EventHandler eh) { Executor executor = getExecutor(getExecutorServiceType(eh .getEventType())); if (executor == null) { // This happens only when events are submitted after shutdown() was // called, so dropping them should be "ok" since it means we're // shutting down. LOG.error("Cannot submit [" + eh + "] because the executor is missing." + " Is this process shutting down?"); } else { executor.submit(eh); } } /** * Subscribe to updates before and after processing instances of * {@link EventHandler.EventType}. Currently only one listener per event * type. * * @param type * Type of event we're registering listener for * @param listener * The listener to run. */ public void registerListener(final EventHandler.EventType type, final EventHandlerListener listener) { this.eventHandlerListeners.put(type, listener); } /** * Stop receiving updates before and after processing instances of * {@link EventHandler.EventType} * * @param type * Type of event we're registering listener for * @return The listener we removed or null if we did not remove it. */ public EventHandlerListener unregisterListener( final EventHandler.EventType type) { return this.eventHandlerListeners.remove(type); } public Map<String, ExecutorStatus> getAllExecutorStatuses() { Map<String, ExecutorStatus> ret = Maps.newHashMap(); for (Map.Entry<String, Executor> e : executorMap.entrySet()) { ret.put(e.getKey(), e.getValue().getStatus()); } return ret; } /** * Executor instance. */ static class Executor { // how long to retain excess threads final long keepAliveTimeInMillis = 1000; // the thread pool executor that services the requests final TrackingThreadPoolExecutor threadPoolExecutor; // work queue to use - unbounded queue final BlockingQueue<Runnable> q = new LinkedBlockingQueue<Runnable>(); private final String name; private final Map<EventHandler.EventType, EventHandlerListener> eventHandlerListeners; private static final AtomicLong seqids = new AtomicLong(0); private final long id; protected Executor( String name, int maxThreads, final Map<EventHandler.EventType, EventHandlerListener> eventHandlerListeners) { this.id = seqids.incrementAndGet(); this.name = name; this.eventHandlerListeners = eventHandlerListeners; // create the thread pool executor this.threadPoolExecutor = new TrackingThreadPoolExecutor( maxThreads, maxThreads, keepAliveTimeInMillis, TimeUnit.MILLISECONDS, q); // name the threads for this threadpool ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); tfb.setNameFormat(this.name + "-%d"); this.threadPoolExecutor.setThreadFactory(tfb.build()); } /** * Submit the event to the queue for handling. * * @param event */ void submit(final EventHandler event) { // If there is a listener for this type, make sure we call the // before // and after process methods. EventHandlerListener listener = this.eventHandlerListeners .get(event.getEventType()); if (listener != null) { event.setListener(listener); } this.threadPoolExecutor.execute(event); } public String toString() { return getClass().getSimpleName() + "-" + id + "-" + name; } public ExecutorStatus getStatus() { List<EventHandler> queuedEvents = Lists.newArrayList(); for (Runnable r : q) { if (!(r instanceof EventHandler)) { LOG.warn("Non-EventHandler " + r + " queued in " + name); continue; } queuedEvents.add((EventHandler) r); } List<RunningEventStatus> running = Lists.newArrayList(); for (Map.Entry<Thread, Runnable> e : threadPoolExecutor .getRunningTasks().entrySet()) { Runnable r = e.getValue(); if (!(r instanceof EventHandler)) { LOG.warn("Non-EventHandler " + r + " running in " + name); continue; } running.add(new RunningEventStatus(e.getKey(), (EventHandler) r)); } return new ExecutorStatus(this, queuedEvents, running); } } /** * A subclass of ThreadPoolExecutor that keeps track of the Runnables that * are executing at any given point in time. */ static class TrackingThreadPoolExecutor extends ThreadPoolExecutor { private ConcurrentMap<Thread, Runnable> running = Maps .newConcurrentMap(); public TrackingThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); } @Override protected void afterExecute(Runnable r, Throwable t) { super.afterExecute(r, t); running.remove(Thread.currentThread()); } @Override protected void beforeExecute(Thread t, Runnable r) { Runnable oldPut = running.put(t, r); assert oldPut == null : "inconsistency for thread " + t; super.beforeExecute(t, r); } /** * @return a map of the threads currently running tasks inside this * executor. Each key is an active thread, and the value is the * task that is currently running. Note that this is not a * stable snapshot of the map. */ public ConcurrentMap<Thread, Runnable> getRunningTasks() { return running; } } /** * A snapshot of the status of a particular executor. This includes the * contents of the executor's pending queue, as well as the threads and * events currently being processed. * * This is a consistent snapshot that is immutable once constructed. */ public static class ExecutorStatus { final Executor executor; final List<EventHandler> queuedEvents; final List<RunningEventStatus> running; ExecutorStatus(Executor executor, List<EventHandler> queuedEvents, List<RunningEventStatus> running) { this.executor = executor; this.queuedEvents = queuedEvents; this.running = running; } /** * Dump a textual representation of the executor's status to the given * writer. * * @param out * the stream to write to * @param indent * a string prefix for each line, used for indentation */ public void dumpTo(Writer out, String indent) throws IOException { out.write(indent + "Status for executor: " + executor + "\n"); out.write(indent + "=======================================\n"); out.write(indent + queuedEvents.size() + " events queued, " + running.size() + " running\n"); if (!queuedEvents.isEmpty()) { out.write(indent + "Queued:\n"); for (EventHandler e : queuedEvents) { out.write(indent + " " + e + "\n"); } out.write("\n"); } if (!running.isEmpty()) { out.write(indent + "Running:\n"); for (RunningEventStatus stat : running) { out.write(indent + " Running on thread '" + stat.threadInfo.getThreadName() + "': " + stat.event + "\n"); out.write(ThreadMonitoring.formatThreadInfo( stat.threadInfo, indent + " ")); out.write("\n"); } } out.flush(); } public String toString() { StringBuffer sb = new StringBuffer(); String indent = ""; sb.append(indent + "Status for executor: " + executor + "\n"); sb.append(indent + "=======================================\n"); sb.append(indent + queuedEvents.size() + " events queued, " + running.size() + " running\n"); if (!queuedEvents.isEmpty()) { sb.append(indent + "Queued:\n"); for (EventHandler e : queuedEvents) { sb.append(indent + " " + e + "\n"); } sb.append("\n"); } if (!running.isEmpty()) { sb.append(indent + "Running:\n"); for (RunningEventStatus stat : running) { sb.append(indent + " Running on thread '" + stat.threadInfo.getThreadName() + "': " + stat.event + "\n"); sb.append(ThreadMonitoring.formatThreadInfo( stat.threadInfo, indent + " ")); sb.append("\n"); } } return sb.toString(); } } /** * The status of a particular event that is in the middle of being handled * by an executor. */ public static class RunningEventStatus { final ThreadInfo threadInfo; final EventHandler event; public RunningEventStatus(Thread t, EventHandler event) { this.threadInfo = ThreadMonitoring.getThreadInfo(t); this.event = event; } } } package ExecutorServiceStudy; import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; /** * Abstract base class for all HBase event handlers. Subclasses should implement * the {@link #process()} method. Subclasses should also do all necessary checks * up in their constructor if possible -- check table exists, is disabled, etc. * -- so they fail fast rather than later when process is running. Do it this * way because process be invoked directly but event handlers are also run in an * executor context -- i.e. asynchronously -- and in this case, exceptions * thrown at process time will not be seen by the invoker, not till we implement * a call-back mechanism so the client can pick them up later. * <p> * Event handlers have an {@link EventType}. {@link EventType} is a list of ALL * handler event types. We need to keep a full list in one place -- and as enums * is a good shorthand for an implemenations -- because event handlers can be * passed to executors when they are to be run asynchronously. The hbase * executor, see {@link ExecutorService}, has a switch for passing event type to * executor. * <p> * Event listeners can be installed and will be called pre- and post- process if * this EventHandler is run in a Thread (its a Runnable so if its {@link #run()} * method gets called). Implement {@link EventHandlerListener}s, and registering * using {@link #setListener(EventHandlerListener)}. * * @see ExecutorService */ public abstract class EventHandler implements Runnable, Comparable<Runnable> { private static final Log LOG = LogFactory.getLog(EventHandler.class); // type of event this object represents protected EventType eventType; // sequence id generator for default FIFO ordering of events protected static AtomicLong seqids = new AtomicLong(0); // sequence id for this event private final long seqid; // Listener to call pre- and post- processing. May be null. private EventHandlerListener listener; // Time to wait for events to happen, should be kept short protected final int waitingTimeForEvents; /** * This interface provides pre- and post-process hooks for events. */ public interface EventHandlerListener { /** * Called before any event is processed * * @param event * The event handler whose process method is about to be * called. */ public void beforeProcess(EventHandler event); /** * Called after any event is processed * * @param event * The event handler whose process method is about to be * called. */ public void afterProcess(EventHandler event); } static public class EventType { public String eventType; /** * Constructor */ public EventType(String value) { this.eventType = value; } } /** * Default base class constructor. */ public EventHandler(EventType eventType, int waitingTimeForEvents) { this.eventType = eventType; seqid = seqids.incrementAndGet(); this.waitingTimeForEvents = waitingTimeForEvents; } public EventHandler(EventType eventType) { this.eventType = eventType; seqid = seqids.incrementAndGet(); this.waitingTimeForEvents = 1000; } public void run() { try { if (getListener() != null) getListener().beforeProcess(this); process(); if (getListener() != null) getListener().afterProcess(this); } catch (Throwable t) { LOG.error("Caught throwable while processing event " + eventType, t); } } /** * This method is the main processing loop to be implemented by the various * subclasses. * * @throws IOException */ public abstract void process() throws IOException; /** * Return the event type * * @return The event type. */ public EventType getEventType() { return this.eventType; } /** * Get the priority level for this handler instance. This uses natural * ordering so lower numbers are higher priority. * <p> * Lowest priority is Integer.MAX_VALUE. Highest priority is 0. * <p> * Subclasses should override this method to allow prioritizing handlers. * <p> * Handlers with the same priority are handled in FIFO order. * <p> * * @return Integer.MAX_VALUE by default, override to set higher priorities */ public int getPriority() { return Integer.MAX_VALUE; } /** * @return This events' sequence id. */ public long getSeqid() { return this.seqid; } /** * Default prioritized runnable comparator which implements a FIFO ordering. * <p> * Subclasses should not override this. Instead, if they want to implement * priority beyond FIFO, they should override {@link #getPriority()}. */ @Override public int compareTo(Runnable o) { EventHandler eh = (EventHandler) o; if (getPriority() != eh.getPriority()) { return (getPriority() < eh.getPriority()) ? -1 : 1; } return (this.seqid < eh.seqid) ? -1 : 1; } /** * @return Current listener or null if none set. */ public synchronized EventHandlerListener getListener() { return listener; } /** * @param listener * Listener to call pre- and post- {@link #process()}. */ public synchronized void setListener(EventHandlerListener listener) { this.listener = listener; } @Override public String toString() { return "Event #" + getSeqid() + " of type " + eventType + " (" + getInformativeName() + ")"; } /** * Event implementations should override thie class to provide an * informative name about what event they are handling. For example, * event-specific information such as which region or server is being * processed should be included if possible. */ public String getInformativeName() { return this.getClass().toString(); } } package ExecutorServiceStudy; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; public abstract class ThreadMonitoring { private static final ThreadMXBean threadBean = ManagementFactory .getThreadMXBean(); private static final int STACK_DEPTH = 20; public static ThreadInfo getThreadInfo(Thread t) { long tid = t.getId(); return threadBean.getThreadInfo(tid, STACK_DEPTH); } /** * Format the given ThreadInfo object as a String. * * @param indent * a prefix for each line, used for nested indentation */ public static String formatThreadInfo(ThreadInfo threadInfo, String indent) { StringBuilder sb = new StringBuilder(); appendThreadInfo(sb, threadInfo, indent); return sb.toString(); } /** * Print all of the thread's information and stack traces. * * @param sb * @param info * @param indent */ public static void appendThreadInfo(StringBuilder sb, ThreadInfo info, String indent) { boolean contention = threadBean.isThreadContentionMonitoringEnabled(); if (info == null) { sb.append(indent).append( "Inactive (perhaps exited while monitoring was done)\n"); return; } String taskName = getTaskName(info.getThreadId(), info.getThreadName()); sb.append(indent).append("Thread ").append(taskName).append(":\n"); Thread.State state = info.getThreadState(); sb.append(indent).append(" State: ").append(state).append("\n"); sb.append(indent).append(" Blocked count: ") .append(info.getBlockedCount()).append("\n"); sb.append(indent).append(" Waited count: ") .append(info.getWaitedCount()).append("\n"); if (contention) { sb.append(indent) .append(" Blocked time: " + info.getBlockedTime()) .append("\n"); sb.append(indent).append(" Waited time: " + info.getWaitedTime()) .append("\n"); } if (state == Thread.State.WAITING) { sb.append(indent).append(" Waiting on ") .append(info.getLockName()).append("\n"); } else if (state == Thread.State.BLOCKED) { sb.append(indent).append(" Blocked on ") .append(info.getLockName()).append("\n"); sb.append(indent) .append(" Blocked by ") .append(getTaskName(info.getLockOwnerId(), info.getLockOwnerName())).append("\n"); } sb.append(indent).append(" Stack:").append("\n"); for (StackTraceElement frame : info.getStackTrace()) { sb.append(indent).append(" ").append(frame.toString()) .append("\n"); } } private static String getTaskName(long id, String name) { if (name == null) { return Long.toString(id); } return id + " (" + name + ")"; } }
Test case:
package ExecutorServiceStudy.test; import ExecutorServiceStudy.EventHandler.EventType; public class AppEventTypes { public static EventType eventtype1_1 = new EventType("e-type1_1"); public static EventType eventtype1_2 = new EventType("e-type1_2"); public static EventType eventtype2_1 = new EventType("e-type2_1"); public static EventType eventtype2_2 = new EventType("e-type2_2"); } package ExecutorServiceStudy.test; import ExecutorServiceStudy.ExecutorService.ExecutorType; public class AppExecutorTypes { public static ExecutorType executortype1 = new ExecutorType("executor1"); public static ExecutorType executortype2 = new ExecutorType("executor2"); } package ExecutorServiceStudy.test; import java.io.IOException; import ExecutorServiceStudy.EventHandler; public class ConcretEventHandler1_1 extends EventHandler{ public ConcretEventHandler1_1(EventType eventType) { super(eventType); } @Override public void process() throws IOException { while(true){ //System.out.println("this is ConcretEventHandler 1.1"); // try { // Thread.sleep(2000); // } catch (InterruptedException e) { // e.printStackTrace(); // } } } } package ExecutorServiceStudy.test; import java.util.Map; import ExecutorServiceStudy.EventHandler; import ExecutorServiceStudy.ExecutorService; import ExecutorServiceStudy.ExecutorService.ExecutorStatus; public class ExecutorServiceTest { static ExecutorService service = new ExecutorService("service test"); static EventHandler.EventHandlerListener sleepListener = new EventHandler.EventHandlerListener() { public void beforeProcess(EventHandler event) { // try { // Thread.sleep(1000); // System.err.println(Thread.currentThread().getId() + "-pre"); // } catch (InterruptedException e) { // e.printStackTrace(); // } System.err.println(Thread.currentThread().getId() + "-pre"); } public void afterProcess(EventHandler event) { // try { // Thread.sleep(500); // System.err.println(Thread.currentThread().getId() + "-post"); // } catch (InterruptedException e) { // e.printStackTrace(); // } System.err.println(Thread.currentThread().getId() + "-post"); } }; public static void main(String args[]) { EventHandler event11 = new ConcretEventHandler1_1( AppEventTypes.eventtype1_1); event11.setListener(sleepListener); EventHandler event12 = new ConcretEventHandler1_2( AppEventTypes.eventtype1_2); event12.setListener(sleepListener); EventHandler event21 = new ConcretEventHandler2_1( AppEventTypes.eventtype2_1); event21.setListener(sleepListener); EventHandler event22 = new ConcretEventHandler2_2( AppEventTypes.eventtype2_2); event22.setListener(sleepListener); service.registerEventToExecutor(AppEventTypes.eventtype1_1, AppExecutorTypes.executortype1); service.registerEventToExecutor(AppEventTypes.eventtype1_2, AppExecutorTypes.executortype1); service.registerEventToExecutor(AppEventTypes.eventtype2_1, AppExecutorTypes.executortype2); service.registerEventToExecutor(AppEventTypes.eventtype2_2, AppExecutorTypes.executortype2); System.out.println(service.getInnerExecutors()); // open the pools service.startExecutorService(AppExecutorTypes.executortype1, 1); service.startExecutorService(AppExecutorTypes.executortype2, 1); // submit events to service (each pool) service.submit(event11); service.submit(event12); service.submit(event21); service.submit(event22); while (true) { Map<String, ExecutorStatus> status = service .getAllExecutorStatuses(); for (String k : status.keySet()) { System.err.println(k + " : " + status.get(k).toString()); } try { Thread.sleep(1000); } catch (InterruptedException e) { service.shutdown(); e.printStackTrace(); } } } }