Java 多线程读取文件并统计词频 实例 出神入化的《ThreadPoolExecutor》

重在展示多线程ThreadPoolExecutor的使用，和线程同步器CountDownLatch，以及相关CAS的原子操作和线程安全的Map/队列。

ThreadPool主线程

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReferenceArray;

/**
 * ClassName: ThreadPool
 * Description:
 * date: 2021/1/16 18:24
 *
 * @author hewei
 */
public class ThreadPool {
    /**存储词的队列**/
    private static ArrayBlockingQueue<String> oneWordQueue = new ArrayBlockingQueue<String>(Integer.MAX_VALUE / 4);
    /**存储词的次数**/
    private static ConcurrentHashMap<String, AtomicInteger> wordCount = new ConcurrentHashMap<String, AtomicInteger>();
    /**判断中英文的正则**/
    private static String[] englishOrChinaWordCompile = {"[a-zA-Z]+", "[\u4e00-\u9fa5]"};
    private static long waitTimeSecond = 5;
    /**已知会创建10个线程池**/
    private static CountDownLatch countDownLatch = new CountDownLatch(10);
    /**存储词的次数排序**/
    private static AtomicReferenceArray wordCountArray=new AtomicReferenceArray<String>(Integer.MAX_VALUE/16);

    public static void main(String[] args) {
        BlockingQueue<Runnable> blockingQueue = new LinkedBlockingDeque<>(10000);
        ThreadPoolExecutor readFilePool = new ThreadPoolExecutor(5, 5,
                waitTimeSecond, TimeUnit.SECONDS, blockingQueue, new UserThreadFactory("Study-WordCountReadFile-"),
                new UserRejectHandler());
        //executor.allowCoreThreadTimeOut(true);
        Long beginTime = System.currentTimeMillis();
        //读取D盘的文件
        File file = new File("E:\tmp");
        if (file.isDirectory()) {
            File[] files = file.listFiles();
            // 定义文件类型
            for (File one : files) {
                String fileName = one.getName();
                readFilePool.execute(new WordCountReadFileTask(countDownLatch, "E:\tmp\" + fileName, oneWordQueue, englishOrChinaWordCompile));
            }
        }
        ThreadPoolExecutor sortWordPool = new ThreadPoolExecutor(5, 5,
                waitTimeSecond, TimeUnit.SECONDS, blockingQueue, new UserThreadFactory("Study-WordCount-"),
                new UserRejectHandler());
        //executor1.allowCoreThreadTimeOut(true);
        for (int i = 0; i < 5; i++) {
            sortWordPool.execute(new WordCountTask(countDownLatch, wordCount, oneWordQueue, waitTimeSecond,null));
        }
        try {
            countDownLatch.await();
            readFilePool.shutdown();
            sortWordPool.shutdown();
            // 写出到文件
            List<Map.Entry<String, AtomicInteger>> list = new ArrayList(wordCount.entrySet());
            Comparator com = new Comparator<Map.Entry<String, AtomicInteger>>(){
                @Override
                public int compare(Map.Entry<String, AtomicInteger> o1, Map.Entry<String, AtomicInteger> o2) {
                    return ((Integer)o2.getValue().get()).compareTo((Integer) o1.getValue().get());
                }
            };
            list.sort(com);
            // 写出到文件
            BufferedWriter bw = new BufferedWriter(new FileWriter("E:\read.txt"));
            for(int i=0;i<list.size();i++){
                if(i<10) {
                    System.out.println("单词  " + list.get(i).getKey() + ",次数  " + list.get(i).getValue());
                }
                bw.write("单词  "+ list.get(i).getKey()+",次数  "+ list.get(i).getValue());
                bw.newLine();
            }
            bw.flush();
            bw.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

UserThreadFactory

import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * ClassName: UserThreadFactory
 * Description:自定义线程创建工厂
 * date: 2021/1/16 18:26
 *
 * @author hewei
 */
public class UserThreadFactory implements ThreadFactory {
    /**
     * 自定义线程名称前缀
     **/
    private final String prefixName;
    /**
     * 线程计数器 从1开始
     */
    private final AtomicInteger threadNumber = new AtomicInteger(1);

    public UserThreadFactory(String prefixName) {
        this.prefixName = prefixName;
    }

    @Override
    public Thread newThread(Runnable runnable) {
        //创建线程
        String name = prefixName + threadNumber.getAndIncrement();
        return new WorkThread(runnable,name);
    }
    /**
     *自定义工作线程，定义线程名称有助于对jvm问题排查
     */
    class WorkThread extends Thread {
        /**
         * 线程名称
         */
        private String name;

        /**
         * @param target  执行的方法
         * @param name 线程的名称
         */
        public WorkThread(Runnable target, String name) {
            super(target);
            super.setName(name);
            this.name=name;
            System.out.println("创建："+name);
        }

        @Override
        public void run() {
            try {
                /**
                 * super.run()等同于target.run()
                 */
                super.run();
            } finally {
                System.out.println("结束线程:" + name);
            }
        }
    }
}

WordCountReadFileTask

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * ClassName: WordCountTask
 * Description:
 * date: 2021/1/17 19:48
 *
 * @author hewei
 */
public class WordCountReadFileTask implements Runnable {
    private String filePathAndName;
    private ArrayBlockingQueue<String> oneWordQueue;
    private  String[] englishOrChinaWordCompile;
    private  CountDownLatch countDownLatch;

    public WordCountReadFileTask(CountDownLatch countDownLatch,String filePathAndName, ArrayBlockingQueue<String> oneWordQueue, String[] englishOrChinaWordCompile) {
        this.countDownLatch=countDownLatch;
        this.filePathAndName = filePathAndName;
        this.oneWordQueue = oneWordQueue;
        this.englishOrChinaWordCompile = englishOrChinaWordCompile;
    }

    @Override
    public void run() {
        try {
            BufferedReader br = new BufferedReader(new FileReader(filePathAndName));
            StringBuffer sb = new StringBuffer();
            List<String> strList=new ArrayList<String>();
            String line = "";
            while((line=br.readLine())!=null){
                sb.append(line);
                /**
                 * 为了保证不超过Integer.max_value
                 */
                if(sb.length()>50000000) {
                    strList.add(sb.toString());
                    /**
                     * 清空StringBuffer
                     * 1.delete,从到到尾
                     * 2.new 新的对象。但会丢弃老对象加速gc到来
                     * 3.setlength=0,不符合这里的场景
                     */
                    sb.delete(0,sb.length());
                }
            }
            if(sb!=null){
                strList.add(sb.toString());
            }
            br.close();
            for(String words:strList) {
                for (String oneCompile : englishOrChinaWordCompile) {
                    //正则
                    Pattern p = Pattern.compile(oneCompile);
                    Matcher matcher = p.matcher(words);
                    while (matcher.find()) {
                        /**
                         * 添加一个元素，如果队列满，则阻塞等待队列被消费腾出空间来
                         */
                        oneWordQueue.put(matcher.group());
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            countDownLatch.countDown();
        }
    }
}

WordCountTask

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReferenceArray;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * ClassName: WordCountTask
 * Description:
 * date: 2021/1/17 19:48
 *
 * @author hewei
 */
public class WordCountTask implements Runnable {
    private ArrayBlockingQueue<String> oneWordQueue;
    private ConcurrentHashMap<String, AtomicInteger> wordCount;
    private long waitTimeSecond;
    private  CountDownLatch countDownLatch;
    private static AtomicReferenceArray wordCountArray;

    public WordCountTask(CountDownLatch countDownLatch,ConcurrentHashMap<String,AtomicInteger> wordCount,
                         ArrayBlockingQueue<String> oneWordQueue,long waitTimeSecond,AtomicReferenceArray wordCountArray) {
        this.wordCountArray=wordCountArray;
        this.countDownLatch=countDownLatch;
        this.oneWordQueue = oneWordQueue;
        this.wordCount=wordCount;
        this.waitTimeSecond=waitTimeSecond;
    }

    @Override
    public void run() {
        try {
            String oneWord;
            AtomicInteger nowCount;
            while ((oneWord=oneWordQueue.poll(waitTimeSecond, TimeUnit.SECONDS))!=null) {
                /**
                 * 循环从队列里取出元素，然后加入到map中
                 * 在加入map过程中，代码块会有指令重排问题。所以每一步都需要加判断。
                 * 所以需要每一个操作都要具有原子性。
                 */
                if((nowCount=wordCount.get(oneWord))==null){
                    nowCount=new AtomicInteger(1);
                    AtomicInteger ifExistCount=wordCount.putIfAbsent(oneWord,nowCount);
                    if(ifExistCount!=null) {
                        ifExistCount.getAndIncrement();
                    }
                }else{
                    nowCount.getAndIncrement();
                }
                /**
                 * 实时排序，该排序依赖线程安全
                 * 略
                 */
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            countDownLatch.countDown();
        }
    }
}