使用DFA算法对敏感词进行过滤

项目目录结构如下：

其中resources资源目录中：

stopwd.txt ：停顿词，匹配时间直接过滤。

wd.txt：敏感词库。

1、WordFilter敏感词过滤类：

package com.skyer.sensitivewdfilter;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * 思路： 创建一个FilterSet，枚举了0~65535的所有char是否是某个敏感词开头的状态
 * 
 * 判断是否是 敏感词开头 | | 是 不是 获取头节点 OK--下一个字 然后逐级遍历，DFA算法
 */
public class WordFilter {

    private static final FilterSet set = new FilterSet(); // 存储首字
    private static final Map<Integer, WordNode> nodes = new HashMap<Integer, WordNode>(1024, 1); // 存储节点
    private static final Set<Integer> stopwdSet = new HashSet<Integer>(); // 停顿词
    private static final char SIGN = '*'; // 敏感词过滤替换

    static {
        try {
            long a = System.nanoTime();
            init();
            a = System.nanoTime() - a;
            System.out.println("加载时间 : " + a + "ns");
            System.out.println("加载时间 : " + a / 1000000 + "ms");
        } catch (Exception e) {
            throw new RuntimeException("初始化过滤器失败");
        }
    }

    private static void init() {
        // 获取敏感词
        addSensitiveWord(readWordFromFile("wd.txt"));
        addStopWord(readWordFromFile("stopwd.txt"));
    }

    /**
     * 增加敏感词
     */
    private static List<String> readWordFromFile(String path) {
        List<String> words;
        BufferedReader br = null;
        try {
            br = new BufferedReader(new InputStreamReader(WordFilter.class.getClassLoader().getResourceAsStream(path)));
            words = new ArrayList<String>(1200);
            for (String buf = ""; (buf = br.readLine()) != null;) {
                if (buf == null || buf.trim().equals(""))
                    continue;
                words.add(buf);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            try {
                if (br != null)
                    br.close();
            } catch (IOException e) {
            }
        }
        return words;
    }

    /**
     * 增加停顿词
     */
    private static void addStopWord(final List<String> words) {
        if (words != null && words.size() > 0) {
            char[] chs;
            for (String curr : words) {
                chs = curr.toCharArray();
                for (char c : chs) {
                    stopwdSet.add(charConvert(c));
                }
            }
        }
    }

    /**
     * 添加DFA节点
     */
    private static void addSensitiveWord(final List<String> words) {
        if (words != null && words.size() > 0) {
            char[] chs;
            int fchar;
            int lastIndex;
            WordNode fnode; // 首字母节点
            for (String curr : words) {
                chs = curr.toCharArray();
                fchar = charConvert(chs[0]);
                if (!set.contains(fchar)) {// 没有首字定义
                    set.add(fchar);// 首字标志位 可重复add
                    fnode = new WordNode(fchar, chs.length == 1);
                    nodes.put(fchar, fnode);
                } else {
                    fnode = nodes.get(fchar);
                    if (!fnode.isLast() && chs.length == 1)
                        fnode.setLast(true);
                }
                lastIndex = chs.length - 1;
                for (int i = 1; i < chs.length; i++) {
                    fnode = fnode.addIfNoExist(charConvert(chs[i]), i == lastIndex);
                }
            }
        }
    }

    /**
     * 过滤判断 将敏感词转化为成屏蔽词
     */
    public static final String doFilter(final String src) {
        char[] chs = src.toCharArray();
        int length = chs.length;
        int currc;
        int k;
        WordNode node;
        for (int i = 0; i < length; i++) {
            currc = charConvert(chs[i]);
            if (!set.contains(currc)) {
                continue;
            }
            node = nodes.get(currc);
            if (node == null)
                continue;
            boolean couldMark = false;
            int markNum = -1;
            if (node.isLast()) {
                couldMark = true;
                markNum = 0;
            }
            k = i;
            for (; ++k < length;) {
                int temp = charConvert(chs[k]);
                if (stopwdSet.contains(temp))
                    continue;
                node = node.querySub(temp);
                if (node == null)
                    break;
                if (node.isLast()) {
                    couldMark = true;
                    markNum = k - i;
                }
            }
            if (couldMark) {
                for (k = 0; k <= markNum; k++) {
                    chs[k + i] = SIGN;
                }
                i = i + markNum;
            }
        }

        return new String(chs);
    }

    /**
     * 是否包含敏感词
     */
    public static final boolean isContains(final String src) {
        char[] chs = src.toCharArray();
        int length = chs.length;
        int currc;
        int k;
        WordNode node;
        for (int i = 0; i < length; i++) {
            currc = charConvert(chs[i]);
            if (!set.contains(currc)) {
                continue;
            }
            node = nodes.get(currc);
            if (node == null)
                continue;
            boolean couldMark = false;
            if (node.isLast()) {
                couldMark = true;
            }
            k = i;
            for (; ++k < length;) {
                int temp = charConvert(chs[k]);
                if (stopwdSet.contains(temp))
                    continue;
                node = node.querySub(temp);
                if (node == null)
                    break;
                if (node.isLast()) {
                    couldMark = true;
                }
            }
            if (couldMark) {
                return true;
            }
        }

        return false;
    }

    /**
     * 大写转化为小写 全角转化为半角
     */
    private static int charConvert(char src) {
        int r = BCConvert.qj2bj(src);
        return (r >= 'A' && r <= 'Z') ? r + 32 : r;
    }

}

其中：

      isContains ：是否包含敏感词

     doFilter：过滤敏感词

2、WordNode敏感词节点：

package com.skyer.sensitivewdfilter;

import java.util.LinkedList;
import java.util.List;

public class WordNode {

    private int value; // 节点名称

    private List<WordNode> subNodes; // 子节点

    private boolean isLast; // 默认false

    public WordNode(int value) {
        this.value = value;
    }

    public WordNode(int value, boolean isLast) {
        this.value = value;
        this.isLast = isLast;
    }

    /**
     * @return 就是传入的subNode
     */
    private WordNode addSubNode(final WordNode subNode) {
        if (subNodes == null)
            subNodes = new LinkedList<WordNode>();
        subNodes.add(subNode);
        return subNode;
    }

    /**
     * 有就直接返回该子节点， 没有就创建添加并返回该子节点
     */
    public WordNode addIfNoExist(final int value, final boolean isLast) {
        if (subNodes == null) {
            return addSubNode(new WordNode(value, isLast));
        }
        for (WordNode subNode : subNodes) {
            if (subNode.value == value) {
                if (!subNode.isLast && isLast)
                    subNode.isLast = true;
                return subNode;
            }
        }
        return addSubNode(new WordNode(value, isLast));
    }

    public WordNode querySub(final int value) {
        if (subNodes == null) {
            return null;
        }
        for (WordNode subNode : subNodes) {
            if (subNode.value == value)
                return subNode;
        }
        return null;
    }

    public boolean isLast() {
        return isLast;
    }

    public void setLast(boolean isLast) {
        this.isLast = isLast;
    }

    @Override
    public int hashCode() {
        return value;
    }

}

3、测试类：

package com.skyer.test;

import org.junit.Test;

import com.skyer.sensitivewdfilter.WordFilter;

public class TestSensitivewd {

    @Test
    public void TestFilter() {
        String s = ""; // 这里写你要过滤的句子（我这里不能写，否则会给博客园屏蔽掉）
        System.out.println("解析问题： " + s);
        System.out.println("解析字数 : " + s.length());
        String re;
        long nano = System.nanoTime();
        re = WordFilter.doFilter(s);
        nano = (System.nanoTime() - nano);
        System.out.println("解析时间 : " + nano + "ns");
        System.out.println("解析时间 : " + nano / 1000000 + "ms");
        System.out.println(re);
        System.out.println();

        nano = System.nanoTime();
        System.out.println("是否包含敏感词： " + WordFilter.isContains(s));
        nano = (System.nanoTime() - nano);
        System.out.println("解析时间 : " + nano + "ns");
        System.out.println("解析时间 : " + nano / 1000000 + "ms");
    }

}

4、测试结果：

原文参考：http://blog.csdn.net/fengshizty/article/details/52373005

DFA知识：http://www.cnblogs.com/naaoveGIS/archive/2016/10/14/5960352.html