ikanalyzer分词，计算信息熵排序分词结果

因需求，现需分词接口，故记录之。

1、需要依赖：

<!-- https://mvnrepository.com/artifact/com.janeluo/ikanalyzer -->
        <dependency>
            <groupId>com.janeluo</groupId>
            <artifactId>ikanalyzer</artifactId>
            <version>2012_u6</version>
        </dependency>

2、完整代码如下：

public JSONArray entropy(String content, Integer quantity) throws Exception {
        List<String> words = extract(DelHtmlTagUtil.delHTMLTag(content), quantity);
        JSONArray array = calculateWordEntropy(words);
        return array;
    }

/**
     * 传入String类型的文章，智能提取单词放入list
     *
     * @param content  传入分词的内容
     * @param quantity 截取关键字在几个单词以上的数量，默认为1
     * @return
     */
    private List<String> extract(String content, Integer quantity) throws IOException {
        List<String> list = Lists.newArrayList();
        StringReader reader = new StringReader(content);
        IKSegmenter ik = new IKSegmenter(reader, true);
        Lexeme lex = null;
        while ((lex = ik.next()) != null) {
            //String typeString = lex.getLexemeTypeString();  词语类型
            String word = lex.getLexemeText();
            if (word.length() > quantity) {//判断截取关键字在几个单词以上的数量
                list.add(word);
            }
        }
        return list;
    }

    private JSONArray calculateWordEntropy(List<String> words) throws Exception{

        int length = words.size();
        ArrayList<String[]> wordList = new ArrayList<String[]>();
        // 将分好的词每3个一组存到数组中
        for (int i = 0; i < length; i++) {

            String[] wordSeg = new String[3];
            if (i == 0) {
                wordSeg[0] = "null";
                wordSeg[1] = words.get(i);
                wordSeg[2] = words.get(i + 1);
            } else if (i == length - 1) {
                wordSeg[0] = words.get(i - 1);
                wordSeg[1] = words.get(i);
                wordSeg[2] = "null";
            } else {
                wordSeg[0] = words.get(i - 1);
                wordSeg[1] = words.get(i);
                wordSeg[2] = words.get(i + 1);
            }

            wordList.add(wordSeg);

        }
        // 去除重复的词
        List<String> lists = Lists.newArrayList();
        for (int l = 0; l < length; l++) {
            lists.add(words.get(l));
        }
        List<String> tempList = Lists.newArrayList();
        for (String str : lists) {
            if (!(tempList.contains(str))) {
                tempList.add(str);
            }
        }
        String[] wordClean = new String[tempList.size()];
        for (int m = 0; m < tempList.size(); m++) {
            wordClean[m] = tempList.get(m);
        }
        // 统计每个词的词频
        int[] frequent = new int[wordClean.length];
        for (int j = 0; j < wordClean.length; j++) {
            int count = 0;
            for (int k = 0; k < words.size(); k++) {
                if (wordClean[j].equals(words.get(k))) {
                    count++;
                }
            }
            frequent[j] = count;
        }
        // 将三元组中中间的那个词相同的存到一个list中，然后计算该词的信息熵
        double[] allEntropy = new double[wordClean.length];
        for (int n = 0; n < wordClean.length; n++) {
            ArrayList<String[]> wordSegList = new ArrayList<String[]>();
            int count = 1;
            for (int p = 0; p < wordList.size(); p++) {
                String[] wordSegStr = wordList.get(p);
                if (wordSegStr[1].equals(wordClean[n])) {
                    count++;
                    wordSegList.add(wordSegStr);
                }
            }
            String[] leftword = new String[wordSegList.size()];
            String[] rightword = new String[wordSegList.size()];
            // 计算左信息熵
            for (int i = 0; i < wordSegList.size(); i++) {
                String[] left = wordSegList.get(i);
                leftword[i] = left[0];
            }
            // 去除左边重复的词
            List<String> listsLeft = new ArrayList<String>();
            for (int l = 0; l < leftword.length; l++) {
                listsLeft.add(leftword[l]);
            }
            List<String> tempListLeft = new ArrayList<String>();
            for (String str : listsLeft) {
                if (!(tempListLeft.contains(str))) {
                    tempListLeft.add(str);
                }
            }
            String[] leftWordClean = new String[tempListLeft.size()];
            for (int m = 0; m < tempListLeft.size(); m++) {
                leftWordClean[m] = tempListLeft.get(m);
            }
            // 统计左边每个词的词频
            int[] leftFrequent = new int[leftWordClean.length];
            for (int j = 0; j < leftWordClean.length; j++) {
                int leftcount = 0;
                for (int k = 0; k < leftword.length; k++) {
                    if (leftWordClean[j].equals(leftword[k])) {
                        leftcount++;
                    }
                }
                leftFrequent[j] = leftcount;
            }
            // 计算左熵值
            double leftEntropy = 0;
            for (int i = 0; i < leftFrequent.length; i++) {
                double a = (double) leftFrequent[i] / count;
                double b = Math.log((double) leftFrequent[i] / count);
                leftEntropy += -a * b;
                // leftEntropy +=
                // (-(double)(leftFrequent[i]/count))*Math.log((double)(leftFrequent[i]/count));
            }
            // 计算右信息熵
            for (int i = 0; i < wordSegList.size(); i++) {
                String[] right = wordSegList.get(i);
                rightword[i] = right[2];
            }
            // 去除右边重复的词
            List<String> listsRight = new ArrayList<String>();
            for (int l = 0; l < rightword.length; l++) {
                listsRight.add(rightword[l]);
            }
            List<String> tempListRight = new ArrayList<String>();
            for (String str : listsRight) {
                if (!(tempListRight.contains(str))) {
                    tempListRight.add(str);
                }
            }
            String[] rightWordClean = new String[tempListRight.size()];
            for (int m = 0; m < tempListRight.size(); m++) {
                rightWordClean[m] = tempListRight.get(m);
            }
            // 统计右边每个词的词频
            int[] rightFrequent = new int[rightWordClean.length];
            for (int j = 0; j < rightWordClean.length; j++) {
                int rightcount = 0;
                for (int k = 0; k < rightword.length; k++) {
                    if (rightWordClean[j].equals(rightword[k])) {
                        rightcount++;
                    }
                }
                rightFrequent[j] = rightcount;
            }
            // 计算右熵值
            double rightEntropy = 0.0;
            for (int i = 0; i < rightFrequent.length; i++) {
                double a = (double) rightFrequent[i] / count;
                double b = Math.log((double) rightFrequent[i] / count);
                rightEntropy += -a * b;
                // rightEntropy +=
                // (-(double)(rightFrequent[i]/count))*Math.log((double)(rightFrequent[i]/count));
            }
            // 计算词的总信息熵
            double wordEntropy = leftEntropy + rightEntropy;
            allEntropy[n] = wordEntropy;

        }
        JSONArray list = new JSONArray();
        for (int i = 0; i < allEntropy.length; i++) {
            JSONObject obj = new JSONObject();
            obj.put("name", wordClean[i]);
            obj.put("entropy", allEntropy[i]);
            list.add(obj);
        }
        Collections.sort(list, (o1, o2) -> {
            Double d1 = ((JSONObject) o1).getDouble("entropy");
            Double d2 = ((JSONObject) o2).getDouble("entropy");
            return d2.compareTo(d1);
        });

        return list;
    }