lucene 分词器

分词器
作用：切分关键词的。
在什么地方使用到了：在建立索引和搜索时。

原文：An IndexWriter creates and maintains an index.
1，切分：
An
IndexWriter
creates
and
maintains
an
index
.
2，去除停用词
IndexWriter
creates
maintains
index
3，转为小写
indexwriter
creates
maintains
index

package cn.itcast.e_analyzer;

import java.io.StringReader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
import org.apache.lucene.analysis.cn.ChineseAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.Version;
import org.junit.Test;
import org.wltea.analyzer.lucene.IKAnalyzer;

public class TestAnalyzer {

    @Test
    public void test() throws Exception {
        String enText = "An IndexWriter creates and maintains an index.";
        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30);
        testAnalyzer(analyzer, enText);
        
        String cnText = "传智播客准备Lucene的开发环境";
        testAnalyzer(analyzer, cnText); // 单字分词
        
        testAnalyzer(new ChineseAnalyzer(), cnText); // 单字分词
        testAnalyzer(new CJKAnalyzer(Version.LUCENE_30), cnText); // 二分法分词
        testAnalyzer(new IKAnalyzer(), cnText); // 词库分词（重点）
    }

    /**
     * 使用指定的分词器对指定的文本进行分词，并打印出分出的词
     * 
     * @param analyzer
     * @param text
     * @throws Exception
     */
    private void testAnalyzer(Analyzer analyzer, String text) throws Exception {
        System.out.println("当前使用的分词器：" + analyzer.getClass().getSimpleName());
        TokenStream tokenStream = analyzer.tokenStream("content", new StringReader(text));
        tokenStream.addAttribute(TermAttribute.class);
        while (tokenStream.incrementToken()) {
            TermAttribute termAttribute = tokenStream.getAttribute(TermAttribute.class);
            //分词条件
            System.out.println(termAttribute.term());
        }
        System.out.println();
    }

}