中文分词系列(一)

关于中文分词的一些资料网上资料很多，大家可以自己去了解了解，今天这里只关注代码怎么写。

中文分词主要可以归纳为“规则分词”、“统计分词”以及“规则+统计”三个主要派别，今天主要了解“规则分词”中常见的正向、逆向和双向最大化匹配，这三个都是基于现在词典做的，所以得准备一个中文词典，一行一个词。

一.正向最大化匹配

　　描述：

　　　　1.找到词典中最长的词，记下长度L

　　　　2.从 “左向右” 取长度为L的字符串，查找词典进行匹配，若匹配成功，则将这个词切分出来。若匹配失败，将这个字符串的最后一个字符去掉，将剩下的串作为新的匹配串　　　　进行匹配。如此重复下去，直到切完。

二.逆向最大化匹配

　　描述：

　　　　1.找到词典中最长的词，记下长度L

　　　　2.从 ”右向左“ 取长度为L的字符串，查找词典进行匹配，若匹配成功，则将这个词切分出来。若匹配失败，将这个字符串的最前面一个字符去掉，将剩下的串作为新的匹配　　　　串进行匹配。如此重复下去，直到切完。

三.双向最大化匹配

　　描述：

　　　　1.将正向和逆向进行比较，先取词数切分最少的作为结果。

四.代码采用python

　　　　1.load 词典

　　

#!/usr/bin/env python
# -*- coding:utf-8 -*-

#这是词典路径
dictPath = '../resource/dict.txt'

def loadDict():
    print('load dict...')
    dictionary = dict()
    maximum = 0
    # read resource
    with open(dictPath, 'r', encoding='utf8') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            str = line.split(' ')
            dictionary[str[0]] = str[2]
            wordLength = len(line)
            if wordLength > maximum:
                maximum = wordLength #词典中最长的词的长度
    return dictionary, maximum

　　　　2.核心方法

#!/usr/bin/env python
# -*- coding:utf-8 -*-

from word_segmentation.regulation.ReverseMaximumMatchMethod import RMM
from word_segmentation.regulation.MaximumMatchMethod import MM
from word_segmentation.regulation.BiDirectctionMatchMethod import BDMM
from word_segmentation.util.LoadDict import loadDict

class RegulationMatch(object):
    def __init__(self):
        self.dictionary, self.maximum = loadDict()

    def cut(self, text, method):
        #逆向
        if method == 'RMM':
            return RMM.cut(text, self.dictionary, self.maximum)
        #正向
        if method == 'MM':
            return MM.cut(text, self.dictionary, self.maximum)
        #双向
        if method == 'BDMM':
            return BDMM.cut(text, self.dictionary, self.maximum)

#!/usr/bin/env python
# -*- coding:utf-8 -*-

'''
词和词性
'''
class Word(object) :
    def __init__(self, token, property):
        self.__token = token
        self.__property = property
    #单词
    def getToken(self):
        return self.__token
    #词性
    def getProperty(self):
        return self.__property

#!/usr/bin/env python
# -*- coding:utf-8 -*-

from word_segmentation.regulation.Word import Word

'''
正向最大化匹配
MaximumMatchMethod
'''
class MM(object):
    def __init__(self):
        pass

    @staticmethod
    def cut(text, dictionary, maximum):
        result = []
        textLength = len(text)
        start = 0
        while textLength > 0:
            word = None
            for size in range(maximum, 0, -1):
                if textLength - size < 0:
                    continue
                piece = text[start:(start + size)]
                if dictionary.__contains__(piece):
                    word = piece
                    result.append(Word(piece, dictionary.get(piece)))
                    textLength -= size
                    start += size
                    break
            if word is None:
                textLength -= 1
        return result

#!/usr/bin/env python
# -*- coding:utf-8 -*-

from word_segmentation.regulation.Word import Word

'''
逆向最大化匹配
ReverseMaximumMatchMethod
'''
class RMM(object):
    def __init__(self):
        pass

    @staticmethod
    def cut(text, dictionary, maximum):
        result = []
        textLength = len(text)
        while textLength > 0:
            word = None
            for size in range(maximum, 0, -1):
                if textLength - size < 0:
                    continue
                piece = text[(textLength - size) : textLength]
                if dictionary.__contains__(piece):
                    word = piece
                    result.append(Word(piece, dictionary.get(piece)))
                    textLength -= size
                    break
            if word is None:
                textLength -= 1
        return result[::-1]

# -*- coding:utf-8 -*-

from word_segmentation.regulation.MaximumMatchMethod import MM
from word_segmentation.regulation.ReverseMaximumMatchMethod import RMM

'''
    比较正向最大匹配和逆向最大匹配结果:
    1.如果分词数量结果不同，那么取分词数量较少的那个
    2.如果分词数量结果相同
        a.分词结果相同，可以返回任何一个
        b.分词结果不同，返回单字数比较少的那个
        c.分词结果不同，单字数相同，返回谁呢（可以返回逆向分词结果）
'''
class BDMM(object):
    def __init__(self):
        pass

    @staticmethod
    def cut(text, dictionary, maximum):
        mmResult = MM.cut(text, dictionary, maximum)
        rmmResult = RMM.cut(text, dictionary, maximum)
        mmSegment = []
        rmmSegment = []
        for word in mmResult:
            mmSegment.append(word.getToken())
            # print('token = %s, property = %s' %(word.getToken(), word.getProperty()))
        for word in rmmResult:
            rmmSegment.append(word.getToken())

        if mmSegment.__len__() < rmmSegment.__len__():
            return mmResult
        elif mmSegment.__len__() == rmmSegment.__len__():
            flag = True
            for segment in mmSegment:
                if segment not in rmmSegment:
                    flag = False
                    break
            if flag:
                return mmResult
            else:
                mmSingleWords = 0
                rmmSingleWords = 0
                for word in mmSegment:
                    if len(word) == 1:
                        mmSingleWords += 1
                for word in rmmSegment:
                    if len(word) == 1:
                        rmmSingleWords += 1
                if mmSingleWords < rmmSingleWords:
                    return mmResult
                else:
                    return rmmResult
        else:
            return rmmResult

#!/usr/bin/env python
# -*- coding:utf-8 -*-

from word_segmentation.regulation.ReverseMaximumMatchMethod import RMM
import word_segmentation.regulation.MaximumMatchMethod
import word_segmentation.regulation.BiDirectctionMatchMethod
from word_segmentation.regulation.RegulationMatchMthod import RegulationMatch

def test():
    pass
if __name__ == '__main__':
    text = '各国有各国的困难…'
    print('分词：')
    print('各国有各国的困难…')
    regulation = RegulationMatch()
    mmResult = regulation.cut(text, 'MM')
    rmmResult = regulation.cut(text, 'RMM')
    bdmmResult = regulation.cut(text, 'BDMM')
    mmSegment = []
    rmmSegment = []
    bdmmSegment = []
    for word in mmResult:
        mmSegment.append(word.getToken())
        #print('token = %s, property = %s' %(word.getToken(), word.getProperty()))
    for word in rmmResult:
        rmmSegment.append(word.getToken())
    for word in bdmmResult:
        bdmmSegment.append(word.getToken())

    print('正向匹配: %s'  % mmSegment)
    print('逆向匹配: %s'  % rmmSegment)
    print('双向匹配: %s' % bdmmSegment)