数据挖掘之聚类算法Ａpriori总结

项目中有时候需要用到对数据进行关联分析，比如分析一个小商店中顾客购买习惯.

package com.data.algorithm;

import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;

/**
 * *********************************************************
 * <p/>
 * Author:     XiJun.Gong
 * Date:       2017-01-20 15:06
 * Version:    default 1.0.0
 * Class description：
 * <p/>
 * *********************************************************
 */

class EOC {

    private static final Logger logger = LoggerFactory.getLogger(EOC.class);
    private Map<String, Integer> fmap;  //forward map
    private Map<Integer, String> bmap;  //backward map
    private List<Map<String, Integer>> elements = null;

    private Integer maxDimension;

    public EOC(final String pathFile, String separatSeq) {

        BufferedReader bufferedReader = null;
        try {
            this.fmap = Maps.newHashMap();
            this.bmap = Maps.newHashMap();
            this.elements = Lists.newArrayList();
            maxDimension = 0;
            bufferedReader = new BufferedReader(
                    new InputStreamReader(
                            new FileInputStream(pathFile), "UTF-8"));
            String _line = null;
            Integer keyValue = null, mapIndex = 0;
            while ((_line = bufferedReader.readLine()) != null) {
                Map<String, Integer> lineMap = Maps.newHashMap();
                if (_line.trim().length() > 1) {
                    if (separatSeq.trim().length() < 1) {
                        separatSeq = ",";
                    }
                    for (String word : Splitter.on(separatSeq).split(_line)) {
                        word = word.trim();
                        if (null == (keyValue = fmap.get(word))) {
                            keyValue = mapIndex++;
                        }
                        fmap.put(word, keyValue);
                        bmap.put(keyValue, word);
                        lineMap.put(word, keyValue);
                    }
                    if (maxDimension < lineMap.size())
                        maxDimension = lineMap.size();
                    elements.add(lineMap);
                }
            }
        } catch (Exception e) {
            logger.error("读取文件出错 , 错误原因:{}", e);
        } finally {
            if (bufferedReader != null) {
                try {
                    bufferedReader.close();
                } catch (IOException e) {
                    logger.error("bufferedReader , 错误原因:{}", e);
                }
            }
        }
    }

    public Integer getMaxDimension() {
        return maxDimension;
    }

    public float getRateOfSet(Collection<Integer> elementChild) {
        float rateCnt = 0f;
        int allSize = 1;
        for (Map<String, Integer> eMap : elements) {
            boolean flag = true;
            for (Integer element : elementChild) {
                if (null == eMap.get(bmap.get(element))) {
                    flag = false;
                    break;
                }
            }
            if (flag) rateCnt += 1;
        }
        return rateCnt / ((allSize = elements.size()) > 1 ? (float) allSize : 1.0f);
    }

    public Set<Integer> getElements() {

        return new HashSet<Integer>(fmap.values());
    }

    public Integer queryByKey(String key) {
        return fmap.get(key);
    }

    public String queryByValue(Integer value) {
        return bmap.get(value);
    }
}

public class Apriori {
    private static final Logger logger = LoggerFactory.getLogger(Apriori.class);
    private EOC eoc = null;
    private Integer maxDimension;
    private final float exp = 1e-4f;

    public Apriori(final String pathFile, String separatSeq, Integer maxDimension) {
        this(pathFile, separatSeq);
        this.maxDimension = maxDimension;
    }

    public Apriori(final String pathFile, String separatSeq) {
        this.eoc = new EOC(pathFile, separatSeq);
        this.maxDimension = this.eoc.getMaxDimension();
    }

    public void work(float confidenceLevel) {
        List<Set<Integer>> listElement = null;
        ArrayList<Set<Integer>> middleWareElement = null;
        Map<Set<Integer>, Float> maps = null;
        listElement = Lists.newArrayList();
        for (Integer element : this.eoc.getElements()) {
            Set<Integer> set = new HashSet<Integer>();
            set.add(element);
            listElement.add(set);
        }
        maps = Maps.newHashMap();
        middleWareElement = Lists.newArrayList();
        for (int i = 1; i < this.maxDimension; i++) {
            for (Set<Integer> tmpSet : listElement) {
                float rate = eoc.getRateOfSet(tmpSet);
                if (confidenceLevel - exp <= rate)
                    maps.put(tmpSet, rate);
            }
            System.out.println("+++++++++++第 " + i + " 维度关联数据+++++++++++");
            output(maps);
            listElement.clear();
            middleWareElement.addAll(maps.keySet());
            maps.clear();
            for (int j = 0; j < middleWareElement.size(); j++) {
                Set<Integer> tmpSet = middleWareElement.get(j);
                for (int k = j + 1; k < middleWareElement.size(); k++) {
                    Set<Integer> setChild = middleWareElement.get(k);
                    for (Integer label : setChild) {
                        if (!tmpSet.contains(label)) {
                            Set<Integer> newElement = new HashSet<Integer>(tmpSet);
                            newElement.add(label);
                            if (!listElement.contains(newElement)) {
                                listElement.add(newElement);
                                break;
                            }
                        }
                    }
                }
            }
            middleWareElement.clear();
        }
    }

    public void output(Map<Set<Integer>, Float> maps) {
        for (Map.Entry<Set<Integer>, Float> iter : maps.entrySet()) {
            for (Integer integer : iter.getKey()) {
                System.out.print(eoc.queryByValue(integer) + " ");
            }
            System.out.println(iter.getValue()*100+"%");
        }
    }
}

　　

package com.data.algorithm;


/**
 * *********************************************************
 * <p/>
 * Author:     XiJun.Gong
 * Date:       2017-01-17 17:57
 * Version:    default 1.0.0
 * Class description：
 * <p/>
 * *********************************************************
 */
public class Main {
    public static void main(String args[]) {
        Apriori apriori = new Apriori("/home/com/src/main/java/com/qunar/data/algorithm/demo.data", ",");
        apriori.work(0.5f);
    }
}

+++++++++++第 1 维度关联数据+++++++++++
苹果 50.0%
西红柿 75.0%
香蕉 75.0%
矿泉水 75.0%
+++++++++++第 2 维度关联数据+++++++++++
苹果 西红柿 50.0%
西红柿 香蕉 50.0%
西红柿 矿泉水 50.0%
香蕉 矿泉水 75.0%
+++++++++++第 3 维度关联数据+++++++++++
西红柿 香蕉 矿泉水 50.0%