MapReduce的WordCount

package com.mengyao.hadoop.mapreduce;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
 * 输入文件目录为HDFS上的/mapreduces/word.txt，内容如下：
 *         hadoop    zookeeper    hbase    hive
 *         flume    sqoop    pig    mahout
 *         hadoop    spark    mllib    hive    zookeeper
 *         hadoop    storm    kafka    redis    zookeeper
 *
 * 输出目录为HDFS上的/mapreduces/wordcount/
 *     _SUCCESS空文件表示作业执行成功（如果是_FAILD文件则失败）
 *  part-r-00000文件表示作业的结果，内容如下：
 *      flume    1
 *      hadoop    3
 *      hbase    1
 *      hive    2
 *      kafka    1
 *      mahout    1
 *      mllib    1
 *      pig    1
 *      redis    1
 *      spark    1
 *      sqoop    1
 *      storm    1
 *      zookeeper    3
 *  
 * @author mengyao
 *
 */
public class WordCount extends Configured implements Tool {

    static class WordCountMapper extends Mapper<LongWritable, Text, Text, LongWritable> {
        
        private Text outputKey;
        private LongWritable outputValue;
        
        @Override
        protected void setup(Context context)
                throws IOException, InterruptedException {
            this.outputKey = new Text();
            this.outputValue = new LongWritable(1L);
        }
        
        @Override
        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            final String[] words = value.toString().split("\t");
            for (String word : words) {
                this.outputKey.set(word);
                context.write(this.outputKey, this.outputValue);            
            }
        }
    }
    
    static class WordCountReducer extends Reducer<Text, LongWritable, Text, LongWritable> {
        
        private Text outputKey;
        private LongWritable outputValue;
        
        @Override
        protected void setup(Context context)
                throws IOException, InterruptedException {
            this.outputKey = new Text();
            this.outputValue = new LongWritable();
        }
        
        @Override
        protected void reduce(Text key, Iterable<LongWritable> value, Context context)
                throws IOException, InterruptedException {
            long count = 0L;
            for (LongWritable item : value) {
                count += item.get();
            }
            this.outputKey.set(key);
            this.outputValue.set(count);
            context.write(this.outputKey, this.outputValue);
        }
    }
    
    @Override
    public int run(String[] args) throws Exception {
        Job job = Job.getInstance(getConf(), WordCount.class.getSimpleName());
        job.setJarByClass(WordCount.class);
        
        job.setInputFormatClass(TextInputFormat.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        
        job.setMapperClass(WordCountMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);
        
        job.setCombinerClass(WordCountReducer.class);
        
        job.setReducerClass(WordCountReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);
        
        return job.waitForCompletion(true)?0:1;
    }

    public static int createJob(String[] args) {
        Configuration conf = new Configuration();
        conf.set("dfs.datanode.socket.write.timeout", "7200000");
        conf.set("mapreduce.input.fileinputformat.split.minsize", "268435456");
        conf.set("mapreduce.input.fileinputformat.split.maxsize", "536870912");
        int status = 0;
        
        try {
            status = ToolRunner.run(conf, new WordCount(), args);
        } catch (Exception e) {
            e.printStackTrace();
        }
        
        return status;
    }
    
    public static void main(String[] args) {
        args = new String[]{"/mapreduces/word.txt", "/mapreduces/wordcount"};
        if (args.length!=2) {
            System.out.println("Usage: "+WordCount.class.getName()+" Input paramters <INPUT_PATH> <OUTPUT_PATH>");
        } else {
            int status = createJob(args);
            System.exit(status);
        }
    }

}