MapReduce -- 统计天气信息

示例

数据：

1949-10-01 14:21:02    34c
1949-10-02 14:01:02    36c
1950-01-01 11:21:02    32c
1950-10-01 12:21:02    37c
1951-12-01 12:21:02    23c
1951-12-02 12:21:02    45c
1951-12-03 12:21:02    50c
1951-12-23 12:21:02    33c
1950-10-02 12:21:02    41c
1950-10-03 12:21:02    27c
1951-07-01 12:21:02    45c
1951-07-02 12:21:02    46c
1951-07-03 12:21:03    47c

要求：

将每年每月中的气温排名前三的数据找出来

实现：

1.每一年用一个reduce任务处理;

2.创建自定义数据类型，存储 [年-月-日-温度];

2.自己实现排序函数 根据 [年-月-温度] 降序排列，也可以在定义数据类型中进行排序;

3.自己实现分组函数，对 [年-月] 分组，reduce的key是分组结果，根据相同的分组值，统计reduce的value值，只统计三个值就可以，因为已经实现了自排序函数。

注意点：

1.自定义数据类型的使用;

2.自定义排序类的使用;

3.自定义分组类的使用，分组类对那些数据进行分组;

4.自定义分区类，分区类与reduce job个数的关系;

示例代码：

RunJob.java  

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;

/**
 * weather 统计天气信息
 *
 * 数据：
 *  1999-10-01 14:21:02    34c
 *  1999-11-02 13:01:02    30c
 *
 * 要求：
 * 将每年的每月中气温排名前三的数据找出来
 *
 * 实现：
 * 1.每一年用一个reduce任务处理;
 * 2.创建自定义数据类型，存储 [年-月-日-温度];
 * 2.自己实现排序函数 根据 [年-月-温度] 降序排列，也可以在定义数据类型中进行排序;
 * 3.自己实现分组函数，对 [年-月] 分组，reduce的key是分组结果，根据相同的分组值，统计reduce的value值，只统计三个值就可以，因为已经实现了自排序函数。
 *
 * Created by Edward on 2016/7/11.
 */
public class RunJob {

    public static void main(String[] args)
    {
        //access hdfs's user
        System.setProperty("HADOOP_USER_NAME","root");

        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", "hdfs://node1:8020");


        try {
            FileSystem fs = FileSystem.get(conf);

            Job job = Job.getInstance(conf);
            job.setJarByClass(RunJob.class);
            job.setMapperClass(MyMapper.class);
            job.setReducerClass(MyReducer.class);

            //需要指定 map out 的 key 和 value
            job.setOutputKeyClass(InfoWritable.class);
            job.setOutputValueClass(Text.class);

            //设置分区 继承 HashPartitioner
            job.setPartitionerClass(YearPartition.class);
            //根据年份创建指定数量的reduce task
            job.setNumReduceTasks(3);

            //设置排序 继承 WritableComparator
            //job.setSortComparatorClass(SortComparator.class);

            //设置分组 继承 WritableComparator 对reduce中的key进行分组
            job.setGroupingComparatorClass(GroupComparator.class);

            FileInputFormat.addInputPath(job, new Path("/test/weather/input"));

            Path path = new Path("/test/weather/output");
            if(fs.exists(path))//如果目录存在，则删除目录
            {
                fs.delete(path,true);
            }
            FileOutputFormat.setOutputPath(job, path);

            boolean b = job.waitForCompletion(true);
            if(b)
            {
                System.out.println("OK");
            }

        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    public static class MyMapper extends Mapper<LongWritable, Text, InfoWritable, Text > {

        private static SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] str = value.toString().split("	");

            try {
                Date date = sdf.parse(str[0]);
                Calendar c = Calendar.getInstance();
                c.setTime(date);
                int year = c.get(Calendar.YEAR);
                int month = c.get(Calendar.MONTH)+1;
                int day = c.get(Calendar.DAY_OF_MONTH);

                double temperature = Double.parseDouble(str[1].substring(0,str[1].length()-1));

                InfoWritable info = new InfoWritable();
                info.setYear(year);
                info.setMonth(month);
                info.setDay(day);
                info.setTemperature(temperature);

                context.write(info, value);

            } catch (ParseException e) {
                e.printStackTrace();
            }
        }
    }

    public static class MyReducer extends Reducer<InfoWritable, Text, Text, NullWritable> {
        @Override
        protected void reduce(InfoWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            int i=0;
            for(Text t: values)
            {
                i++;
                if(i>3)
                    break;
                context.write(t, NullWritable.get());
            }
        }
    }
}

InfoWritable.java 

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/**
 * 自定义数据类型 继承 WritableComparable
 * 【年-月-日-温度】
 * Created by Edward on 2016/7/11.
 */
public class InfoWritable implements WritableComparable<InfoWritable> {

    private int year;
    private int month;
    private int day;
    private double temperature;

    public void setYear(int year) {
        this.year = year;
    }

    public void setMonth(int month) {
        this.month = month;
    }

    public void setDay(int day) {
        this.day = day;
    }

    public void setTemperature(double temperature) {
        this.temperature = temperature;
    }

    public int getYear() {
        return year;
    }

    public int getMonth() {
        return month;
    }

    public int getDay() {
        return day;
    }

    public double getTemperature() {
        return temperature;
    }

    /**
     *
     * 对象比较，对温度进行倒序排序
     */
    @Override
    public int compareTo(InfoWritable o) {

        int result = Integer.compare(this.getYear(),o.getYear());
        if(result == 0)
        {
            result = Integer.compare(this.getMonth(),o.getMonth());
            if(result == 0)
            {
                return -Double.compare(this.getTemperature(), o.getTemperature());
            }
            else
                return result;
        }
        else
            return result;

        //return this==o?0:-1;
    }

    @Override
    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeInt(this.year);
        dataOutput.writeInt(this.month);
        dataOutput.writeInt(this.day);
        dataOutput.writeDouble(this.temperature);
    }

    @Override
    public void readFields(DataInput dataInput) throws IOException {
        this.year = dataInput.readInt();
        this.month = dataInput.readInt();
        this.day = dataInput.readInt();
        this.temperature = dataInput.readDouble();
    }
}

YearPartition.java

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

/**
 *
 * 创建分区，通过key中的year来创建分区
 *
 * Created by Edward on 2016/7/11.
 */
public class YearPartition extends HashPartitioner <InfoWritable, Text>{
    @Override
    public int getPartition(InfoWritable key, Text value, int numReduceTasks) {
        return key.getYear()%numReduceTasks;
    }
}

GroupComparator.java

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

/**
 * 创建分组类，继承WritableComparator
 * 【年-月】
 * Created by Edward on 2016/7/11.
 */
public class GroupComparator extends WritableComparator {

    GroupComparator()
    {
        super(InfoWritable.class, true);
    }

    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        InfoWritable ia = (InfoWritable)a;
        InfoWritable ib = (InfoWritable)b;

        int result = Integer.compare(ia.getYear(),ib.getYear());
        if(result == 0)
        {
            return Integer.compare(ia.getMonth(),ib.getMonth());
        }
        else
            return result;
    }
}

SortComparator.java

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

/**
 * 排序类，继承WritableComparator
 * 排序规则【年-月-温度】 温度降序
 * Created by Edward on 2016/7/11.
 */
public class SortComparator extends WritableComparator {

    /**
     * 调用父类的构造函数
     */
    SortComparator()
    {
        super(InfoWritable.class, true);
    }


    /**
     * 比较两个对象的大小，使用降序排列
     * @param a
     * @param b
     * @return
     */
    @Override
    public int compare(WritableComparable a, WritableComparable b) {

        InfoWritable ia = (InfoWritable)a;
        InfoWritable ib = (InfoWritable)b;

        int result = Integer.compare(ia.getYear(),ib.getYear());
        if(result == 0)
        {
            result = Integer.compare(ia.getMonth(),ib.getMonth());
            if(result == 0)
            {
                return -Double.compare(ia.getTemperature(), ib.getTemperature());
            }
            else
                return result;
        }
        else
            return result;
    }
}