Mapreduce 订单分组案例

程序执行流程如下:

map()-->getPartition()分区--->write()(序列化,每一行都顺序执行这三个方法)--->readFields()---->compareTo()排序---->readFields()--->分组compare--->reduce()

对于每一行的内容,依次执行map()-->getPartition()---->write()(序列化,会在指定的输出目录下生成temporary目录),全部序列化完成之后执行readFields(),之后进行排序,全部排序完毕后才会再次执行readFields(),

之后进行分组,分出一组,执行一次reduce()写出数据

1.OrderBean(分了区之后就不需要对OrderId进行排序了,因为同一分区OrderId相同)

package mapreduce.order;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.WritableComparable;

public class OrderBean implements WritableComparable<OrderBean>{
    private String orderId;
    private Double price;
    
    public String getOrderId() {
        return orderId;
    }

    public void setOrderId(String orderId) {
        this.orderId = orderId;
    }

    public Double getPrice() {
        return price;
    }

    public void setPrice(Double price) {
        this.price = price;
    }

    public OrderBean() {
        super();
    }

    /**
     * 序列化
     * @param out
     * @throws IOException
     */
    @Override
    public void write(DataOutput out) throws IOException {
        out.writeUTF(orderId);
        out.writeDouble(price);
    }

    /**
     * 反序列化
     * @param in
     * @throws IOException
     */
    @Override
    public void readFields(DataInput in) throws IOException {
        orderId = in.readUTF();
        price   = in.readDouble();
    }

    
    @Override
    public String toString() {
        return orderId + "	" + price;
    }

    /**
     * 二次排序(如果进行了分区,可以直接对price排序,不再需要二次排序)
     * @param o
     * @return
     */
    @Override
    public int compareTo(OrderBean o) {
        //先比较orderId(字符串)
    //    int result = this.getOrderId().compareTo(o.getOrderId());
        
        //相同再比较price
    //    if(result == 0) {
            //降序,数字的compareTo() -1表示小于
        int    result = this.getPrice() > o.getPrice()?-1:1;
    //    }
        return result;
    }
}

2.OrderMapper

package mapreduce.order;

import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class OrderMapper extends Mapper<LongWritable, Text, OrderBean, NullWritable> {
    OrderBean order = new OrderBean();

    @Override
    protected void map(LongWritable key, Text value,
            Mapper<LongWritable, Text, OrderBean, NullWritable>.Context context)
            throws IOException, InterruptedException {

        // 1.读取一行
        String line = value.toString();

        // 2.切割并封装对象
        String[] split = line.split("	");
        order.setOrderId(split[0]);
        order.setPrice(Double.parseDouble(split[2]));

        // 3.输出
        context.write(order, NullWritable.get());
    }
}

3.OrderPartioner

package mapreduce.order;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Partitioner;

/**
 * 分区
 * @author tele
 *
 */
public class OrderPartioner extends Partitioner<OrderBean, NullWritable> {
    /**
     * 默认是根据orderBean的hashcode进行分区
     * 现在更改为orderBean对象的orderId进行分区,这样相同orderId的即可进入同一分区
     */
    @Override
    public int getPartition(OrderBean key, NullWritable value, int numReduceTasks) {
        return (key.getOrderId().hashCode() & Integer.MAX_VALUE) % numReduceTasks;
    }
}

4.OrderGroupComparator

package mapreduce.order;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

/**
 * reduce分组的方法
 * 
 * @author tele
 *
 */
public class OrderGroupComparator extends WritableComparator {
    /**
     * 确保类型转换成功,能够创建实例,如果是false,创建的实例为null
     */
    protected OrderGroupComparator() {
        super(OrderBean.class, true);
    }

    /**
     * reduce默认是根据key进行分组,即此处的orderBean,重写之后
     * 根据orderBean的orderId进行分组,相同的orderId的orderBean将会分到同一组
     */
    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        OrderBean bean1 = (OrderBean) a;
        OrderBean bean2 = (OrderBean) b;
        return bean1.getOrderId().compareTo(bean2.getOrderId());
    }
}

5.OrderReducer(reduce()每次读入一组)

package mapreduce.order;

import java.io.IOException;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;

public class OrderReducer extends Reducer<OrderBean, NullWritable, OrderBean, NullWritable> {
    @Override
    protected void reduce(OrderBean key, Iterable<NullWritable> value,
            Reducer<OrderBean, NullWritable, OrderBean, NullWritable>.Context context)
            throws IOException, InterruptedException {
        context.write(key, NullWritable.get());
    }
}

6.OrderDriver(必须设置分组类)

package mapreduce.order;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * 
 * @author tele
 *
 */
public class OrderDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        // 1.获得job实例
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        // 2.设置driverclass
        job.setJarByClass(OrderDriver.class);
        // mapclass
        job.setMapperClass(OrderMapper.class);
        // reduceclass
        job.setReducerClass(OrderReducer.class);

        // 3.map输入输出数据类型
        job.setMapOutputKeyClass(OrderBean.class);
        job.setMapOutputValueClass(NullWritable.class);

        // 4.最终输出数据类型
        job.setOutputKeyClass(OrderBean.class);
        job.setOutputValueClass(NullWritable.class);

        // 7.设置分区类
        job.setPartitionerClass(OrderPartioner.class);
        job.setNumReduceTasks(3);

        // 8.设置分组类
        job.setGroupingComparatorClass(OrderGroupComparator.class);

        // 5.输入与输出路径
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        // 6.提交与退出
        boolean result = job.waitForCompletion(true);
        System.exit(result ? 0 : 1);

    }
}