MapReduce_partition

package MapReduce;

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.LongWritable;  
import org.apache.hadoop.io.NullWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Partitioner;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.util.GenericOptionsParser;
 
  
public class MyPartitioner {  
    private static final String INPUT_PATH = "hdfs://h201:9000/user/hadoop/input_par";
    private static final String OUTPUT_PATH = "hdfs://h201:9000/user/hadoop/output";
      
    public static class MyPartitionerMap extends Mapper<LongWritable, Text, Text, Text> {  
          
        protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)  
        throws java.io.IOException, InterruptedException {  
              
            String arr_value[] = value.toString().split("	");  
            if (arr_value.length > 3) {  
                context.write(new Text("long"), value);  
            } else if (arr_value.length < 3) {  
                context.write(new Text("short"), value);  
            } else {  
                context.write(new Text("right"), value);  
            }  
        }  
    }  
  
    /** 
    * partitioner的输入就是map的输出 
    *  
    * @author Administrator 
    */  
    public static class MyPartitionerPar extends Partitioner<Text, Text> {  
      
        @Override  
        public int getPartition(Text key, Text value, int numPartitions) {  
            int result = 0;  
            /*********************************************************************/  
            /***key.toString().equals("long")  must use toString()!!!!  ***/  
            /***开始的时候我没有用 ，导致都在一个区里，结果也在一个reduce输出文件中。  ***/  
            /********************************************************************/  
            if (key.toString().equals("long")) {  
                result = 0 % numPartitions;  
            } else if (key.toString().equals("short")) {  
                result = 1 % numPartitions;  
            } else if (key.toString().equals("right")) {  
                result = 2 % numPartitions;  
            }  
            return result;  
        }  
    }  
  
    public static class MyPartitionerReduce extends Reducer<Text, Text, Text, Text> {  
        protected void reduce(Text key, java.lang.Iterable<Text> value, Context context) throws java.io.IOException,  
        InterruptedException {  
            for (Text val : value) {  
                context.write(key, val);  
                //context.write(key, val);  
            }  
        }  
    }  
  
    public static void main(String[] args) throws Exception {  
        Configuration conf = new Configuration();
        /*
        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();  
        if (otherArgs.length != 2) {  
            System.err.println("Usage: MyPartitioner <in> <out>");  
            System.exit(2);  
        }
        */
        conf.set("mapred.jar","mp1.jar");
        final FileSystem fileSystem = FileSystem.get(new URI(OUTPUT_PATH), conf);
        fileSystem.delete(new Path(OUTPUT_PATH), true);
        Job job = new Job(conf, "MyPartitioner");  
        job.setNumReduceTasks(3);  
          
        job.setJarByClass(MyPartitioner.class);  
          
        job.setMapperClass(MyPartitionerMap.class);  
        job.setCombinerClass(MyPartitionerReduce.class);
        
        job.setMapOutputKeyClass(Text.class);  
        job.setMapOutputValueClass(Text.class);  
          
        job.setPartitionerClass(MyPartitionerPar.class);  
        job.setReducerClass(MyPartitionerReduce.class);  
          
        job.setOutputKeyClass(Text.class);  
        job.setOutputValueClass(Text.class);  
          
        FileInputFormat.setInputPaths(job, INPUT_PATH);  
        FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH));  
        System.exit(job.waitForCompletion(true) ? 0 : 1);  
    }  
}  

vim partititon

aa      1       2
bb      2       22
cc      11
dd      1
ee      99      99      999
ff      12      23      123

[hadoop@h201 ~]$ hadoop fs -cat /user/hadoop/output/part-r-00001
18/06/10 17:55:02 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
short   dd      1
short   cc      11
[hadoop@h201 ~]$ hadoop fs -cat /user/hadoop/output/part-r-00000
18/06/10 17:55:16 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
long    ff      12      23      123
long    ee      99      99      999
[hadoop@h201 ~]$ hadoop fs -cat /user/hadoop/output/part-r-00002
18/06/10 18:01:37 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
right   bb      2       22
right   aa      1       2