MapReduce_TopK

vim topk.txt

a　　1000
b　　2000
c　　90000
d　　88
e　　999999
f　　9998
g　　13223

package MapReduce;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.TreeMap;  
  
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IntWritable;  
import org.apache.hadoop.io.LongWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
  
public class TopK {  
    private static final String INPUT_PATH = "hdfs://h201:9000/user/hadoop/input_TopK";
    private static final String OUTPUT_PATH = "hdfs://h201:9000/user/hadoop/output";
    public static final int K = 2;  
      
    public static class KMap extends Mapper<LongWritable,Text,IntWritable,Text> {    
        TreeMap<Integer, String> map = new TreeMap<Integer, String>();    
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {                
            String line = value.toString();  
            if(line.trim().length() > 0 && line.indexOf("	") != -1) {  //indexof如果没有找到字符串，则返回-1                  
                String[] arr = line.split("	", 2);  
                String name = arr[0];  
                Integer num = Integer.parseInt(arr[1]);  
                map.put(num, name);      
                if(map.size() > K) {  
                    map.remove(map.firstKey());  
                }  
            }  
        }  
        @Override  
        protected void cleanup(  
                Mapper<LongWritable, Text, IntWritable, Text>.Context context) throws IOException, InterruptedException {                
            for(Integer num : map.keySet()) {  
                context.write(new IntWritable(num), new Text(map.get(num)));  
            }                
        }           
    }             
    public static class KReduce extends Reducer<IntWritable, Text, IntWritable, Text> {            
        TreeMap<Integer, String> map = new TreeMap<Integer, String>();           
        public void reduce(IntWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {                    
            map.put(key.get(), values.iterator().next().toString());  
            if(map.size() > K) {  
                map.remove(map.firstKey());  
            }  
        }  
        @Override  
        protected void cleanup(Reducer<IntWritable, Text, IntWritable, Text>.Context context) throws IOException, InterruptedException {  
            for(Integer num : map.keySet()) {  
                context.write(new IntWritable(num), new Text(map.get(num)));  
            }  
        }  
    }  
  
    public static void main(String[] args) throws IOException, URISyntaxException {  
        // TODO Auto-generated method stub  
          
        Configuration conf = new Configuration();
        conf.set("mapred.jar","tk.jar");
        final FileSystem fileSystem = FileSystem.get(new URI(OUTPUT_PATH), conf);
        fileSystem.delete(new Path(OUTPUT_PATH), true);
        try {  
            Job job = new Job(conf, "my own word count"); 
        
            job.setJarByClass(TopK.class);  
            job.setMapperClass(KMap.class);  
            job.setCombinerClass(KReduce.class);  
            job.setReducerClass(KReduce.class);  
            job.setOutputKeyClass(IntWritable.class);  
            job.setOutputValueClass(Text.class);  
            FileInputFormat.setInputPaths(job, INPUT_PATH);  
            FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH));  
            System.out.println(job.waitForCompletion(true));  
        } catch (IOException e) {  
            // TODO Auto-generated catch block  
            e.printStackTrace();  
        } catch (ClassNotFoundException e) {  
            // TODO Auto-generated catch block  
            e.printStackTrace();  
        } catch (InterruptedException e) {  
            // TODO Auto-generated catch block  
            e.printStackTrace();  
        }   
    }  
}  

/usr/jdk1.7.0_25/bin/javac TopK.java

/usr/jdk1.7.0_25/bin/jar cvf tk.jar TopK*class

[hadoop@h201 ~]$ hadoop fs -cat /user/hadoop/output/part-r-00000
18/06/10 15:49:24 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
90000   c
999999  e