map/reduce实现数据去重

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class Dedpu extends Configured implements Tool {
/**
 * 数据去重
 * 数据样例：
 * 输入数据
 * 2006-6-9 a
 * 2006-6-10 b
 * 2006-6-9 a
 * 结果数据
 * 2006-6-9 a
 * 2006-6-10 b
 * 设计思路：
 * Map阶段 <时间，字符>
 * Reduce阶段输入<时间，list<字符>>，去除重复的字符，输出
 * 
 * **/
    public static class Map extends Mapper<LongWritable,Text,Text,Text>{
        public void map(LongWritable key,Text value,Context context)throws IOException, InterruptedException{
            String line=value.toString();
            Text myvalue=new Text("");
            context.write(new Text(line), myvalue);
//            StringTokenizer tokenizer=new StringTokenizer(line);
//            String datestr="",datastr="";
//            while(tokenizer.hasMoreTokens())
//            {
//                datestr=tokenizer.nextToken();
//                datastr=tokenizer.nextToken();
//                context.write(new Text(datestr), new Text(datastr));
//                
//            }
        }
        
    }
    
    public static class Reduce extends Reducer<Text,Text,Text,Text>{
        public void reduce(Text key,Iterable<Text>values,Context context)throws IOException,InterruptedException{

            context.write(key, new Text(""));
//            ArrayList  arr=new ArrayList();
//            Text mykey=key;
//            for(Text txt:values)
//            {
//                
//                if(!arr.contains(txt.toString())){
//                    arr.add(txt.toString());
//                }
//                    
//                
//            }
//            for(int i=0;i<arr.size();i++){
//                context.write(mykey, new Text(arr.get(i).toString()));
//                
//            }
            
        
            
        }
        
    }
    
    public int run(String[] args)throws Exception
    {
        Configuration conf=new Configuration();
        Job job=new Job(conf,"Data Depution");
        job.setJarByClass(Dedpu.class);
        
        job.setMapperClass(Map.class);
        job.setCombinerClass(Reduce.class);
        job.setReducerClass(Reduce.class);
        
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        
        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);
        
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        
        boolean success=job.waitForCompletion(true);
        return success?0:1;
        
    }
    
    public static void main(String[] args) throws Exception{
        int ret=ToolRunner.run(new Dedpu(), args);
        System.exit(ret);
    }
}