• hadoop 倒排索引-分布式作业二


    import java.io.IOException;
    import java.util.StringTokenizer;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.input.FileSplit;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    public class InvertedIndex {
    
    	public static class InversedIndexMapper extends Mapper<Object, Text, Text, Text> {
    		
    		private Text outKey = new Text();
    		private Text outVal = new Text();
    		
    		@Override
    		public void map (Object key,Text value,Context context) {
    			StringTokenizer tokens = new StringTokenizer(value.toString());
    			FileSplit split = (FileSplit) context.getInputSplit();
    			while(tokens.hasMoreTokens()) {
    				String token = tokens.nextToken();
    				try {
    					outKey.set(token + ":" + split.getPath());
    					outVal.set("1");
    					context.write(outKey, outVal);
    				} catch (IOException e) {
    					e.printStackTrace();
    				} catch (InterruptedException e) {
    					e.printStackTrace();
    				}
    			}
    			
    			System.out.println("Map phase finished ...");
    		}
    	}
    	
    	public static class InversedIndexCombiner extends Reducer<Text, Text, Text, Text> {
    		
    		private Text outKey = new Text();
    		private Text outVal = new Text();
    		
    		@Override
    		public void reduce(Text key,Iterable<Text> values,Context context) {
    			String[] keys = key.toString().split(":");
    			int sum = 0;
    			for(Text val : values) {
    				sum += Integer.parseInt(val.toString());
    			}
    			try {
    				outKey.set(keys[0]);
    				int index = keys[keys.length-1].lastIndexOf('/');
    				outVal.set(keys[keys.length-1].substring(index+1) + ":" + sum);
    				context.write(outKey, outVal);
    			} catch (IOException e) {
    				e.printStackTrace();
    			} catch (InterruptedException e) {
    				e.printStackTrace();
    			}
    
    			System.out.println("Combine phase finished ...");
    		}
    		
    	}
    	
    	public static class InversedIndexReducer extends Reducer<Text, Text, Text, Text> {
    		
    		@Override
    		public void reduce (Text key,Iterable<Text> values,Context context) {
    			StringBuffer sb = new StringBuffer();
    			for(Text text : values) {
    				sb.append(text.toString() + " ,");
    			}
    			try {
    				context.write(key, new Text(sb.toString()));
    			} catch (IOException e) {
    				e.printStackTrace();
    			} catch (InterruptedException e) {
    				e.printStackTrace();
    			}
    
    			System.out.println("Reduce phase finished ...");
    		}
    	}
    	
    	public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    		Configuration conf = new Configuration();
    		@SuppressWarnings("deprecation")
    		Job job = new Job(conf,"index inverted");
    		
    		job.setJarByClass(InvertedIndex.class);
    		job.setMapperClass(InversedIndexMapper.class);
    		job.setCombinerClass(InversedIndexCombiner.class);
    		job.setReducerClass(InversedIndexReducer.class);
    		job.setMapOutputKeyClass(Text.class);
    		job.setMapOutputValueClass(Text.class);
    		job.setOutputKeyClass(Text.class);
    		job.setOutputValueClass(Text.class);
    
    		job.setNumReduceTasks(1);
    		
    		FileInputFormat.addInputPath(job, new Path("input"));
    		FileOutputFormat.setOutputPath(job, new Path("output"));
    		if(job.waitForCompletion(true))
    		{
    			System.out.println("All job finished ...");
    			System.exit(0);
    		}
    	}
    
    }
    

      

  • 相关阅读:
    【转】人类的心理行为模式----《影响力》笔记
    c++类型形参的实参的受限转换
    如何确定网页和查询的相关性
    英语美文摘要
    特殊的字符数组
    c++中的虚函数
    [C#] 常用工具类——应用程序属性信息访问类
    c#常用工具类:文件和二进制转换
    [C#] 常用工具类——文件操作类
    Lamda和Linq语法对比详细
  • 原文地址:https://www.cnblogs.com/shenbingyu/p/4940676.html
Copyright © 2020-2023  润新知