• 云计算-MapReduce


    Hadoop示例程序WordCount详解及实例http://blog.csdn.net/xw13106209/article/details/6116323

    hadoop中使用MapReduce编程实例(转)http://eric-gcm.iteye.com/blog/1807468

    【mapreduce进阶编程二】奇偶行分别求和http://www.aboutyun.com/forum.php?mod=viewthread&tid=9360

    hadoop2.2.0 MapReduce求和并排序http://www.cnblogs.com/mengyao/p/4151509.html

    MapReduce求最大值http://blog.csdn.net/lzm1340458776/article/details/43227759

    慢慢看!

    求最大值:

    package org.apache.hadoop.examples;

    import java.io.IOException;
    import java.util.StringTokenizer;

    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.conf.Configured;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Partitioner;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.util.GenericOptionsParser;
    import org.apache.hadoop.util.Tool;
    import org.apache.hadoop.util.ToolRunner;

    public class Score2 {
    private static String SPILT = " ";
    private static int max = -10000;

    public static class ScoreMapper extends Mapper<LongWritable, Text, Text, Text> {

    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

    int num = Integer.valueOf(String.valueOf(value));

    if (max < num)
    max = num;
    context.write(new Text(String.valueOf(max)), new Text(""));
    }
    }

    public static class MaxReducer extends Reducer<Text, Text, Text, Text> {
    @Override
    public void reduce(Text key, Iterable<Text> value, Context context) throws IOException, InterruptedException {
    int num = Integer.valueOf(String.valueOf(key));

    if (num == max) {
    context.write(new Text("max:"), new Text(String.valueOf(num)));
    }
    }
    }


    public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
    System.out.println("please input at least 2 arguments");
    System.exit(2);
    }

    Job job = Job.getInstance(conf, "max");
    job.setJarByClass(Score2.class);
    job.setMapperClass(ScoreMapper.class);
    job.setReducerClass(MaxReducer.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.setNumReduceTasks(1);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
    }
     
    啊啊啊啊啊啊

    import java.io.IOException;
    import java.util.Iterator;
    import java.util.StringTokenizer;

    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.DoubleWritable;
    import org.apache.hadoop.io.DoubleWritable.Comparator;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.WritableComparable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Mapper.Context;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.util.GenericOptionsParser;

    public class max {
      public static class Map1 extends

         Mapper<Object, Text, Text, IntWritable> {

             public void map(Object key, Text value, Context context)

             throws IOException, InterruptedException {
              int count=0;
              String line = value.toString();
              String s[]=line.split(" ");
              for (int i=0;i<s.length;i++)
              {
               //if (Integer.parseInt(s[i])<min)//s[i]bian int
                //min=Integer.parseInt(s[i]);

               char c[] = s[i].toCharArray();
               if (c[c.length-1]=='h')
                count++;

              }
              context.write(new Text("1"), new IntWritable(count));
              //context.write(new Text(key.tostring),new IntWritable(max));

             }

         }

      public static class Reduce1 extends
      Reducer<Text, IntWritable, Text, IntWritable> {
          public void reduce(Text key, Iterable<IntWritable> values,
          Context context) throws IOException, InterruptedException {
           //int min=1000000;
           int sum=0;
              Iterator<IntWritable> iterator = values.iterator();  //
              while (iterator.hasNext()) {
               int n=iterator.next().get();

              // if (n<min)
               // min=n;
             sum=sum+n;
              }
          context.write(new Text(" "), new IntWritable(sum));
          }
      }

      public static void main(String[] args) throws Exception {

             Configuration conf = new Configuration();
             FileSystem fs = FileSystem.get(conf);
             conf.set("mapred.job.tracker", "127.0.0.1:9000");
             String[] otherArgs = new GenericOptionsParser(conf, args)
                     .getRemainingArgs();
             if (otherArgs.length != 2) { //input output

                 System.err.println("Usage: Data Deduplication <in> <out><out>");
                 System.exit(2);
             }
             Job job = Job.getInstance(conf,"min");

             job.setJarByClass(max.class);

             job.setMapperClass(Map1.class);
             job.setMapOutputKeyClass(Text.class);
             job.setMapOutputValueClass(IntWritable.class);

             job.setReducerClass(Reduce1.class);
             job.setOutputKeyClass(Text.class);
             job.setOutputValueClass(IntWritable.class);

             FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
             Path outpath = new Path(otherArgs[1]);
             if (fs.exists(outpath))
             {
              fs.delete(outpath,true);
             }
             FileOutputFormat.setOutputPath(job, outpath);


             if (job.waitForCompletion(true))
             {

               System.exit(0);

             }

         }


    }

  • 相关阅读:
    数据库操作语句大全(sql)
    尚未在 Web 服务器上注册ASP.NET 4.5。安装VS15后的问题(转)
    ASP.NET通过http/https的POST方式,发送和接受XML文件内容(转)
    C# 的关键字详细介绍(转)
    12个css高级技巧.html
    CentOS上安装elasticsearch
    springboot例子
    ajax的XmlHttpRequest对象常用方法
    阿里云oss操作
    Python出现Could not find a version that satisfies the requirement openpyxl (from versions: )
  • 原文地址:https://www.cnblogs.com/ximiaomiao/p/7003759.html
Copyright © 2020-2023  润新知