使用hadoop mapreduce分析mongodb数据：（2）

在上一篇使用hadoop mapreduce分析mongodb数据：（１）中，介绍了如何使用Hadoop MapReduce连接MongoDB数据库以及如何处理数据库，本文结合一个案例来进一步说明Hadoop MapReduce处理MongoDB的细节

• 原始数据

• > db.stackin.find({})
  { "_id" : ObjectId("575ce909aa02c3b21f1be0bb"), "summary" : "good good day",  "url" : "url_1" }
  { "_id" : ObjectId("575ce909aa02c3b21f1be0bc"), "summary" : "hello world good world", "url" : "url_2" }
  { "_id" : ObjectId("575ce909aa02c3b21f1be0bd"), "summary" : "hello world good hello good", "url" : "url_3" }
  { "_id" : ObjectId("575ce909aa02c3b21f1be0be"), "summary" : "hello world hello",  "url" : "url_4" }

  每一个记录表示一个网页，summary对应的值是网页的文章，url对应的值是该文章的链接

• 目标结果

• > db.stackout.find({})
  { "_id" : "world", "data" : [  {  "url_2" : 2 },  {  "url_3" : 1 },  {  "url_4" : 1 } ], "index" : 0, "maxindex" : 3 }
  { "_id" : "good", "data" : [  {  "url_1" : 2 },  {  "url_2" : 1 },  {  "url_3" : 2 } ], "index" : 0, "maxindex" : 3 }
  { "_id" : "day", "data" : [  {  "url_1" : 1 } ], "index" : 0, "maxindex" : 1 }
  { "_id" : "hello", "data" : [  {  "url_2" : 1 },  {  "url_3" : 2 },  {  "url_4" : 2 } ], "index" : 0, "maxindex" : 3 }

  我们需要统计每个单词在每个网页中分别出现的次数，从结果可知，单词world在每个url出现的次数

• 设计代码
  

  import java.util.*; 
  import java.io.*;
  
  import org.bson.*;
  
  import com.mongodb.hadoop.MongoInputFormat;
  import com.mongodb.hadoop.MongoOutputFormat;
  import com.mongodb.hadoop.io.BSONWritable;
  
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.io.*;
  import org.apache.hadoop.mapreduce.*;
  
  
  public class WordCount {
  
      public static class TokenizerMapper extends Mapper<Object, BSONObject, Text, BSONWritable> {
          //private final static 
          private Text word = new Text();
  
          public void map(Object key, BSONObject value, Context context ) 
                  throws IOException, InterruptedException {
              String url = value.get("url").toString();
              StringTokenizer itr = new StringTokenizer(value.get("summary").toString().
                      replaceAll("\p{Punct}|\d","").replaceAll("
  ", " ").replace("
  ", " ").
                      replace("
  ", " ").toLowerCase());
              while (itr.hasMoreTokens()) {
                  word.set(itr.nextToken());
                  BasicBSONObject urlCounts = new BasicBSONObject();
                  urlCounts.put(url, 1);
                  context.write(word, new BSONWritable(urlCounts));
              }
          }
      }
  
      public static class IntSumReducer extends Reducer<Text, BSONWritable, Text, BSONWritable> {
          //private BasicBSONObject result = new BasicBSONObject();
          
          public void reduce(Text key, Iterable<BSONWritable> values, Context context)
              throws IOException, InterruptedException {
              HashMap<String, Integer> mymap = new HashMap<String, Integer>();
              BasicBSONObject result = new BasicBSONObject();
              BasicBSONObject urlcount = new BasicBSONObject();
              for (BSONWritable val : values) {
                  @SuppressWarnings("unchecked")
                  BSONObject temp2 = val.getDoc();
                  @SuppressWarnings("unchecked")
                  HashMap<String, Integer> temp = (HashMap<String, Integer>) val.getDoc().toMap();
                  for (Map.Entry<String, Integer> entry :  temp.entrySet()) {
                      if (mymap.containsKey(entry.getKey())) {
                          mymap.put(entry.getKey(), entry.getValue()+1);
                      }
                      else {
                          mymap.put(entry.getKey(), 1);
                      }
                  }
              }
              result.putAll(mymap);
              context.write(key, new BSONWritable(result));
          }
      }
  
      public static void main(String[] args) throws Exception {
          Configuration conf = new Configuration();
          conf.set( "mongo.input.uri" , "mongodb://localhost/stackoverflow.stackin" );
          conf.set( "mongo.output.uri" , "mongodb://localhost/stackoverflow.stackout" );
          @SuppressWarnings("deprecation")
          Job job = new Job(conf, "word count");
          job.setJarByClass(WordCount.class);
          job.setMapperClass(TokenizerMapper.class);
          //job.setCombinerClass(IntSumReducer.class);
          job.setReducerClass(IntSumReducer.class);
          job.setMapOutputKeyClass(Text.class);
          job.setMapOutputValueClass(BSONWritable.class);
          job.setOutputKeyClass(Text.class);
          job.setOutputValueClass(BSONWritable.class);
          job.setInputFormatClass( MongoInputFormat.class );
          job.setOutputFormatClass( MongoOutputFormat.class );
          System.exit(job.waitForCompletion(true) ? 0 : 1);
      }
  }

  设计的思路是，在map部分得到一个word以及键为url值为1的Bson对象，然后写入content中。对应的，在reduce部分对传入的值进行统计。

总结：本案例很简单，但是需要明白Hadoop MapReduce的原理以及mongo-hadoop API中对象的使用。如果有疑问，可以在评论区提出～