hadoop实战

hadoop实战  上面一份代码是hdfs上text的读写

每个key之间是根据hadoop自带的有序，value根据相同key排序，可以在reduce中执行

partition可以有序的分配reduce，防止一个reduce中数据过大

package com.company;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;


public class Main {
    public static void main(String[] args) throws Exception {
        try {
            Configuration config = new Configuration();
            FileSystem hdfs = FileSystem.get(config);
            FSDataInputStream hdfsInStream = hdfs.open(new Path("input"));
            BufferedReader br = new BufferedReader(new InputStreamReader(hdfsInStream, "utf-8"));
            FSDataOutputStream os = hdfs.create(new Path("output"));

            String line = br.readLine();
            while (line != null) {
                line = br.readLine();
                int x = Integer.parseInt("1");
                int y = Integer.parseInt("2");

                String sp[] = line.split("	", -1);
                for (int i = 0; i < sp.length; i++) {
                    System.out.print(sp[i] + "   ");
                }
                System.out.println("");
            }
            br.close();
            hdfsInStream.close();
            os.write("outputtext".getBytes("UTF-8"));
        } catch (IOException e) {
            e.printStackTrace();
        }


    }
}







import java.io.IOException;
import java.util.*;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.mapred.JobConf;

public class WordCount {

 public static class Map extends Mapper<LongWritable, Text, Text, Text> {

    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] parts = value.toString().split("	",-1);
    if(parts.length != 2)
       return;
    String Label = "";
    String Ip = "";
    String Num = parts[1];
    int length = Num.length();
    int ned = 9 - length;
    while(ned-- > 0) {
        Num = "0" + Num;
    }
    String Uid = "";
    int dex1 = parts[0].indexOf(' ');
    int dex2 = parts[0].indexOf('1');
    Label = parts[0].substring(0,dex1);
    Ip = parts[0].substring(dex2+1);
    Uid = parts[0].substring(dex1+1,dex2);
    Text Word = new Text(Uid + " " + Label);
    Text Val = new Text(Num);
    Text Word = new Text(Uid + " " + Label);
    Text Val = new Text(Num);
    context.write(Word, Val);
    }
 }

  public static class KeyPartitioner extends Partitioner<Text,Text> implements Configurable {
    int partition[] = new int[15];

    @Override
    public void setConf(Configuration config) {//implements Configurable 在partition开始时运行，且只执行1次
        try { 
            FileSystem hdfs = FileSystem.get(config);
            FSDataInputStream hdfsInStream = hdfs.open(new Path("input"));//本地hdfs上的文件
            BufferedReader br = new BufferedReader(new InputStreamReader(hdfsInStream, "utf-8"));//读取得到分割点的文件

            String line = br.readLine();
            while (line != null) {
                String sp[] = line.split(" ", -1);
                int x = Integer.parseInt(sp[0]);
                int y = Integer.parseInt(sp[1]);
                partition[y] = x;
                line = br.readLine();
            }
            partition[0] = 0;
            partition[10] = 65535;
            br.close();
            hdfsInStream.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

        @Override
    public Configuration getConf() {
        return null;
    }

    /*public void configure(JobConf config) {
        try {
            FileSystem hdfs = FileSystem.get(config);
            FSDataInputStream hdfsInStream = hdfs.open(new Path("/group/lwantispam/shenneng.ysn/operator/data"));
            BufferedReader br = new BufferedReader(new InputStreamReader(hdfsInStream, "utf-8"));
            FSDataInputStream hdfsInStream = hdfs.open(new Path("/group/lwantispam/shenneng.ysn/operator/data"));
            BufferedReader br = new BufferedReader(new InputStreamReader(hdfsInStream, "utf-8"));

            String line = br.readLine();
            while (line != null) {
                String sp[] = line.split(" ", -1);
                int x = Integer.parseInt(sp[0]);
                int y = Integer.parseInt(sp[1]);
                partition[y] = x;
                line = br.readLine();
            }
            partition[0] = 0;
            partition[10] = 65535;
            br.close();
            hdfsInStream.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }*/
    public int low(int k) {//取下限
        int l = 0,r = 10;
        while(l <= r) {
            int mid = (l + r) >> 1;
            if(partition[mid] < k) {
                l = mid + 1;
            } else {
                r = mid - 1;
            }
        }
        return r + 1;
    }

    public int up(int k) {//取上限
        int l = 0, r = 10;
        while(l <= r) {
            int mid = (l + r) >> 1;
            if(partition[mid] > k) {
                r = mid - 1;
            } else {
                l = mid + 1;
            }
        }
        return l - 1;
        }
        return l - 1;
    }

    @Override
    public int getPartition(Text key,Text value,int numPartitions) {
        int keynumber = Integer.parseInt(value.toString());
        int left = low(keynumber);
        int right = up(keynumber);
        int len = right - left + 1;
        if (len > 1) {
            int part = (int) (Math.random() * 1000) % len + left - 1;
            return part;
        } else {
            return left-1;
        }
     }
  }

 public static class Reduce extends Reducer<Text, Text, Text, Text> {

    @Override                                            //可以防止重写出错
    public void reduce(Text key, Iterable<Text> values, Context context)//要照格式
      throws IOException, InterruptedException {
        List<Integer> tips = new ArrayList<Integer>();
        for(Text value : values) {
            int tmp = Integer.parseInt(value.toString());
            tips.add(tmp);
        }
        Collections.sort(tips);
        for(int i = 0; i < tips.size(); i++) {
            System.out.print(String.valueOf(tips.get(i)) + ";");//可以输出在集群
            context.write(key, new Text(String.valueOf(tips.get(i))));
        }
    }
 }

 public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    Job job = new Job(conf, "wordcount");
    job.setJarByClass( WordCount.class ) ;
    job.setOutputKeyClass(Text.class);
    job.setJarByClass( WordCount.class ) ;
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);

    job.setNumReduceTasks(10);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    job.setPartitionerClass(KeyPartitioner.class);

    FileInputFormat.setInputPaths(job, new Path("datainfile"));
    FileOutputFormat.setOutputPath(job, new Path("dataoutfile"));

    job.waitForCompletion(true);
    //System.out.println("hello world");
 }

}

 finally

many log about
url 1...
wordcount
url sum (get the total)

sum 1... (count all sum)
wordcount
sum Sum
get partition point ---〉

class Go{
    int[] item = new int[10009];
    int total = 0;
    public void add(int x,int y) {
        item[x] = y;
        total += y;
    }

    public String print() {
        int pre = 0;
        int step = total/10 + 1;
        int st = 1;
        String out = "";
        for(int i = 1; i < 10000; i++) {
            pre += item[i];
            while(st * step <= pre) {
                out += i + " " + st + "
";
                st++;
            }
        }
        return out;
    }
}


Go getPartition = new Go();
    try {
        Configuration config = new Configuration();
        FileSystem hdfs = FileSystem.get(config);
         FSDataInputStream hdfsInStream = hdfs.open(new Path("input"));
         BufferedReader br = new BufferedReader(new InputStreamReader(
               hdfsInStream, "utf-8"));
        FSDataOutputStream os = hdfs.create(new Path("output"));
         String line = br.readLine();
         while (line != null) {
            String sp[] = line.split("	",-1);
            int x = Integer.parseInt(sp[0]);
            int y = Integer.parseInt(sp[1]);
            getPartition.add(x,y);
            line = br.readLine();
         }
         br.close();
         hdfsInStream.close();
        String out = getPartition.print();
        os.write(out.getBytes("UTF-8"));
      } catch (IOException e) {
         e.printStackTrace();
      }

url sum
partition
reduce sort
get order url and order sum;

***the code is on the top**