练习 : 自定义sink mysql hbase

mysql

package sink;

//import com.util.Propss;
//import com.bean.Sku;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.lang.reflect.Field;
import java.sql.*;

/**
 * @Description:
 * @Author: liuyuan
 * @Times : 2021/9/24 20:09
 */

//自定义SINK RichSinkFuncation
public class MySQLSink extends RichSinkFunction<String> {
    private static Connection conn;
    private static PreparedStatement pre;

    private static String database;
    private static String sql;
    private static Class T;

    public MySQLSink(String database,String sql,Class T){
        this.database=database;
        this.sql=sql;
        this.T=T;
    }

    @Override
    public void open(Configuration parameters) throws Exception {
        Class.forName("com.mysql.jdbc.Driver");
        conn = DriverManager.getConnection("jdbc:mysql://hadoop106:3306/"+database, "root", "root");
        conn.setAutoCommit(true);
    }

    @Override
    public void invoke(String value, Context context) throws Exception {
        String[] split = value.split(",");
        Field[] declaredFields = T.getDeclaredFields();
        pre = conn.prepareStatement(sql);
        for (int i = 0; i < declaredFields.length; i++) {
            if(declaredFields[i].getType().toString().equals("class java.lang.String")){
                pre.setString((i+1),split[i]);
            }
            if(declaredFields[i].getType().toString().equals("class java.lang.Integer")){
                pre.setInt((i+1),Integer.valueOf(split[i]));
            }
            if(declaredFields[i].getType().toString().equals("class java.lang.Double")){
                pre.setDouble((i+1),Double.valueOf(split[i]));
            }
            if(declaredFields[i].getType().toString().equals("class java.lang.Long")){
                pre.setLong((i+1),Long.valueOf(split[i]));
            }
        }
        pre.execute();
    }

    @Override
    public void close() throws Exception {
        pre.close();
        conn.close();

    }
}

hbase

package sink;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.util.Bytes;

import java.io.IOException;
import java.lang.reflect.Field;
import java.util.*;

public class HBaseSink extends RichSinkFunction<String> {
    private Connection connection;
    private Class T;
    private String tableName;
    private String[] fieldsName;
    List<Put> list=new ArrayList<Put>();
    
    public static String[] getFiledName(Class T) {
        Field[] fields =T.getClass().getDeclaredFields();
        String[] fieldName = new String[fields.length];
        for (int i = 0; i < fieldName.length; i++) {
            fieldName[i] = fields[i].getName();

        }
        return fieldName;
    }

    public HBaseSink(Class T, String tableName){
        this.T=T;
        this.tableName=tableName;
        this.fieldsName=getFiledName(T);
    }

    @Override
    public void open(Configuration parameters) throws Exception {
        connection= HBase_Util.getConf();
    }

    @Override
    public void invoke(String value, Context context) throws Exception {
        String[] s1 = value.split(",");
        Table table = connection.getTable(TableName.valueOf(tableName));
//        String rowkey = UUID.randomUUID().toString().replaceAll("-", "");
        Put put = new Put(Bytes.toBytes(s1[0]));
        for (int i = 0; i < fieldsName.length; i++) {
            put.addColumn(Bytes.toBytes("info"),Bytes.toBytes(fieldsName[i]),Bytes.toBytes(s1[i]));
            list.add(put);
        }
        table.put(list);
    }

    @Override
    public void close() throws Exception {
        connection.close();
    }


    public static  class HBase_Util {
        static org.apache.hadoop.conf.Configuration con = HBaseConfiguration.create();
        static org.apache.hadoop.conf.Configuration conf = Propss.setConf(con);
        static Connection connection;
        static HBaseAdmin admin;
        static Table t;

        static {
            try {
                connection = ConnectionFactory.createConnection(conf);
                admin = (HBaseAdmin)connection.getAdmin();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        //获取 conn
        public static Connection getConf(){
            //创建HBase的配置对象
            org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
            //设置hbase配置属性
            conf.set("hbase.zookeeper.quorum","hadoop106,hadoop107,hadoop108");
            conf.set("hbase.zookeeper.property.clientPort","2181");
            Connection connection=null;
            //通过连接函数，创建连接对象
            try {
                connection = ConnectionFactory.createConnection(conf);

            } catch (IOException e) {
                e.printStackTrace();
            }
            return connection;
        }

        //建表
        public static void build_Table(String tableName,List<String> FamilyNames) throws Exception {
            TableDescriptorBuilder buider = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName));
            for (String columnName : FamilyNames) {
                ColumnFamilyDescriptor info = ColumnFamilyDescriptorBuilder.of(Bytes.toBytes(columnName));
                buider.setColumnFamily(info);
            }
            TableDescriptor build = buider.build();
            admin.createTable(build);
            System.out.println("____build_done____");
        }

        //插入一条数据
        public static void insert_Row(String tableName,String row,String Family,String qualifier,String value) throws Exception {
            t = connection.getTable(TableName.valueOf(tableName));
            Put put = new Put(Bytes.toBytes(row));
            Put put1 = put.addColumn(Bytes.toBytes(Family), Bytes.toBytes(qualifier), Bytes.toBytes(value));
            t.put(put1);
            System.out.println("____insert_Row_done____");
        }

        //插入num条数据
        public  static  void insert_Batch(String tableName,String row,String Family,String qualifier,String value,Integer num) throws Exception {
            t = connection.getTable(TableName.valueOf(tableName));
            List<Put> list=new ArrayList<>();
            for (int i = 0; i < num; i++) {
                String s = UUID.randomUUID().toString().replaceAll("-", "");
                Put puts = new Put(Bytes.toBytes(s));
                Put putss = puts.addColumn(Bytes.toBytes(Family), Bytes.toBytes(qualifier), Bytes.toBytes(value+i));
                list.add(putss);
            }
            t.put(list);
            System.out.println("____insert_Batch_done____");
        }

        //删除表
        public static void drop_Table(String tableName) throws Exception {
            if (admin.tableExists(TableName.valueOf(tableName))){
                admin.disableTable(TableName.valueOf(tableName));
                admin.deleteTable(TableName.valueOf(tableName));
                System.out.println("____drop_Table_done____");
            }else {
                System.out.println("____no_such_Table_found____");
            }

        }

        //删除一条数据
        public static void  delete_Row(String tableName,String row) throws Exception {
            t = connection.getTable(TableName.valueOf(tableName));
            Delete delete = new Delete(Bytes.toBytes(row));
            t.delete(delete);
            System.out.println("____delete_Row_done____");
        }

        //特定列过滤查询
        public static void scan_Filter(String tableName, String Family, String qualifier, CompareOperator compare, byte[] value) throws Exception {
            t = connection.getTable(TableName.valueOf(tableName));
            SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes(Family), Bytes.toBytes(qualifier), compare, value);
            Scan scan = new Scan();
            Scan scan1 = scan.setFilter(filter);
            ResultScanner scanner = t.getScanner(scan1);
            Iterator<Result> iterator = scanner.iterator();
            while (iterator.hasNext()){
                Cell[] cells = iterator.next().rawCells();
                System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____");
                for (Cell cell : cells) {
                    System.out.print(new String(CellUtil.cloneRow(cell)));
                    System.out.print(" - ");
                    System.out.print(new String(CellUtil.cloneFamily(cell)));
                    System.out.print(" - ");
                    System.out.print(new String(CellUtil.cloneQualifier(cell)));
                    System.out.print(" - ");
                    System.out.println(new String(CellUtil.cloneValue(cell)));
                }

            }
            System.out.println("____scan_Filter_done____");
        }
        //查询一条数据
        public static void scan_Row(String tableName,String row)throws Exception{
            t = connection.getTable(TableName.valueOf(tableName));
            Get get = new Get(Bytes.toBytes(row));
            Result result = t.get(get);
            Cell[] cells = result.rawCells();
            for (Cell cell : cells) {
                System.out.print(new String(CellUtil.cloneRow(cell)));
                System.out.print(" - ");
                System.out.print(new String(CellUtil.cloneFamily(cell)));
                System.out.print(" - ");
                System.out.print(new String(CellUtil.cloneQualifier(cell)));
                System.out.print(" - ");
                System.out.println(new String(CellUtil.cloneValue(cell)));
            }
            System.out.println("____scan_Row_done____");
        }
        //区间查询数据
        public static void scan_Rows(String tableName,String row1,String row2)throws Exception{
            t = connection.getTable(TableName.valueOf(tableName));
            Scan sc=new Scan(Bytes.toBytes(row1),Bytes.toBytes(row2));
            ResultScanner scanner = t.getScanner(sc);
            Iterator<Result> iterator = scanner.iterator();
            System.out.println("____前闭后开____");
            while (iterator.hasNext()){
                Result next = iterator.next();
                Cell[] cells = next.rawCells();
                System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____");
                for (Cell cell : cells) {
                    System.out.print(new String(CellUtil.cloneRow(cell)));
                    System.out.print(" - ");
                    System.out.print(new String(CellUtil.cloneFamily(cell)));
                    System.out.print(" - ");
                    System.out.print(new String(CellUtil.cloneQualifier(cell)));
                    System.out.print(" - ");
                    System.out.println(new String(CellUtil.cloneValue(cell)));
                }
            }
            System.out.println("____scan_Rows_done____");
        }
        //查询一条特定列族数据
        public static void get_value_by_family(String tableName,String row,String family)throws Exception{
            t = connection.getTable(TableName.valueOf(tableName));
            Get get = new Get(Bytes.toBytes(row));
            get.addFamily(Bytes.toBytes(family));
            Result result = t.get(get);
            Cell[] cells = result.rawCells();
            for (Cell cell : cells) {
                System.out.print(new String(CellUtil.cloneRow(cell)));
                System.out.print(" - ");
                System.out.print(new String(CellUtil.cloneFamily(cell)));
                System.out.print(" - ");
                System.out.print(new String(CellUtil.cloneQualifier(cell)));
                System.out.print(" - ");
                System.out.println(new String(CellUtil.cloneValue(cell)));
            }
            System.out.println("____get_value_by_family_done____");
        }

        //查询一条特定列数据
        public static void get_value_by_qualifier(String tableName,String row,String family,String qualifier)throws Exception{
            t = connection.getTable(TableName.valueOf(tableName));
            Get get = new Get(Bytes.toBytes(row));
            get.addColumn(Bytes.toBytes(family),Bytes.toBytes(qualifier));
            Result result = t.get(get);
            Cell[] cells = result.rawCells();
            for (Cell cell : cells) {
                System.out.print(new String(CellUtil.cloneRow(cell)));
                System.out.print(" - ");
                System.out.print(new String(CellUtil.cloneFamily(cell)));
                System.out.print(" - ");
                System.out.print(new String(CellUtil.cloneQualifier(cell)));
                System.out.print(" - ");
                System.out.println(new String(CellUtil.cloneValue(cell)));
            }
            System.out.println("____get_value_by_qualifier_done____");
        }

        //全查某表
        public static void  scan_All(String tableName) throws Exception {
            t = connection.getTable(TableName.valueOf(tableName));
            Scan sc=new Scan();
            ResultScanner scanner = t.getScanner(sc);
            Iterator<Result> iterator = scanner.iterator();
            while (iterator.hasNext()){
                Result next = iterator.next();
                Cell[] cells = next.rawCells();
                System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____");
                for (Cell cell : cells) {
                    System.out.print(new String(CellUtil.cloneRow(cell)));
                    System.out.print(" - ");
                    System.out.print(new String(CellUtil.cloneFamily(cell)));
                    System.out.print(" - ");
                    System.out.print(new String(CellUtil.cloneQualifier(cell)));
                    System.out.print(" - ");
                    System.out.println(new String(CellUtil.cloneValue(cell)));
                }
            }
            System.out.println("____scan_All_done____");
        }
        //查看所有表
        public static  void list() throws Exception {
            TableName[] tableNames = admin.listTableNames();
            for (TableName tableName : tableNames) {
                System.out.println(tableName.toString());
            }

        }
        //查看所有表结构
        public static  void desc_Table(String tableName) throws Exception {
            List<TableDescriptor> tableDescriptors = admin.listTableDescriptors();
            Iterator<TableDescriptor> iterator = tableDescriptors.iterator();
            while (iterator.hasNext()){
                TableDescriptor next = iterator.next();
                if (next.getTableName().toString().equals(tableName)){
                    System.out.println(next);
                }
            }
            System.out.println("____list_done____");

        }


        //关流
        public static  void stop() throws Exception {
            connection.close();
            System.out.println("____connection_close_done____");
        }


        public static class Propss {
            public static Properties producer_Props = new Properties();
            public static Properties consumer_Props = new Properties();
            public static HashMap<String, Object> kafka_Consumer = new HashMap<>();
            public static HashMap<String, Object> kafka_Producer = new HashMap<>();

            public static org.apache.hadoop.conf.Configuration setConf(org.apache.hadoop.conf.Configuration conf){
                conf.set("hbase.zookeeper.quorum","hadoop106,hadoop107,hadoop108");
                conf.set("hbae.zookeeper.property.client","2181");
                return conf;
            }
            static {

                kafka_Producer.put("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092");
                //0、1 和 all：0表示只要把消息发送出去就返回成功；1表示只要Leader收到消息就返回成功；all表示所有副本都写入数据成功才算成功
                kafka_Producer.put("acks", "all");
                //重试次数
                kafka_Producer.put("retries", Integer.MAX_VALUE);
                //批处理的字节数
                kafka_Producer.put("batch.size", 16384);
                //批处理的延迟时间，当批次数据未满之时等待的时间
                kafka_Producer.put("linger.ms", 1);
                //用来约束KafkaProducer能够使用的内存缓冲的大小的，默认值32MB
                kafka_Producer.put("buffer.memory", 33554432);
                // properties.put("value.serializer",
                // "org.apache.kafka.common.serialization.ByteArraySerializer");
                // properties.put("key.serializer",
                // "org.apache.kafka.common.serialization.ByteArraySerializer");
                kafka_Producer.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
                kafka_Producer.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");

                kafka_Consumer.put("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092");
                kafka_Consumer.put("group.id", "com-test");
                //from beginning
                kafka_Consumer.put("auto.offset.reset", "earliest");
                kafka_Consumer.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
                kafka_Consumer.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

                producer_Props.setProperty("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092");
                producer_Props.setProperty("ack", "all");
                producer_Props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
                producer_Props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
                producer_Props.put("auto.offset.reset", "earliest");

                consumer_Props.setProperty("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092");
                consumer_Props.setProperty("group.id", "com-test");
                consumer_Props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
                consumer_Props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
                consumer_Props.put("auto.offset.reset", "earliest");
            }


        }

    }

}