Flink测试程序以及发布到服务器

测试程序：从kafka中读取车辆定位数据，进行计算过滤，将超出指定电子围栏区域的定位数据保存到mysql，以便于后续进一步处理。

该测试程序源自业务部门的需求：对监控的车辆行驶轨迹进行实时分析，如果超出电子围栏区域则需要及时报警处理。由于这个场景的实时性要求高，因此比较适合用flink来直接处理。如果先将车辆定位数据保存到doris，再进行定时计算，则在时效性上可能就无法保证了。

由于flink目前只支持java跟scala，为了写程序，被迫学起了java。本例使用idea来编写（题外话：java跟c#的很多概念、甚至语法都已经非常接近了，写起代码来并不费劲；费劲是需要理解idea的配置管理、包引用、编译发布环境等这些概念，对于习惯了使用宇宙第一ide的vs新手来说，确实非常头疼(╥﹏╥)）。

项目的pom配置如下：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>catsti.flink</groupId>
    <artifactId>kafka-compute-mysql</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <flink.version>1.13.2</flink.version>
        <java.version>1.8</java.version>
        <scala.binary.version>2.11</scala.binary.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.34</version>
        </dependency>
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-dbcp2</artifactId>
            <version>2.1.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.11</artifactId>
            <version>1.11.3</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.28</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>1.7.25</version>
            <scope>compile</scope>
        </dependency>
    </dependencies>
</project>

主类代码如下：

import comm.JsonHelper;
import comm.PolygonUtil;
import entity.p_cargis;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;

import java.awt.geom.Point2D;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

public class DataCompute {
    public static void main(String[] args) throws Exception{

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //设置重启机制
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, Time.of(10, TimeUnit.SECONDS)));//每10秒重启一次，尝试3次

        //设置kafka属性
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "172.18.88.33:9092");
        props.setProperty("group.id","g1");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("auto.offset.reset", "latest");

        //读取kafka消息
        SingleOutputStreamOperator<p_cargis> pgis = env.addSource(new FlinkKafkaConsumer<>(
                "flinktest1",        //需要读取的kafka消息主题
                new SimpleStringSchema(),   //
                props)).map(string -> JsonHelper.getJsonToBean(string, p_cargis.class));//将kafka消息（json格式）转化为p_cargis实体

        //按照车牌号分组
        pgis.keyBy(p_cargis::getCph);

        //对数据进行过滤
        SingleOutputStreamOperator<p_cargis> filter = pgis.filter(new FilterFunction<p_cargis>() {
            @Override
            public boolean filter(p_cargis p_cargis) throws Exception {
                //判断数据是否在指定区域内
                Point2D.Double point = new Point2D.Double();
                point.x = p_cargis.lat;
                point.y = p_cargis.lon;
                return !PolygonUtil.isInPolygon(point,GetPolygonRule.getTestPolygonRule());
            }
        });

        filter.print();

        //将过滤出来的数据保存到mysql
        filter.addSink(new SinkToMySql()).name("Save to Mysql");

        //启动执行
        env.execute("测试程序（读kafka，计算车辆定位数据是否超出范围，将超出数据写入mysql）");
    }
}

多边形工具类（用于判定指定点位是否在指定的区域内）：

package comm;

import java.awt.geom.Point2D;
import java.util.List;

public class PolygonUtil {
    /**
     * 地球半径
     */
    private static double EARTH_RADIUS = 6378138.0;

    private static double rad(double d)
    {
        return d * Math.PI / 180.0;
    }

    /**
     * 计算是否在圆上（单位/千米）
     *
     * @Title: GetDistance
     * @Description: TODO()
     * @param radius 半径
     * @param lat1  纬度
     * @param lng1  经度
     * @return
     * @return double
     * @throws
     */
     public static boolean isInCircle(double radius,double lat1, double lng1, double lat2, double lng2)
     {
         double radLat1 = rad(lat1);
         double radLat2 = rad(lat2);
         double a = radLat1 - radLat2;
         double b = rad(lng1) - rad(lng2);
         double s = 2 * Math.asin(Math.sqrt(Math.pow(Math.sin(a/2),2) + Math.cos(radLat1)*Math.cos(radLat2)*Math.pow(Math.sin(b/2),2)));
         s = s * EARTH_RADIUS;
         s = Math.round(s * 10000) / 10000;
         if(s > radius) {//不在圆上
             return false;
         }else {
             return true;
         }
     }

    /**
     * 是否在矩形区域内
     * @Title: isInArea
     * @Description: TODO()
     * @param lat 测试点经度
     * @param lng 测试点纬度
     * @param minLat 纬度范围限制1
     * @param maxLat 纬度范围限制2
     * @param minLng 经度限制范围1
     * @param maxLng 经度范围限制2
     * @return
     * @return boolean
     * @throws
     */
    public static boolean isInRectangleArea(double lat,double lng,double minLat,
                                            double maxLat,double minLng,double maxLng){
        if(isInRange(lat, minLat, maxLat)){//如果在纬度的范围内
            if(minLng*maxLng>0){
                if(isInRange(lng, minLng, maxLng)){
                    return true;
                }else {
                    return false;
                }
            }else {
                if(Math.abs(minLng)+Math.abs(maxLng)<180){
                    if(isInRange(lng, minLng, maxLng)){
                        return true;
                    }else {
                        return false;
                    }
                }else{
                    double left = Math.max(minLng, maxLng);
                    double right = Math.min(minLng, maxLng);
                    if(isInRange(lng, left, 180)||isInRange(lng, right,-180)){
                        return true;
                    }else {
                        return false;
                    }
                }
            }
        }else{
            return false;
        }
    }

    /**
     * 判断是否在经纬度范围内
     * @Title: isInRange
     * @Description: TODO()
     * @param point
     * @param left
     * @param right
     * @return
     * @return boolean
     * @throws
     */
    public static boolean isInRange(double point, double left,double right){
        if(point>=Math.min(left, right)&&point<=Math.max(left, right)){
            return true;
        }else {
            return false;
        }
    }

    /**
     * 判断点是否在多边形内
     * @Title: IsPointInPoly
     * @Description: TODO()
     * @param point 测试点
     * @param pts 多边形的点
     * @return
     * @return boolean
     * @throws
     */
    public static boolean isInPolygon(Point2D.Double point, List<Point2D.Double> pts){

        int N = pts.size();
        boolean boundOrVertex = true;
        int intersectCount = 0;//交叉点数量
        double precision = 2e-10; //浮点类型计算时候与0比较时候的容差
        Point2D.Double p1, p2;//临近顶点
        Point2D.Double p = point; //当前点

        p1 = pts.get(0);
        for(int i = 1; i <= N; ++i){
            if(p.equals(p1)){
                return boundOrVertex;
            }

            p2 = pts.get(i % N);
            if(p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)){
                p1 = p2;
                continue;
            }

            //射线穿过算法
            if(p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)){
                if(p.y <= Math.max(p1.y, p2.y)){
                    if(p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)){
                        return boundOrVertex;
                    }

                    if(p1.y == p2.y){
                        if(p1.y == p.y){
                            return boundOrVertex;
                        }else{
                            ++intersectCount;
                        }
                    }else{
                        double xinters = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
                        if(Math.abs(p.y - xinters) < precision){
                            return boundOrVertex;
                        }

                        if(p.y < xinters){
                            ++intersectCount;
                        }
                    }
                }
            }else{
                if(p.x == p2.x && p.y <= p2.y){
                    Point2D.Double p3 = pts.get((i+1) % N);
                    if(p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)){
                        ++intersectCount;
                    }else{
                        intersectCount += 2;
                    }
                }
            }
            p1 = p2;
        }
        if(intersectCount % 2 == 0){//偶数在多边形外
            return false;
        } else { //奇数在多边形内
            return true;
        }
    }
}

车辆定位数据实体类：

package entity;

public class p_cargis {
    public String cph;//车牌号
    public int cpys;//车牌颜色
    public String dt;//定位数据生成时间
    public double lat;//经度
    public double lon;//纬度
    public int xssd;//行驶速度


    public p_cargis() {
    }

    public String getCph(){
        return cph;
    }

    public int getCpys(){
        return cpys;
    }

    public String getDt() {
        return dt;
    }

    public double getLat() {
        return lat;
    }

    public double getLon() {
        return lon;
    }

    public int getXssd() {
        return xssd;
    }

    public void setCph(String cph) {
        this.cph = cph;
    }

    public void setCpys(int cpys) {
        this.cpys = cpys;
    }

    public void setDt(String dt) {
        this.dt = dt;
    }

    public void setLat(double lat) {
        this.lat = lat;
    }

    public void setLon(double lon) {
        this.lon = lon;
    }

    public void setXssd(int xssd) {
        this.xssd = xssd;
    }
}

将过滤出来的数据写入mysql，通过SinkToMySql类完成：

import entity.p_cargis;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;


/**
 * 将结果保存到数据库
 * */
public class SinkToMySql extends RichSinkFunction<p_cargis> {
    PreparedStatement ps;
    private Connection connection;

    /**
     * open() 方法中建立连接，这样不用每次 invoke 的时候都要建立连接和释放连接
     *
     * @param parameters
     * @throws Exception
     */
    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        connection = getConnection();
        String sql = "insert into T_ResultData(datatype, dataname, datavalue) values(?, ?, ?);";
        ps = this.connection.prepareStatement(sql);
    }

    @Override
    public void close() throws Exception {
        super.close();
        //关闭连接和释放资源
        if (connection != null) {
            connection.close();
        }
        if (ps != null) {
            ps.close();
        }
    }

    /**
     * 每条数据的插入都要调用一次 invoke() 方法
     *
     * @param value
     * @param context
     * @throws Exception
     */
    @Override
    public void invoke(p_cargis value, Context context) throws Exception {
        insertDB();
    }

    private void insertDB(p_cargis value) throws Exception {
        //组装数据，执行插入操作
        ps.setInt(1, value.getCpys());
        ps.setString(2, value.getCph());
        ps.setString(3, String.valueOf(value.getLat()) + "-" + String.valueOf(value.getLon()));
        ps.executeUpdate();
    }

    private static Connection getConnection() {
        Connection con = null;
        try {
            Class.forName("com.mysql.jdbc.Driver");
            con = DriverManager.getConnection("jdbc:mysql://172.16.170.74:3306/source_data?useUnicode=true&characterEncoding=UTF-8", "username", "password");
        } catch (Exception e) {
            System.out.println("-----------mysql get connection has exception , msg = "+ e.getMessage());
        }
        return con;
    }
}

整个项目结构如下：

编译发布生成jar包，然后到flink服务器上add上去：

 之后submit就可以运行起来了：

 使用一个测试程序，向kafka里写测试数据：

 很快mysql里产生了经过flink代码过滤出来的预警数据（不在指定区域内的车辆定位数据）：