flink 结合 springboot 实现单独运行 ( Java jar xxx.jar)

环境  springboot 2.7.6 + flink13.1

说明:  主要是想单独运行多个flink 项目, 小项目, 数据量少,充分使用 状态后端

运行  java -jar  xxx.jar 

1: 创建springboot项目: (pom 见下面)

实例1:  

/***************************
 *<pre>
 * @PACKAGE : com.sea.flinkboot.helloword
 *
 * @Author    :  Sea
 *
 * @Date    : 4/29/22 6:14 PM
 *
 * @Desc    :
 *</pre>
 ***************************/

/**
 * 方式1, 仅仅是希望通过 java -jar 单独启动
 */
@Slf4j
@Component
public class StreamWordCount {
    public static void main(String[] args) throws  Exception {
//        ParameterTool parameterTool = ParameterTool.fromArgs(args);
//        String host = parameterTool.get("host");
//        Integer port = parameterTool.getInt("port");
        StreamExecutionEnvironment evn = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> lineDSS = evn.socketTextStream("localhost", 9999);
//        DataStreamSource<String> lineDSS = evn.readTextFile("src/test/resource/input/words.txt");
        SingleOutputStreamOperator<Tuple2<String, Long>> sum = lineDSS.flatMap((String line, Collector<String> words) -> {
            Arrays.asList(line.split(" ")).forEach(words::collect);
        }).returns(Types.STRING).map(w -> Tuple2.of(w, 1L)).returns(Types.TUPLE(Types.STRING, Types.LONG)).keyBy(t -> t.f0).sum(1);
        sum.print();
        evn.execute();
    }

    /**
     * 开启一个单独的线程,避免堵塞其他项目启动
     */
    @PostConstruct
    public void start(){
        new Thread(()->{
            try {
                StreamWordCount.main(null);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }).run();

    }
}

鸡肋使用: (调用spring 容器中的方法)(在单个算子内部,启动一个ApplicationContext,然后获取bean )

@EnableAutoConfiguration
@MapperScan(basePackages = {"com.xxx.bigdata.xxx.mapper"})
public class SimpleSink extends RichSinkFunction<String> {
 
 
    TeacherInfoMapper teacherInfoMapper;
 
    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        SpringApplication application = new SpringApplication(SimpleSink.class);
        application.setBannerMode(Banner.Mode.OFF);
        ApplicationContext context = application.run(new String[]{});
        teacherInfoMapper = context.getBean(TeacherInfoMapper.class);
    }
 
    @Override
    public void close() throws Exception {
        super.close();
    }
 
    @Override
    public void invoke(String value, Context context) throws Exception {
        List<TeacherInfo> teacherInfoList = teacherInfoMapper.selectByPage(0, 100);
        teacherInfoList.stream().forEach(teacherInfo -> System.out.println("teacherinfo:" + teacherInfo.getTeacherId() + "," + teacherInfo.getTimeBit() + "," + teacherInfo.getWeek()));
    }
}

提示:

 1.替换log

 2. 添加log4j2.xml

依赖: pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.6.7</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.sea</groupId>
    <artifactId>sea-flinkboot</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>sea-flinkboot</name>
    <description>Demo project for Spring Boot</description>
    <properties>
        <java.version>1.8</java.version>
        <spring-cloud.version>Hoxton.SR6</spring-cloud.version>
        <flink.version>1.13.1</flink.version>
        <scala.binary.version>2.12</scala.binary.version>
    </properties>
    <dependencies>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
            <exclusions> <!--移除框架中的logback依赖-->
                <exclusion>
                    <groupId>org.springframework.boot</groupId>
                    <artifactId>spring-boot-starter-logging</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <!--Flink默认使用log4j作为具体的日志实现,与springboot 默认的logback 有冲突-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-log4j2</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.12</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- table and sql-->
        <!--
           <dependency>
               <groupId>org.apache.flink</groupId>
               <artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
               <version>${flink.version}</version>
           </dependency>
           <dependency>
               <groupId>org.apache.flink</groupId>
               <artifactId>flink-table-planner-blink_${scala.binary.version}</artifactId>
               <version>${flink.version}</version>
           </dependency>
           <dependency>
               <groupId>org.apache.flink</groupId>
               <artifactId>flink-table-common</artifactId>
               <version>${flink.version}</version>
           </dependency>-->

        <!-- hadoop client-->
     <!--           <dependency>
                    <groupId>org.apache.hadoop</groupId>
                    <artifactId>hadoop-client</artifactId>
                    <version>2.7.5</version>
                    <scope>provided</scope>
                </dependency>-->

        <!-- flink kafka -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!--Redis 连接器依赖-->
<!--        <dependency>
            <groupId>org.apache.bahir</groupId>
            <artifactId>flink-connector-redis_2.11</artifactId>
            <version>1.0</version>
        </dependency>-->

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch7_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!-- sink jdbc mysql-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.47</version>
        </dependency>

        <!--      <dependency>
                  <groupId>org.springframework.boot</groupId>
                  <artifactId>spring-boot-starter-web</artifactId>
              </dependency>-->

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>

        <!-- https://mvnrepository.com/artifact/com.ververica/flink-connector-mongodb-cdc -->
        <!--doc "https://ververica.github.io/flink-cdc-connectors/master/content/connectors/mongodb-cdc.html-->
<!--
        <dependency>
            <groupId>com.ververica</groupId>
            <artifactId>flink-connector-mongodb-cdc</artifactId>
            <version>2.2.0</version>
        </dependency>
-->

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.79</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-autoconfigure</artifactId>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-resources-plugin</artifactId>
            </plugin>
        </plugins>
        <!--如果需要集群发布部署,使用如下打包-->
        <!--<plugins>
            &lt;!&ndash; 编译插件 &ndash;&gt;
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.6.0</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>3.3.0</version>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                    <archive>
                        <manifest>
                            &lt;!&ndash; 可以设置jar包的入口类(可选) &ndash;&gt;
                            <mainClass>org.example.CmiotToCassandra</mainClass>
                        </manifest>
                    </archive>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>-->
    </build>

</project>

log4j2.xml

<?xml version="1.0" encoding="UTF-8"?>
<!-- status表示log4j2本身的日志信息打印级别，和下面的level，不是一个概念 -->
<!--TRACE < DEBUG < INFO < WARN < ERROR < FATAL < OFF-->
<Configuration status="OFF" monitorInterval="30">
    <Properties>
        <Property name="LOG_PATTERN">
            %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} -&#45;&#45; [%15.15t] %-40.40c{1.} : %m%n%ex
        </Property>
        <Property name="LOG_FILE_PATH">./logs/flinkboot</Property>
    </Properties>
    <Appenders>
        <Console name="ConsoleAppender" target="SYSTEM_OUT" follow="true">
            <PatternLayout pattern="${LOG_PATTERN}"/>
        </Console>
        <!-- Rolling File Appender -->
        <RollingFile name="FileAppender" fileName="${LOG_FILE_PATH}/sea-finkboot.log"
                     filePattern="${LOG_FILE_PATH}/spring-boot-log4j2-demo-%d{yyyy-MM-dd}-%i.log">
            <PatternLayout>
                <Pattern>${LOG_PATTERN}</Pattern>
            </PatternLayout>
            <!-- 只记录ERROR级别日志信息，程序打印的其他信息不会被记录 -->
            <!-- 此level设置的日志级别，是过滤日志文件中打印出的日志信息，和Root的level有所区别 -->
<!--            <Filters> <ThresholdFilter level="ERROR" onMatch="ACCEPT" onMismatch="DENY" /></Filters>-->
            <Policies>
                <SizeBasedTriggeringPolicy size="100MB" />
                <!-- 每天创建一个日志文件 -->
                <TimeBasedTriggeringPolicy interval="1" />
            </Policies>
            <DefaultRolloverStrategy max="20"/>
        </RollingFile>
    </Appenders>
    <Loggers>
        <!-- 用来单独指定日志的形式，比如要为指定包下的class指定不同的日志级别等。 -->
        <!--<Logger name="com.easyjijin.demo.springbootlog4j2" level="debug" additivity="false">
            <AppenderRef ref="ConsoleAppender" />
            <AppenderRef ref="FileAppender"/>
        </Logger>-->

        <!-- AsyncRoot - 异步记录日志 - 需要LMAXDisruptor的支持 -->
        <!-- <AsyncRootlevel="info" additivity="false">
          <AppenderRef ref="Console"/>
          <AppenderRef ref="FileAppender"/>
        </AsyncRoot> -->

        <!-- All < Trace < Debug < Info < Warn < Error < Fatal < OFF. -->
        　　　　 <!-- 程序会打印高于或等于所设置级别的日志，设置的日志等级越高，打印出来的日志就越少。-->
        <!-- 此level设置的日志级别，是过滤项目中输出的日志信息，和ThresholdFilter的level有所区别 -->
        <Root level="INFO">
            <AppenderRef ref="ConsoleAppender" level="INFO"/>
            <AppenderRef ref="FileAppender" level="error"/>
        </Root>
    </Loggers>
</Configuration>