环境 springboot 2.7.6 + flink13.1
说明: 主要是想单独运行多个flink 项目, 小项目, 数据量少,充分使用 状态后端
运行 java -jar xxx.jar
1: 创建springboot项目: (pom 见下面)
实例1:
/*************************** *<pre> * @PACKAGE : com.sea.flinkboot.helloword * * @Author : Sea * * @Date : 4/29/22 6:14 PM * * @Desc : *</pre> ***************************/ /** * 方式1, 仅仅是希望通过 java -jar 单独启动 */ @Slf4j @Component public class StreamWordCount { public static void main(String[] args) throws Exception { // ParameterTool parameterTool = ParameterTool.fromArgs(args); // String host = parameterTool.get("host"); // Integer port = parameterTool.getInt("port"); StreamExecutionEnvironment evn = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSource<String> lineDSS = evn.socketTextStream("localhost", 9999); // DataStreamSource<String> lineDSS = evn.readTextFile("src/test/resource/input/words.txt"); SingleOutputStreamOperator<Tuple2<String, Long>> sum = lineDSS.flatMap((String line, Collector<String> words) -> { Arrays.asList(line.split(" ")).forEach(words::collect); }).returns(Types.STRING).map(w -> Tuple2.of(w, 1L)).returns(Types.TUPLE(Types.STRING, Types.LONG)).keyBy(t -> t.f0).sum(1); sum.print(); evn.execute(); } /** * 开启一个单独的线程,避免堵塞其他项目启动 */ @PostConstruct public void start(){ new Thread(()->{ try { StreamWordCount.main(null); } catch (Exception e) { e.printStackTrace(); } }).run(); } }
鸡肋使用: (调用spring 容器中的方法)(在单个算子内部,启动一个ApplicationContext,然后获取bean )
@EnableAutoConfiguration @MapperScan(basePackages = {"com.xxx.bigdata.xxx.mapper"}) public class SimpleSink extends RichSinkFunction<String> { TeacherInfoMapper teacherInfoMapper; @Override public void open(Configuration parameters) throws Exception { super.open(parameters); SpringApplication application = new SpringApplication(SimpleSink.class); application.setBannerMode(Banner.Mode.OFF); ApplicationContext context = application.run(new String[]{}); teacherInfoMapper = context.getBean(TeacherInfoMapper.class); } @Override public void close() throws Exception { super.close(); } @Override public void invoke(String value, Context context) throws Exception { List<TeacherInfo> teacherInfoList = teacherInfoMapper.selectByPage(0, 100); teacherInfoList.stream().forEach(teacherInfo -> System.out.println("teacherinfo:" + teacherInfo.getTeacherId() + "," + teacherInfo.getTimeBit() + "," + teacherInfo.getWeek())); } }
提示:
1.替换log
2. 添加log4j2.xml
依赖: pom.xml
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-parent</artifactId> <version>2.6.7</version> <relativePath/> <!-- lookup parent from repository --> </parent> <groupId>com.sea</groupId> <artifactId>sea-flinkboot</artifactId> <version>0.0.1-SNAPSHOT</version> <name>sea-flinkboot</name> <description>Demo project for Spring Boot</description> <properties> <java.version>1.8</java.version> <spring-cloud.version>Hoxton.SR6</spring-cloud.version> <flink.version>1.13.1</flink.version> <scala.binary.version>2.12</scala.binary.version> </properties> <dependencies> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-runtime-web_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter</artifactId> <exclusions> <!--移除框架中的logback依赖--> <exclusion> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-logging</artifactId> </exclusion> </exclusions> </dependency> <!--Flink默认使用log4j作为具体的日志实现,与springboot 默认的logback 有冲突--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-log4j2</artifactId> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-clients_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-java</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-streaming-java_2.12</artifactId> <version>${flink.version}</version> </dependency> <!-- table and sql--> <!-- <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-table-planner-blink_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-table-common</artifactId> <version>${flink.version}</version> </dependency>--> <!-- hadoop client--> <!-- <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>2.7.5</version> <scope>provided</scope> </dependency>--> <!-- flink kafka --> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <!--Redis 连接器依赖--> <!-- <dependency> <groupId>org.apache.bahir</groupId> <artifactId>flink-connector-redis_2.11</artifactId> <version>1.0</version> </dependency>--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-elasticsearch7_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <!-- sink jdbc mysql--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-jdbc_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.47</version> </dependency> <!-- <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> </dependency>--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> <optional>true</optional> </dependency> <!-- https://mvnrepository.com/artifact/com.ververica/flink-connector-mongodb-cdc --> <!--doc "https://ververica.github.io/flink-cdc-connectors/master/content/connectors/mongodb-cdc.html--> <!-- <dependency> <groupId>com.ververica</groupId> <artifactId>flink-connector-mongodb-cdc</artifactId> <version>2.2.0</version> </dependency> --> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.2.79</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-autoconfigure</artifactId> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-maven-plugin</artifactId> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-resources-plugin</artifactId> </plugin> </plugins> <!--如果需要集群发布部署,使用如下打包--> <!--<plugins> <!– 编译插件 –> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <version>3.6.0</version> <configuration> <source>1.8</source> <target>1.8</target> <encoding>UTF-8</encoding> </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> <version>3.3.0</version> <configuration> <descriptorRefs> <descriptorRef>jar-with-dependencies</descriptorRef> </descriptorRefs> <archive> <manifest> <!– 可以设置jar包的入口类(可选) –> <mainClass>org.example.CmiotToCassandra</mainClass> </manifest> </archive> </configuration> <executions> <execution> <id>make-assembly</id> <phase>package</phase> <goals> <goal>single</goal> </goals> </execution> </executions> </plugin> </plugins>--> </build> </project>
log4j2.xml
<?xml version="1.0" encoding="UTF-8"?> <!-- status表示log4j2本身的日志信息打印级别，和下面的level，不是一个概念 --> <!--TRACE < DEBUG < INFO < WARN < ERROR < FATAL < OFF--> <Configuration status="OFF" monitorInterval="30"> <Properties> <Property name="LOG_PATTERN"> %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex </Property> <Property name="LOG_FILE_PATH">./logs/flinkboot</Property> </Properties> <Appenders> <Console name="ConsoleAppender" target="SYSTEM_OUT" follow="true"> <PatternLayout pattern="${LOG_PATTERN}"/> </Console> <!-- Rolling File Appender --> <RollingFile name="FileAppender" fileName="${LOG_FILE_PATH}/sea-finkboot.log" filePattern="${LOG_FILE_PATH}/spring-boot-log4j2-demo-%d{yyyy-MM-dd}-%i.log"> <PatternLayout> <Pattern>${LOG_PATTERN}</Pattern> </PatternLayout> <!-- 只记录ERROR级别日志信息，程序打印的其他信息不会被记录 --> <!-- 此level设置的日志级别，是过滤日志文件中打印出的日志信息，和Root的level有所区别 --> <!-- <Filters> <ThresholdFilter level="ERROR" onMatch="ACCEPT" onMismatch="DENY" /></Filters>--> <Policies> <SizeBasedTriggeringPolicy size="100MB" /> <!-- 每天创建一个日志文件 --> <TimeBasedTriggeringPolicy interval="1" /> </Policies> <DefaultRolloverStrategy max="20"/> </RollingFile> </Appenders> <Loggers> <!-- 用来单独指定日志的形式，比如要为指定包下的class指定不同的日志级别等。 --> <!--<Logger name="com.easyjijin.demo.springbootlog4j2" level="debug" additivity="false"> <AppenderRef ref="ConsoleAppender" /> <AppenderRef ref="FileAppender"/> </Logger>--> <!-- AsyncRoot - 异步记录日志 - 需要LMAXDisruptor的支持 --> <!-- <AsyncRootlevel="info" additivity="false"> <AppenderRef ref="Console"/> <AppenderRef ref="FileAppender"/> </AsyncRoot> --> <!-- All < Trace < Debug < Info < Warn < Error < Fatal < OFF. --> <!-- 程序会打印高于或等于所设置级别的日志，设置的日志等级越高，打印出来的日志就越少。--> <!-- 此level设置的日志级别，是过滤项目中输出的日志信息，和ThresholdFilter的level有所区别 --> <Root level="INFO"> <AppenderRef ref="ConsoleAppender" level="INFO"/> <AppenderRef ref="FileAppender" level="error"/> </Root> </Loggers> </Configuration>