一 整体结构
项目根pom
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <parent> <artifactId>dtsf</artifactId> <groupId>com.suning.dtsf</groupId> <version>RDRS-V3.8.0.rc</version> </parent> <modelVersion>4.0.0</modelVersion> <artifactId>dtsf-worker</artifactId> <packaging>pom</packaging> <modules> <module>dtsf-worker-core</module> <module>dtsf-worker-api</module> <module>dtsf-worker-writer-kafka</module> <module>dtsf-worker-reader-replformysql</module> <module>dtsf-worker-sink-reader</module> <module>dtsf-worker-sink-writer</module> <module>dtsf-worker-writer-hbase</module> <module>dtsf-worker-sink-pgwriter</module> </modules> </project>
二 core项目
1 package.xml
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd"> <id>worker</id> <formats> <format>dir</format> <format>tar.gz</format> </formats> <includeBaseDirectory>false</includeBaseDirectory> <fileSets> <fileSet> <directory>src/main/bin</directory> <outputDirectory>/bin</outputDirectory> <includes> <include>*.*</include> </includes> <directoryMode>775</directoryMode> </fileSet> <fileSet> <directory>../../dtsf-biz/target/classes/conf</directory> <outputDirectory>conf</outputDirectory> <includes> <include>*.*</include> </includes> </fileSet> </fileSets> <files> <file> <source>target/classes/logback.xml</source> <outputDirectory>conf</outputDirectory> </file> <file> <source>target/classes/worker.properties</source> <outputDirectory>conf</outputDirectory> </file> <file> <source>target/classes/javaopts</source> <outputDirectory>conf</outputDirectory> </file> </files> <dependencySets> <dependencySet> <useProjectArtifact>true</useProjectArtifact> <outputDirectory>/lib</outputDirectory> <scope>runtime</scope> <useTransitiveFiltering>true</useTransitiveFiltering> <excludes> <exclude>slf4j-nop*</exclude> </excludes> </dependencySet> </dependencySets> </assembly>
startup.sh
#!/bin/bash if [ -f "/etc/profile" ];then source /etc/profile fi current_path=`pwd` case "`uname`" in Linux) bin_abs_path=$(readlink -f $(dirname $0)) ;; *) bin_abs_path=`cd $(dirname $0); pwd` ;; esac base=${bin_abs_path}/.. worker_conf=$base/conf/worker.properties logback_configurationFile=$base/conf/logback.xml java_opts_file=$base/conf/javaopts export LANG=en_US.UTF-8 export BASE=$base if [ -e $java_opts_file ]; then source $java_opts_file fi if [ -f $base/bin/worker.pid ] ; then ps -fe | grep dtsf-worker | grep -v grep > /dev/null 2>&1 if [ $? -ne 0 ];then echo "dtsf worker is hung up" rm -f worker.pid else echo "found worker.pid , Please run stop.sh first ,then startup.sh" 2>&2 exit 1 fi fi if [ ! -d $base/logs/worker ] ; then mkdir -p $base/logs/worker fi if [ ! -d $base/logs/gc ] ; then mkdir -p $base/logs/gc fi ## set java path if [ -z "$JAVA" ] ; then JAVA=$(which java) fi if [ -z "$JAVA" ]; then echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2 exit 1 fi case "$#" in 0 ) ;; 1 ) var=$* if [ -f $var ] ; then worker_conf=$var else echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN." exit fi;; 2 ) var=$1 if [ -f $var ] ; then worker_conf=$var else if [ "$1" = "debug" ]; then DEBUG_PORT=$2 DEBUG_SUSPEND="y" JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND" fi fi;; * ) echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN." exit;; esac str=`file -L $JAVA | grep 64-bit` if [ -n "$JAVA_OPTS_CONF" ]; then JAVA_OPTS=$JAVA_OPTS_CONF elif [ -n "$str" ]; then JAVA_OPTS="-server -Xms32G -Xmx32G -Xmn2048m -XX:SurvivorRatio=2 -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${BASE}/logs/gc/gc-worker-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=20 -XX:GCLogFileSize=1024K" else JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${BASE}/logs/gc/gc-worker-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=20 -XX:GCLogFileSize=1024K" fi JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" # -Dcom.sun.management.jmxremote.port=9333 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false WORKER_OPTS="-DappName=dtsf-worker -Dlogback.configurationFile=$logback_configurationFile -Djava.opts.file=$java_opts_file -Dworker.conf=$worker_conf -Dworker.home=$base" if [ -e $worker_conf -a -e $logback_configurationFile ] then for i in $base/lib/*; do CLASSPATH=$i:"$CLASSPATH"; done CLASSPATH="$base/conf:$CLASSPATH"; echo "cd to $bin_abs_path for workaround relative path" cd $bin_abs_path echo LOG CONFIGURATION : $logback_configurationFile echo worker conf : $worker_conf echo CLASSPATH :$CLASSPATH echo JAVA_OPTS :$JAVA_OPTS $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $WORKER_OPTS -classpath .:$CLASSPATH com.suning.dtsf.worker.core.boot.WorkerBooter 1>>$base/logs/worker/worker.log 2>&1 & echo $! > $base/bin/worker.pid echo "cd to $current_path for continue" cd $current_path else echo "worker conf("$worker_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!" fi echo "worker has been started." #echo "append worker cron script" #if [ ! -f "/var/spool/cron/root" ];then # touch /var/spool/cron/root #fi #grep "startup.sh" /var/spool/cron/root #if [ $? -ne 0 ];then # cron_asterisk="* * * * *" # cron_content="$base/bin/startup.sh" # echo "$cron_asterisk sh $cron_content" >> /var/spool/cron/root # if [ -f "/etc/init.d/crond" ];then # /etc/init.d/crond restart # fi #fi
stop.sh
#!/bin/bash echo "uninstall cron" grep "startup.sh" /var/spool/cron/root if [ $? -eq 0 ];then sed -i "/startup.sh/d" /var/spool/cron/root if [ ! -f "/etc/init.d/crond" ];then /etc/init.d/crond restart fi fi cygwin=false; linux=false; case "`uname`" in CYGWIN*) cygwin=true ;; Linux*) linux=true ;; esac get_pid() { STR=$1 PID=$2 if $cygwin; then JAVA_CMD="$JAVA_HOMEinjava" JAVA_CMD=`cygpath --path --unix $JAVA_CMD` JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` else if $linux; then if [ ! -z "$PID" ]; then JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` else JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` fi else if [ ! -z "$PID" ]; then JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` else JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'` fi fi fi echo $JAVA_PID; } base=`dirname $0`/.. pidfile=$base/bin/worker.pid if [ ! -f "$pidfile" ];then echo "worker is not running. exists" exit fi pid=`cat $pidfile` if [ "$pid" == "" ] ; then pid=`get_pid "appName=dtsf-worker"` fi echo -e "`hostname`: stopping worker $pid ... " kill $pid LOOPS=0 while (true); do gpid=`get_pid "appName=dtsf-worker" "$pid"` if [ "$gpid" == "" ] ; then echo "Oook! cost:$LOOPS" if [ -f "$pidfile" ];then `rm $pidfile` fi break; fi let LOOPS=LOOPS+1 sleep 1 done
pom.xml
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <parent> <artifactId>dtsf-worker</artifactId> <groupId>com.suning.dtsf</groupId> <version>RDRS-V3.8.0.rc</version> </parent> <modelVersion>4.0.0</modelVersion> <artifactId>dtsf-worker-core</artifactId> <dependencies> ...... </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> <source>8</source> <target>8</target> </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> <version>3.1.0</version> <configuration> <archive> <manifest> <mainClass>com.ucar.datalink.worker.core.boot.WorkerBooter</mainClass> </manifest> </archive> <excludes> <exclude>**/logback.xml</exclude> <exclude>**/worker.properties</exclude> </excludes> </configuration> </plugin> <plugin> <artifactId>maven-assembly-plugin</artifactId> <!--<version>2.2.1</version>--> <configuration> <archive> <manifest> <mainClass>com.ucar.datalink.worker.core.boot.WorkerBooter</mainClass> </manifest> </archive> <finalName>dtsf</finalName> <outputDirectory>${project.parent.parent.build.directory}</outputDirectory> <descriptors> <descriptor>src/main/assembly/package.xml</descriptor> </descriptors> </configuration> <executions> <execution> <phase>package</phase> <goals> <goal>single</goal> </goals> </execution> </executions> </plugin> </plugins> </build> </project>
三 reader-mysql
package.xml
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd"> <id>reader-mysql</id> <formats> <format>dir</format> </formats> <includeBaseDirectory>false</includeBaseDirectory> <dependencySets> <dependencySet> <useProjectArtifact>true</useProjectArtifact> <outputDirectory>/</outputDirectory> <includes> <include>com.suning.dtsf:dtsf-worker-reader-replformysql</include> </includes> </dependencySet> <dependencySet> <useProjectArtifact>false</useProjectArtifact> <outputDirectory>/lib</outputDirectory> <scope>runtime</scope> <useTransitiveFiltering>true</useTransitiveFiltering> <excludes> <exclude>com.suning.dtsf:dtsf-worker-api</exclude> <exclude>fastjson*</exclude> <exclude>lz4*</exclude> <exclude>slf4j*</exclude> <exclude>log4j</exclude> <exclude>connect-api*</exclude> <exclude>kafka-clients*</exclude> <exclude>commons-logging*</exclude> <exclude>simpleclient*</exclude> </excludes> </dependencySet> </dependencySets> </assembly>
pom.xml
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <parent> <artifactId>dtsf-worker</artifactId> <groupId>com.suning.dtsf</groupId> <version>RDRS-V3.8.0.rc</version> </parent> <modelVersion>4.0.0</modelVersion> <artifactId>dtsf-worker-reader-replformysql</artifactId> <dependencies> ...... </dependencies> <build> <plugins> <plugin> <artifactId>maven-assembly-plugin</artifactId> <!--<version>2.2.1</version>--> <configuration> <descriptors> <descriptor>src/main/assembly/package.xml</descriptor> </descriptors> <finalName>reader-replformysql</finalName> <appendAssemblyId>false</appendAssemblyId> <outputDirectory>${project.parent.parent.build.directory}/dtsf-worker/plugin/reader </outputDirectory> </configuration> <executions> <execution> <phase>package</phase> <goals> <goal>single</goal> </goals> </execution> </executions> </plugin> </plugins> </build> </project>