集群资源队列监控：Grafana

集群资源队列监控：Grafana

common.sh

#!/bin/sh

. ~/.bashrc

home=$(cd `dirname $0`; cd ..; pwd)
bin_home=$home/bin
config_home=$home/conf
logs_home=$home/logs
lib_home=$home/lib
data_home=$home/data

cluster=$1
day=$2

if [ "${cluster}" = "" -o "${day}" = "" ]; then
    echo "请指定参数集群和日期(例：yth 20191203)："
    exit 0
fi

day_timestamp=`date -d "${day}" +%s`
day_timestamp_1=`date -d "${day} +1 day" +%s`
day_str0=`date -d @${day_timestamp} "+%Y-%m-%d"`
day_str1=`date -d @${day_timestamp} "+%Y/%m/%d"`
day_str2=`date -d @${day_timestamp_1} "+%Y-%m-%d"`job_parse_input_path

conf=${config_home}/${cluster}.conf

flink_home=`cat ${conf} | grep ^flink_home | awk -F"=" '{print $2}'`

fsimage_binary_path=`cat ${conf} | grep ^fsimage_binary_path | awk -F"=" '{print $2}'`
fsimage_txt_path=`cat ${conf} | grep ^fsimage_txt_path | awk -F"=" '{print $2}'`
fsimage_org_hdfs_path=`cat ${conf} | grep ^fsimage_org_hdfs_path | awk -F"=" '{print $2}'`
fsimage_detail_hdfs_path=`cat ${conf} | grep ^fsimage_detail_hdfs_path | awk -F"=" '{print $2}'`

fsimage_org_hdfs_path=${fsimage_org_hdfs_path}/day=${day}/cluster=${cluster}
fsimage_detail_hdfs_path=${fsimage_detail_hdfs_path}/day=${day}/cluster=${cluster}

yarn_queue_offline=`cat ${conf} | grep ^yarn_queue_offline | awk -F"=" '{print $2}'`

#作业解析参数
job_parse_input_path=`cat ${conf} | grep ^job_parse_input_path | awk -F"=" '{print $2}'`
job_parse_output_path=`cat ${conf} | grep ^job_parse_output_path | awk -F"=" '{print $2}'`

start-fsimage-detail.sh

#!/bin/sh

home=$(cd `dirname $0`; cd ..; pwd)
. ${home}/bin/common.sh

hdfs dfs -mkdir -p ${fsimage_detail_hdfs_path}
hdfs dfs -rm -r ${fsimage_detail_hdfs_path}

hadoop jar ${lib_home}/ops-fsimage-detail-2.0-release.jar \
com.djt.fsimage.detail.mrdetail.FsImageLogMR \
-Dmapreduce.job.queuename=${yarn_queue_offline} \
-DinputPath=${fsimage_org_hdfs_path} \
-DoutputPath=${fsimage_detail_hdfs_path} \
-Dday=${day} \
-DreduceCount=20

if [ "${hive_on}" == "true" ]
then
    hive -e "alter table xunshan.dwd_fsimage_detail add partition (day=${day}, cluster='${cluster}') location '${fsimage_detail_hdfs_path}'"
fi

start-fsimage-oiv.sh

#!/bin/sh

home=$(cd `dirname $0`; cd ..; pwd)
. ${home}/bin/common.sh

export HADOOP_HEAPSIZE=20000

fsimage_binary_name=`ls ${fsimage_binary_path} | grep ${cluster} | grep ${day}`

fsimage_binary_file=${fsimage_binary_path}/${fsimage_binary_name}

fsimage_txt_name=${fsimage_binary_name}.txt
fsimage_txt_file=${fsimage_txt_path}/${fsimage_txt_name}

hdfs oiv -p Delimited -i ${fsimage_binary_file} -o ${fsimage_txt_file}

hdfs dfs -mkdir -p ${fsimage_org_hdfs_path}
hdfs dfs -rm ${fsimage_org_hdfs_path}/${fsimage_txt_name}
hdfs dfs -put ${fsimage_txt_file} ${fsimage_org_hdfs_path}

echo -e "==========> load to hive start ==========>"
hive -e "ALTER TABLE xunshan.dwd_fsimage_org DROP IF EXISTS PARTITION(day=${day},cluster='${cluster}');"
hive -e "ALTER TABLE xunshan.dwd_fsimage_org ADD PARTITION(day=${day},cluster='${cluster}') LOCATION '${fsimage_org_hdfs_path}';"
echo -e "==========> load to hive end ==========>"

start-job-parse.sh

#!/bin/sh

home=$(cd `dirname $0`; cd ..; pwd)
. ${home}/bin/common.sh

${flink_home}/bin/flink run -c com.djt.job.parse.JobMain -m yarn-cluster -yjm 8192  -yn 10 -ys 4 -p 40 -ytm 8192 -yqu ${yarn_queue_offline} \
${lib_home}/ops-job-parse-2.0-release.jar \
--cluster ${cluster} \
--day ${day} \
--job.input.path ${job_parse_input_path} \
--job.output.path ${job_parse_output_path}

start-yarn-monitor.sh

#!/bin/sh

home=$(cd `dirname $0`; cd ..; pwd)
. ${home}/bin/common.sh

${flink_home}/bin/flink run -c com.djt.yarn.monitor.Main -m yarn-cluster -yjm 1024  -yn 1 -ys 1 -p 1 -ytm 1024 -yqu root.realtime \
${lib_home}/ops-yarn-monitor-2.0-release.jar \
--clusterNames cluster001 \
--cluster001.hosts ops2:8088,ops2:8088 \
--influxdb.url http://work1:9092 \
--influxdb.username admin \
--influxdb.password admin \
--influxdb.database yarn