使用 mtools 搭架 mongodb 的自动化日志分析平台

mtools 是一款优秀的开源工具，是由 mongodb 公司的工程师个人维护的，在git上可以下载：
https://github.com/rueckstiess/mtools/tree/master

创建平台目录

# mkdir /apps
# cd /apps
# ls -l
总用量 16
-rw-r--r-- 1 root root  225 6月   5 18:20 log_done.log
-rwxr--r-- 1 root root  420 6月   5 10:24 log_done.sh
drwxr-xr-x 2 root root 4096 6月   5 18:00 mongodb_log_trade
drwxr-xr-x 2 root root    6 6月   5 18:28 mongodb_out_trade
drwxr-xr-x 2 root root  151 6月   5 18:25 mongodb_script

# tree /apps
/apps
├── log_done.log
├── log_done.sh
├── mongodb_log_trade
├── mongodb_out_trade
└── mongodb_script
    ├── mongodb_log_scp.sh
    ├── mongodb_mail_trade.sh
    └── mongodb_out_trade.sh


3 directories, 5 files

log_done.sh

# cat log_done.sh
#!/bin/bash

#20180601  peiyb  add mtools deal mongodb log
CDATE=`date +%Y%m%d`

#############################################
#
#mongodb 的日志
#
#############################################

/apps/mongodb_script/mongodb_log_scp.sh
/apps/mongodb_script/mongodb_out_trade.sh
/apps/mongodb_script/mongodb_mail_trade.sh

mongodb_log_scp.sh

需要事先配置好ssh到各mongodb机器的ssh免密登录。关于如何配置ssh信任可以参考另一篇博客：
https://blog.csdn.net/ctypyb2002/article/details/80572181

# cat mongodb_log_scp.sh 
#!/bin/bash

#20180601  peiyb  add mtools deal mongodb log
CDATE=`date +%Y%m%d`

#############################################
#
#拷贝 mongodb trade 的日志
#
#############################################

rm -f /apps/mongodb_log_trade/*

echo "`date ` scp mongodb trade log file"
scp root@192.168.56.100:/var/log/mongodb/mongod.log-$CDATE /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.100
scp root@192.168.56.101:/var/log/mongodb/mongod.log-$CDATE /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.101
scp root@192.168.56.102:/var/log/mongodb/mongod.log-$CDATE /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.102
scp root@192.168.56.103:/var/log/mongodb/mongod.log-$CDATE /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.103

iconv -f ascii -t gbk -c /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.100 -o /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.100-gbk
iconv -f ascii -t gbk -c /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.101 -o /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.101-gbk
iconv -f ascii -t gbk -c /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.102 -o /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.102-gbk
iconv -f ascii -t gbk -c /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.103 -o /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.103-gbk

rm -f /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.100
rm -f /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.101
rm -f /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.102
rm -f /apps/mongodb_log_trade/mongod.log-$CDATE-192.168.56.103

mongodb_out_trade.sh

# cat mongodb_out_trade.sh 
#!/bin/bash

#20180602  peiyb  add mtools deal mongodb log
CDATE=`date +%Y%m%d`
LOGBASEDIR='/apps/mongodb_log_trade'
OUTBASEDIR='/apps/mongodb_out_trade'

rm -f ${OUTBASEDIR}/*

#############################################
#
#使用 mtools mloginfo 处理 connections
#
#############################################

for LFILE in `ls ${LOGBASEDIR}/`
do
 echo $LFILE

 #基本信息分析
 /bin/mloginfo ${LOGBASEDIR}/${LFILE} --restarts --rsinfo --rsstate --connections --connstats --distinct  > ${OUTBASEDIR}/state-${LFILE}.txt

 #分析查询次数
 /bin/mloginfo ${LOGBASEDIR}/${LFILE} --queries --sort count > ${OUTBASEDIR}/query-${LFILE}.txt

 #分析超过10s的慢查询
 /bin/mlogfilter ${LOGBASEDIR}/${LFILE} --slow 10000 --shorten 200 > ${OUTBASEDIR}/slow_query_10s-${LFILE}.txt

 #分析慢扫描
 /bin/mlogfilter ${LOGBASEDIR}/${LFILE} --scan --shorten 200 > ${OUTBASEDIR}/slow_scan-${LFILE}.txt

done

mongodb_mail_trade.sh

# cat mongodb_mail_trade.sh 
#!/bin/bash

#20180602  peiyb  add mail send
CDATE=`date +%Y%m%d`
LOGBASEDIR='/apps/mongodb_log_trade'
OUTBASEDIR='/apps/mongodb_out_trade'
MAILLIST='peiyb@163.com'
ATTACHLIST=''

#############################################
#
#使用 linux mailx 发送 mongodb trade 生成的out文件
#
#############################################

#获取不同机器个数
for LFILE in `ls ${OUTBASEDIR}/ |grep -i "mongod.log"|cut -d- -f4 | sort | uniq`
do
    ATTACHLIST=''
    #拼接找出符合的文件
    for AFILE in `ls ${OUTBASEDIR}/ |grep -i $LFILE`
    do
        ATTACHLIST=$ATTACHLIST' -a '$OUTBASEDIR'/'$AFILE
    done
    #echo $ATTACHLIST

    #发送邮件
    echo "见附件" | /bin/mailx  -s "[db][mg001][log analyze][$LFILE]" $ATTACHLIST $MAILLIST   
done

添加 crontab

00 18 * * * /apps/log_done.sh >> /apps/log_done.log 2>&1

出现错误

this tool can't parse input from stdin.

mlogfilter: error: unrecognized arguments: /apps/mongodb_log_trade/mongod.log-20180605-192.168.56.100-gbk

晚上查找后需要通过 script 命令解决
script -c “[executable string]”

00 18 * * * script -c "/apps/log_done.sh >> /apps/log_done.log"

查看运行的进程

# ps -ef|grep -i apps |grep -v grep
root      6633     1  0 18:00 ?        00:00:00 script -c /apps/log_done.sh >> /apps/log_done.log
root      6634  6633  0 18:00 pts/3    00:00:00 sh -c /apps/log_done.sh >> /apps/log_done.log
root      6638  6634  0 18:00 pts/3    00:00:00 /bin/bash /apps/log_done.sh
root      6679  6638  0 18:00 pts/3    00:00:00 /bin/bash /apps/mongodb_script/mongodb_out_trade.sh
root      6683  6679 99 18:00 pts/3    00:03:02 /bin/python /bin/mloginfo /apps/mongodb_log_trade/mongod.log-20180605-192.168.56.100-gbk --restarts --rsstate --connections --connstats --distinct

参考：
https://github.com/rueckstiess/mtools/issues/304
https://stackoverflow.com/questions/1401002/trick-an-application-into-thinking-its-stdout-is-a-terminal-not-a-pipe