巡检服务

[root@zabbixwxsec server_check]# cat start.sh 
#!/bin/bash
#start server check
#Tue Nov 13 17:47:03 CST 2018
#diao
#v1.0
#>server.information
#echo "server check start......"
#echo "服务器基础资源检查：" >>/root/server_check/server.information
#echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>/root/server_check/server.information
#cd /root/server_check/source_check;/usr/bin/python inspection.py >> /root/server_check/server.information
#echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++" >> /root/server_check/server.information
#echo >> /root/server_check/server.information
#echo >> /root/server_check/server.information
#echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++" >> /root/server_check/server.information
#echo "服务器基础进程检查：" >> /root/server_check/server.information
#cd /root/server_check/proc_check;/bin/bash check_proc.sh >> /root/server_check/server.information
#echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++" >> /root/server_check/server.information
#echo >> /root/server_check/server.information
#echo >> /root/server_check/server.information
#echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++" >> /root/server_check/server.information
#echo "服务器日志信息检查：" >> /root/server_check/server.information
#cd /root/server_check/log_check;/bin/bash log_analyse_test.sh >>  /root/server_check/server.information
#echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++" >> /root/server_check/server.information
#echo "server check finish......"



echo "server check start......"
echo "服务器基础资源检查："
echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++"
cd /root/server_check/source_check;/usr/bin/python inspection.py
echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echo ""
echo ""
echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echo "服务器基础进程检查："
cd /root/server_check/proc_check;/bin/bash check_proc.sh
echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echo ""
echo ""
echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echo "服务器日志信息检查："
cd /root/server_check/log_check;/bin/bash log_analyse_test.sh
echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echo "server check finish......"

cd /root/server_check/source_check;/usr/bin/python inspection.py

[root@zabbixwxsec source_check]# cat inspection.py 
#!/usr/bin/env python
# coding:utf8

import ConfigParser
import paramiko
import re
import threading
import time
import zipfile
import shutil
import os
import sys

#用来处理ssh连接
class Inspection(threading.Thread):
    #用于获取一个IP列表和备份应用列表
    def __init__(self):
        threading.Thread.__init__(self)
        cp = ConfigParser.SafeConfigParser()
        cp.read('cfg') 
        self.ip_list = cp.sections()
        self.backup_file = ["nginx", "oracle", "mysql", "tomcat"]
    
    #用于处理ssh连接
    def ssh_connection(self, host, port, user, pwd):
    try:
            ssh = paramiko.SSHClient()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(host, port, username=user, password=pwd)
            return ssh 
    except:
        print host,"无法登陆, 请在配置列表中将其删除后重试！"
        sys.exit(3)

    #用于命令执行
    def SSH_Command_execution(self, conn, cmd):
        stdin,stdout,stderr = conn.exec_command(cmd)
        return stdout.read()

    #用于日志文件传输
    def Log_file_transfer(self, ip, port, username, password, spath, dpath):
        ssh = paramiko.Transport((ip, int(port)))
    ssh.connect(username=username, password=password)
    sftp = paramiko.SFTPClient.from_transport(ssh)
    sftp.get(spath,dpath)
        ssh.close()        
        
    
    #用于读取配置文件的用户名和密码
    def Conf(self, ip):
        cp = ConfigParser.SafeConfigParser()
        cp.read('cfg')
        port = cp.get(ip,"port")
        user = cp.get(ip,"user")
        passwd = cp.get(ip,"password")
        return ip, port, user, passwd
    
    #用于巡检功能整合
    def scaffold(self, ip,  cmd):
        ip, port, user, passwd = self.Conf(ip)
        conn = self.ssh_connection(ip, int(port), user, passwd)
        res = self.SSH_Command_execution(conn, cmd)
    conn.close()
        return res

    #用于日志功能整合
    def Log_shipping(self, ip, spath, dpath):
    ip, port, user, passwd = self.Conf(ip)
    self.Log_file_transfer(ip, port, user, passwd, spath, dpath)

#各种巡检项
class Main_body(Inspection):
    def __init__(self):
        Inspection.__init__(self)
   
    def Log_Check(self, ip):
    now_time  = time.strftime('%Y%m%d')
        self.scaffold(ip, "/usr/bin/python /root/log.py")
    self.Log_shipping(ip, "/root/error_log/"+now_time+".zip", "/root/error_log/"+ip+".zip")
    z = zipfile.ZipFile("/root/error_log/"+ip+".zip", 'r')
    f = z.namelist()
    for name in f: 
            f_handle=open("/root/error_log/"+name,"wb") 
               f_handle.write(z.read(name))       
            f_handle.close() 
    z.close()
    shutil.move("/root/error_log/root/error_log/"+now_time+".log", "/root/error_log/"+ip+".log")
    os.remove("/root/error_log/"+ip+".zip")
    return "/root/error_log/"+ip+".log"

    def Cpu_Check(self, ip):
        _cmd = self.scaffold(ip, "top -b -n 1")
        used = re.search('(d.d)(s|\%)',_cmd).group(1)
        return used

    def Memory_Check(self, ip):
        _cmd = self.scaffold(ip, "free -m")
        if re.search('(.[a-z]+)(6|7)',self.scaffold(ip, "uname -a")).group(2) == "6":
            used = re.search('(buffers/cache:s+)([0-9]+)',_cmd).group(2)
            total = re.search('(Mem:s+)([0-9]+)',_cmd).group(2)
            count = float(used) / float( total) * 100
            return count
        elif re.search('(.[a-z]+)(6|7)',self.scaffold(ip, "uname -a")).group(2) == "7":
            used = re.search('(Mem:)s+(d+)s+(d+)',_cmd).group(3)
            total = re.search('(Mem:)s+(d+)s+(d+)',_cmd).group(2)
            count = float(used) / float(total) * 100
            return count

    def Disk_Check(self, ip):
        _cmd = self.scaffold(ip, "df -h")
        gen_used = re.search('(d+\%) (/)',_cmd).group(1)
        return gen_used
        
    def Backup_Check(self, ip):
        cp = ConfigParser.SafeConfigParser()
        cp.read('cfg')
    time_nowadays = time.strftime('%Y%m%d')
    for path in self.backup_file:
        try:
        if cp.get(ip, path):
            tmp_1 = cp.get(ip, path)
            file_path = tmp_1.format(time = time_nowadays)
            _cmd = self.scaffold(ip, "[ -e %s ] && echo $?" %file_path)

            if _cmd:
                    return True, path
            else:
                return False, path
        else:
                    pass
            except Exception as e:
            continue

    #数据库检查
    def Oracle_Check(self, ip): 
        pass
        
    #用于主体运行
    def run(self):
        for ip in self.ip_list:
            print ">>>"+ip
            print "+  cpu使用率为" + self.Cpu_Check(ip) + "%"
            print "+  内存使用率为%.1f" %self.Memory_Check(ip) + "%"
            print "+  磁盘使用率为" + self.Disk_Check(ip)

        try :
            status,path = self.Backup_Check(ip)
            if status:
            print "+  今天的%s备份文件存在" %path
            else:
            print "+  今天的%s备份文件不存在" %path
        except Exception as e:
            pass
        print  "+  错误日志存放在 %s" %self.Log_Check(ip)
        print
    print "当前时间是 %s" %(time.strftime('%Y-%m-%d %H:%m:%S'))


if __name__ == "__main__":
    s = Main_body()
    s.start()
    for t in threading.enumerate():  
        if t is threading.currentThread():
            continue
        t.join() 

[root@zabbixwxsec source_check]# cat cfg
#[10.16.183.191]
#user = root
#password = xxxxx
#port = 22
#oracle = "/opt/bak/file/10.191-tomcat7-{time}.tar.gz"

[10.12.16.7]
user = root
password = xxx
port = 22
mysql = "/home/bak/usms-db{time}.dmp"

[10.12.16.8]
user = root
password = xxxx
port = 22
mysql = "/home/bak/usms-db{time}.dmp"

[10.12.16.9]
user = root
password = xxxx
port = 22
tomcat = "/opt/bak/file/10.32-tomcat-{time}.tar.gz"

cd /root/server_check/proc_check;/bin/bash check_proc.sh

[root@zabbixwxsec proc_check]# cat check_proc.sh 
#!/bin/bash
#check server proc
#author
#Sun Jul 22 10:31:34 CST 2018
#Tue Nov 13 15:00:00 CST 2018(changed)
#v1.0--->v1.1

#get ip list and server information
/usr/bin/python /root/server_check/proc_check/ssh_server.py >proc.txt
cat /root/server_check/proc_check/proc.txt |awk -F"[" '{print $1}' >ip.txt

#output server information by regulation
while read line
do
    #get current server information by "grep" commond and output $line file
    cat proc.txt |grep "$line" > ${line}.txt
    #defining output format 
    echo ">>>>IP地址:$line"
    echo -e "proc		status"
    #get process nums
    nums=`cat /root/server_check/proc_check/proc| grep "$line" |awk -F" " '{print $2 }'`
    #proc 1
    proc_name=`cat /root/server_check/proc_check/proc| grep "$line" |awk -F" " '{print $3 }'`
    echo -ne "${proc_name}		"
    resu_name=`cat /root/server_check/proc_check/${line}.txt| grep $proc_name` &>/dev/null
    if [ $? -eq 0 ]; then
        echo "running"
    else
        echo "not running"
    fi
    if [ $nums -eq 1 ]; then
        continue
    fi
    #exit
    #proc 2
    proc_name=`cat /root/server_check/proc_check/proc| grep "$line" |awk -F" " '{print $4 }'`
    echo -ne "${proc_name}		"
    resu_name=`cat /root/server_check/proc_check/${line}.txt| grep $proc_name` &>/dev/null
    if [ $? -eq 0 ]; then
        echo "running"
    else
        echo "not running"
    fi
    if [ $nums -eq 2 ]; then
        continue
    fi
    #proc 3
    proc_name=`cat /root/server_check/proc_check/proc| grep "$line" |awk -F" " '{print $5 }'`
    echo -ne "${proc_name}		"
    resu_name=`cat /root/server_check/proc_check/${line}.txt| grep $proc_name` &>/dev/null
    if [ $? -eq 0 ]; then
        echo "running"
    else
        echo "not running"
    fi
    if [ $nums -eq 3 ]; then
        continue
    fi
    #proc 4
    proc_name=`cat /root/server_check/proc_check/proc| grep "$line" |awk -F" " '{print $6 }'`
    echo -ne "${proc_name}		"
    resu_name=`cat /root/server_check/proc_check/${line}.txt| grep $proc_name` &>/dev/null
    if [ $? -eq 0 ]; then
        echo "running"
    else
        echo "not running"
    fi
    if [ $nums -eq 4 ]; then
        continue
    fi
    #proc 5
    proc_name=`cat /root/server_check/proc_check/proc| grep "$line" |awk -F" " '{print $7 }'`
    echo -ne "${proc_name}		"
    resu_name=`cat /root/server_check/proc_check/${line}.txt| grep $proc_name` &>/dev/null
    if [ $? -eq 0 ]; then
        echo "running"
    else
        echo "not running"
    fi
    if [ $nums -eq 5 ]; then
        continue
    fi
    
done < ip.txt
rm -rf *.txt

[root@zabbixwxsec proc_check]# cat ssh_server.py 
#!/usr/bin/python
#encoding=utf8
#功能：实现服务器进程监控
#作者：彼岸花的微笑
#时间：Sun Jul 22 10:31:34 CST 2018
#版本：v1.0
import paramiko
import sys
import os

#读取服务器信息文件
def Check_proc(ipaddr,username,password):
    try:
        ssh=paramiko.SSHClient()
        ssh.load_system_host_keys()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(hostname=ipaddr,username=username,password=password,timeout=5)
        stdin, stdout, stderr = ssh.exec_command('ps -ef |egrep "tomcat|mongodb|oracle|grid|mysql|java" |grep -v "grep"')
        sys.stdout.write(ipaddr);print stdout.readlines()
        ssh.close()
    except:
        print ipaddr,"无法登录，请在配置文件中删除后重试!"
        sys.exit()


if __name__ == "__main__":
#    ret1 = Check_proc('10.16.183.191','root','xxx}')
    ret1 = Check_proc('10.12.16.7','root','xxx')
    ret1 = Check_proc('10.12.16.8','root','xxx')
    ret1 = Check_proc('10.12.16.9','root','xxx')

[root@zabbixwxsec proc_check]# cat proc 
#10.16.183.191 5 tomcat mongodb oracle mysql java
10.12.16.7   1 mysql
10.12.16.8   1 mysql
10.12.16.9   3 tomcat mongodb java

cd /root/server_check/log_check;/bin/bash log_analyse_test.sh

[root@zabbixwxsec log_check]# cat log_analyse_test.sh 
#!/bin/bash
#logs analyse
#Tue Nov 13 14:27:49 CST 2018
#diao
#get log messages and ip list
/usr/bin/python get_log_message.py >message.log
cat message.log |awk -F" " '{print $1}' |sort |uniq > ip.txt

while read line
do
    cat message.log |grep "^${line}" > "$line"_message.txt
    echo ">>>>>$line"
    #secure log host
    message=`cat "$line"_message.txt |grep "sec1" |awk -F"[" '{print $2}'|awk -F"]" '{print $1}'`
    if [ "$message" != "" ]; then
        echo "安全主机:存在多个访问该主机的IP地址，请注意查看！"
    else
        echo "安全主机:未发现访问该主机的IP地址！"
    fi
    #secure log error
    message=`cat "$line"_message.txt |grep "sec2" |awk -F"[" '{print $2}'|awk -F"]" '{print $1}'`
    if [ "$message" = "" ]; then
        echo "安全日志:未发现日志错误信息！"
    else
        echo "安全日志:发现日志错误信息，请及时处理！"
    fi
    #messages log error
    message=`cat "$line"_message.txt |grep "system" |awk -F"[" '{print $2}'|awk -F"]" '{print $1}'`
    if [ "$message" = "" ]; then
        echo "系统日志:未发现日志错误信息！"
    else
        echo "系统日志:发现日志错误信息，请及时处理！"
    fi
    #kernel log error
    message=`cat "$line"_message.txt |grep "dmesg" |awk -F"[" '{print $2}'|awk -F"]" '{print $1}'`
    if [ "$message" = "" ]; then
        echo "内核日志:未发现日志错误信息！"
    else
        echo "内核日志:发现日志错误信息，请及时处理！"
    fi
    #mail log error
    message=`cat "$line"_message.txt |grep "mail" |awk -F"[" '{print $2}'|awk -F"]" '{print $1}'`
    if [ "$message" = "" ]; then
        echo "邮件日志:未发现日志错误信息！"
    else
        echo "安全日志:发现日志错误信息，请及时处理！"
    fi
    #cron log error
    message=`cat "$line"_message.txt |grep "cron" |awk -F"[" '{print $2}'|awk -F"]" '{print $1}'`
    if [ "$message" = "" ]; then
        echo "定时日志:未发现日志错误信息！"
    else
        echo "安全日志:发现日志错误信息，请及时处理！"
    fi
    #boot log error
    message=`cat "$line"_message.txt |grep "boot" |awk -F"[" '{print $2}'|awk -F"]" '{print $1}'`
    if [ "$message" = "" ]; then
        echo "启动日志:未发现日志错误信息！"
    else
        echo "安全日志:发现日志错误信息，请及时处理！"
    fi
done < ip.txt
rm -rf *.txt
rm -rf message.log

[root@zabbixwxsec log_check]# cat get_log_message.py 
#!/usr/bin/python
#encoding=utf8
import sys
import paramiko
import os
test_fail='result.fail'
pass_file=open('passwd','r')
for line in pass_file:
    inform=line.split()
    ipaddr=inform[0]
    username=inform[1]
    password=inform[2]
    try:
        ssh=paramiko.SSHClient()
        ssh.load_system_host_keys()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(hostname=ipaddr,username=username,password=password,timeout=5)
        stdin, stdout, stderr = ssh.exec_command('egrep -o "([0-9]{1,3}.){3}[0-9]{1,3}" /var/log/secure |grep -v ":" |sort |uniq')
        sys.stdout.write(ipaddr+"	{sec1	}	");print stdout.readlines()
        stdin, stdout, stderr = ssh.exec_command('egrep "error|Error|Failed" /var/log/secure')
        sys.stdout.write(ipaddr+"	{sec2	}	");print stdout.readlines()
        stdin, stdout, stderr = ssh.exec_command('egrep "error|Error|Failed" /var/log/messages')
        sys.stdout.write(ipaddr+"	{system	}	");print stdout.readlines()
        stdin, stdout, stderr = ssh.exec_command('egrep "error|Error|Failed" /var/log/dmesg')
        sys.stdout.write(ipaddr+"	{dmesg	}	");print stdout.readlines()
        stdin, stdout, stderr = ssh.exec_command('egrep "error|Error|Failed|warn" /var/log/maillog')
        sys.stdout.write(ipaddr+"	{mail	}	");print stdout.readlines()
        stdin, stdout, stderr = ssh.exec_command('egrep "error|Error|Failed" /var/log/cron')
        sys.stdout.write(ipaddr+"	{cron	}	");print stdout.readlines()
        stdin, stdout, stderr = ssh.exec_command('egrep "error|Error|Failed" /var/log/boot.log')
        sys.stdout.write(ipaddr+"	{boot	}	");print stdout.readlines()
        ssh.close()
    except:
        print ipaddr,"无法登录，请在配置文件中删除后重试!"
        sys.exit()
pass_file.close()

[root@zabbixwxsec log_check]# cat passwd 
10.16.183.191 root xxx
10.12.16.7 root xx
10.12.16.8 root xxx
10.12.16.9 root xxx