rsync 的配置 一、服务端： 1.安装、 cd /home/rongda mkdir rsync [root@rongda rongda]# ls rsync/ log rsync.passwd run tool yum localinstall rsync -y 【默认安装的是rsync-3.1.2-4.el7.x86_64版本的】 2.配置 [root@backup ~]# vim /etc/rsyncd.conf uid = rsync gid = rsync port = 873 fake super = yes use chroot = no max connections = 200 timeout = 600 ignore errors read only = false list = false hosts allow = 0.0.0.0/0 auth users = rsync_backup secrets file = /home/rongda/rsync/rsync.passwd log file = /home/rongda/rsync/log/rsyncd.log pid file = /home/rongda/rsync/run/rsyncd.pid lock file = /home/rongda/rsync/run/rsyncd.lock ##################################### [backup] path = /backup #3.初始化的操作 3.1）创建Rsync进程运行的用户 （ rsync ） [root@backup ~]# useradd -M -s /sbin/nologin rsync [root@backup ~]# id rsync uid=1008(rsync) gid=1008(rsync) 组=1008(rsync) 3.2) 定义虚拟用户名rsync_bakcup的密码. mkdir /home/rongda/rsync/ echo "rsync_backup:123456" > /home/rongda/rsync/rsync.passwd chmod 600 /home/rongda/rsync/rsync.passwd 3.3) 创建存储的数据的目录 /backup [root@backup ~]# mkdir /backup [root@backup ~]# chown -R rsync.rsync /backup/ 3.4)创建启动时的日志目录及存放进程号的目录 mkdir /home/rongda/rsync/log/ mkdir /home/rongda/rsync/run #4.启动并检测 [root@backup ~]# systemctl start rsyncd [root@backup ~]# systemctl enable rsyncd [root@backup ~]# systemctl status rsyncd [root@backup ~]# netstat -lntp |grep 873 #5.客户端可进行一波测试 无需安装此服务 mkdir /home/rongda/rsync/ echo "123456" > /home/rongda/rsync/rsync.passwd chmod 600 /home/rongda/rsync/rsync.passwd [root@nfs ~]# rsync -avz --password-file=/home/rongda/rsync/rsync.passwd $Path rsync_backup@172.16.10.65::backup 二、客户端上传并推送数据 1.编写脚本:备份的脚本 vim scripts.sh #!/usr/bin/bash #0.将目录需要用的IP，时间，定义为变量 export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin Addr=$(hostname -I |awk '{print $1}') Time=$(date +%F) Path=/backup DirName="${Addr}_${Time}" current_time=`date +%F-%H-%M` #准备一个目录 mkdir -p "$Path/$DirName.full" #将文件备份到该目录中-mongodb /home/rongda/mongodb/bin/mongodump -h 192.168.254.101:27017 --authenticationDatabase admin -o $Path/$DirName.full/mongodb > /dev/null 2>&1 cp -rp /home/rongda/mongodb/data/ $Path/$DirName.full/mongodb/ > /dev/null 2>&1 #备份mysql的数据 mkdir $Path/$DirName.full/mysql/ #/usr/bin/mysqldump -uib_test -piB_test2019 ib_test >>$Path/$DirName.full/mysql/backupdb.sql > /dev/null 2>&1 /usr/bin/mysqldump -uroot -pGotoipo20!9 ib_test >>$Path/$DirName.full/mysql/ib_test.sql #备份fastdfs的数据 #cp -rp /home/rongda/fastdfs/file $Path/$DirName.full/fastdfs/ > /dev/null 2>&1 mkdir $Path/$DirName.full/fastdfs/ -p cp -rp /home/rongda/fastdfs/file $Path/$DirName.full/fastdfs/ #备份es的数据 cd /home/rongda/backup/es/ /bin/sh esdump.sh && mkdir $Path/$DirName.full/es/ && cp *.tar.gz $Path/$DirName.full/es/ > /dev/null 2>&1 && rm -fr /home/rongda/backup/es/esdump-$current_time*.tar.gz && rm -fr /home/rongda/backup/es/esdump-$current_time && cp -rp /home/rongda/elasticsearch-2.4.5/data $Path/$DirName.full/es/ > /dev/null 2>&1 #使用rsync命令进行远程传输 #rsync -avz --password-file=/home/rongda/rsync/rsync.passwd $Path/$DirName.full rsync_backup@192.168.254.102::backup 2.推送的脚本 [root@rongda rongda]# cat scripts_push.sh #!/usr/bin/bash #0.将目录需要用的IP，时间，定义为变量 export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin Addr=$(hostname -I |awk '{print $1}') Time=$(date +%F) Path=/backup DirName="${Addr}_${Time}" current_time=`date +%F-%H-%M` #1.我们要准备一个目录 mkdir -p $Path/check #2.将文件目录备份到该目录下 tar czf $Path/$DirName.full.tar.gz $Path/$DirName.full && md5sum $Path/$DirName.full.tar.gz > $Path/check/check_flag_$Time rm -fr $Path/$DirName.full #3.将文件进行传输使用rsync命令进行远程传输 rsync -avz --password-file=/home/rongda/rsync/rsync.passwd $Path/$DirName.full.tar.gz rsync_backup@192.168.254.102::backup rsync -avz --password-file=/home/rongda/rsync/rsync.passwd $Path/check/check_flag_$Time rsync_backup@192.168.254.102::backup 3.编写定时任务： [root@nfs backup]# crontab -l #backup * 01 * * * /usr/bin/sh /home/rongda/scripts.sh &>/dev/null #push * 02 * * * /usr/bin/sh /home/rongda/scripts_push.sh &>/dev/null 三、服务端检查客户端传过来的数据是否是完整的 [root@rongda rongda]# cat script_check_data.sh #!/usr/bin/bash #检查客户端推送过来的数据是否是完整的 #将目录定义为时间变量 export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin Time=$(date +%F) Path=/backup #1.找到当天的check_flag文件,校验数据是否是完整的，并且将校验结果存储至某个文件中 mkdir $Path/check_result find $Path/ -type f -name "check_flag_$Time" | xargs md5sum -c > $Path/check_result/result_$Time #2.将推送过来的校验值删除掉 find $Path/ -type f -name "check_flag_$Time" | xargs rm -fr #2.服务端仅保留6个月的备份数据,其余的全部删除 #find $Path -type d -mtime +180 | xargs rm -rf 编写定时任务： [root@rongda rongda]# crontab -l #check * 04 * * * /usr/bin/sh /home/rongda/script_check_data.sh &>/dev/null =========================================== es部分的备份 cd /home/rongda/backup/es vim esdump.sh #!/bin/bash es_host=192.168.254.101 es_port=9200 es_index=wisdom_doc #scp_url=192.168.6.230:/home/ current_time=`date +%F-%H-%M` mkdir ./esdump-$current_time /home/rongda/backup/es/esdump dump --host $es_host --port $es_port --index $es_index --directory /home/rongda/backup/es/esdump-$current_time --chunk 500 --format bulk_indexing tar -zcf esdump-$current_time.tar.gz esdump-$current_time #scp esdump-$current_time.tar.gz $scp_url