MySQL、MongoDB、Redis数据库Docker镜像制作
在多台主机上进行数据库部署时，如果使用传统的MySQL的交互式的安装方式将会重复很多遍。如果做成镜像，那么我们只需要make once，就run everywhere了。
本文的Dockerfile内容包括MySQL、MongoDB、Redis、JDK、FastDFS
Dockerfile样例
下面的样例中，在运行前需要将所需的软件包或文件夹放到tar目录下，其中还包括已经提前编译好的FastDFS。指定了数据库需要的存储文件夹（镜像之外的可挂载的目录）以及容器运行时对外开放的端口。还指定了容器启动后执行的脚本：entrypoint.sh
#Dockerfile for my specific purpose:ZH-Dist-Crawler
#pull base image
FROM ubuntu:14.04
#MAINTAINER abc
#the default user is root
#USER root
#add specific source to /etc/apt/sources.list mirrors.163.com
RUN echo 'deb http://mirrors.163.com/ubuntu/ trusty main restricted universe multiverse' > /etc/apt/sources.list &&
echo 'deb http://mirrors.163.com/ubuntu/ trusty-updates main restricted universe multiverse' >> /etc/apt/sources.list
#install some basic tools:ssh wget/curl .etc
# Remove unneeded /var/lib/apt/lists/* after install to reduce the docker image size (tens or hundrands of megabytes)
# -qq used for hide apt-get's output
RUN apt-get -qqy update ;
#apt-get install -y openssh-server wget ;
apt-get -y install gcc make
#NOTICE:comment must be in seperate line
#create directories
RUN
#mkdir -p /var/run/sshd ;
#mkdir -p /root/.ssh ;
#for saving dump.rdb and read redis.conf
mkdir -p /data/redis &&
mkdir -p /data/mongo/db /data/mongo/log &&
#FastDFS #tmp dir will be removed
mkdir -p /tmp/fdfs &&
mkdir -p /data/fdfs/tracker /data/fdfs/storage &&
#mkdir -p /etc/mysql &&
#when install mysql from ubuntu source,it will be created
mkdir -p /opt/redis &&
mkdir -p /opt/mongo
#install mysql from apt source
#RUN
# apt-get update &&
# DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server &&
# rm -rf /var/lib/apt/lists/* &&
# sed -i 's/^(bind-addresss.*)/# 1/' /etc/mysql/my.cnf &&
# sed -i 's/^(log_errors.*)/# 1/' /etc/mysql/my.cnf &&
# echo "mysqld_safe &" > /tmp/config &&
# echo "mysqladmin --silent --wait=30 ping || exit 1" >> /tmp/config &&
# echo "mysql -e 'GRANT ALL PRIVILEGES ON *.* TO "root"@"%" WITH GRANT OPTION;'" >> /tmp/config &&
# bash /tmp/config &&
# rm -f /tmp/config
#add files to image:redis mongodb fastdfs
#COPY is same as 'ADD' but without the tar and remote url handling.recomment to use COPY
COPY tar/fdfs /tmp/fdfs
COPY tar/redis /opt/redis
COPY tar/mongo /opt/mongo
#environment for reference from bootstrap.sh to start the servers
ENV REDIS_CONFIG /data/redis/redis.conf
#ENV MONGO_CONFIG /data/mongo/mongodb.conf
#ENV MYSQL_CONFIG /data/mysql/my.cnf
ENV FDFS_TRACKER_CONFIG /data/fdfs/tracker.conf
ENV FDFS_STORAGE_CONFIG /data/fdfs/storage.conf
#tar/fdfs directory should include the compiled x64 files,will be installed to /usr/bin
#install FastDFS and move configs to the /data/xxx directory,and this directory may also include data
#can be override by add the COLUME.
#WORKDIR /tmp/fdfs
RUN
#install to /usr/bin
cd /tmp/fdfs/libfastcommon &&
./make.sh install &&
cd /tmp/fdfs/FastDFS &&
./make.sh install &&
mv /tmp/fdfs/*.conf /data/fdfs/ &&
rm -rf /tmp/fdfs &&
#other configs
mv /opt/redis/redis.conf $REDIS_CONFIG
#specify the version
ENV JAVA_MAJOR 7
ENV JAVA_MINOR 80
ENV JAVA_HOME /opt/java/jdk1.$JAVA_MAJOR.0_$JAVA_MINOR
COPY tar/jdk-"$JAVA_MAJOR"u$JAVA_MINOR-linux-x64.tar.gz /jdk.tgz
RUN
JDK_FILE=/jdk.tgz &&
mkdir -p /opt/java &&
tar -xzf $JDK_FILE -C /opt/java &&
chmod ugo+x $JAVA_HOME/bin/* &&
rm $JDK_FILE
#install mysql from binary local file
COPY tar/mysql-5.5.49-linux2.6-x86_64.tar.gz /mysql.tgz
# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
RUN groupadd -r mysql && useradd -r -g mysql mysql
#libaio.so.1 is needed by mysqld to run
RUN
AR=/mysql.tgz
&& mkdir -p /usr/local/mysql
&& tar -xzf $AR -C /usr/local/mysql --strip-components=1
&& rm $AR
&& rm -rf /usr/local/mysql/mysql-test /usr/local/mysql/sql-bench
&& rm -rf /usr/local/mysql/bin/*-debug /usr/local/mysql/bin/*_embedded
&& find /usr/local/mysql -type f -name "*.a" -delete
&& apt-get -y update && apt-get install -y libaio1 binutils && rm -rf /var/lib/apt/lists/*
&& { find /usr/local/mysql -type f -executable -exec strip --strip-all '{}' + || true; }
&& apt-get purge -y --auto-remove binutils gcc make
&& apt-get clean
#change timezone to UTC+8 ,-f means force replace if exist destination file
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
# replicate some of the way the APT package configuration works,create running state dir before running mysqld,or will show error:Bind on unix socket: No such file or directory
RUN mkdir -p /etc/mysql/conf.d
&& {
echo '[mysqld]';
echo 'skip-host-cache';
echo 'skip-name-resolve';
echo 'user = mysql';
echo 'datadir = /var/lib/mysql';
echo '!includedir /etc/mysql/conf.d/';
} > /etc/mysql/my.cnf
&& mkdir -p /var/run/mysqld
&& chown mysql:mysql /var/run/mysqld
# can remove some pre-installed softwares such as python3
ENV REDIS /opt/redis
ENV MONGO /opt/mongo
# PATH
ENV PATH $PATH:$JAVA_HOME/bin:$REDIS:$MONGO/bin:/usr/local/mysql/bin:/usr/local/mysql/scripts
#define mountable directories
VOLUME ["/data/mongo"]
VOLUME ["/etc/mysql/conf.d", "/var/lib/mysql"]
VOLUME ["/data/redis"]
VOLUME ["/data/fdfs"]
COPY bootstrap.sh /entrypoint.sh
COPY mysql-entrypoint.sh /mysql-entrypoint.sh
#will always be excuted and can only be excuted in container
ENTRYPOINT ["/entrypoint.sh"]
# Define default command.
#CMD ["mysqld_safe"] #CMD is as params to ENTRYPOINT and can be override when run the image
#expose ports,the host can be equal to this,better to specific a different one,such as 6379+10001
# redis #mysql #mongodb #fdfs_tracker #fdfs_storage
EXPOSE 6379 3306 27017 22122 23000
启动脚本entrypoint.sh
#!/bin/bash
#run by docker ENTRYPOINYT
#NOTE that this script does not call interactive shell at last,so you must run 'docker exec <id or name> bash' to get the shell,and type Ctrl+D to exit that bash,but has no relation to this script's signal handling
# trap SIGTERM and call terminate() ,INT for Ctrl+C
trap terminate TERM
trap terminate INT
#####!!!WARNING!!!#####
#MUST MAKE SURE THAT THIS SCIRPT IS RUN AS PID 1, docker will send SIGTERM to pid 1 process only.Luckily, when run as ENTRYPOINT,its pid is 1,but shell would not forward signals to its children.
#The optional way is run Supervisord as pid 1 and use it to manage daemon services.
function terminate()
{
#should set a long time between SIGTERM and SIGKILL by docker stop ----time=30 (the default inteval is 10s)
echo "** Trapped SIGTERM(-15) or INT(-2),stopping DBs..."
# MONGOPID=`ps -ef | grep 'mongod' | grep -v grep | awk '{print $2}'`
# if [ ! -z $MONGOPID ]; then
#not null then send signal,kill default send -15(-TERM)
# kill -15 $MONGOPID
# fi
#killall send signal to process by name,if success no output,else output info to stderr
#killall: command not found,reason:in ubuntu:14.04 docker image,it does not include the cmd,use pkill instead
pkill mysqld #-15
pkill redis-server
#Perform a blocking SAVE if at least one save point is configured.
#if configured saving ratio and directory,kill -15 redis-server is same as call:
# redis-cli shutdown save
pkill fdfs_trackerd 2> /dev/null
pkill fdfs_storaged 2> /dev/null
pkill mongod
#the signal handling is asynchronized (multi process),so wait some time in the main process in case that the mongd terminate early than redis which is in other process.
#when main process exit without wait children,the docker container will exit,forcelly kill other process
#for that I have put 'wait' cmd in the last,there is no need to sleep a period of time
#sleep 5
}
function show_usage()
{
echo "Usage: $0 {master|slave} " #specIP
}
function start_mongo_bg()
{
echo "Starting MongoDB server"
ulimit -v unlimited
ulimit -n 100000
#echo never > /sys/kernel/mm/transparent_hugepage/enabled #should be run in host
#echo never > /sys/kernel/mm/transparent_hugepage/defrag
#--fork seems to fork tweence
mongod --quiet --dbpath=/data/mongo/db --logpath=/data/mongo/log/mongod.log & # --port 37017 --rest
}
function start_mysql_bg()
{
echo 'starting mysql'
/mysql-entrypoint.sh mysqld $MYSQL_PARAM &
}
function start_redis_bg()
{
#sysctl vm.overcommit_memory=1 #should be run in host
#no need to add '&' when set daemonize yes
redis-server $REDIS_CONFIG
}
#not used by now,maybe block the terminate() echo display
function looping()
{
while :; do
sleep 300
done
}
if [ $# -lt 1 ];then
show_usage
exit 0
fi
#add ip to eth0, set --cap-add=NET_ADMIN when use command 'docker run'
#echo "adding ip:$2 to eth0"
#/sbin/ip addr add $2 dev eth0
#param can be master/slave
case "$1" in
master)
echo 'start master dbs'
fdfs_trackerd $FDFS_TRACKER_CONFIG
start_mysql_bg
start_redis_bg
start_mongo_bg #run in background
#must use wait to wait children,or would not receive signal and handle through terminate()
echo 'waiting chidren exit...'
wait
echo 'docker container exit'
;;
slave)
echo 'start slave'
fdfs_storaged $FDFS_STORAGE_CONFIG
start_mongo_bg
echo 'waiting chidren exit...'
wait
echo 'docker container exit'
;;
*)
echo 'Usage: $0 {master|slave}'
exit 1
esac
#if the main thread exit and docker container will exit,so need to prevent it run as daemon
#looping
mysql-entrypoint.sh
#!/bin/bash
set -eo pipefail
#$@ is all of the parameters passed to the script : mysqld -xxx
# if command starts with an option, prepend mysqld,${VAR:start0:num} return a substr
if [ "${1:0:1}" = '-' ]; then
set -- mysqld "$@"
fi
# skip setup if they want an option that stops mysqld
wantHelp=
for arg; do
case "$arg" in
-'?'|--help|--print-defaults|-V|--version)
wantHelp=1
break
;;
esac
done
if [ ! -z "$wantHelp" ]; then
echo 'if you want to store the db data when container exit,you must use -v /path/externalDir:/var/lib/mysql'
echo 'else specify one of MYSQL_ROOT_PASSWORD, MYSQL_ALLOW_EMPTY_PASSWORD will install new mysql at startup'
echo '=========mongod --help==========='
echo `mongod --help`
exit 0
fi
DATADIR='/var/lib/mysql'
BASEDIR='/usr/local/mysql'
#-a:and -z:True if string is empty
if [ "$1" = 'mysqld' ]; then
# Get config
#DATADIR="$("$@" --verbose --help --log-bin-index=`mktemp -u` 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')"
#if not exit /var/lib/mysql/mysql dir then run mysql_install_db # -a -z "$MYSQL_RANDOM_ROOT_PASSWORD"
if [ ! -d "$DATADIR/mysql" ]; then
if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" ]; then
echo >&2 'error: database is uninitialized and password option is not specified '
echo >&2 ' You need to specify one of MYSQL_ROOT_PASSWORD, MYSQL_ALLOW_EMPTY_PASSWORD'
exit 1
fi
mkdir -p "$DATADIR"
chown -R mysql:mysql "$DATADIR"
echo 'Initializing database'
#下面的语句不要在末尾加2>&1 /dev/null可能会被识别为mysqld的参数，况且非bash可能不支持2>&1
mysql_install_db --user=mysql --datadir="$DATADIR" --rpm --basedir=$BASEDIR
echo 'Database initialized'
"$@" --skip-networking --basedir=$BASEDIR &
pid="$!"
mysql=( mysql --protocol=socket -uroot )
for i in {30..0}; do
if echo 'SELECT 1' | "${mysql[@]}" &> /dev/null; then
break
fi
echo 'MySQL init process in progress...'
sleep 1
done
if [ "$i" = 0 ]; then
echo >&2 'MySQL init process failed.'
exit 1
fi
if [ -z "$MYSQL_INITDB_SKIP_TZINFO" ]; then
# sed is for https://bugs.mysql.com/bug.php?id=20545
mysql_tzinfo_to_sql /usr/share/zoneinfo | sed 's/Local time zone must be set--see zic manual page/FCTY/' | "${mysql[@]}" mysql
fi
# if [ ! -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then
# MYSQL_ROOT_PASSWORD="$(pwgen -1 32)"
# echo "GENERATED ROOT PASSWORD: $MYSQL_ROOT_PASSWORD"
# fi
"${mysql[@]}" <<-EOSQL
-- What's done in this file shouldn't be replicated
-- or products like mysql-fabric won't work
SET @@SESSION.SQL_LOG_BIN=0;
DELETE FROM mysql.user ;
CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;
GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ;
FLUSH PRIVILEGES ;
EOSQL
if [ ! -z "$MYSQL_ROOT_PASSWORD" ]; then
mysql+=( -p"${MYSQL_ROOT_PASSWORD}" )
fi
if [ "$MYSQL_DATABASE" ]; then
echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" | "${mysql[@]}"
mysql+=( "$MYSQL_DATABASE" )
fi
if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then
echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" | "${mysql[@]}"
if [ "$MYSQL_DATABASE" ]; then
echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" | "${mysql[@]}"
fi
echo 'FLUSH PRIVILEGES ;' | "${mysql[@]}"
fi
echo
# for f in /docker-entrypoint-initdb.d/*; do
# case "$f" in
# *.sh) echo "$0: running $f"; . "$f" ;;
# *.sql) echo "$0: running $f"; "${mysql[@]}" < "$f"; echo ;;
# *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${mysql[@]}"; echo ;;
# *) echo "$0: ignoring $f" ;;
# esac
# echo
# done
if [ ! -z "$MYSQL_ONETIME_PASSWORD" ]; then
echo >&2
echo >&2 'Sorry, this version of MySQL does not support "PASSWORD EXPIRE" (required for MYSQL_ONETIME_PASSWORD).'
echo >&2
fi
#kill the mysqld
if ! kill -s TERM "$pid" || ! wait "$pid"; then
echo >&2 'MySQL init process failed.'
exit 1
fi
echo
echo 'MySQL init process done. Ready for start up.'
echo
fi
chown -R mysql:mysql "$DATADIR"
fi
#re start the mysqld
echo "formally start using cmd:$@"
exec "$@"
构建镜像.etc
构建镜像
docker build -t db:1.0 .
删除已经退出的容器
docker rm -v `docker ps -aq`
在ubuntu低版本，如12.04（内核小于3.8）上安装docker的方法(低版本的docker使用上会有一些问题)：
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys D8576A8BA88D21E9
sudo sh -c "echo deb http://mirror.yandex.ru/mirrors/docker/ docker main > /etc/apt/sources.list.d/docker.list"
sudo apt-get update
sudo apt-get install lxc-docker 可以加上--no-install-recommends避免安装git
总共安装了aufs-tools cgroup-lite liberror-perl lxc-docker lxc-docker-1.7.1
安装完docker的deb包后会自动创建docker组（groupadd docker）
并且启动docker，然后The UNIX socket /var/run/docker.sock is now readable and writable by members of the docker group.
可以将用户加入docker组来免去sudo权限：usermod -a -G docker user1 注销再登录就可以了（如果是ssh登录的，退出再重新登或者执行一次性的ssh shell命令就是登录后执行命令立刻退出）
镜像在节点之间的拷贝：
1.直接打包镜像docker save mynewimage > /tmp/mynewimage.tar
2.将容器保存为镜像再打包
docker commit 3a09b2588478 mynewimage
3.加载镜像 docker load < /tmp/mynewimage.tar
host主机上进行的设置
#docker容器中不能对内核的参数进行设置，是只读的，对于数据库需要调优的情况，需要在host上进行设置
#mongo
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
#redis
sysctl vm.overcommit_memory=1
run-image.sh启动镜像成为容器
#!/bin/bash
# --cap-add=NET_ADMIN 特殊权限，容许设置网络
#-d 后台#--net='host' 使用host的网络配置，hostname，interface完全一样，不是隔离的了
#-it name:1.0
#容器运行起来后再启动数据库：docker exec mycontainer /path/to/script
#first clear the exited containers,-v to remove associated volumns(really useful)
#function for get this script's dir,used for locate relative resource files
function get_script_dir ()
{
SOURCE="${BASH_SOURCE[0]}"
# While $SOURCE is a symlink, resolve it
while [ -h "$SOURCE" ]; do
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$( readlink "$SOURCE" )"
# If $SOURCE was a relative symlink (so no "/" as prefix, need to resolve it relative to the symlink base directory
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
return "$DIR"
}
param='master'
if [ "$1" == "slave" ];then
param='slave'
elif [ -z "$1" ];then
echo 'param: master/slave [bootstrap.sh] #last param specify if mount a external bootstrap.sh'
exit 0
fi
if [ ! -z "$2" ];then
VOL_OPT="-v $2:/entrypoint.sh"
fi
#or you maybe account with name confict
containers=`docker rm -v $(docker ps -a -q -f status=exited) 2> /dev/null`
if [ ! -z "$containers" ];then
echo 'removed exited container:'
echo $containers
fi
#--net='host' #-t表示分配一个伪终端
#决定采用默认桥接方式，只需要对host进行端口映射，爬虫程序中的ip地址仍使用host的ip。
#如果要在单机上测试，仍需要配置网络
extra_param="" #--cap-add=NET_ADMIN
PRE="$HOME/data"
if [ $param == 'master' ];then
NAME='zh'
hostname='zh-master'
names="--name $NAME -h $hostname"
# extra_param=$extra_param" --net=host"
# if set --net=host,the port mapping will be no use
PORT='-p 16379:6379 -p 13306:3306 -p 37017:27017 -p 22122:22122'
VOL="-v $PRE/mongo/:/data/mongo -v $PRE/redis:/data/redis -v $PRE/mysql/mysql/:/var/lib/mysql -v $PRE/mysql/conf.d/:/etc/mysql/conf.d -v $PRE/mysql/log/:/var/log/mysql -v $PRE/fdfs/:/data/fdfs"
ENVs='-e MYSQL_ROOT_PASSWORD=f '
else
NAME='zh-s'
#hostname='zh-'
names="--name $NAME"
extra_param=$extra_param" --link zh:zh-m"
PORT='-p 47017:27017 -p 23000:23000'
#mongo's data area cannot be conflict with the master's setting
VOL="-v $PRE/mongo.s/:/data/mongo -v $PRE/fdfs/:/data/fdfs"
fi
echo "container: $NAME starting.."
CID=$(docker run $extra_param
$names $ENVs $PORT $VOL $VOL_OPT
-d -t zhihu:1.0
$param
#192.168.1.10
#$1 $2
)
echo "id-> ${CID:0:5}"
#可以通过-e MYSQL_PARAM=--character-set-server=utf8mb4 --xxx给mysql传递参数
容器退出后可直接使用下述命令启动：
docker start 名字 / ID