4 案例演示 自定义镜像运行Nginx及tomcat服务并基于NFS实现动静分离

一 环境前期说明

准备好harbor和nfs服务器，nfs服务：172.31.7.122
harbor服务为 harbor.magedu.com，并且提前创建好项目

二 准备docker镜像

2.1 创建4个基础镜像，centos,nginx,tomcat,jdk

2.1.1 centos镜像

dockfile

[root@k8s-master1 centos]# cat Dockerfile 
#自定义Centos 基础镜像
FROM centos:7.9.2009 
MAINTAINER Jack.Zhang  2973707860@qq.com

ADD filebeat-7.12.1-x86_64.rpm /tmp
RUN yum install -y /tmp/filebeat-7.12.1-x86_64.rpm vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop &&  rm -rf /etc/localtime /tmp/filebeat-7.12.1-x86_64.rpm && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime 

构建和上传

[root@k8s-master1 centos]# cat build-command.sh 
#!/bin/bash
docker build -t  harbor.magedu.com/baseimages/magedu-centos-base:7.9.2009 .

docker push harbor.magedu.com/baseimages/magedu-centos-base:7.9.2009

2.1.2 jdk镜像构建

目录结构

dockfile

[root@k8s-master1 jdk-1.8.212]# cat Dockerfile 
#JDK Base Image
FROM harbor.magedu.com/baseimages/magedu-centos-base:7.9.2009 

MAINTAINER zhangshijie "zhangshijie@magedu.net"


ADD jdk-8u212-linux-x64.tar.gz /usr/local/src/
RUN ln -sv /usr/local/src/jdk1.8.0_212 /usr/local/jdk 
ADD profile /etc/profile


ENV JAVA_HOME /usr/local/jdk
ENV JRE_HOME $JAVA_HOME/jre
 
ENV CLASSPATH $JAVA_HOME/lib/:$JRE_HOME/lib/
ENV PATH $PATH:$JAVA_HOME/bin

构建并上传

#!/bin/bash
docker build -t harbor.magedu.com/pub-images/jdk-base:v8.212  .
sleep 1
docker push  harbor.magedu.com/pub-images/jdk-base:v8.212

2.1.3 nginx镜像构建

dockfile内容

[root@k8s-master1 nginx-base]# cat Dockerfile 
#Nginx Base Image
FROM harbor.magedu.com/baseimages/magedu-centos-base:7.9.2009 

MAINTAINER  zhangshijie@magedu.net

RUN yum install -y vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop
ADD nginx-1.20.2.tar.gz /usr/local/src/
RUN cd /usr/local/src/nginx-1.20.2 && ./configure  && make && make install && ln -sv  /usr/local/nginx/sbin/nginx /usr/sbin/nginx  &&rm -rf /usr/local/src/nginx-1.20.2.tar.gz 

构建并上传

#!/bin/bash
docker build -t harbor.magedu.com/pub-images/nginx-base:v1.20.2  .
sleep 1
docker push  harbor.magedu.com/pub-images/nginx-base:v1.20.2

2.1.4 tomcat镜像构建

dockfile内容：

[root@k8s-master1 tomcat-base-8.5.43]# cat Dockerfile 
#Tomcat 8.5.43基础镜像
FROM harbor.magedu.com/pub-images/jdk-base:v8.212 

MAINTAINER zhangshijie "zhangshijie@magedu.net"

RUN mkdir /apps /data/tomcat/webapps /data/tomcat/logs -pv 
ADD apache-tomcat-8.5.43.tar.gz  /apps
RUN useradd tomcat -u 2050 && ln -sv /apps/apache-tomcat-8.5.43 /apps/tomcat && chown -R tomcat.tomcat /apps /data

构建并上传

[root@k8s-master1 tomcat-base-8.5.43]# cat build-command.sh 
#!/bin/bash
docker build -t harbor.magedu.com/pub-images/tomcat-base:v8.5.43  .
sleep 3
docker push  harbor.magedu.com/pub-images/tomcat-base:v8.5.43

2.2 创建两个业务镜像，tomcat，nginx(k8s使用的)

目录结构

2.2.1 tomcat镜像构建

dockfile内容

#tomcat web1
FROM harbor.magedu.com/pub-images/tomcat-base:v8.5.43

ADD catalina.sh /apps/tomcat/bin/catalina.sh
ADD server.xml /apps/tomcat/conf/server.xml #配置文件
#ADD myapp/* /data/tomcat/webapps/myapp/
ADD app1.tar.gz /data/tomcat/webapps/myapp/ #代码目录
ADD run_tomcat.sh /apps/tomcat/bin/run_tomcat.sh #启动脚本
#ADD filebeat.yml /etc/filebeat/filebeat.yml 
RUN chown  -R tomcat.tomcat /data/ /apps/
#ADD filebeat-7.5.1-x86_64.rpm /tmp/
#RUN cd /tmp && yum localinstall -y filebeat-7.5.1-amd64.deb

EXPOSE 8080 8443

CMD ["/apps/tomcat/bin/run_tomcat.sh"]

构建并上传,这个脚本执行的时候需要传个参数，就是你代码的版本号

#!/bin/bash
TAG=$1
docker build -t  harbor.magedu.com/magedu/tomcat-app1:${TAG} .
sleep 3
docker push  harbor.magedu.com/magedu/tomcat-app1:${TAG}

2.2.2 nginx镜像构建

dockfile内容

#Nginx 1.20.2
FROM harbor.magedu.com/pub-images/nginx-base:v1.20.2 


RUN useradd tomcat -u 2050 
ADD nginx.conf /usr/local/nginx/conf/nginx.conf
ADD app1.tar.gz  /usr/local/nginx/html/webapp/ #静态文件路径
ADD index.html  /usr/local/nginx/html/index.html #静态文件路径

#静态资源挂载路径
RUN mkdir -p /usr/local/nginx/html/webapp/static /usr/local/nginx/html/webapp/images && chown tomcat.tomcat -R /usr/local/nginx/html/webapp/static /usr/local/nginx/html/webapp/images 

EXPOSE 80 443

CMD ["nginx"] 

构建并上传,也要传递个参数

[root@k8s-master1 nginx]# cat build-command.sh 
#!/bin/bash
TAG=$1
docker build -t harbor.magedu.com/magedu/nginx-web1:${TAG} .
echo "镜像构建完成，即将上传到harbor"
sleep 1
docker push harbor.magedu.com/magedu/nginx-web1:${TAG}
echo "镜像上传到harbor完成"

nginx.conf配置文件说明：我们要用nginx代理tomcat,所以server后面要写tomcat的svc地址

user  tomcat tomcat;
worker_processes  auto;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;
daemon off;

events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

upstream  tomcat_webserver {
        server magedu-tomcat-app1-service.magedu.svc.magedu.local:80; 
}

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   html;
            index  index.html index.htm;
        }

        location /webapp {
            root   html;
            index  index.html index.htm;
        }

        location /myapp {
             proxy_pass  http://tomcat_webserver;
             proxy_set_header   Host    $host;
             proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Real-IP $remote_addr;
        }
}

三 准备yaml文件

3.1 nginx的yaml

kubectl apply -f /yaml/docker/yaml/magedu/nginx/nginx.yaml

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: magedu-nginx-deployment-label
  name: magedu-nginx-deployment
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: magedu-nginx-selector
  template:
    metadata:
      labels:
        app: magedu-nginx-selector
    spec:
      containers:
      - name: magedu-nginx-container
        image: harbor.magedu.com/magedu/nginx-web1:202205041446  #nginx业务镜像地址
        #command: ["/apps/tomcat/bin/run_tomcat.sh"]
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 80
          protocol: TCP
          name: http
        - containerPort: 443
          protocol: TCP
          name: https
        env:
        - name: "password"
          value: "123456"
        - name: "age"
          value: "20"

        volumeMounts:
        - name: magedu-images
          mountPath: /usr/local/nginx/html/webapp/images
          readOnly: false
        - name: magedu-static
          mountPath: /usr/local/nginx/html/webapp/static
          readOnly: false
      volumes:
      - name: magedu-images
        nfs:
          server: 172.31.7.122
          path: /data/k8sdata/magedu/images 
      - name: magedu-static
        nfs:
          server: 172.31.7.122
          path: /data/k8sdata/magedu/static
      #nodeSelector:
      #  group: magedu

    

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: magedu-nginx-service-label
  name: magedu-nginx-service
  namespace: magedu
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
    nodePort: 30090
  - name: https
    port: 443
    protocol: TCP
    targetPort: 443
    nodePort: 30091
  selector:
    app: magedu-nginx-selector

3.2 tomcat的yaml

kubectl apply -f tomcat-app1.yaml

kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: magedu-tomcat-app1-deployment-label
  name: magedu-tomcat-app1-deployment
  namespace: magedu
spec:
  replicas: 2
  selector:
    matchLabels:
      app: magedu-tomcat-app1-selector
  template:
    metadata:
      labels:
        app: magedu-tomcat-app1-selector
    spec:
      containers:
      - name: magedu-tomcat-app1-container
        image: harbor.magedu.com/magedu/tomcat-app1:202205041153 
        #command: ["/apps/tomcat/bin/run_tomcat.sh"]
        imagePullPolicy: IfNotPresent
        #imagePullPolicy: Always
        ports:
        - containerPort: 8080
          protocol: TCP
          name: http
        env:
        - name: "password"
          value: "123456"
        - name: "age"
          value: "18"
        resources:
          limits:
            cpu: 1
            memory: "512Mi"
          requests:
            cpu: 500m
            memory: "512Mi"
        volumeMounts:
        - name: magedu-images
          mountPath: /usr/local/nginx/html/webapp/images
          readOnly: false
        - name: magedu-static
          mountPath: /usr/local/nginx/html/webapp/static
          readOnly: false
      volumes:
      - name: magedu-images
        nfs:
          server: 172.31.7.122
          path: /data/k8sdata/magedu/images
      - name: magedu-static
        nfs:
          server: 172.31.7.122
          path: /data/k8sdata/magedu/static
#      nodeSelector:
#        project: magedu
#        app: tomcat
---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: magedu-tomcat-app1-service-label
  name: magedu-tomcat-app1-service
  namespace: magedu
spec:
  #type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 8080
    #nodePort: 30092
  selector:
    app: magedu-tomcat-app1-selector

创建完两个yaml之后，查看结果

四 验证结果

最终通过域名访问，这里用haproxy，和keppalived产生的虚拟Ip

4.1 haproxy配置

将 www.mysite.com域名解析到172.31.7.188 ,更改host文件

4.2 访问后端tomcat内容

myapp是我后端tomcat提供的服务，有两个pod，默认轮训方式访问，如图所示：

4.3 访问nginx

4.3 访问nginx里的图片，存在于nfs存储上