• 【Linux】【Services】【SaaS】Docker+kubernetes(10. 利用反向代理实现服务高可用)


    1. 简介

    1.1. 由于K8S并没有自己的集群,所以需要借助其他软件来实现,公司的生产环境使用的是Nginx,想要支持TCP转发要额外安装模块,测试环境中我就使用HAPROXY了

    1.2. 由于是做实现,我用Nginx又重新实现了一下HAPROXY的TCP转发,都会在本文中介绍

    1.3. Haproxy配置参数详解请参考我前面的文章:http://www.cnblogs.com/demonzk/p/6904029.html

    2. 环境

    2.1. 机器列表

    功能与组件 机器名 服务IP 管理IP VIP Processor Cores RAM Storage 备注
    HAPROXY HCTJOSDR01 10.30.2.48 172.16.0.48 172.16.0.148        
    HAPROXY HCTJOSDR02 10.30.2.49 172.16.0.49 172.16.0.149        

    2.2. 架构图

    2.3. 软件版本

    haproxy    1.5.18-6.el7

    keepalived    1.3.5-1.el7

    nginx      1.12.2-1.el7

    3. 安装与基础配置

    3.1. haproxy

    yum安装

    yum -y install haproxy

    配置haproxy日志,修改/etc/rsyslog.conf

    #去掉下面两行的注释
    $ModLoad imudp
    $UDPServerRun 514

    添加一个配置文件/etc/rsyslog.d/haproxy.conf

    local2.*                       /var/log/haproxy.log

    修改/etc/sysconfig/rsyslog

    #-r是允许接受外部日志
    #-c 是说兼容syslog v2
    #-m 是说每隔多长时间加一个时间戳,0表示不加
    SYSLOGD_OPTIONS="-r -c 2"

    修改haproxy配置文件,删掉没用的,添加状态监控页面

    global
        # to have these messages end up in /var/log/haproxy.log you will
        # need to:
        #
        # 1) configure syslog to accept network log events.  This is done
        #    by adding the '-r' option to the SYSLOGD_OPTIONS in
        #    /etc/sysconfig/syslog
        #
        # 2) configure local2 events to go to the /var/log/haproxy.log
        #   file. A line like the following can be added to
        #   /etc/sysconfig/syslog
        #
        #    local2.*                       /var/log/haproxy.log
        #
        log         127.0.0.1 local2
    
        chroot      /var/lib/haproxy
        pidfile     /var/run/haproxy.pid
        maxconn     4000
        user        haproxy
        group       haproxy
        daemon
    
        # turn on stats unix socket
        stats socket /var/lib/haproxy/stats
    
    #---------------------------------------------------------------------
    # common defaults that all the 'listen' and 'backend' sections will
    # use if not designated in their block
    #---------------------------------------------------------------------
    defaults
        mode                    http
        log                     global
        option                  httplog
        option                  dontlognull
        option http-server-close
        option forwardfor       except 127.0.0.0/8
        option                  redispatch
        retries                 3
        timeout http-request    10s
        timeout queue           1m
        timeout connect         10s
        timeout client          1m
        timeout server          1m
        timeout http-keep-alive 10s
        timeout check           10s
        maxconn                 3000
    
    #状态监控页面
    listen stats 0.0.0.0:9001
        stats enable
        stats uri /haproxyadmin?stats
        stats realm HAProxy Statistics
        stats auth admin:admin
        stats admin if TRUE

    启用

    systemctl start haproxy && systemctl enable haproxy

    3.1. 或者使用Nginx做TCP转发(本次使用Mysql做例子)

    安装

    yum -y install nginx

    在/etc/nginx/nginx.conf中添加下面这段

    stream {
    
        log_format tcp_proxy '$remote_addr [$time_local] '
                     '$protocol $status $bytes_sent $bytes_received '
                     '$session_time "$upstream_addr" '
                     '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
        include /etc/nginx/tcp.d/*.conf;
    }

    在/etc/nginx/tcp.d/mysql.hccos.cn.conf中写入如下内容

    server {
        listen          3306;
        proxy_connect_timeout 5s;
        proxy_timeout 30s;
            proxy_pass mysql;
        }
    upstream mysql {
        server 172.16.0.25:3306 max_fails=3 fail_timeout=10s;
        server 172.16.0.26:3306 max_fails=3 fail_timeout=10s;
        server 172.16.0.27:3306 max_fails=3 fail_timeout=10s;
    }

    3.2. keepalived安装

    yum安装

    yum -y install keepalived

    修改/etc/sysconfig/keepalived

    KEEPALIVED_OPTIONS="-D -d -S 0"

    修改/etc/rsyslog.d/keepalived.conf

    local0.*    /var/log/keepalived.log

    在两台机器上修改/etc/keepalived/keepalived.conf

    10.30.2.48

    global_defs {
       notification_email {
         eric.zhangtj@homecredit.cn
       }
       notification_email_from eric.zhangtj@homecredit.cn
       smtp_server 10.25.8.2
       smtp_connect_timeout 30
       router_id LVS_DEVEL
    }
    
    
    vrrp_script check_haproxy {
    script "killall -0 haproxy"
            interval 1
            weight 21
    }
    
    vrrp_script chk_mantaince_down {
       script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
       interval 1
       weight 2
    }
    
    
    vrrp_instance VI_148 {
        state MASTER
        interface ens192
        virtual_router_id 22
        garp_master_delay 1
        mcast_src_ip 172.16.0.48
        lvs_sync_daemon_interface ens192
        priority 110
        advert_int 2
        authentication {
            auth_type PASS
            auth_pass 1111
        }
        track_interface {
        ens192
        }
        virtual_ipaddress {
            172.16.0.148/24 dev ens192 label ens192:0
        }
        track_script {
        check_haproxy
        chk_mantaince_down
        }
    }
    
    vrrp_instance VI_149 {
        state BACKUP
        interface ens192
        virtual_router_id 23
        garp_master_delay 1
        mcast_src_ip 172.16.0.49
        lvs_sync_daemon_interface ens192
        priority 100
        advert_int 2
        authentication {
            auth_type PASS
            auth_pass 1111
        }
        track_interface {
            ens192
        }
        virtual_ipaddress {
            172.16.0.149/24 dev ens192 label ens192:1
        }
        track_script {
        check_haproxy
        chk_mantaince_down
        }
    }

    10.30.2.49

    global_defs {
       notification_email {
         eric.zhangtj@homecredit.cn
       }
       notification_email_from eric.zhangtj@homecredit.cn
       smtp_server 10.25.8.2
       smtp_connect_timeout 30
       router_id LVS_DEVEL
    }
    
    
    vrrp_script check_haproxy {
    script "killall -0 haproxy"
            interval 1
            weight 21
    }
    
    vrrp_script chk_mantaince_down {
       script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
       interval 1
       weight 2
    }
    
    vrrp_instance VI_148 {
        state BACKUP
        interface ens192
        virtual_router_id 22
        garp_master_delay 1
        mcast_src_ip 172.16.0.48
        lvs_sync_daemon_interface ens192
        priority 100
        advert_int 2
        authentication {
            auth_type PASS
            auth_pass 1111
        }
        track_interface {
        ens192
        }
        virtual_ipaddress {
            172.16.0.148/24 dev ens192 label ens192:0
        }
        track_script {
        check_haproxy
        chk_mantaince_down
        }
    }
    
    vrrp_instance VI_49 {
        state MASTER
        interface ens192
        virtual_router_id 23
        garp_master_delay 1
        mcast_src_ip 172.16.0.49
        lvs_sync_daemon_interface ens192
        priority 110
        advert_int 2
        authentication {
            auth_type PASS
            auth_pass 1111
        }
        track_interface {
            ens192
        }
        virtual_ipaddress {
            172.16.0.149/24 dev ens192 label ens192:1
        }
        track_script {
        check_haproxy
        chk_mantaince_down
        }
    }

    启用服务

    systemctl start keepalived && systemctl enable keepalived

    3.3. 内核参数

     修改/etc/sysctl.conf

    # Controls IP packet forwarding
    # 开启IP转发功能
    net.ipv4.ip_forward = 1
    
    # 开启允许绑定非本机的IP
    net.ipv4.ip_nonlocal_bind = 1
    sysctl -p
  • 相关阅读:
    并发编程-协程
    并发编程--线程
    并发编程--进程
    并发编程--操作系统介绍
    套接字Socket
    网络基础
    异常处理
    面向对象-常用模块
    面向对象进阶(反射)
    C#项目优化
  • 原文地址:https://www.cnblogs.com/demonzk/p/8404085.html
Copyright © 2020-2023  润新知