openresty ngx_http_upstream_check_module 模块安装

实际上测试了tengine以及其他的ngx_http_upstream_check_module发现都不好使，反而发现ngx_healthcheck_module 值得使用

构建

基于docker，复用了openresty 官方的dockerfile 构建模式，核心流程如下
clone ngx_healthcheck_module 模块，patch 布丁到openresty，执行构建

• dockerfile
  集成了部分tengine的模块

 

# Dockerfile - alpine

# https://github.com/openresty/docker-openresty

 

ARG RESTY_IMAGE_BASE="alpine"

ARG RESTY_IMAGE_TAG="3.15"

 

FROM ${RESTY_IMAGE_BASE}:${RESTY_IMAGE_TAG}

 

LABEL maintainer="Evan Wies <evan@neomantra.net>"

 

# Docker Build Arguments

ARG RESTY_IMAGE_BASE="alpine"

ARG RESTY_IMAGE_TAG="3.15"

ARG RESTY_VERSION="1.19.9.1"

ARG RESTY_OPENSSL_VERSION="1.1.1n"

ARG RESTY_OPENSSL_PATCH_VERSION="1.1.1f"

ARG RESTY_OPENSSL_URL_BASE="https://www.openssl.org/source"

ARG RESTY_PCRE_VERSION="8.44"

ARG RESTY_PCRE_SHA256="aecafd4af3bd0f3935721af77b889d9024b2e01d96b58471bd91a3063fb47728"

ARG RESTY_J="1"

ARG RESTY_CONFIG_OPTIONS="\

    --with-compat \

    --with-file-aio \

    --with-http_addition_module \

    --with-http_auth_request_module \

    --with-http_dav_module \

    --with-http_flv_module \

    --with-http_geoip_module=dynamic \

    --with-http_gunzip_module \

    --with-http_gzip_static_module \

    --with-http_image_filter_module=dynamic \

    --with-http_mp4_module \

    --with-http_random_index_module \

    --with-http_realip_module \

    --with-http_secure_link_module \

    --with-http_slice_module \

    --with-http_ssl_module \

    --with-http_stub_status_module \

    --with-http_sub_module \

    --with-http_v2_module \

    --with-http_xslt_module=dynamic \

    --with-ipv6 \

    --with-mail \

    --with-mail_ssl_module \

    --with-md5-asm \

    --with-pcre-jit \

    --with-sha1-asm \

    --with-stream \

    --with-stream_ssl_module \

    --add-module=/tmp/ngx_healthcheck_module-master \

    --add-module=/tmp/tengine-2.3.3/modules/ngx_http_concat_module \

    --add-module=/tmp/tengine-2.3.3/modules/ngx_http_trim_filter_module \

    --add-module=/tmp/tengine-2.3.3/modules/ngx_http_footer_filter_module \ 

    --with-threads \

    "

ARG RESTY_CONFIG_OPTIONS_MORE=""

ARG RESTY_LUAJIT_OPTIONS="--with-luajit-xcflags='-DLUAJIT_NUMMODE=2 -DLUAJIT_ENABLE_LUA52COMPAT'"

 

ARG RESTY_ADD_PACKAGE_BUILDDEPS=""

ARG RESTY_ADD_PACKAGE_RUNDEPS=""

ARG RESTY_EVAL_PRE_CONFIGURE=""

ARG RESTY_EVAL_POST_MAKE=""

 

# These are not intended to be user-specified

ARG _RESTY_CONFIG_DEPS="--with-pcre \

    --with-cc-opt='-DNGX_LUA_ABORT_AT_PANIC -I/usr/local/openresty/pcre/include -I/usr/local/openresty/openssl/include' \

    --with-ld-opt='-L/usr/local/openresty/pcre/lib -L/usr/local/openresty/openssl/lib -Wl,-rpath,/usr/local/openresty/pcre/lib:/usr/local/openresty/openssl/lib' \

    "

 

LABEL resty_image_base="${RESTY_IMAGE_BASE}"

LABEL resty_image_tag="${RESTY_IMAGE_TAG}"

LABEL resty_version="${RESTY_VERSION}"

LABEL resty_openssl_version="${RESTY_OPENSSL_VERSION}"

LABEL resty_openssl_patch_version="${RESTY_OPENSSL_PATCH_VERSION}"

LABEL resty_openssl_url_base="${RESTY_OPENSSL_URL_BASE}"

LABEL resty_pcre_version="${RESTY_PCRE_VERSION}"

LABEL resty_pcre_sha256="${RESTY_PCRE_SHA256}"

LABEL resty_config_options="${RESTY_CONFIG_OPTIONS}"

LABEL resty_config_options_more="${RESTY_CONFIG_OPTIONS_MORE}"

LABEL resty_config_deps="${_RESTY_CONFIG_DEPS}"

LABEL resty_add_package_builddeps="${RESTY_ADD_PACKAGE_BUILDDEPS}"

LABEL resty_add_package_rundeps="${RESTY_ADD_PACKAGE_RUNDEPS}"

LABEL resty_eval_pre_configure="${RESTY_EVAL_PRE_CONFIGURE}"

LABEL resty_eval_post_make="${RESTY_EVAL_POST_MAKE}"

 

COPY ngx_healthcheck_module-master.tar.gz /tmp/ngx_healthcheck_module-master.tar.gz

 

RUN apk add --no-cache --virtual .build-deps \

        build-base \

        coreutils \

        curl \

        gd-dev \

        geoip-dev \

        libxslt-dev \

        linux-headers \

        make \

        perl-dev \

        readline-dev \

        zlib-dev \

        ${RESTY_ADD_PACKAGE_BUILDDEPS} \

    && apk add --no-cache \

        gd \

        geoip \

        libgcc \

        libxslt \

        zlib \

        ${RESTY_ADD_PACKAGE_RUNDEPS} \

    && cd /tmp \

    && if [ -n "${RESTY_EVAL_PRE_CONFIGURE}" ]; then eval $(echo ${RESTY_EVAL_PRE_CONFIGURE}); fi \

    && cd /tmp \

    && curl -fSL "${RESTY_OPENSSL_URL_BASE}/openssl-${RESTY_OPENSSL_VERSION}.tar.gz" -o openssl-${RESTY_OPENSSL_VERSION}.tar.gz \

    && tar xzf openssl-${RESTY_OPENSSL_VERSION}.tar.gz \

    && cd openssl-${RESTY_OPENSSL_VERSION} \

    && if [ $(echo ${RESTY_OPENSSL_VERSION} | cut -c 1-5) = "1.1.1" ] ; then \

        echo 'patching OpenSSL 1.1.1 for OpenResty' \

        && curl -s https://raw.githubusercontent.com/openresty/openresty/master/patches/openssl-${RESTY_OPENSSL_PATCH_VERSION}-sess_set_get_cb_yield.patch | patch -p1 ; \

    fi \

    && if [ $(echo ${RESTY_OPENSSL_VERSION} | cut -c 1-5) = "1.1.0" ] ; then \

        echo 'patching OpenSSL 1.1.0 for OpenResty' \

        && curl -s https://raw.githubusercontent.com/openresty/openresty/ed328977028c3ec3033bc25873ee360056e247cd/patches/openssl-1.1.0j-parallel_build_fix.patch | patch -p1 \

        && curl -s https://raw.githubusercontent.com/openresty/openresty/master/patches/openssl-${RESTY_OPENSSL_PATCH_VERSION}-sess_set_get_cb_yield.patch | patch -p1 ; \

    fi \

    && ./config \

      no-threads shared zlib -g \

      enable-ssl3 enable-ssl3-method \

      --prefix=/usr/local/openresty/openssl \

      --libdir=lib \

      -Wl,-rpath,/usr/local/openresty/openssl/lib \

    && make -j${RESTY_J} \

    && make -j${RESTY_J} install_sw \

    && cd /tmp \

    && curl -fSL https://downloads.sourceforge.net/project/pcre/pcre/${RESTY_PCRE_VERSION}/pcre-${RESTY_PCRE_VERSION}.tar.gz -o pcre-${RESTY_PCRE_VERSION}.tar.gz \

    && echo "${RESTY_PCRE_SHA256}  pcre-${RESTY_PCRE_VERSION}.tar.gz" | shasum -a 256 --check \

    && tar xzf pcre-${RESTY_PCRE_VERSION}.tar.gz \

    && cd /tmp/pcre-${RESTY_PCRE_VERSION} \

    && ./configure \

        --prefix=/usr/local/openresty/pcre \

        --disable-cpp \

        --enable-jit \

        --enable-utf \

        --enable-unicode-properties \

    && make -j${RESTY_J} \

    && make -j${RESTY_J} install \

    && cd /tmp \

    && curl -fSL https://openresty.org/download/openresty-${RESTY_VERSION}.tar.gz -o openresty-${RESTY_VERSION}.tar.gz \

    && curl -fSL https://github.com/alibaba/tengine/archive/refs/tags/2.3.3.tar.gz -o tengine-2.3.3.tar.gz \

    && tar xzf openresty-${RESTY_VERSION}.tar.gz \

    && tar xzf tengine-2.3.3.tar.gz \

    && tar xzf ngx_healthcheck_module-master.tar.gz\

    && cd /tmp/openresty-${RESTY_VERSION}/bundle/nginx-1.19.9 \

    && patch -p1  < /tmp/ngx_healthcheck_module-master/nginx_healthcheck_for_nginx_1.19+.patch \

    && cd /tmp/openresty-${RESTY_VERSION} \

    && eval ./configure -j${RESTY_J} ${_RESTY_CONFIG_DEPS} ${RESTY_CONFIG_OPTIONS} ${RESTY_CONFIG_OPTIONS_MORE} ${RESTY_LUAJIT_OPTIONS} \

    && make -j${RESTY_J} \

    && make -j${RESTY_J} install \

    && cd /tmp \

    && if [ -n "${RESTY_EVAL_POST_MAKE}" ]; then eval $(echo ${RESTY_EVAL_POST_MAKE}); fi \

    && rm -rf \

        openssl-${RESTY_OPENSSL_VERSION}.tar.gz openssl-${RESTY_OPENSSL_VERSION} \

        pcre-${RESTY_PCRE_VERSION}.tar.gz pcre-${RESTY_PCRE_VERSION} \

        openresty-${RESTY_VERSION}.tar.gz openresty-${RESTY_VERSION} \

        ngx_healthcheck_module-master.tar.gz ngx_healthcheck_module-master \

        tengine-2.3.3.tar.gz tengine-2.3.3 \

    && apk del .build-deps \

    && mkdir -p /var/run/openresty \

    && ln -sf /dev/stdout /usr/local/openresty/nginx/logs/access.log \

    && ln -sf /dev/stderr /usr/local/openresty/nginx/logs/error.log

 

# Add additional binaries into PATH for convenience

ENV PATH=$PATH:/usr/local/openresty/luajit/bin:/usr/local/openresty/nginx/sbin:/usr/local/openresty/bin

 

# Copy nginx configuration files

COPY nginx.conf /usr/local/openresty/nginx/conf/nginx.conf

COPY nginx.vh.default.conf /etc/nginx/conf.d/default.conf

 

CMD ["/usr/local/openresty/bin/openresty", "-g", "daemon off;"]

 

# Use SIGQUIT instead of default SIGTERM to cleanly drain requests

# See https://github.com/openresty/docker-openresty/blob/master/README.md#tips--pitfalls

STOPSIGNAL SIGQUIT

参考使用

• docker-compose

version: '3'

services:

  app:

    image: dalongrong/openresty-tengine:latest

    volumes:

      - "./nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf"

    privileged: true

    cap_add:

       - ALL

    ports:

      - "80:80"

  app2:

    image: dalongrong/openresty-tengine:latest

  app3:

    image: dalongrong/openresty-tengine:latest

• nginx.conf

worker_processes  1;

user root;  

events {

    worker_connections  1024;

}

http {

    include       mime.types;

    default_type  application/octet-stream;

    sendfile        on;

    keepalive_timeout  65;

    gzip  on;

    upstream cluster1 {

        # simple round-robin

        server app2:80;

        server app3:80;

        check interval=1000 rise=2 fall=5 timeout=1000 type=http;

        check_http_send "HEAD / HTTP/1.0\r\n\r\n";

        check_http_expect_alive http_2xx http_3xx;

    }  

    server {

        listen       80;

        server_name  localhost;

        charset utf-8;

        default_type text/html;

        location / {

            proxy_set_header Host $http_host;

            proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for; 

            client_body_buffer_size 10M;

            client_max_body_size 10G;

            proxy_buffers 1024 4k;

            proxy_read_timeout 300;

            proxy_connect_timeout 80;

            proxy_pass http://cluster1;  

        }

        location /status {

            healthcheck_status;

        }

        error_page   500 502 503 504  /50x.html;

        location = /50x.html {

            root   html;

        }    

    }    

}

• 效果
  当然直接prometheus metrics 可以直接使用

说明

我已经构建好了一个镜像，可以直接使用dalongrong/openresty-tengine:latest，还是得吐槽下tengine了问题不少，而且基本不维护了（好像今年又要开始维护了。。。。）,如果为了方便监控，推荐也集成上nginx 的vts模块,真得好好学学nginx 模块开发了，不然解决问题就有点费事了

参考资料

https://github.com/zhouchangxun/ngx_healthcheck_module
https://github.com/alibaba/tengine/tree/master/modules
https://github.com/vozlt/nginx-module-vts