ansible安装etcd 的play-book配置
1.目录结构
roles/etcd ├── defaults │ └── main.yml ├── files │ ├── jwt_RS256 │ └── jwt_RS256.pub ├── README.md ├── tasks │ ├── configure.yml │ ├── install.yml │ └── main.yml └── templates ├── etcd.conf.j2 └── etcd.service.j2
2.主配置
etcd]# more etcd/defaults/main.ym
--- software_files_path: "/usr/local" software_install_path: "/usr/local/etcd" etcd_version: "3.3.22" etcdctl_api_version: "3" etcd_file: "etcd-v{{ etcd_version }}-linux-amd64.tar.gz" etcd_file_path: "{{ software_files_path }}/{{ etcd_file }}" etcd_file_url: "https://github.com/coreos/etcd/releases/download/v{{ etcd_version }}/{{ etcd_file }}" etcd_port: 2379 etcd_peer_port: 2380 etcd_home: "/data/etcd_data" etcd_datadir: "{{ etcd_home }}/data" etcd_confdir: "{{ etcd_home }}/conf" etcd_wardir: "{{ etcd_home }}/war" etcd_jwtkey: "{{ etcd_home }}/key" etcd_user: "apache" etcd_name: "etcd_cluster_0" etcd_service_name: "etcd{% if etcd_port != 2379 %}{{ etcd_port }}{% endif %}" etcd_initial_advertise_peer_urls: "http://{{ ansible_default_ipv4.address }}:{{ etcd_peer_port }}" etcd_listen_peer_urls: "http://{{ ansible_default_ipv4.address }}:{{ etcd_peer_port }}" etcd_listen_client_urls: "http://{{ ansible_default_ipv4.address }}:{{ etcd_port }},http://127.0.0.1:{{ etcd_port }}" etcd_advertise_client_urls: "http://{{ ansible_default_ipv4.address }}:2{{ etcd_port }}" etcd_trusted_ca_file: false etcd_cert_file: false etcd_key_file: false etcd_peer_trusted_ca_file: false etcd_peer_cert_file: false etcd_peer_key_file: false etcd_auto_tls: false etcd_discovery: false etcd_initial_cluster: false etcd_initial_cluster_token: "etcd_cluster" etcd_proxy: false etcd_force_new_cluster: false etcd_debug: false etcd_gomaxprocs: 1
# 秘钥文件
etcd]# more etcd/files/jwt_RS256
-----BEGIN RSA PRIVATE KEY----- MIIJKQIBAAKCAgEAqPkVQJ2ATlOk8p13rvExMWYH+i7b8keQea6CEGOdd+EPLQlq SJFf5FxkQ3TVF5m3SUP5smMqWVGV9AizQfcV7UpQFkAzcDmP5NODZDqVHBSZku5n +zw793eh6qxw1Swxjvn57b+BBvWmLzhFuZZWrD9dajt+0xztkDR/pbx8iQ3te3X4 Vq3iYr9IgqbkrnFb8FoL4sE3BE0PMtiMRln7S5WTel9gT3ruiuoT7E80MeDDTBhW RwbaZXBeuDWyZJt9HjOEkwpDyCOrILR5dslhJICQoT6/LAu14MRg5uHlPglJUbDU Zri/2ImG3xEqMz24RhTzlCEM2GY6m/hEunQSl4vIvpMDXHxji+AWeq5v78zT18cs rxLHoVC7SCBk3hZZZS41grFi5nY6bIA++DWK0AN+M1a2rgkNVEQ5GK67vA+iar9A uRH9MV6vMIkEqdEQ5RTmbzHHC9jwX05fdKfx338Y1gpWrS7ITEkBF1aG0mIbjAn4 xxx -----END RSA PRIVATE KEY----- etcd/files/jwt_RS256.pub -----BEGIN PUBLIC KEY----- MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAqPkVQJ2ATlOk8p13rvEx MWYH+i7b8keQea6CEGOdd+EPLQlqSJFf5FxkQ3TVF5m3SUP5smMqWVGV9AizQfcV xxx -----END PUBLIC KEY-----
etcd]# more etcd/tasks/configure.yml
--- - name: configure | Create etcd Directory. file: path={{ item }} state=directory owner={{ etcd_user }} group={{ etcd_user }} with_items: - "{{ etcd_datadir }}" - "{{ etcd_wardir }}" - "{{ etcd_confdir }}" - "{{ etcd_jwtkey }}" - name: Copy jwt public key file copy: src: jwt_RS256 dest: "{{etcd_jwtkey}}/jwt_RS256" owner: "{{ etcd_user }}" group: "{{ etcd_user }}" mode: '0644' - name: Copy jwt private key file copy: src: jwt_RS256.pub dest: "{{etcd_jwtkey}}/jwt_RS256.pub" owner: "{{ etcd_user }}" group: "{{ etcd_user }}" mode: '0644' - name: configure | Setup etcd.conf file. template: dest: "{{ etcd_confdir }}/etcd.conf" mode: 0644 src: etcd.conf.j2 owner: "{{ etcd_user }}" group: "{{ etcd_user }}" - block: - name: configure | Setup systemd etcd service. template: dest: "/usr/lib/systemd/system/{{ etcd_service_name }}.service" mode: 0644 src: etcd.service.j2 - name: "configure | Ensure {{ etcd_service_name }} is running for centos7." systemd: "name={{ etcd_service_name }} state=started enabled=true" ignore_errors: true - name: "configure | Wait {{ etcd_port }} port is opening." wait_for: port={{ etcd_port }} when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7" - name: configure | Change the initial cluster state to already exist. replace: path: "{{ etcd_confdir }}/etcd.conf" regexp: "initial-cluster-state: 'new'" replace: "initial-cluster-state: 'existing'"
etcd]# more etcd/tasks/install.yml
--- - name: install | Check if etcd local file is already configured. stat: path={{ etcd_file_path }} connection: local register: etcd_file_result - name: install | Create software directory. file: path={{ software_files_path }} state=directory connection: local when: not etcd_file_result.stat.exists - name: install | Download etcd file. get_url: url={{ etcd_file_url }} dest={{ software_files_path }} connection: local when: not etcd_file_result.stat.exists - name: install | Confirm the existence of the installation directory. file: path={{ software_install_path }} state=directory - name: install | Copy etcd file to agent. unarchive: src: "{{ etcd_file_path }}" dest: "{{ software_install_path }}" creates: "{{ software_install_path}}/etcd-v{{ etcd_version }}-linux-amd64" - name: install | Check if etcd remote soft link is already configured. stat: path="{{ software_install_path }}/etcd" register: etcd_soft_link_result - name: install | Create etcd dir soft link. file: "src={{ software_install_path }}/etcd-v{{ etcd_version }}-linux-amd64 dest={{ software_install_path }}/etcd state=link" when: not etcd_soft_link_result.stat.exists - name: install | Config environment variable. lineinfile: dest=/etc/profile line='export PATH={{ software_install_path }}/etcd:$PATH' - name: install | Config etcdctl API version environment variable. lineinfile: dest=/etc/profile line='export ETCDCTL_API={{ etcdctl_api_version }}' - name: install | Create etcd group. group: name={{ etcd_user }} - name: install | Create etcd user. user: name={{ etcd_user }} group=users shell=/bin/bash
# 配置和安装
# more etcd/tasks/main.yml
--- - import_tasks: install.yml - import_tasks: configure.yml
# 模板配置
etcd]# more etcd/templates/etcd.conf.j2
# This is the configuration file for the etcd server. # Human-readable name for this member. name: '{{ etcd_name }}' # Path to the data directory. data-dir: {{ etcd_datadir }} # Path to the dedicated wal directory. wal-dir: {{ etcd_wardir }} # Number of committed transactions to trigger a snapshot to disk. snapshot-count: 10000 # Time (in milliseconds) of a heartbeat interval. heartbeat-interval: 100 # Time (in milliseconds) for an election to timeout. election-timeout: 1000 # Raise alarms when backend size exceeds the given quota. 0 means use the # default quota. quota-backend-bytes: 0 # List of comma separated URLs to listen on for peer traffic. listen-peer-urls: {{ etcd_listen_peer_urls }} # List of comma separated URLs to listen on for client traffic. listen-client-urls: {{ etcd_listen_client_urls }} # Maximum number of snapshot files to retain (0 is unlimited). max-snapshots: 5 # Maximum number of wal files to retain (0 is unlimited). max-wals: 5 # Comma-separated white list of origins for CORS (cross-origin resource sharing). cors: # List of this member's peer URLs to advertise to the rest of the cluster. # The URLs needed to be a comma-separated list. initial-advertise-peer-urls: {{ etcd_initial_advertise_peer_urls }} # List of this member's client URLs to advertise to the public. # The URLs needed to be a comma-separated list. advertise-client-urls: {{ etcd_advertise_client_urls }} {% if etcd_discovery %} # Discovery URL used to bootstrap the cluster. discovery: {{ etcd_discovery }} # Valid values include 'exit', 'proxy' discovery-fallback: 'proxy' # HTTP proxy to use for traffic to discovery service. discovery-proxy: # DNS domain used to bootstrap initial cluster. discovery-srv: {% endif %} {% if etcd_initial_cluster %} # Initial cluster configuration for bootstrapping. initial-cluster: {{ etcd_initial_cluster }} # Initial cluster token for the etcd cluster during bootstrap. initial-cluster-token: '{{ etcd_initial_cluster_token }}' # auth token auth-token: 'jwt,pub-key={{ etcd_jwtkey }}/jwt_RS256.pub,priv-key={{ etcd_jwtkey }}/jwt_RS256,sign-method=RS256' # Initial cluster state ('new' or 'existing'). initial-cluster-state: 'new' {% endif %} # Reject reconfiguration requests that would cause quorum loss. strict-reconfig-check: false # Accept etcd V2 client requests enable-v2: true {% if etcd_proxy %} # Valid values include 'on', 'readonly', 'off' proxy: 'on' # Time (in milliseconds) an endpoint will be held in a failed state. proxy-failure-wait: 5000 # Time (in milliseconds) of the endpoints refresh interval. proxy-refresh-interval: 30000 # Time (in milliseconds) for a dial to timeout. proxy-dial-timeout: 1000 # Time (in milliseconds) for a write to timeout. proxy-write-timeout: 5000 # Time (in milliseconds) for a read to timeout. proxy-read-timeout: 0 {% endif %} client-transport-security: {% if etcd_trusted_ca_file %} # DEPRECATED: Path to the client server TLS CA file. ca-file: {{ etcd_trusted_ca_file }} {% endif %} {% if etcd_cert_file %} # Path to the client server TLS cert file. cert-file: {{ etcd_cert_file }} {% endif %} {% if etcd_key_file %} # Path to the client server TLS key file. key-file: {{ etcd_key_file }} {% endif %} {% if etcd_trusted_ca_file and etcd_cert_file and etcd_key_file %} # Enable client cert authentication. client-cert-auth: true {% endif %} {% if etcd_trusted_ca_file %} # Path to the client server TLS trusted CA key file. trusted-ca-file: {{ etcd_trusted_ca_file }} {% endif %} {% if etcd_auto_tls %} # Client TLS using generated certificates auto-tls: {{ etcd_auto_tls }} {% endif %} peer-transport-security: {% if etcd_peer_trusted_ca_file %} # DEPRECATED: Path to the peer server TLS CA file. ca-file: {{ etcd_peer_trusted_ca_file }} {% endif %} {% if etcd_peer_cert_file %} # Path to the peer server TLS cert file. cert-file: {{ etcd_peer_cert_file }} {% endif %} {% if etcd_peer_key_file %} # Path to the peer server TLS key file. key-file: {{ etcd_peer_key_file }} {% endif %} {% if etcd_peer_trusted_ca_file and etcd_peer_key_file and etcd_peer_cert_file %} # Enable peer client cert authentication. client-cert-auth: true {% endif %} {% if etcd_peer_trusted_ca_file %} # Path to the peer server TLS trusted CA key file. trusted-ca-file: {{ etcd_peer_trusted_ca_file }} {% endif %} {% if etcd_auto_tls %} # Peer TLS using generated certificates. auto-tls: {{ etcd_auto_tls }} {% endif %} # Enable debug-level logging for etcd. debug: {{ etcd_debug }} # Specify a particular log level for each etcd package (eg: 'etcdmain=CRITICAL,etcdserver=DEBUG'. log-package-levels: # Force to create a new one member cluster. force-new-cluster: {{ etcd_force_new_cluster }}
# systemctl 服务管理
etcd]# more etcd/templates/etcd.service.j2
[Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target [Service] Type=notify WorkingDirectory={{ etcd_datadir }} User={{ etcd_user }} TimeoutStartSec=10 # set GOMAXPROCS to number of processors ExecStart=/bin/bash -c "GOMAXPROCS={{ etcd_gomaxprocs }} {{ software_install_path }}/etcd/etcd --config-file="{{ etcd_confdir }}/etcd.conf"" Restart=on-failure LimitNOFILE=65536 [Install] WantedBy=multi-user.target
3.入口文件
ansible]# more etcd_cluster.yml
## 示例：ansible-playbook --ask-valut-pass -i online/yt etcd_cluster.yml # # 注意 etcd_name 的值 和 etcd_initial_cluster 中的名称值要一致 # 如: etcd_initial_cluster: "custorm-etcd01=http://x.x.x.x:2380,http://x.x.x.x:2380,http://x.x.x.x:2380" # etcd_name: "custorm-etcd01" - hosts: szlocal_etcd01 vars: - etcd_initial_cluster: "szlocal_etcd01=http://10.10.18.214:2380,szlocal_etcd02=http://192.168.11.36:2380,szlocal_etcd03=http://10.10.17.14:2380" roles: - { role: etcd, etcd_name: "szlocal_etcd01" } - hosts: szlocal_etcd02 vars: - etcd_initial_cluster: "szlocal_etcd01=http://10.10.18.214:2380,szlocal_etcd02=http://192.168.11.36:2380,szlocal_etcd03=http://10.10.17.14:2380" roles: - { role: etcd, etcd_name: "szlocal_etcd02" } - hosts: szlocal_etcd03 vars: - etcd_initial_cluster: "szlocal_etcd01=http://10.10.18.214:2380,szlocal_etcd02=http://192.168.11.36:2380,szlocal_etcd03=http://10.10.17.14:2380" roles: - { role: etcd, etcd_name: "szlocal_etcd03" }
# 主机列表
ansible]# more etcd_test_hosts
[szlocal_etcd] # test_etcd szlocal_etcd01 ansible_host=10.10.18.214 ansible_port=2016 ansible_user=root ansible_ssh_pass='pass' szlocal_etcd02 ansible_host=192.168.11.36 ansible_port=2016 ansible_user=root ansible_ssh_pass='pass' szlocal_etcd03 ansible_host=10.10.17.14 ansible_port=2016 ansible_user=root ansible_ssh_pass='pass'
4.常用的etcd命令
# 查看etcd的健康状态
``` export ETCDCTL_API=3 # 带认证 etcdctl --endpoints='http://172.18.44.39:2379,http://172.18.54.153:2379,http://172.18.54.154:2379' --user=root:111 endpoint health # 无认证 etcdctl --endpoints='http://172.25.130.150:2379,http://172.25.130.153:2379,http://172.25.130.154:2379' endpoint health etcdctl --endpoints='http://172.30.0.7:2379,http://172.30.0.8:2379,http://172.30.0.11:2379' endpoint health ``` # 开启用户认证，只需要在集群中任意一台上执行就行 创建用户认证 ``` etcdctl user add root etcdctl --endpoints=http://127.0.0.1:2379 user add root 会提示输入密码 ``` 查看用户 ``` # 无认证 etcdctl user list # 带认证 etcdctl --user root:123456 user list ``` # 给用户赋权限 ``` etcdctl user grant-role root root etcdctl user grant-role core_srv_user root etcdctl --endpoints=http://127.0.0.1:2379 --user=root:'123456' user add core_srv_user ``` # 默认是关闭的，需要开启 ``` etcdctl auth enable ``` # 测试写入信息 ``` etcdctl --endpoints=http://127.0.0.1:2379 --user=root:123456 put /test 'test' etcdctl --endpoints=http://127.0.0.1:2379 get /test etcdctl --endpoints=http://127.0.0.1:2379 del /test
# 使用playbook安装etcd
ansible-playbook --ask-valut-pass -i online/yt etcd_cluster.yml
秘钥生成：
# gen key pair for RS256
openssl genrsa -out jwt_RS256 4096
openssl rsa -in jwt_RS256 -pubout > jwt_RS256.pub
etcd server flags
--auth-token=jwt,pub-key=/srv/jwt_RS256.pub,priv-key=/srv/jwt_RS256,sign-method=RS256
https://github.com/etcd-io/etcd/issues/10144