阿里云基于弹性网卡实现高可用ip

一、环境介绍

阿里云服务器CentOS 7.9

[root@keep1 ~]# cat /etc/hosts
::1    localhost    localhost.localdomain    localhost6    localhost6.localdomain6
127.0.0.1    localhost    localhost.localdomain    localhost4    localhost4.localdomain4

172.16.208.195 keep1
172.16.208.196 keep2

二、创建阿里云弹性网卡

 

 

三、安装keepalived

yum install keepalived -y

 

四、安装安装自动配置eni网卡（2台都需安装）

阿里云的eni只对CentOS 7.3和CentOS 6.8能自动识别到网卡，其他版本需要配置网卡参数

官方文档
https://www.alibabacloud.com/help/zh/doc-detail/56955.htm?spm=a2c63.p38356.b99.398.5367779fyWo75G

 

1、安装自动配置eni网卡(系统需高于CentOS 7.3版本)

wget https://image-offline.oss-cn-hangzhou.aliyuncs.com/multi-nic-util/multi-nic-util-0.6.tgz
tar -zxvf multi-nic-util-0.6.tgz
cd multi-nic-util-0.6
bash install.sh

五、创建阿里云的accessKeyId和accessSecret，授权ECS管理权限（AliyunECSFullAccess）

 

六、go环境安装

go sdk安装文档:https://help.aliyun.com/document_detail/63640.htm?spm=5176.12573546.1318498.5.433c97a5QYvwVA
api文档:https://www.alibabacloud.com/help/zh/doc-detail/25485.htm?spm=a2c63.p38356.b99.657.25a92647ynVJk3

1、golang安装与代理配置

#golang安装
yum install -y golang 
#添加代理
go env -w GO111MODULE=on
go env -w GOPROXY=https://goproxy.cn,direct

七、创建eni项目

1、创建目录

mkdir  /go/eni/ -p

2、创建main脚本文件

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"
	"github.com/buger/jsonparser"
	"time"
)

type aliyunAuth struct {
	regionId string
	accessKeyId string
	accessSecret string
}


func (self aliyunAuth) ecsBindingEni(NetworkInterfaceId,InstanceId string) {
	client, err := ecs.NewClientWithAccessKey(self.regionId,self.accessKeyId,self.accessSecret)
	request := ecs.CreateAttachNetworkInterfaceRequest()
	request.Scheme = "https"
	request.NetworkInterfaceId = NetworkInterfaceId
	request.InstanceId = InstanceId
	response, err := client.AttachNetworkInterface(request)
	if err != nil {
		fmt.Print(err.Error())
	}
	fmt.Printf("response is %#v
", response)
}

func (self aliyunAuth) ecsRemoveEni(NetworkInterfaceId,InstanceId string) {
	client, err := ecs.NewClientWithAccessKey(self.regionId,self.accessKeyId,self.accessSecret)
	request := ecs.CreateDetachNetworkInterfaceRequest()
	request.Scheme = "https"
	request.NetworkInterfaceId = NetworkInterfaceId
	request.InstanceId = InstanceId

	response, err := client.DetachNetworkInterface(request)
	if err != nil {
		fmt.Print(err.Error())
	}
	fmt.Printf("response is %#v
", response)
}

func (self aliyunAuth) getEcsId( eni_ip string) string {
	client, err := ecs.NewClientWithAccessKey(self.regionId,self.accessKeyId,self.accessSecret)
	request := ecs.CreateDescribeNetworkInterfacesRequest()
	request.Scheme = "https"
	request.PrimaryIpAddress = eni_ip
	response, err := client.DescribeNetworkInterfaces(request)
	result, _ := json.Marshal(response)
	ecs_id, _ := jsonparser.GetString(result,"NetworkInterfaceSets","NetworkInterfaceSet","[0]","InstanceId")
	if err != nil {
		fmt.Print(err.Error())
	}
	fmt.Printf("response is %#v
", response)

	return ecs_id
}

func main() {
	var (
		behavior         = flag.String("b","attach","执行操作")
		eniInterfaceId   = flag.String("n", "", "弹性网卡Id")
		ecsId            = flag.String("e", "", "ECSId")
		eniIp            = flag.String("i", "", "弹性网卡Ip")

	)
	flag.Parse()

	var eni = new(aliyunAuth)
	eni.regionId = "cn-hangzhou"
	eni.accessKeyId = "xxxxxxxxxxxx"
	eni.accessSecret = "xxxxxxxxxxxxxx"

	switch *behavior {
	case "attach" :
		ecs_id := eni.getEcsId(*eniIp)
		if ecs_id == "" {
			eni.ecsBindingEni(*eniInterfaceId,*ecsId)
		}else {
			if *ecsId == ecs_id {
				break
			}
			eni.ecsRemoveEni(*eniInterfaceId,ecs_id)
			for i := 0;i < 10; i++ {
				time.Sleep(time.Millisecond*500)
				if eni.getEcsId(*eniIp) == ""{
					eni.ecsBindingEni(*eniInterfaceId,*ecsId)
					break
				}
			}

		}

	case "remove":
		eni.ecsRemoveEni(*eniInterfaceId,*ecsId)
	default:
		fmt.Println("Usage: { attach|remove }")

	}
}

　

3、go mod与go vendor

[root@keep1 ~]# cd /go/eni/
[root@keep1 eni]# ls
main.go
[root@keep1 eni]# go mod init  eni
go: creating new go.mod: module eni
[root@keep1 eni]# go mod vendor
go: finding module for package github.com/buger/jsonparser
go: finding module for package github.com/aliyun/alibaba-cloud-sdk-go/services/ecs
go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.865
go: downloading github.com/buger/jsonparser v1.1.1
go: found github.com/aliyun/alibaba-cloud-sdk-go/services/ecs in github.com/aliyun/alibaba-cloud-sdk-go v1.61.865
go: found github.com/buger/jsonparser in github.com/buger/jsonparser v1.1.1
go: downloading github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af
go: downloading github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742
go: downloading gopkg.in/ini.v1 v1.42.0
go: downloading github.com/json-iterator/go v1.1.5
go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd

4、编译

go build  -o eni   main.go 

八、keepalived检查脚本

[root@keep2 ~]# cat /etc/keepalived/check_eni.sh 
#!/bin/bash

KEEPALIVED_VIP=192.168.1.100
ENI_VIP=172.31.0.240

if [ `ip a s eth0|grep ${KEEPALIVED_VIP}|wc -l` -eq 1 ]; then
    if [ `ip a s eth1|grep $ENI_VIP|wc -l` -eq 1 ]; then
        ping -c 1 -w 1 $ENI_VIP >/dev/null
        [ $? -ne 0 ]&& exit 1
    fi    

fi

九、keepalived配置文件

1、主配置文件

! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc  #发送邮箱
   }
   notification_email_from Alexandre.Cassen@firewall.loc #邮箱地址
   smtp_server 127.0.0.1   #邮件服务器地址
   smtp_connect_timeout 30  
   router_id LVS_DEVEL-01   #主机名，每个节点不同即可
}

vrrp_script check_LVS {
    script "/etc/keepalived/check_eni.sh" #脚本路径
    interval 2 #2秒执行一次
    timeout 2  #超时时长
    weight -2  #脚本执行失败优先级降低2
    rise 3     #连续3次执行脚本成功表示ok
    fall 3     #连续3次脚本执行失败表示失败
}


vrrp_instance VI_1 {
    state MASTER   #节点主从，在另一个节点上为BACKUP
    interface eth0  #IP地址漂移到的网卡
    virtual_router_id 51  #多个节点必须相同
    priority 100   #优先级，备用节点的值必须低于主节点的值
    advert_int 1   #通告间隔1秒
    authentication { 
        auth_type PASS   #预共享密钥认证
        auth_pass 1111   #密钥
    }

unicast_src_ip 172.16.208.195  #本机地址
    unicast_peer {
        172.16.208.196    #目标地址
    }
    notify_master "/go/eni/eni -b attach -n 弹性网卡Id  -e ECSId -i 172.16.208.200"
    track_script {
        check_LVS
    }

    virtual_ipaddress {
        192.168.1.100/24  dev eth0  #VIP地址
    }
}

2、备配置

! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL-02
}
vrrp_script check_LVS {
    script "/etc/keepalived/check_eni.sh"
    interval 2 
    timeout 2  
    weight -2  
    rise 3    
    fall 3     
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }

unicast_src_ip 172.16.208.196
    unicast_peer {
        172.16.208.195
    }
    notify_master "/go/eni/eni -b attach -n 弹性网卡Id  -e ECSId -i 弹性网卡Ip"

    track_script {
        check_LVS
    }

    virtual_ipaddress {
        192.168.1.100/24 dev eth0 
    }
}

3、启动keepalived

systemctl  start keepalived.service 

十、测试

keep1服务器停止keepalived后查看eni的ip是否会绑定到keep2服务器上

1、keep1服务器操作

[root@keep1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:16:3e:12:e6:15 brd ff:ff:ff:ff:ff:ff
    inet 172.16.208.195/20 brd 172.16.223.255 scope global dynamic eth0
       valid_lft 315354146sec preferred_lft 315354146sec
    inet 192.168.1.100/24 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::216:3eff:fe12:e615/64 scope link 
       valid_lft forever preferred_lft forever
8: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:16:3e:16:d2:e5 brd ff:ff:ff:ff:ff:ff
    inet 172.16.208.200/20 brd 172.16.223.255 scope global dynamic eth1
       valid_lft 315359996sec preferred_lft 315359996sec
    inet6 fe80::216:3eff:fe16:d2e5/64 scope link 
       valid_lft forever preferred_lft forever
[root@keep1 ~]# systemctl  stop keepalived.service 

2、keep2服务器查看（弹性网卡已绑定到keep2服务器）

[root@keep2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:16:3e:16:c4:39 brd ff:ff:ff:ff:ff:ff
    inet 172.16.208.196/20 brd 172.16.223.255 scope global dynamic eth0
       valid_lft 315353960sec preferred_lft 315353960sec
    inet 192.168.1.100/24 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::216:3eff:fe16:c439/64 scope link 
       valid_lft forever preferred_lft forever
9: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:16:3e:16:d2:e5 brd ff:ff:ff:ff:ff:ff
    inet 172.16.208.200/20 brd 172.16.223.255 scope global dynamic eth1
       valid_lft 315359985sec preferred_lft 315359985sec
    inet6 fe80::216:3eff:fe16:d2e5/64 scope link 
       valid_lft forever preferred_lft forever