kubeproxy

cmd\kube-proxy\proxy.go

func main() {
	// 新构建一个proxy运行命令，结合之前scheduler，我们可以看出k8s跑各组件的套路是一样的
	command := app.NewProxyCommand() --->cmd\kube-proxy\app\server.go
	code := cli.Run(command)
	os.Exit(code)
}

cmd\kube-proxy\app\server.go

NewProxyCommand
	cmd := &cobra.Command{
		Run: func(cmd *cobra.Command, args []string) {
		...
			if err := opts.Run(); err != nil {
				klog.ErrorS(err, "Error running ProxyServer")
				os.Exit(1)
			}
	}


Run
	...
	proxyServer, err := NewProxyServer(o) --->cmd\kube-proxy\app\server_others.go
	...
	return o.runLoop()


runLoop
	go func() {
		err := o.proxyServer.Run()
		o.errCh <- err
	}()

cmd\kube-proxy\app\server_others.go

// 由于我们一般都是在linux系统上跑k8s，所以代理开启的模式一般为ipvs或者iptables
func NewProxyServer(o *Options) (*ProxyServer, error) {
	return newProxyServer(o.config, o.CleanupAndExit, o.master)
}


**newProxyServer**
	...
	// 即便指定使用ipvs模式，仍然需要经过验证
	canUseIPVS, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface, config.IPVS.Scheduler) --->pkg\proxy\ipvs\proxier.go
	...
	if proxyMode == proxyModeIPTables {
		...
		proxier, err = iptables.NewDualStackProxier() --->pkg\proxy\iptables\proxier.go
	} else {
		proxier, err = ipvs.NewDualStackProxier() --->pkg\proxy\ipvs\proxier.go
	}

pkg\proxy\ipvs\proxier.go

// 该函数就是ipvs模式必须的一些模块的验证比对，如果失败则会降级成iptables模式
func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler string) (bool, error)

pkg\proxy\iptables\proxier.go

// kube-proxy已经支持并开启了ipv6
NewDualStackProxier
	ipv4Proxier, err := NewProxier(ipt[0], sysctl,
		exec, syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[0], hostname,
		nodeIP[0], recorder, healthzServer, ipFamilyMap[v1.IPv4Protocol])
	ipv6Proxier, err := NewProxier(ipt[1], sysctl,
		exec, syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[1], hostname,
		nodeIP[1], recorder, healthzServer, ipFamilyMap[v1.IPv6Protocol])
	return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil --->pkg\proxy\metaproxier\meta_proxier.go


var _ proxy.Provider = &Proxier{}


NewProxier
	go ipt.Monitor(kubeProxyCanaryChain, []utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter},
		proxier.syncProxyRules, syncPeriod, wait.NeverStop)


// 对比几个pkg\proxy\metaproxier\meta_proxier.go里的接口方法可以发现，具体实现是基于syncProxyRules设置具体实现的，这个方法居然有800多行...
// 这是iptables触发的所有代码了，其他配置iptables规则的地方只有在iptablesInit处了
// This is where all of the iptables-save/restore calls happen.
// The only other iptables rules are those that are setup in iptablesInit()
// This assumes proxier.mu is NOT held
// 这个方法体现出的问题有很多
// 1.代码量稍多，调用频繁效率低
// 2.有新变更时就要把以前的全扫一遍再跟新的合并然后落盘
// 3.iptables是规则链表，创建一条逻辑加了N条规则。随集群规模增加iptables规则链表会很多，查询效率低
// 4.逻辑上感觉处理的不太好，规则先全都加完再删
**syncProxyRules**
	// 开头直接上锁
	proxier.mu.Lock()
	defer proxier.mu.Unlock()
	...
	// 获取svc跟endpoint的变更信息
	serviceUpdateResult := proxier.serviceMap.Update(proxier.serviceChanges) --->pkg\proxy\service.go
	endpointUpdateResult := proxier.endpointsMap.Update(proxier.endpointsChanges) --->pkg\proxy\endpoints.go
	...
	// 对svc所用到ClusterIP/ExternalIP/LoadBalancerIP及NodePort都会有追踪清理工作
	for _, svcPortName := range endpointUpdateResult.StaleServiceNames {
		if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && conntrack.IsClearConntrackNeeded(svcInfo.Protocol()) {
			conntrackCleanupServiceIPs.Insert(svcInfo.ClusterIP().String())
			for _, extIP := range svcInfo.ExternalIPStrings() {
				conntrackCleanupServiceIPs.Insert(extIP)
			}
			for _, lbIP := range svcInfo.LoadBalancerIPStrings() {
				conntrackCleanupServiceIPs.Insert(lbIP)
			}
			nodePort := svcInfo.NodePort()
			if svcInfo.Protocol() == v1.ProtocolUDP && nodePort != 0 {
				conntrackCleanupServiceNodePorts.Insert(nodePort)
			}
		}
	}
	...
	// 如果失败了，则会按幂增长时间重试
	success := false
	defer func() {
		if !success {
			klog.InfoS("Sync failed", "retryingTime", proxier.syncPeriod)
			proxier.syncRunner.RetryAfter(proxier.syncPeriod)
		}
	}()
	...
	// 创建并链接kube链，没懂他每次还要这里双验证然后拼固定参数的含义，感觉完全可以优化掉成常量
	for _, jump := range iptablesJumpChains {
		...
		args := append(jump.extraArgs,
			"-m", "comment", "--comment", jump.comment,
			"-j", string(jump.dstChain),
		)
		...
	}
	...
	// 获取原本的的Filter、NAT规则并构造成字典
	existingFilterChains := make(map[utiliptables.Chain][]byte)
	proxier.existingFilterChainsData.Reset()
	err := proxier.iptables.SaveInto(utiliptables.TableFilter, proxier.existingFilterChainsData)
		existingFilterChains = utiliptables.GetChainLines(utiliptables.TableFilter, proxier.existingFilterChainsData.Bytes())
	existingNATChains := make(map[utiliptables.Chain][]byte)
	proxier.iptablesData.Reset()
	err = proxier.iptables.SaveInto(utiliptables.TableNAT, proxier.iptablesData)
		existingNATChains = utiliptables.GetChainLines(utiliptables.TableNAT, proxier.iptablesData.Bytes())
	...
	// 将原本的的Filter、NAT规则回写到对应buffer里
	proxier.filterChains.Write("*filter")
	proxier.natChains.Write("*nat")
	for _, chainName := range []utiliptables.Chain{kubeServicesChain, kubeExternalServicesChain, kubeForwardChain, kubeNodePortsChain} {
		if chain, ok := existingFilterChains[chainName]; ok {
			proxier.filterChains.WriteBytes(chain)
		} else {
			proxier.filterChains.Write(utiliptables.MakeChainLine(chainName))
		}
	}
	for _, chainName := range []utiliptables.Chain{kubeServicesChain, kubeNodePortsChain, kubePostroutingChain, KubeMarkMasqChain} {
		if chain, ok := existingNATChains[chainName]; ok {
			proxier.natChains.WriteBytes(chain)
		} else {
			proxier.natChains.Write(utiliptables.MakeChainLine(chainName))
		}
	}
	...
	// 构造postrouting规则写到buffer里
	proxier.natRules.Write(
		"-A", string(kubePostroutingChain),
		"-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", proxier.masqueradeMark, proxier.masqueradeMark),
		"-j", "RETURN",
	)
	proxier.natRules.Write(
		"-A", string(kubePostroutingChain),
		"-j", "MARK", "--xor-mark", proxier.masqueradeMark,
	)
	masqRule := []string{
		"-A", string(kubePostroutingChain),
		"-m", "comment", "--comment", `"kubernetes service traffic requiring SNAT"`,
		"-j", "MASQUERADE",
	}
	if proxier.iptables.HasRandomFully() {
		masqRule = append(masqRule, "--random-fully")
	}
	proxier.natRules.Write(masqRule)
	...
	// 构造伪装规则写到buffer里
	proxier.natRules.Write(
		"-A", string(KubeMarkMasqChain),
		"-j", "MARK", "--or-mark", proxier.masqueradeMark,
	)
	...
	...
	**// 该方法最核心逻辑，为每个svc构建对应规则，共计550行**
	for svcName, svc := range proxier.serviceMap {
		// 获取一些关于svc及其endpoints的信息
		svcInfo, ok := svc.(*serviceInfo)
		allEndpoints := proxier.endpointsMap[svcName]
		...
		var hasEndpoints, hasLocalReadyEndpoints, hasLocalServingTerminatingEndpoints bool
		for _, ep := range allEndpoints {
			if ep.IsReady() {
				hasEndpoints = true
				if ep.GetIsLocal() {
					hasLocalReadyEndpoints = true
				}
			} else if svc.NodeLocalExternal() && utilfeature.DefaultFeatureGate.Enabled(features.ProxyTerminatingEndpoints) {
				// 需要svc配置允许使用不可用后端才会去验证
				if ep.IsServing() && ep.IsTerminating() {
					hasEndpoints = true
					if ep.GetIsLocal() {
						hasLocalServingTerminatingEndpoints = true
					}
				}
			}
		}
		useTerminatingEndpoints := !hasLocalReadyEndpoints && hasLocalServingTerminatingEndpoints
		for _, ep := range allEndpoints {
			...
			// 添加到转发到endpoints的规则
			if svc.NodeLocalExternal() && epInfo.IsLocal {
				if useTerminatingEndpoints {
					if epInfo.Serving && epInfo.Terminating {
						localEndpointChains = append(localEndpointChains, endpointChain)
						endpointInUse = true
					}
				} else if epInfo.Ready {
					localEndpointChains = append(localEndpointChains, endpointChain)
					endpointInUse = true
				}
			}
		}
		...
		// 记录之前的svc链
		// 记录之前的lb类型链
		// 创建clusterIP类型规则
		args = append(args[:0],
				"-m", "comment", "--comment", fmt.Sprintf(`"%s cluster IP"`, svcNameString),
				"-m", protocol, "-p", protocol,
				"-d", utilproxy.ToCIDR(svcInfo.ClusterIP()),
				"--dport", strconv.Itoa(svcInfo.Port()),
			)
		// 创建externalIPs类型规则
		args = append(args[:0],
					"-m", "comment", "--comment", fmt.Sprintf(`"%s external IP"`, svcNameString),
					"-m", protocol, "-p", protocol,
					"-d", utilproxy.ToCIDR(netutils.ParseIPSloppy(externalIP)),
					"--dport", strconv.Itoa(svcInfo.Port()),
				)
		// 创建load-balancer类型规则
		args = append(args[:0],
						"-A", string(kubeServicesChain),
						"-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString),
						"-m", protocol, "-p", protocol,
						"-d", utilproxy.ToCIDR(netutils.ParseIPSloppy(ingress)),
						"--dport", strconv.Itoa(svcInfo.Port()),
					)
		// 创建节点端口转发类型规则
		args = append(args[:0],
					"-m", "comment", "--comment", svcNameString,
					"-m", protocol, "-p", protocol,
					"--dport", strconv.Itoa(svcInfo.NodePort()),
				)
	}
	...
	...
	// 清理不用的规则
	for chain := range existingNATChains {
		if !activeNATChains[chain] {
			chainString := string(chain)
			if !strings.HasPrefix(chainString, "KUBE-SVC-") && !strings.HasPrefix(chainString, "KUBE-SEP-") && !strings.HasPrefix(chainString, "KUBE-FW-") && !strings.HasPrefix(chainString, "KUBE-XLB-") {
				// 不是咱自己的不能乱操作
				continue
			}
			// 必须按iptables的规则多谢一条用来删除它的链规则
			proxier.natChains.WriteBytes(existingNATChains[chain])
			proxier.natRules.Write("-X", chainString)
		}
	}
	...
	// 将所有表的新规则落盘
	err = proxier.iptables.RestoreAll(proxier.iptablesData.Bytes(), utiliptables.NoFlushTables, utiliptables.RestoreCounters)
	// 此次修改成功
	success = true


type iptablesJumpChain struct {
	table     utiliptables.Table
	dstChain  utiliptables.Chain
	srcChain  utiliptables.Chain
	comment   string
	extraArgs []string
}

var iptablesJumpChains = []iptablesJumpChain{
	{utiliptables.TableFilter, kubeExternalServicesChain, utiliptables.ChainInput, "kubernetes externally-visible service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}},
	{utiliptables.TableFilter, kubeExternalServicesChain, utiliptables.ChainForward, "kubernetes externally-visible service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}},
	{utiliptables.TableFilter, kubeNodePortsChain, utiliptables.ChainInput, "kubernetes health check service ports", nil},
	{utiliptables.TableFilter, kubeServicesChain, utiliptables.ChainForward, "kubernetes service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}},
	{utiliptables.TableFilter, kubeServicesChain, utiliptables.ChainOutput, "kubernetes service portals", []string{"-m", "conntrack", "--ctstate", "NEW"}},
	{utiliptables.TableFilter, kubeForwardChain, utiliptables.ChainForward, "kubernetes forwarding rules", nil},
	{utiliptables.TableNAT, kubeServicesChain, utiliptables.ChainOutput, "kubernetes service portals", nil},
	{utiliptables.TableNAT, kubeServicesChain, utiliptables.ChainPrerouting, "kubernetes service portals", nil},
	{utiliptables.TableNAT, kubePostroutingChain, utiliptables.ChainPostrouting, "kubernetes postrouting rules", nil},
}

pkg\proxy\ipvs\proxier.go

**syncProxyRules**
	// 跟iptables的前置流程一样
	proxier.mu.Lock()
	defer proxier.mu.Unlock()
	...
	// 获取svc跟endpoint的变更信息
	serviceUpdateResult := proxier.serviceMap.Update(proxier.serviceChanges) --->pkg\proxy\service.go
	endpointUpdateResult := proxier.endpointsMap.Update(proxier.endpointsChanges) --->pkg\proxy\endpoints.go
	...
	// 对svc所用到ClusterIP/ExternalIP/LoadBalancerIP及NodePort都会有追踪清理工作
	for _, svcPortName := range endpointUpdateResult.StaleServiceNames {
		if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && conntrack.IsClearConntrackNeeded(svcInfo.Protocol()) {
			conntrackCleanupServiceIPs.Insert(svcInfo.ClusterIP().String())
			for _, extIP := range svcInfo.ExternalIPStrings() {
				conntrackCleanupServiceIPs.Insert(extIP)
			}
			for _, lbIP := range svcInfo.LoadBalancerIPStrings() {
				conntrackCleanupServiceIPs.Insert(lbIP)
			}
			nodePort := svcInfo.NodePort()
			if svcInfo.Protocol() == v1.ProtocolUDP && nodePort != 0 {
				conntrackCleanupServiceNodePorts.Insert(nodePort)
			}
		}
	}
	// ipvs模式依然会使用iptables的一些规则，这是因为ipvs的设计是做负载均衡的，不支持过滤钩子这种功能，所以依然由iptables实现
	proxier.filterChains.Write("*filter")
	proxier.natChains.Write("*nat")
	proxier.createAndLinkKubeChain()
	
	**// 为每个svc添加对应ipvs设置，该段代码共计457行**
	for svcName, svc := range proxier.serviceMap {
		// 记录原本的规则
		for _, e := range proxier.endpointsMap[svcName] {
			...
			proxier.ipsetList[kubeLoopBackIPSet].activeEntries.Insert(entry.String())
		}
		// clusterIP类型
		proxier.ipsetList[kubeClusterIPSet].activeEntries.Insert(entry.String())
		// 这里会有SessionAffinity对比，可以将同一请求转发给相同后端，SessionAffinity默认10800秒
		if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP {
			serv.Flags |= utilipvs.FlagPersistent
			serv.Timeout = uint32(svcInfo.StickyMaxAgeSeconds())
		}
		// externalIPs类型
		proxier.ipsetList[kubeExternalIPSet].activeEntries.Insert(entry.String())
		// load-balancer类型
		proxier.ipsetList[kubeLoadBalancerSet].activeEntries.Insert(entry.String())
		// Nodeports类型
		nodePortSet.activeEntries.Insert(entry.String())
	}
	
	// 设置ipvs规则
	for _, set := range proxier.ipsetList {
		set.syncIPSetEntries()
	}
	
	// 后面继续是iptables的一些补充工作

// 这个文件里可以看到所有的操作都是ipv4、ipv6顺序执行的，即kube-proxy已经具备了ipv6生产能力

pkg\proxy\metaproxier\meta_proxier.go

func NewMetaProxier(ipv4Proxier, ipv6Proxier proxy.Provider) proxy.Provider {
	return proxy.Provider(&metaProxier{
		ipv4Proxier: ipv4Proxier,
		ipv6Proxier: ipv6Proxier,
	}) --->pkg\proxy\xxx\proxier.go
}