k8s helm 运用与自建helm仓库chartmuseum

搭建私有 helm 仓库 chartmuseum

chartmuseum 是一个开源的 Helm Chart Repository，支持多种后端存储，包括 GCS，S3 等。
#环境
1.k8s 1.16 集群
2.有nfs 可供挂载的存储

#创建 chartmuseum 私有仓库，准备配置文件
$ cat chartmuseum.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: chartmuseum
  name: chartmuseum
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: chartmuseum
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: chartmuseum
    spec:
      containers:
      - image: k8s-harbor01.gdfsxxds.rjyun/xy/chartmuseum:latest
        imagePullPolicy: IfNotPresent
        name: chartmuseum
        ports:
        - containerPort: 8080
          protocol: TCP
        env:
        - name: DEBUG
          value: "1"
        - name: STORAGE
          value: local
        - name: STORAGE_LOCAL_ROOTDIR
          value: /charts
        resources:
          limits:
            cpu: 500m
            memory: 256Mi
          requests:
            cpu: 100m
            memory: 64Mi
        volumeMounts:
        - mountPath: /charts
          name: charts-volume
      volumes:
      - name: charts-volume
        nfs:
          path: /data/upload/charts
          server: 172.24.119.30
      restartPolicy: Always



--- 

apiVersion: v1
kind: Service
metadata:
  name: chartmuseum
  namespace: kube-system
spec:
  ports:
    - port: 8080
      protocol: TCP
      targetPort: 8080
  selector:
    app: chartmuseum


--- 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: chartmuseum
  namespace: kube-system
spec:
  rules:
  - host: chart-xxx.com
    http:
      paths:
      - path: /
        backend:
          serviceName: chartmuseum
          servicePort: 8080

# 创建
kubectl apply -f chartmuseum.yaml 
#检查
kubectl get pods -n kube-system

# 成功显示欢迎信息
浏览器访问  http://chart-xxx.com

# 添加 helm repo
helm repo add chartmuseum http://chart-xxx.com
helm repo list

创建应用并上传到 chartmuseum

#创建 chart 
mkdir /root/helm && cd /root/helm
helm create myapp
#修改模板配置
cd /root/helm/myapp
$ tree
.
├── charts
├── Chart.yaml
├── templates
│   ├── deployment.yaml
│   ├── _helpers.tpl
│   ├── ingress.yaml
│   ├── NOTES.txt
│   ├── serviceaccount.yaml
│   ├── service.yaml
│   └── tests
│       └── test-connection.yaml
└── values.yaml

# values.yaml 配置
$ cat values.yaml 

# Default values for deploy.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

replicaCount: 1

log: ""

image:
  repository: nginx
  tag: stable
  pullPolicy: IfNotPresent
  imagePullSecrets: regcred

nameOverride: ""
fullnameOverride: ""

service:
  type: ClusterIP
  port: 80
  annotations: {}
    # prometheus.io/probe: "true"
    # prometheus.io/port: "80"

ingress:
  enabled: true
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/enable-cors:  "true"
  paths: 
    - /
  hosts:
    - chart-example.local
  tls: false


## Configure extra options for liveness and readiness probes
## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes)
livenessProbe: {}
#   tcpSocket:
#     port: 80
#   initialDelaySeconds: 3
#   periodSeconds: 3
readinessProbe: {}
#   httpGet:
#     path: /
#     port: 80
#   initialDelaySeconds: 5
#   periodSeconds: 5

resources: {}
  # We usually recommend not to specify default resources and to leave this as a conscious
  # choice for the user. This also increases chances charts run on environments with little
  # resources, such as Minikube. If you do want to specify resources, uncomment the following
  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
  # limits:
  #  cpu: 100m
  #  memory: 128Mi
  # requests:
  #  cpu: 100m
  #  memory: 128Mi

nodeSelector: {}

tolerations: []

affinity: {}

## EXAMPLE:
##   extraConfigmapMounts:
##     - name: airflow-webserver-config
##       mountPath: /etc/supervisord.conf
##       configMap: laravel-config
##       readOnly: true
##       subPath: hotniuer2-config-production
##
extraConfigmapMounts: []

## EXAMPLE:
##   services:
##     - name: grpc
##       port: 80
##
services: []


# Chart.yaml 配置
$ cat Chart.yaml 
apiVersion: v1
appVersion: "1.0"
description: A Helm chart for Kubernetes
name: myapp
version: 0.9.0

#template 配置

deployment.yaml 配置：
*注意： deployment 名称是修改过的，

$ cat deployment.yaml 

apiVersion: apps/v1
kind: Deployment
metadata:
  name: "{{ .Release.Name }}-deployment"
  labels:
    app: {{ .Release.Name }}
    helm.sh/chart: {{ include "deploy.chart" . }}
    app.kubernetes.io/name: {{ include "deploy.fullname" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      app: "{{ .Release.Name }}"
  template:
    metadata:
      labels:
        app: "{{ .Release.Name }}"
        log: "{{ .Values.log }}"
    spec:
      imagePullSecrets:
      - name: {{ .Values.image.imagePullSecrets }}
      containers:
        - name: {{ .Release.Name }}
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          ports:
            - name: http
              containerPort: {{ .Values.service.port }}
              protocol: TCP
            {{- range .Values.services }}
            - name: {{ .name }}
              containerPort: {{ .port }}
              protocol: TCP
            {{- end }}
          {{- with .Values.livenessProbe }}
          livenessProbe:
            {{- toYaml . | nindent 12 }}
          {{- end }}
          {{- with .Values.readinessProbe }}
          readinessProbe:
            {{- toYaml . | nindent 12 }}
          {{- end }}
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
          env:
          {{- range $key, $val := .Values.env }}
          - name: {{ $key }}
            value: {{ $val | quote }}
          {{- end }}
          volumeMounts:
            {{- range .Values.extraConfigmapMounts }}
            - name: {{ .name }}
              mountPath: {{ .mountPath }}
              readOnly: {{ .readOnly }}
              {{- if .subPath }}
              subPath: {{ .subPath }}
              {{- end }}
            {{- end }}
          terminationMessagePolicy: FallbackToLogsOnError
      volumes:
        {{- range .Values.extraConfigmapMounts }}
        - name: {{ .name }}
          configMap:
            name: {{ .configMap }}
        {{- end }}
      {{- with .Values.nodeSelector }}
      nodeSelector:
        {{- toYaml . | nindent 8 }}
      {{- end }}
    {{- with .Values.affinity }}
      affinity:
        {{- toYaml . | nindent 8 }}
    {{- end }}
    {{- with .Values.tolerations }}
      tolerations:
        {{- toYaml . | nindent 8 }}
    {{- end }}



#service 配置 
*注意： service 名称是修改过的
$ cat service.yaml 

apiVersion: v1
kind: Service
metadata:
  name: "{{ .Release.Name }}-service"
  labels:
    app: {{ .Release.Name }}
    helm.sh/chart: {{ include "deploy.chart" . }}
    app.kubernetes.io/name: {{ include "deploy.name" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  {{- with .Values.service.annotations }}
  annotations:
    {{- toYaml . | nindent 4 }}
  {{- end }}
spec:
  type: {{ .Values.service.type }}
  ports:
    - port: {{ .Values.service.port }}
      targetPort: http
      protocol: TCP
      name: http
    {{- range .Values.services }}
    - name: {{ .name }}
      port: {{ .port }}
      protocol: TCP
      targetPort: {{ .port }}
    {{- end }}
  selector:
    app: {{ .Release.Name }}


#ingress 配置 
* 注意：{{- $fullName := .Release.Name -}}  是修改后的 
$ cat ingress.yaml 

{{- if .Values.ingress.enabled -}}
{{- $fullName := .Release.Name -}}
{{- $ingressPaths := .Values.ingress.paths -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: "{{ .Release.Name }}-ingress"
  labels:
    app: {{ .Release.Name }}
    app.kubernetes.io/name: {{ include "deploy.name" . }}
    helm.sh/chart: {{ include "deploy.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  {{- with .Values.ingress.annotations }}
  annotations:
    {{- toYaml . | nindent 4 }}
  {{- end }}
spec:
{{- if .Values.ingress.tls }}
  tls:
    - hosts:
      {{- range .Values.ingress.hosts }}
        - {{ . | quote }}
      {{- end }}
      secretName: {{ $fullName }}-tls
{{- end }}
  rules:
  {{- range .Values.ingress.hosts }}
    - host: {{ . | quote }}
      http:
        paths:
        {{- range $ingressPaths }}
          - path: {{ . }}
            backend:
              serviceName: "{{ $fullName }}-service"
              servicePort: http
        {{- end }}
  {{- end }}
{{- end }}



# 检查语法
cd /root/helm/myapp
helm  lint myapp

# 打包 
helm package myapp 
# 上传
curl  --data-binary "@myapp-0.9.0.tgz"   http://chart-xxx.com/api/charts

通过 chartmuseum 安装软件

# 查找版本
helm search myapp

# 安装
helm install --name myapp chartmuseum/myapp

# 查看
helm list
helm status myapp

# 删除
helm del --purge myapp 

更新 charts 并上传到 chartmuseum

cd /root/helm/myapp

vi values.yaml
# 调整 replicaCount 为 2
replicaCount: 2

vi Chart.yaml
# 把 version 改为 0.10.0
version: 0.10.0

# 测试，打包
cd /root/helm
helm lint myapp
helm package myapp

# 上传
curl --data-binary "@myapp-0.10.0.tgz"  http://chart-xxx.com/api/charts
# 更新本地缓存然后可以查看已有的 charts：
helm repo update
helm search chartmuseum/

# 若之前没有删除，可以进行更新
helm upgrade myapp chartmuseum/myapp

# 回滚
helm rollback myapp 1

集合jenkins 使用

# upgrade -i 在k8s 集群上没有项目则创建，有项目则更新
helm upgrade  -i   mytestnginxhelm2  chartmuseum/myapp --set service.port=80,ingress={mytest-xxx.com},image.repository=k8s-harbor01.gdfsxxds.rjyun/xy/nginx,image.tag=stable  --namespace=default