k8s 连接glusterfs 集群使用

准备glusterfs 存储

glusterfs volume 为public，挂载在服务器的/data/public/ 下，创建项目目录projects
mkdir /data/public/projects/

创建单独测试项目数据目录volume-test-nginx 
mkdir /data/public/projects/volume-test-nginx

创建k8s 名称空间 testnginx

kubectl create ns testnginx

准备glusterfs endpoint 配置

{
  "kind": "Endpoints",
  "apiVersion": "v1",
  "metadata": {
    "name": "glusterfs-cluster",
    "namespace": "testnginx"       #指定endpoint 名称空间
  },
  "subsets": [
    {
      "addresses": [
        {
          "ip": "10.65.0.19"
        }
      ],
      "ports": [
        {
          "port": 1000
        }
      ]
    },
    {
      "addresses": [
        {
          "ip": "10.65.0.20"
        }
      ],
      "ports": [
        {
          "port": 1000
        }
      ]
    },
    {
      "addresses": [
        {
          "ip": "10.65.0.21"
        }
      ],
      "ports": [
        {
          "port": 1000
        }
      ]
    },
    {
      "addresses": [
        {
          "ip": "10.65.0.22"
        }
      ],
      "ports": [
        {
          "port": 1000
        }
      ]
    },
    {
      "addresses": [
        {
          "ip": "10.65.0.23"
        }
      ],
      "ports": [
        {
          "port": 1000
        }
      ]
    },
    {
      "addresses": [
        {
          "ip": "10.65.0.24"
        }
      ],
      "ports": [
        {
          "port": 1000
        }
      ]
    },
    {
      "addresses": [
        {
          "ip": "10.65.0.25"
        }
      ],
      "ports": [
        {
          "port": 1000
        }
      ]
    },
    {
      "addresses": [
        {
          "ip": "10.65.0.26"
        }
      ],
      "ports": [
        {
          "port": 1000
        }
      ]
    },
    {
      "addresses": [
        {
          "ip": "10.65.0.27"
        }
      ],
      "ports": [
        {
          "port": 1000
        }
      ]
    }
  ]
}

准备 glusterfs svc 配置文件

# cat glusterfs-service.json 
{
  "kind": "Service",
  "apiVersion": "v1",
  "metadata": {
    "name": "glusterfs-cluster",
    "namespace": "testnginx"
  },
  "spec": {
    "ports": [
      {"port": 1000}
    ]
  }
}

准备 glusterfs pv

# cat pv.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  namespace: testnginx
  name: gluster-volume-testnginx
spec:
  capacity:
    storage: 3Gi
  accessModes: ["ReadWriteMany","ReadOnlyMany"]
  glusterfs:
    endpoints: "glusterfs-cluster"
    path: "public/projects/volume-test-nginx" #public 为glusterfs volume，projects/volume-test-nginx 为单独项目路径
    readOnly: false

准备 glusterfs Pvc 配置

# cat pvc.yaml 
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  namespace: testnginx
  name: gluster-volume-testnginx
spec:
  accessModes: ["ReadWriteMany"]
  resources:
    requests:
      storage: 3Gi

准备glusterfs 测试nginx 配置文件

# cat dp.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: testnginx
  namespace: testnginx
  labels:
    app.name: testnginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app.name: testnginx
  template:
    metadata:
      labels:
        app.name: testnginx
    spec:
      containers:
      - name: testnginx
        image: nginx:latest
        ports:
        - containerPort: 80
        volumeMounts:
        - name: glusterfsvol
          mountPath: "/usr/share/nginx/html"
      volumes:
      - name: glusterfsvol
        persistentVolumeClaim:
          claimName: gluster-volume-testnginx

生成配置文件

kubectl apply  -f glusterfs-endpoints.json
kubectl apply  -f glusterfs-service.json
kubectl apply  -f pv.yaml
kubectl apply  -f pvc.yaml
kubectl apply  -f dp.yaml

查看nginx 数据目录

# kubectl exec -it -n testnginx  testnginx-67f47689d9-zcgrd bash
root@testnginx-67f47689d9-zcgrd:/# cd /usr/share/nginx/html/
root@testnginx-67f47689d9-zcgrd:/usr/share/nginx/html# ls
index.html
root@testnginx-67f47689d9-zcgrd:/usr/share/nginx/html# cat index.html 
111

查看 glusterfs 存储
[root@lgy-storage1 10.65.0.1 /data/public/projects/volume-test-nginx ] 
# cat index.html 
111