ceph cluster 部署 (cephadm)

https://docs.ceph.com/en/pacific/cephadm
https://www.cnblogs.com/st2021/p/14970266.html
https://blog.csdn.net/get_set/article/details/108092248

ceph

hostname	category	hardware	eth0 - public	eth1	eth2 - cluster	eth3	gateway
vm-201	ceph-mon	core*1 / 2g / 20GB	192.168.100.201	10.0.100.201	10.0.110.201	10.0.120.201	192.168.100.1
vm-202	ceph-mon	core*1 / 2g / 20GB	192.168.100.202	10.0.100.202	10.0.110.202	10.0.120.202	192.168.100.1
vm-203	ceph-mon	core*1 / 2g / 20GB	192.168.100.203	10.0.100.203	10.0.110.203	10.0.120.203	192.168.100.1
vm-204	ceph-osd	core*4 / 4g / 20GB,10GBx2,30GBx4	192.168.100.204	10.0.100.204	10.0.110.204	10.0.120.204	192.168.100.1
vm-205	ceph-osd	core*4 / 4g / 20GB,10GBx2,30GBx4	192.168.100.205	10.0.100.205	10.0.110.205	10.0.120.205	192.168.100.1
vm-206	ceph-osd	core*4 / 4g / 20GB,10GBx2,30GBx4	192.168.100.206	10.0.100.206	10.0.110.206	10.0.120.206	192.168.100.1

CentOS Linux release 7.9.2009 (Core)

1. ssh信任登录

ssh-keygen -b 1024 -t rsa -P '' -f ~/.ssh/id_rsa

for i in {202..206}; do ssh-copy-id -i .ssh/id_rsa.pub 192.168.100.$i; done

2. 静态指向

cat > /etc/hosts <<EOF
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

# admin
192.168.100.210 vm-210

# openstack
192.168.100.211 vm-211
192.168.100.212 vm-212
192.168.100.213 vm-213
192.168.100.214 vm-214
192.168.100.215 vm-215
192.168.100.216 vm-216
192.168.100.217 vm-217
192.168.100.218 vm-218
192.168.100.219 vm-219

# k8s
192.168.100.191 vm-191
192.168.100.192 vm-192
192.168.100.193 vm-193
192.168.100.194 vm-194
192.168.100.195 vm-195
192.168.100.196 vm-196
192.168.100.197 vm-197
192.168.100.207 vm-207
192.168.100.198 vm-198

# ceph
192.168.100.201 vm-201
192.168.100.202 vm-202
192.168.100.203 vm-203
192.168.100.204 vm-204
192.168.100.205 vm-205
192.168.100.206 vm-206
EOF

for i in {202..206}; do scp /etc/hosts vm-$i:/etc; done

3. docker

cat > /etc/yum.repos.d/docker-ce.repo << EOF
[docker-ce-stable]
name=Docker CE Stable - \$basearch
baseurl=https://mirrors.nju.edu.cn/docker-ce/linux/centos/\$releasever/\$basearch/stable
enabled=1
gpgcheck=0
gpgkey=https://mirrors.nju.edu.cn/docker-ce/linux/centos/gpg
EOF

yum install -y docker-ce  && systemctl restart docker

cat > /etc/docker/daemon.json << EOF
{
  "registry-mirrors": ["https://registry.docker-cn.com", "http://hub-mirror.c.163.com", "https://docker.mirrors.ustc.edu.cn"],
  "insecure-registries": ["https://192.168.100.198:5000"],
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

systemctl restart docker  && systemctl enable docker 
docker version && docker info

for i in {202..206}; do scp /etc/yum.repos.d/docker-ce.repo vm-$i:/etc/yum.repos.d; done
for i in {202..206}; do ssh vm-$i 'yum install -y docker-ce  && systemctl restart docker'; done

for i in {202..206}; do scp /etc/docker/daemon.json vm-$i:/etc/docker/; done
for i in {202..206}; do ssh vm-$i 'systemctl enable docker && systemctl restart docker'; done

4. python

pip3 install pip -U -i https://pypi.tuna.tsinghua.edu.cn/simple
pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

for i in {202..206}; do ssh vm-$i 'pip3 install pip -U -i https://pypi.tuna.tsinghua.edu.cn/simple; pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple'; done

5. time & lvm2

yum install -y chrony lvm2; systemctl enable chronyd && systemctl restart chronyd

for i in {202..206}; do ssh vm-$i 'yum install -y chrony lvm2; systemctl enable chronyd && systemctl restart chronyd'; done

6. cephadm

curl -k https://raw.fastgit.org/ceph/ceph/v15.2.15/src/cephadm/cephadm -o /usr/sbin/cephadm
chmod 755 /usr/sbin/cephadm

sed -e 's|quay.io/ceph/ceph:v15|192.168.100.198:5000/ceph/ceph:v15.2.13|g' \
    -e 's|quay.io/prometheus/prometheus:v2.18.1|192.168.100.198:5000/prometheus/prometheus:2.31.1|g' \
    -e 's|quay.io/prometheus/node-exporter:v0.18.1|192.168.100.198:5000/prometheus/node-exporter:1.2.2|g' \
    -e 's|quay.io/prometheus/alertmanager:v0.20.0|192.168.100.198:5000/prometheus/alertmanager:0.23.0|g' \
    -e 's|quay.io/ceph/ceph-grafana:6.7.4|192.168.100.198:5000/ceph/ceph-grafana:6.7.4|g' \
    -i /usr/sbin/cephadm

head -n 10 /usr/sbin/cephadm
#!/usr/bin/python3

# Default container images -----------------------------------------------------
DEFAULT_IMAGE = '192.168.100.198:5000/ceph/ceph:v15.2.13'
DEFAULT_IMAGE_IS_MASTER = False
DEFAULT_PROMETHEUS_IMAGE = '192.168.100.198:5000/prometheus/prometheus:2.31.1'
DEFAULT_NODE_EXPORTER_IMAGE = '192.168.100.198:5000/prometheus/node-exporter:1.2.2'
DEFAULT_ALERT_MANAGER_IMAGE = '192.168.100.198:5000/prometheus/alertmanager:0.23.0'
DEFAULT_GRAFANA_IMAGE = '192.168.100.198:5000/ceph/ceph-grafana:6.7.4'
# ------------------------------------------------------------------------------

for i in {202..203}; do scp /usr/sbin/cephadm vm-$i:/usr/sbin; done
for i in {202..203}; do ssh vm-$i 'chmod 755 /usr/sbin/cephadm'; done

7. bootstrap

# cephadm bootstrap --mon-ip 192.168.100.201 

Ceph Dashboard is now available at:

	     URL: https://vm-201:8443/
	    User: admin
	Password: fkrf1t3p89

You can access the Ceph CLI with:

	sudo /usr/sbin/cephadm shell --fsid 75600af8-477e-11ec-85f6-f2b532235db6 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring

Please consider enabling telemetry to help improve Ceph:

	ceph telemetry on

For more information see:

	https://docs.ceph.com/docs/master/mgr/telemetry/

Bootstrap complete.

# cephadm shell -- ceph -s
Inferring fsid 75600af8-477e-11ec-85f6-f2b532235db6
Inferring config /var/lib/ceph/75600af8-477e-11ec-85f6-f2b532235db6/mon.vm-201/config
Using recent ceph image 192.168.100.198:5000/ceph/ceph@sha256:0368cf225b3a13b7bdeb3d81ecf370a62931ffa5ff87af880d66aebae74f910a
  cluster:
    id:     75600af8-477e-11ec-85f6-f2b532235db6
    health: HEALTH_WARN
            OSD count 0 < osd_pool_default_size 3
 
  services:
    mon: 1 daemons, quorum vm-201 (age 8m)
    mgr: vm-201.feujdg(active, since 8m)
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:  

# or 

cephadm add-repo --release octopus
sed -e 's|download.ceph.com|mirrors.nju.edu.cn/ceph|g' -i /etc/yum.repos.d/ceph.repo
cephadm install ceph-common 

ceph -s 

ceph config set global public_network 192.168.100.0/24
ceph config set global cluster_network 10.100.110.0/24
ceph config set global public_network_interface eth0
ceph config set global cluster_network_interface eth2

8. unmanaged

ceph orch apply mon --unmanaged
ceph orch apply mgr --unmanaged
ceph orch apply osd --all-available-devices --unmanaged

9. add hosts

ceph cephadm get-pub-key > ~/ceph.pub
for i in {202..206}; do ssh-copy-id -f -i ~/ceph.pub root@vm-$i; done

ceph orch host add vm-201 192.168.100.201 --labels mon mgr rgw
ceph orch host add vm-202 192.168.100.202 --labels mon mgr rgw
ceph orch host add vm-203 192.168.100.203 --labels mon mgr rgw

ceph orch host add vm-204 192.168.100.204 --labels osd
ceph orch host add vm-205 192.168.100.205 --labels osd
ceph orch host add vm-206 192.168.100.206 --labels osd

# ceph orch host ls
HOST    ADDR             LABELS  STATUS  
vm-201  192.168.100.201  mon             
vm-202  192.168.100.202  mon             
vm-203  192.168.100.203  mon             
vm-204  192.168.100.204  osd             
vm-205  192.168.100.205  osd             
vm-206  192.168.100.206  osd

10. deploy mon

ceph orch apply mon vm-201,vm-202,vm-203

11. deploy mgr

ceph orch apply mgr vm-201,vm-202,vm-203

12. deploy osd

# ceph orch device ls --wide
Hostname  Path      Type  Transport  RPM      Vendor  Model          Serial       Size   Health   Ident  Fault  Available  Reject Reasons  
vm-204    /dev/sdb  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi6  32.2G  Unknown  N/A    N/A    Yes                        
vm-204    /dev/sdc  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi5  32.2G  Unknown  N/A    N/A    Yes                        
vm-204    /dev/sdd  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi4  32.2G  Unknown  N/A    N/A    Yes                        
vm-204    /dev/sde  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi3  32.2G  Unknown  N/A    N/A    Yes                        
vm-204    /dev/sdf  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi2  10.7G  Unknown  N/A    N/A    Yes                        
vm-204    /dev/sdg  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi1  10.7G  Unknown  N/A    N/A    Yes                        
vm-205    /dev/sdb  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi6  32.2G  Unknown  N/A    N/A    Yes                        
vm-205    /dev/sdc  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi5  32.2G  Unknown  N/A    N/A    Yes                        
vm-205    /dev/sdd  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi4  32.2G  Unknown  N/A    N/A    Yes                        
vm-205    /dev/sde  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi3  32.2G  Unknown  N/A    N/A    Yes                        
vm-205    /dev/sdf  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi2  10.7G  Unknown  N/A    N/A    Yes                        
vm-205    /dev/sdg  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi1  10.7G  Unknown  N/A    N/A    Yes                        
vm-206    /dev/sdb  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi6  32.2G  Unknown  N/A    N/A    Yes                        
vm-206    /dev/sdc  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi5  32.2G  Unknown  N/A    N/A    Yes                        
vm-206    /dev/sdd  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi4  32.2G  Unknown  N/A    N/A    Yes                        
vm-206    /dev/sde  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi3  32.2G  Unknown  N/A    N/A    Yes                        
vm-206    /dev/sdf  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi2  10.7G  Unknown  N/A    N/A    Yes                        
vm-206    /dev/sdg  hdd   Unknown    Unknown  QEMU    QEMU HARDDISK  drive-scsi1  10.7G  Unknown  N/A    N/A    Yes  

cat > /tmp/osds.yaml << EOF
service_type: osd
service_id: osd_using_paths
placement:
  hosts:
    - vm-204
    - vm-205
    - vm-206
spec:
  data_devices:
    paths:
    - /dev/sdb
    - /dev/sdc
    - /dev/sdd
    - /dev/sde
  db_devices:
    paths:
    - /dev/sdf
  wal_devices:
    paths:
    - /dev/sdg
EOF

ceph orch apply -i /tmp/osds.yaml 

# ceph osd tree
ID  CLASS  WEIGHT   TYPE NAME        STATUS  REWEIGHT  PRI-AFF
-1         0.38031  root default                              
-5         0.12677      host vm-204                           
 1    hdd  0.03169          osd.1        up   1.00000  1.00000
 4    hdd  0.03169          osd.4        up   1.00000  1.00000
 7    hdd  0.03169          osd.7        up   1.00000  1.00000
10    hdd  0.03169          osd.10       up   1.00000  1.00000
-3         0.12677      host vm-205                           
 0    hdd  0.03169          osd.0        up   1.00000  1.00000
 3    hdd  0.03169          osd.3        up   1.00000  1.00000
 6    hdd  0.03169          osd.6        up   1.00000  1.00000
 9    hdd  0.03169          osd.9        up   1.00000  1.00000
-7         0.12677      host vm-206                           
 2    hdd  0.03169          osd.2        up   1.00000  1.00000
 5    hdd  0.03169          osd.5        up   1.00000  1.00000
 8    hdd  0.03169          osd.8        up   1.00000  1.00000
11    hdd  0.03169          osd.11       up   1.00000  1.00000

# ceph osd df 
ID  CLASS  WEIGHT   REWEIGHT  SIZE     RAW USE  DATA     OMAP    META      AVAIL    %USE   VAR   PGS  STATUS
 1    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   4 KiB  1024 MiB   29 GiB  10.79  1.00    0      up
 4    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   3 KiB  1024 MiB   29 GiB  10.79  1.00    0      up
 7    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   4 KiB  1024 MiB   29 GiB  10.79  1.00    1      up
10    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   5 KiB  1024 MiB   29 GiB  10.79  1.00    0      up
 0    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   4 KiB  1024 MiB   29 GiB  10.79  1.00    0      up
 3    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   2 KiB  1024 MiB   29 GiB  10.79  1.00    0      up
 6    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   3 KiB  1024 MiB   29 GiB  10.79  1.00    0      up
 9    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   4 KiB  1024 MiB   29 GiB  10.79  1.00    1    down
 2    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   3 KiB  1024 MiB   29 GiB  10.79  1.00    0      up
 5    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   5 KiB  1024 MiB   29 GiB  10.79  1.00    1      up
 8    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   3 KiB  1024 MiB   29 GiB  10.79  1.00    0      up
11    hdd  0.03169   1.00000   32 GiB  3.5 GiB   11 MiB   4 KiB  1024 MiB   29 GiB  10.79  1.00    0      up
                       TOTAL  390 GiB   42 GiB  132 MiB  50 KiB    12 GiB  348 GiB  10.79                   
MIN/MAX VAR: 1.00/1.00  STDDEV: 0

13. deploy rgw

radosgw-admin realm create --rgw-realm=default-realm --default
radosgw-admin zonegroup create --rgw-zonegroup=default-zonegroup --master --default
radosgw-admin zone create --rgw-zonegroup=default-zonegroup --rgw-zone=default-zone --master --default
radosgw-admin period update --rgw-realm=default-realm --commit 

ceph orch apply rgw default-realm default-zone --unmanaged
ceph orch daemon add rgw default-realm default-zone --placement="vm-201 vm-202 vm-203"

# ceph -s
  cluster:
    id:     cc84d9b4-4830-11ec-a506-f2b532235db6
    health: HEALTH_WARN
            1 failed cephadm daemon(s)
 
  services:
    mon: 3 daemons, quorum vm-201,vm-202,vm-203 (age 65m)
    mgr: vm-201.zgeeaz(active, since 94m), standbys: vm-202.tzqyjz, vm-203.tosmgb
    osd: 12 osds: 12 up (since 18m), 12 in (since 19m)
    rgw: 3 daemons active (default-realm.default-zone.vm-201.gbechp, default-realm.default-zone.vm-202.ojodvg, default-realm.default-zone.vm-203.ecllzd)
 
  task status:
 
  data:
    pools:   5 pools, 105 pgs
    objects: 201 objects, 7.4 KiB
    usage:   42 GiB used, 348 GiB / 390 GiB avail
    pgs:     105 active+clean
 
  io:
    client:   43 KiB/s rd, 170 B/s wr, 43 op/s rd, 23 op/s wr

pool

ceph config set mon mon_allow_pool_delete true

ceph osd pool rm .rgw.root .rgw.root --yes-i-really-really-mean-it

service

ceph orch rm node-exporter
ceph orch apply node-exporter '*'
ceph orch redeploy node-exporter

* cluster network没找到解决方法

相关阅读:
sql developer 中文乱码解决办法
 ubuntu oracle数据库18c安装
 ubuntu Oracle SQL Developer 安装
 web.xml is missing and <failOnMissingWebXml> is set to true
MySQL设置快速删除
 Annoying “Remote System Explorer Operation” causing freeze for couple of seconds
安装程序时出现2502 2503错误解决方法
 Smoke Testing（冒烟测试）
MySQL5.7.11免安装版的安装和配置：解决MYSQL 服务无法启动问题
 Error 2503 and 2502 when installing/uninstalling on Windows 10
原文地址：https://www.cnblogs.com/liujitao79/p/15567219.html