ceph从新启动后lvm消失问题
从新启动ceph集群后 有几台机器lvm莫名消失了.
集群就炸了啊,赶紧 nodown noout norecover 一套标记走起来.
vgscan pvscan 都搜不到,对应osd也起不来
[root@ceph-62 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 9.1T 0 disk
sdb 8:16 0 9.1T 0 disk
sdc 8:32 0 9.1T 0 disk
sdd 8:48 0 9.1T 0 disk
sde 8:64 0 9.1T 0 disk
sdf 8:80 0 9.1T 0 disk
sdg 8:96 0 9.1T 0 disk
sdh 8:112 0 9.1T 0 disk
sdi 8:128 0 9.1T 0 disk
sdj 8:144 0 9.1T 0 disk
sdk 8:160 0 9.1T 0 disk
sdl 8:176 0 9.1T 0 disk
sdm 8:192 0 9.1T 0 disk
sdn 8:208 0 9.1T 0 disk
sdo 8:224 0 9.1T 0 disk
sdp 8:240 0 9.1T 0 disk
sdq 65:0 0 9.1T 0 disk
sdr 65:16 0 9.1T 0 disk
sds 65:32 0 9.1T 0 disk
sdt 65:48 0 223.6G 0 disk
├─sdt1 65:49 0 200M 0 part /boot/efi
├─sdt2 65:50 0 1G 0 part /boot
└─sdt3 65:51 0 222.4G 0 part
├─centos-root 253:0 0 50G 0 lvm /
├─centos-swap 253:1 0 4G 0 lvm [SWAP]
└─centos-home 253:2 0 168.4G 0 lvm /home
sdu 65:64 0 9.1T 0 disk
一看凉了,正常的是这样的
[root@ceph-59 ~]# date
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 9.1T 0 disk
└─ceph--ebc2b2fc--be8e--48af--9729--8149f9a3960e-osd--data--c7ac8edc--4187--46ca--9569--b35faaf1dd9c 253:13 0 9.1T 0 lvm
sdb 8:16 0 9.1T 0 disk
└─ceph--3e5b8aba--ec7e--4448--b8c5--ea3e3fe7b4a5-osd--data--11b74eff--5a3c--432e--8ec9--aa776c9d4c64 253:14 0 9.1T 0 lvm
sdc 8:32 0 9.1T 0 disk
└─ceph--8800da1a--2764--4370--b8d7--8a0b1332bfc4-osd--data--fd68fe83--b142--4b55--a3d4--aa7b728febec 253:15 0 9.1T 0 lvm
sdd 8:48 0 9.1T 0 disk
└─ceph--ebd8ee23--cada--4717--8933--3cc5cdbb9840-osd--data--603de199--c6e3--4a78--a2cf--1cda25bd3d02 253:9 0 9.1T 0 lvm
sde 8:64 0 9.1T 0 disk
└─ceph--ce85d728--29a1--4faa--92f4--5288801cf7c0-osd--data--4dfb9363--52bf--498b--9417--9624b99f0a95 253:10 0 9.1T 0 lvm
sdf 8:80 0 9.1T 0 disk
└─ceph--9dbd5617--aa78--45cb--b80a--e33944890518-osd--data--c1cd41dd--3042--4949--9551--c081b2fe418d 253:11 0 9.1T 0 lvm
sdg 8:96 0 9.1T 0 disk
└─ceph--3ce3bcda--0250--44cd--ac93--50d585ef5ad5-osd--data--101b4271--cf4c--4ec5--aa53--4627d4feb698 253:12 0 9.1T 0 lvm
sdh 8:112 0 9.1T 0 disk
└─ceph--86368e74--660f--4772--8844--bf9c29e4e730-osd--data--3bc3cc6c--5850--4c83--a67a--51b024addce6 253:19 0 9.1T 0 lvm
sdi 8:128 0 9.1T 0 disk
└─ceph--79778284--af9a--416a--814b--22bb1c732794-osd--data--fcd6b24d--e48e--4315--9bcd--d54fba335815 253:20 0 9.1T 0 lvm
sdj 8:144 0 9.1T 0 disk
└─ceph--bf7c58a5--a9e5--4b77--81c3--ee75634754f7-osd--data--8e10bfa5--ce46--46fe--ba2d--ba70a78eb17b 253:21 0 9.1T 0 lvm
-
先看看 /etc/lvm/backup/*
[root@ceph-59 ~]# ls /etc/lvm/backup/ centos ceph-3e5b8aba-ec7e-4448-b8c5-ea3e3fe7b4a5 ceph-9dbd5617-aa78-45cb-b80a-e33944890518 ceph-ebd8ee23-cada-4717-8933-3cc5cdbb9840 ceph-0033060c-5010-4bd5-9859-78ffe5ceff27 ceph-6e4dedf0-130c-4b76-a3be-d2520b69f522 ceph-ab9e39dd-dde1-41c8-8bc1-11936adbf85a ceph-f3220c11-eb66-45ba-9525-fb7017c45d4f ceph-28354762-18d9-4302-b11c-946832b7dceb ceph-716552a4-af1b-4ceb-b034-a31195152391 ceph-bf7c58a5-a9e5-4b77-81c3-ee75634754f7 ceph-f95e3fc6-c7ee-48e8-bc70-ab40e995a868 ceph-2be8cb74-f986-4893-b3f6-784e1b128e01 ceph-79778284-af9a-416a-814b-22bb1c732794 ceph-c7094bd4-d4a4-427c-add7-0eabd83ee1ba ceph-36ef94cd-7f9a-4ecc-9d12-4268289d60cd ceph-86368e74-660f-4772-8844-bf9c29e4e730 ceph-ce85d728-29a1-4faa-92f4-5288801cf7c0 ceph-3ce3bcda-0250-44cd-ac93-50d585ef5ad5 ceph-8800da1a-2764-4370-b8d7-8a0b1332bfc4 ceph-ebc2b2fc-be8e-48af-9729-8149f9a3960e**推荐看 ls /etc/lvm/archive/ **
因为这里版本比较多些.
[root@ceph-59 ~]# ls /etc/lvm/archive/* /etc/lvm/archive/centos_00000-1556371822.vg /etc/lvm/archive/ceph-8800da1a-2764-4370-b8d7-8a0b1332bfc4_00002-498819197.vg /etc/lvm/archive/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27_00000-191613742.vg /etc/lvm/archive/ceph-8800da1a-2764-4370-b8d7-8a0b1332bfc4_00003-196229818.vg /etc/lvm/archive/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27_00001-1071232553.vg /etc/lvm/archive/ceph-8800da1a-2764-4370-b8d7-8a0b1332bfc4_00004-1508357692.vg /etc/lvm/archive/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27_00002-1833047220.vg /etc/lvm/archive/ceph-8800da1a-2764-4370-b8d7-8a0b1332bfc4_00005-482948107.vg /etc/lvm/archive/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27_00003-1002373647.vg /etc/lvm/archive/ceph-8800da1a-2764-4370-b8d7-8a0b1332bfc4_00006-695412169.vg /etc/lvm/archive/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27_00004-882636099.vg /etc/lvm/archive/ceph-9dbd5617-aa78-45cb-b80a-e33944890518_00000-1186551457.vg /etc/lvm/archive/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27_00005-645028611.vg /etc/lvm/archive/ceph-9dbd5617-aa78-45cb-b80a-e33944890518_00001-1556927714.vg /etc/lvm/archive/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27_00006-2143351603.vg /etc/lvm/archive/ceph-9dbd5617-aa78-45cb-b80a-e33944890518_00002-1272062293.vg /etc/lvm/archive/ceph-28354762-18d9-4302-b11c-946832b7dceb_00000-1313439840.vg /etc/lvm/archive/ceph-9dbd5617-aa78-45cb-b80a-e33944890518_00003-163927944.vg /etc/lvm/archive/ceph-28354762-18d9-4302-b11c-946832b7dceb_00001-1936313617.vg /etc/lvm/archive/ceph-9dbd5617-aa78-45cb-b80a-e33944890518_00004-427518662.vg /etc/lvm/archive/ceph-28354762-18d9-4302-b11c-946832b7dceb_00002-1765261441.vg /etc/lvm/archive/ceph-9dbd5617-aa78-45cb-b80a-e33944890518_00005-544615481.vg /etc/lvm/archive/ceph-28354762-18d9-4302-b11c-946832b7dceb_00003-1532647703.vg /etc/lvm/archive/ceph-9dbd5617-aa78-45cb-b80a-e33944890518_00006-671781823.vg /etc/lvm/archive/ceph-28354762-18d9-4302-b11c-946832b7dceb_00004-1642162971.vg /etc/lvm/archive/ceph-ab9e39dd-dde1-41c8-8bc1-11936adbf85a_00000-2137077032.vg /etc/lvm/archive/ceph-28354762-18d9-4302-b11c-946832b7dceb_00005-66156891.vg /etc/lvm/archive/ceph-ab9e39dd-dde1-41c8-8bc1-11936adbf85a_00001-533416413.vg /etc/lvm/archive/ceph-28354762-18d9-4302-b11c-946832b7dceb_00006-151645212.vg /etc/lvm/archive/ceph-ab9e39dd-dde1-41c8-8bc1-11936adbf85a_00002-266310388.vg /etc/lvm/archive/ceph-2be8cb74-f986-4893-b3f6-784e1b128e01_00000-1665211260.vg -
随便打开一个看看内容
[root@ceph-59 ~]# cat /etc/lvm/backup/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27 # Generated by LVM2 version 2.02.185(2)-RHEL7 (2019-05-13): contents = "Text Format Volume Group" version = 1 description = "Created *after* executing 'vgchange -ay ceph-0033060c-5010-4bd5-9859-78ffe5ceff27'" creation_host = "ceph-59" # Linux ceph-59 3.10.0-1062.el7.x86_64 #1 SMP Wed Aug 7 18:08:02 UTC 2019 x86_64 creation_time = 1611333863 ceph-0033060c-5010-4bd5-9859-78ffe5ceff27 { id = "WIT8wF-kJdI-SS3E-5ue8-e3Ws-Awk3-AnkDkz" seqno = 6 format = "lvm2" # informational status = ["RESIZEABLE", "READ", "WRITE"] flags = [] extent_size = 8192 # 4 Megabytes max_lv = 0 max_pv = 0 metadata_copies = 0 physical_volumes { pv0 { id = "EIGfcN-503D-MVDf-AwK1-SDtA-A4Fw-mvBqpe" device = "/dev/sdn" # Hint only status = ["ALLOCATABLE"] flags = [] dev_size = 19532873728 # 9.0957 Terabytes pe_start = 2048 pe_count = 2384383 # 9.0957 Terabytes } } logical_volumes { osd-data-d9fa7e7d-57f6-437c-9b2b-c9652ec0697d { id = "WtB7QP-qOxT-hV2b-3a2X-zPFS-09aP-HVKbc0" status = ["READ", "WRITE", "VISIBLE"] flags = [] tags = ["ceph.osdspec_affinity=", "ceph.vdo=0", "ceph.osd_id=208", "ceph.osd_fsid=c71e53d3-97e8-4f97-a356-1a4bb1187d45", "ceph.cluster_name=ceph", "ceph.cluster_fsid=9f4c8519-821e-434a-9e85-cdef908b808c", "ceph.encrypted=0", "ceph.cephx_lockbox_secret=", "ceph.type=block", "ceph.crush_device_class=None", "ceph.block_device=/dev/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27/osd-data-d9fa7e7d-57f6-437c-9b2b-c9652ec0697d", "ceph.block_uuid=WtB7QP-qOxT-hV2b-3a2X-zPFS-09aP-HVKbc0"] creation_time = 1601402387 # 2020-09-30 01:59:47 +0800 creation_host = "ceph-59" segment_count = 1 segment1 { start_extent = 0 extent_count = 2384383 # 9.0957 Terabytes type = "striped" stripe_count = 1 # linear stripes = [ "pv0", 0 ] } } } } -
挑点我们一会就用的
pv0 { id = "EIGfcN-503D-MVDf-AwK1-SDtA-A4Fw-mvBqpe" device = "/dev/sdn" # Hint onlydevice 是设备名 id是设备uuid
tags = ["ceph.osdspec_affinity=", "ceph.vdo=0", "ceph.osd_id=208", "ceph.osd_fsid=c71e53d3-97e8-4f97-a356-1a4bb1187d45", "ceph.cluster_name=ceph", "ceph.cluster_fsid=9f4c8519-821e-434a-9e85-cdef908b808c", "ceph.encrypted=0", "ceph.cephx_lockbox_secret=", "ceph.type=block", "ceph.crush_device_class=None", "ceph.block_device=/dev/ceph-0033060c-5010-4bd5-9859-78ffe5ceff27/osd-data-d9fa7e7d-57f6-437c-9b2b-c9652ec0697d", "ceph.block_uuid=WtB7QP-qOxT-hV2b-3a2X-zPFS-09aP-HVKbc0"]ceph.osd_id 是osd的id ceph.osd_fsid osd的fsid
下面是临时写的一个垃圾脚本,基本能跳过已经存在vg不修复,根据备份的信息来自动恢复vg然后自动挂载和启动osd服务并加入集群.
import os
import re
import time
rea=r'id = "([w+-*w*]+)"s*device = "([w/]*)"'
reb=r'ceph.osd_id=(d+)", "ceph.osd_fsid=([w*-*w*]+)"'
# with open(r'C:UsersmakeitDesktopceph-0033060c-5010-4bd5-9859-78ffe5ceff27','r') as f:
def getAllLvmBackFile():
backs=os.listdir('/etc/lvm/backup/')
flist=[]
for i in backs:
if 'ceph' in i:
flist.append(os.path.join('/etc/lvm/backup/',i))
return flist
def getTargetLvms():
flist=getAllLvmBackFile()
cmd_get_old_vgs='''vgs|grep ceph|awk '{print $1}' >/tmp/oldlvms'''
os.system(cmd_get_old_vgs)
goodlvms=''
with open('/tmp/oldlvms','r') as f:
godlvms=f.readlines()
for l in goodlvms:
l=l.strip()
for i in flist:
# print(i,l,i.find(l))
if i.find(l)!=-1:
print('remove ',i)
flist.remove(i)
break
return flist
def fixOneOSD(filename):
content=''
with open(filename,'r') as f:
content=f.read()
r=re.search(rea,content)
uuid=''
dev=''
if r:
uuid=r.group(1).replace('
','').replace('
','')
dev=r.group(2)
print(uuid,dev)
#recovery lvm
cmd_clear_part='dd if=/dev/zero of={} bs=1k count=6'.format(dev)
os.system(cmd_clear_part)
print('create pv ')
cmd_create_new_pv='pvcreate --force --uuid {} --restorefile {} {}'.format(uuid,filename,dev)
cmd_restore_vg='vgcfgrestore {}'.format(os.path.split(filename)[1])
cmd_active_vg='vgchange -ay {}'.format(os.path.split(filename)[1])
os.system(cmd_create_new_pv)
print('cmd_restore_vg ')
os.system(cmd_restore_vg)
print('cmd_active_vg')
os.system(cmd_active_vg)
# start osd
r=re.search(reb,content)
osdid=''
osdfsid=''
if r:
osdid=r.group(1)
osdfsid=r.group(2)
print(osdid,osdfsid)
cmd_star_mount='systemctl start ceph-volume@lvm-{osdid}-{osdfsid};systemctl enable ceph-volume@lvm-{osdid}-{osdfsid}'.format(osdid=osdid,osdfsid=osdfsid)
cmd_start_osd='systemctl start ceph-osd@{osdid};systemctl enable ceph-osd@{osdid}'.format(osdid=osdid)
print('cmd_star_mount ')
os.system(cmd_star_mount)
print('cmd_start_osd ')
os.system(cmd_start_osd)
def main():
''
# badlvms=getTargetLvms()
# for i in badlvms:
# fixOneOSD(i)
# time.sleep(3)
fixOneOSD(os.path.join('/etc/lvm/backup/','ceph-54ced685-fec0-4725-8df3-e78ad313d223'))
fixOneOSD(os.path.join('/etc/lvm/backup/','ceph-891ebdb2-3ba9-47c4-b2d4-788c4b0c1a2c'))
if __name__ == "__main__":
main()
*** 经测试 发现一个问题,有的机器从新启动的时候/dev/sda有可能会变成/dev/sdb这种情况.***
上面脚本就处理不好了,目前我是根据 /var/lib/ceph/osd/ceph-*/里面信息人工推测设备名
如果设备名不匹配服务是起不来的,会报告keyring 不匹配等错误, 执行 ceph-osd -i $osdid --flush-journal 也会报告该错误.
最笨的方法就一个一个测试,如果是对的就服务就能起来,osd就可以加入到集群中.
反复从新启动机器,只要盘里面数据没问题,基本都可恢复,ceph还是挺抗揍的.
如果确定设备名后修改/dev/sdn 到正确设备名即可
pv0 {
id = "EIGfcN-503D-MVDf-AwK1-SDtA-A4Fw-mvBqpe"
device = "/dev/sdn" # Hint only
然后从修改脚本,
fixOneOSD(os.path.join('/etc/lvm/backup/','ceph-891ebdb2-3ba9-47c4-b2d4-788c4b0c1a2c'))
从新运行脚本. 正常是要从新启动机器的. 不然执行会出问题.