auto lo
iface lo inet loopback
auto enp50s0f0
iface enp50s0f0 inet static
address 192.168.1.6
netmask 255.255.255.0
iface enp46s0f1 inet manual
iface enp0s20f0u1u6 inet manual
iface enp50s0f1 inet manual
iface enp50s0f2 inet manual
iface enp50s0f3 inet manual
iface enp28s0f0 inet manual
iface enp28s0f1 inet manual
iface enp46s0f0 inet manual
auto bond0
iface bond0 inet manual
bond-slaves enp28s0f0 enp46s0f0
bond-miimon 100
bond-mode balance-rr
auto bond0.7
iface bond0.7 inet manual
auto bond0.8
iface bond0.8 inet manual
auto bond1
iface bond1 inet static
address 192.168.3.6
netmask 255.255.255.0
bond-slaves enp28s0f1 enp46s0f1
bond-miimon 100
bond-mode balance-rr
auto vmbr8
iface vmbr8 inet manual
bridge-ports bond0.8
bridge-stp off
bridge-fd 0
auto vmbr7
iface vmbr7 inet static
address 192.168.7.206
netmask 255.255.255.0
gateway 192.168.7.254
bridge-ports bond0.7
bridge-stp off
bridge-fd 0
----------------------------------------------
rm -rf /etc/apt/sources.list.d/pve-enterprise.list
添加社区源
wget -q -O- 'http://download.proxmox.com/debian/pve/dists/stretch/proxmox-ve-release-5.x.gpg' | apt-key add -
echo "deb http://download.proxmox.com/debian/pve stretch pve-no-subscription" > /etc/apt/sources.list.d/pve-no-subscription.list
debian源改为阿里源
> /etc/apt/sources.list
deb http://mirrors.aliyun.com/debian stretch main contrib non-free
deb-src http://mirrors.aliyun.com/debian stretch main contrib non-free
deb http://mirrors.aliyun.com/debian stretch-updates main contrib non-free
deb-src http://mirrors.aliyun.com/debian stretch-updates main contrib non-free
deb http://mirrors.aliyun.com/debian-security stretch/updates main contrib non-free
deb-src http://mirrors.aliyun.com/debian-security stretch/updates main contrib non-free
apt clean all 没有用
apt update && apt dist-upgrade -y
这一步较慢，主要是pve-kernel，用了1个半小时，平均20kB/s，下午15：00-17：00
还有就是ceph-common，ceph-mon,ceph-osd也较慢，平均20kB/s
ceph版本是12.2.12
pve版本是5.4-4
慢的原因主要是这个网站的问题http://download.proxmox.com/debian/ceph-luminous stretch/main
Partition table scan:
MBR: protective
BSD: not present
APM: not present
GPT: present
root@pve04:~# gdisk /dev/sdk GPT fdisk (gdisk) version 1.0.1 Partition table scan: MBR: protective BSD: not present APM: not present GPT: present Found valid GPT with protective MBR; using GPT. Command (? for help): p Disk /dev/sdk: 249938560 sectors, 119.2 GiB Logical sector size: 512 bytes Disk identifier (GUID): ADDBB660-E1C5-4799-A896-E55C3556BF6D Partition table holds up to 128 entries First usable sector is 34, last usable sector is 249938526 Partitions will be aligned on 2-sector boundaries Total free space is 0 sectors (0 bytes) Number Start (sector) End (sector) Size Code Name 1 34 2047 1007.0 KiB EF02 2 2048 1050623 512.0 MiB EF00 3 1050624 249938526 118.7 GiB 8E00 root@cu-pve05:~# gdisk /dev/sdk GPT fdisk (gdisk) version 1.0.1 Partition table scan: MBR: protective BSD: not present APM: not present GPT: present Found valid GPT with protective MBR; using GPT. Command (? for help): p Disk /dev/sdk: 249938560 sectors, 119.2 GiB Logical sector size: 512 bytes Disk identifier (GUID): FE3DB5F2-5990-48C3-BF94-E3AC1A59C97A Partition table holds up to 128 entries First usable sector is 34, last usable sector is 249938526 Partitions will be aligned on 2-sector boundaries Total free space is 0 sectors (0 bytes) Number Start (sector) End (sector) Size Code Name 1 34 2047 1007.0 KiB EF02 2 2048 1050623 512.0 MiB EF00 3 1050624 249938526 118.7 GiB 8E00 Command (? for help): root@pve04:~# sdk 8:160 0 119.2G 0 disk ├─sdk1 8:161 0 1007K 0 part ├─sdk2 8:162 0 512M 0 part /boot/efi └─sdk3 8:163 0 118.7G 0 part ├─pve-swap 253:0 0 8G 0 lvm [SWAP] ├─pve-root 253:1 0 29.5G 0 lvm / ├─pve-data_tmeta 253:2 0 1G 0 lvm │ └─pve-data 253:4 0 64.4G 0 lvm └─pve-data_tdata 253:3 0 64.4G 0 lvm └─pve-data 253:4 0 64.4G 0 lvm root@cu-pve05:~# sdk 8:160 0 119.2G 0 disk ├─sdk1 8:161 0 1007K 0 part ├─sdk2 8:162 0 512M 0 part /boot/efi └─sdk3 8:163 0 118.7G 0 part ├─pve-swap 253:0 0 8G 0 lvm [SWAP] ├─pve-root 253:1 0 29.5G 0 lvm / ├─pve-data_tmeta 253:2 0 1G 0 lvm │ └─pve-data 253:4 0 64.4G 0 lvm └─pve-data_tdata 253:3 0 64.4G 0 lvm └─pve-data 253:4 0 64.4G 0 lvm root@cu-pve06:~# sdk 8:160 0 119.2G 0 disk ├─sdk1 8:161 0 1007K 0 part ├─sdk2 8:162 0 512M 0 part /boot/efi └─sdk3 8:163 0 118.7G 0 part ├─pve-swap 253:0 0 8G 0 lvm [SWAP] ├─pve-root 253:1 0 29.5G 0 lvm / ├─pve-data_tmeta 253:2 0 1G 0 lvm │ └─pve-data 253:4 0 64.4G 0 lvm └─pve-data_tdata 253:3 0 64.4G 0 lvm └─pve-data 253:4 0 64.4G 0 lvm
清除磁盘分区，以便在web界面上能看到osd，因为之前做过，所以再次做的时候会有这个问题
ceph-disk list
ceph-disk zap /dev/sd[abcdefghij]
root@cu-pve04:~# ceph-disk list /dev/dm-0 swap, swap /dev/dm-1 other, xfs, mounted on / /dev/dm-2 other, unknown /dev/dm-3 other, unknown /dev/dm-4 other, unknown /dev/loop0 other, unknown /dev/loop1 other, unknown /dev/loop2 other, unknown /dev/loop3 other, unknown /dev/loop4 other, unknown /dev/loop5 other, unknown /dev/loop6 other, unknown /dev/loop7 other, unknown /dev/sda : /dev/sda1 ceph block.db, for /dev/sdb1 /dev/sda2 ceph block.db, for /dev/sdc1 /dev/sda3 ceph block.db, for /dev/sde1 /dev/sda4 ceph block.db, for /dev/sdf1 /dev/sdb : /dev/sdb1 ceph data, prepared, unknown cluster 439800b8-b1cd-43bd-b365-8f7b33ba5994, osd.16, block /dev/sdb2, block.db /dev/sda1 /dev/sdb2 ceph block, for /dev/sdb1 /dev/sdc : /dev/sdc1 ceph data, prepared, unknown cluster 439800b8-b1cd-43bd-b365-8f7b33ba5994, osd.17, block /dev/sdc2, block.db /dev/sda2 /dev/sdc2 ceph block, for /dev/sdc1 /dev/sdd : /dev/sdd1 ceph block.db, for /dev/sdg1 /dev/sdd2 ceph block.db, for /dev/sdh1 /dev/sdd3 ceph block.db, for /dev/sdi1 /dev/sdd4 ceph block.db, for /dev/sdj1 /dev/sde : /dev/sde1 ceph data, prepared, unknown cluster 439800b8-b1cd-43bd-b365-8f7b33ba5994, osd.18, block /dev/sde2, block.db /dev/sda3 /dev/sde2 ceph block, for /dev/sde1 /dev/sdf : /dev/sdf1 ceph data, prepared, unknown cluster 439800b8-b1cd-43bd-b365-8f7b33ba5994, osd.19, block /dev/sdf2, block.db /dev/sda4 /dev/sdf2 ceph block, for /dev/sdf1 /dev/sdg : /dev/sdg1 ceph data, prepared, unknown cluster 439800b8-b1cd-43bd-b365-8f7b33ba5994, osd.20, block /dev/sdg2, block.db /dev/sdd1 /dev/sdg2 ceph block, for /dev/sdg1 /dev/sdh : /dev/sdh1 ceph data, prepared, unknown cluster 439800b8-b1cd-43bd-b365-8f7b33ba5994, osd.21, block /dev/sdh2, block.db /dev/sdd2 /dev/sdh2 ceph block, for /dev/sdh1 /dev/sdi : /dev/sdi1 ceph data, prepared, unknown cluster 439800b8-b1cd-43bd-b365-8f7b33ba5994, osd.22, block /dev/sdi2, block.db /dev/sdd3 /dev/sdi2 ceph block, for /dev/sdi1 /dev/sdj : /dev/sdj1 ceph data, prepared, unknown cluster 439800b8-b1cd-43bd-b365-8f7b33ba5994, osd.23, block /dev/sdj2, block.db /dev/sdd4 /dev/sdj2 ceph block, for /dev/sdj1 /dev/sdk : /dev/sdk1 other, 21686148-6449-6e6f-744e-656564454649 /dev/sdk2 other, vfat, mounted on /boot/efi /dev/sdk3 other, LVM2_member root@cu-pve04:~# -------------------------------------------------------------------------- 清除原有分区表 root@cu-pve05:~# ceph-disk zap /dev/sd[abcdefghij] root@cu-pve04:~# ceph-disk zap /dev/sdb 110+0 records in 110+0 records out 115343360 bytes (115 MB, 110 MiB) copied, 0.989543 s, 117 MB/s /dev/sdb1: 4 bytes were erased at offset 0x00000000 (xfs): 58 46 53 42 100+0 records in 100+0 records out 104857600 bytes (105 MB, 100 MiB) copied, 0.475141 s, 221 MB/s Caution: invalid backup GPT header, but valid main header; regenerating backup header from main header. Warning! Main and backup partition tables differ! Use the 'c' and 'e' options on the recovery & transformation menu to examine the two tables. Warning! One or more CRCs don't match. You should repair the disk! **************************************************************************** Caution: Found protective or hybrid MBR and corrupt GPT. Using GPT, but disk verification and recovery are STRONGLY recommended. **************************************************************************** GPT data structures destroyed! You may now partition the disk using fdisk or other utilities. Creating new GPT entries. The operation has completed successfully. root@cu-pve04:~# -------------------------------------------------------------------------- web界面上创建osd的日志 create OSD on /dev/sdb (bluestore) using device '/dev/sda' for block.db wipe disk/partition: /dev/sdb 200+0 records in 200+0 records out 209715200 bytes (210 MB, 200 MiB) copied, 0.921603 s, 228 MB/s Creating new GPT entries. GPT data structures destroyed! You may now partition the disk using fdisk or other utilities. Creating new GPT entries. The operation has completed successfully. Setting name! partNum is 0 REALLY setting name! The operation has completed successfully. prepare_device: OSD will not be hot-swappable if block.db is not the same device as the osd data Setting name! partNum is 0 REALLY setting name! The operation has completed successfully. The operation has completed successfully. Setting name! partNum is 1 REALLY setting name! The operation has completed successfully. The operation has completed successfully. meta-data=/dev/sdb1 isize=2048 agcount=4, agsize=6400 blks = sectsz=4096 attr=2, projid32bit=1 = crc=1 finobt=1, sparse=0, rmapbt=0, reflink=0 data = bsize=4096 blocks=25600, imaxpct=25 = sunit=0 swidth=0 blks naming =version 2 bsize=4096 ascii-ci=0 ftype=1 log =internal log bsize=4096 blocks=1608, version=2 = sectsz=4096 sunit=1 blks, lazy-count=1 realtime =none extsz=4096 blocks=0, rtextents=0 Warning: The kernel is still using the old partition table. The new table will be used at the next reboot or after you run partprobe(8) or kpartx(8) The operation has completed successfully. TASK OK 创建8个osd,一个ssd缓存盘对应4个hdd数据盘。 -------------------------------------------------------------------------- root@cu-pve04:~# ceph-disk list /dev/dm-0 swap, swap /dev/dm-1 other, xfs, mounted on / /dev/dm-2 other, unknown /dev/dm-3 other, unknown /dev/dm-4 other, unknown /dev/loop0 other, unknown /dev/loop1 other, unknown /dev/loop2 other, unknown /dev/loop3 other, unknown /dev/loop4 other, unknown /dev/loop5 other, unknown /dev/loop6 other, unknown /dev/loop7 other, unknown /dev/sda : /dev/sda1 ceph block.db, for /dev/sdb1 /dev/sda2 ceph block.db, for /dev/sdc1 /dev/sda3 ceph block.db, for /dev/sde1 /dev/sda4 ceph block.db, for /dev/sdf1 /dev/sdb : /dev/sdb1 ceph data, active, cluster ceph, osd.0, block /dev/sdb2, block.db /dev/sda1 /dev/sdb2 ceph block, for /dev/sdb1 /dev/sdc : /dev/sdc1 ceph data, active, cluster ceph, osd.1, block /dev/sdc2, block.db /dev/sda2 /dev/sdc2 ceph block, for /dev/sdc1 /dev/sdd : /dev/sdd1 ceph block.db, for /dev/sdg1 /dev/sdd2 ceph block.db, for /dev/sdh1 /dev/sdd3 ceph block.db, for /dev/sdi1 /dev/sdd4 ceph block.db, for /dev/sdj1 /dev/sde : /dev/sde1 ceph data, active, cluster ceph, osd.2, block /dev/sde2, block.db /dev/sda3 /dev/sde2 ceph block, for /dev/sde1 /dev/sdf : /dev/sdf1 ceph data, active, cluster ceph, osd.3, block /dev/sdf2, block.db /dev/sda4 /dev/sdf2 ceph block, for /dev/sdf1 /dev/sdg : /dev/sdg1 ceph data, active, cluster ceph, osd.4, block /dev/sdg2, block.db /dev/sdd1 /dev/sdg2 ceph block, for /dev/sdg1 /dev/sdh : /dev/sdh1 ceph data, active, cluster ceph, osd.5, block /dev/sdh2, block.db /dev/sdd2 /dev/sdh2 ceph block, for /dev/sdh1 /dev/sdi : /dev/sdi1 ceph data, active, cluster ceph, osd.6, block /dev/sdi2, block.db /dev/sdd3 /dev/sdi2 ceph block, for /dev/sdi1 /dev/sdj : /dev/sdj1 ceph data, active, cluster ceph, osd.7, block /dev/sdj2, block.db /dev/sdd4 /dev/sdj2 ceph block, for /dev/sdj1 /dev/sdk : /dev/sdk1 other, 21686148-6449-6e6f-744e-656564454649 /dev/sdk2 other, vfat, mounted on /boot/efi /dev/sdk3 other, LVM2_member root@cu-pve04:~#