[root@fuel ~]# ssh 10.254.3.2
Warning: Permanently added '10.254.3.2' (RSA) to the list of known hosts.
root@10.254.3.2's password:
Last login: Mon Sep 7 10:21:39 2015 from 10.254.3.1
[root@node-129 ~]# ssh node-8
Last login: Sun Sep 6 11:04:27 2015 from 10.254.3.1
[root@node-8 ~]# ceph status
cluster 2f103745-25e6-4fbf-a948-70ee630c5cc2
health HEALTH_WARN 4 pgs down; 3 pgs incomplete; 1 pgs peering; 4 pgs stuck inactive; 4 pgs stuck unclean; pool volumes has too few pgs
monmap e3: 3 mons at {node-129=10.133.1.3:6789/0,node-136=10.133.1.9:6789/0,node-137=10.133.1.10:6789/0}, election epoch 21680, quorum 0,1,2 node-129,node-136,node-137
osdmap e68678: 43 osds: 42 up, 42 in
pgmap v8513312: 2752 pgs, 13 pools, 6903 GB data, 1722 kobjects
13764 GB used, 952 TB / 965 TB avail
3 down+incomplete
2748 active+clean
1 down+peering
client io 1667 kB/s wr, 184 op/s
[root@node-8 ~]# cd /etc/init.d
[root@node-8 init.d]# service ceph status
=== osd.29 ===
osd.29: not running.
[root@node-8 init.d]# service ceph start
=== osd.29 ===
create-or-move updated item name 'osd.29' weight 24.01 at location {host=node-8,root=default} to crush map
Starting Ceph osd.29 on node-8...
starting osd.29 at :/0 osd_data /var/lib/ceph/osd/ceph-29 /var/lib/ceph/osd/ceph-29/journal
=== osd.29 ===
Starting Ceph osd.29 on node-8...already running
[root@node-8 init.d]# service ceph status
=== osd.29 ===
osd.29: running {"version":"0.80.7"}
[root@node-8 init.d]# ssh node-129
Warning: Permanently added 'node-129,10.133.1.3' (RSA) to the list of known hosts.
Last login: Mon Sep 7 10:22:40 2015 from 10.254.3.1
[root@node-129 ~]# ceph status
cluster 2f103745-25e6-4fbf-a948-70ee630c5cc2
health HEALTH_WARN 3 pgs backfill; 2 pgs backfilling; 3 pgs down; 4 pgs incomplete; 1 pgs peering; 9 pgs recovering; 5 pgs stuck inactive; 13 pgs stuck unclean; recovery 101755/3642852 objects degraded (2.793%); pool volumes has too few pgs
monmap e3: 3 mons at {node-129=10.133.1.3:6789/0,node-136=10.133.1.9:6789/0,node-137=10.133.1.10:6789/0}, election epoch 21680, quorum 0,1,2 node-129,node-136,node-137
osdmap e68682: 43 osds: 43 up, 43 in
pgmap v8513390: 2752 pgs, 13 pools, 6903 GB data, 1722 kobjects
14236 GB used, 975 TB / 989 TB avail
101755/3642852 objects degraded (2.793%)
1 remapped
3 down+incomplete
3 active+remapped+wait_backfill
10 active+remapped
2 active+remapped+backfilling
2722 active+clean
1 remapped+peering
9 active+recovering
1 incomplete
client io 18656 B/s wr, 5 op/s
[root@node-129 ~]# ceph osd tree
# id weight type name up/down reweight
-1 989.5 root default
-2 22.06 host node-133
0 22.06 osd.0 up 1
-3 22.06 host node-132
1 22.06 osd.1 up 1
-4 22.06 host node-131
2 22.06 osd.2 up 1
-5 22.06 host node-160
3 22.06 osd.3 up 1
-6 22.06 host node-147
24 22.06 osd.24 up 1
-7 22.06 host node-158
4 22.06 osd.4 up 1
-8 22.06 host node-145
7 22.06 osd.7 up 1
-9 22.06 host node-150
6 22.06 osd.6 up 1
-10 22.06 host node-156
8 22.06 osd.8 up 1
-11 22.06 host node-153
9 22.06 osd.9 up 1
-12 22.06 host node-159
11 22.06 osd.11 up 1
-13 22.06 host node-162
10 22.06 osd.10 up 1
-14 22.06 host node-161
12 22.06 osd.12 up 1
-15 22.06 host node-144
13 22.06 osd.13 up 1
-16 22.06 host node-146
14 22.06 osd.14 up 1
-17 22.06 host node-152
15 22.06 osd.15 up 1
-18 22.06 host node-157
17 22.06 osd.17 up 1
-19 22.06 host node-148
5 22.06 osd.5 up 1
-20 0 host node-151
-21 22.06 host node-149
20 22.06 osd.20 up 1
-22 0 host node-143
-23 22.06 host node-142
19 22.06 osd.19 up 1
-24 22.06 host node-155
16 22.06 osd.16 up 1
-25 22.06 host node-154
23 22.06 osd.23 up 1
-26 24.01 host node-1
18 24.01 osd.18 up 1
-27 24.01 host node-2
21 24.01 osd.21 up 1
-28 24.01 host node-3
22 24.01 osd.22 up 1
-29 24.01 host node-4
25 24.01 osd.25 up 1
-30 24.01 host node-5
26 24.01 osd.26 up 1
-31 24.01 host node-6
27 24.01 osd.27 up 1
-32 24.01 host node-7
28 24.01 osd.28 up 1
-33 24.01 host node-8
29 24.01 osd.29 up 1
-34 24.01 host node-9
30 24.01 osd.30 up 1
-35 24.01 host node-10
31 24.01 osd.31 up 1
-36 24.01 host node-11
32 24.01 osd.32 up 1
-37 24.01 host node-12
33 24.01 osd.33 up 1
-38 24.01 host node-13
34 24.01 osd.34 up 1
-39 24.01 host node-14
35 24.01 osd.35 up 1
-40 24.01 host node-15
36 24.01 osd.36 up 1
-41 24.01 host node-16
37 24.01 osd.37 up 1
-42 24.01 host node-17
38 24.01 osd.38 up 1
-43 24.01 host node-18
39 24.01 osd.39 up 1
-44 24.01 host node-19
40 24.01 osd.40 up 1
-45 24.01 host node-20
41 24.01 osd.41 up 1
-46 24.01 host node-21
42 24.01 osd.42 up 1
[root@node-129 ~]# ssh node-8
Last login: Mon Sep 7 10:22:59 2015 from 10.133.1.3
[root@node-8 ~]# cd /var/log
[root@node-8 log]# cd ceph
[root@node-8 ceph]# ls
ceph-osd.11.log ceph-osd.29.log-20150901.gz ceph-osd.29.log-20150905.gz
ceph-osd.11.log-20150604.gz ceph-osd.29.log-20150902.gz ceph-osd.29.log-20150906.gz
ceph-osd.11.log-20150605.gz ceph-osd.29.log-20150903.gz ceph-osd.29.log-20150907.gz
ceph-osd.29.log ceph-osd.29.log-20150904.gz
[root@node-8 ceph]# tail -n 100 ceph-osd.11.log
[root@node-8 ceph]# tail -n 100 ceph-osd.29.log
2015-09-07 10:23:53.880231 7f3663a0b7a0 0 ceph version 0.80.7 (6c0127fcb58008793d3c8b62d925bc91963672a3), process ceph-osd, pid 4201
2015-09-07 10:23:54.023144 7f3663a0b7a0 0 filestore(/var/lib/ceph/osd/ceph-29) mount detected xfs (libxfs)
2015-09-07 10:23:54.023180 7f3663a0b7a0 1 filestore(/var/lib/ceph/osd/ceph-29) disabling 'filestore replica fadvise' due to known issues with fadvise(DONTNEED) on xfs
2015-09-07 10:23:54.174461 7f3663a0b7a0 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-29) detect_features: FIEMAP ioctl is supported and appears to work
2015-09-07 10:23:54.174487 7f3663a0b7a0 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-29) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
2015-09-07 10:23:54.249462 7f3663a0b7a0 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-29) detect_features: syscall(SYS_syncfs, fd) fully supported
2015-09-07 10:23:54.249834 7f3663a0b7a0 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-29) detect_feature: extsize is disabled by conf
2015-09-07 10:23:54.875698 7f3663a0b7a0 0 filestore(/var/lib/ceph/osd/ceph-29) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
2015-09-07 10:23:54.877593 7f3663a0b7a0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway
2015-09-07 10:23:54.877611 7f3663a0b7a0 1 journal _open /var/lib/ceph/osd/ceph-29/journal fd 20: 2147483648 bytes, block size 4096 bytes, directio = 1, aio = 0
2015-09-07 10:23:55.080493 7f3663a0b7a0 1 journal _open /var/lib/ceph/osd/ceph-29/journal fd 20: 2147483648 bytes, block size 4096 bytes, directio = 1, aio = 0
2015-09-07 10:23:55.081302 7f3663a0b7a0 1 journal close /var/lib/ceph/osd/ceph-29/journal
2015-09-07 10:23:55.094775 7f3663a0b7a0 0 filestore(/var/lib/ceph/osd/ceph-29) mount detected xfs (libxfs)
2015-09-07 10:23:55.183751 7f3663a0b7a0 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-29) detect_features: FIEMAP ioctl is supported and appears to work
2015-09-07 10:23:55.183778 7f3663a0b7a0 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-29) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
2015-09-07 10:23:55.400956 7f3663a0b7a0 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-29) detect_features: syscall(SYS_syncfs, fd) fully supported
2015-09-07 10:23:55.401048 7f3663a0b7a0 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-29) detect_feature: extsize is disabled by conf
2015-09-07 10:23:55.489634 7f3663a0b7a0 0 filestore(/var/lib/ceph/osd/ceph-29) mount: WRITEAHEAD journal mode explicitly enabled in conf
2015-09-07 10:23:55.491056 7f3663a0b7a0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway
2015-09-07 10:23:55.491070 7f3663a0b7a0 1 journal _open /var/lib/ceph/osd/ceph-29/journal fd 21: 2147483648 bytes, block size 4096 bytes, directio = 1, aio = 0
2015-09-07 10:23:55.492196 7f3663a0b7a0 1 journal _open /var/lib/ceph/osd/ceph-29/journal fd 21: 2147483648 bytes, block size 4096 bytes, directio = 1, aio = 0
2015-09-07 10:23:55.544064 7f3663a0b7a0 0 <cls> cls/hello/cls_hello.cc:271: loading cls_hello
2015-09-07 10:23:55.600770 7f3663a0b7a0 0 osd.29 68635 crush map has features 1107558400, adjusting msgr requires for clients
2015-09-07 10:23:55.600796 7f3663a0b7a0 0 osd.29 68635 crush map has features 1107558400 was 8705, adjusting msgr requires for mons
2015-09-07 10:23:55.600800 7f3663a0b7a0 0 osd.29 68635 crush map has features 1107558400, adjusting msgr requires for osds
2015-09-07 10:23:55.600815 7f3663a0b7a0 0 osd.29 68635 load_pgs
2015-09-07 10:23:57.462595 7f3663a0b7a0 0 osd.29 68635 load_pgs opened 144 pgs
2015-09-07 10:23:57.468481 7f3650a66700 0 osd.29 68635 ignoring osdmap until we have initialized
2015-09-07 10:23:57.468669 7f3650a66700 0 osd.29 68635 ignoring osdmap until we have initialized
2015-09-07 10:24:04.070596 7f3663a0b7a0 0 osd.29 68635 done with init, starting boot process
[root@node-8 ceph]# tail -n 100 ceph-osd.29.log | grep ERROR
[root@node-8 ceph]# tail -n 100 ceph-osd.29.log | grep error
[root@node-8 ceph]# tail -n 100 ceph-osd.29.log | grep Error
[root@node-8 ceph]# tail -n 1000 ceph-osd.29.log | grep Error
[root@node-8 ceph]# tail -n 1000 ceph-osd.29.log | grep error
[root@node-8 ceph]# tail -n 1000 ceph-osd.29.log | grep ERROR