Note: "permalinks" may not be as permanent as we would like,
direct links of old sources may well be a few messages off.
On 2017-08-09 20:38, Lars Ellenberg wrote:
> IO stack (e.g. lsblk ; lsblk -t ; lsblk -D) may be interesting,
> as well as the drbd configuration (drbdadm dump ),
> the kernel logs around the "crash" if possible,
> and your "best guess" as to what the box may have been doing
> when it became unresponsive.
This is with 9.0.9rc1 on kvm08 only (one01 and kvm12 have 9.0.8) and
prior to starting the VM (one01) on kvm08. I get the same result if all
three systems have 9.0.9.
[root at kvm08 ~]# modinfo drbd
filename:
/lib/modules/3.10.0-514.26.2.el7.x86_64/extra/drbd/drbd.ko
alias: block-major-147-*
license: GPL
version: 9.0.9rc1-1
description: drbd - Distributed Replicated Block Device v9.0.9rc1-1
author: Philipp Reisner <phil at linbit.com>, Lars Ellenberg
<lars at linbit.com>
rhelversion: 7.3
srcversion: 0A810366A84785163288DD2
depends: libcrc32c
vermagic: 3.10.0-514.26.2.el7.x86_64 SMP mod_unload modversions
parm: minor_count:Approximate number of drbd devices (1-255)
(uint)
parm: disable_sendpage:bool
parm: allow_oos:DONT USE! (bool)
parm: enable_faults:int
parm: fault_rate:int
parm: fault_count:int
parm: fault_devs:int
parm: two_phase_commit_fail:int
parm: usermode_helper:string
[root at kvm08 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 1.8T 0 disk
├─sda1 8:1 0 200M 0 part /boot/efi
├─sda2 8:2 0 1G 0 part
│ └─md126 9:126 0 1024M 0 raid1 /boot
└─sda3 8:3 0 1.8T 0 part
└─md127 9:127 0 3.7T 0 raid10
├─vg_sys-lv_sysroot 253:0 0 20G 0 lvm /
├─vg_sys-lv_swap 253:1 0 8G 0 lvm [SWAP]
├─vg_sys-lv_drbd_pool_tmeta 253:2 0 4M 0 lvm
│ └─vg_sys-lv_drbd_pool-tpool 253:4 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool 253:5 0 1T 0 lvm
├─vg_sys-lv_drbd_pool_tdata 253:3 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool-tpool 253:4 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool 253:5 0 1T 0 lvm
├─vg_sys-lv_one_pool_tmeta 253:6 0 4M 0 lvm
│ └─vg_sys-lv_one_pool-tpool 253:8 0 30G 0 lvm
│ ├─vg_sys-lv_one_pool 253:9 0 30G 0 lvm
│ └─vg_sys-lv_one01_disk01 253:16 0 100G 0 lvm
├─vg_sys-lv_one_pool_tdata 253:7 0 30G 0 lvm
│ └─vg_sys-lv_one_pool-tpool 253:8 0 30G 0 lvm
│ ├─vg_sys-lv_one_pool 253:9 0 30G 0 lvm
│ └─vg_sys-lv_one01_disk01 253:16 0 100G 0 lvm
├─vg_sys-lv_sys_pool_tmeta 253:10 0 32M 0 lvm
│ └─vg_sys-lv_sys_pool-tpool 253:12 0 30G 0 lvm
│ ├─vg_sys-lv_sys_pool 253:13 0 30G 0 lvm
│ ├─vg_sys-lv_save 253:14 0 16G 0 lvm
/var/lib/libvirt/save
│ └─vg_sys-lv_images 253:15 0 10G 0 lvm
/var/lib/libvirt/images
├─vg_sys-lv_sys_pool_tdata 253:11 0 30G 0 lvm
│ └─vg_sys-lv_sys_pool-tpool 253:12 0 30G 0 lvm
│ ├─vg_sys-lv_sys_pool 253:13 0 30G 0 lvm
│ ├─vg_sys-lv_save 253:14 0 16G 0 lvm
/var/lib/libvirt/save
│ └─vg_sys-lv_images 253:15 0 10G 0 lvm
/var/lib/libvirt/images
├─vg_sys-.drbdctrl_0 253:17 0 4M 0 lvm
│ └─drbd0 147:0 0 4M 0 disk
└─vg_sys-.drbdctrl_1 253:18 0 4M 0 lvm
└─drbd1 147:1 0 4M 0 disk
sdb 8:16 0 1.8T 0 disk
├─sdb1 8:17 0 200M 0 part /boot/efi1
├─sdb2 8:18 0 1G 0 part
│ └─md126 9:126 0 1024M 0 raid1 /boot
└─sdb3 8:19 0 1.8T 0 part
└─md127 9:127 0 3.7T 0 raid10
├─vg_sys-lv_sysroot 253:0 0 20G 0 lvm /
├─vg_sys-lv_swap 253:1 0 8G 0 lvm [SWAP]
├─vg_sys-lv_drbd_pool_tmeta 253:2 0 4M 0 lvm
│ └─vg_sys-lv_drbd_pool-tpool 253:4 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool 253:5 0 1T 0 lvm
├─vg_sys-lv_drbd_pool_tdata 253:3 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool-tpool 253:4 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool 253:5 0 1T 0 lvm
├─vg_sys-lv_one_pool_tmeta 253:6 0 4M 0 lvm
│ └─vg_sys-lv_one_pool-tpool 253:8 0 30G 0 lvm
│ ├─vg_sys-lv_one_pool 253:9 0 30G 0 lvm
│ └─vg_sys-lv_one01_disk01 253:16 0 100G 0 lvm
├─vg_sys-lv_one_pool_tdata 253:7 0 30G 0 lvm
│ └─vg_sys-lv_one_pool-tpool 253:8 0 30G 0 lvm
│ ├─vg_sys-lv_one_pool 253:9 0 30G 0 lvm
│ └─vg_sys-lv_one01_disk01 253:16 0 100G 0 lvm
├─vg_sys-lv_sys_pool_tmeta 253:10 0 32M 0 lvm
│ └─vg_sys-lv_sys_pool-tpool 253:12 0 30G 0 lvm
│ ├─vg_sys-lv_sys_pool 253:13 0 30G 0 lvm
│ ├─vg_sys-lv_save 253:14 0 16G 0 lvm
/var/lib/libvirt/save
│ └─vg_sys-lv_images 253:15 0 10G 0 lvm
/var/lib/libvirt/images
├─vg_sys-lv_sys_pool_tdata 253:11 0 30G 0 lvm
│ └─vg_sys-lv_sys_pool-tpool 253:12 0 30G 0 lvm
│ ├─vg_sys-lv_sys_pool 253:13 0 30G 0 lvm
│ ├─vg_sys-lv_save 253:14 0 16G 0 lvm
/var/lib/libvirt/save
│ └─vg_sys-lv_images 253:15 0 10G 0 lvm
/var/lib/libvirt/images
├─vg_sys-.drbdctrl_0 253:17 0 4M 0 lvm
│ └─drbd0 147:0 0 4M 0 disk
└─vg_sys-.drbdctrl_1 253:18 0 4M 0 lvm
└─drbd1 147:1 0 4M 0 disk
sdc 8:32 0 1.8T 0 disk
├─sdc1 8:33 0 200M 0 part /boot/efi2
├─sdc2 8:34 0 1G 0 part
│ └─md126 9:126 0 1024M 0 raid1 /boot
└─sdc3 8:35 0 1.8T 0 part
└─md127 9:127 0 3.7T 0 raid10
├─vg_sys-lv_sysroot 253:0 0 20G 0 lvm /
├─vg_sys-lv_swap 253:1 0 8G 0 lvm [SWAP]
├─vg_sys-lv_drbd_pool_tmeta 253:2 0 4M 0 lvm
│ └─vg_sys-lv_drbd_pool-tpool 253:4 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool 253:5 0 1T 0 lvm
├─vg_sys-lv_drbd_pool_tdata 253:3 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool-tpool 253:4 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool 253:5 0 1T 0 lvm
├─vg_sys-lv_one_pool_tmeta 253:6 0 4M 0 lvm
│ └─vg_sys-lv_one_pool-tpool 253:8 0 30G 0 lvm
│ ├─vg_sys-lv_one_pool 253:9 0 30G 0 lvm
│ └─vg_sys-lv_one01_disk01 253:16 0 100G 0 lvm
├─vg_sys-lv_one_pool_tdata 253:7 0 30G 0 lvm
│ └─vg_sys-lv_one_pool-tpool 253:8 0 30G 0 lvm
│ ├─vg_sys-lv_one_pool 253:9 0 30G 0 lvm
│ └─vg_sys-lv_one01_disk01 253:16 0 100G 0 lvm
├─vg_sys-lv_sys_pool_tmeta 253:10 0 32M 0 lvm
│ └─vg_sys-lv_sys_pool-tpool 253:12 0 30G 0 lvm
│ ├─vg_sys-lv_sys_pool 253:13 0 30G 0 lvm
│ ├─vg_sys-lv_save 253:14 0 16G 0 lvm
/var/lib/libvirt/save
│ └─vg_sys-lv_images 253:15 0 10G 0 lvm
/var/lib/libvirt/images
├─vg_sys-lv_sys_pool_tdata 253:11 0 30G 0 lvm
│ └─vg_sys-lv_sys_pool-tpool 253:12 0 30G 0 lvm
│ ├─vg_sys-lv_sys_pool 253:13 0 30G 0 lvm
│ ├─vg_sys-lv_save 253:14 0 16G 0 lvm
/var/lib/libvirt/save
│ └─vg_sys-lv_images 253:15 0 10G 0 lvm
/var/lib/libvirt/images
├─vg_sys-.drbdctrl_0 253:17 0 4M 0 lvm
│ └─drbd0 147:0 0 4M 0 disk
└─vg_sys-.drbdctrl_1 253:18 0 4M 0 lvm
└─drbd1 147:1 0 4M 0 disk
sdd 8:48 0 1.8T 0 disk
├─sdd1 8:49 0 200M 0 part /boot/efi3
├─sdd2 8:50 0 1G 0 part
│ └─md126 9:126 0 1024M 0 raid1 /boot
└─sdd3 8:51 0 1.8T 0 part
└─md127 9:127 0 3.7T 0 raid10
├─vg_sys-lv_sysroot 253:0 0 20G 0 lvm /
├─vg_sys-lv_swap 253:1 0 8G 0 lvm [SWAP]
├─vg_sys-lv_drbd_pool_tmeta 253:2 0 4M 0 lvm
│ └─vg_sys-lv_drbd_pool-tpool 253:4 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool 253:5 0 1T 0 lvm
├─vg_sys-lv_drbd_pool_tdata 253:3 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool-tpool 253:4 0 1T 0 lvm
│ └─vg_sys-lv_drbd_pool 253:5 0 1T 0 lvm
├─vg_sys-lv_one_pool_tmeta 253:6 0 4M 0 lvm
│ └─vg_sys-lv_one_pool-tpool 253:8 0 30G 0 lvm
│ ├─vg_sys-lv_one_pool 253:9 0 30G 0 lvm
│ └─vg_sys-lv_one01_disk01 253:16 0 100G 0 lvm
├─vg_sys-lv_one_pool_tdata 253:7 0 30G 0 lvm
│ └─vg_sys-lv_one_pool-tpool 253:8 0 30G 0 lvm
│ ├─vg_sys-lv_one_pool 253:9 0 30G 0 lvm
│ └─vg_sys-lv_one01_disk01 253:16 0 100G 0 lvm
├─vg_sys-lv_sys_pool_tmeta 253:10 0 32M 0 lvm
│ └─vg_sys-lv_sys_pool-tpool 253:12 0 30G 0 lvm
│ ├─vg_sys-lv_sys_pool 253:13 0 30G 0 lvm
│ ├─vg_sys-lv_save 253:14 0 16G 0 lvm
/var/lib/libvirt/save
│ └─vg_sys-lv_images 253:15 0 10G 0 lvm
/var/lib/libvirt/images
├─vg_sys-lv_sys_pool_tdata 253:11 0 30G 0 lvm
│ └─vg_sys-lv_sys_pool-tpool 253:12 0 30G 0 lvm
│ ├─vg_sys-lv_sys_pool 253:13 0 30G 0 lvm
│ ├─vg_sys-lv_save 253:14 0 16G 0 lvm
/var/lib/libvirt/save
│ └─vg_sys-lv_images 253:15 0 10G 0 lvm
/var/lib/libvirt/images
├─vg_sys-.drbdctrl_0 253:17 0 4M 0 lvm
│ └─drbd0 147:0 0 4M 0 disk
└─vg_sys-.drbdctrl_1 253:18 0 4M 0 lvm
└─drbd1 147:1 0 4M 0 disk
[root at kvm08 ~]# lsblk -t
NAME ALIGNMENT MIN-IO OPT-IO PHY-SEC
LOG-SEC ROTA SCHED RQ-SIZE RA WSAME
sda 0 512 0 512
512 1 cfq 128 4096 0B
├─sda1 0 512 0 512
512 1 cfq 128 4096 0B
├─sda2 0 512 0 512
512 1 cfq 128 4096 0B
│ └─md126 0 512 0 512
512 1 128 4096 0B
└─sda3 0 512 0 512
512 1 cfq 128 4096 0B
└─md127 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sysroot 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_swap 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_drbd_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_drbd_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_one_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_one_pool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one01_disk01 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_one_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_one_pool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one01_disk01 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sys_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_sys_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_sys_pool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_save 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_images 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sys_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_sys_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_sys_pool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_save 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_images 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-.drbdctrl_0 0 524288 1048576 512
512 1 128 4096 0B
│ └─drbd0 0 524288 1048576 512
512 1 128 4096 0B
└─vg_sys-.drbdctrl_1 0 524288 1048576 512
512 1 128 4096 0B
└─drbd1 0 524288 1048576 512
512 1 128 4096 0B
sdb 0 512 0 512
512 1 cfq 128 4096 0B
├─sdb1 0 512 0 512
512 1 cfq 128 4096 0B
├─sdb2 0 512 0 512
512 1 cfq 128 4096 0B
│ └─md126 0 512 0 512
512 1 128 4096 0B
└─sdb3 0 512 0 512
512 1 cfq 128 4096 0B
└─md127 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sysroot 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_swap 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_drbd_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_drbd_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_one_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_one_pool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one01_disk01 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_one_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_one_pool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one01_disk01 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sys_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_sys_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_sys_pool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_save 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_images 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sys_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_sys_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_sys_pool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_save 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_images 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-.drbdctrl_0 0 524288 1048576 512
512 1 128 4096 0B
│ └─drbd0 0 524288 1048576 512
512 1 128 4096 0B
└─vg_sys-.drbdctrl_1 0 524288 1048576 512
512 1 128 4096 0B
└─drbd1 0 524288 1048576 512
512 1 128 4096 0B
sdc 0 512 0 512
512 1 cfq 128 4096 0B
├─sdc1 0 512 0 512
512 1 cfq 128 4096 0B
├─sdc2 0 512 0 512
512 1 cfq 128 4096 0B
│ └─md126 0 512 0 512
512 1 128 4096 0B
└─sdc3 0 512 0 512
512 1 cfq 128 4096 0B
└─md127 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sysroot 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_swap 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_drbd_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_drbd_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_one_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_one_pool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one01_disk01 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_one_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_one_pool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one01_disk01 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sys_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_sys_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_sys_pool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_save 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_images 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sys_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_sys_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_sys_pool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_save 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_images 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-.drbdctrl_0 0 524288 1048576 512
512 1 128 4096 0B
│ └─drbd0 0 524288 1048576 512
512 1 128 4096 0B
└─vg_sys-.drbdctrl_1 0 524288 1048576 512
512 1 128 4096 0B
└─drbd1 0 524288 1048576 512
512 1 128 4096 0B
sdd 0 512 0 512
512 1 cfq 128 4096 0B
├─sdd1 0 512 0 512
512 1 cfq 128 4096 0B
├─sdd2 0 512 0 512
512 1 cfq 128 4096 0B
│ └─md126 0 512 0 512
512 1 128 4096 0B
└─sdd3 0 512 0 512
512 1 cfq 128 4096 0B
└─md127 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sysroot 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_swap 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_drbd_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_drbd_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_drbd_pool 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_one_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_one_pool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one01_disk01 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_one_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_one_pool 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_one01_disk01 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sys_pool_tmeta 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_sys_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_sys_pool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_save 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_images 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-lv_sys_pool_tdata 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_sys_pool-tpool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_sys_pool 0 524288 1048576 512
512 1 128 4096 0B
│ ├─vg_sys-lv_save 0 524288 1048576 512
512 1 128 4096 0B
│ └─vg_sys-lv_images 0 524288 1048576 512
512 1 128 4096 0B
├─vg_sys-.drbdctrl_0 0 524288 1048576 512
512 1 128 4096 0B
│ └─drbd0 0 524288 1048576 512
512 1 128 4096 0B
└─vg_sys-.drbdctrl_1 0 524288 1048576 512
512 1 128 4096 0B
└─drbd1 0 524288 1048576 512
512 1 128 4096 0B
[root at kvm08 ~]# lsblk -D
NAME DISC-ALN DISC-GRAN DISC-MAX DISC-ZERO
sda 0 0B 0B 0
├─sda1 0 0B 0B 0
├─sda2 0 0B 0B 0
│ └─md126 0 0B 0B 0
└─sda3 0 0B 0B 0
└─md127 0 0B 512K 0
├─vg_sys-lv_sysroot 0 0B 0B 0
├─vg_sys-lv_swap 0 0B 0B 0
├─vg_sys-lv_drbd_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool-tpool 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool 0 0B 0B 0
├─vg_sys-lv_drbd_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool-tpool 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool 0 0B 0B 0
├─vg_sys-lv_one_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_one_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_one_pool 0 0B 0B 0
│ └─vg_sys-lv_one01_disk01 0 512K 16G 0
├─vg_sys-lv_one_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_one_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_one_pool 0 0B 0B 0
│ └─vg_sys-lv_one01_disk01 0 512K 16G 0
├─vg_sys-lv_sys_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_sys_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_sys_pool 0 0B 0B 0
│ ├─vg_sys-lv_save 0 512K 16G 0
│ └─vg_sys-lv_images 0 512K 16G 0
├─vg_sys-lv_sys_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_sys_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_sys_pool 0 0B 0B 0
│ ├─vg_sys-lv_save 0 512K 16G 0
│ └─vg_sys-lv_images 0 512K 16G 0
├─vg_sys-.drbdctrl_0 0 0B 0B 0
│ └─drbd0 0 0B 0B 0
└─vg_sys-.drbdctrl_1 0 0B 0B 0
└─drbd1 0 0B 0B 0
sdb 0 0B 0B 0
├─sdb1 0 0B 0B 0
├─sdb2 0 0B 0B 0
│ └─md126 0 0B 0B 0
└─sdb3 0 0B 0B 0
└─md127 0 0B 512K 0
├─vg_sys-lv_sysroot 0 0B 0B 0
├─vg_sys-lv_swap 0 0B 0B 0
├─vg_sys-lv_drbd_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool-tpool 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool 0 0B 0B 0
├─vg_sys-lv_drbd_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool-tpool 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool 0 0B 0B 0
├─vg_sys-lv_one_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_one_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_one_pool 0 0B 0B 0
│ └─vg_sys-lv_one01_disk01 0 512K 16G 0
├─vg_sys-lv_one_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_one_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_one_pool 0 0B 0B 0
│ └─vg_sys-lv_one01_disk01 0 512K 16G 0
├─vg_sys-lv_sys_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_sys_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_sys_pool 0 0B 0B 0
│ ├─vg_sys-lv_save 0 512K 16G 0
│ └─vg_sys-lv_images 0 512K 16G 0
├─vg_sys-lv_sys_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_sys_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_sys_pool 0 0B 0B 0
│ ├─vg_sys-lv_save 0 512K 16G 0
│ └─vg_sys-lv_images 0 512K 16G 0
├─vg_sys-.drbdctrl_0 0 0B 0B 0
│ └─drbd0 0 0B 0B 0
└─vg_sys-.drbdctrl_1 0 0B 0B 0
└─drbd1 0 0B 0B 0
sdc 0 0B 0B 0
├─sdc1 0 0B 0B 0
├─sdc2 0 0B 0B 0
│ └─md126 0 0B 0B 0
└─sdc3 0 0B 0B 0
└─md127 0 0B 512K 0
├─vg_sys-lv_sysroot 0 0B 0B 0
├─vg_sys-lv_swap 0 0B 0B 0
├─vg_sys-lv_drbd_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool-tpool 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool 0 0B 0B 0
├─vg_sys-lv_drbd_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool-tpool 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool 0 0B 0B 0
├─vg_sys-lv_one_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_one_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_one_pool 0 0B 0B 0
│ └─vg_sys-lv_one01_disk01 0 512K 16G 0
├─vg_sys-lv_one_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_one_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_one_pool 0 0B 0B 0
│ └─vg_sys-lv_one01_disk01 0 512K 16G 0
├─vg_sys-lv_sys_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_sys_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_sys_pool 0 0B 0B 0
│ ├─vg_sys-lv_save 0 512K 16G 0
│ └─vg_sys-lv_images 0 512K 16G 0
├─vg_sys-lv_sys_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_sys_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_sys_pool 0 0B 0B 0
│ ├─vg_sys-lv_save 0 512K 16G 0
│ └─vg_sys-lv_images 0 512K 16G 0
├─vg_sys-.drbdctrl_0 0 0B 0B 0
│ └─drbd0 0 0B 0B 0
└─vg_sys-.drbdctrl_1 0 0B 0B 0
└─drbd1 0 0B 0B 0
sdd 0 0B 0B 0
├─sdd1 0 0B 0B 0
├─sdd2 0 0B 0B 0
│ └─md126 0 0B 0B 0
└─sdd3 0 0B 0B 0
└─md127 0 0B 512K 0
├─vg_sys-lv_sysroot 0 0B 0B 0
├─vg_sys-lv_swap 0 0B 0B 0
├─vg_sys-lv_drbd_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool-tpool 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool 0 0B 0B 0
├─vg_sys-lv_drbd_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool-tpool 0 0B 0B 0
│ └─vg_sys-lv_drbd_pool 0 0B 0B 0
├─vg_sys-lv_one_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_one_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_one_pool 0 0B 0B 0
│ └─vg_sys-lv_one01_disk01 0 512K 16G 0
├─vg_sys-lv_one_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_one_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_one_pool 0 0B 0B 0
│ └─vg_sys-lv_one01_disk01 0 512K 16G 0
├─vg_sys-lv_sys_pool_tmeta 0 0B 0B 0
│ └─vg_sys-lv_sys_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_sys_pool 0 0B 0B 0
│ ├─vg_sys-lv_save 0 512K 16G 0
│ └─vg_sys-lv_images 0 512K 16G 0
├─vg_sys-lv_sys_pool_tdata 0 0B 0B 0
│ └─vg_sys-lv_sys_pool-tpool 0 0B 0B 0
│ ├─vg_sys-lv_sys_pool 0 0B 0B 0
│ ├─vg_sys-lv_save 0 512K 16G 0
│ └─vg_sys-lv_images 0 512K 16G 0
├─vg_sys-.drbdctrl_0 0 0B 0B 0
│ └─drbd0 0 0B 0B 0
└─vg_sys-.drbdctrl_1 0 0B 0B 0
└─drbd1 0 0B 0B 0
[root at kvm08 ~]# drbdadm dump
# /etc/drbd.conf
global {
usage-count yes;
}
common {
}
# resource .drbdctrl on kvm08.devnull.net.nz: not ignored, not stacked
# defined at /etc/drbd.d/drbdctrl.res:1
resource .drbdctrl {
volume 0 {
device minor 0;
disk /dev/vg_sys/.drbdctrl_0;
meta-disk internal;
}
volume 1 {
device minor 1;
disk /dev/vg_sys/.drbdctrl_1;
meta-disk internal;
}
on one01.devnull.net.nz {
node-id 0;
address ipv4 192.168.48.25:6999;
}
on kvm08.devnull.net.nz {
node-id 1;
address ipv4 192.168.48.23:6999;
}
on kvm12.devnull.net.nz {
node-id 2;
address ipv4 192.168.48.24:6999;
}
connection-mesh {
hosts one01.devnull.net.nz kvm08.devnull.net.nz
kvm12.devnull.net.nz;
net {
protocol C;
}
}
net {
cram-hmac-alg sha256;
shared-secret "xsVULIQ+jgn1HQXGqmW4";
allow-two-primaries no;
}
}
This is /var/log/messages from the time that the VM one01 was started up
until the box stopped responding.
Aug 10 15:50:35 kvm08 systemd: Started Virtual machine log manager.
Aug 10 15:50:35 kvm08 systemd: Starting Virtual machine log manager...
Aug 10 15:50:35 kvm08 kernel: tun: Universal TUN/TAP device driver, 1.6
Aug 10 15:50:35 kvm08 kernel: tun: (C) 1999-2004 Max Krasnyansky
<maxk at qualcomm.com>
Aug 10 15:50:35 kvm08 NetworkManager[1079]: <info> [1502337035.9787]
manager: (vnet0): new Tun device
(/org/freedesktop/NetworkManager/Devices/14)
Aug 10 15:50:36 kvm08 kernel: device vnet0 entered promiscuous mode
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0178]
device (vnet0): state change: unmanaged -> unavailable (reason
'connection-assumed') [10 20 41]
Aug 10 15:50:36 kvm08 kernel: mgt-br: port 2(vnet0) entered forwarding
state
Aug 10 15:50:36 kvm08 kernel: mgt-br: port 2(vnet0) entered forwarding
state
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0192]
keyfile: add connection in-memory
(57653b42-13d2-40f4-b26e-a91dec446be6,"vnet0")
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0197]
device (vnet0): state change: unavailable -> disconnected (reason
'connection-assumed') [20 30 41]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0203]
device (vnet0): Activation: starting connection 'vnet0'
(57653b42-13d2-40f4-b26e-a91dec446be6)
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0207]
device (vnet0): state change: disconnected -> prepare (reason 'none')
[30 40 0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0211]
device (vnet0): state change: prepare -> config (reason 'none') [40 50
0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0213]
device (vnet0): state change: config -> ip-config (reason 'none') [50 70
0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0214]
device (mgt-br): bridge port vnet0 was attached
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0214]
device (vnet0): Activation: connection 'vnet0' enslaved, continuing
activation
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0217]
device (vnet0): state change: ip-config -> secondaries (reason 'none')
[70 90 0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0219]
device (vnet0): state change: secondaries -> activated (reason 'none')
[90 100 0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0925]
device (vnet0): Activation: successful, device activated.
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.0937]
manager: (vnet1): new Tun device
(/org/freedesktop/NetworkManager/Devices/15)
Aug 10 15:50:36 kvm08 dbus-daemon: dbus[1071]: [system] Activating via
systemd: service name='org.freedesktop.nm_dispatcher'
unit='dbus-org.freedesktop.nm-dispatcher.service'
Aug 10 15:50:36 kvm08 dbus[1071]: [system] Activating via systemd:
service name='org.freedesktop.nm_dispatcher'
unit='dbus-org.freedesktop.nm-dispatcher.service'
Aug 10 15:50:36 kvm08 systemd: Starting Network Manager Script
Dispatcher Service...
Aug 10 15:50:36 kvm08 dbus[1071]: [system] Successfully activated
service 'org.freedesktop.nm_dispatcher'
Aug 10 15:50:36 kvm08 dbus-daemon: dbus[1071]: [system] Successfully
activated service 'org.freedesktop.nm_dispatcher'
Aug 10 15:50:36 kvm08 nm-dispatcher: req:1 'up' [vnet0]: new request (4
scripts)
Aug 10 15:50:36 kvm08 nm-dispatcher: req:1 'up' [vnet0]: start running
ordered scripts...
Aug 10 15:50:36 kvm08 systemd: Started Network Manager Script Dispatcher
Service.
Aug 10 15:50:36 kvm08 kernel: device vnet1 entered promiscuous mode
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1188]
device (vnet1): state change: unmanaged -> unavailable (reason
'connection-assumed') [10 20 41]
Aug 10 15:50:36 kvm08 kernel: storage-br: port 2(vnet1) entered
forwarding state
Aug 10 15:50:36 kvm08 kernel: storage-br: port 2(vnet1) entered
forwarding state
Aug 10 15:50:36 kvm08 systemd: Unit iscsi.service cannot be reloaded
because it is inactive.
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1200]
keyfile: add connection in-memory
(f66ca2a2-5ee5-42b4-8823-6765e82f3b06,"vnet1")
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1203]
device (vnet1): state change: unavailable -> disconnected (reason
'connection-assumed') [20 30 41]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1208]
device (vnet1): Activation: starting connection 'vnet1'
(f66ca2a2-5ee5-42b4-8823-6765e82f3b06)
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1211]
device (vnet1): state change: disconnected -> prepare (reason 'none')
[30 40 0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1214]
device (vnet1): state change: prepare -> config (reason 'none') [40 50
0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1216]
device (vnet1): state change: config -> ip-config (reason 'none') [50 70
0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1216]
device (storage-br): bridge port vnet1 was attached
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1216]
device (vnet1): Activation: connection 'vnet1' enslaved, continuing
activation
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1218]
device (vnet1): state change: ip-config -> secondaries (reason 'none')
[70 90 0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1219]
device (vnet1): state change: secondaries -> activated (reason 'none')
[90 100 0]
Aug 10 15:50:36 kvm08 NetworkManager[1079]: <info> [1502337036.1374]
device (vnet1): Activation: successful, device activated.
Aug 10 15:50:36 kvm08 nm-dispatcher: req:2 'up' [vnet1]: new request (4
scripts)
Aug 10 15:50:36 kvm08 nm-dispatcher: req:2 'up' [vnet1]: start running
ordered scripts...
Aug 10 15:50:36 kvm08 systemd: Unit iscsi.service cannot be reloaded
because it is inactive.
Aug 10 15:50:36 kvm08 dbus-daemon: dbus[1071]: [system] Activating via
systemd: service name='org.freedesktop.machine1'
unit='dbus-org.freedesktop.machine1.service'
Aug 10 15:50:36 kvm08 dbus[1071]: [system] Activating via systemd:
service name='org.freedesktop.machine1'
unit='dbus-org.freedesktop.machine1.service'
Aug 10 15:50:36 kvm08 systemd: Created slice Virtual Machine and
Container Slice.
Aug 10 15:50:36 kvm08 systemd: Starting Virtual Machine and Container
Slice.
Aug 10 15:50:36 kvm08 systemd: Starting Virtual Machine and Container
Registration Service...
Aug 10 15:50:36 kvm08 dbus[1071]: [system] Successfully activated
service 'org.freedesktop.machine1'
Aug 10 15:50:36 kvm08 dbus-daemon: dbus[1071]: [system] Successfully
activated service 'org.freedesktop.machine1'
Aug 10 15:50:36 kvm08 systemd: Started Virtual Machine and Container
Registration Service.
Aug 10 15:50:36 kvm08 systemd-machined: New machine qemu-1-one01.
Aug 10 15:50:36 kvm08 systemd: Started Virtual Machine qemu-1-one01.
Aug 10 15:50:36 kvm08 systemd: Starting Virtual Machine qemu-1-one01.
Aug 10 15:50:36 kvm08 kvm: 1 guest now active
Aug 10 15:50:36 kvm08 journal: libvirt version: 2.0.0, package:
10.el7_3.9 (CentOS BuildSystem <http://bugs.centos.org>,
2017-05-25-20:52:28, c1bm.rdu2.centos.org)
Aug 10 15:50:36 kvm08 journal: hostname: kvm08.devnull.net.nz
Aug 10 15:50:36 kvm08 journal: End of file while reading data:
Input/output error
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x611
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x639
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x641
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x619
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x60d
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x3f8
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x3f9
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x3fa
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x630
Aug 10 15:50:49 kvm08 kernel: kvm [4831]: vcpu0 unhandled rdmsr: 0x631
Aug 10 15:50:51 kvm08 kernel: mgt-br: port 2(vnet0) entered forwarding
state
Aug 10 15:50:51 kvm08 kernel: storage-br: port 2(vnet1) entered
forwarding state
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Handshake to peer 0 successful: Agreed network protocol version 112
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Feature flags enabled on protocol level: 0x7 TRIM THIN_RESYNC
WRITE_SAME.
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz: Peer
authenticated using 32 bytes HMAC
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Starting ack_recv thread (from drbd_r_.drbdctr [3054])
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Preparing remote state change 4113410068 (primary_nodes=2,
weak_nodes=FFFFFFFFFFFFFFF8)
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Committing remote state change 4113410068
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Preparing remote state change 2601336323 (primary_nodes=2,
weak_nodes=FFFFFFFFFFFFFFF8)
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Committing remote state change 2601336323
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz: conn(
Connecting -> Connected ) peer( Unknown -> Secondary )
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz: sock
was shut down by peer
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz: conn(
Connected -> BrokenPipe ) peer( Secondary -> Unknown )
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
ack_receiver terminated
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Terminating ack_recv thread
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Connection closed
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz: conn(
BrokenPipe -> Unconnected )
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Restarting receiver thread
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz: conn(
Unconnected -> Connecting )
Aug 10 15:50:59 kvm08 kernel: drbd .drbdctrl tcp:one01.devnull.net.nz:
Error receiving initial packet
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Preparing remote state change 2153237908 (primary_nodes=2,
weak_nodes=FFFFFFFFFFFFFFF9)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Aborting remote state change 2153237908
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Preparing remote state change 3731470544 (primary_nodes=2,
weak_nodes=FFFFFFFFFFFFFFF9)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Aborting remote state change 3731470544
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Preparing remote state change 3183590660 (primary_nodes=2,
weak_nodes=FFFFFFFFFFFFFFF9)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Aborting remote state change 3183590660
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Preparing remote state change 1167464219 (primary_nodes=2,
weak_nodes=FFFFFFFFFFFFFFF9)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl kvm12.devnull.net.nz:
Aborting remote state change 1167464219
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Handshake to peer 0 successful: Agreed network protocol version 112
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Feature flags enabled on protocol level: 0x7 TRIM THIN_RESYNC
WRITE_SAME.
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz: Peer
authenticated using 32 bytes HMAC
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Starting ack_recv thread (from drbd_r_.drbdctr [3054])
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Preparing remote state change 2100492907 (primary_nodes=2,
weak_nodes=FFFFFFFFFFFFFFF8)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz:
Committing remote state change 2100492907
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl one01.devnull.net.nz: conn(
Connecting -> Connected ) peer( Unknown -> Secondary )
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0: current_size: 8112
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: c_size: 8112 u_size: 0 d_size: 8112 max_size: 8112
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: la_size: 8112 my_usize: 0 my_max_size: 8112
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: node_id: 0 idx: 0 bm-uuid: 0x7c7d4005e24d2c70
flags: 0x10 max_size: 8112 (DUnknown)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0: my node_id: 1
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
kvm12.devnull.net.nz: node_id: 2 idx: 1 bm-uuid: 0x0 flags: 0x11
max_size: 8112 (UpToDate)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: calling drbd_determine_dev_size()
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: node_id: 0 idx: 0 bm-uuid: 0x7c7d4005e24d2c70
flags: 0x10 max_size: 8112 (DUnknown)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0: my node_id: 1
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
kvm12.devnull.net.nz: node_id: 2 idx: 1 bm-uuid: 0x0 flags: 0x11
max_size: 8112 (UpToDate)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: drbd_sync_handshake:
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: self
9C13275B839ECD75:7C7D4005E24D2C70:56663C92713D4592:39067985F6A30308
bits:0 flags:120
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: peer
7C7D4005E24D2C70:0000000000000000:7C7D4005E24D2C70:56663C92713D4592
bits:0 flags:24
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: uuid_compare()=2 by rule 70
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: pdsk( DUnknown -> Inconsistent ) repl( Off ->
WFBitMapS )
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: send bitmap stats [Bytes(packets)]: plain 0(0),
RLE 20(1), total 20; compression: 87.5%
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1: current_size: 8112
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: c_size: 8112 u_size: 0 d_size: 8112 max_size: 8112
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: la_size: 8112 my_usize: 0 my_max_size: 8112
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: node_id: 0 idx: 0 bm-uuid: 0x8473cede44c3a0ca
flags: 0x10 max_size: 8112 (DUnknown)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1: my node_id: 1
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
kvm12.devnull.net.nz: node_id: 2 idx: 1 bm-uuid: 0x0 flags: 0x11
max_size: 8112 (UpToDate)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: calling drbd_determine_dev_size()
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: node_id: 0 idx: 0 bm-uuid: 0x8473cede44c3a0ca
flags: 0x10 max_size: 8112 (DUnknown)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1: my node_id: 1
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
kvm12.devnull.net.nz: node_id: 2 idx: 1 bm-uuid: 0x0 flags: 0x11
max_size: 8112 (UpToDate)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: drbd_sync_handshake:
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: self
621527FE9CE3F5D5:8473CEDE44C3A0CA:CBFFFFCC9D2E2580:947C2F6D7DF40874
bits:0 flags:120
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: peer
8473CEDE44C3A0CA:0000000000000000:CBFFFFCC9D2E2580:947C2F6D7DF40874
bits:0 flags:24
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: uuid_compare()=2 by rule 70
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: pdsk( DUnknown -> Inconsistent ) repl( Off ->
WFBitMapS )
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: send bitmap stats [Bytes(packets)]: plain 0(0),
RLE 20(1), total 20; compression: 87.5%
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: receive bitmap stats [Bytes(packets)]: plain 0(0),
RLE 20(1), total 20; compression: 87.5%
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: helper command: /sbin/drbdadm before-resync-source
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: helper command: /sbin/drbdadm before-resync-source
exit code 0 (0x0)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: repl( WFBitMapS -> SyncSource )
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: Began resync as SyncSource (will sync 0 KB [0 bits
set]).
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: Resync done (total 1 sec; paused 0 sec; 0 K/sec)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/0 drbd0
one01.devnull.net.nz: pdsk( Inconsistent -> UpToDate ) repl( SyncSource
-> Established )
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: receive bitmap stats [Bytes(packets)]: plain 0(0),
RLE 20(1), total 20; compression: 87.5%
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: helper command: /sbin/drbdadm before-resync-source
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: helper command: /sbin/drbdadm before-resync-source
exit code 0 (0x0)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: repl( WFBitMapS -> SyncSource )
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: Began resync as SyncSource (will sync 0 KB [0 bits
set]).
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: Resync done (total 1 sec; paused 0 sec; 0 K/sec)
Aug 10 15:51:00 kvm08 kernel: drbd .drbdctrl/1 drbd1
one01.devnull.net.nz: pdsk( Inconsistent -> UpToDate ) repl( SyncSource
-> Established )
Aug 10 15:51:15 kvm08 systemd: Created slice user-9869.slice.
Aug 10 15:51:15 kvm08 systemd: Starting user-9869.slice.
Aug 10 15:51:15 kvm08 systemd-logind: New session 3 of user oneadmin.
Aug 10 15:51:15 kvm08 systemd: Started Session 3 of user oneadmin.
Aug 10 15:51:15 kvm08 systemd: Starting Session 3 of user oneadmin.
Aug 10 15:51:25 kvm08 drbdmanaged[1756]: INFO updating storage
pool information
As to what the box is doing at the time - running the VM one01 and a
DRBDManage cluster with no resources in it.
one01 is an OpenNeubula frontend and both kvm08 and kvm12 are OpenNebula
nodes. OpenNebula has no VMs and only the default datastores configured
and is basically doing nothing at this point.
Hope that that helps. Let me know if there is anything else.