Note: "permalinks" may not be as permanent as we would like,
direct links of old sources may well be a few messages off.
To DRBD's guys:
After removing and restoring one VM (vm106) I found a strange
situation on my 3 nodes PVE with DRBD9
on the Primary node:
root at mpve2:~# drbdadm status
.drbdctrl role:Primary
volume:0 disk:UpToDate
volume:1 disk:UpToDate
mpve1 role:Secondary
volume:0 peer-disk:UpToDate
volume:1 peer-disk:UpToDate
mpve3 role:Secondary
volume:0 peer-disk:UpToDate
volume:1 peer-disk:UpToDate
.....omitted...
vm-106-disk-1 role:Primary
disk:UpToDate
mpve1 connection:StandAlone
mpve3 role:Secondary
peer-disk:UpToDate
on the node that appear in StandAlone:
root at mpve1:~# drbdsetup status
.drbdctrl role:Secondary
volume:0 disk:UpToDate
volume:1 disk:UpToDate
mpve2 role:Primary
volume:0 peer-disk:UpToDate
volume:1 peer-disk:UpToDate
mpve3 role:Secondary
volume:0 peer-disk:UpToDate
volume:1 peer-disk:UpToDate
.....omitted...
vm-106-disk-2 role:Secondary
disk:Inconsistent
mpve2 connection:StandAlone
mpve3 connection:Connecting
As you can see there is not a vm-106-disk-1 but there is a
vm-106-disk-2, inexistent on other two nodes.
But with locate I can found on mpve1:
root at mpve1:~# locate vm-106
/dev/drbd/by-disk/drbdpool/vm-106-disk-2_00
/dev/drbd/by-res/vm-106-disk-2
/dev/drbd/by-res/vm-106-disk-2/0
/dev/drbdpool/vm-106-disk-1_00
/dev/drbdpool/vm-106-disk-2_00
/var/lib/drbd.d/drbdmanage_vm-106-disk-1.res.q
/var/lib/drbd.d/drbdmanage_vm-106-disk-2.res
root at mpve1:~#
on mpve2:
root at mpve2:~# locate vm-106-disk
/dev/drbd/by-disk/drbdpool/vm-106-disk-1_00
/dev/drbd/by-res/vm-106-disk-1
/dev/drbd/by-res/vm-106-disk-1/0
/dev/drbdpool/vm-106-disk-1_00
/var/lib/drbd.d/drbdmanage_vm-106-disk-1.res
root at mpve2:~#
on mpve3:
root at mpve3:~# locate vm-106-disk
/dev/drbd/by-disk/drbdpool/vm-106-disk-1_00
/dev/drbd/by-res/vm-106-disk-1
/dev/drbd/by-res/vm-106-disk-1/0
/dev/drbdpool/vm-106-disk-1_00
/dev/drbdpool/vm-106-disk-2_00
/var/lib/drbd.d/drbdmanage_vm-106-disk-1.res
root at mpve3:~#
Even stranger is the output of lsblk (accluded only a portion)
root at mpve1:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sdb 8:16 0 7.3T 0 disk
--sdb1 8:17 0 7.3T 0 part
-drbdpool-lvol0 251:1 0 128M 0 lvm
-drbdpool-lvol1 251:2 0 128M 0 lvm
-drbdpool-drbdthinpool_tmeta 251:3 0 120M 0 lvm
| --drbdpool-drbdthinpool-tpool 251:5 0 7.3T 0 lvm
| -drbdpool-drbdthinpool 251:6 0 7.3T 0 lvm
| -drbdpool-vm--102--disk--1_00 251:9 0 146G 0 lvm
| | --drbd102 147:102 0 146G 1 disk
| -drbdpool-vm--103--disk--1_00 251:10 0 146G 0 lvm
| | --drbd103 147:103 0 146G 1 disk
| -drbdpool-vm--104--disk--1_00 251:11 0 146G 0 lvm
| | --drbd104 147:104 0 146G 1 disk
| -drbdpool-vm--100--disk--1_00 251:12 0 146.1G 0 lvm
| | --drbd116 147:116 0 146G 1 disk
| -drbdpool-vm--101--disk--1_00 251:13 0 146.1G 0 lvm
| | --drbd117 147:117 0 146G 1 disk
| -drbdpool-vm--107--disk--1_00 251:14 0 146.1G 0 lvm
| | --drbd122 147:122 0 146G 1 disk
| -drbdpool-vm--108--disk--1_00 251:15 0 146.1G 0 lvm
| | --drbd123 147:123 0 146G 1 disk
| -drbdpool-vm--106--disk--1_00 251:16 0 146.1G 0 lvm
| -drbdpool-vm--109--disk--1_00 251:19 0 146.1G 0 lvm
| | --drbd125 147:125 0 146G 1 disk
| --drbdpool-vm--106--disk--2_00 251:20 0 146.1G 0 lvm
| --drbd124 147:124 0 146G 1 disk
-drbdpool-drbdthinpool_tdata 251:4 0 7.3T 0 lvm
| --drbdpool-drbdthinpool-tpool 251:5 0 7.3T 0 lvm
| -drbdpool-drbdthinpool 251:6 0 7.3T 0 lvm
| -drbdpool-vm--102--disk--1_00 251:9 0 146G 0 lvm
| | --drbd102 147:102 0 146G 1 disk
| -drbdpool-vm--103--disk--1_00 251:10 0 146G 0 lvm
| | --drbd103 147:103 0 146G 1 disk
| -drbdpool-vm--104--disk--1_00 251:11 0 146G 0 lvm
| | --drbd104 147:104 0 146G 1 disk
| -drbdpool-vm--100--disk--1_00 251:12 0 146.1G 0 lvm
| | --drbd116 147:116 0 146G 1 disk
| -drbdpool-vm--101--disk--1_00 251:13 0 146.1G 0 lvm
| | --drbd117 147:117 0 146G 1 disk
| -drbdpool-vm--107--disk--1_00 251:14 0 146.1G 0 lvm
| | --drbd122 147:122 0 146G 1 disk
| -drbdpool-vm--108--disk--1_00 251:15 0 146.1G 0 lvm
| | --drbd123 147:123 0 146G 1 disk
| -drbdpool-vm--106--disk--1_00 251:16 0 146.1G 0 lvm
| -drbdpool-vm--109--disk--1_00 251:19 0 146.1G 0 lvm
| | --drbd125 147:125 0 146G 1 disk
| --drbdpool-vm--106--disk--2_00 251:20 0 146.1G 0 lvm
| --drbd124 147:124 0 146G 1 disk
-drbdpool-.drbdctrl_0 251:7 0 4M 0 lvm
| --drbd0 147:0 0 4M 1 disk
--drbdpool-.drbdctrl_1 251:8 0 4M 0 lvm
--drbd1 147:1 0 4M 1 disk
root at mpve2:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sdb 8:16 0 7.3T 0 disk
--sdb1 8:17 0 7.3T 0 part
-drbdpool-drbdthinpool_tmeta 251:1 0 120M 0 lvm
| --drbdpool-drbdthinpool-tpool 251:3 0 7.3T 0 lvm
| -drbdpool-drbdthinpool 251:4 0 7.3T 0 lvm
| -drbdpool-vm--102--disk--1_00 251:7 0 146G 0 lvm
| | --drbd102 147:102 0 146G 1 disk
| -drbdpool-vm--103--disk--1_00 251:8 0 146G 0 lvm
| | --drbd103 147:103 0 146G 1 disk
| -drbdpool-vm--104--disk--1_00 251:9 0 146G 0 lvm
| | --drbd104 147:104 0 146G 0 disk
| -drbdpool-vm--100--disk--1_00 251:10 0 146.1G 0 lvm
| | --drbd116 147:116 0 146G 0 disk
| -drbdpool-vm--101--disk--1_00 251:11 0 146.1G 0 lvm
| | --drbd117 147:117 0 146G 0 disk
| -drbdpool-vm--106--disk--1_00 251:12 0 146.1G 0 lvm
| | --drbd126 147:126 0 146G 0 disk
| -drbdpool-vm--107--disk--1_00 251:13 0 146.1G 0 lvm
| | --drbd122 147:122 0 146G 0 disk
| -drbdpool-vm--108--disk--1_00 251:14 0 146.1G 0 lvm
| | --drbd123 147:123 0 146G 1 disk
| --drbdpool-vm--109--disk--1_00 251:17 0 146.1G 0 lvm
| --drbd125 147:125 0 146G 1 disk
-drbdpool-drbdthinpool_tdata 251:2 0 7.3T 0 lvm
| --drbdpool-drbdthinpool-tpool 251:3 0 7.3T 0 lvm
| -drbdpool-drbdthinpool 251:4 0 7.3T 0 lvm
| -drbdpool-vm--102--disk--1_00 251:7 0 146G 0 lvm
| | --drbd102 147:102 0 146G 1 disk
| -drbdpool-vm--103--disk--1_00 251:8 0 146G 0 lvm
| | --drbd103 147:103 0 146G 1 disk
| -drbdpool-vm--104--disk--1_00 251:9 0 146G 0 lvm
| | --drbd104 147:104 0 146G 0 disk
| -drbdpool-vm--100--disk--1_00 251:10 0 146.1G 0 lvm
| | --drbd116 147:116 0 146G 0 disk
| -drbdpool-vm--101--disk--1_00 251:11 0 146.1G 0 lvm
| | --drbd117 147:117 0 146G 0 disk
| -drbdpool-vm--106--disk--1_00 251:12 0 146.1G 0 lvm
| | --drbd126 147:126 0 146G 0 disk
| -drbdpool-vm--107--disk--1_00 251:13 0 146.1G 0 lvm
| | --drbd122 147:122 0 146G 0 disk
| -drbdpool-vm--108--disk--1_00 251:14 0 146.1G 0 lvm
| | --drbd123 147:123 0 146G 1 disk
| --drbdpool-vm--109--disk--1_00 251:17 0 146.1G 0 lvm
| --drbd125 147:125 0 146G 1 disk
-drbdpool-.drbdctrl_0 251:5 0 4M 0 lvm
| --drbd0 147:0 0 4M 0 disk
--drbdpool-.drbdctrl_1 251:6 0 4M 0 lvm
--drbd1 147:1 0 4M 0 disk
root at mpve3:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 3.7T 0 disk
-sda1 8:1 0 1007K 0 part
-sda2 8:2 0 127M 0 part
--sda3 8:3 0 3.7T 0 part
-pve-root 251:0 0 96G 0 lvm /
-pve-swap 251:16 0 62G 0 lvm [SWAP]
--pve-data 251:17 0 3.5T 0 lvm /var/lib/vz
sdb 8:16 0 7.3T 0 disk
--sdb1 8:17 0 7.3T 0 part
-drbdpool-drbdthinpool_tmeta 251:1 0 120M 0 lvm
| --drbdpool-drbdthinpool-tpool 251:3 0 7.3T 0 lvm
| -drbdpool-drbdthinpool 251:4 0 7.3T 0 lvm
| -drbdpool-vm--102--disk--1_00 251:7 0 146G 0 lvm
| | --drbd102 147:102 0 146G 1 disk
| -drbdpool-vm--103--disk--1_00 251:8 0 146G 0 lvm
| | --drbd103 147:103 0 146G 1 disk
| -drbdpool-vm--104--disk--1_00 251:9 0 146G 0 lvm
| | --drbd104 147:104 0 146G 1 disk
| -drbdpool-vm--105--disk--2_00 251:10 0 32G 0 lvm
| -drbdpool-vm--100--disk--1_00 251:11 0 146.1G 0 lvm
| | --drbd116 147:116 0 146G 1 disk
| -drbdpool-vm--101--disk--1_00 251:12 0 146.1G 0 lvm
| | --drbd117 147:117 0 146G 1 disk
| -drbdpool-vm--107--disk--1_00 251:13 0 146.1G 0 lvm
| | --drbd122 147:122 0 146G 1 disk
| -drbdpool-vm--108--disk--1_00 251:14 0 146.1G 0 lvm
| | --drbd123 147:123 0 146G 1 disk
| -drbdpool-vm--106--disk--1_00 251:15 0 146.1G 0 lvm
| | --drbd126 147:126 0 146G 1 disk
| -drbdpool-vm--106--disk--2_00 251:18 0 146.1G 0 lvm
| --drbdpool-vm--109--disk--1_00 251:19 0 146.1G 0 lvm
| --drbd125 147:125 0 146G 1 disk
-drbdpool-drbdthinpool_tdata 251:2 0 7.3T 0 lvm
| --drbdpool-drbdthinpool-tpool 251:3 0 7.3T 0 lvm
| -drbdpool-drbdthinpool 251:4 0 7.3T 0 lvm
| -drbdpool-vm--102--disk--1_00 251:7 0 146G 0 lvm
| | --drbd102 147:102 0 146G 1 disk
| -drbdpool-vm--103--disk--1_00 251:8 0 146G 0 lvm
| | --drbd103 147:103 0 146G 1 disk
| -drbdpool-vm--104--disk--1_00 251:9 0 146G 0 lvm
| | --drbd104 147:104 0 146G 1 disk
| -drbdpool-vm--105--disk--2_00 251:10 0 32G 0 lvm
| -drbdpool-vm--100--disk--1_00 251:11 0 146.1G 0 lvm
| | --drbd116 147:116 0 146G 1 disk
| -drbdpool-vm--101--disk--1_00 251:12 0 146.1G 0 lvm
| | --drbd117 147:117 0 146G 1 disk
| -drbdpool-vm--107--disk--1_00 251:13 0 146.1G 0 lvm
| | --drbd122 147:122 0 146G 1 disk
| -drbdpool-vm--108--disk--1_00 251:14 0 146.1G 0 lvm
| | --drbd123 147:123 0 146G 1 disk
| -drbdpool-vm--106--disk--1_00 251:15 0 146.1G 0 lvm
| | --drbd126 147:126 0 146G 1 disk
| -drbdpool-vm--106--disk--2_00 251:18 0 146.1G 0 lvm
| --drbdpool-vm--109--disk--1_00 251:19 0 146.1G 0 lvm
| --drbd125 147:125 0 146G 1 disk
-drbdpool-.drbdctrl_0 251:5 0 4M 0 lvm
| --drbd0 147:0 0 4M 1 disk
--drbdpool-.drbdctrl_1 251:6 0 4M 0 lvm
--drbd1 147:1 0 4M 1 disk
As you can see there is on mpve1 and mpve3 a drbdpool-vm--106--disk--2_00
moreover there is on mpve3 a drbdpool-vm--105--disk--2_00, but the VM
105 is not more existent.
The question is: why there are this kind of misalignement and how to
solve them, considering that the system is in production? (for me can
be not a problem to remove vm106 and restore from a backup, but I
want to avoid problems on the restored VM).
And how to avoid this kind of problems in the future?
Thanks in advance,
Michele