Note: "permalinks" may not be as permanent as we would like,
direct links of old sources may well be a few messages off.
Dear all, I have a strange Problem on a two node cluster, with an DRBD volume. Each time I try to create a new logical volume I see this error: c-r0212034:~# lvcreate -L 2G --name foo rep device-mapper: reload ioctl failed: Invalid argument Aborting. Failed to activate new LV to wipe the start of it. I red a lot about this in the internet, but unfortunately it was not possible to fix it till yet. Please find some helpful output below: # pvcreate /dev/drbd0 Physical volume "/dev/drbd0" successfully created # vgcreate rep /dev/drbd0 Volume group "rep" successfully created # pvscan PV /dev/sda2 VG pve lvm2 [930.82 GB / 836.00 MB free] PV /dev/drbd0 VG rep lvm2 [699.98 GB / 699.98 GB free] # pvs PV VG Fmt Attr PSize PFree /dev/dm-3 rep lvm2 a- 699.98G 699.98G /dev/sda2 pve lvm2 a- 930.82G 836.00M LVM-Filter in lvm.conf: filter = ["a|sd.*|", "a|drbd.*|", "r|.*|"] My drbd.conf: global { usage-count no; # minor-count dialog-refresh disable-ip-verification } common { protocol C; handlers { # fence-peer "/usr/lib/drbd/crm-fence-peer.sh"; # split-brain "/usr/lib/drbd/notify-split-brain.sh root"; # out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root"; # before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k"; # after-resync-target /usr/lib/drbd/unsnapshot-resync-target-lvm.sh; } startup { # wfc-timeout degr-wfc-timeout outdated-wfc-timeout wait-after-sb } disk { # on-io-error fencing use-bmbv no-disk-barrier no-disk-flushes # no-disk-drain no-md-flushes max-bio-bvecs } net { # sndbuf-size rcvbuf-size timeout connect-int ping-int ping-timeout max-buffers # max-epoch-size ko-count allow-two-primaries cram-hmac-alg shared-secret # after-sb-0pri after-sb-1pri after-sb-2pri data-integrity-alg no-tcp-cork } syncer { # rate after al-extents use-rle cpu-mask verify-alg csums-alg rate 1000M; } } resource r0 { protocol C; startup { wfc-timeout 15; # wfc-timeout can be dangerous (http://forum.proxmox.com/threads/3465-Is-it-safe-to-use-wfc-timeout-in-DRBD-configuration) degr-wfc-timeout 60; become-primary-on both; } net { cram-hmac-alg sha1; shared-secret "my-secret"; allow-two-primaries; after-sb-0pri discard-zero-changes; after-sb-1pri discard-secondary; after-sb-2pri disconnect; } on c-r0212034 { device /dev/drbd0; disk /dev/pve/r0; address 10.8.11.1:7788; meta-disk internal; } on c-r0212035 { device /dev/drbd0; disk /dev/pve/r0; address 10.8.11.2:7788; meta-disk internal; } } Syslog: Oct 8 23:06:30 c-r0212034 kernel: device-mapper: table: 253:4: linear: dm-linear: Device lookup failed Oct 8 23:06:30 c-r0212034 kernel: device-mapper: ioctl: error adding target to table # lvcreate -L 2G -n foo rep -vvvv #lvmcmdline.c:914 Processing: lvcreate -L 2G -n foo rep -vvvv #lvmcmdline.c:917 O_DIRECT will be used #config/config.c:950 Setting global/locking_type to 1 #locking/locking.c:223 File-based locking selected. #config/config.c:927 Setting global/locking_dir to /var/lock/lvm #activate/activate.c:362 Getting target version for linear #ioctl/libdm-iface.c:1606 dm version OF [16384] #ioctl/libdm-iface.c:1606 dm versions OF [16384] #activate/activate.c:362 Getting target version for striped #ioctl/libdm-iface.c:1606 dm versions OF [16384] #lvcreate.c:302 Setting logging type to disk #config/config.c:950 Setting activation/mirror_region_size to 512 #lvcreate.c:895 Finding volume group "rep" #locking/file_locking.c:164 Locking /var/lock/lvm/V_rep WB #filters/filter-regex.c:172 /dev/ram0: Skipping (regex) #filters/filter-regex.c:172 /dev/loop0: Skipping (regex) #device/dev-io.c:439 Opened /dev/sda RO #device/dev-io.c:260 /dev/sda: size is 1953124352 sectors #device/dev-io.c:340 WARNING: /dev/sda already opened read-only #device/dev-io.c:515 /dev/sda: Immediate close attempt while still referenced #device/dev-io.c:485 Closed /dev/sda #device/dev-io.c:439 Opened /dev/sda RW O_DIRECT #device/dev-io.c:134 /dev/sda: block size is 4096 bytes #filters/filter.c:124 /dev/sda: Skipping: Partition table signature found #device/dev-io.c:485 Closed /dev/sda #device/dev-io.c:439 Opened /dev/drbd0 RW O_DIRECT #device/dev-io.c:134 /dev/drbd0: block size is 4096 bytes #label/label.c:160 /dev/drbd0: lvm2 label detected #cache/lvmcache.c:965 lvmcache: /dev/drbd0: now in VG #orphans_lvm2 (#orphans_lvm2) #format_text/format-text.c:1095 /dev/drbd0: Found metadata at 8704 size 632 (in area at 4096 size 192512) for rep (JwEqO3-epv4-evmI-hlcg-owfl-pjrT-foel0n) #cache/lvmcache.c:965 lvmcache: /dev/drbd0: now in VG rep with 1 mdas #cache/lvmcache.c:752 lvmcache: /dev/drbd0: setting rep VGID to JwEqO3epv4evmIhlcgowflpjrTfoel0n #cache/lvmcache.c:1002 lvmcache: /dev/drbd0: VG rep: Set creation host to c-r0212034. #filters/filter-regex.c:172 /dev/dm-0: Skipping (regex) #filters/filter-regex.c:172 /dev/ram1: Skipping (regex) #filters/filter-regex.c:172 /dev/loop1: Skipping (regex) #device/dev-io.c:439 Opened /dev/sda1 RW O_DIRECT #device/dev-io.c:134 /dev/sda1: block size is 4096 bytes #label/label.c:184 /dev/sda1: No label detected #label/label.c:287 <backtrace> #device/dev-io.c:485 Closed /dev/sda1 #filters/filter-regex.c:172 /dev/root: Skipping (regex) #filters/filter-regex.c:172 /dev/ram2: Skipping (regex) #filters/filter-regex.c:172 /dev/loop2: Skipping (regex) #device/dev-io.c:439 Opened /dev/sda2 RW O_DIRECT #device/dev-io.c:134 /dev/sda2: block size is 4096 bytes #label/label.c:160 /dev/sda2: lvm2 label detected #cache/lvmcache.c:965 lvmcache: /dev/sda2: now in VG #orphans_lvm2 (#orphans_lvm2) #format_text/format-text.c:1095 /dev/sda2: Found metadata at 18432 size 1590 (in area at 4096 size 192512) for pve (KLwqKx-bXU4-4cZO-ZtxL-tO0g-8Cop-Ex5ubf) #cache/lvmcache.c:965 lvmcache: /dev/sda2: now in VG pve with 1 mdas #cache/lvmcache.c:752 lvmcache: /dev/sda2: setting pve VGID to KLwqKxbXU44cZOZtxLtO0g8CopEx5ubf #cache/lvmcache.c:1002 lvmcache: /dev/sda2: VG pve: Set creation host to c-r0212034. #device/dev-io.c:485 Closed /dev/sda2 #filters/filter-regex.c:172 /dev/dm-2: Skipping (regex) #filters/filter-regex.c:172 /dev/ram3: Skipping (regex) #filters/filter-regex.c:172 /dev/loop3: Skipping (regex) #device/dev-io.c:439 Opened /dev/dm-3 RW O_DIRECT #device/dev-io.c:134 /dev/dm-3: block size is 4096 bytes #label/label.c:160 /dev/dm-3: lvm2 label detected #cache/lvmcache.c:1133 Duplicate PV ayC0bjchgYw0zf2ye3nTDIYbn3qjLwge on /dev/drbd0 - using dm /dev/dm-3 #cache/lvmcache.c:965 lvmcache: /dev/dm-3: now in VG #orphans_lvm2 (#orphans_lvm2) with 1 mdas #format_text/format-text.c:1095 /dev/dm-3: Found metadata at 8704 size 632 (in area at 4096 size 192512) for rep (JwEqO3-epv4-evmI-hlcg-owfl-pjrT-foel0n) #cache/lvmcache.c:965 lvmcache: /dev/dm-3: now in VG rep with 1 mdas #cache/lvmcache.c:752 lvmcache: /dev/dm-3: setting rep VGID to JwEqO3epv4evmIhlcgowflpjrTfoel0n #cache/lvmcache.c:1002 lvmcache: /dev/dm-3: VG rep: Set creation host to c-r0212034. #filters/filter-regex.c:172 /dev/ram4: Skipping (regex) #filters/filter-regex.c:172 /dev/loop4: Skipping (regex) #filters/filter-regex.c:172 /dev/ram5: Skipping (regex) #filters/filter-regex.c:172 /dev/loop5: Skipping (regex) #filters/filter-regex.c:172 /dev/ram6: Skipping (regex) #filters/filter-regex.c:172 /dev/loop6: Skipping (regex) #filters/filter-regex.c:172 /dev/ram7: Skipping (regex) #filters/filter-regex.c:172 /dev/loop7: Skipping (regex) #filters/filter-regex.c:172 /dev/ram8: Skipping (regex) #filters/filter-regex.c:172 /dev/ram9: Skipping (regex) #filters/filter-regex.c:172 /dev/ram10: Skipping (regex) #filters/filter-regex.c:172 /dev/ram11: Skipping (regex) #filters/filter-regex.c:172 /dev/ram12: Skipping (regex) #filters/filter-regex.c:172 /dev/ram13: Skipping (regex) #filters/filter-regex.c:172 /dev/ram14: Skipping (regex) #filters/filter-regex.c:172 /dev/ram15: Skipping (regex) #label/label.c:270 Using cached label for /dev/dm-3 #label/label.c:270 Using cached label for /dev/dm-3 #format_text/format-text.c:476 Read rep metadata (5) from /dev/dm-3 at 8704 size 632 #metadata/pv_manip.c:272 /dev/dm-3 0: 0 179194: NULL(0:0) #format_text/archiver.c:123 Archiving volume group "rep" metadata (seqno 5). #metadata/lv_manip.c:1822 Creating logical volume foo #metadata/pv_map.c:47 Allowing allocation on /dev/dm-3 start PE 0 length 179194 #metadata/pv_manip.c:272 /dev/dm-3 0: 0 512: foo(0:0) #metadata/pv_manip.c:272 /dev/dm-3 1: 512 178682: NULL(0:0) #format_text/format-text.c:566 Writing rep metadata to /dev/dm-3 at 9728 len 881 #format_text/format-text.c:662 Pre-Committing rep metadata (6) to /dev/dm-3 header at 4096 #format_text/archiver.c:363 Creating volume group backup "/etc/lvm/backup/rep" (seqno 6). #format_text/format-text.c:852 Writing rep metadata to /etc/lvm/backup/.lvm_c-r0212034_8388_1357441257 #format_text/format-text.c:896 Committing rep metadata (6) #format_text/format-text.c:897 Renaming /etc/lvm/backup/rep.tmp to /etc/lvm/backup/rep #format_text/format-text.c:662 Committing rep metadata (6) to /dev/dm-3 header at 4096 #locking/file_locking.c:244 Locking LV JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH (R) #metadata/metadata.c:1858 Finding volume group for uuid JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH #label/label.c:270 Using cached label for /dev/dm-3 #label/label.c:270 Using cached label for /dev/dm-3 #format_text/format-text.c:476 Read rep metadata (6) from /dev/dm-3 at 9728 size 881 #metadata/metadata.c:1864 Found volume group "rep" #activate/activate.c:447 Getting device info for rep-foo #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH NF [16384] #ioctl/libdm-iface.c:1606 dm info JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH NF [16384] #ioctl/libdm-iface.c:1606 dm info rep-foo NF [16384] #mm/memlock.c:99 Locking memory #mm/memlock.c:130 memlock_count inc to 1 #config/config.c:927 Setting activation/missing_stripe_filler to /dev/ioerror #activate/dev_manager.c:632 Getting device info for rep-foo [LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH] #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH OF [16384] #ioctl/libdm-iface.c:1606 dm info JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH OF [16384] #ioctl/libdm-iface.c:1606 dm info rep-foo OF [16384] #activate/dev_manager.c:632 Getting device info for rep-foo-real [LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-real] #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-real OF [16384] #ioctl/libdm-iface.c:1606 dm info JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-real OF [16384] #ioctl/libdm-iface.c:1606 dm info rep-foo-real OF [16384] #activate/dev_manager.c:632 Getting device info for rep-foo-cow [LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-cow] #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-cow OF [16384] #ioctl/libdm-iface.c:1606 dm info JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-cow OF [16384] #ioctl/libdm-iface.c:1606 dm info rep-foo-cow OF [16384] #activate/dev_manager.c:632 Getting device info for rep-foo_mlog [LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-_mlog] #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-_mlog OF [16384] #ioctl/libdm-iface.c:1606 dm info JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-_mlog OF [16384] #ioctl/libdm-iface.c:1606 dm info rep-foo_mlog OF [16384] #activate/dev_manager.c:823 Checking kernel supports striped segment type for foo #ioctl/libdm-iface.c:1606 dm deps (253:3) OF [16384] #libdm-deptree.c:1187 Creating rep-foo #ioctl/libdm-iface.c:1606 dm create rep-foo LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH NF [16384] #libdm-common.c:607 rep-foo: Stacking NODE_ADD (253,4) 0:6 0660 #libdm-deptree.c:1463 Loading rep-foo table #libdm-deptree.c:1413 Adding target: 0 4194304 linear 253:3 384 #ioctl/libdm-iface.c:1606 dm table (253:4) OF [16384] #ioctl/libdm-iface.c:1606 dm reload (253:4) NF [16384] #ioctl/libdm-iface.c:1623 device-mapper: reload ioctl failed: Invalid argument #libdm-deptree.c:1538 <backtrace> #activate/dev_manager.c:1089 <backtrace> #activate/dev_manager.c:1116 <backtrace> #activate/activate.c:579 <backtrace> #mm/memlock.c:118 Unlocking memory #mm/memlock.c:137 memlock_count dec to 0 #libdm-common.c:312 Created /dev/mapper/rep-foo #lvcreate.c:803 Aborting. Failed to activate new LV to wipe the start of it. #locking/file_locking.c:239 Locking LV JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH (NL) #metadata/metadata.c:1858 Finding volume group for uuid JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH #label/label.c:270 Using cached label for /dev/dm-3 #label/label.c:270 Using cached label for /dev/dm-3 #format_text/format-text.c:476 Read rep metadata (6) from /dev/dm-3 at 9728 size 881 #metadata/metadata.c:1864 Found volume group "rep" #activate/activate.c:447 Getting device info for rep-foo #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH OF [16384] #mm/memlock.c:99 Locking memory #mm/memlock.c:130 memlock_count inc to 1 #activate/dev_manager.c:632 Getting device info for rep-foo [LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH] #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH OF [16384] #ioctl/libdm-iface.c:1606 dm deps (253:4) OF [16384] #activate/dev_manager.c:632 Getting device info for rep-foo-real [LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-real] #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-real OF [16384] #ioctl/libdm-iface.c:1606 dm info JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-real OF [16384] #ioctl/libdm-iface.c:1606 dm info rep-foo-real OF [16384] #activate/dev_manager.c:632 Getting device info for rep-foo-cow [LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-cow] #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-cow OF [16384] #ioctl/libdm-iface.c:1606 dm info JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-cow OF [16384] #ioctl/libdm-iface.c:1606 dm info rep-foo-cow OF [16384] #activate/dev_manager.c:632 Getting device info for rep-foo_mlog [LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-_mlog] #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-_mlog OF [16384] #ioctl/libdm-iface.c:1606 dm info JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH-_mlog OF [16384] #ioctl/libdm-iface.c:1606 dm info rep-foo_mlog OF [16384] #ioctl/libdm-iface.c:1606 dm info (253:4) OF [16384] #libdm-deptree.c:830 Removing rep-foo (253:4) #ioctl/libdm-iface.c:1606 dm remove (253:4) NF [16384] #libdm-common.c:623 rep-foo: Stacking NODE_DEL (replaces other stacked ops) #mm/memlock.c:118 Unlocking memory #mm/memlock.c:137 memlock_count dec to 0 #libdm-common.c:374 Removed /dev/mapper/rep-foo #locking/file_locking.c:234 Unlocking LV JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH #metadata/metadata.c:1858 Finding volume group for uuid JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH #label/label.c:270 Using cached label for /dev/dm-3 #label/label.c:270 Using cached label for /dev/dm-3 #format_text/format-text.c:476 Read rep metadata (6) from /dev/dm-3 at 9728 size 881 #metadata/metadata.c:1864 Found volume group "rep" #activate/activate.c:447 Getting device info for rep-foo #ioctl/libdm-iface.c:1606 dm info LVM-JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH NF [16384] #ioctl/libdm-iface.c:1606 dm info JwEqO3epv4evmIhlcgowflpjrTfoel0nRC7uHcquGNP3nLrV0xKm4ugMvFEVs1PH NF [16384] #ioctl/libdm-iface.c:1606 dm info rep-foo NF [16384] #metadata/pv_manip.c:272 /dev/dm-3 0: 0 512: NULL(0:0) #metadata/pv_manip.c:272 /dev/dm-3 1: 512 178682: NULL(0:0) #format_text/format-text.c:566 Writing rep metadata to /dev/dm-3 at 10752 len 632 #format_text/format-text.c:662 Pre-Committing rep metadata (7) to /dev/dm-3 header at 4096 #format_text/archiver.c:363 Creating volume group backup "/etc/lvm/backup/rep" (seqno 7). #format_text/format-text.c:852 Writing rep metadata to /etc/lvm/backup/.lvm_c-r0212034_8388_227330571 #format_text/format-text.c:896 Committing rep metadata (7) #format_text/format-text.c:897 Renaming /etc/lvm/backup/rep.tmp to /etc/lvm/backup/rep #format_text/format-text.c:662 Committing rep metadata (7) to /dev/dm-3 header at 4096 #locking/file_locking.c:59 Unlocking /var/lock/lvm/V_rep #device/dev-io.c:485 Closed /dev/drbd0 #device/dev-io.c:485 Closed /dev/dm-3 It looks like there is a problem with the LVM filter?? I already tried a lot of different filters, but it didnt helped. I have the same configuration perfectly running on an other cluster. Home somebody of can help me! Thanks a lot... Best regards, Markus