Note: "permalinks" may not be as permanent as we would like,
direct links of old sources may well be a few messages off.
Hi,
i am using drbd since years and had never any complainments. last week i started to test it on a new environment:
server: debian jessie 8.7 with custom kernel 4.9.15 (.config attached)
drbd-utils: installed via aptitude 8.9.2rc1
drbd kernel Version: 8.4.7 nothing changed
file system: disk > lvs2 > drbd > xen vm
|---------|
| host1 |-----------------------------------
|---------|
| |
| |
| | br255 (bond255 / drbd sync) br0 (bond0)
| |
| |
|---------|
| host2 |-----------------------------------
|---------|
the problem is, I had very difficulties to get the sync to work, various boot's. now I see cat /proc/drbd that drbd starts to sync. the problem is, it interrupt all the time. when start over it doesn't start at the last break, it starts from the beginning 0%.
any idea on this?
bellow I sent my config's and the syslog, I hope someone see what im doing wrong ☹
regards walter.
/etc/network/interfaces
***
auto lo
iface lo inet loopback
auto bond0
iface bond0 inet manual
slaves eth0 eth1
bond-mode 4
bond-miimon 100
bond-lacp-rate 4
auto br0
iface br0 inet static
address 10.41.10.42
netmask 255.255.255.0
gateway 10.41.10.1
bridge_ports bond0
bridge_maxwait 0
bridge_stp off
bridge_fd 0
auto bond255
iface bond255 inet manual
slaves eth2 eth3
bond_mode 4
bond-miimon 100
bond-lacp-rate 4
mtu 9000
auto br255
iface br255 inet static
address 10.255.255.2
netmask 255.255.255.0
bridge_ports bond255
bridge_maxwait 0
bridge_stp off
bridge_fd 0
***
/etc/drbd.d/abo.drbd.res
***
resource drbd_host11 {
device /dev/drbd11;
on srv-ldje-xend01 {
disk /dev/vgdata/host11;
meta-disk /dev/vgdata/drbd-meta[11];
address 10.255.255.1:7711;
}
on srv-ldje-xend02 {
disk /dev/vgdata/host11;
meta-disk /dev/vgdata/drbd-meta[11];
address 10.255.255.2:7711;
}
}
resource drbd_host12 {
device /dev/drbd12;
on srv-ldje-xend01 {
disk /dev/vgdata/host12;
meta-disk /dev/vgdata/drbd-meta[12];
address 10.255.255.1:7712;
}
on srv-ldje-xend02 {
disk /dev/vgdata/host12;
meta-disk /dev/vgdata/drbd-meta[12];
address 10.255.255.2:7712;
}
}
...
resource drbd_host18 {
device /dev/drbd18;
on srv-ldje-xend01 {
disk /dev/vgdata/host18;
meta-disk /dev/vgdata/drbd-meta[18];
address 10.255.255.1:7718;
}
on srv-ldje-xend02 {
disk /dev/vgdata/host18;
meta-disk /dev/vgdata/drbd-meta[18];
address 10.255.255.2:7718;
}
}
***
/etc/drbd.d/global_common.conf
***
global {
usage-count no;
minor-count 64;
}
common {
protocol C;
handlers {
pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
}
startup {
wfc-timeout 90;
degr-wfc-timeout 30;
outdated-wfc-timeout 10;
}
disk {
on-io-error pass_on;
fencing dont-care;
}
net {
unplug-watermark 20480;
sndbuf-size 512k;
timeout 30;
ping-int 10;
ping-timeout 20;
max-buffers 20480;
max-epoch-size 16384;
ko-count 4;
#allow-two-primaries;
after-sb-0pri discard-zero-changes;
after-sb-1pri discard-secondary;
after-sb-2pri disconnect;
}
syncer {
al-extents 31;
rate 100M;
verify-alg crc32c;
csums-alg crc32c;
}
}
***
cat /proc/drbd > broken pipe
***
11: cs:SyncTarget ro:Secondary/Primary ds:Inconsistent/UpToDate C r-----
ns:0 nr:0 dw:0 dr:1114152 al:0 bm:0 lo:2 pe:0 ua:0 ap:0 ep:1 wo:f oos:24673672
[>....................] sync'ed: 4.4% (24092/25180)M
finish: 3:33:46 speed: 1,920 (3,592) want: 17,560 K/sec (stalled)
12: cs:BrokenPipe ro:Secondary/Unknown ds:Inconsistent/DUnknown C r-----
ns:0 nr:91892 dw:91892 dr:708100 al:0 bm:0 lo:2 pe:0 ua:113 ap:0 ep:1 wo:f oos:3199900
13: cs:BrokenPipe ro:Secondary/Unknown ds:Inconsistent/DUnknown C r-----
ns:0 nr:2540 dw:2540 dr:613396 al:0 bm:0 lo:2 pe:0 ua:5 ap:0 ep:1 wo:f oos:3791536
14: cs:BrokenPipe ro:Secondary/Unknown ds:Inconsistent/DUnknown C r-----
ns:0 nr:64 dw:64 dr:806344 al:0 bm:0 lo:2 pe:0 ua:1 ap:0 ep:1 wo:f oos:31372296
15: cs:SyncTarget ro:Secondary/Primary ds:Inconsistent/UpToDate C r-----
ns:0 nr:0 dw:0 dr:803176 al:0 bm:0 lo:2 pe:0 ua:0 ap:0 ep:1 wo:f oos:26737988
[>....................] sync'ed: 3.0% (26108/26892)M
finish: 4:58:58 speed: 1,488 (2,588) want: 15,080 K/sec (stalled)
16: cs:NetworkFailure ro:Secondary/Unknown ds:Inconsistent/DUnknown C r-----
ns:0 nr:808 dw:808 dr:711044 al:0 bm:0 lo:2 pe:0 ua:0 ap:0 ep:1 wo:f oos:3538372
17: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
18: cs:BrokenPipe ro:Secondary/Unknown ds:Inconsistent/DUnknown C r-----
ns:0 nr:283496 dw:264040 dr:491100 al:0 bm:0 lo:42 pe:0 ua:163 ap:0 ep:1 wo:f oos:49669412
***
/var/log/syslog
***
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.262132] block drbd12: Began resync as SyncSource (will sync 3832200 KB [958050 bits set]).
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.262164] block drbd16: updated sync UUID 99D6C7DC9F4B907B:4701327128F1188B:4700327128F1188B:46FF327128F1188B
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.262222] block drbd12: updated sync UUID D6A040E6709CF1B7:4401BB4FE91C8B0D:4400BB4FE91C8B0D:43FFBB4FE91C8B0D
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.264843] block drbd13: helper command: /sbin/drbdadm before-resync-source minor-13 exit code 0 (0x0)
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.264880] block drbd13: conn( WFBitMapS -> SyncSource )
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.264929] block drbd13: Began resync as SyncSource (will sync 4399228 KB [1099807 bits set]).
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.264966] block drbd13: updated sync UUID 809A75B995CF4ACF:722B36DD40B6DBB3:722A36DD40B6DBB3:722936DD40B6DBB3
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.265529] block drbd15: helper command: /sbin/drbdadm before-resync-source minor-15 exit code 0 (0x0)
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.265565] block drbd15: conn( WFBitMapS -> SyncSource )
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.265623] block drbd15: Began resync as SyncSource (will sync 27541164 KB [6885291 bits set]).
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.265658] block drbd15: updated sync UUID CDC2BC0312DAA9E7:206A78EBAED58B95:206978EBAED58B95:206878EBAED58B95
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.268561] block drbd11: helper command: /sbin/drbdadm before-resync-source minor-11 exit code 0 (0x0)
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.268612] block drbd11: conn( WFBitMapS -> SyncSource )
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.268688] block drbd11: Began resync as SyncSource (will sync 25787824 KB [6446956 bits set]).
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.268725] block drbd11: updated sync UUID CEBC1744CCA42951:4BCB73E2A193A8D5:4BCA73E2A193A8D5:4BC973E2A193A8D5
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.280416] block drbd18: receive bitmap stats [Bytes(packets)]: plain 0(0), RLE 342(1), total 342; compression: 100.0%
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.280426] block drbd18: helper command: /sbin/drbdadm before-resync-source minor-18
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.285937] block drbd18: helper command: /sbin/drbdadm before-resync-source minor-18 exit code 0 (0x0)
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.285972] block drbd18: conn( WFBitMapS -> SyncSource )
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.286020] block drbd18: Began resync as SyncSource (will sync 49941016 KB [12485254 bits set]).
Mar 21 21:47:26 srv-ldje-xend02 kernel: [21006.286056] block drbd18: updated sync UUID C057EBB56C9AA199:7CD1B0838059A665:7CD0B0838059A665:7CCFB0838059A665
Mar 21 21:47:29 srv-ldje-xend02 kernel: [21009.567682] block drbd17: Resync done (total 3 sec; paused 0 sec; 0 K/sec)
Mar 21 21:47:29 srv-ldje-xend02 kernel: [21009.567693] block drbd17: updated UUIDs E578D4A6A4B5FC93:0000000000000000:D836D0DA52465B95:D835D0DA52465B95
Mar 21 21:47:29 srv-ldje-xend02 kernel: [21009.567715] block drbd17: conn( SyncSource -> Connected ) pdsk( Inconsistent -> UpToDate )
Mar 21 21:47:37 srv-ldje-xend02 kernel: [21018.024694] drbd drbd_host18: [drbd_w_drbd_hos/1035] sock_sendmsg time expired, ko = 3
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21036.968588] block drbd13: Remote failed to finish a request within 12140ms > ko-count (4) * timeout (30 * 0.1s)
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21036.968763] drbd drbd_host13: peer( Secondary -> Unknown ) conn( SyncSource -> Timeout )
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21036.968894] drbd drbd_host13: ack_receiver terminated
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21036.968898] drbd drbd_host13: Terminating drbd_a_drbd_hos
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21037.005192] drbd drbd_host13: Connection closed
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21037.005376] drbd drbd_host13: conn( Timeout -> Unconnected )
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21037.005399] drbd drbd_host13: receiver terminated
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21037.005401] drbd drbd_host13: Restarting receiver thread
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21037.005403] drbd drbd_host13: receiver (re)started
Mar 21 21:47:56 srv-ldje-xend02 kernel: [21037.005554] drbd drbd_host13: conn( Unconnected -> WFConnection )
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21037.992566] block drbd12: Remote failed to finish a request within 12156ms > ko-count (4) * timeout (30 * 0.1s)
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21037.992737] drbd drbd_host12: peer( Secondary -> Unknown ) conn( SyncSource -> Timeout )
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21037.992985] drbd drbd_host12: ack_receiver terminated
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21037.992988] drbd drbd_host12: Terminating drbd_a_drbd_hos
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21038.033233] drbd drbd_host12: Connection closed
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21038.033419] drbd drbd_host12: conn( Timeout -> Unconnected )
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21038.033442] drbd drbd_host12: receiver terminated
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21038.033444] drbd drbd_host12: Restarting receiver thread
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21038.033446] drbd drbd_host12: receiver (re)started
Mar 21 21:47:57 srv-ldje-xend02 kernel: [21038.033597] drbd drbd_host12: conn( Unconnected -> WFConnection )
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.272573] block drbd16: Remote failed to finish a request within 12072ms > ko-count (4) * timeout (30 * 0.1s)
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.272738] drbd drbd_host16: peer( Secondary -> Unknown ) conn( SyncSource -> Timeout )
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.272959] drbd drbd_host16: ack_receiver terminated
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.272963] drbd drbd_host16: Terminating drbd_a_drbd_hos
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.313175] drbd drbd_host16: Connection closed
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.313370] drbd drbd_host16: conn( Timeout -> Unconnected )
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.313394] drbd drbd_host16: receiver terminated
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.313396] drbd drbd_host16: Restarting receiver thread
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.313398] drbd drbd_host16: receiver (re)started
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.313549] drbd drbd_host16: conn( Unconnected -> WFConnection )
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.528575] block drbd18: Remote failed to finish a request within 12044ms > ko-count (4) * timeout (30 * 0.1s)
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.528746] drbd drbd_host18: peer( Secondary -> Unknown ) conn( SyncSource -> Timeout )
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.528989] drbd drbd_host18: ack_receiver terminated
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.528993] drbd drbd_host18: Terminating drbd_a_drbd_hos
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.561120] drbd drbd_host18: Connection closed
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.561305] drbd drbd_host18: conn( Timeout -> Unconnected )
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.561328] drbd drbd_host18: receiver terminated
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.561330] drbd drbd_host18: Restarting receiver thread
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.561332] drbd drbd_host18: receiver (re)started
Mar 21 21:47:59 srv-ldje-xend02 kernel: [21039.561483] drbd drbd_host18: conn( Unconnected -> WFConnection )
Mar 21 21:48:50 srv-ldje-xend02 kernel: [21090.705770] drbd drbd_host13: Error receiving initial packet
Mar 21 21:48:50 srv-ldje-xend02 kernel: [21090.708719] drbd drbd_host13: initial packet S crossed
...
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.727034] drbd drbd_host12: conn( WFConnection -> WFReportParams )
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.727059] drbd drbd_host12: Starting ack_recv thread (from drbd_r_drbd_hos [1041])
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768367] block drbd18: drbd_sync_handshake:
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768378] block drbd18: self C057EBB56C9AA199:7CD1B0838059A665:7CD0B0838059A665:7CCFB0838059A665 bits:12458576 flags:0
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768384] block drbd18: peer 7CD1B0838059A664:0000000000000000:0000000000000000:0000000000000000 bits:12442897 flags:0
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768388] block drbd18: uuid_compare()=1 by rule 70
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768393] block drbd18: Becoming sync source due to disk states.
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768410] block drbd16: drbd_sync_handshake:
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768416] block drbd18: peer( Unknown -> Secondary ) conn( WFReportParams -> WFBitMapS )
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768423] block drbd16: self 99D6C7DC9F4B907B:4701327128F1188B:4700327128F1188B:46FF327128F1188B bits:999096 flags:0
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768429] block drbd16: peer 4701327128F1188A:0000000000000000:0000000000000000:0000000000000000 bits:999063 flags:0
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768434] block drbd16: uuid_compare()=1 by rule 70
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768438] block drbd16: Becoming sync source due to disk states.
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.768461] block drbd16: peer( Unknown -> Secondary ) conn( WFReportParams -> WFBitMapS )
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.769199] block drbd16: send bitmap stats [Bytes(packets)]: plain 0(0), RLE 37(1), total 37; compression: 100.0%
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.769295] block drbd12: drbd_sync_handshake:
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.769303] block drbd12: self D6A040E6709CF1B7:4401BB4FE91C8B0D:4400BB4FE91C8B0D:43FFBB4FE91C8B0D bits:894678 flags:0
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.769315] block drbd12: peer 4401BB4FE91C8B0C:0000000000000000:0000000000000000:0000000000000000 bits:894457 flags:0
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.769323] block drbd12: uuid_compare()=1 by rule 70
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.769328] block drbd12: Becoming sync source due to disk states.
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.769350] block drbd12: peer( Unknown -> Secondary ) conn( WFReportParams -> WFBitMapS )
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.769969] block drbd18: send bitmap stats [Bytes(packets)]: plain 0(0), RLE 351(1), total 351; compression: 100.0%
Mar 21 21:48:52 srv-ldje-xend02 kernel: [21092.770139] block drbd12: send bitmap stats [Bytes(packets)]: plain 0(0), RLE 372(1), total 372; compression: 100.0%
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.728446] drbd drbd_host13: Handshake successful: Agreed network protocol version 101
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.728452] drbd drbd_host13: Feature flags enabled on protocol level: 0x7 TRIM THIN_RESYNC WRITE_SAME.
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.728693] drbd drbd_host13: conn( WFConnection -> WFReportParams )
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.728718] drbd drbd_host13: Starting ack_recv thread (from drbd_r_drbd_hos [1043])
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.764372] block drbd13: drbd_sync_handshake:
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.764381] block drbd13: self 809A75B995CF4ACF:722B36DD40B6DBB3:722A36DD40B6DBB3:722936DD40B6DBB3 bits:1046366 flags:0
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.764388] block drbd13: peer 722B36DD40B6DBB2:0000000000000000:0000000000000000:0000000000000000 bits:1046265 flags:0
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.764392] block drbd13: uuid_compare()=1 by rule 70
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.764396] block drbd13: Becoming sync source due to disk states.
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.764419] block drbd13: peer( Unknown -> Secondary ) conn( WFReportParams -> WFBitMapS )
Mar 21 21:48:54 srv-ldje-xend02 kernel: [21094.765202] block drbd13: send bitmap stats [Bytes(packets)]: plain 0(0), RLE 310(1), total 310; compression: 100.0%
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.113018] block drbd16: receive bitmap stats [Bytes(packets)]: plain 0(0), RLE 37(1), total 37; compression: 100.0%
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.113030] block drbd16: helper command: /sbin/drbdadm before-resync-source minor-16
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.113315] block drbd13: receive bitmap stats [Bytes(packets)]: plain 0(0), RLE 310(1), total 310; compression: 100.0%
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.113327] block drbd13: helper command: /sbin/drbdadm before-resync-source minor-13
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.114243] block drbd12: receive bitmap stats [Bytes(packets)]: plain 0(0), RLE 372(1), total 372; compression: 100.0%
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.114254] block drbd12: helper command: /sbin/drbdadm before-resync-source minor-12
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.116073] block drbd18: receive bitmap stats [Bytes(packets)]: plain 0(0), RLE 351(1), total 351; compression: 100.0%
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.116086] block drbd18: helper command: /sbin/drbdadm before-resync-source minor-18
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.122206] block drbd16: helper command: /sbin/drbdadm before-resync-source minor-16 exit code 0 (0x0)
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.122244] block drbd16: conn( WFBitMapS -> SyncSource )
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.122297] block drbd16: Began resync as SyncSource (will sync 3996396 KB [999099 bits set]).
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.122335] block drbd16: updated sync UUID 99D6C7DC9F4B907B:4702327128F1188B:4701327128F1188B:4700327128F1188B
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.123018] block drbd13: helper command: /sbin/drbdadm before-resync-source minor-13 exit code 0 (0x0)
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.123057] block drbd13: conn( WFBitMapS -> SyncSource )
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.123108] block drbd13: Began resync as SyncSource (will sync 4185484 KB [1046371 bits set]).
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.123145] block drbd13: updated sync UUID 809A75B995CF4ACF:722C36DD40B6DBB3:722B36DD40B6DBB3:722A36DD40B6DBB3
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.124259] block drbd18: helper command: /sbin/drbdadm before-resync-source minor-18 exit code 0 (0x0)
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.124291] block drbd18: conn( WFBitMapS -> SyncSource )
Mar 21 21:48:57 srv-ldje-xend02 kernel: [21098.124339] block drbd18: Began resync as SyncSource (will sync 49834304 KB [12458576 bits set]).
Mar 21 21:48:58 srv-ldje-xend02 kernel: [21098.124374] block drbd18: updated sync UUID C057EBB56C9AA199:7CD2B0838059A665:7CD1B0838059A665:7CD0B0838059A665
Mar 21 21:48:58 srv-ldje-xend02 kernel: [21098.127469] block drbd12: helper command: /sbin/drbdadm before-resync-source minor-12 exit code 0 (0x0)
Mar 21 21:48:58 srv-ldje-xend02 kernel: [21098.127504] block drbd12: conn( WFBitMapS -> SyncSource )
Mar 21 21:48:58 srv-ldje-xend02 kernel: [21098.127553] block drbd12: Began resync as SyncSource (will sync 3578732 KB [894683 bits set]).
Mar 21 21:48:58 srv-ldje-xend02 kernel: [21098.127588] block drbd12: updated sync UUID D6A040E6709CF1B7:4402BB4FE91C8B0D:4401BB4FE91C8B0D:4400BB4FE91C8B0D
Mar 21 21:49:18 srv-ldje-xend02 kernel: [21118.216207] block drbd16: Remote failed to finish a request within 12008ms > ko-count (4) * timeout (30 * 0.1s)
Mar 21 21:49:18 srv-ldje-xend02 kernel: [21118.216378] drbd drbd_host16: peer( Secondary -> Unknown ) conn( SyncSource -> Timeout )
Mar 21 21:49:18 srv-ldje-xend02 kernel: [21118.216615] drbd drbd_host16: ack_receiver terminated
Mar 21 21:49:18 srv-ldje-xend02 kernel: [21118.216619] drbd drbd_host16: Terminating drbd_a_drbd_hos
Mar 21 21:49:18 srv-ldje-xend02 kernel: [21118.257185] drbd drbd_host16: Connection closed
Mar 21 21:49:18 srv-ldje-xend02 kernel: [21118.257370] drbd drbd_host16: conn( Timeout -> Unconnected )
Mar 21 21:49:18 srv-ldje-xend02 kernel: [21118.257393] drbd drbd_host16: receiver terminated
Mar 21 21:49:18 srv-ldje-xend02 kernel: [21118.257395] drbd drbd_host16: Restarting receiver thread
Mar 21 21:49:18 srv-ldje-xend02 kernel: [21118.257397] drbd drbd_host16: receiver (re)started
***