Note: "permalinks" may not be as permanent as we would like,
direct links of old sources may well be a few messages off.
I have been struggeling with a performance problem for some time now
and I hope to get some fresh ideas here.
I have two mirrored nodes (Primary/Secondary) and the drbd-device is
on a HW-raid5 with eight drives (total of 6.4 TiB). What first caught
my attention was that the sync speed was very low - only around 25,000
K/Sec. It should be close to the sync speed that I have set to 110
MB/Sec (my NIC is 1Gbit/s).
Since it would take several days to sync the nodes I created a smaller
partition (10 GB) and ran some tests:
---
function dd_test() {
dev=${1}
desc=${2}
for bs in $(seq 500 1 520); do
echo -n "$desc ${dev} ${bs} "
dd if=/dev/zero of=${dev} bs=${bs}M count=1 oflag=direct 2>&1 | \
tail -n1 | cut -d\ -f1,6,8
done
}
TEST_RESOURCE=share
TEST_DEVICE=$(drbdadm sh-dev $TEST_RESOURCE)
TEST_LL_DEVICE=$(drbdadm sh-ll-dev $TEST_RESOURCE)
drbdadm primary $TEST_RESOURCE
dd_test ${TEST_DEVICE} "connected "
drbdadm disconnect ${TEST_RESOURCE}
dd_test ${TEST_DEVICE} disconnected
sleep 1
drbdadm down ${TEST_RESOURCE}
dd_test "${TEST_LL_DEVICE} " "raw_device "
---
The results puzzles me somewhat. There is a large drop in performance
when the size exceeds 510 MB (I have ran tests up to 2GB). The speed
increases when the second node is not connected, but the relative
performance drop becomes even larger. On the "raw" device there is no
such drop in performance. So my conclusion is that this has something
to do with DRBD that I can not understand...
Any ideas?
Typical results from the tests:
---
Test Device MB Bytes Secs Speed (MB/s)
connected /dev/drbd2 500 524288000 8.22073 63.8
connected /dev/drbd2 501 525336576 4.96216 106
connected /dev/drbd2 502 526385152 4.94810 106
connected /dev/drbd2 503 527433728 5.17096 102
connected /dev/drbd2 504 528482304 5.11144 103
connected /dev/drbd2 505 529530880 5.14627 103
connected /dev/drbd2 506 530579456 5.08018 104
connected /dev/drbd2 507 531628032 5.06370 105
connected /dev/drbd2 508 532676608 5.11654 104
connected /dev/drbd2 509 533725184 5.12548 104
connected /dev/drbd2 510 534773760 8.70478 61.4
connected /dev/drbd2 511 535822336 7.53861 71.1
connected /dev/drbd2 512 536870912 8.30908 64.6
connected /dev/drbd2 513 537919488 8.09685 66.4
connected /dev/drbd2 514 538968064 7.88830 68.3
connected /dev/drbd2 515 540016640 7.68320 70.3
connected /dev/drbd2 516 541065216 8.38615 64.5
connected /dev/drbd2 517 542113792 8.65357 62.6
connected /dev/drbd2 518 543162368 8.63435 62.9
connected /dev/drbd2 519 544210944 8.47207 64.2
connected /dev/drbd2 520 545259520 7.68356 71.0
disconnected /dev/drbd2 500 524288000 6.20871 84.4
disconnected /dev/drbd2 501 525336576 1.23159 427
disconnected /dev/drbd2 502 526385152 1.19747 440
disconnected /dev/drbd2 503 527433728 1.24428 424
disconnected /dev/drbd2 504 528482304 1.36408 387
disconnected /dev/drbd2 505 529530880 1.35296 391
disconnected /dev/drbd2 506 530579456 1.30420 407
disconnected /dev/drbd2 507 531628032 1.17819 451
disconnected /dev/drbd2 508 532676608 1.22635 434
disconnected /dev/drbd2 509 533725184 1.29133 413
disconnected /dev/drbd2 510 534773760 4.25663 126
disconnected /dev/drbd2 511 535822336 3.53406 152
disconnected /dev/drbd2 512 536870912 3.73568 144
disconnected /dev/drbd2 513 537919488 3.70945 145
disconnected /dev/drbd2 514 538968064 3.28342 164
disconnected /dev/drbd2 515 540016640 3.62062 149
disconnected /dev/drbd2 516 541065216 3.87234 140
disconnected /dev/drbd2 517 542113792 4.37600 124
disconnected /dev/drbd2 518 543162368 4.19390 130
disconnected /dev/drbd2 519 544210944 3.85436 141
disconnected /dev/drbd2 520 545259520 4.11660 132
raw_device /dev/sdc1 500 524288000 1.09239 480
raw_device /dev/sdc1 501 525336576 1.09424 480
raw_device /dev/sdc1 502 526385152 1.28691 409
raw_device /dev/sdc1 503 527433728 1.40394 376
raw_device /dev/sdc1 504 528482304 1.15559 457
raw_device /dev/sdc1 505 529530880 1.18005 449
raw_device /dev/sdc1 506 530579456 1.24624 426
raw_device /dev/sdc1 507 531628032 1.22667 433
raw_device /dev/sdc1 508 532676608 1.41159 377
raw_device /dev/sdc1 509 533725184 1.19987 445
raw_device /dev/sdc1 510 534773760 1.14022 469
raw_device /dev/sdc1 511 535822336 1.28896 416
raw_device /dev/sdc1 512 536870912 1.23961 433
raw_device /dev/sdc1 513 537919488 1.35534 397
raw_device /dev/sdc1 514 538968064 1.11606 483
raw_device /dev/sdc1 515 540016640 1.26940 425
raw_device /dev/sdc1 516 541065216 1.19242 454
raw_device /dev/sdc1 517 542113792 1.30275 416
raw_device /dev/sdc1 518 543162368 1.36742 397
raw_device /dev/sdc1 519 544210944 1.25283 434
raw_device /dev/sdc1 520 545259520 1.40227 389
---
drbdadm dump:
---
DRBD module version: 8.3.13
userland version: 8.3.11
you should upgrade your drbd tools!
# /etc/drbd.conf
common {
protocol C;
net {
after-sb-0pri discard-zero-changes;
sndbuf-size 0;
max-buffers 8000;
max-epoch-size 8000;
}
disk {
fencing resource-only;
}
syncer {
rate 110M;
}
handlers {
pri-on-incon-degr "/usr/lib64/drbd/notify-pri-on-incon-degr.sh; /usr/lib64/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
pri-lost-after-sb "/usr/lib64/drbd/notify-pri-lost-after-sb.sh; /usr/lib64/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
local-io-error "/usr/lib64/drbd/notify-io-error.sh; /usr/lib64/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
fence-peer /usr/lib64/drbd/crm-fence-peer.sh;
after-resync-target /usr/lib64/drbd/crm-unfence-peer.sh;
split-brain "/usr/lib64/drbd/notify-split-brain.sh root";
out-of-sync "/usr/lib64/drbd/notify-out-of-sync.sh root";
}
}
# resource ha_share on node2.lan: not ignored, not stacked
resource ha_share {
on node2.lan {
device /dev/drbd1 minor 1;
disk /dev/mapper/system_vg-ha_share;
address ipv4 192.168.133.11:7788;
meta-disk internal;
}
on node1.lan {
device /dev/drbd1 minor 1;
disk /dev/mapper/system_vg-ha_share;
address ipv4 192.168.133.10:7788;
meta-disk internal;
}
}
# resource share on node2.lan: not ignored, not stacked
resource share {
on node2.lan {
device /dev/drbd2 minor 2;
disk /dev/sdc1;
address ipv4 192.168.133.11:7789;
meta-disk internal;
}
on node1.lan {
device /dev/drbd2 minor 2;
disk /dev/sdc1;
address ipv4 192.168.133.10:7789;
meta-disk internal;
}
}
---