Note: "permalinks" may not be as permanent as we would like,
direct links of old sources may well be a few messages off.
Hello,All:
I used to google for the solution,but get no answers.
Somebody says DRBD8.4 doesn't work with sdp?Really?
I have two nodes(Centos 6.4 with kernel 2.6.32-358.el6.x86_64),and two
Mellanox IB cards.Now,I want to use DRBD to sync data between two nodes.But
the DRBD status is always "WFconnect".
[root at node1 home]# cat /proc/drbd
version: 8.4.2 (api:1/proto:86-101)
GIT-hash: 7ad5f850d711223713d6dcadc3dd48860321070c build by
root at localhost.localdomain, 2013-09-30 16:29:29
0: cs:WFConnection ro:Secondary/Unknown ds:Inconsistent/Outdated C r----s
ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:10485404
I have loaded the ib_sdp.ko and pre_loaded lib_sdp.so
[root at node36 ofa_kernel-3.1]# lsmod | grep sdp
ib_sdp 129946 0
rdma_cm 43237 2 rdma_ucm,ib_sdp
ib_core 126865 13
rdma_ucm,ib_ucm,ib_sdp,rdma_cm,iw_cm,ib_ipoib,ib_cm,ib_uverbs,ib_umad,mlx5_ib,mlx4_ib,ib_sa,ib_mad
mlx_compat 32626 18
rdma_ucm,ib_ucm,ib_sdp,rdma_cm,iw_cm,ib_ipoib,ib_cm,ib_uverbs,ib_umad,mlx5_ib,mlx5_core,mlx4_en,mlx4_ib,ib_sa,ib_mad,ib_core,ib_addr,mlx4_core
ipv6 321422 41
ib_sdp,ib_ipoib,ib_core,ib_addr,ip6t_REJECT,nf_conntrack_ipv6,nf_defrag_ipv6
[root at node1 ofa_kernel-3.1]# export
declare -x G_BROKEN_FILENAMES="1"
declare -x HISTCONTROL="ignoredups"
declare -x HISTSIZE="1000"
declare -x HOME="/root"
declare -x HOSTNAME="node1"
declare -x LANG="en_US.UTF-8"
declare -x LD_PRELOAD="/usr/lib64/libsdp.so"
declare -x LESSOPEN="|/usr/bin/lesspipe.sh %s"
declare -x LIBSDP_CONFIG_FILE="/etc/libsdp.conf"
declare -x LOGNAME="root"
Here is part of my configure:
resource res_sdp {
meta-disk internal;
disk /dev/mapper/p1-lv1;
device /dev/drbd0;
on node1 {
address sdp *****:7789;
}
on node36 {
address sdp *****:7789;
}
}
And I get some log of ib_sdp.ko.
Feb 28 15:24:20 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_CONNECT_RESPONSE handled
Feb 28 15:24:20 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_CONNECT_RESPONSE done. status 0
Feb 28 15:24:27 node1 kernel: sdp_cma_handler:438 sdp_sock( 1465:0 7789:0):
event: RDMA_CM_EVENT_CONNECT_REQUEST
Feb 28 15:24:27 node1 kernel: sdp_connect_handler:178 sdp_sock( 1465:0
7789:0): sdp_connect_handler ffff88086ee7ac00 -> ffff88086ef41000
Feb 28 15:24:27 node1 kernel: sdp_init_sock:1325 sdp_sock( 1465:0 7789:0):
sdp_init_sock
Feb 28 15:24:27 node1 kernel: sdp_init_qp:111 sdp_sock( 1465:0 7789:41102):
sdp_init_qp
Feb 28 15:24:27 node1 kernel: sdp_init_qp:114 sdp_sock( 1465:0 7789:41102):
Max sges: 32
Feb 28 15:24:27 node1 kernel: sdp_init_qp:117 sdp_sock( 1465:0 7789:41102):
Setting max send sge to: 9
Feb 28 15:24:27 node1 kernel: sdp_init_qp:120 sdp_sock( 1465:0 7789:41102):
Setting max recv sge to: 9
Feb 28 15:24:27 node1 kernel: sdp_init_qp:151 sdp_sock( 1465:0 7789:41102):
sdp_init_qp done
Feb 28 15:24:27 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
7789:41102): sdp_connect_handler:300 - set state: TCP_LISTEN ->
TCP_SYN_RECV 0x480
Feb 28 15:24:27 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0 7789:0):
event: RDMA_CM_EVENT_CONNECT_REQUEST handled
Feb 28 15:24:27 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0 7789:0):
event: RDMA_CM_EVENT_CONNECT_REQUEST done. status 0
Feb 28 15:24:28 node1 kernel: sdp_cma_handler:438 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_DISCONNECTED
Feb 28 15:24:28 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
58262:7789): sdp_set_error:591 - set state: TCP_ESTABLISHED -> TCP_CLOSE
0xffffffff
Feb 28 15:24:28 node1 kernel: sdp_disconnected_handler:400 sdp_sock( 1465:0
58262:7789): sdp_disconnected_handler
Feb 28 15:24:28 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_DISCONNECTED handled
Feb 28 15:24:28 node1 kernel: sdp_reset_sk:492 sdp_sock( 1465:0
58262:7789): sdp_reset_sk
Feb 28 15:24:28 node1 kernel: sdp_reset_sk:501 sdp_sock( 1465:0
58262:7789): setting state to error
Feb 28 15:24:28 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
58262:7789): sdp_set_error:591 - set state: TCP_CLOSE -> TCP_CLOSE
0xffffffff
Feb 28 15:24:28 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_DISCONNECTED done. status -104
Feb 28 15:24:28 node1 kernel: sdp_destroy_work:1238 sdp_sock( 1615:6
58262:7789): sdp_destroy_work: refcnt 2
Feb 28 15:24:28 node1 kernel: sdp_do_posts:816 sdp_sock( 1614:11
58262:7789): QP is deactivated
Feb 28 15:24:28 node1 kernel: sdp_do_posts:816 sdp_sock( 1614:11
58262:7789): QP is deactivated
Feb 28 15:24:28 node1 kernel: sdp_destroy_qp:242 sdp_sock( 1615:6
58262:7789): destroying qp
Feb 28 15:24:28 node1 kernel: _sdp_exch_state:559 sdp_sock( 1615:0
58262:7789): sdp_destroy_work:1264 - set state: TCP_CLOSE -> TCP_CLOSE
0xffffffff
Feb 28 15:24:31 node1 kernel: sdp_cma_handler:438 sdp_sock( 1465:0 7789:0):
event: RDMA_CM_EVENT_CONNECT_REQUEST
Feb 28 15:24:31 node1 kernel: sdp_connect_handler:178 sdp_sock( 1465:0
7789:0): sdp_connect_handler ffff88086ee7ac00 -> ffff88086dd43000
Feb 28 15:24:31 node1 kernel: sdp_init_sock:1325 sdp_sock( 1465:0 7789:0):
sdp_init_sock
Feb 28 15:24:31 node1 kernel: sdp_init_qp:111 sdp_sock( 1465:0 7789:40747):
sdp_init_qp
Feb 28 15:24:31 node1 kernel: sdp_init_qp:114 sdp_sock( 1465:0 7789:40747):
Max sges: 32
Feb 28 15:24:31 node1 kernel: sdp_init_qp:117 sdp_sock( 1465:0 7789:40747):
Setting max send sge to: 9
Feb 28 15:24:31 node1 kernel: sdp_init_qp:120 sdp_sock( 1465:0 7789:40747):
Setting max recv sge to: 9
Feb 28 15:24:31 node1 kernel: sdp_init_qp:151 sdp_sock( 1465:0 7789:40747):
sdp_init_qp done
Feb 28 15:24:31 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
7789:40747): sdp_connect_handler:300 - set state: TCP_LISTEN ->
TCP_SYN_RECV 0x480
Feb 28 15:24:31 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0 7789:0):
event: RDMA_CM_EVENT_CONNECT_REQUEST handled
Feb 28 15:24:31 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0 7789:0):
event: RDMA_CM_EVENT_CONNECT_REQUEST done. status 0
Feb 28 15:24:31 node1 kernel: sdp_cma_handler:438 sdp_sock( 1465:0
7789:40747): event: RDMA_CM_EVENT_ESTABLISHED
Feb 28 15:24:31 node1 kernel: sdp_connected_handler:358 sdp_sock( 1465:0
7789:40747): sdp_connected_handler
Feb 28 15:24:31 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
7789:40747): sdp_connected_handler:363 - set state: TCP_SYN_RECV ->
TCP_ESTABLISHED 0x8
Feb 28 15:24:31 node1 kernel: sdp_set_default_moderation:361 sdp_sock(
1465:0 7789:40747): Using adaptive interrupt moderation
Feb 28 15:24:31 node1 kernel: sdp_set_default_moderation:374 sdp_sock(
1465:0 7789:40747): Default coalesing params for buf size:32784 -
moder_cnt:10 moder_time:16
Feb 28 15:24:31 node1 kernel: sdp_connected_handler:383 sdp_sock( 1465:0
7789:0): sdp_connected_handler child connection established
Feb 28 15:24:31 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0
7789:40747): event: RDMA_CM_EVENT_ESTABLISHED handled
Feb 28 15:24:31 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0
7789:40747): event: RDMA_CM_EVENT_ESTABLISHED done. status 0
Feb 28 15:24:32 node1 kernel: sdp_do_posts:816 sdp_sock( 3662:11
58262:7789): QP is deactivated
Feb 28 15:24:32 node1 kernel: sdp_close:751 sdp_sock( 3662:11 58262:7789):
sdp_close
Feb 28 15:24:32 node1 kernel: sdp_delete_keepalive_timer:275 sdp_sock(
3662:11 58262:7789): sdp_delete_keepalive_timer
Feb 28 15:24:32 node1 kernel: sdp_close:834 sdp_sock( 3662:11 58262:7789):
sdp_close:834 - sock_put(SOCK_REF_ALIVE) - refcount = 1 from withing
sk_common_release
Feb 28 15:24:32 node1 kernel: sdp_unhash:2671 sdp_sock( 3662:11
58262:7789): sdp_unhash
Feb 28 15:24:32 node1 kernel: sdp_destruct:596 sdp_sock( 3662:11
58262:7789): sdp_destruct
Feb 28 15:24:32 node1 kernel: sdp_destroy_resources:537 sdp_sock( 3662:11
58262:7789): sdp_destroy_resources
Feb 28 15:24:32 node1 kernel: sdp_destroy_qp:242 sdp_sock( 3662:11
58262:7789): destroying qp
Feb 28 15:24:32 node1 kernel: sdp_destroy_resources:572 sdp_sock( 3662:11
58262:7789): sdp_destroy_resources done; releasing sock
Feb 28 15:24:32 node1 kernel: sdp_destruct:648 sdp_sock( 3662:11
58262:7789): sdp_destruct done
Feb 28 15:24:32 node1 kernel: sdp_create_ipvx_socket:2838 sdp_sock( 3662:11
-1:-1): type 1 protocol 6
Feb 28 15:24:32 node1 kernel: sdp_init_sock:1325 sdp_sock( 3662:11 0:0):
sdp_init_sock
Feb 28 15:24:32 node1 kernel: sdp_get_port:216 sdp_sock( 3662:11 0:0):
sdp_get_port: 192.168.1.31:0
Feb 28 15:24:32 node1 kernel: sdp_ipv4_connect:964 sdp_sock( 3662:11
59104:0): sdp_ipv4_connect 192.168.1.31:57574 -> 192.168.1.36:7789
Feb 28 15:24:32 node1 kernel: _sdp_exch_state:559 sdp_sock( 3662:11
59104:0): sdp_connect:1003 - set state: TCP_CLOSE -> TCP_SYN_SENT 0x80
Feb 28 15:24:32 node1 kernel: sdp_cma_handler:438 sdp_sock( 1009:1
59104:0): event: RDMA_CM_EVENT_ADDR_RESOLVED
Feb 28 15:24:32 node1 kernel: sdp_cma_handler:657 sdp_sock( 1009:1
59104:0): event: RDMA_CM_EVENT_ADDR_RESOLVED handled
Feb 28 15:24:32 node1 kernel: sdp_cma_handler:671 sdp_sock( 1009:1
59104:0): event: RDMA_CM_EVENT_ADDR_RESOLVED done. status 0
Feb 28 15:24:32 node1 kernel: sdp_cma_handler:438 sdp_sock( 1599:6
59104:0): event: RDMA_CM_EVENT_ROUTE_RESOLVED
And what did the sdpnetstat say?
[root at node1 home]# sdpnetstat -S
Active Internet connections (w/o servers)
Proto Recv-Q Send-Q Local Address Foreign Address
State
sdp 0 0 node1:office-tools node36:59756
ESTABLISHED
sdp 0 0 node1:office-tools node36:52031
ESTABLISHED
sdp 0 0 node1:office-tools node36:52658
ESTABLISHED
sdp 0 0 node1:54297 node36:office-tools
ESTABLISHED
*Any insight would be greatly appreciated.*
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.linbit.com/pipermail/drbd-user/attachments/20160228/c229bf77/attachment.htm>