Note: "permalinks" may not be as permanent as we would like,
direct links of old sources may well be a few messages off.
On 29/02/2016 2:00 AM, "翟果" <zhai.guo at datatom.com> wrote:
>
> Hello,All:
> I used to google for the solution,but get no answers.
> Somebody says DRBD8.4 doesn't work with sdp?Really?
As far as I can see thats not true:
http://drbd.linbit.com/users-guide-8.4/s-replication-transports.html
> I have two nodes(Centos 6.4 with kernel 2.6.32-358.el6.x86_64),and
two Mellanox IB cards.Now,I want to use DRBD to sync data between two
nodes.But the DRBD status is always "WFconnect".
>
> [root at node1 home]# cat /proc/drbd
> version: 8.4.2 (api:1/proto:86-101)
> GIT-hash: 7ad5f850d711223713d6dcadc3dd48860321070c build by
root at localhost.localdomain, 2013-09-30 16:29:29
> 0: cs:WFConnection ro:Secondary/Unknown ds:Inconsistent/Outdated C r----s
> ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f
oos:10485404
>
>
> I have loaded the ib_sdp.ko and pre_loaded lib_sdp.so
>
> [root at node36 ofa_kernel-3.1]# lsmod | grep sdp
> ib_sdp 129946 0
> rdma_cm 43237 2 rdma_ucm,ib_sdp
> ib_core 126865 13
rdma_ucm,ib_ucm,ib_sdp,rdma_cm,iw_cm,ib_ipoib,ib_cm,ib_uverbs,ib_umad,mlx5_ib,mlx4_ib,ib_sa,ib_mad
> mlx_compat 32626 18
rdma_ucm,ib_ucm,ib_sdp,rdma_cm,iw_cm,ib_ipoib,ib_cm,ib_uverbs,ib_umad,mlx5_ib,mlx5_core,mlx4_en,mlx4_ib,ib_sa,ib_mad,ib_core,ib_addr,mlx4_core
> ipv6 321422 41
ib_sdp,ib_ipoib,ib_core,ib_addr,ip6t_REJECT,nf_conntrack_ipv6,nf_defrag_ipv6
>
> [root at node1 ofa_kernel-3.1]# export
> declare -x G_BROKEN_FILENAMES="1"
> declare -x HISTCONTROL="ignoredups"
> declare -x HISTSIZE="1000"
> declare -x HOME="/root"
> declare -x HOSTNAME="node1"
> declare -x LANG="en_US.UTF-8"
> declare -x LD_PRELOAD="/usr/lib64/libsdp.so"
> declare -x LESSOPEN="|/usr/bin/lesspipe.sh %s"
> declare -x LIBSDP_CONFIG_FILE="/etc/libsdp.conf"
> declare -x LOGNAME="root"
>
> Here is part of my configure:
> resource res_sdp {
> meta-disk internal;
> disk /dev/mapper/p1-lv1;
> device /dev/drbd0;
>
> on node1 {
> address sdp *****:7789;
> }
> on node36 {
> address sdp *****:7789;
> }
> }
>
> And I get some log of ib_sdp.ko.
>
> Feb 28 15:24:20 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_CONNECT_RESPONSE handled
> Feb 28 15:24:20 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_CONNECT_RESPONSE done. status 0
> Feb 28 15:24:27 node1 kernel: sdp_cma_handler:438 sdp_sock( 1465:0
7789:0): event: RDMA_CM_EVENT_CONNECT_REQUEST
> Feb 28 15:24:27 node1 kernel: sdp_connect_handler:178 sdp_sock( 1465:0
7789:0): sdp_connect_handler ffff88086ee7ac00 -> ffff88086ef41000
> Feb 28 15:24:27 node1 kernel: sdp_init_sock:1325 sdp_sock( 1465:0
7789:0): sdp_init_sock
> Feb 28 15:24:27 node1 kernel: sdp_init_qp:111 sdp_sock( 1465:0
7789:41102): sdp_init_qp
> Feb 28 15:24:27 node1 kernel: sdp_init_qp:114 sdp_sock( 1465:0
7789:41102): Max sges: 32
> Feb 28 15:24:27 node1 kernel: sdp_init_qp:117 sdp_sock( 1465:0
7789:41102): Setting max send sge to: 9
> Feb 28 15:24:27 node1 kernel: sdp_init_qp:120 sdp_sock( 1465:0
7789:41102): Setting max recv sge to: 9
> Feb 28 15:24:27 node1 kernel: sdp_init_qp:151 sdp_sock( 1465:0
7789:41102): sdp_init_qp done
> Feb 28 15:24:27 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
7789:41102): sdp_connect_handler:300 - set state: TCP_LISTEN ->
TCP_SYN_RECV 0x480
> Feb 28 15:24:27 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0
7789:0): event: RDMA_CM_EVENT_CONNECT_REQUEST handled
> Feb 28 15:24:27 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0
7789:0): event: RDMA_CM_EVENT_CONNECT_REQUEST done. status 0
> Feb 28 15:24:28 node1 kernel: sdp_cma_handler:438 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_DISCONNECTED
> Feb 28 15:24:28 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
58262:7789): sdp_set_error:591 - set state: TCP_ESTABLISHED -> TCP_CLOSE
0xffffffff
> Feb 28 15:24:28 node1 kernel: sdp_disconnected_handler:400 sdp_sock(
1465:0 58262:7789): sdp_disconnected_handler
> Feb 28 15:24:28 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_DISCONNECTED handled
> Feb 28 15:24:28 node1 kernel: sdp_reset_sk:492 sdp_sock( 1465:0
58262:7789): sdp_reset_sk
> Feb 28 15:24:28 node1 kernel: sdp_reset_sk:501 sdp_sock( 1465:0
58262:7789): setting state to error
> Feb 28 15:24:28 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
58262:7789): sdp_set_error:591 - set state: TCP_CLOSE -> TCP_CLOSE
0xffffffff
> Feb 28 15:24:28 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0
58262:7789): event: RDMA_CM_EVENT_DISCONNECTED done. status -104
> Feb 28 15:24:28 node1 kernel: sdp_destroy_work:1238 sdp_sock( 1615:6
58262:7789): sdp_destroy_work: refcnt 2
> Feb 28 15:24:28 node1 kernel: sdp_do_posts:816 sdp_sock( 1614:11
58262:7789): QP is deactivated
> Feb 28 15:24:28 node1 kernel: sdp_do_posts:816 sdp_sock( 1614:11
58262:7789): QP is deactivated
> Feb 28 15:24:28 node1 kernel: sdp_destroy_qp:242 sdp_sock( 1615:6
58262:7789): destroying qp
> Feb 28 15:24:28 node1 kernel: _sdp_exch_state:559 sdp_sock( 1615:0
58262:7789): sdp_destroy_work:1264 - set state: TCP_CLOSE -> TCP_CLOSE
0xffffffff
> Feb 28 15:24:31 node1 kernel: sdp_cma_handler:438 sdp_sock( 1465:0
7789:0): event: RDMA_CM_EVENT_CONNECT_REQUEST
> Feb 28 15:24:31 node1 kernel: sdp_connect_handler:178 sdp_sock( 1465:0
7789:0): sdp_connect_handler ffff88086ee7ac00 -> ffff88086dd43000
> Feb 28 15:24:31 node1 kernel: sdp_init_sock:1325 sdp_sock( 1465:0
7789:0): sdp_init_sock
> Feb 28 15:24:31 node1 kernel: sdp_init_qp:111 sdp_sock( 1465:0
7789:40747): sdp_init_qp
> Feb 28 15:24:31 node1 kernel: sdp_init_qp:114 sdp_sock( 1465:0
7789:40747): Max sges: 32
> Feb 28 15:24:31 node1 kernel: sdp_init_qp:117 sdp_sock( 1465:0
7789:40747): Setting max send sge to: 9
> Feb 28 15:24:31 node1 kernel: sdp_init_qp:120 sdp_sock( 1465:0
7789:40747): Setting max recv sge to: 9
> Feb 28 15:24:31 node1 kernel: sdp_init_qp:151 sdp_sock( 1465:0
7789:40747): sdp_init_qp done
> Feb 28 15:24:31 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
7789:40747): sdp_connect_handler:300 - set state: TCP_LISTEN ->
TCP_SYN_RECV 0x480
> Feb 28 15:24:31 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0
7789:0): event: RDMA_CM_EVENT_CONNECT_REQUEST handled
> Feb 28 15:24:31 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0
7789:0): event: RDMA_CM_EVENT_CONNECT_REQUEST done. status 0
> Feb 28 15:24:31 node1 kernel: sdp_cma_handler:438 sdp_sock( 1465:0
7789:40747): event: RDMA_CM_EVENT_ESTABLISHED
> Feb 28 15:24:31 node1 kernel: sdp_connected_handler:358 sdp_sock( 1465:0
7789:40747): sdp_connected_handler
> Feb 28 15:24:31 node1 kernel: _sdp_exch_state:559 sdp_sock( 1465:0
7789:40747): sdp_connected_handler:363 - set state: TCP_SYN_RECV ->
TCP_ESTABLISHED 0x8
> Feb 28 15:24:31 node1 kernel: sdp_set_default_moderation:361 sdp_sock(
1465:0 7789:40747): Using adaptive interrupt moderation
> Feb 28 15:24:31 node1 kernel: sdp_set_default_moderation:374 sdp_sock(
1465:0 7789:40747): Default coalesing params for buf size:32784 -
moder_cnt:10 moder_time:16
> Feb 28 15:24:31 node1 kernel: sdp_connected_handler:383 sdp_sock( 1465:0
7789:0): sdp_connected_handler child connection established
> Feb 28 15:24:31 node1 kernel: sdp_cma_handler:657 sdp_sock( 1465:0
7789:40747): event: RDMA_CM_EVENT_ESTABLISHED handled
> Feb 28 15:24:31 node1 kernel: sdp_cma_handler:671 sdp_sock( 1465:0
7789:40747): event: RDMA_CM_EVENT_ESTABLISHED done. status 0
> Feb 28 15:24:32 node1 kernel: sdp_do_posts:816 sdp_sock( 3662:11
58262:7789): QP is deactivated
> Feb 28 15:24:32 node1 kernel: sdp_close:751 sdp_sock( 3662:11
58262:7789): sdp_close
> Feb 28 15:24:32 node1 kernel: sdp_delete_keepalive_timer:275 sdp_sock(
3662:11 58262:7789): sdp_delete_keepalive_timer
> Feb 28 15:24:32 node1 kernel: sdp_close:834 sdp_sock( 3662:11
58262:7789): sdp_close:834 - sock_put(SOCK_REF_ALIVE) - refcount = 1 from
withing sk_common_release
> Feb 28 15:24:32 node1 kernel: sdp_unhash:2671 sdp_sock( 3662:11
58262:7789): sdp_unhash
> Feb 28 15:24:32 node1 kernel: sdp_destruct:596 sdp_sock( 3662:11
58262:7789): sdp_destruct
> Feb 28 15:24:32 node1 kernel: sdp_destroy_resources:537 sdp_sock( 3662:11
58262:7789): sdp_destroy_resources
> Feb 28 15:24:32 node1 kernel: sdp_destroy_qp:242 sdp_sock( 3662:11
58262:7789): destroying qp
> Feb 28 15:24:32 node1 kernel: sdp_destroy_resources:572 sdp_sock( 3662:11
58262:7789): sdp_destroy_resources done; releasing sock
> Feb 28 15:24:32 node1 kernel: sdp_destruct:648 sdp_sock( 3662:11
58262:7789): sdp_destruct done
> Feb 28 15:24:32 node1 kernel: sdp_create_ipvx_socket:2838 sdp_sock(
3662:11 -1:-1): type 1 protocol 6
> Feb 28 15:24:32 node1 kernel: sdp_init_sock:1325 sdp_sock( 3662:11 0:0):
sdp_init_sock
> Feb 28 15:24:32 node1 kernel: sdp_get_port:216 sdp_sock( 3662:11 0:0):
sdp_get_port: 192.168.1.31:0
> Feb 28 15:24:32 node1 kernel: sdp_ipv4_connect:964 sdp_sock( 3662:11
59104:0): sdp_ipv4_connect 192.168.1.31:57574 -> 192.168.1.36:7789
> Feb 28 15:24:32 node1 kernel: _sdp_exch_state:559 sdp_sock( 3662:11
59104:0): sdp_connect:1003 - set state: TCP_CLOSE -> TCP_SYN_SENT 0x80
> Feb 28 15:24:32 node1 kernel: sdp_cma_handler:438 sdp_sock( 1009:1
59104:0): event: RDMA_CM_EVENT_ADDR_RESOLVED
> Feb 28 15:24:32 node1 kernel: sdp_cma_handler:657 sdp_sock( 1009:1
59104:0): event: RDMA_CM_EVENT_ADDR_RESOLVED handled
> Feb 28 15:24:32 node1 kernel: sdp_cma_handler:671 sdp_sock( 1009:1
59104:0): event: RDMA_CM_EVENT_ADDR_RESOLVED done. status 0
> Feb 28 15:24:32 node1 kernel: sdp_cma_handler:438 sdp_sock( 1599:6
59104:0): event: RDMA_CM_EVENT_ROUTE_RESOLVED
>
> And what did the sdpnetstat say?
> [root at node1 home]# sdpnetstat -S
> Active Internet connections (w/o servers)
> Proto Recv-Q Send-Q Local Address Foreign Address
State
> sdp 0 0 node1:office-tools node36:59756
ESTABLISHED
> sdp 0 0 node1:office-tools node36:52031
ESTABLISHED
> sdp 0 0 node1:office-tools node36:52658
ESTABLISHED
> sdp 0 0 node1:54297 node36:office-tools
ESTABLISHED
>
> Any insight would be greatly appreciated.
>
>
>
> _______________________________________________
> drbd-user mailing list
> drbd-user at lists.linbit.com
> http://lists.linbit.com/mailman/listinfo/drbd-user
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.linbit.com/pipermail/drbd-user/attachments/20160229/0c9e3bc2/attachment.htm>