[DRBD-user] Multiple DRBD stacked resources with pacemaker

[matth ingersoll]

I'm trying to figure out the proper/preferred method for configuring
multiple stacked DRBD resources with pacemaker.  The layers involved
from bottom to top:

a. drbd
b. stacked drbd IP (ocf:heartbeat:IPaddr2)
c. drbd stacked resources

-- (resource group "rg_test") --
d. lvm volume groups (ocf:heartbeat:LVM)
e. iscsitarget (IET- ocf:heartbeat:iSCSITarget)
f. iscsi logical unit (ocf:heartbeat:iSCSILogicalUnit)

The first example from
http://www.drbd.org/users-guide/s-pacemaker-stacked-resources.html is
fairly easy to follow and understand but searching the web, finding a
multi stacked resource example and then mixing in another resource
group on top of it is proving difficult.

An example of what I have tried (and many variations of this):

primitive p_drbd_r0 ocf:linbit:drbd \
	params drbd_resource="r0"
primitive p_drbd_r1 ocf:linbit:drbd \
	params drbd_resource="r1"

primitive p_drbd_r0-U ocf:linbit:drbd \
	params drbd_resource="r0-U"

primitive p_drbd_r0-U ocf:linbit:drbd \
	params drbd_resource="r0-U"
primitive p_drbd_r1-U ocf:linbit:drbd \
	params drbd_resource="r1-U"

primitive p_ip_stacked ocf:heartbeat:IPaddr2 \
	params ip="192.168.42.1" nic="eth0"

primitive res_iscsi_ip_eth5.3009 ocf:heartbeat:IPaddr2 \
        params ip="10.50.9.145" cidr_netmask="28" nic="eth5.3009"
primitive res_lu_vg0_lv0 ocf:heartbeat:iSCSILogicalUnit \
        params target_iqn="iqn.2011-05.net.example:3009.1" lun="0"
path="/dev/vg0/lv0"
primitive res_lvm_vg0 ocf:heartbeat:LVM \
        params volgrpname="vg0"
primitive res_target_3009 ocf:heartbeat:iSCSITarget \
        params iqn="iqn.2011-05.net.example:3009.1" tid="1"
allowed_initiators="10.50.9.145/28"

ms ms_drbd_r0 p_drbd_r0 \
	meta master-max="1" master-node-max="1" \
        clone-max="2" clone-node-max="1" \
        notify="true" globally-unique="false"
ms ms_drbd_r1 p_drbd_r1 \
	meta master-max="1" master-node-max="1" \
        clone-max="2" clone-node-max="1" \
        notify="true" globally-unique="false"

ms ms_drbd_r0-U p_drbd_r0-U \
	meta master-max="1" clone-max="1" \
        clone-node-max="1" master-node-max="1" \
        notify="true" globally-unique="false"
ms ms_drbd_r1-U p_drbd_r1-U \
	meta master-max="1" clone-max="1" \
        clone-node-max="1" master-node-max="1" \
        notify="true" globally-unique="false"

group rg_test res_iscsi_ip_eth5.3009 res_lvm_vg0  res_target_3009
res_lu_vg0_lv0 \
	meta target-role="Stopped"

location cli-standby-rg_test rg_test \
        rule $id="cli-standby-rule-rg_test" -inf: #uname eq node01

colocation c_drbd_r0-U_on_drbd_r0 \
        inf: ms_drbd_r0-U ms_drbd_r0:Master
colocation c_drbd_r1-U_on_drbd_r1 \
        inf: ms_drbd_r1-U ms_drbd_r1:Master

colocation c_drbd_r0-U_on_ip \
        inf: ms_drbd_r0-U p_ip_stacked
colocation c_drbd_r1-U_on_ip \
        inf: ms_drbd_r1-U p_ip_stacked

colocation c_ip_on_r0_master \
        inf: p_ip_stacked ms_drbd_r0:Master
colocation c_ip_on_r1_master \
        inf: p_ip_stacked ms_drbd_r1:Master

order o_ip_before_r0-U \
        inf: p_ip_stacked ms_drbd_r0-U:start
order o_ip_before_r1-U \
        inf: p_ip_stacked ms_drbd_r1-U:start

order o_drbd_r0_before_r0-U \
        inf: ms_drbd_r0:promote ms_drbd_r0-U:start
order o_drbd_r1_before_r1-U \
        inf: ms_drbd_r1:promote ms_drbd_r1-U:start

property $id="cib-bootstrap-options" \
        dc-version="1.0.9-74392a28b7f31d7ddc86689598bd23114f58978b" \
        cluster-infrastructure="Heartbeat" \
        stonith-enabled="false" \
        no-quorum-policy="ignore" \
        default-resource-stickiness="200" \
        last-lrm-refresh="1305158205"

I can get it to sometimes work (possibly not from the above example
config) but it appears to be getting stuck on when transitioning the
stacked IP address - things get out of order.  I have also confirmed
that by manually starting each layer outside of heartbeat/crm, it
works fine.  Any help is much appreciated.

Regards,

Matth