[DRBD-user] Second time failover not working

hcb hormazdb at gmail.com
Tue Nov 23 14:58:56 CET 2010


Hi,

I have a 2-node cluster that is setup using heartbeat/pacemaker and drbd
used to provide a HA mysql service.

I was running some failover tests to check if it was all working properly
and ran into a slight issue.
When i pull the plug on node 1 i.e. the primary node, the secondary node
becomes active without any issues. 
After rebooting node 1 it becomes the secondary node. 

This is expected behavior. (node1 became secondary and node2 became primary)
Also /proc/drbd shows that primary and secondary are uptodate

However now when i pull the plug on node 2 i.e now the primary node , node 1
does not failover to become the primary node again. The error that i receive
is that node 1 is outofdate and it refuses to promote to primary without an
uptodate disk. Shouldnt it automatically re-promote node 1 to primary?

/proc/drbd starts fluctuating between 
1:
 0: cs:WFConnection ro:Secondary/Unknown ds:UpToDate/DUnknown C r----
    ns:116 nr:296 dw:528 dr:3408 al:7 bm:22 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b
oos:84
2:
0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C r----
    ns:116 nr:296 dw:544 dr:3466 al:7 bm:52 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b
oos:84

Here are the config files:

ha.cf

autojoin none
bcast eth1
warntime 5
deadtime 15
initdead 60
keepalive 2
crm respawn
node node-1
node node-2

crm config file

<cib validate-with="pacemaker-1.0" crm_feature_set="3.0.1" have-quorum="1"
dc-uuid="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c" admin_epoch="0" epoch="124"
num_updates="472">
  <configuration>
    <crm_config>
      <cluster_property_set id="cib-bootstrap-options">
        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version"
value="1.0.5-3840e6b5a305ccb803d29b468556739e75532d56"/>
        <nvpair id="cib-bootstrap-options-cluster-infrastructure"
name="cluster-infrastructure" value="Heartbeat"/>
        <nvpair id="cib-bootstrap-options-stonith-enabled"
name="stonith-enabled" value="false"/>
        <nvpair name="last-lrm-refresh"
id="cib-bootstrap-options-last-lrm-refresh" value="1290500431"/>
        <nvpair id="cib-bootstrap-options-no-quorum-policy"
name="no-quorum-policy" value="ignore"/>
      </cluster_property_set>
    </crm_config>
    <nodes>
      <node id="8b86ee1d-9bea-4b51-b809-286c5be30c15" uname="node-2"
type="normal">
        <instance_attributes
id="nodes-8b86ee1d-9bea-4b51-b809-286c5be30c15">
          <nvpair id="nodes-8b86ee1d-9bea-4b51-b809-286c5be30c15-standby"
name="standby" value="off"/>
        </instance_attributes>
      </node>
      <node uname="node-1" type="normal"
id="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
        <instance_attributes
id="nodes-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
          <nvpair name="standby"
id="nodes-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c-standby" value="off"/>
        </instance_attributes>
      </node>
    </nodes>
    <resources>
      <master id="ms_drbd_mysql">
        <meta_attributes id="ms_drbd_mysql-meta_attributes">
          <nvpair id="ms_drbd_mysql-meta_attributes-master-max"
name="master-max" value="1"/>
          <nvpair id="ms_drbd_mysql-meta_attributes-master-node-max"
name="master-node-max" value="1"/>
          <nvpair id="ms_drbd_mysql-meta_attributes-clone-max"
name="clone-max" value="2"/>
          <nvpair id="ms_drbd_mysql-meta_attributes-clone-node-max"
name="clone-node-max" value="1"/>
          <nvpair id="ms_drbd_mysql-meta_attributes-notify" name="notify"
value="true"/>
          <nvpair name="target-role"
id="ms_drbd_mysql-meta_attributes-target-role" value="Started"/>
        </meta_attributes>
        <primitive class="ocf" id="drbd_mysql" provider="linbit"
type="drbd">
          <instance_attributes id="drbd_mysql-instance_attributes">
            <nvpair id="drbd_mysql-instance_attributes-drbd_resource"
name="drbd_resource" value="r0"/>
          </instance_attributes>
          <operations>
            <op id="drbd_mysql-monitor-15s" interval="15s" name="monitor"/>
          </operations>
        </primitive>
      </master>
      <group id="mysql">
        <primitive class="ocf" provider="heartbeat" type="Filesystem"
id="fs_mysql">
          <instance_attributes id="fs_mysql-instance_attributes">
            <nvpair name="device" id="fs_mysql-instance_attributes-device"
value="/dev/drbd0"/>
            <nvpair id="fs_mysql-instance_attributes-directory"
name="directory" value="/mnt/drbd"/>
            <nvpair id="fs_mysql-instance_attributes-fstype" name="fstype"
value="ext3"/>
          </instance_attributes>
          <meta_attributes id="fs_mysql-meta_attributes">
            <nvpair name="target-role"
id="fs_mysql-meta_attributes-target-role" value="Started"/>
          </meta_attributes>
        </primitive>
        <primitive class="ocf" id="ip_mysql" provider="heartbeat"
type="IPaddr2">
          <instance_attributes id="ip_mysql-instance_attributes">
            <nvpair id="ip_mysql-instance_attributes-ip" name="ip"
value="192.168.42.85"/>
            <nvpair id="ip_mysql-instance_attributes-nic" name="nic"
value="eth0"/>
          </instance_attributes>
        </primitive>
        <primitive class="lsb" type="mysql" id="mysqld">
          <meta_attributes id="mysqld-meta_attributes">
            <nvpair name="target-role"
id="mysqld-meta_attributes-target-role" value="Started"/>
          </meta_attributes>
        </primitive>
        <meta_attributes id="mysql-meta_attributes">
          <nvpair id="mysql-meta_attributes-is-managed" name="is-managed"
value="true"/>
        </meta_attributes>
      </group>
    </resources>
    <constraints>
      <rsc_order first="ms_drbd_mysql" first-action="promote"
id="mysql_after_drbd" score="INFINITY" then="mysql" then-action="start"/>
      <rsc_colocation id="mysql_on_drbd" rsc="mysql" score="INFINITY"
with-rsc="ms_drbd_mysql" with-rsc-role="Master"/>
    </constraints>
    <rsc_defaults/>
    <op_defaults/>
  </configuration>
  <status>
    <node_state uname="node-2" ha="active" in_ccm="true" crmd="online"
shutdown="0" join="member" expected="member"
id="8b86ee1d-9bea-4b51-b809-286c5be30c15"
crm-debug-origin="do_update_resource">
      <lrm id="8b86ee1d-9bea-4b51-b809-286c5be30c15">
        <lrm_resources>
          <lrm_resource id="fs_mysql" type="Filesystem" class="ocf"
provider="heartbeat">
            <lrm_rsc_op id="fs_mysql_monitor_0" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="5:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:7;5:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="3" rc-code="7" op-status="0" interval="0" last-run="1290517413"
last-rc-change="1290517413" exec-time="40" queue-time="0"
op-digest="6e170ae78fb628de2bc66c50e403c073"/>
          </lrm_resource>
          <lrm_resource id="ip_mysql" type="IPaddr2" class="ocf"
provider="heartbeat">
            <lrm_rsc_op id="ip_mysql_monitor_0" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="6:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:7;6:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="4" rc-code="7" op-status="0" interval="0" last-run="1290517413"
last-rc-change="1290517413" exec-time="30" queue-time="0"
op-digest="eeb1174061459dd7928cd3e1222390ba"/>
          </lrm_resource>
          <lrm_resource id="mysqld" type="mysql" class="lsb">
            <lrm_rsc_op id="mysqld_monitor_0" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="7:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:7;7:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="5" rc-code="7" op-status="0" interval="0" last-run="1290517413"
last-rc-change="1290517413" exec-time="30" queue-time="0"
op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
          </lrm_resource>
          <lrm_resource id="drbd_mysql:0" type="drbd" class="ocf"
provider="linbit">
            <lrm_rsc_op id="drbd_mysql:0_monitor_0" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="4:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;4:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="2" rc-code="0" op-status="0" interval="0" last-run="1290517413"
last-rc-change="1290517413" exec-time="70" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
            <lrm_rsc_op id="drbd_mysql:0_monitor_15000" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="8:110:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;8:110:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="6" rc-code="0" op-status="0" interval="15000" last-run="1290517415"
last-rc-change="1290517415" exec-time="60" queue-time="0"
op-digest="db2fb3216fa7207fdc0b2bf31bbd09c1"/>
          </lrm_resource>
        </lrm_resources>
      </lrm>
      <transient_attributes id="8b86ee1d-9bea-4b51-b809-286c5be30c15">
        <instance_attributes
id="status-8b86ee1d-9bea-4b51-b809-286c5be30c15">
          <nvpair
id="status-8b86ee1d-9bea-4b51-b809-286c5be30c15-master-drbd_mysql:0"
name="master-drbd_mysql:0" value="10000"/>
          <nvpair
id="status-8b86ee1d-9bea-4b51-b809-286c5be30c15-probe_complete"
name="probe_complete" value="true"/>
        </instance_attributes>
      </transient_attributes>
    </node_state>
    <node_state uname="node-1" ha="active" in_ccm="true" crmd="online"
shutdown="0" join="member" expected="member"
id="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c"
crm-debug-origin="do_update_resource">
      <transient_attributes id="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
        <instance_attributes
id="status-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
          <nvpair
id="status-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c-probe_complete"
name="probe_complete" value="true"/>
          <nvpair name="master-drbd_mysql:1"
id="status-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c-master-drbd_mysql:1"
value="10000"/>
        </instance_attributes>
      </transient_attributes>
      <lrm id="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
        <lrm_resources>
          <lrm_resource id="drbd_mysql:1" type="drbd" class="ocf"
provider="linbit">
            <lrm_rsc_op id="drbd_mysql:1_monitor_0" operation="monitor"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="5:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:7;5:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="2" rc-code="7" op-status="0" interval="0" last-run="1290516543"
last-rc-change="1290516543" exec-time="60" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
            <lrm_rsc_op id="drbd_mysql:1_start_0" operation="start"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="13:1:0:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:0;13:1:0:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="6" rc-code="0" op-status="0" interval="0" last-run="1290516545"
last-rc-change="1290516545" exec-time="170" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
            <lrm_rsc_op id="drbd_mysql:1_demote_0" operation="demote"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="6:107:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;6:107:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="358" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="200" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
            <lrm_rsc_op id="drbd_mysql:1_promote_0" operation="promote"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="7:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;7:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="361" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="70" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
            <lrm_rsc_op id="drbd_mysql:1_post_notify_promote_0"
operation="notify" crm-debug-origin="build_active_RAs"
crm_feature_set="3.0.1"
transition-key="53:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;53:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="362" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="60" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
            <lrm_rsc_op id="drbd_mysql:1_pre_notify_start_0"
operation="notify" crm-debug-origin="do_update_resource"
crm_feature_set="3.0.1"
transition-key="55:109:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;55:109:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="366" rc-code="0" op-status="0" interval="0" last-run="1290517456"
last-rc-change="1290517456" exec-time="50" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
          </lrm_resource>
          <lrm_resource id="mysqld" type="mysql" class="lsb">
            <lrm_rsc_op id="mysqld_monitor_0" operation="monitor"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="8:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:7;8:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="5" rc-code="7" op-status="0" interval="0" last-run="1290516543"
last-rc-change="1290516543" exec-time="30" queue-time="0"
op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
            <lrm_rsc_op id="mysqld_start_0" operation="start"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="34:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;34:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="365" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="1190" queue-time="0"
op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
          </lrm_resource>
          <lrm_resource id="fs_mysql" type="Filesystem" class="ocf"
provider="heartbeat">
            <lrm_rsc_op id="fs_mysql_monitor_0" operation="monitor"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="6:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:7;6:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="3" rc-code="7" op-status="0" interval="0" last-run="1290516543"
last-rc-change="1290516543" exec-time="20" queue-time="0"
op-digest="6e170ae78fb628de2bc66c50e403c073"/>
            <lrm_rsc_op id="fs_mysql_stop_0" operation="stop"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="32:99:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;32:99:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="332" rc-code="0" op-status="0" interval="0" last-run="1290517083"
last-rc-change="1290517083" exec-time="50" queue-time="0"
op-digest="6e170ae78fb628de2bc66c50e403c073"/>
            <lrm_rsc_op id="fs_mysql_start_0" operation="start"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="32:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;32:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="363" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="100" queue-time="0"
op-digest="6e170ae78fb628de2bc66c50e403c073"/>
          </lrm_resource>
          <lrm_resource id="ip_mysql" type="IPaddr2" class="ocf"
provider="heartbeat">
            <lrm_rsc_op id="ip_mysql_monitor_0" operation="monitor"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="7:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:7;7:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="4" rc-code="7" op-status="0" interval="0" last-run="1290516543"
last-rc-change="1290516543" exec-time="30" queue-time="0"
op-digest="eeb1174061459dd7928cd3e1222390ba"/>
            <lrm_rsc_op id="ip_mysql_start_0" operation="start"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="33:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;33:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="364" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="40" queue-time="0"
op-digest="eeb1174061459dd7928cd3e1222390ba"/>
          </lrm_resource>
        </lrm_resources>
      </lrm>
    </node_state>
  </status>
</cib>

drbd.conf

global { usage-count no; }
common { syncer { rate 100M; } }
resource r0 {
        protocol C;
        startup {
                wfc-timeout  15;
                degr-wfc-timeout 60;
        }
        net {
                cram-hmac-alg sha1;
                shared-secret "capsilon";
        }
        on node-1 {
                device /dev/drbd0;
                disk /dev/sda3;
                address 192.168.42.189:7788;
                meta-disk internal;
        }
        on node-2 {
                device /dev/drbd0;
                disk /dev/sda3;
                address 192.168.42.199:7788;
                meta-disk internal;
        }
}
Also does the disk size(60gb) matter in any way?

If any more info is required please ping me.
Thanks,
HCB

-- 
View this message in context: http://old.nabble.com/Second-time-failover-not-working-tp30283804p30283804.html
Sent from the DRBD - User mailing list archive at Nabble.com.




More information about the drbd-user mailing list