Note: "permalinks" may not be as permanent as we would like,
direct links of old sources may well be a few messages off.
Hi,
I have a 2-node cluster that is setup using heartbeat/pacemaker and drbd
used to provide a HA mysql service.
I was running some failover tests to check if it was all working properly
and ran into a slight issue.
When i pull the plug on node 1 i.e. the primary node, the secondary node
becomes active without any issues.
After rebooting node 1 it becomes the secondary node.
This is expected behavior. (node1 became secondary and node2 became primary)
Also /proc/drbd shows that primary and secondary are uptodate
However now when i pull the plug on node 2 i.e now the primary node , node 1
does not failover to become the primary node again. The error that i receive
is that node 1 is outofdate and it refuses to promote to primary without an
uptodate disk. Shouldnt it automatically re-promote node 1 to primary?
/proc/drbd starts fluctuating between
1:
0: cs:WFConnection ro:Secondary/Unknown ds:UpToDate/DUnknown C r----
ns:116 nr:296 dw:528 dr:3408 al:7 bm:22 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b
oos:84
2:
0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C r----
ns:116 nr:296 dw:544 dr:3466 al:7 bm:52 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b
oos:84
Here are the config files:
ha.cf
autojoin none
bcast eth1
warntime 5
deadtime 15
initdead 60
keepalive 2
crm respawn
node node-1
node node-2
crm config file
<cib validate-with="pacemaker-1.0" crm_feature_set="3.0.1" have-quorum="1"
dc-uuid="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c" admin_epoch="0" epoch="124"
num_updates="472">
<configuration>
<crm_config>
<cluster_property_set id="cib-bootstrap-options">
<nvpair id="cib-bootstrap-options-dc-version" name="dc-version"
value="1.0.5-3840e6b5a305ccb803d29b468556739e75532d56"/>
<nvpair id="cib-bootstrap-options-cluster-infrastructure"
name="cluster-infrastructure" value="Heartbeat"/>
<nvpair id="cib-bootstrap-options-stonith-enabled"
name="stonith-enabled" value="false"/>
<nvpair name="last-lrm-refresh"
id="cib-bootstrap-options-last-lrm-refresh" value="1290500431"/>
<nvpair id="cib-bootstrap-options-no-quorum-policy"
name="no-quorum-policy" value="ignore"/>
</cluster_property_set>
</crm_config>
<nodes>
<node id="8b86ee1d-9bea-4b51-b809-286c5be30c15" uname="node-2"
type="normal">
<instance_attributes
id="nodes-8b86ee1d-9bea-4b51-b809-286c5be30c15">
<nvpair id="nodes-8b86ee1d-9bea-4b51-b809-286c5be30c15-standby"
name="standby" value="off"/>
</instance_attributes>
</node>
<node uname="node-1" type="normal"
id="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
<instance_attributes
id="nodes-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
<nvpair name="standby"
id="nodes-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c-standby" value="off"/>
</instance_attributes>
</node>
</nodes>
<resources>
<master id="ms_drbd_mysql">
<meta_attributes id="ms_drbd_mysql-meta_attributes">
<nvpair id="ms_drbd_mysql-meta_attributes-master-max"
name="master-max" value="1"/>
<nvpair id="ms_drbd_mysql-meta_attributes-master-node-max"
name="master-node-max" value="1"/>
<nvpair id="ms_drbd_mysql-meta_attributes-clone-max"
name="clone-max" value="2"/>
<nvpair id="ms_drbd_mysql-meta_attributes-clone-node-max"
name="clone-node-max" value="1"/>
<nvpair id="ms_drbd_mysql-meta_attributes-notify" name="notify"
value="true"/>
<nvpair name="target-role"
id="ms_drbd_mysql-meta_attributes-target-role" value="Started"/>
</meta_attributes>
<primitive class="ocf" id="drbd_mysql" provider="linbit"
type="drbd">
<instance_attributes id="drbd_mysql-instance_attributes">
<nvpair id="drbd_mysql-instance_attributes-drbd_resource"
name="drbd_resource" value="r0"/>
</instance_attributes>
<operations>
<op id="drbd_mysql-monitor-15s" interval="15s" name="monitor"/>
</operations>
</primitive>
</master>
<group id="mysql">
<primitive class="ocf" provider="heartbeat" type="Filesystem"
id="fs_mysql">
<instance_attributes id="fs_mysql-instance_attributes">
<nvpair name="device" id="fs_mysql-instance_attributes-device"
value="/dev/drbd0"/>
<nvpair id="fs_mysql-instance_attributes-directory"
name="directory" value="/mnt/drbd"/>
<nvpair id="fs_mysql-instance_attributes-fstype" name="fstype"
value="ext3"/>
</instance_attributes>
<meta_attributes id="fs_mysql-meta_attributes">
<nvpair name="target-role"
id="fs_mysql-meta_attributes-target-role" value="Started"/>
</meta_attributes>
</primitive>
<primitive class="ocf" id="ip_mysql" provider="heartbeat"
type="IPaddr2">
<instance_attributes id="ip_mysql-instance_attributes">
<nvpair id="ip_mysql-instance_attributes-ip" name="ip"
value="192.168.42.85"/>
<nvpair id="ip_mysql-instance_attributes-nic" name="nic"
value="eth0"/>
</instance_attributes>
</primitive>
<primitive class="lsb" type="mysql" id="mysqld">
<meta_attributes id="mysqld-meta_attributes">
<nvpair name="target-role"
id="mysqld-meta_attributes-target-role" value="Started"/>
</meta_attributes>
</primitive>
<meta_attributes id="mysql-meta_attributes">
<nvpair id="mysql-meta_attributes-is-managed" name="is-managed"
value="true"/>
</meta_attributes>
</group>
</resources>
<constraints>
<rsc_order first="ms_drbd_mysql" first-action="promote"
id="mysql_after_drbd" score="INFINITY" then="mysql" then-action="start"/>
<rsc_colocation id="mysql_on_drbd" rsc="mysql" score="INFINITY"
with-rsc="ms_drbd_mysql" with-rsc-role="Master"/>
</constraints>
<rsc_defaults/>
<op_defaults/>
</configuration>
<status>
<node_state uname="node-2" ha="active" in_ccm="true" crmd="online"
shutdown="0" join="member" expected="member"
id="8b86ee1d-9bea-4b51-b809-286c5be30c15"
crm-debug-origin="do_update_resource">
<lrm id="8b86ee1d-9bea-4b51-b809-286c5be30c15">
<lrm_resources>
<lrm_resource id="fs_mysql" type="Filesystem" class="ocf"
provider="heartbeat">
<lrm_rsc_op id="fs_mysql_monitor_0" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="5:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:7;5:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="3" rc-code="7" op-status="0" interval="0" last-run="1290517413"
last-rc-change="1290517413" exec-time="40" queue-time="0"
op-digest="6e170ae78fb628de2bc66c50e403c073"/>
</lrm_resource>
<lrm_resource id="ip_mysql" type="IPaddr2" class="ocf"
provider="heartbeat">
<lrm_rsc_op id="ip_mysql_monitor_0" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="6:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:7;6:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="4" rc-code="7" op-status="0" interval="0" last-run="1290517413"
last-rc-change="1290517413" exec-time="30" queue-time="0"
op-digest="eeb1174061459dd7928cd3e1222390ba"/>
</lrm_resource>
<lrm_resource id="mysqld" type="mysql" class="lsb">
<lrm_rsc_op id="mysqld_monitor_0" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="7:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:7;7:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="5" rc-code="7" op-status="0" interval="0" last-run="1290517413"
last-rc-change="1290517413" exec-time="30" queue-time="0"
op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
<lrm_resource id="drbd_mysql:0" type="drbd" class="ocf"
provider="linbit">
<lrm_rsc_op id="drbd_mysql:0_monitor_0" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="4:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;4:109:7:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="2" rc-code="0" op-status="0" interval="0" last-run="1290517413"
last-rc-change="1290517413" exec-time="70" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
<lrm_rsc_op id="drbd_mysql:0_monitor_15000" operation="monitor"
crm-debug-origin="do_update_resource" crm_feature_set="3.0.1"
transition-key="8:110:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;8:110:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="6" rc-code="0" op-status="0" interval="15000" last-run="1290517415"
last-rc-change="1290517415" exec-time="60" queue-time="0"
op-digest="db2fb3216fa7207fdc0b2bf31bbd09c1"/>
</lrm_resource>
</lrm_resources>
</lrm>
<transient_attributes id="8b86ee1d-9bea-4b51-b809-286c5be30c15">
<instance_attributes
id="status-8b86ee1d-9bea-4b51-b809-286c5be30c15">
<nvpair
id="status-8b86ee1d-9bea-4b51-b809-286c5be30c15-master-drbd_mysql:0"
name="master-drbd_mysql:0" value="10000"/>
<nvpair
id="status-8b86ee1d-9bea-4b51-b809-286c5be30c15-probe_complete"
name="probe_complete" value="true"/>
</instance_attributes>
</transient_attributes>
</node_state>
<node_state uname="node-1" ha="active" in_ccm="true" crmd="online"
shutdown="0" join="member" expected="member"
id="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c"
crm-debug-origin="do_update_resource">
<transient_attributes id="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
<instance_attributes
id="status-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
<nvpair
id="status-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c-probe_complete"
name="probe_complete" value="true"/>
<nvpair name="master-drbd_mysql:1"
id="status-41dbeeb0-e32e-49c2-b720-b9bc377f0c9c-master-drbd_mysql:1"
value="10000"/>
</instance_attributes>
</transient_attributes>
<lrm id="41dbeeb0-e32e-49c2-b720-b9bc377f0c9c">
<lrm_resources>
<lrm_resource id="drbd_mysql:1" type="drbd" class="ocf"
provider="linbit">
<lrm_rsc_op id="drbd_mysql:1_monitor_0" operation="monitor"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="5:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:7;5:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="2" rc-code="7" op-status="0" interval="0" last-run="1290516543"
last-rc-change="1290516543" exec-time="60" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
<lrm_rsc_op id="drbd_mysql:1_start_0" operation="start"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="13:1:0:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:0;13:1:0:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="6" rc-code="0" op-status="0" interval="0" last-run="1290516545"
last-rc-change="1290516545" exec-time="170" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
<lrm_rsc_op id="drbd_mysql:1_demote_0" operation="demote"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="6:107:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;6:107:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="358" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="200" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
<lrm_rsc_op id="drbd_mysql:1_promote_0" operation="promote"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="7:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;7:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="361" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="70" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
<lrm_rsc_op id="drbd_mysql:1_post_notify_promote_0"
operation="notify" crm-debug-origin="build_active_RAs"
crm_feature_set="3.0.1"
transition-key="53:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;53:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="362" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="60" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
<lrm_rsc_op id="drbd_mysql:1_pre_notify_start_0"
operation="notify" crm-debug-origin="do_update_resource"
crm_feature_set="3.0.1"
transition-key="55:109:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;55:109:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="366" rc-code="0" op-status="0" interval="0" last-run="1290517456"
last-rc-change="1290517456" exec-time="50" queue-time="0"
op-digest="c0e018b73fdf522b6cdd355e125af15e"/>
</lrm_resource>
<lrm_resource id="mysqld" type="mysql" class="lsb">
<lrm_rsc_op id="mysqld_monitor_0" operation="monitor"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="8:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:7;8:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="5" rc-code="7" op-status="0" interval="0" last-run="1290516543"
last-rc-change="1290516543" exec-time="30" queue-time="0"
op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
<lrm_rsc_op id="mysqld_start_0" operation="start"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="34:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;34:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="365" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="1190" queue-time="0"
op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
<lrm_resource id="fs_mysql" type="Filesystem" class="ocf"
provider="heartbeat">
<lrm_rsc_op id="fs_mysql_monitor_0" operation="monitor"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="6:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:7;6:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="3" rc-code="7" op-status="0" interval="0" last-run="1290516543"
last-rc-change="1290516543" exec-time="20" queue-time="0"
op-digest="6e170ae78fb628de2bc66c50e403c073"/>
<lrm_rsc_op id="fs_mysql_stop_0" operation="stop"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="32:99:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;32:99:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="332" rc-code="0" op-status="0" interval="0" last-run="1290517083"
last-rc-change="1290517083" exec-time="50" queue-time="0"
op-digest="6e170ae78fb628de2bc66c50e403c073"/>
<lrm_rsc_op id="fs_mysql_start_0" operation="start"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="32:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;32:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="363" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="100" queue-time="0"
op-digest="6e170ae78fb628de2bc66c50e403c073"/>
</lrm_resource>
<lrm_resource id="ip_mysql" type="IPaddr2" class="ocf"
provider="heartbeat">
<lrm_rsc_op id="ip_mysql_monitor_0" operation="monitor"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="7:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
transition-magic="0:7;7:1:7:1febe546-0b42-40bb-816d-81ebd85fb690"
call-id="4" rc-code="7" op-status="0" interval="0" last-run="1290516543"
last-rc-change="1290516543" exec-time="30" queue-time="0"
op-digest="eeb1174061459dd7928cd3e1222390ba"/>
<lrm_rsc_op id="ip_mysql_start_0" operation="start"
crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1"
transition-key="33:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
transition-magic="0:0;33:108:0:cad1ffc4-9f46-422d-b7e9-e71f179f4ad2"
call-id="364" rc-code="0" op-status="0" interval="0" last-run="1290517085"
last-rc-change="1290517085" exec-time="40" queue-time="0"
op-digest="eeb1174061459dd7928cd3e1222390ba"/>
</lrm_resource>
</lrm_resources>
</lrm>
</node_state>
</status>
</cib>
drbd.conf
global { usage-count no; }
common { syncer { rate 100M; } }
resource r0 {
protocol C;
startup {
wfc-timeout 15;
degr-wfc-timeout 60;
}
net {
cram-hmac-alg sha1;
shared-secret "capsilon";
}
on node-1 {
device /dev/drbd0;
disk /dev/sda3;
address 192.168.42.189:7788;
meta-disk internal;
}
on node-2 {
device /dev/drbd0;
disk /dev/sda3;
address 192.168.42.199:7788;
meta-disk internal;
}
}
Also does the disk size(60gb) matter in any way?
If any more info is required please ping me.
Thanks,
HCB
--
View this message in context: http://old.nabble.com/Second-time-failover-not-working-tp30283804p30283804.html
Sent from the DRBD - User mailing list archive at Nabble.com.