From 914435c74aa92f2b527e2f7432f8afdf6871d71e Mon Sep 17 00:00:00 2001 From: Simon P. Graham Date: Sun, 6 Jan 2008 23:17:15 -0500 Subject: [PATCH] Ensure that state updates cannot be done out of order whilst running receive_state. This fixes a race condition introduced in the change to always schedule after-state processing to worker thread since this can run before or after the receive_state code itself leading to out of order messages. --- drbd/drbd_int.h | 2 + drbd/drbd_main.c | 55 +++++++++++++++++++++++++++++++++++-------------- drbd/drbd_receiver.c | 23 ++++++++++++++------ 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/drbd/drbd_int.h b/drbd/drbd_int.h index d2db85d..40223e9 100644 --- a/drbd/drbd_int.h +++ b/drbd/drbd_int.h @@ -947,9 +947,11 @@ extern void drbd_free_sock(drbd_dev *mdev); extern int drbd_send(drbd_dev *mdev, struct socket *sock, void* buf, size_t size, unsigned msg_flags); extern int drbd_send_protocol(drbd_dev *mdev); +extern int _drbd_send_uuids(drbd_dev *mdev); extern int drbd_send_uuids(drbd_dev *mdev); extern int drbd_send_sync_uuid(drbd_dev *mdev, u64 val); extern int drbd_send_sizes(drbd_dev *mdev); +extern int _drbd_send_state(drbd_dev *mdev); extern int drbd_send_state(drbd_dev *mdev); extern int _drbd_send_cmd(drbd_dev *mdev, struct socket *sock, Drbd_Packet_Cmd cmd, Drbd_Header *h, diff --git a/drbd/drbd_main.c b/drbd/drbd_main.c index 4ff56f7..8e4eb80 100644 --- a/drbd/drbd_main.c +++ b/drbd/drbd_main.c @@ -1347,11 +1347,13 @@ int drbd_send_protocol(drbd_dev *mdev) (Drbd_Header*)&p,sizeof(p)); } -int drbd_send_uuids(drbd_dev *mdev) +/* Hold sock mutex before calling this */ +int _drbd_send_uuids(drbd_dev *mdev) { Drbd_GenCnt_Packet p; - int i; + int i, ok=0; u64 uuid_flags = 0; + struct socket *sock = mdev->data.socket; if(!inc_local_if_state(mdev,Negotiating)) return 1; // ok. @@ -1369,9 +1371,23 @@ int drbd_send_uuids(drbd_dev *mdev) p.uuid[UUID_FLAGS] = cpu_to_be64(uuid_flags); dec_local(mdev); + + if(likely(sock != NULL)) { + ok =_drbd_send_cmd(mdev,sock,ReportUUIDs, + (Drbd_Header*)&p,sizeof(p),0); + } + + return ok; +} - return drbd_send_cmd(mdev,USE_DATA_SOCKET,ReportUUIDs, - (Drbd_Header*)&p,sizeof(p)); +int drbd_send_uuids(drbd_dev *mdev) +{ + int ok; + down(&mdev->data.mutex); + ok = _drbd_send_uuids(mdev); + up(&mdev->data.mutex); + + return ok; } int drbd_send_sync_uuid(drbd_dev *mdev, u64 val) @@ -1415,6 +1431,23 @@ int drbd_send_sizes(drbd_dev *mdev) return ok; } +/* Hold socket mutex before calling this */ +int _drbd_send_state(drbd_dev *mdev) +{ + struct socket *sock = mdev->data.socket; + Drbd_State_Packet p; + int ok = 0; + + p.state = cpu_to_be32(mdev->state.i); + + if(likely(sock != NULL)) { + ok =_drbd_send_cmd(mdev,sock,ReportState, + (Drbd_Header*)&p,sizeof(p),0); + } + + return ok; +} + /** * drbd_send_state: * Informs the peer about our state. Only call it when @@ -1424,20 +1457,10 @@ int drbd_send_sizes(drbd_dev *mdev) */ int drbd_send_state(drbd_dev *mdev) { - struct socket *sock; - Drbd_State_Packet p; - int ok = 0; + int ok; down(&mdev->data.mutex); - - p.state = cpu_to_be32(mdev->state.i); /* Within the send mutex */ - sock = mdev->data.socket; - - if (likely(sock != NULL)) { - ok = _drbd_send_cmd(mdev, sock, ReportState, - (Drbd_Header*)&p, sizeof(p), 0); - } - + ok = _drbd_send_state(mdev); up(&mdev->data.mutex); return ok; diff --git a/drbd/drbd_receiver.c b/drbd/drbd_receiver.c index 98076dd..d4d6312 100644 --- a/drbd/drbd_receiver.c +++ b/drbd/drbd_receiver.c @@ -2431,10 +2431,15 @@ STATIC int receive_state(drbd_dev *mdev, Drbd_Header *h) drbd_conns_t nconn,oconn; drbd_state_t ns,peer_state; int rv; + + /** + * Ensure no other thread sends state whilst we are running + **/ + down(&mdev->data.mutex); - ERR_IF(h->length != (sizeof(*p)-sizeof(*h))) return FALSE; + ERR_IF(h->length != (sizeof(*p)-sizeof(*h))) goto fail; if (drbd_recv(mdev, h->payload, h->length) != h->length) - return FALSE; + goto fail; peer_state.i = be32_to_cpu(p->state); @@ -2459,7 +2464,7 @@ STATIC int receive_state(drbd_dev *mdev, Drbd_Header *h) if (cr) nconn=drbd_sync_handshake(mdev, peer_state.role, peer_state.disk); dec_local(mdev); - if(nconn == conn_mask) return FALSE; + if(nconn == conn_mask) goto fail; } spin_lock_irq(&mdev->req_lock); @@ -2480,18 +2485,18 @@ STATIC int receive_state(drbd_dev *mdev, Drbd_Header *h) if(rv < SS_Success) { drbd_force_state(mdev,NS(conn,Disconnecting)); - return FALSE; + goto fail; } if (oconn > WFReportParams ) { if( nconn > Connected && peer_state.conn <= Connected) { // we want resync, peer has not yet decided to sync... - drbd_send_uuids(mdev); - drbd_send_state(mdev); + _drbd_send_uuids(mdev); + _drbd_send_state(mdev); } else if (nconn == Connected && peer_state.disk == Negotiating) { // peer is waiting for us to respond... - drbd_send_state(mdev); + _drbd_send_state(mdev); } } @@ -2500,7 +2505,11 @@ STATIC int receive_state(drbd_dev *mdev, Drbd_Header *h) /* FIXME assertion for (gencounts do not diverge) */ drbd_md_sync(mdev); // update connected indicator, la_size, ... + up(&mdev->data.mutex); return TRUE; + fail: + up(&mdev->data.mutex); + return FALSE; } STATIC int receive_sync_uuid(drbd_dev *mdev, Drbd_Header *h) -- 1.5.4.rc1