[DRBD-cvs] drbd by phil; Yet another implementation of the sync_g...
drbd-user@lists.linbit.com
drbd-user@lists.linbit.com
Sun, 18 Jan 2004 11:12:39 +0100 (CET)
DRBD CVS committal
Author : phil
Module : drbd
Dir : drbd/drbd
Modified Files:
Tag: rel-0_7-branch
drbd.h drbd_dsender.c drbd_fs.c drbd_int.h drbd_main.c
drbd_receiver.c
Log Message:
Yet another implementation of the sync_groups functionality.
Unforunately something does not yet work. w_resume_next_sg gets queued,
but the dsender does not process it. stange. need to do further
debugging.
===================================================================
RCS file: /var/lib/cvs/drbd/drbd/drbd/drbd.h,v
retrieving revision 1.34.2.16
retrieving revision 1.34.2.17
diff -u -3 -r1.34.2.16 -r1.34.2.17
--- drbd.h 17 Nov 2003 14:21:38 -0000 1.34.2.16
+++ drbd.h 18 Jan 2004 10:12:38 -0000 1.34.2.17
@@ -150,9 +150,9 @@
SkippedSyncT,
WFBitMapS,
WFBitMapT,
- SyncSource,
- SyncTarget,
- PausedSyncS, // is sync source, but higher priority groups first
+ SyncSource, // The distance between original state and pause
+ SyncTarget, // state must be the same for source and target. (+2)
+ PausedSyncS, // see _drbd_rs_resume() and _drbd_rs_pause()
PausedSyncT, // is sync target, but higher priority groups first
} Drbd_CState;
===================================================================
RCS file: /var/lib/cvs/drbd/drbd/drbd/Attic/drbd_dsender.c,v
retrieving revision 1.1.2.53
retrieving revision 1.1.2.54
diff -u -3 -r1.1.2.53 -r1.1.2.54
--- drbd_dsender.c 18 Jan 2004 07:26:50 -0000 1.1.2.53
+++ drbd_dsender.c 18 Jan 2004 10:12:38 -0000 1.1.2.54
@@ -78,27 +78,9 @@
return 0;
}
-STATIC drbd_dev *ds_find_osg(drbd_dev *mdev)
-{
- drbd_dev *odev;
- int i;
-
- for (i=0; i < minor_count; i++) {
- odev = drbd_conf + i;
- if ( odev->sync_conf.group < mdev->sync_conf.group
- && odev->cstate > Connected ) {
- return odev;
- }
- }
-
- return 0;
-}
-
void resync_timer_fn(unsigned long data)
{
- drbd_dev* mdev;
-
- mdev = (drbd_dev*) data;
+ drbd_dev* mdev = (drbd_dev*) data;
drbd_queue_work(mdev,&mdev->data.work,&mdev->resync_work);
}
@@ -106,34 +88,13 @@
STATIC int w_make_resync_request(drbd_dev* mdev, struct drbd_work* w)
{
struct Pending_read *pr;
- struct Drbd_Conf *odev;
sector_t sector;
int number,i,size;
PARANOIA_BUG_ON(w != &mdev->resync_work);
- // wait_for_other_sync_groups
- odev = ds_find_osg(mdev);
- if (odev) {
- spin_lock_irq(&odev->req_lock);
- if (odev->cstate > Connected) {
- list_add_tail(&w->list,&odev->cstate_hook);
- spin_unlock_irq(&odev->req_lock);
- INFO("Syncer waits for sync group %i\n",
- odev->sync_conf.group);
- drbd_send_short_cmd(mdev,SyncStop);
- set_cstate(mdev,PausedSyncT);
- return 1;
- }
- // state change while we were looking at it. never mind ...
- spin_unlock_irq(&odev->req_lock);
- }
+ if(mdev->cstate < Connected ) return 1; // connection was lost...
- if (mdev->cstate == PausedSyncT) {
- INFO("resumed synchronisation.\n");
- drbd_send_short_cmd(mdev,SyncCont);
- set_cstate(mdev,SyncTarget);
- }
D_ASSERT(mdev->cstate == SyncTarget);
#define SLEEP_TIME (HZ/10)
@@ -157,7 +118,7 @@
if (sector == MBDS_DONE) {
INVALIDATE_MAGIC(pr);
mempool_free(pr,drbd_pr_mempool);
- mdev->resync_work.cb = w_resync_inactive;
+ mdev->resync_work.cb = w_resync_inactive; //TODO ööö
return 1;
}
@@ -197,12 +158,12 @@
drbd_md_write(mdev);
}
mdev->rs_total = 0;
- set_cstate(mdev,Connected);
// assert that all bit-map parts are cleared.
D_ASSERT(list_empty(&mdev->resync->lru));
- w->cb = w_resync_inactive;
- INIT_LIST_HEAD(&w->list);
+ // w->cb = w_resync_inactive; // look into done set_cstate()
+
+ set_cstate(mdev,Connected);
return 1;
}
@@ -240,7 +201,151 @@
return ok;
}
-void drbd_start_resync(struct Drbd_Conf *mdev, Drbd_CState side)
+
+STATIC void drbd_global_lock(void)
+{
+ int i;
+
+ local_irq_disable();
+ for (i=0; i < minor_count; i++) {
+ spin_lock(&drbd_conf[i].req_lock);
+ }
+}
+
+STATIC void drbd_global_unlock(void)
+{
+ int i;
+
+ for (i=0; i < minor_count; i++) {
+ spin_unlock(&drbd_conf[i].req_lock);
+ }
+ local_irq_enable();
+}
+
+STATIC void _drbd_rs_resume(drbd_dev *mdev)
+{
+ Drbd_CState ns;
+
+ ns = mdev->cstate - (PausedSyncS - SyncSource);
+ D_ASSERT(ns == SyncSource || ns == SyncTarget);
+
+ INFO("Syncer continues.\n");
+ _set_cstate(mdev,ns);
+
+ mdev->resync_work.cb = w_make_resync_request;
+ _drbd_dequeue_work(&mdev->data.work,&mdev->resync_work);
+}
+
+
+STATIC void _drbd_rs_pause(drbd_dev *mdev)
+{
+ Drbd_CState ns;
+
+ D_ASSERT(mdev->cstate == SyncSource || mdev->cstate == SyncTarget);
+ ns = mdev->cstate + (PausedSyncS - SyncSource);
+
+ del_timer_sync(&mdev->resync_timer);
+ _drbd_dequeue_work(&mdev->data.work,&mdev->resync_work);
+ mdev->resync_work.cb = w_resync_inactive;
+
+ _set_cstate(mdev,ns);
+ INFO("Syncer waits for sync group.\n");
+}
+
+STATIC int _drbd_pause_higher_sg(drbd_dev *mdev)
+{
+ drbd_dev *odev;
+ int i,rv=0;
+
+ for (i=0; i < minor_count; i++) {
+ odev = drbd_conf + i;
+ if ( odev->sync_conf.group > mdev->sync_conf.group
+ && ( odev->cstate == SyncSource ||
+ odev->cstate == SyncTarget ) ) {
+ _drbd_rs_pause(odev);
+ rv = 1;
+ }
+ }
+
+ return rv;
+}
+
+STATIC int _drbd_lower_sg_running(drbd_dev *mdev)
+{
+ drbd_dev *odev;
+ int i,rv=0;
+
+ for (i=0; i < minor_count; i++) {
+ odev = drbd_conf + i;
+ if ( odev->sync_conf.group < mdev->sync_conf.group
+ && ( odev->cstate == SyncSource ||
+ odev->cstate == SyncTarget ) ) {
+ rv = 1;
+ }
+ }
+
+ return rv;
+}
+
+int w_resume_next_sg(drbd_dev* mdev, struct drbd_work* w)
+{
+ drbd_dev *odev;
+ int i,ng=10000;
+
+ PARANOIA_BUG_ON(w != &mdev->resync_work);
+
+ WARN("w_resume_next_sg() called.\n");
+
+ drbd_global_lock();
+
+ for (i=0; i < minor_count; i++) {
+ odev = drbd_conf + i;
+ if ( odev->sync_conf.group > mdev->sync_conf.group
+ && odev->sync_conf.group < ng ) {
+ ng = odev->sync_conf.group;
+ }
+ }
+
+ WARN("ng = %d\n",ng);
+
+ for (i=0; i < minor_count; i++) {
+ odev = drbd_conf + i;
+ if ( odev->sync_conf.group == ng ) {
+ _drbd_rs_resume(odev);
+ WARN("odev = %p\n",odev);
+ }
+ }
+
+ drbd_global_unlock();
+ w->cb = w_resync_inactive;
+
+ return 1;
+}
+
+void drbd_alter_sg(drbd_dev *mdev, int ng)
+{
+ int c = 0;
+ int d = (ng - mdev->sync_conf.group);
+
+ drbd_global_lock();
+ mdev->sync_conf.group = ng;
+
+ if( ( mdev->cstate == PausedSyncS ||
+ mdev->cstate == PausedSyncT ) && ( d < 0 ) ) {
+ if(_drbd_pause_higher_sg(mdev)) c=1;
+ else if(!_drbd_lower_sg_running(mdev)) c=1;
+ if(c) _drbd_rs_resume(mdev);
+ }
+
+ if( ( mdev->cstate == SyncSource ||
+ mdev->cstate == SyncTarget ) && ( d > 0 ) ) {
+ if(_drbd_lower_sg_running(mdev)) c=1;
+ if(c) _drbd_rs_pause(mdev);
+ }
+ drbd_global_unlock();
+}
+
+void drbd_start_resync(drbd_dev *mdev, Drbd_CState side)
{
set_cstate(mdev,side);
mdev->rs_left=mdev->rs_total;
@@ -271,6 +376,13 @@
}
drbd_md_write(mdev);
+
+ drbd_global_lock();
+ _drbd_pause_higher_sg(mdev);
+ if(_drbd_lower_sg_running(mdev)) {
+ _drbd_rs_pause(mdev);
+ }
+ drbd_global_unlock();
}
int drbd_worker(struct Drbd_thread *thi)
@@ -310,7 +422,7 @@
spin_lock_irq(&mdev->req_lock);
if (!list_empty(&mdev->data.work.q)) {
w = list_entry(mdev->data.work.q.next,struct drbd_work,list);
- list_del(&w->list);
+ list_del_init(&w->list);
}
spin_unlock_irq(&mdev->req_lock);
===================================================================
RCS file: /var/lib/cvs/drbd/drbd/drbd/drbd_fs.c,v
retrieving revision 1.28.2.44
retrieving revision 1.28.2.45
diff -u -3 -r1.28.2.44 -r1.28.2.45
--- drbd_fs.c 15 Jan 2004 14:26:29 -0000 1.28.2.44
+++ drbd_fs.c 18 Jan 2004 10:12:38 -0000 1.28.2.45
@@ -487,6 +487,42 @@
return 0;
}
+STATIC int drbd_ioctl_set_syncer(struct Drbd_Conf *mdev,
+ struct ioctl_syncer_config* arg)
+{
+ struct syncer_config sc;
+
+ if(copy_from_user(&sc,&arg->config,sizeof(sc))) return -EFAULT;
+
+ mdev->sync_conf.rate = sc.rate;
+ mdev->sync_conf.use_csums = sc.use_csums;
+ mdev->sync_conf.skip = sc.skip;
+ mdev->sync_conf.al_extents = sc.al_extents;
+
+ if ( !mdev->act_log ||
+ mdev->act_log->nr_elements != mdev->sync_conf.al_extents ) {
+ struct lru_cache *n,*t;
+ n = lc_alloc(mdev->sync_conf.al_extents,
+ sizeof(struct lc_element), mdev);
+ // FIXME if (n==NULL) scream out loud ...
+ // FIXME if (still_in_use) BUG();
+ spin_lock_irq(&mdev->al_lock);
+ t = mdev->act_log;
+ mdev->act_log = n;
+ spin_unlock_irq(&mdev->al_lock);
+ if (t) lc_free(t);
+ }
+
+ if (mdev->cstate > WFConnection)
+ drbd_send_sync_param(mdev,&sc);
+
+ drbd_alter_sg(mdev, sc.group);
+
+ return 0;
+}
+
+
+
int drbd_ioctl(struct inode *inode, struct file *file,
unsigned int cmd, unsigned long arg)
{
@@ -557,29 +593,8 @@
break;
case DRBD_IOCTL_SET_SYNC_CONFIG:
- // PARANOIA check plausibility of values.
- err = copy_from_user(&drbd_conf[minor].sync_conf,
- &(((struct ioctl_syncer_config*)arg)->config),
- sizeof(struct syncer_config));
- // THINK > WFReportParams? Connected?
- if (mdev->cstate > WFConnection)
- drbd_send_sync_param(mdev);
- // TODO Need to signal dsender() ?
-
- if ( !mdev->act_log ||
- mdev->act_log->nr_elements != mdev->sync_conf.al_extents )
- {
- struct lru_cache *n,*t;
- n = lc_alloc(mdev->sync_conf.al_extents,
- sizeof(struct lc_element), mdev);
- // FIXME if (n==NULL) scream out loud ...
- // FIXME if (still_in_use) BUG();
- spin_lock_irq(&mdev->al_lock);
- t = mdev->act_log;
- mdev->act_log = n;
- spin_unlock_irq(&mdev->al_lock);
- if (t) lc_free(t);
- }
+ err = drbd_ioctl_set_syncer(mdev,
+ (struct ioctl_syncer_config*) arg);
break;
case DRBD_IOCTL_GET_CONFIG:
===================================================================
RCS file: /var/lib/cvs/drbd/drbd/drbd/drbd_int.h,v
retrieving revision 1.58.2.97
retrieving revision 1.58.2.98
diff -u -3 -r1.58.2.97 -r1.58.2.98
--- drbd_int.h 18 Jan 2004 07:26:50 -0000 1.58.2.97
+++ drbd_int.h 18 Jan 2004 10:12:38 -0000 1.58.2.98
@@ -348,8 +348,6 @@
DataRequest, // Used to ask for a data block
RSDataRequest, // Used to ask for a data block
SyncParam,
- SyncStop,
- SyncCont,
MAX_CMD,
MayIgnore = 0x100, // Flag only to test if (cmd > MayIgnore) ...
MAX_OPT_CMD,
@@ -378,8 +376,6 @@
case DataRequest : return "DataRequest";
case RSDataRequest : return "RSDataRequest";
case SyncParam : return "SyncParam";
- case SyncStop : return "SyncStop";
- case SyncCont : return "SyncCont";
case MAX_CMD : return "MAX_CMD";
case MayIgnore : return "MayIgnore";
case MAX_OPT_CMD : return "MAX_OPT_CMD";
@@ -685,7 +681,6 @@
Drbd_State state;
Drbd_CState cstate;
wait_queue_head_t cstate_wait;
- struct list_head cstate_hook; // processed if cstate changes.
wait_queue_head_t state_wait; // TODO: Remove state_wait.
Drbd_State o_state;
unsigned long int la_size; // last agreed disk size
@@ -748,7 +743,7 @@
*************************/
// drbd_main.c
-extern void set_cstate(drbd_dev* mdev,Drbd_CState cs);
+extern void _set_cstate(drbd_dev* mdev,Drbd_CState cs);
extern void drbd_thread_start(struct Drbd_thread *thi);
extern void _drbd_thread_stop(struct Drbd_thread *thi, int restart, int wait);
extern void drbd_free_resources(drbd_dev *mdev);
@@ -762,7 +757,7 @@
extern int drbd_send_param(drbd_dev *mdev);
extern int drbd_send_cmd(drbd_dev *mdev, struct socket *sock,
Drbd_Packet_Cmd cmd, Drbd_Header *h, size_t size);
-extern int drbd_send_sync_param(drbd_dev *mdev);
+extern int drbd_send_sync_param(drbd_dev *mdev, struct syncer_config *sc);
extern int drbd_send_cstate(drbd_dev *mdev);
extern int drbd_send_b_ack(drbd_dev *mdev, u32 barrier_nr,
u32 set_size);
@@ -877,12 +872,14 @@
// drbd_dsender.c
extern int drbd_worker(struct Drbd_thread *thi);
extern void enslaved_read_bh_end_io(struct buffer_head *bh, int uptodate);
+extern void drbd_alter_sg(drbd_dev *mdev, int ng);
extern void drbd_start_resync(drbd_dev *mdev, Drbd_CState side);
// worker callbacks
-extern int w_e_end_data_req (drbd_dev *mdev, struct drbd_work*w);
-extern int w_e_end_rsdata_req (drbd_dev *mdev, struct drbd_work*w);
-extern int w_resync_finished (drbd_dev *mdev, struct drbd_work*w);
-extern int w_resync_inactive (drbd_dev *mdev, struct drbd_work*w);
+extern int w_e_end_data_req (drbd_dev *mdev, struct drbd_work *w);
+extern int w_e_end_rsdata_req (drbd_dev *mdev, struct drbd_work *w);
+extern int w_resync_finished (drbd_dev *mdev, struct drbd_work *w);
+extern int w_resync_inactive (drbd_dev *mdev, struct drbd_work *w);
+extern int w_resume_next_sg (drbd_dev* mdev, struct drbd_work *w);
// drbd_receiver.c
extern int drbd_release_ee(drbd_dev* mdev,struct list_head* list);
@@ -999,11 +996,26 @@
* inline helper functions
*************************/
-static inline void
-_drbd_queue_work(drbd_dev *mdev, struct drbd_work_queue *q,
- struct drbd_work *w)
+static inline void set_cstate(drbd_dev* mdev,Drbd_CState ns)
{
unsigned long flags;
+ spin_lock_irqsave(&mdev->req_lock,flags);
+ _set_cstate(mdev,ns);
+ spin_unlock_irqrestore(&mdev->req_lock,flags);
+}
+
+static inline void
+_drbd_dequeue_work(struct drbd_work_queue *q, struct drbd_work *w)
+{
+ if(!list_empty(&w->list)) {
+ list_del_init(&w->list);
+ down(&q->s); // Should! never sleep.
+ }
+}
+
+static inline void
+_drbd_queue_work(struct drbd_work_queue *q, struct drbd_work *w)
+{
list_add_tail(&w->list,&q->q);
up(&q->s);
}
===================================================================
RCS file: /var/lib/cvs/drbd/drbd/drbd/drbd_main.c,v
retrieving revision 1.73.2.99
retrieving revision 1.73.2.100
diff -u -3 -r1.73.2.99 -r1.73.2.100
--- drbd_main.c 17 Jan 2004 20:17:38 -0000 1.73.2.99
+++ drbd_main.c 18 Jan 2004 10:12:38 -0000 1.73.2.100
@@ -313,27 +313,19 @@
}
#endif
-void set_cstate(drbd_dev* mdev,Drbd_CState cs)
+void _set_cstate(drbd_dev* mdev,Drbd_CState ns)
{
- struct list_head *le;
- struct drbd_work *w;
- struct Drbd_Conf *that_dev;
- unsigned long flags;
+ Drbd_CState os;
- spin_lock_irqsave(&mdev->req_lock,flags);
- mdev->cstate = cs;
+ os = mdev->cstate;
+ mdev->cstate = ns;
+ wake_up_interruptible(&mdev->cstate_wait);
- while(!list_empty(&mdev->cstate_hook)) {
- le = mdev->cstate_hook.next;
- w = list_entry(le,struct drbd_work,list);
- list_del(le);
- that_dev = container_of(w,struct Drbd_Conf,resync_work);
- PARANOIA_BUG_ON(!IS_VALID_MDEV(mdev));
- list_add_tail(&w->list,&that_dev->data.work.q);
+ if ( ( os==SyncSource || os==SyncTarget ) && ns <= Connected ) {
+ WARN("queuing w_resume_next_sg().\n");
+ mdev->resync_work.cb = w_resume_next_sg;
+ _drbd_dequeue_work(&mdev->data.work,&mdev->resync_work);
}
- spin_unlock_irqrestore(&mdev->req_lock,flags);
-
- wake_up_interruptible(&mdev->cstate_wait);
}
STATIC int drbd_thread_setup(void* arg)
@@ -516,20 +508,20 @@
return ok;
}
-int drbd_send_sync_param(drbd_dev *mdev)
+int drbd_send_sync_param(drbd_dev *mdev, struct syncer_config *sc)
{
Drbd_SyncParam_Packet p;
int ok;
- p.rate = cpu_to_be32(mdev->sync_conf.rate);
- p.use_csums = cpu_to_be32(mdev->sync_conf.use_csums);
- p.skip = cpu_to_be32(mdev->sync_conf.skip);
- p.group = cpu_to_be32(mdev->sync_conf.group);
+ p.rate = cpu_to_be32(sc->rate);
+ p.use_csums = cpu_to_be32(sc->use_csums);
+ p.skip = cpu_to_be32(sc->skip);
+ p.group = cpu_to_be32(sc->group);
ok = drbd_send_cmd(mdev,mdev->data.socket,SyncParam,(Drbd_Header*)&p,sizeof(p));
if ( ok
&& (mdev->cstate == SkippedSyncS || mdev->cstate == SkippedSyncT)
- && !mdev->sync_conf.skip )
+ && !sc->skip )
{
set_cstate(mdev,WFReportParams);
ok = drbd_send_param(mdev);
@@ -1029,13 +1021,11 @@
mdev->pr_lock = SPIN_LOCK_UNLOCKED;
mdev->send_task_lock = SPIN_LOCK_UNLOCKED;
- INIT_LIST_HEAD(&mdev->cstate_hook);
INIT_LIST_HEAD(&mdev->free_ee);
INIT_LIST_HEAD(&mdev->active_ee);
INIT_LIST_HEAD(&mdev->sync_ee);
INIT_LIST_HEAD(&mdev->done_ee);
INIT_LIST_HEAD(&mdev->read_ee);
- // INIT_LIST_HEAD(&mdev->rdone_ee);
INIT_LIST_HEAD(&mdev->busy_blocks);
INIT_LIST_HEAD(&mdev->app_reads);
INIT_LIST_HEAD(&mdev->resync_reads);
===================================================================
RCS file: /var/lib/cvs/drbd/drbd/drbd/drbd_receiver.c,v
retrieving revision 1.97.2.87
retrieving revision 1.97.2.88
diff -u -3 -r1.97.2.87 -r1.97.2.88
--- drbd_receiver.c 17 Jan 2004 21:18:38 -0000 1.97.2.87
+++ drbd_receiver.c 18 Jan 2004 10:12:38 -0000 1.97.2.88
@@ -1192,7 +1192,7 @@
mdev->sync_conf.rate = be32_to_cpu(p->rate);
mdev->sync_conf.use_csums = be32_to_cpu(p->use_csums);
mdev->sync_conf.skip = be32_to_cpu(p->skip);
- mdev->sync_conf.group = be32_to_cpu(p->group);
+ drbd_alter_sg(mdev, be32_to_cpu(p->group));
if ( (mdev->cstate == SkippedSyncS || mdev->cstate == SkippedSyncT)
&& !mdev->sync_conf.skip )
@@ -1482,22 +1482,6 @@
return TRUE; // cannot fail, only deadlock :)
}
-STATIC int receive_SyncStop(drbd_dev *mdev, Drbd_Header *h)
-{
- D_ASSERT(mdev->cstate == SyncSource);
- set_cstate(mdev,PausedSyncS);
- INFO("Syncer waits for sync group\n");
- return TRUE; // cannot fail ?
-}
-
-STATIC int receive_SyncCont(drbd_dev *mdev, Drbd_Header *h)
-{
- D_ASSERT(mdev->cstate == PausedSyncS);
- set_cstate(mdev,SyncSource);
- INFO("resumed synchronisation.\n");
- return TRUE; // cannot fail ?
-}
-
typedef int (*drbd_cmd_handler_f)(drbd_dev*,Drbd_Header*);
static drbd_cmd_handler_f drbd_default_handler[] = {
@@ -1519,8 +1503,6 @@
[DataRequest] = receive_DataRequest,
[RSDataRequest] = receive_DataRequest, //receive_RSDataRequest,
[SyncParam] = receive_SyncParam,
- [SyncStop] = receive_SyncStop,
- [SyncCont] = receive_SyncCont,
};
static drbd_cmd_handler_f *drbd_cmd_handler = drbd_default_handler;