[Drbd-dev] DRBD on container, lxc

[Ken-ichirou MATSUZAWA]

 Hello Evgeniy, list

I did not know where I should post, here is my starting point. I
started to try DRBD on lxc but did not work. I thought first thing
to do is makeing connector to net namespace aware. Appended patch
may have many issue but I noticed only a few though. 

First this does not add connector sock to struct net, but just
holding sock as a list and send message to all net namespace. I
referred lib/kobject_uevent.c

Next I could not know the plan of relationship sock and work queue,
that is, call cn_queue_alloc_dev() with NULL sock.

Finaly, is this a correct way to run DRBD on lxc?

Would I ask your advice on something?
thanks,


---

 drivers/connector/connector.c |  191 +++++++++++++++++++++++++++++++++++------
 include/linux/connector.h     |    6 ++
 2 files changed, 171 insertions(+), 26 deletions(-)

diff --git a/drivers/connector/connector.c b/drivers/connector/connector.c
index 05117f1..38f4abe 100644
--- a/drivers/connector/connector.c
+++ b/drivers/connector/connector.c
@@ -32,6 +32,7 @@
 #include <linux/spinlock.h>
 
 #include <net/sock.h>
+#include <net/net_namespace.h>
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Evgeniy Polyakov <zbr at ioremap.net>");
@@ -42,6 +43,78 @@ static struct cn_dev cdev;
 
 static int cn_already_initialized;
 
+#ifdef CONFIG_NET
+struct cn_sock {
+	struct list_head list;
+	struct sock *sk;
+};
+static DEFINE_SPINLOCK(cn_sock_lock);
+
+static int send_firm_group(struct cn_msg *msg, u32 group, gfp_t gfp_mask)
+{
+	unsigned int size;
+	struct sk_buff *skb;
+	struct nlmsghdr *nlh;
+	struct cn_msg *data;
+	struct cn_dev *dev = &cdev;
+	struct cn_sock *cn_sk;
+	int err = 0;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(cn_sk, &dev->nls_list, list) {
+		struct sock *nls = cn_sk->sk;
+
+		size = NLMSG_SPACE(sizeof(*msg) + msg->len);
+		skb = alloc_skb(size, gfp_mask);
+		if (!skb) {
+			err =  -ENOMEM;
+			goto nlmsg_failure;
+		}
+
+		nlh = NLMSG_PUT(skb, 0, msg->seq, NLMSG_DONE, size - sizeof(*nlh));
+		data = NLMSG_DATA(nlh);
+		memcpy(data, msg, sizeof(*data) + msg->len);
+		NETLINK_CB(skb).dst_group = group;
+
+		err = netlink_broadcast(nls, skb, 0, group, gfp_mask);
+		if (err && err != -ESRCH)
+			goto nlmsg_failure;
+
+		err = 0;
+	}
+
+nlmsg_failure:
+	rcu_read_unlock();
+
+	return err;
+}
+
+#else /* CONFIG_NET */
+static int send_firm_group(struct cn_msg *msg, u32 group, gfp_t gfp_mask)
+{
+	unsigned int size;
+	struct sk_buff *skb;
+	struct nlmsghdr *nlh;
+	struct cn_msg *data;
+	struct cn_dev *dev = &cdev;
+	
+	if (!netlink_has_listeners(dev->nls, group))
+		return -ESRCH;
+
+	size = NLMSG_SPACE(sizeof(*msg) + msg->len);
+	skb = alloc_skb(size, gfp_mask);
+	if (!skb)
+		return -ENOMEM;
+
+	nlh = NLMSG_PUT(skb, 0, msg->seq, NLMSG_DONE, size - sizeof(*nlh));
+	data = NLMSG_DATA(nlh);
+	memcpy(data, msg, sizeof(*data) + msg->len);
+	NETLINK_CB(skb).dst_group = group;
+
+	return netlink_broadcast(dev->nls, skb, 0, group, gfp_mask);
+}
+#endif /* CONFIG_NET */
+
 /*
  * msg->seq and msg->ack are used to determine message genealogy.
  * When someone sends message it puts there locally unique sequence
@@ -66,10 +139,6 @@ static int cn_already_initialized;
 int cn_netlink_send(struct cn_msg *msg, u32 __group, gfp_t gfp_mask)
 {
 	struct cn_callback_entry *__cbq;
-	unsigned int size;
-	struct sk_buff *skb;
-	struct nlmsghdr *nlh;
-	struct cn_msg *data;
 	struct cn_dev *dev = &cdev;
 	u32 group = 0;
 	int found = 0;
@@ -92,28 +161,7 @@ int cn_netlink_send(struct cn_msg *msg, u32 __group, gfp_t gfp_mask)
 		group = __group;
 	}
 
-	if (!netlink_has_listeners(dev->nls, group))
-		return -ESRCH;
-
-	size = NLMSG_SPACE(sizeof(*msg) + msg->len);
-
-	skb = alloc_skb(size, gfp_mask);
-	if (!skb)
-		return -ENOMEM;
-
-	nlh = NLMSG_PUT(skb, 0, msg->seq, NLMSG_DONE, size - sizeof(*nlh));
-
-	data = NLMSG_DATA(nlh);
-
-	memcpy(data, msg, sizeof(*data) + msg->len);
-
-	NETLINK_CB(skb).dst_group = group;
-
-	return netlink_broadcast(dev->nls, skb, 0, group, gfp_mask);
-
-nlmsg_failure:
-	kfree_skb(skb);
-	return -EINVAL;
+	return send_firm_group(msg, group, gfp_mask);
 }
 EXPORT_SYMBOL_GPL(cn_netlink_send);
 
@@ -272,6 +320,96 @@ static const struct file_operations cn_file_ops = {
 	.release = single_release
 };
 
+#ifdef CONFIG_NET
+static int cn_net_init(struct net *net)
+{
+	struct cn_dev *dev = &cdev;
+	struct cn_sock *cn_sk;
+
+	cn_sk = kzalloc(sizeof(struct cn_sock), GFP_KERNEL);
+	if (!cn_sk)
+		return -ENOMEM;
+
+	cn_sk->sk = netlink_kernel_create(net, NETLINK_CONNECTOR,
+					 CN_NETLINK_USERS + 0xf,
+					 dev->input, NULL, THIS_MODULE);
+	if (!cn_sk->sk) {
+		printk(KERN_ERR
+		       "connector: unable to create netlink socket!\n");
+		kfree(cn_sk);
+		return -EIO;
+	}
+
+	spin_lock(&cn_sock_lock);
+	list_add_tail_rcu(&cn_sk->list, &dev->nls_list);
+	spin_unlock(&cn_sock_lock);
+
+	proc_net_fops_create(net, "connector", S_IRUGO, &cn_file_ops);
+
+ 	return 0;
+}
+
+static void cn_net_exit(struct net *net) {
+	struct cn_dev *dev = &cdev;
+	struct cn_sock *cn_sk;
+	int found = 0;
+
+	spin_lock(&cn_sock_lock);
+	list_for_each_entry(cn_sk, &dev->nls_list, list) {
+		if (sock_net(cn_sk->sk) == net) {
+			list_del_rcu(&cn_sk->list);
+			found = 1;
+			break;
+		}
+	}
+	spin_unlock(&cn_sock_lock);
+
+	if (!found)
+		return;
+
+	proc_net_remove(net, "connector");
+	netlink_kernel_release(cn_sk->sk);
+	kfree(cn_sk);
+}
+
+static struct pernet_operations cn_net_ops = {
+	.init = cn_net_init,
+	.exit = cn_net_exit,
+};
+
+static int __devinit cn_init(void)
+{
+	int err = 0;
+	struct cn_dev *dev = &cdev;
+
+	INIT_LIST_HEAD(&dev->nls_list);
+	dev->input = cn_rx_skb;
+
+	dev->cbdev = cn_queue_alloc_dev("cqueue", NULL);
+	if (!dev->cbdev)
+		return -EINVAL;
+
+	err = register_pernet_subsys(&cn_net_ops);
+	if (err) {
+		cn_queue_free_dev(dev->cbdev);
+		return err;
+	}
+
+	cn_already_initialized = 1;
+	
+	return 0;
+}
+
+static void __devexit cn_fini(void)
+{
+	struct cn_dev *dev = &cdev;
+
+	cn_already_initialized = 0;
+	unregister_pernet_subsys(&cn_net_ops);
+	cn_queue_free_dev(dev->cbdev);
+}
+	
+#else /* CONFIG_NET */
 static int __devinit cn_init(void)
 {
 	struct cn_dev *dev = &cdev;
@@ -308,6 +446,7 @@ static void __devexit cn_fini(void)
 	cn_queue_free_dev(dev->cbdev);
 	netlink_kernel_release(dev->nls);
 }
+#endif /* CONFIG_NET */
 
 subsys_initcall(cn_init);
 module_exit(cn_fini);
diff --git a/include/linux/connector.h b/include/linux/connector.h
index 7e8ca75..e0b9f4c 100644
--- a/include/linux/connector.h
+++ b/include/linux/connector.h
@@ -122,7 +122,13 @@ struct cn_dev {
 	struct cb_id id;
 
 	u32 seq, groups;
+
+#ifdef CONFIG_NET
+	struct list_head nls_list;
+#else /* CONFIG_NET */
 	struct sock *nls;
+#endif /* CONFIG_NET */
+
 	void (*input) (struct sk_buff *skb);
 
 	struct cn_queue_dev *cbdev;
-- 
1.7.2.5