1 Subject: [PATCH 01/03] block: prepare I/O context code for BFQ
3 BFQ uses struct cfq_io_context to store its per-process per-device data,
4 reusing the same code for cic handling of CFQ. The code is not shared
5 ATM to minimize the impact of these patches.
7 This patch introduces a new hlist to each io_context to store all the
8 cic's allocated by BFQ to allow calling the right destructor on module
9 unload; the radix tree used for cic lookup needs to be duplicated
10 because it can contain dead keys inserted by a scheduler and later
11 retrieved by the other one.
13 Update the io_context exit and free paths to take care also of
16 Change the type of cfqq inside struct cfq_io_context to void *
17 to use it also for BFQ per-queue data.
19 A new bfq-specific ioprio_changed field is necessary, too, to avoid
20 clobbering cfq's one, so switch ioprio_changed to a bitmap, with one
21 element per scheduler.
23 Signed-off-by: Fabio Checconi <fabio@gandalf.sssup.it>
24 Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
26 block/blk-ioc.c | 27 ++++++++++++++++-----------
27 block/cfq-iosched.c | 10 +++++++---
28 fs/ioprio.c | 9 +++++++--
29 include/linux/iocontext.h | 18 +++++++++++++++---
30 4 files changed, 45 insertions(+), 19 deletions(-)
32 diff --git a/block/blk-ioc.c b/block/blk-ioc.c
33 index 012f065..c18bb82 100644
37 #include <linux/module.h>
38 #include <linux/init.h>
39 #include <linux/bio.h>
40 +#include <linux/bitmap.h>
41 #include <linux/blkdev.h>
42 #include <linux/bootmem.h> /* for max_pfn/max_low_pfn */
46 static struct kmem_cache *iocontext_cachep;
48 -static void cfq_dtor(struct io_context *ioc)
49 +static void hlist_sched_dtor(struct io_context *ioc, struct hlist_head *list)
51 - if (!hlist_empty(&ioc->cic_list)) {
52 + if (!hlist_empty(list)) {
53 struct cfq_io_context *cic;
55 - cic = list_entry(ioc->cic_list.first, struct cfq_io_context,
57 + cic = list_entry(list->first, struct cfq_io_context, cic_list);
61 @@ -41,7 +41,9 @@ int put_io_context(struct io_context *ioc)
63 if (ioc->aic && ioc->aic->dtor)
64 ioc->aic->dtor(ioc->aic);
67 + hlist_sched_dtor(ioc, &ioc->cic_list);
68 + hlist_sched_dtor(ioc, &ioc->bfq_cic_list);
71 kmem_cache_free(iocontext_cachep, ioc);
72 @@ -51,18 +53,18 @@ int put_io_context(struct io_context *ioc)
74 EXPORT_SYMBOL(put_io_context);
76 -static void cfq_exit(struct io_context *ioc)
77 +static void hlist_sched_exit(struct io_context *ioc, struct hlist_head *list)
81 - if (!hlist_empty(&ioc->cic_list)) {
82 + if (!hlist_empty(list)) {
83 struct cfq_io_context *cic;
85 - cic = list_entry(ioc->cic_list.first, struct cfq_io_context,
87 + cic = list_entry(list->first, struct cfq_io_context, cic_list);
94 /* Called by the exitting task */
95 @@ -78,7 +80,8 @@ void exit_io_context(void)
96 if (atomic_dec_and_test(&ioc->nr_tasks)) {
97 if (ioc->aic && ioc->aic->exit)
98 ioc->aic->exit(ioc->aic);
100 + hlist_sched_exit(ioc, &ioc->cic_list);
101 + hlist_sched_exit(ioc, &ioc->bfq_cic_list);
105 @@ -93,13 +96,15 @@ struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
106 atomic_set(&ret->refcount, 1);
107 atomic_set(&ret->nr_tasks, 1);
108 spin_lock_init(&ret->lock);
109 - ret->ioprio_changed = 0;
110 + bitmap_zero(ret->ioprio_changed, IOC_IOPRIO_CHANGED_BITS);
112 ret->last_waited = jiffies; /* doesn't matter... */
113 ret->nr_batch_requests = 0; /* because this is 0 */
115 INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH);
116 INIT_HLIST_HEAD(&ret->cic_list);
117 + INIT_RADIX_TREE(&ret->bfq_radix_root, GFP_ATOMIC | __GFP_HIGH);
118 + INIT_HLIST_HEAD(&ret->bfq_cic_list);
119 ret->ioc_data = NULL;
122 diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
123 index 1e2aff8..ab503ec 100644
124 --- a/block/cfq-iosched.c
125 +++ b/block/cfq-iosched.c
126 @@ -1417,7 +1417,6 @@ static void changed_ioprio(struct io_context *ioc, struct cfq_io_context *cic)
127 static void cfq_ioc_set_ioprio(struct io_context *ioc)
129 call_for_each_cic(ioc, changed_ioprio);
130 - ioc->ioprio_changed = 0;
133 static struct cfq_queue *
134 @@ -1664,8 +1663,13 @@ cfq_get_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
138 - smp_read_barrier_depends();
139 - if (unlikely(ioc->ioprio_changed))
141 + * test_and_clear_bit() implies a memory barrier, paired with
142 + * the wmb() in fs/ioprio.c, so the value seen for ioprio is the
145 + if (unlikely(test_and_clear_bit(IOC_CFQ_IOPRIO_CHANGED,
146 + ioc->ioprio_changed)))
147 cfq_ioc_set_ioprio(ioc);
150 diff --git a/fs/ioprio.c b/fs/ioprio.c
151 index da3cc46..9296ac3 100644
156 static int set_task_ioprio(struct task_struct *task, int ioprio)
160 struct io_context *ioc;
162 if (task->uid != current->euid &&
163 @@ -53,12 +53,17 @@ static int set_task_ioprio(struct task_struct *task, int ioprio)
167 + /* let other ioc users see the new values */
169 task->io_context = ioc;
173 ioc->ioprio = ioprio;
174 - ioc->ioprio_changed = 1;
175 + /* make sure schedulers see the new ioprio value */
177 + for (i = 0; i < IOC_IOPRIO_CHANGED_BITS; i++)
178 + set_bit(i, ioc->ioprio_changed);
182 diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h
183 index 08b987b..335bbb9 100644
184 --- a/include/linux/iocontext.h
185 +++ b/include/linux/iocontext.h
190 +#include <linux/bitmap.h>
191 #include <linux/radix-tree.h>
192 #include <linux/rcupdate.h>
194 @@ -30,12 +31,11 @@ struct as_io_context {
199 struct cfq_io_context {
201 unsigned long dead_key;
203 - struct cfq_queue *cfqq[2];
206 struct io_context *ioc;
208 @@ -60,6 +60,16 @@ struct cfq_io_context {
212 + * Indexes into the ioprio_changed bitmap. A bit set indicates that
213 + * the corresponding I/O scheduler needs to see a ioprio update.
216 + IOC_CFQ_IOPRIO_CHANGED,
217 + IOC_BFQ_IOPRIO_CHANGED,
218 + IOC_IOPRIO_CHANGED_BITS
222 * I/O subsystem state of the associated processes. It is refcounted
223 * and kmalloc'ed. These could be shared between processes.
225 @@ -71,7 +81,7 @@ struct io_context {
228 unsigned short ioprio;
229 - unsigned short ioprio_changed;
230 + DECLARE_BITMAP(ioprio_changed, IOC_IOPRIO_CHANGED_BITS);
233 * For request batching
234 @@ -82,6 +92,8 @@ struct io_context {
235 struct as_io_context *aic;
236 struct radix_tree_root radix_root;
237 struct hlist_head cic_list;
238 + struct radix_tree_root bfq_radix_root;
239 + struct hlist_head bfq_cic_list;