Branch data Line data Source code
1 : : /* Kernel thread helper functions.
2 : : * Copyright (C) 2004 IBM Corporation, Rusty Russell.
3 : : *
4 : : * Creation is done via kthreadd, so that we get a clean environment
5 : : * even if we're invoked from userspace (think modprobe, hotplug cpu,
6 : : * etc.).
7 : : */
8 : : #include <linux/sched.h>
9 : : #include <linux/kthread.h>
10 : : #include <linux/completion.h>
11 : : #include <linux/err.h>
12 : : #include <linux/cpuset.h>
13 : : #include <linux/unistd.h>
14 : : #include <linux/file.h>
15 : : #include <linux/export.h>
16 : : #include <linux/mutex.h>
17 : : #include <linux/slab.h>
18 : : #include <linux/freezer.h>
19 : : #include <linux/ptrace.h>
20 : : #include <linux/uaccess.h>
21 : : #include <trace/events/sched.h>
22 : :
23 : : static DEFINE_SPINLOCK(kthread_create_lock);
24 : : static LIST_HEAD(kthread_create_list);
25 : : struct task_struct *kthreadd_task;
26 : :
27 : : struct kthread_create_info
28 : : {
29 : : /* Information passed to kthread() from kthreadd. */
30 : : int (*threadfn)(void *data);
31 : : void *data;
32 : : int node;
33 : :
34 : : /* Result passed back to kthread_create() from kthreadd. */
35 : : struct task_struct *result;
36 : : struct completion *done;
37 : :
38 : : struct list_head list;
39 : : };
40 : :
41 : : struct kthread {
42 : : unsigned long flags;
43 : : unsigned int cpu;
44 : : void *data;
45 : : struct completion parked;
46 : : struct completion exited;
47 : : };
48 : :
49 : : enum KTHREAD_BITS {
50 : : KTHREAD_IS_PER_CPU = 0,
51 : : KTHREAD_SHOULD_STOP,
52 : : KTHREAD_SHOULD_PARK,
53 : : KTHREAD_IS_PARKED,
54 : : };
55 : :
56 : : #define __to_kthread(vfork) \
57 : : container_of(vfork, struct kthread, exited)
58 : :
59 : : static inline struct kthread *to_kthread(struct task_struct *k)
60 : : {
61 : 4419662 : return __to_kthread(k->vfork_done);
62 : : }
63 : :
64 : : static struct kthread *to_live_kthread(struct task_struct *k)
65 : : {
66 : 496 : struct completion *vfork = ACCESS_ONCE(k->vfork_done);
67 [ + - ][ # # ]: 496 : if (likely(vfork))
[ # # ]
68 : 496 : return __to_kthread(vfork);
69 : : return NULL;
70 : : }
71 : :
72 : : /**
73 : : * kthread_should_stop - should this kthread return now?
74 : : *
75 : : * When someone calls kthread_stop() on your kthread, it will be woken
76 : : * and this will return true. You should then return, and your return
77 : : * value will be passed through to kthread_stop().
78 : : */
79 : 0 : bool kthread_should_stop(void)
80 : : {
81 : 1952690 : return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
82 : : }
83 : : EXPORT_SYMBOL(kthread_should_stop);
84 : :
85 : : /**
86 : : * kthread_should_park - should this kthread park now?
87 : : *
88 : : * When someone calls kthread_park() on your kthread, it will be woken
89 : : * and this will return true. You should then do the necessary
90 : : * cleanup and call kthread_parkme()
91 : : *
92 : : * Similar to kthread_should_stop(), but this keeps the thread alive
93 : : * and in a park position. kthread_unpark() "restarts" the thread and
94 : : * calls the thread function again.
95 : : */
96 : 0 : bool kthread_should_park(void)
97 : : {
98 : 1890370 : return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(current)->flags);
99 : : }
100 : :
101 : : /**
102 : : * kthread_freezable_should_stop - should this freezable kthread return now?
103 : : * @was_frozen: optional out parameter, indicates whether %current was frozen
104 : : *
105 : : * kthread_should_stop() for freezable kthreads, which will enter
106 : : * refrigerator if necessary. This function is safe from kthread_stop() /
107 : : * freezer deadlock and freezable kthreads should use this function instead
108 : : * of calling try_to_freeze() directly.
109 : : */
110 : 0 : bool kthread_freezable_should_stop(bool *was_frozen)
111 : : {
112 : : bool frozen = false;
113 : :
114 : : might_sleep();
115 : :
116 [ # # ]: 0 : if (unlikely(freezing(current)))
117 : 0 : frozen = __refrigerator(true);
118 : :
119 [ # # ]: 0 : if (was_frozen)
120 : 0 : *was_frozen = frozen;
121 : :
122 : 0 : return kthread_should_stop();
123 : : }
124 : : EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);
125 : :
126 : : /**
127 : : * kthread_data - return data value specified on kthread creation
128 : : * @task: kthread task in question
129 : : *
130 : : * Return the data value specified when kthread @task was created.
131 : : * The caller is responsible for ensuring the validity of @task when
132 : : * calling this function.
133 : : */
134 : 0 : void *kthread_data(struct task_struct *task)
135 : : {
136 : 2498132 : return to_kthread(task)->data;
137 : : }
138 : :
139 : : /**
140 : : * probe_kthread_data - speculative version of kthread_data()
141 : : * @task: possible kthread task in question
142 : : *
143 : : * @task could be a kthread task. Return the data value specified when it
144 : : * was created if accessible. If @task isn't a kthread task or its data is
145 : : * inaccessible for any reason, %NULL is returned. This function requires
146 : : * that @task itself is safe to dereference.
147 : : */
148 : 0 : void *probe_kthread_data(struct task_struct *task)
149 : : {
150 : : struct kthread *kthread = to_kthread(task);
151 : 0 : void *data = NULL;
152 : :
153 : 0 : probe_kernel_read(&data, &kthread->data, sizeof(data));
154 : 0 : return data;
155 : : }
156 : :
157 : 0 : static void __kthread_parkme(struct kthread *self)
158 : : {
159 : 495 : __set_current_state(TASK_PARKED);
160 [ - + ]: 495 : while (test_bit(KTHREAD_SHOULD_PARK, &self->flags)) {
161 [ # # ]: 0 : if (!test_and_set_bit(KTHREAD_IS_PARKED, &self->flags))
162 : 0 : complete(&self->parked);
163 : 0 : schedule();
164 : 0 : __set_current_state(TASK_PARKED);
165 : : }
166 : 495 : clear_bit(KTHREAD_IS_PARKED, &self->flags);
167 : 495 : __set_current_state(TASK_RUNNING);
168 : 495 : }
169 : :
170 : 0 : void kthread_parkme(void)
171 : : {
172 : 0 : __kthread_parkme(to_kthread(current));
173 : 0 : }
174 : :
175 : 0 : static int kthread(void *_create)
176 : : {
177 : : /* Copy data: it's on kthread's stack */
178 : : struct kthread_create_info *create = _create;
179 : 495 : int (*threadfn)(void *data) = create->threadfn;
180 : 495 : void *data = create->data;
181 : : struct completion *done;
182 : : struct kthread self;
183 : : int ret;
184 : :
185 : 495 : self.flags = 0;
186 : 495 : self.data = data;
187 : : init_completion(&self.exited);
188 : : init_completion(&self.parked);
189 : 495 : current->vfork_done = &self.exited;
190 : :
191 : : /* If user was SIGKILLed, I release the structure. */
192 : 990 : done = xchg(&create->done, NULL);
193 [ - + ]: 495 : if (!done) {
194 : 0 : kfree(create);
195 : 0 : do_exit(-EINTR);
196 : : }
197 : : /* OK, tell user we're spawned, wait for stop or wakeup */
198 : 495 : __set_current_state(TASK_UNINTERRUPTIBLE);
199 : 495 : create->result = current;
200 : 495 : complete(done);
201 : 495 : schedule();
202 : :
203 : : ret = -EINTR;
204 : :
205 [ + - ]: 495 : if (!test_bit(KTHREAD_SHOULD_STOP, &self.flags)) {
206 : 495 : __kthread_parkme(&self);
207 : 495 : ret = threadfn(data);
208 : : }
209 : : /* we can't just return, we must preserve "self" on stack */
210 : 1 : do_exit(ret);
211 : : }
212 : :
213 : : /* called from do_fork() to get node information for about to be created task */
214 : 0 : int tsk_fork_get_node(struct task_struct *tsk)
215 : : {
216 : : #ifdef CONFIG_NUMA
217 : : if (tsk == kthreadd_task)
218 : : return tsk->pref_node_fork;
219 : : #endif
220 : 1122984 : return numa_node_id();
221 : : }
222 : :
223 : 0 : static void create_kthread(struct kthread_create_info *create)
224 : : {
225 : : int pid;
226 : :
227 : : #ifdef CONFIG_NUMA
228 : : current->pref_node_fork = create->node;
229 : : #endif
230 : : /* We want our own signal handler (we take no signals by default). */
231 : 495 : pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);
232 [ - + ]: 495 : if (pid < 0) {
233 : : /* If user was SIGKILLed, I release the structure. */
234 : 0 : struct completion *done = xchg(&create->done, NULL);
235 : :
236 [ # # ]: 0 : if (!done) {
237 : 0 : kfree(create);
238 : 0 : return;
239 : : }
240 : 0 : create->result = ERR_PTR(pid);
241 : 0 : complete(done);
242 : : }
243 : : }
244 : :
245 : : /**
246 : : * kthread_create_on_node - create a kthread.
247 : : * @threadfn: the function to run until signal_pending(current).
248 : : * @data: data ptr for @threadfn.
249 : : * @node: memory node number.
250 : : * @namefmt: printf-style name for the thread.
251 : : *
252 : : * Description: This helper function creates and names a kernel
253 : : * thread. The thread will be stopped: use wake_up_process() to start
254 : : * it. See also kthread_run().
255 : : *
256 : : * If thread is going to be bound on a particular cpu, give its node
257 : : * in @node, to get NUMA affinity for kthread stack, or else give -1.
258 : : * When woken, the thread will run @threadfn() with @data as its
259 : : * argument. @threadfn() can either call do_exit() directly if it is a
260 : : * standalone thread for which no one will call kthread_stop(), or
261 : : * return when 'kthread_should_stop()' is true (which means
262 : : * kthread_stop() has been called). The return value should be zero
263 : : * or a negative error number; it will be passed to kthread_stop().
264 : : *
265 : : * Returns a task_struct or ERR_PTR(-ENOMEM).
266 : : */
267 : 0 : struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
268 : : void *data, int node,
269 : : const char namefmt[],
270 : : ...)
271 : : {
272 : 495 : DECLARE_COMPLETION_ONSTACK(done);
273 : : struct task_struct *task;
274 : : struct kthread_create_info *create = kmalloc(sizeof(*create),
275 : : GFP_KERNEL);
276 : :
277 [ + - ]: 495 : if (!create)
278 : : return ERR_PTR(-ENOMEM);
279 : 495 : create->threadfn = threadfn;
280 : 495 : create->data = data;
281 : 495 : create->node = node;
282 : 495 : create->done = &done;
283 : :
284 : : spin_lock(&kthread_create_lock);
285 : 495 : list_add_tail(&create->list, &kthread_create_list);
286 : : spin_unlock(&kthread_create_lock);
287 : :
288 : 495 : wake_up_process(kthreadd_task);
289 : : /*
290 : : * Wait for completion in killable state, for I might be chosen by
291 : : * the OOM killer while kthreadd is trying to allocate memory for
292 : : * new kernel thread.
293 : : */
294 [ - + ]: 495 : if (unlikely(wait_for_completion_killable(&done))) {
295 : : /*
296 : : * If I was SIGKILLed before kthreadd (or new kernel thread)
297 : : * calls complete(), leave the cleanup of this structure to
298 : : * that thread.
299 : : */
300 [ # # ]: 0 : if (xchg(&create->done, NULL))
301 : : return ERR_PTR(-ENOMEM);
302 : : /*
303 : : * kthreadd (or new kernel thread) will call complete()
304 : : * shortly.
305 : : */
306 : 0 : wait_for_completion(&done);
307 : : }
308 : 495 : task = create->result;
309 [ + - ]: 495 : if (!IS_ERR(task)) {
310 : : static const struct sched_param param = { .sched_priority = 0 };
311 : : va_list args;
312 : :
313 : 495 : va_start(args, namefmt);
314 : 495 : vsnprintf(task->comm, sizeof(task->comm), namefmt, args);
315 : 495 : va_end(args);
316 : : /*
317 : : * root may have changed our (kthreadd's) priority or CPU mask.
318 : : * The kernel thread should not inherit these properties.
319 : : */
320 : 495 : sched_setscheduler_nocheck(task, SCHED_NORMAL, ¶m);
321 : 495 : set_cpus_allowed_ptr(task, cpu_all_mask);
322 : : }
323 : 495 : kfree(create);
324 : 495 : return task;
325 : : }
326 : : EXPORT_SYMBOL(kthread_create_on_node);
327 : :
328 : 0 : static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state)
329 : : {
330 : : /* Must have done schedule() in kthread() before we set_task_cpu */
331 [ # # ]: 0 : if (!wait_task_inactive(p, state)) {
332 : 0 : WARN_ON(1);
333 : 0 : return;
334 : : }
335 : : /* It's safe because the task is inactive. */
336 : 0 : do_set_cpus_allowed(p, cpumask_of(cpu));
337 : 0 : p->flags |= PF_NO_SETAFFINITY;
338 : : }
339 : :
340 : : /**
341 : : * kthread_bind - bind a just-created kthread to a cpu.
342 : : * @p: thread created by kthread_create().
343 : : * @cpu: cpu (might not be online, must be possible) for @k to run on.
344 : : *
345 : : * Description: This function is equivalent to set_cpus_allowed(),
346 : : * except that @cpu doesn't need to be online, and the thread must be
347 : : * stopped (i.e., just returned from kthread_create()).
348 : : */
349 : 0 : void kthread_bind(struct task_struct *p, unsigned int cpu)
350 : : {
351 : 0 : __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
352 : 0 : }
353 : : EXPORT_SYMBOL(kthread_bind);
354 : :
355 : : /**
356 : : * kthread_create_on_cpu - Create a cpu bound kthread
357 : : * @threadfn: the function to run until signal_pending(current).
358 : : * @data: data ptr for @threadfn.
359 : : * @cpu: The cpu on which the thread should be bound,
360 : : * @namefmt: printf-style name for the thread. Format is restricted
361 : : * to "name.*%u". Code fills in cpu number.
362 : : *
363 : : * Description: This helper function creates and names a kernel thread
364 : : * The thread will be woken and put into park mode.
365 : : */
366 : 0 : struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
367 : : void *data, unsigned int cpu,
368 : : const char *namefmt)
369 : : {
370 : : struct task_struct *p;
371 : :
372 : 0 : p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
373 : : cpu);
374 [ # # ]: 0 : if (IS_ERR(p))
375 : : return p;
376 : 0 : set_bit(KTHREAD_IS_PER_CPU, &to_kthread(p)->flags);
377 : 0 : to_kthread(p)->cpu = cpu;
378 : : /* Park the thread to get it out of TASK_UNINTERRUPTIBLE state */
379 : 0 : kthread_park(p);
380 : 0 : return p;
381 : : }
382 : :
383 : 0 : static void __kthread_unpark(struct task_struct *k, struct kthread *kthread)
384 : : {
385 : 496 : clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
386 : : /*
387 : : * We clear the IS_PARKED bit here as we don't wait
388 : : * until the task has left the park code. So if we'd
389 : : * park before that happens we'd see the IS_PARKED bit
390 : : * which might be about to be cleared.
391 : : */
392 [ - + ]: 496 : if (test_and_clear_bit(KTHREAD_IS_PARKED, &kthread->flags)) {
393 [ # # ]: 0 : if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
394 : 0 : __kthread_bind(k, kthread->cpu, TASK_PARKED);
395 : 0 : wake_up_state(k, TASK_PARKED);
396 : : }
397 : 496 : }
398 : :
399 : : /**
400 : : * kthread_unpark - unpark a thread created by kthread_create().
401 : : * @k: thread created by kthread_create().
402 : : *
403 : : * Sets kthread_should_park() for @k to return false, wakes it, and
404 : : * waits for it to return. If the thread is marked percpu then its
405 : : * bound to the cpu again.
406 : : */
407 : 0 : void kthread_unpark(struct task_struct *k)
408 : : {
409 : : struct kthread *kthread = to_live_kthread(k);
410 : :
411 [ # # ]: 0 : if (kthread)
412 : 0 : __kthread_unpark(k, kthread);
413 : 0 : }
414 : :
415 : : /**
416 : : * kthread_park - park a thread created by kthread_create().
417 : : * @k: thread created by kthread_create().
418 : : *
419 : : * Sets kthread_should_park() for @k to return true, wakes it, and
420 : : * waits for it to return. This can also be called after kthread_create()
421 : : * instead of calling wake_up_process(): the thread will park without
422 : : * calling threadfn().
423 : : *
424 : : * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
425 : : * If called by the kthread itself just the park bit is set.
426 : : */
427 : 0 : int kthread_park(struct task_struct *k)
428 : : {
429 : : struct kthread *kthread = to_live_kthread(k);
430 : : int ret = -ENOSYS;
431 : :
432 [ # # ]: 0 : if (kthread) {
433 [ # # ]: 0 : if (!test_bit(KTHREAD_IS_PARKED, &kthread->flags)) {
434 : 0 : set_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
435 [ # # ]: 0 : if (k != current) {
436 : 0 : wake_up_process(k);
437 : 0 : wait_for_completion(&kthread->parked);
438 : : }
439 : : }
440 : : ret = 0;
441 : : }
442 : 0 : return ret;
443 : : }
444 : :
445 : : /**
446 : : * kthread_stop - stop a thread created by kthread_create().
447 : : * @k: thread created by kthread_create().
448 : : *
449 : : * Sets kthread_should_stop() for @k to return true, wakes it, and
450 : : * waits for it to exit. This can also be called after kthread_create()
451 : : * instead of calling wake_up_process(): the thread will exit without
452 : : * calling threadfn().
453 : : *
454 : : * If threadfn() may call do_exit() itself, the caller must ensure
455 : : * task_struct can't go away.
456 : : *
457 : : * Returns the result of threadfn(), or %-EINTR if wake_up_process()
458 : : * was never called.
459 : : */
460 : 0 : int kthread_stop(struct task_struct *k)
461 : : {
462 : : struct kthread *kthread;
463 : : int ret;
464 : :
465 : : trace_sched_kthread_stop(k);
466 : :
467 : 496 : get_task_struct(k);
468 : : kthread = to_live_kthread(k);
469 [ + - ]: 496 : if (kthread) {
470 : 496 : set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
471 : 496 : __kthread_unpark(k, kthread);
472 : 496 : wake_up_process(k);
473 : 496 : wait_for_completion(&kthread->exited);
474 : : }
475 : 496 : ret = k->exit_code;
476 : : put_task_struct(k);
477 : :
478 : : trace_sched_kthread_stop_ret(ret);
479 : 496 : return ret;
480 : : }
481 : : EXPORT_SYMBOL(kthread_stop);
482 : :
483 : 0 : int kthreadd(void *unused)
484 : : {
485 : 0 : struct task_struct *tsk = current;
486 : :
487 : : /* Setup a clean context for our children to inherit. */
488 : 0 : set_task_comm(tsk, "kthreadd");
489 : 0 : ignore_signals(tsk);
490 : 0 : set_cpus_allowed_ptr(tsk, cpu_all_mask);
491 : : set_mems_allowed(node_states[N_MEMORY]);
492 : :
493 : 0 : current->flags |= PF_NOFREEZE;
494 : :
495 : : for (;;) {
496 : 495 : set_current_state(TASK_INTERRUPTIBLE);
497 [ + - ]: 495 : if (list_empty(&kthread_create_list))
498 : 495 : schedule();
499 : 495 : __set_current_state(TASK_RUNNING);
500 : :
501 : : spin_lock(&kthread_create_lock);
502 [ + + ]: 990 : while (!list_empty(&kthread_create_list)) {
503 : : struct kthread_create_info *create;
504 : :
505 : 495 : create = list_entry(kthread_create_list.next,
506 : : struct kthread_create_info, list);
507 : 495 : list_del_init(&create->list);
508 : : spin_unlock(&kthread_create_lock);
509 : :
510 : 495 : create_kthread(create);
511 : :
512 : : spin_lock(&kthread_create_lock);
513 : : }
514 : : spin_unlock(&kthread_create_lock);
515 : : }
516 : :
517 : : return 0;
518 : : }
519 : :
520 : 0 : void __init_kthread_worker(struct kthread_worker *worker,
521 : : const char *name,
522 : : struct lock_class_key *key)
523 : : {
524 : 0 : spin_lock_init(&worker->lock);
525 : : lockdep_set_class_and_name(&worker->lock, key, name);
526 : 0 : INIT_LIST_HEAD(&worker->work_list);
527 : 0 : worker->task = NULL;
528 : 0 : }
529 : : EXPORT_SYMBOL_GPL(__init_kthread_worker);
530 : :
531 : : /**
532 : : * kthread_worker_fn - kthread function to process kthread_worker
533 : : * @worker_ptr: pointer to initialized kthread_worker
534 : : *
535 : : * This function can be used as @threadfn to kthread_create() or
536 : : * kthread_run() with @worker_ptr argument pointing to an initialized
537 : : * kthread_worker. The started kthread will process work_list until
538 : : * the it is stopped with kthread_stop(). A kthread can also call
539 : : * this function directly after extra initialization.
540 : : *
541 : : * Different kthreads can be used for the same kthread_worker as long
542 : : * as there's only one kthread attached to it at any given time. A
543 : : * kthread_worker without an attached kthread simply collects queued
544 : : * kthread_works.
545 : : */
546 : 0 : int kthread_worker_fn(void *worker_ptr)
547 : : {
548 : : struct kthread_worker *worker = worker_ptr;
549 : : struct kthread_work *work;
550 : :
551 [ # # ]: 0 : WARN_ON(worker->task);
552 : 0 : worker->task = current;
553 : : repeat:
554 : 0 : set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */
555 : :
556 [ # # ]: 0 : if (kthread_should_stop()) {
557 : 0 : __set_current_state(TASK_RUNNING);
558 : : spin_lock_irq(&worker->lock);
559 : 0 : worker->task = NULL;
560 : : spin_unlock_irq(&worker->lock);
561 : 0 : return 0;
562 : : }
563 : :
564 : : work = NULL;
565 : : spin_lock_irq(&worker->lock);
566 [ # # ]: 0 : if (!list_empty(&worker->work_list)) {
567 : : work = list_first_entry(&worker->work_list,
568 : : struct kthread_work, node);
569 : 0 : list_del_init(&work->node);
570 : : }
571 : 0 : worker->current_work = work;
572 : : spin_unlock_irq(&worker->lock);
573 : :
574 [ # # ]: 0 : if (work) {
575 : 0 : __set_current_state(TASK_RUNNING);
576 : 0 : work->func(work);
577 [ # # ]: 0 : } else if (!freezing(current))
578 : 0 : schedule();
579 : :
580 : : try_to_freeze();
581 : : goto repeat;
582 : : }
583 : : EXPORT_SYMBOL_GPL(kthread_worker_fn);
584 : :
585 : : /* insert @work before @pos in @worker */
586 : : static void insert_kthread_work(struct kthread_worker *worker,
587 : : struct kthread_work *work,
588 : : struct list_head *pos)
589 : : {
590 : : lockdep_assert_held(&worker->lock);
591 : :
592 : : list_add_tail(&work->node, pos);
593 : 0 : work->worker = worker;
594 [ # # ][ # # ]: 0 : if (likely(worker->task))
[ # # ]
595 : 0 : wake_up_process(worker->task);
596 : : }
597 : :
598 : : /**
599 : : * queue_kthread_work - queue a kthread_work
600 : : * @worker: target kthread_worker
601 : : * @work: kthread_work to queue
602 : : *
603 : : * Queue @work to work processor @task for async execution. @task
604 : : * must have been created with kthread_worker_create(). Returns %true
605 : : * if @work was successfully queued, %false if it was already pending.
606 : : */
607 : 0 : bool queue_kthread_work(struct kthread_worker *worker,
608 : : struct kthread_work *work)
609 : : {
610 : : bool ret = false;
611 : : unsigned long flags;
612 : :
613 : 0 : spin_lock_irqsave(&worker->lock, flags);
614 [ # # ]: 0 : if (list_empty(&work->node)) {
615 : 0 : insert_kthread_work(worker, work, &worker->work_list);
616 : : ret = true;
617 : : }
618 : : spin_unlock_irqrestore(&worker->lock, flags);
619 : 0 : return ret;
620 : : }
621 : : EXPORT_SYMBOL_GPL(queue_kthread_work);
622 : :
623 : : struct kthread_flush_work {
624 : : struct kthread_work work;
625 : : struct completion done;
626 : : };
627 : :
628 : 0 : static void kthread_flush_work_fn(struct kthread_work *work)
629 : : {
630 : : struct kthread_flush_work *fwork =
631 : : container_of(work, struct kthread_flush_work, work);
632 : 0 : complete(&fwork->done);
633 : 0 : }
634 : :
635 : : /**
636 : : * flush_kthread_work - flush a kthread_work
637 : : * @work: work to flush
638 : : *
639 : : * If @work is queued or executing, wait for it to finish execution.
640 : : */
641 : 0 : void flush_kthread_work(struct kthread_work *work)
642 : : {
643 : 0 : struct kthread_flush_work fwork = {
644 : : KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
645 : 0 : COMPLETION_INITIALIZER_ONSTACK(fwork.done),
646 : : };
647 : : struct kthread_worker *worker;
648 : : bool noop = false;
649 : :
650 : : retry:
651 : 0 : worker = work->worker;
652 [ # # ]: 0 : if (!worker)
653 : 0 : return;
654 : :
655 : : spin_lock_irq(&worker->lock);
656 [ # # ]: 0 : if (work->worker != worker) {
657 : : spin_unlock_irq(&worker->lock);
658 : : goto retry;
659 : : }
660 : :
661 [ # # ]: 0 : if (!list_empty(&work->node))
662 : : insert_kthread_work(worker, &fwork.work, work->node.next);
663 [ # # ]: 0 : else if (worker->current_work == work)
664 : 0 : insert_kthread_work(worker, &fwork.work, worker->work_list.next);
665 : : else
666 : : noop = true;
667 : :
668 : : spin_unlock_irq(&worker->lock);
669 : :
670 [ # # ]: 0 : if (!noop)
671 : 0 : wait_for_completion(&fwork.done);
672 : : }
673 : : EXPORT_SYMBOL_GPL(flush_kthread_work);
674 : :
675 : : /**
676 : : * flush_kthread_worker - flush all current works on a kthread_worker
677 : : * @worker: worker to flush
678 : : *
679 : : * Wait until all currently executing or pending works on @worker are
680 : : * finished.
681 : : */
682 : 0 : void flush_kthread_worker(struct kthread_worker *worker)
683 : : {
684 : 0 : struct kthread_flush_work fwork = {
685 : : KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
686 : 0 : COMPLETION_INITIALIZER_ONSTACK(fwork.done),
687 : : };
688 : :
689 : 0 : queue_kthread_work(worker, &fwork.work);
690 : 0 : wait_for_completion(&fwork.done);
691 : 0 : }
692 : : EXPORT_SYMBOL_GPL(flush_kthread_worker);
|
