Branch data Line data Source code
1 : : #ifndef _LINUX_JUMP_LABEL_H
2 : : #define _LINUX_JUMP_LABEL_H
3 : :
4 : : /*
5 : : * Jump label support
6 : : *
7 : : * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 : : * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
9 : : *
10 : : * Jump labels provide an interface to generate dynamic branches using
11 : : * self-modifying code. Assuming toolchain and architecture support the result
12 : : * of a "if (static_key_false(&key))" statement is a unconditional branch (which
13 : : * defaults to false - and the true block is placed out of line).
14 : : *
15 : : * However at runtime we can change the branch target using
16 : : * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
17 : : * object and for as long as there are references all branches referring to
18 : : * that particular key will point to the (out of line) true block.
19 : : *
20 : : * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
21 : : * must be considered absolute slow paths (machine wide synchronization etc.).
22 : : * OTOH, since the affected branches are unconditional their runtime overhead
23 : : * will be absolutely minimal, esp. in the default (off) case where the total
24 : : * effect is a single NOP of appropriate size. The on case will patch in a jump
25 : : * to the out-of-line block.
26 : : *
27 : : * When the control is directly exposed to userspace it is prudent to delay the
28 : : * decrement to avoid high frequency code modifications which can (and do)
29 : : * cause significant performance degradation. Struct static_key_deferred and
30 : : * static_key_slow_dec_deferred() provide for this.
31 : : *
32 : : * Lacking toolchain and or architecture support, it falls back to a simple
33 : : * conditional branch.
34 : : *
35 : : * struct static_key my_key = STATIC_KEY_INIT_TRUE;
36 : : *
37 : : * if (static_key_true(&my_key)) {
38 : : * }
39 : : *
40 : : * will result in the true case being in-line and starts the key with a single
41 : : * reference. Mixing static_key_true() and static_key_false() on the same key is not
42 : : * allowed.
43 : : *
44 : : * Not initializing the key (static data is initialized to 0s anyway) is the
45 : : * same as using STATIC_KEY_INIT_FALSE.
46 : : *
47 : : */
48 : :
49 : : #include <linux/types.h>
50 : : #include <linux/compiler.h>
51 : : #include <linux/bug.h>
52 : :
53 : : extern bool static_key_initialized;
54 : :
55 : : #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \
56 : : "%s used before call to jump_label_init", \
57 : : __func__)
58 : :
59 : : #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
60 : :
61 : : struct static_key {
62 : : atomic_t enabled;
63 : : /* Set lsb bit to 1 if branch is default true, 0 ot */
64 : : struct jump_entry *entries;
65 : : #ifdef CONFIG_MODULES
66 : : struct static_key_mod *next;
67 : : #endif
68 : : };
69 : :
70 : : # include <asm/jump_label.h>
71 : : # define HAVE_JUMP_LABEL
72 : : #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
73 : :
74 : : enum jump_label_type {
75 : : JUMP_LABEL_DISABLE = 0,
76 : : JUMP_LABEL_ENABLE,
77 : : };
78 : :
79 : : struct module;
80 : :
81 : : #include <linux/atomic.h>
82 : : #ifdef HAVE_JUMP_LABEL
83 : :
84 : : #define JUMP_LABEL_TRUE_BRANCH 1UL
85 : :
86 : : static
87 : : inline struct jump_entry *jump_label_get_entries(struct static_key *key)
88 : : {
89 : : return (struct jump_entry *)((unsigned long)key->entries
90 : : & ~JUMP_LABEL_TRUE_BRANCH);
91 : : }
92 : :
93 : : static inline bool jump_label_get_branch_default(struct static_key *key)
94 : : {
95 : : if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
96 : : return true;
97 : : return false;
98 : : }
99 : :
100 : : static __always_inline bool static_key_false(struct static_key *key)
101 : : {
102 : : return arch_static_branch(key);
103 : : }
104 : :
105 : : static __always_inline bool static_key_true(struct static_key *key)
106 : : {
107 : : return !static_key_false(key);
108 : : }
109 : :
110 : : extern struct jump_entry __start___jump_table[];
111 : : extern struct jump_entry __stop___jump_table[];
112 : :
113 : : extern void jump_label_init(void);
114 : : extern void jump_label_lock(void);
115 : : extern void jump_label_unlock(void);
116 : : extern void arch_jump_label_transform(struct jump_entry *entry,
117 : : enum jump_label_type type);
118 : : extern void arch_jump_label_transform_static(struct jump_entry *entry,
119 : : enum jump_label_type type);
120 : : extern int jump_label_text_reserved(void *start, void *end);
121 : : extern void static_key_slow_inc(struct static_key *key);
122 : : extern void static_key_slow_dec(struct static_key *key);
123 : : extern void jump_label_apply_nops(struct module *mod);
124 : :
125 : : #define STATIC_KEY_INIT_TRUE ((struct static_key) \
126 : : { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
127 : : #define STATIC_KEY_INIT_FALSE ((struct static_key) \
128 : : { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
129 : :
130 : : #else /* !HAVE_JUMP_LABEL */
131 : :
132 : : struct static_key {
133 : : atomic_t enabled;
134 : : };
135 : :
136 : : static __always_inline void jump_label_init(void)
137 : : {
138 : 0 : static_key_initialized = true;
139 : : }
140 : :
141 : : static __always_inline bool static_key_false(struct static_key *key)
142 : : {
143 [ + ][ + + ]: 1784933431 : if (unlikely(atomic_read(&key->enabled) > 0))
[ + + ][ + + ]
[ + + ][ + + ]
[ + + ][ + + ]
[ # # ][ + + ]
[ + - ][ + - ]
[ + + ][ + - ]
[ + - ][ # # ]
[ + ][ + ]
[ + + ][ + # ]
[ # # ][ + # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ + + ][ + + ]
[ + + ][ + + ]
[ + - ][ + + ]
[ + - ][ + # ]
[ + - ][ + - ]
[ + # ][ + + ]
[ + - ][ + # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ + - ][ + + ]
[ + + ][ + + ]
[ + + ][ + + ]
[ + - ][ + - ]
[ + ][ # # ]
[ # # ][ # # ]
[ + - ][ + - ]
[ + ][ + + ]
[ + - ][ + - ]
[ # # ][ + - ]
[ + ][ # # ]
[ + - ][ + - ]
[ + ][ + ][ + ]
[ + - ][ + - ]
[ + ][ # # ]
[ + - ][ # # ]
[ + - ][ + ]
[ + - ][ + - ]
[ + + ][ + # ]
[ # # ][ # ]
[ # ][ + ]
[ # # ][ + - ]
[ + - ][ + # ]
[ + # ][ # # ]
[ + + ][ + - ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ + - ][ # # ]
[ + - ][ + - ]
[ + ][ + + ]
[ + - ][ # # ]
[ + ][ + - ]
[ + - ][ + - ]
[ + - ][ # # ]
[ + - ][ + - ]
[ + - ][ + - ]
144 : : return true;
145 : : return false;
146 : : }
147 : :
148 : : static __always_inline bool static_key_true(struct static_key *key)
149 : : {
150 : : if (likely(atomic_read(&key->enabled) > 0))
151 : : return true;
152 : : return false;
153 : : }
154 : :
155 : : static inline void static_key_slow_inc(struct static_key *key)
156 : : {
157 [ # # ][ # # ]: 0 : STATIC_KEY_CHECK_USE();
[ # # ]
158 : 0 : atomic_inc(&key->enabled);
159 : : }
160 : :
161 : : static inline void static_key_slow_dec(struct static_key *key)
162 : : {
163 [ # # ][ # # ]: 0 : STATIC_KEY_CHECK_USE();
[ # # ]
164 : 0 : atomic_dec(&key->enabled);
165 : : }
166 : :
167 : : static inline int jump_label_text_reserved(void *start, void *end)
168 : : {
169 : : return 0;
170 : : }
171 : :
172 : : static inline void jump_label_lock(void) {}
173 : : static inline void jump_label_unlock(void) {}
174 : :
175 : : static inline int jump_label_apply_nops(struct module *mod)
176 : : {
177 : : return 0;
178 : : }
179 : :
180 : : #define STATIC_KEY_INIT_TRUE ((struct static_key) \
181 : : { .enabled = ATOMIC_INIT(1) })
182 : : #define STATIC_KEY_INIT_FALSE ((struct static_key) \
183 : : { .enabled = ATOMIC_INIT(0) })
184 : :
185 : : #endif /* HAVE_JUMP_LABEL */
186 : :
187 : : #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
188 : : #define jump_label_enabled static_key_enabled
189 : :
190 : : static inline bool static_key_enabled(struct static_key *key)
191 : : {
192 : 0 : return (atomic_read(&key->enabled) > 0);
193 : : }
194 : :
195 : : #endif /* _LINUX_JUMP_LABEL_H */
|
