Branch data Line data Source code
1 : : #ifndef _LINUX_JUMP_LABEL_H
2 : : #define _LINUX_JUMP_LABEL_H
3 : :
4 : : /*
5 : : * Jump label support
6 : : *
7 : : * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 : : * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
9 : : *
10 : : * Jump labels provide an interface to generate dynamic branches using
11 : : * self-modifying code. Assuming toolchain and architecture support the result
12 : : * of a "if (static_key_false(&key))" statement is a unconditional branch (which
13 : : * defaults to false - and the true block is placed out of line).
14 : : *
15 : : * However at runtime we can change the branch target using
16 : : * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
17 : : * object and for as long as there are references all branches referring to
18 : : * that particular key will point to the (out of line) true block.
19 : : *
20 : : * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
21 : : * must be considered absolute slow paths (machine wide synchronization etc.).
22 : : * OTOH, since the affected branches are unconditional their runtime overhead
23 : : * will be absolutely minimal, esp. in the default (off) case where the total
24 : : * effect is a single NOP of appropriate size. The on case will patch in a jump
25 : : * to the out-of-line block.
26 : : *
27 : : * When the control is directly exposed to userspace it is prudent to delay the
28 : : * decrement to avoid high frequency code modifications which can (and do)
29 : : * cause significant performance degradation. Struct static_key_deferred and
30 : : * static_key_slow_dec_deferred() provide for this.
31 : : *
32 : : * Lacking toolchain and or architecture support, it falls back to a simple
33 : : * conditional branch.
34 : : *
35 : : * struct static_key my_key = STATIC_KEY_INIT_TRUE;
36 : : *
37 : : * if (static_key_true(&my_key)) {
38 : : * }
39 : : *
40 : : * will result in the true case being in-line and starts the key with a single
41 : : * reference. Mixing static_key_true() and static_key_false() on the same key is not
42 : : * allowed.
43 : : *
44 : : * Not initializing the key (static data is initialized to 0s anyway) is the
45 : : * same as using STATIC_KEY_INIT_FALSE.
46 : : *
47 : : */
48 : :
49 : : #include <linux/types.h>
50 : : #include <linux/compiler.h>
51 : : #include <linux/bug.h>
52 : :
53 : : extern bool static_key_initialized;
54 : :
55 : : #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \
56 : : "%s used before call to jump_label_init", \
57 : : __func__)
58 : :
59 : : #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
60 : :
61 : : struct static_key {
62 : : atomic_t enabled;
63 : : /* Set lsb bit to 1 if branch is default true, 0 ot */
64 : : struct jump_entry *entries;
65 : : #ifdef CONFIG_MODULES
66 : : struct static_key_mod *next;
67 : : #endif
68 : : };
69 : :
70 : : # include <asm/jump_label.h>
71 : : # define HAVE_JUMP_LABEL
72 : : #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
73 : :
74 : : enum jump_label_type {
75 : : JUMP_LABEL_DISABLE = 0,
76 : : JUMP_LABEL_ENABLE,
77 : : };
78 : :
79 : : struct module;
80 : :
81 : : #include <linux/atomic.h>
82 : : #ifdef HAVE_JUMP_LABEL
83 : :
84 : : #define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL
85 : : #define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL
86 : : #define JUMP_LABEL_TYPE_MASK 1UL
87 : :
88 : : static
89 : : inline struct jump_entry *jump_label_get_entries(struct static_key *key)
90 : : {
91 : : return (struct jump_entry *)((unsigned long)key->entries
92 : : & ~JUMP_LABEL_TYPE_MASK);
93 : : }
94 : :
95 : : static inline bool jump_label_get_branch_default(struct static_key *key)
96 : : {
97 : : if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
98 : : JUMP_LABEL_TYPE_TRUE_BRANCH)
99 : : return true;
100 : : return false;
101 : : }
102 : :
103 : : static __always_inline bool static_key_false(struct static_key *key)
104 : : {
105 : : return arch_static_branch(key);
106 : : }
107 : :
108 : : static __always_inline bool static_key_true(struct static_key *key)
109 : : {
110 : : return !static_key_false(key);
111 : : }
112 : :
113 : : extern struct jump_entry __start___jump_table[];
114 : : extern struct jump_entry __stop___jump_table[];
115 : :
116 : : extern void jump_label_init(void);
117 : : extern void jump_label_lock(void);
118 : : extern void jump_label_unlock(void);
119 : : extern void arch_jump_label_transform(struct jump_entry *entry,
120 : : enum jump_label_type type);
121 : : extern void arch_jump_label_transform_static(struct jump_entry *entry,
122 : : enum jump_label_type type);
123 : : extern int jump_label_text_reserved(void *start, void *end);
124 : : extern void static_key_slow_inc(struct static_key *key);
125 : : extern void static_key_slow_dec(struct static_key *key);
126 : : extern void jump_label_apply_nops(struct module *mod);
127 : :
128 : : #define STATIC_KEY_INIT_TRUE ((struct static_key) \
129 : : { .enabled = ATOMIC_INIT(1), \
130 : : .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
131 : : #define STATIC_KEY_INIT_FALSE ((struct static_key) \
132 : : { .enabled = ATOMIC_INIT(0), \
133 : : .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
134 : :
135 : : #else /* !HAVE_JUMP_LABEL */
136 : :
137 : : struct static_key {
138 : : atomic_t enabled;
139 : : };
140 : :
141 : : static __always_inline void jump_label_init(void)
142 : : {
143 : 0 : static_key_initialized = true;
144 : : }
145 : :
146 : : static __always_inline bool static_key_false(struct static_key *key)
147 : : {
148 [ + + ][ + + ]: 1645432459 : if (unlikely(atomic_read(&key->enabled) > 0))
[ + - ][ + + ]
[ + + ][ + + ]
[ + + ][ + + ]
[ # # ][ + + ]
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ][ # # ]
[ + - ][ + - ]
[ + - ][ # # ]
[ + - ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ + + ]
[ + + ][ + + ]
[ + + ][ + - ]
[ + + ][ + - ]
[ + # ][ + - ]
[ + - ][ + # ]
[ + # ][ + # ]
[ + + ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ + - ]
[ + - ][ + - ]
[ # # ][ + - ]
[ # # ][ + + ]
[ + + ][ + + ]
[ + + ][ + - ]
[ + ][ # # ]
[ # # ][ + - ]
[ + + ][ + ]
[ + + ][ + ]
[ + - ][ + - ]
[ # # ][ + - ]
[ + - ][ + - ]
[ + - ][ + - ]
[ + ][ + ][ + ]
[ + + ][ + - ]
[ + ][ # # ]
[ + - ][ # # ]
[ + - ][ + ]
[ + - ][ + - ]
[ + - ][ + + ]
[ + - ][ - ]
[ + ][ + - ]
[ # # ][ + # ]
[ + # ][ + # ]
[ + + ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ + - ]
[ # # ][ + - ]
[ + # ][ # ]
[ - ][ - ]
[ # # ][ + ]
[ + - ][ + - ]
[ + ][ + - ]
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ][ + - ]
149 : : return true;
150 : : return false;
151 : : }
152 : :
153 : : static __always_inline bool static_key_true(struct static_key *key)
154 : : {
155 [ - + ][ - + ]: 21821 : if (likely(atomic_read(&key->enabled) > 0))
[ - + ][ - + ]
[ # # ][ # # ]
156 : : return true;
157 : : return false;
158 : : }
159 : :
160 : : static inline void static_key_slow_inc(struct static_key *key)
161 : : {
162 [ # # ][ # # ]: 0 : STATIC_KEY_CHECK_USE();
[ # # ]
163 : 0 : atomic_inc(&key->enabled);
164 : : }
165 : :
166 : : static inline void static_key_slow_dec(struct static_key *key)
167 : : {
168 [ # # ][ # # ]: 0 : STATIC_KEY_CHECK_USE();
[ # # ][ # # ]
169 : 0 : atomic_dec(&key->enabled);
170 : : }
171 : :
172 : : static inline int jump_label_text_reserved(void *start, void *end)
173 : : {
174 : : return 0;
175 : : }
176 : :
177 : : static inline void jump_label_lock(void) {}
178 : : static inline void jump_label_unlock(void) {}
179 : :
180 : : static inline int jump_label_apply_nops(struct module *mod)
181 : : {
182 : : return 0;
183 : : }
184 : :
185 : : #define STATIC_KEY_INIT_TRUE ((struct static_key) \
186 : : { .enabled = ATOMIC_INIT(1) })
187 : : #define STATIC_KEY_INIT_FALSE ((struct static_key) \
188 : : { .enabled = ATOMIC_INIT(0) })
189 : :
190 : : #endif /* HAVE_JUMP_LABEL */
191 : :
192 : : #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
193 : : #define jump_label_enabled static_key_enabled
194 : :
195 : : static inline bool static_key_enabled(struct static_key *key)
196 : : {
197 : 0 : return (atomic_read(&key->enabled) > 0);
198 : : }
199 : :
200 : : #endif /* _LINUX_JUMP_LABEL_H */
|
