Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef LINUX_HARDIRQ_H #define LINUX_HARDIRQ_H #include <linux/context_tracking_state.h> #include <linux/preempt.h> #include <linux/lockdep.h> #include <linux/ftrace_irq.h> #include <linux/sched.h> #include <linux/vtime.h> #include <asm/hardirq.h> extern void synchronize_irq(unsigned int irq); extern bool synchronize_hardirq(unsigned int irq); #ifdef CONFIG_NO_HZ_FULL void __rcu_irq_enter_check_tick(void); #else static inline void __rcu_irq_enter_check_tick(void) { } #endif static __always_inline void rcu_irq_enter_check_tick(void) { if (context_tracking_enabled()) __rcu_irq_enter_check_tick(); } /* * It is safe to do non-atomic ops on ->hardirq_context, * because NMI handlers may not preempt and the ops are * always balanced, so the interrupted value of ->hardirq_context * will always be restored. */ #define __irq_enter() \ do { \ preempt_count_add(HARDIRQ_OFFSET); \ lockdep_hardirq_enter(); \ account_hardirq_enter(current); \ } while (0) /* * Like __irq_enter() without time accounting for fast * interrupts, e.g. reschedule IPI where time accounting * is more expensive than the actual interrupt. */ #define __irq_enter_raw() \ do { \ preempt_count_add(HARDIRQ_OFFSET); \ lockdep_hardirq_enter(); \ } while (0) /* * Enter irq context (on NO_HZ, update jiffies): */ void irq_enter(void); /* * Like irq_enter(), but RCU is already watching. */ void irq_enter_rcu(void); /* * Exit irq context without processing softirqs: */ #define __irq_exit() \ do { \ account_hardirq_exit(current); \ lockdep_hardirq_exit(); \ preempt_count_sub(HARDIRQ_OFFSET); \ } while (0) /* * Like __irq_exit() without time accounting */ #define __irq_exit_raw() \ do { \ lockdep_hardirq_exit(); \ preempt_count_sub(HARDIRQ_OFFSET); \ } while (0) /* * Exit irq context and process softirqs if needed: */ void irq_exit(void); /* * Like irq_exit(), but return with RCU watching. */ void irq_exit_rcu(void); #ifndef arch_nmi_enter #define arch_nmi_enter() do { } while (0) #define arch_nmi_exit() do { } while (0) #endif #ifdef CONFIG_TINY_RCU static inline void rcu_nmi_enter(void) { } static inline void rcu_nmi_exit(void) { } #else extern void rcu_nmi_enter(void); extern void rcu_nmi_exit(void); #endif /* * NMI vs Tracing * -------------- * * We must not land in a tracer until (or after) we've changed preempt_count * such that in_nmi() becomes true. To that effect all NMI C entry points must * be marked 'notrace' and call nmi_enter() as soon as possible. */ /* * nmi_enter() can nest up to 15 times; see NMI_BITS. */ #define __nmi_enter() \ do { \ lockdep_off(); \ arch_nmi_enter(); \ BUG_ON(in_nmi() == NMI_MASK); \ __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ } while (0) #define nmi_enter() \ do { \ __nmi_enter(); \ lockdep_hardirq_enter(); \ rcu_nmi_enter(); \ instrumentation_begin(); \ ftrace_nmi_enter(); \ instrumentation_end(); \ } while (0) #define __nmi_exit() \ do { \ BUG_ON(!in_nmi()); \ __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) #define nmi_exit() \ do { \ instrumentation_begin(); \ ftrace_nmi_exit(); \ instrumentation_end(); \ rcu_nmi_exit(); \ lockdep_hardirq_exit(); \ __nmi_exit(); \ } while (0) #endif /* LINUX_HARDIRQ_H */ |