summaryrefslogtreecommitdiff
path: root/arch/powerpc/kernel/irq.c
diff options
context:
space:
mode:
authorChristophe Leroy <christophe.leroy@csgroup.eu>2022-05-18 09:40:15 +0200
committerMichael Ellerman <mpe@ellerman.id.au>2022-06-29 16:47:42 +1000
commit7d7b28b302085e1ec2815bc9f5205af28394c5db (patch)
tree2cfb09d29a1fb12d23f6ddce4189cbb6ace8c8e3 /arch/powerpc/kernel/irq.c
parente93dee186fc95f2058b0c9d2317d8b876b8512db (diff)
powerpc/irq: Split irq.c
More than half of irq.c is dedicated to PPC64. Move PPC64 code out of irq.c into irq_64.c Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://lore.kernel.org/r/9f1a47de80f78d3dd270a7a72f69f55f581c4054.1652859593.git.christophe.leroy@csgroup.eu
Diffstat (limited to 'arch/powerpc/kernel/irq.c')
-rw-r--r--arch/powerpc/kernel/irq.c421
1 files changed, 0 insertions, 421 deletions
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 4db27198b002..4abfe6bd6622 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -67,12 +67,6 @@
#include <asm/softirq_stack.h>
#include <asm/ppc_asm.h>
-#ifdef CONFIG_PPC64
-#include <asm/paca.h>
-#include <asm/firmware.h>
-#include <asm/lv1call.h>
-#include <asm/dbell.h>
-#endif
#define CREATE_TRACE_POINTS
#include <asm/trace.h>
#include <asm/cpu_has_feature.h>
@@ -89,411 +83,6 @@ u32 tau_interrupts(unsigned long cpu);
#endif
#endif /* CONFIG_PPC32 */
-#ifdef CONFIG_PPC64
-
-int distribute_irqs = 1;
-
-static inline notrace unsigned long get_irq_happened(void)
-{
- unsigned long happened;
-
- __asm__ __volatile__("lbz %0,%1(13)"
- : "=r" (happened) : "i" (offsetof(struct paca_struct, irq_happened)));
-
- return happened;
-}
-
-void replay_soft_interrupts(void)
-{
- struct pt_regs regs;
-
- /*
- * Be careful here, calling these interrupt handlers can cause
- * softirqs to be raised, which they may run when calling irq_exit,
- * which will cause local_irq_enable() to be run, which can then
- * recurse into this function. Don't keep any state across
- * interrupt handler calls which may change underneath us.
- *
- * We use local_paca rather than get_paca() to avoid all the
- * debug_smp_processor_id() business in this low level function.
- */
-
- ppc_save_regs(&regs);
- regs.softe = IRQS_ENABLED;
- regs.msr |= MSR_EE;
-
-again:
- if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
- WARN_ON_ONCE(mfmsr() & MSR_EE);
-
- /*
- * Force the delivery of pending soft-disabled interrupts on PS3.
- * Any HV call will have this side effect.
- */
- if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {
- u64 tmp, tmp2;
- lv1_get_version_info(&tmp, &tmp2);
- }
-
- /*
- * Check if an hypervisor Maintenance interrupt happened.
- * This is a higher priority interrupt than the others, so
- * replay it first.
- */
- if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_HMI)) {
- local_paca->irq_happened &= ~PACA_IRQ_HMI;
- regs.trap = INTERRUPT_HMI;
- handle_hmi_exception(&regs);
- if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
- hard_irq_disable();
- }
-
- if (local_paca->irq_happened & PACA_IRQ_DEC) {
- local_paca->irq_happened &= ~PACA_IRQ_DEC;
- regs.trap = INTERRUPT_DECREMENTER;
- timer_interrupt(&regs);
- if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
- hard_irq_disable();
- }
-
- if (local_paca->irq_happened & PACA_IRQ_EE) {
- local_paca->irq_happened &= ~PACA_IRQ_EE;
- regs.trap = INTERRUPT_EXTERNAL;
- do_IRQ(&regs);
- if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
- hard_irq_disable();
- }
-
- if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (local_paca->irq_happened & PACA_IRQ_DBELL)) {
- local_paca->irq_happened &= ~PACA_IRQ_DBELL;
- regs.trap = INTERRUPT_DOORBELL;
- doorbell_exception(&regs);
- if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
- hard_irq_disable();
- }
-
- /* Book3E does not support soft-masking PMI interrupts */
- if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_PMI)) {
- local_paca->irq_happened &= ~PACA_IRQ_PMI;
- regs.trap = INTERRUPT_PERFMON;
- performance_monitor_exception(&regs);
- if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
- hard_irq_disable();
- }
-
- if (local_paca->irq_happened & ~PACA_IRQ_HARD_DIS) {
- /*
- * We are responding to the next interrupt, so interrupt-off
- * latencies should be reset here.
- */
- trace_hardirqs_on();
- trace_hardirqs_off();
- goto again;
- }
-}
-
-#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_PPC_KUAP)
-static inline void replay_soft_interrupts_irqrestore(void)
-{
- unsigned long kuap_state = get_kuap();
-
- /*
- * Check if anything calls local_irq_enable/restore() when KUAP is
- * disabled (user access enabled). We handle that case here by saving
- * and re-locking AMR but we shouldn't get here in the first place,
- * hence the warning.
- */
- kuap_assert_locked();
-
- if (kuap_state != AMR_KUAP_BLOCKED)
- set_kuap(AMR_KUAP_BLOCKED);
-
- replay_soft_interrupts();
-
- if (kuap_state != AMR_KUAP_BLOCKED)
- set_kuap(kuap_state);
-}
-#else
-#define replay_soft_interrupts_irqrestore() replay_soft_interrupts()
-#endif
-
-notrace void arch_local_irq_restore(unsigned long mask)
-{
- unsigned char irq_happened;
-
- /* Write the new soft-enabled value if it is a disable */
- if (mask) {
- irq_soft_mask_set(mask);
- return;
- }
-
- if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
- WARN_ON_ONCE(in_nmi() || in_hardirq());
-
- /*
- * After the stb, interrupts are unmasked and there are no interrupts
- * pending replay. The restart sequence makes this atomic with
- * respect to soft-masked interrupts. If this was just a simple code
- * sequence, a soft-masked interrupt could become pending right after
- * the comparison and before the stb.
- *
- * This allows interrupts to be unmasked without hard disabling, and
- * also without new hard interrupts coming in ahead of pending ones.
- */
- asm_volatile_goto(
-"1: \n"
-" lbz 9,%0(13) \n"
-" cmpwi 9,0 \n"
-" bne %l[happened] \n"
-" stb 9,%1(13) \n"
-"2: \n"
- RESTART_TABLE(1b, 2b, 1b)
- : : "i" (offsetof(struct paca_struct, irq_happened)),
- "i" (offsetof(struct paca_struct, irq_soft_mask))
- : "cr0", "r9"
- : happened);
-
- if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
- WARN_ON_ONCE(!(mfmsr() & MSR_EE));
-
- return;
-
-happened:
- irq_happened = get_irq_happened();
- if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
- WARN_ON_ONCE(!irq_happened);
-
- if (irq_happened == PACA_IRQ_HARD_DIS) {
- if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
- WARN_ON_ONCE(mfmsr() & MSR_EE);
- irq_soft_mask_set(IRQS_ENABLED);
- local_paca->irq_happened = 0;
- __hard_irq_enable();
- return;
- }
-
- /* Have interrupts to replay, need to hard disable first */
- if (!(irq_happened & PACA_IRQ_HARD_DIS)) {
- if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
- if (!(mfmsr() & MSR_EE)) {
- /*
- * An interrupt could have come in and cleared
- * MSR[EE] and set IRQ_HARD_DIS, so check
- * IRQ_HARD_DIS again and warn if it is still
- * clear.
- */
- irq_happened = get_irq_happened();
- WARN_ON_ONCE(!(irq_happened & PACA_IRQ_HARD_DIS));
- }
- }
- __hard_irq_disable();
- local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
- } else {
- if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
- if (WARN_ON_ONCE(mfmsr() & MSR_EE))
- __hard_irq_disable();
- }
- }
-
- /*
- * Disable preempt here, so that the below preempt_enable will
- * perform resched if required (a replayed interrupt may set
- * need_resched).
- */
- preempt_disable();
- irq_soft_mask_set(IRQS_ALL_DISABLED);
- trace_hardirqs_off();
-
- replay_soft_interrupts_irqrestore();
- local_paca->irq_happened = 0;
-
- trace_hardirqs_on();
- irq_soft_mask_set(IRQS_ENABLED);
- __hard_irq_enable();
- preempt_enable();
-}
-EXPORT_SYMBOL(arch_local_irq_restore);
-
-/*
- * This is a helper to use when about to go into idle low-power
- * when the latter has the side effect of re-enabling interrupts
- * (such as calling H_CEDE under pHyp).
- *
- * You call this function with interrupts soft-disabled (this is
- * already the case when ppc_md.power_save is called). The function
- * will return whether to enter power save or just return.
- *
- * In the former case, it will have notified lockdep of interrupts
- * being re-enabled and generally sanitized the lazy irq state,
- * and in the latter case it will leave with interrupts hard
- * disabled and marked as such, so the local_irq_enable() call
- * in arch_cpu_idle() will properly re-enable everything.
- */
-bool prep_irq_for_idle(void)
-{
- /*
- * First we need to hard disable to ensure no interrupt
- * occurs before we effectively enter the low power state
- */
- __hard_irq_disable();
- local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
-
- /*
- * If anything happened while we were soft-disabled,
- * we return now and do not enter the low power state.
- */
- if (lazy_irq_pending())
- return false;
-
- /* Tell lockdep we are about to re-enable */
- trace_hardirqs_on();
-
- /*
- * Mark interrupts as soft-enabled and clear the
- * PACA_IRQ_HARD_DIS from the pending mask since we
- * are about to hard enable as well as a side effect
- * of entering the low power state.
- */
- local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
- irq_soft_mask_set(IRQS_ENABLED);
-
- /* Tell the caller to enter the low power state */
- return true;
-}
-
-#ifdef CONFIG_PPC_BOOK3S
-/*
- * This is for idle sequences that return with IRQs off, but the
- * idle state itself wakes on interrupt. Tell the irq tracer that
- * IRQs are enabled for the duration of idle so it does not get long
- * off times. Must be paired with fini_irq_for_idle_irqsoff.
- */
-bool prep_irq_for_idle_irqsoff(void)
-{
- WARN_ON(!irqs_disabled());
-
- /*
- * First we need to hard disable to ensure no interrupt
- * occurs before we effectively enter the low power state
- */
- __hard_irq_disable();
- local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
-
- /*
- * If anything happened while we were soft-disabled,
- * we return now and do not enter the low power state.
- */
- if (lazy_irq_pending())
- return false;
-
- /* Tell lockdep we are about to re-enable */
- trace_hardirqs_on();
-
- return true;
-}
-
-/*
- * Take the SRR1 wakeup reason, index into this table to find the
- * appropriate irq_happened bit.
- *
- * Sytem reset exceptions taken in idle state also come through here,
- * but they are NMI interrupts so do not need to wait for IRQs to be
- * restored, and should be taken as early as practical. These are marked
- * with 0xff in the table. The Power ISA specifies 0100b as the system
- * reset interrupt reason.
- */
-#define IRQ_SYSTEM_RESET 0xff
-
-static const u8 srr1_to_lazyirq[0x10] = {
- 0, 0, 0,
- PACA_IRQ_DBELL,
- IRQ_SYSTEM_RESET,
- PACA_IRQ_DBELL,
- PACA_IRQ_DEC,
- 0,
- PACA_IRQ_EE,
- PACA_IRQ_EE,
- PACA_IRQ_HMI,
- 0, 0, 0, 0, 0 };
-
-void replay_system_reset(void)
-{
- struct pt_regs regs;
-
- ppc_save_regs(&regs);
- regs.trap = 0x100;
- get_paca()->in_nmi = 1;
- system_reset_exception(&regs);
- get_paca()->in_nmi = 0;
-}
-EXPORT_SYMBOL_GPL(replay_system_reset);
-
-void irq_set_pending_from_srr1(unsigned long srr1)
-{
- unsigned int idx = (srr1 & SRR1_WAKEMASK_P8) >> 18;
- u8 reason = srr1_to_lazyirq[idx];
-
- /*
- * Take the system reset now, which is immediately after registers
- * are restored from idle. It's an NMI, so interrupts need not be
- * re-enabled before it is taken.
- */
- if (unlikely(reason == IRQ_SYSTEM_RESET)) {
- replay_system_reset();
- return;
- }
-
- if (reason == PACA_IRQ_DBELL) {
- /*
- * When doorbell triggers a system reset wakeup, the message
- * is not cleared, so if the doorbell interrupt is replayed
- * and the IPI handled, the doorbell interrupt would still
- * fire when EE is enabled.
- *
- * To avoid taking the superfluous doorbell interrupt,
- * execute a msgclr here before the interrupt is replayed.
- */
- ppc_msgclr(PPC_DBELL_MSGTYPE);
- }
-
- /*
- * The 0 index (SRR1[42:45]=b0000) must always evaluate to 0,
- * so this can be called unconditionally with the SRR1 wake
- * reason as returned by the idle code, which uses 0 to mean no
- * interrupt.
- *
- * If a future CPU was to designate this as an interrupt reason,
- * then a new index for no interrupt must be assigned.
- */
- local_paca->irq_happened |= reason;
-}
-#endif /* CONFIG_PPC_BOOK3S */
-
-/*
- * Force a replay of the external interrupt handler on this CPU.
- */
-void force_external_irq_replay(void)
-{
- /*
- * This must only be called with interrupts soft-disabled,
- * the replay will happen when re-enabling.
- */
- WARN_ON(!arch_irqs_disabled());
-
- /*
- * Interrupts must always be hard disabled before irq_happened is
- * modified (to prevent lost update in case of interrupt between
- * load and store).
- */
- __hard_irq_disable();
- local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
-
- /* Indicate in the PACA that we have an interrupt to replay */
- local_paca->irq_happened |= PACA_IRQ_EE;
-}
-
-#endif /* CONFIG_PPC64 */
-
int arch_show_interrupts(struct seq_file *p, int prec)
{
int j;
@@ -795,13 +384,3 @@ int irq_choose_cpu(const struct cpumask *mask)
return hard_smp_processor_id();
}
#endif
-
-#ifdef CONFIG_PPC64
-static int __init setup_noirqdistrib(char *str)
-{
- distribute_irqs = 0;
- return 1;
-}
-
-__setup("noirqdistrib", setup_noirqdistrib);
-#endif /* CONFIG_PPC64 */