diff -drupN a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
--- a/arch/arm64/mm/context.c	2018-08-06 17:23:04.000000000 +0300
+++ b/arch/arm64/mm/context.c	2022-06-12 05:28:14.000000000 +0300
@@ -233,7 +233,12 @@ switch_mm_fastpath:
 
 	arm64_apply_bp_hardening();
 
-	cpu_switch_mm(mm->pgd, mm);
+	/*
+	 * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when
+	 * emulating PAN.
+	 */
+	if (!system_uses_ttbr0_pan())
+		cpu_switch_mm(mm->pgd, mm);
 }
 
 /* Errata workaround post TTBRx_EL1 update. */