1/*
2 * Based on arch/arm/mm/fault.c
3 *
4 * Copyright (C) 1995  Linus Torvalds
5 * Copyright (C) 1995-2004 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#include <linux/module.h>
22#include <linux/signal.h>
23#include <linux/mm.h>
24#include <linux/hardirq.h>
25#include <linux/init.h>
26#include <linux/kprobes.h>
27#include <linux/uaccess.h>
28#include <linux/page-flags.h>
29#include <linux/sched.h>
30#include <linux/highmem.h>
31#include <linux/perf_event.h>
32
33#include <asm/exception.h>
34#include <asm/debug-monitors.h>
35#include <asm/esr.h>
36#include <asm/system_misc.h>
37#include <asm/pgtable.h>
38#include <asm/tlbflush.h>
39
40static const char *fault_name(unsigned int esr);
41
42/*
43 * Dump out the page tables associated with 'addr' in mm 'mm'.
44 */
45void show_pte(struct mm_struct *mm, unsigned long addr)
46{
47	pgd_t *pgd;
48
49	if (!mm)
50		mm = &init_mm;
51
52	pr_alert("pgd = %p\n", mm->pgd);
53	pgd = pgd_offset(mm, addr);
54	pr_alert("[%08lx] *pgd=%016llx", addr, pgd_val(*pgd));
55
56	do {
57		pud_t *pud;
58		pmd_t *pmd;
59		pte_t *pte;
60
61		if (pgd_none(*pgd) || pgd_bad(*pgd))
62			break;
63
64		pud = pud_offset(pgd, addr);
65		printk(", *pud=%016llx", pud_val(*pud));
66		if (pud_none(*pud) || pud_bad(*pud))
67			break;
68
69		pmd = pmd_offset(pud, addr);
70		printk(", *pmd=%016llx", pmd_val(*pmd));
71		if (pmd_none(*pmd) || pmd_bad(*pmd))
72			break;
73
74		pte = pte_offset_map(pmd, addr);
75		printk(", *pte=%016llx", pte_val(*pte));
76		pte_unmap(pte);
77	} while(0);
78
79	printk("\n");
80}
81
82/*
83 * The kernel tried to access some page that wasn't present.
84 */
85static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
86			      unsigned int esr, struct pt_regs *regs)
87{
88	/*
89	 * Are we prepared to handle this kernel fault?
90	 */
91	if (fixup_exception(regs))
92		return;
93
94	/*
95	 * No handler, we'll have to terminate things with extreme prejudice.
96	 */
97	bust_spinlocks(1);
98	pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
99		 (addr < PAGE_SIZE) ? "NULL pointer dereference" :
100		 "paging request", addr);
101
102	show_pte(mm, addr);
103	die("Oops", regs, esr);
104	bust_spinlocks(0);
105	do_exit(SIGKILL);
106}
107
108/*
109 * Something tried to access memory that isn't in our memory map. User mode
110 * accesses just cause a SIGSEGV
111 */
112static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
113			    unsigned int esr, unsigned int sig, int code,
114			    struct pt_regs *regs)
115{
116	struct siginfo si;
117
118	if (show_unhandled_signals && unhandled_signal(tsk, sig) &&
119	    printk_ratelimit()) {
120		pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n",
121			tsk->comm, task_pid_nr(tsk), fault_name(esr), sig,
122			addr, esr);
123		show_pte(tsk->mm, addr);
124		show_regs(regs);
125	}
126
127	tsk->thread.fault_address = addr;
128	tsk->thread.fault_code = esr;
129	si.si_signo = sig;
130	si.si_errno = 0;
131	si.si_code = code;
132	si.si_addr = (void __user *)addr;
133	force_sig_info(sig, &si, tsk);
134}
135
136static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
137{
138	struct task_struct *tsk = current;
139	struct mm_struct *mm = tsk->active_mm;
140
141	/*
142	 * If we are in kernel mode at this point, we have no context to
143	 * handle this fault with.
144	 */
145	if (user_mode(regs))
146		__do_user_fault(tsk, addr, esr, SIGSEGV, SEGV_MAPERR, regs);
147	else
148		__do_kernel_fault(mm, addr, esr, regs);
149}
150
151#define VM_FAULT_BADMAP		0x010000
152#define VM_FAULT_BADACCESS	0x020000
153
154#define ESR_LNX_EXEC		(1 << 24)
155
156static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
157			   unsigned int mm_flags, unsigned long vm_flags,
158			   struct task_struct *tsk)
159{
160	struct vm_area_struct *vma;
161	int fault;
162
163	vma = find_vma(mm, addr);
164	fault = VM_FAULT_BADMAP;
165	if (unlikely(!vma))
166		goto out;
167	if (unlikely(vma->vm_start > addr))
168		goto check_stack;
169
170	/*
171	 * Ok, we have a good vm_area for this memory access, so we can handle
172	 * it.
173	 */
174good_area:
175	/*
176	 * Check that the permissions on the VMA allow for the fault which
177	 * occurred. If we encountered a write or exec fault, we must have
178	 * appropriate permissions, otherwise we allow any permission.
179	 */
180	if (!(vma->vm_flags & vm_flags)) {
181		fault = VM_FAULT_BADACCESS;
182		goto out;
183	}
184
185	return handle_mm_fault(mm, vma, addr & PAGE_MASK, mm_flags);
186
187check_stack:
188	if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
189		goto good_area;
190out:
191	return fault;
192}
193
194static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
195				   struct pt_regs *regs)
196{
197	struct task_struct *tsk;
198	struct mm_struct *mm;
199	int fault, sig, code;
200	unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC;
201	unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
202
203	tsk = current;
204	mm  = tsk->mm;
205
206	/* Enable interrupts if they were enabled in the parent context. */
207	if (interrupts_enabled(regs))
208		local_irq_enable();
209
210	/*
211	 * If we're in an interrupt or have no user context, we must not take
212	 * the fault.
213	 */
214	if (in_atomic() || !mm)
215		goto no_context;
216
217	if (user_mode(regs))
218		mm_flags |= FAULT_FLAG_USER;
219
220	if (esr & ESR_LNX_EXEC) {
221		vm_flags = VM_EXEC;
222	} else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
223		vm_flags = VM_WRITE;
224		mm_flags |= FAULT_FLAG_WRITE;
225	}
226
227	/*
228	 * As per x86, we may deadlock here. However, since the kernel only
229	 * validly references user space from well defined areas of the code,
230	 * we can bug out early if this is from code which shouldn't.
231	 */
232	if (!down_read_trylock(&mm->mmap_sem)) {
233		if (!user_mode(regs) && !search_exception_tables(regs->pc))
234			goto no_context;
235retry:
236		down_read(&mm->mmap_sem);
237	} else {
238		/*
239		 * The above down_read_trylock() might have succeeded in which
240		 * case, we'll have missed the might_sleep() from down_read().
241		 */
242		might_sleep();
243#ifdef CONFIG_DEBUG_VM
244		if (!user_mode(regs) && !search_exception_tables(regs->pc))
245			goto no_context;
246#endif
247	}
248
249	fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
250
251	/*
252	 * If we need to retry but a fatal signal is pending, handle the
253	 * signal first. We do not need to release the mmap_sem because it
254	 * would already be released in __lock_page_or_retry in mm/filemap.c.
255	 */
256	if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
257		return 0;
258
259	/*
260	 * Major/minor page fault accounting is only done on the initial
261	 * attempt. If we go through a retry, it is extremely likely that the
262	 * page will be found in page cache at that point.
263	 */
264
265	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
266	if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
267		if (fault & VM_FAULT_MAJOR) {
268			tsk->maj_flt++;
269			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
270				      addr);
271		} else {
272			tsk->min_flt++;
273			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
274				      addr);
275		}
276		if (fault & VM_FAULT_RETRY) {
277			/*
278			 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
279			 * starvation.
280			 */
281			mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
282			mm_flags |= FAULT_FLAG_TRIED;
283			goto retry;
284		}
285	}
286
287	up_read(&mm->mmap_sem);
288
289	/*
290	 * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
291	 */
292	if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
293			      VM_FAULT_BADACCESS))))
294		return 0;
295
296	/*
297	 * If we are in kernel mode at this point, we have no context to
298	 * handle this fault with.
299	 */
300	if (!user_mode(regs))
301		goto no_context;
302
303	if (fault & VM_FAULT_OOM) {
304		/*
305		 * We ran out of memory, call the OOM killer, and return to
306		 * userspace (which will retry the fault, or kill us if we got
307		 * oom-killed).
308		 */
309		pagefault_out_of_memory();
310		return 0;
311	}
312
313	if (fault & VM_FAULT_SIGBUS) {
314		/*
315		 * We had some memory, but were unable to successfully fix up
316		 * this page fault.
317		 */
318		sig = SIGBUS;
319		code = BUS_ADRERR;
320	} else {
321		/*
322		 * Something tried to access memory that isn't in our memory
323		 * map.
324		 */
325		sig = SIGSEGV;
326		code = fault == VM_FAULT_BADACCESS ?
327			SEGV_ACCERR : SEGV_MAPERR;
328	}
329
330	__do_user_fault(tsk, addr, esr, sig, code, regs);
331	return 0;
332
333no_context:
334	__do_kernel_fault(mm, addr, esr, regs);
335	return 0;
336}
337
338/*
339 * First Level Translation Fault Handler
340 *
341 * We enter here because the first level page table doesn't contain a valid
342 * entry for the address.
343 *
344 * If the address is in kernel space (>= TASK_SIZE), then we are probably
345 * faulting in the vmalloc() area.
346 *
347 * If the init_task's first level page tables contains the relevant entry, we
348 * copy the it to this task.  If not, we send the process a signal, fixup the
349 * exception, or oops the kernel.
350 *
351 * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt
352 * or a critical region, and should only copy the information from the master
353 * page table, nothing more.
354 */
355static int __kprobes do_translation_fault(unsigned long addr,
356					  unsigned int esr,
357					  struct pt_regs *regs)
358{
359	if (addr < TASK_SIZE)
360		return do_page_fault(addr, esr, regs);
361
362	do_bad_area(addr, esr, regs);
363	return 0;
364}
365
366/*
367 * This abort handler always returns "fault".
368 */
369static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
370{
371	return 1;
372}
373
374static struct fault_info {
375	int	(*fn)(unsigned long addr, unsigned int esr, struct pt_regs *regs);
376	int	sig;
377	int	code;
378	const char *name;
379} fault_info[] = {
380	{ do_bad,		SIGBUS,  0,		"ttbr address size fault"	},
381	{ do_bad,		SIGBUS,  0,		"level 1 address size fault"	},
382	{ do_bad,		SIGBUS,  0,		"level 2 address size fault"	},
383	{ do_bad,		SIGBUS,  0,		"level 3 address size fault"	},
384	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 0 translation fault"	},
385	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 1 translation fault"	},
386	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 2 translation fault"	},
387	{ do_page_fault,	SIGSEGV, SEGV_MAPERR,	"level 3 translation fault"	},
388	{ do_bad,		SIGBUS,  0,		"reserved access flag fault"	},
389	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 access flag fault"	},
390	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 access flag fault"	},
391	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 access flag fault"	},
392	{ do_bad,		SIGBUS,  0,		"reserved permission fault"	},
393	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 permission fault"	},
394	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 permission fault"	},
395	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 permission fault"	},
396	{ do_bad,		SIGBUS,  0,		"synchronous external abort"	},
397	{ do_bad,		SIGBUS,  0,		"asynchronous external abort"	},
398	{ do_bad,		SIGBUS,  0,		"unknown 18"			},
399	{ do_bad,		SIGBUS,  0,		"unknown 19"			},
400	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
401	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
402	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
403	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
404	{ do_bad,		SIGBUS,  0,		"synchronous parity error"	},
405	{ do_bad,		SIGBUS,  0,		"asynchronous parity error"	},
406	{ do_bad,		SIGBUS,  0,		"unknown 26"			},
407	{ do_bad,		SIGBUS,  0,		"unknown 27"			},
408	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk" },
409	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk" },
410	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk" },
411	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk" },
412	{ do_bad,		SIGBUS,  0,		"unknown 32"			},
413	{ do_bad,		SIGBUS,  BUS_ADRALN,	"alignment fault"		},
414	{ do_bad,		SIGBUS,  0,		"debug event"			},
415	{ do_bad,		SIGBUS,  0,		"unknown 35"			},
416	{ do_bad,		SIGBUS,  0,		"unknown 36"			},
417	{ do_bad,		SIGBUS,  0,		"unknown 37"			},
418	{ do_bad,		SIGBUS,  0,		"unknown 38"			},
419	{ do_bad,		SIGBUS,  0,		"unknown 39"			},
420	{ do_bad,		SIGBUS,  0,		"unknown 40"			},
421	{ do_bad,		SIGBUS,  0,		"unknown 41"			},
422	{ do_bad,		SIGBUS,  0,		"unknown 42"			},
423	{ do_bad,		SIGBUS,  0,		"unknown 43"			},
424	{ do_bad,		SIGBUS,  0,		"unknown 44"			},
425	{ do_bad,		SIGBUS,  0,		"unknown 45"			},
426	{ do_bad,		SIGBUS,  0,		"unknown 46"			},
427	{ do_bad,		SIGBUS,  0,		"unknown 47"			},
428	{ do_bad,		SIGBUS,  0,		"unknown 48"			},
429	{ do_bad,		SIGBUS,  0,		"unknown 49"			},
430	{ do_bad,		SIGBUS,  0,		"unknown 50"			},
431	{ do_bad,		SIGBUS,  0,		"unknown 51"			},
432	{ do_bad,		SIGBUS,  0,		"implementation fault (lockdown abort)" },
433	{ do_bad,		SIGBUS,  0,		"unknown 53"			},
434	{ do_bad,		SIGBUS,  0,		"unknown 54"			},
435	{ do_bad,		SIGBUS,  0,		"unknown 55"			},
436	{ do_bad,		SIGBUS,  0,		"unknown 56"			},
437	{ do_bad,		SIGBUS,  0,		"unknown 57"			},
438	{ do_bad,		SIGBUS,  0,		"implementation fault (coprocessor abort)" },
439	{ do_bad,		SIGBUS,  0,		"unknown 59"			},
440	{ do_bad,		SIGBUS,  0,		"unknown 60"			},
441	{ do_bad,		SIGBUS,  0,		"unknown 61"			},
442	{ do_bad,		SIGBUS,  0,		"unknown 62"			},
443	{ do_bad,		SIGBUS,  0,		"unknown 63"			},
444};
445
446static const char *fault_name(unsigned int esr)
447{
448	const struct fault_info *inf = fault_info + (esr & 63);
449	return inf->name;
450}
451
452/*
453 * Dispatch a data abort to the relevant handler.
454 */
455asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
456					 struct pt_regs *regs)
457{
458	const struct fault_info *inf = fault_info + (esr & 63);
459	struct siginfo info;
460
461	if (!inf->fn(addr, esr, regs))
462		return;
463
464	pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
465		 inf->name, esr, addr);
466
467	info.si_signo = inf->sig;
468	info.si_errno = 0;
469	info.si_code  = inf->code;
470	info.si_addr  = (void __user *)addr;
471	arm64_notify_die("", regs, &info, esr);
472}
473
474/*
475 * Handle stack alignment exceptions.
476 */
477asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
478					   unsigned int esr,
479					   struct pt_regs *regs)
480{
481	struct siginfo info;
482
483	info.si_signo = SIGBUS;
484	info.si_errno = 0;
485	info.si_code  = BUS_ADRALN;
486	info.si_addr  = (void __user *)addr;
487	arm64_notify_die("", regs, &info, esr);
488}
489
490static struct fault_info debug_fault_info[] = {
491	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware breakpoint"	},
492	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware single-step"	},
493	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware watchpoint"	},
494	{ do_bad,	SIGBUS,		0,		"unknown 3"		},
495	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch32 BKPT"		},
496	{ do_bad,	SIGTRAP,	0,		"aarch32 vector catch"	},
497	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch64 BRK"		},
498	{ do_bad,	SIGBUS,		0,		"unknown 7"		},
499};
500
501void __init hook_debug_fault_code(int nr,
502				  int (*fn)(unsigned long, unsigned int, struct pt_regs *),
503				  int sig, int code, const char *name)
504{
505	BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
506
507	debug_fault_info[nr].fn		= fn;
508	debug_fault_info[nr].sig	= sig;
509	debug_fault_info[nr].code	= code;
510	debug_fault_info[nr].name	= name;
511}
512
513asmlinkage int __exception do_debug_exception(unsigned long addr,
514					      unsigned int esr,
515					      struct pt_regs *regs)
516{
517	const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
518	struct siginfo info;
519
520	if (!inf->fn(addr, esr, regs))
521		return 1;
522
523	pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n",
524		 inf->name, esr, addr);
525
526	info.si_signo = inf->sig;
527	info.si_errno = 0;
528	info.si_code  = inf->code;
529	info.si_addr  = (void __user *)addr;
530	arm64_notify_die("", regs, &info, 0);
531
532	return 0;
533}
534