65 struct uprobe {  struct
90 	struct uprobe		*uprobe;  argument
369 static int match_uprobe(struct uprobe *l, struct uprobe *r)  in match_uprobe()
386 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)  in __find_uprobe()
388 	struct uprobe u = { .inode = inode, .offset = offset };  in __find_uprobe()
390 	struct uprobe *uprobe;  in __find_uprobe()  local
394 		uprobe = rb_entry(n, struct uprobe, rb_node);  in __find_uprobe()
395 		match = match_uprobe(&u, uprobe);  in __find_uprobe()
397 			atomic_inc(&uprobe->ref);  in __find_uprobe()
398 			return uprobe;  in __find_uprobe()
413 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)  in find_uprobe()
415 	struct uprobe *uprobe;  in find_uprobe()  local
418 	uprobe = __find_uprobe(inode, offset);  in find_uprobe()
421 	return uprobe;  in find_uprobe()
424 static struct uprobe *__insert_uprobe(struct uprobe *uprobe)  in __insert_uprobe()  argument
428 	struct uprobe *u;  in __insert_uprobe()
433 		u = rb_entry(parent, struct uprobe, rb_node);  in __insert_uprobe()
434 		match = match_uprobe(uprobe, u);  in __insert_uprobe()
448 	rb_link_node(&uprobe->rb_node, parent, p);  in __insert_uprobe()
449 	rb_insert_color(&uprobe->rb_node, &uprobes_tree);  in __insert_uprobe()
451 	atomic_set(&uprobe->ref, 2);  in __insert_uprobe()
464 static struct uprobe *insert_uprobe(struct uprobe *uprobe)  in insert_uprobe()  argument
466 	struct uprobe *u;  in insert_uprobe()
469 	u = __insert_uprobe(uprobe);  in insert_uprobe()
475 static void put_uprobe(struct uprobe *uprobe)  in put_uprobe()  argument
477 	if (atomic_dec_and_test(&uprobe->ref))  in put_uprobe()
478 		kfree(uprobe);  in put_uprobe()
481 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)  in alloc_uprobe()
483 	struct uprobe *uprobe, *cur_uprobe;  in alloc_uprobe()  local
485 	uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);  in alloc_uprobe()
486 	if (!uprobe)  in alloc_uprobe()
489 	uprobe->inode = igrab(inode);  in alloc_uprobe()
490 	uprobe->offset = offset;  in alloc_uprobe()
491 	init_rwsem(&uprobe->register_rwsem);  in alloc_uprobe()
492 	init_rwsem(&uprobe->consumer_rwsem);  in alloc_uprobe()
495 	cur_uprobe = insert_uprobe(uprobe);  in alloc_uprobe()
498 		kfree(uprobe);  in alloc_uprobe()
499 		uprobe = cur_uprobe;  in alloc_uprobe()
503 	return uprobe;  in alloc_uprobe()
506 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)  in consumer_add()  argument
508 	down_write(&uprobe->consumer_rwsem);  in consumer_add()
509 	uc->next = uprobe->consumers;  in consumer_add()
510 	uprobe->consumers = uc;  in consumer_add()
511 	up_write(&uprobe->consumer_rwsem);  in consumer_add()
519 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)  in consumer_del()  argument
524 	down_write(&uprobe->consumer_rwsem);  in consumer_del()
525 	for (con = &uprobe->consumers; *con; con = &(*con)->next) {  in consumer_del()
532 	up_write(&uprobe->consumer_rwsem);  in consumer_del()
559 static int copy_insn(struct uprobe *uprobe, struct file *filp)  in copy_insn()  argument
561 	struct address_space *mapping = uprobe->inode->i_mapping;  in copy_insn()
562 	loff_t offs = uprobe->offset;  in copy_insn()
563 	void *insn = &uprobe->arch.insn;  in copy_insn()
564 	int size = sizeof(uprobe->arch.insn);  in copy_insn()
569 		if (offs >= i_size_read(uprobe->inode))  in copy_insn()
585 static int prepare_uprobe(struct uprobe *uprobe, struct file *file,  in prepare_uprobe()  argument
590 	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))  in prepare_uprobe()
594 	down_write(&uprobe->consumer_rwsem);  in prepare_uprobe()
595 	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))  in prepare_uprobe()
598 	ret = copy_insn(uprobe, file);  in prepare_uprobe()
603 	if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))  in prepare_uprobe()
606 	ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);  in prepare_uprobe()
611 	BUG_ON((uprobe->offset & ~PAGE_MASK) +  in prepare_uprobe()
615 	set_bit(UPROBE_COPY_INSN, &uprobe->flags);  in prepare_uprobe()
618 	up_write(&uprobe->consumer_rwsem);  in prepare_uprobe()
629 static bool filter_chain(struct uprobe *uprobe,  in filter_chain()  argument
635 	down_read(&uprobe->consumer_rwsem);  in filter_chain()
636 	for (uc = uprobe->consumers; uc; uc = uc->next) {  in filter_chain()
641 	up_read(&uprobe->consumer_rwsem);  in filter_chain()
647 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,  in install_breakpoint()  argument
653 	ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);  in install_breakpoint()
665 	ret = set_swbp(&uprobe->arch, mm, vaddr);  in install_breakpoint()
675 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)  in remove_breakpoint()  argument
678 	return set_orig_insn(&uprobe->arch, mm, vaddr);  in remove_breakpoint()
681 static inline bool uprobe_is_active(struct uprobe *uprobe)  in uprobe_is_active()  argument
683 	return !RB_EMPTY_NODE(&uprobe->rb_node);  in uprobe_is_active()
690 static void delete_uprobe(struct uprobe *uprobe)  in delete_uprobe()  argument
692 	if (WARN_ON(!uprobe_is_active(uprobe)))  in delete_uprobe()
696 	rb_erase(&uprobe->rb_node, &uprobes_tree);  in delete_uprobe()
698 	RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */  in delete_uprobe()
699 	iput(uprobe->inode);  in delete_uprobe()
700 	put_uprobe(uprobe);  in delete_uprobe()
787 register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)  in register_for_each_vma()  argument
794 	info = build_map_info(uprobe->inode->i_mapping,  in register_for_each_vma()
795 					uprobe->offset, is_register);  in register_for_each_vma()
811 		    file_inode(vma->vm_file) != uprobe->inode)  in register_for_each_vma()
815 		    vaddr_to_offset(vma, info->vaddr) != uprobe->offset)  in register_for_each_vma()
822 				err = install_breakpoint(uprobe, mm, vma, info->vaddr);  in register_for_each_vma()
824 			if (!filter_chain(uprobe,  in register_for_each_vma()
826 				err |= remove_breakpoint(uprobe, mm, info->vaddr);  in register_for_each_vma()
840 static int __uprobe_register(struct uprobe *uprobe, struct uprobe_consumer *uc)  in __uprobe_register()  argument
842 	consumer_add(uprobe, uc);  in __uprobe_register()
843 	return register_for_each_vma(uprobe, uc);  in __uprobe_register()
846 static void __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)  in __uprobe_unregister()  argument
850 	if (WARN_ON(!consumer_del(uprobe, uc)))  in __uprobe_unregister()
853 	err = register_for_each_vma(uprobe, NULL);  in __uprobe_unregister()
855 	if (!uprobe->consumers && !err)  in __uprobe_unregister()
856 		delete_uprobe(uprobe);  in __uprobe_unregister()
878 	struct uprobe *uprobe;  in uprobe_register()  local
893 	uprobe = alloc_uprobe(inode, offset);  in uprobe_register()
894 	if (!uprobe)  in uprobe_register()
900 	down_write(&uprobe->register_rwsem);  in uprobe_register()
902 	if (likely(uprobe_is_active(uprobe))) {  in uprobe_register()
903 		ret = __uprobe_register(uprobe, uc);  in uprobe_register()
905 			__uprobe_unregister(uprobe, uc);  in uprobe_register()
907 	up_write(&uprobe->register_rwsem);  in uprobe_register()
908 	put_uprobe(uprobe);  in uprobe_register()
926 	struct uprobe *uprobe;  in uprobe_apply()  local
930 	uprobe = find_uprobe(inode, offset);  in uprobe_apply()
931 	if (WARN_ON(!uprobe))  in uprobe_apply()
934 	down_write(&uprobe->register_rwsem);  in uprobe_apply()
935 	for (con = uprobe->consumers; con && con != uc ; con = con->next)  in uprobe_apply()
938 		ret = register_for_each_vma(uprobe, add ? uc : NULL);  in uprobe_apply()
939 	up_write(&uprobe->register_rwsem);  in uprobe_apply()
940 	put_uprobe(uprobe);  in uprobe_apply()
953 	struct uprobe *uprobe;  in uprobe_unregister()  local
955 	uprobe = find_uprobe(inode, offset);  in uprobe_unregister()
956 	if (WARN_ON(!uprobe))  in uprobe_unregister()
959 	down_write(&uprobe->register_rwsem);  in uprobe_unregister()
960 	__uprobe_unregister(uprobe, uc);  in uprobe_unregister()
961 	up_write(&uprobe->register_rwsem);  in uprobe_unregister()
962 	put_uprobe(uprobe);  in uprobe_unregister()
966 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)  in unapply_uprobe()  argument
977 		    file_inode(vma->vm_file) != uprobe->inode)  in unapply_uprobe()
981 		if (uprobe->offset <  offset ||  in unapply_uprobe()
982 		    uprobe->offset >= offset + vma->vm_end - vma->vm_start)  in unapply_uprobe()
985 		vaddr = offset_to_vaddr(vma, uprobe->offset);  in unapply_uprobe()
986 		err |= remove_breakpoint(uprobe, mm, vaddr);  in unapply_uprobe()
999 		struct uprobe *u = rb_entry(n, struct uprobe, rb_node);  in find_node_in_range()
1028 	struct uprobe *u;  in build_probe_list()
1038 			u = rb_entry(t, struct uprobe, rb_node);  in build_probe_list()
1045 			u = rb_entry(t, struct uprobe, rb_node);  in build_probe_list()
1064 	struct uprobe *uprobe, *u;  in uprobe_mmap()  local
1081 	list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {  in uprobe_mmap()
1083 		    filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {  in uprobe_mmap()
1084 			unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);  in uprobe_mmap()
1085 			install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);  in uprobe_mmap()
1087 		put_uprobe(uprobe);  in uprobe_mmap()
1286 static unsigned long xol_get_insn_slot(struct uprobe *uprobe)  in xol_get_insn_slot()  argument
1300 			      &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));  in xol_get_insn_slot()
1399 		put_uprobe(tmp->uprobe);  in uprobe_free_utask()
1440 		atomic_inc(&n->uprobe->ref);  in dup_utask()
1518 static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)  in prepare_uretprobe()  argument
1568 	atomic_inc(&uprobe->ref);  in prepare_uretprobe()
1569 	ri->uprobe = uprobe;  in prepare_uretprobe()
1588 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)  in pre_ssout()  argument
1598 	xol_vaddr = xol_get_insn_slot(uprobe);  in pre_ssout()
1605 	err = arch_uprobe_pre_xol(&uprobe->arch, regs);  in pre_ssout()
1611 	utask->active_uprobe = uprobe;  in pre_ssout()
1694 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)  in find_active_uprobe()
1697 	struct uprobe *uprobe = NULL;  in find_active_uprobe()  local
1707 			uprobe = find_uprobe(inode, offset);  in find_active_uprobe()
1710 		if (!uprobe)  in find_active_uprobe()
1716 	if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))  in find_active_uprobe()
1720 	return uprobe;  in find_active_uprobe()
1723 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)  in handler_chain()  argument
1729 	down_read(&uprobe->register_rwsem);  in handler_chain()
1730 	for (uc = uprobe->consumers; uc; uc = uc->next) {  in handler_chain()
1746 		prepare_uretprobe(uprobe, regs); /* put bp at return */  in handler_chain()
1748 	if (remove && uprobe->consumers) {  in handler_chain()
1749 		WARN_ON(!uprobe_is_active(uprobe));  in handler_chain()
1750 		unapply_uprobe(uprobe, current->mm);  in handler_chain()
1752 	up_read(&uprobe->register_rwsem);  in handler_chain()
1758 	struct uprobe *uprobe = ri->uprobe;  in handle_uretprobe_chain()  local
1761 	down_read(&uprobe->register_rwsem);  in handle_uretprobe_chain()
1762 	for (uc = uprobe->consumers; uc; uc = uc->next) {  in handle_uretprobe_chain()
1766 	up_read(&uprobe->register_rwsem);  in handle_uretprobe_chain()
1794 		put_uprobe(ri->uprobe);  in handle_trampoline()
1822 	struct uprobe *uprobe;  in handle_swbp()  local
1835 	uprobe = find_active_uprobe(bp_vaddr, &is_swbp);  in handle_swbp()
1836 	if (!uprobe) {  in handle_swbp()
1863 	if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))  in handle_swbp()
1870 	if (arch_uprobe_ignore(&uprobe->arch, regs))  in handle_swbp()
1873 	handler_chain(uprobe, regs);  in handle_swbp()
1875 	if (arch_uprobe_skip_sstep(&uprobe->arch, regs))  in handle_swbp()
1878 	if (!pre_ssout(uprobe, regs, bp_vaddr))  in handle_swbp()
1883 	put_uprobe(uprobe);  in handle_swbp()
1892 	struct uprobe *uprobe;  in handle_singlestep()  local
1895 	uprobe = utask->active_uprobe;  in handle_singlestep()
1897 		err = arch_uprobe_post_xol(&uprobe->arch, regs);  in handle_singlestep()
1899 		arch_uprobe_abort_xol(&uprobe->arch, regs);  in handle_singlestep()
1903 	put_uprobe(uprobe);  in handle_singlestep()