kernel/events/uprobes.c

89  * Execute out of line area: anonymous executable mapping installed
1437 static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
1450 	if (!area->vaddr) {
1452 		area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
1454 		if (IS_ERR_VALUE(area->vaddr)) {
1455 			ret = area->vaddr;
1460 	vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
1462 				&area->xol_mapping);
1470 	smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
1481 	struct xol_area *area;
1483 	area = kmalloc(sizeof(*area), GFP_KERNEL);
1484 	if (unlikely(!area))
1487 	area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long),
1489 	if (!area->bitmap)
1492 	area->xol_mapping.name = "[uprobes]";
1493 	area->xol_mapping.fault = NULL;
1494 	area->xol_mapping.pages = area->pages;
1495 	area->pages[0] = alloc_page(GFP_HIGHUSER);
1496 	if (!area->pages[0])
1498 	area->pages[1] = NULL;
1500 	area->vaddr = vaddr;
1501 	init_waitqueue_head(&area->wq);
1503 	set_bit(0, area->bitmap);
1504 	atomic_set(&area->slot_count, 1);
1505 	arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
1507 	if (!xol_add_vma(mm, area))
1508 		return area;
1510 	__free_page(area->pages[0]);
1512 	kfree(area->bitmap);
1514 	kfree(area);
1521  * This area will be used for storing instructions for execution out of line.
1523  * Returns the allocated area or NULL.
1528 	struct xol_area *area;
1534 	area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */
1535 	return area;
1539  * uprobe_clear_state - Free the area allocated for slots.
1543 	struct xol_area *area = mm->uprobes_state.xol_area;
1549 	if (!area)
1552 	put_page(area->pages[0]);
1553 	kfree(area->bitmap);
1554 	kfree(area);
1579 static unsigned long xol_take_insn_slot(struct xol_area *area)
1585 		slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1587 			if (!test_and_set_bit(slot_nr, area->bitmap))
1593 		wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
1596 	slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
1597 	atomic_inc(&area->slot_count);
1608 	struct xol_area *area;
1611 	area = get_xol_area();
1612 	if (!area)
1615 	xol_vaddr = xol_take_insn_slot(area);
1619 	arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,
1632 	struct xol_area *area;
1643 	area = tsk->mm->uprobes_state.xol_area;
1644 	vma_end = area->vaddr + PAGE_SIZE;
1645 	if (area->vaddr <= slot_addr && slot_addr < vma_end) {
1649 		offset = slot_addr - area->vaddr;
1654 		clear_bit(slot_nr, area->bitmap);
1655 		atomic_dec(&area->slot_count);
1657 		if (waitqueue_active(&area->wq))
1658 			wake_up(&area->wq);
1788 		uprobe_warn(current, "dup xol area");
1798 	struct xol_area *area;
1812 	area = mm->uprobes_state.xol_area;
1813 	if (!area)
1814 		return uprobe_warn(t, "dup xol area");
1819 	t->utask->dup_xol_addr = area->vaddr;
1825  * Current area->vaddr notion assume the trampoline address is always
1826  * equal area->vaddr.
1832 	struct xol_area *area;
1836 	area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */
1837 	if (area)
1838 		trampoline_vaddr = area->vaddr;