diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 7b6847200b43..169b2e71e9cb 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -766,7 +766,9 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt bpf_map_memcg_exit(old_memcg, new_memcg); return clear_lo32(arena->user_vm_start) + uaddr32; out: - range_tree_set(&arena->rt, pgoff + mapped, page_cnt - mapped); + if (range_tree_set(&arena->rt, pgoff + mapped, page_cnt - mapped)) + pr_warn_ratelimited("bpf_arena: failed to restore free range %ld+%ld after partial alloc\n", + pgoff + mapped, page_cnt - mapped); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); if (mapped) { flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); @@ -881,7 +883,18 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, if (ret) goto defer; - range_tree_set(&arena->rt, pgoff, page_cnt); + ret = range_tree_set(&arena->rt, pgoff, page_cnt); + if (ret) { + /* + * range_tree_set() is failure-atomic, so -ENOMEM leaves the + * range allocated and the pages mapped. Abort the free rather + * than returning pages the free tree does not track; a later + * free of the same range can succeed. + */ + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + bpf_map_memcg_exit(old_memcg, new_memcg); + return; + } init_llist_head(&free_pages); cdata.arena = arena; @@ -977,12 +990,13 @@ static void arena_free_worker(struct work_struct *work) struct llist_node *list, *pos, *t; struct arena_free_span *s; u64 arena_vm_start, user_vm_start; - struct llist_head free_pages; + struct llist_head free_pages, cleared; struct clear_range_data cdata; struct page *page; unsigned long full_uaddr; long kaddr, page_cnt, pgoff; unsigned long flags; + bool retry = false; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { schedule_work(work); @@ -992,28 +1006,43 @@ static void arena_free_worker(struct work_struct *work) bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); init_llist_head(&free_pages); + init_llist_head(&cleared); cdata.arena = arena; cdata.free_pages = &free_pages; arena_vm_start = bpf_arena_get_kern_vm_start(arena); user_vm_start = bpf_arena_get_user_vm_start(arena); list = llist_del_all(&arena->free_spans); - llist_for_each(pos, list) { + llist_for_each_safe(pos, t, list) { s = llist_entry(pos, struct arena_free_span, node); page_cnt = s->page_cnt; kaddr = arena_vm_start + s->uaddr; pgoff = compute_pgoff(arena, s->uaddr); + /* + * Set the range free before clearing PTEs, and requeue the + * span on failure: the PTEs stay intact and the free is + * retried later. Only spans moved to @cleared (PTE clearing + * actually ran) reach the flush/zap/release loop below. + */ + if (range_tree_set(&arena->rt, pgoff, page_cnt)) { + llist_add(&s->node, &arena->free_spans); + retry = true; + continue; + } + /* clear ptes and collect pages in free_pages llist */ apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, apply_range_clear_cb, &cdata); - - range_tree_set(&arena->rt, pgoff, page_cnt); + llist_add(&s->node, &cleared); } raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + if (retry) + irq_work_queue(&arena->free_irq); + /* Iterate the list again without holding spinlock to do the tlb flush and zap_pages */ - llist_for_each_safe(pos, t, list) { + llist_for_each_safe(pos, t, cleared.first) { s = llist_entry(pos, struct arena_free_span, node); page_cnt = s->page_cnt; full_uaddr = clear_lo32(user_vm_start) + s->uaddr; diff --git a/kernel/bpf/range_tree.c b/kernel/bpf/range_tree.c index 2f28886f3ff7..54055b1fe541 100644 --- a/kernel/bpf/range_tree.c +++ b/kernel/bpf/range_tree.c @@ -143,16 +143,22 @@ int range_tree_clear(struct range_tree *rt, u32 start, u32 len) if (rn->rn_start < start && rn->rn_last > last) { u32 old_last = rn->rn_last; + /* + * Pre-allocate the right-half node before modifying + * the tree. If allocation fails we return -ENOMEM + * without altering the range tree. + */ + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + /* Overlaps with the entire clearing range */ range_it_remove(rn, rt); rn->rn_last = start - 1; range_it_insert(rn, rt); - /* Add a range */ - new_rn = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, - NUMA_NO_NODE); - if (!new_rn) - return -ENOMEM; + /* Add right-half range */ new_rn->rn_start = last + 1; new_rn->rn_last = old_last; range_it_insert(new_rn, rt); @@ -193,6 +199,7 @@ int is_range_tree_set(struct range_tree *rt, u32 start, u32 len) int range_tree_set(struct range_tree *rt, u32 start, u32 len) { u32 last = start + len - 1; + struct range_node *new_rn = NULL; struct range_node *right; struct range_node *left; int err; @@ -202,20 +209,40 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) if (left && left->rn_start <= start && left->rn_last >= last) return 0; + /* + * A new node is needed only when the range has no adjacent free + * range on either side. This is known before clearing: any range + * covering start - 1 or last + 1 survives the clear as an adjacent + * piece. Allocate only in that case, before modifying the tree, so + * a failure leaves the range tree unmodified + */ + left = range_it_iter_first(rt, start - 1, start - 1); + right = range_it_iter_first(rt, last + 1, last + 1); + if (!left && !right) { + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + } + /* Clear out everything in the range we want to set. */ err = range_tree_clear(rt, start, len); if (err) - return err; + goto out_free_new; /* Do we have a left-adjacent range ? */ left = range_it_iter_first(rt, start - 1, start - 1); - if (left && left->rn_last + 1 != start) - return -EFAULT; + if (left && left->rn_last + 1 != start) { + err = -EFAULT; + goto out_free_new; + } /* Do we have a right-adjacent range ? */ right = range_it_iter_first(rt, last + 1, last + 1); - if (right && right->rn_start != last + 1) - return -EFAULT; + if (right && right->rn_start != last + 1) { + err = -EFAULT; + goto out_free_new; + } if (left && right) { /* Combine left and right adjacent ranges */ @@ -235,14 +262,16 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) right->rn_start = start; range_it_insert(right, rt); } else { - left = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!left) - return -ENOMEM; - left->rn_start = start; - left->rn_last = last; - range_it_insert(left, rt); + /* No adjacent ranges; use the pre-allocated node */ + new_rn->rn_start = start; + new_rn->rn_last = last; + range_it_insert(new_rn, rt); } return 0; + +out_free_new: + kfree_nolock(new_rn); + return err; } void range_tree_destroy(struct range_tree *rt)