usockets(windows): keep a closed poll alive until the outer tick and libuv are done with it

packages/bun-usockets/src/context.c

@@ -482,7 +482,7 @@ void us_listen_socket_close(struct us_listen_socket_t *ls) {
         struct us_socket_group_t *group = ls->accept_group;
         struct us_loop_t *loop = s->group->loop;
         us_poll_stop((struct us_poll_t *) s, loop);
-        bsd_close_socket(us_poll_fd((struct us_poll_t *) s));
+        us_internal_poll_close_fd((struct us_poll_t *) s);
 
         us_internal_listen_socket_ssl_free(ls);
 

packages/bun-usockets/src/eventing/epoll_kqueue.c

@@ -763,6 +763,10 @@ void us_poll_stop(struct us_poll_t *p, struct us_loop_t *loop) {
     us_internal_loop_update_pending_ready_polls(loop, p, 0, old_events, new_events);
 }
 
+void us_internal_poll_close_fd(struct us_poll_t *p) {
+    bsd_close_socket(us_poll_fd(p));
+}
+
 size_t us_internal_accept_poll_event(struct us_poll_t *p) {
 #ifdef LIBUS_USE_EPOLL
     int fd = us_poll_fd(p);

packages/bun-usockets/src/eventing/libuv.c

@@ -47,8 +47,11 @@ static struct us_socket_t *us_internal_poll_cb_adopted_socket(struct us_poll_t *
   return us_internal_socket_follow_adopted((struct us_socket_t *)wp);
 }
 
-/* uv_poll_t->data always (except for most times after calling us_poll_stop)
- * points to the us_poll_t */
+static void close_cb_free_poll(uv_handle_t *h);
+
+/* uv_poll_t->data always points to the us_poll_t (us_poll_resize moves it to
+ * the replacement block). libuv delivers no poll_cb once us_poll_stop has
+ * disarmed the handle, and nothing re-arms a stopped poll. */
 static void poll_cb(uv_poll_t *p, int status, int events) {
   /* UV_DISCONNECT (Windows AFD): the peer closed its write side. A FIN
    * arriving after this side already half-closed and stopped reading never
@@ -122,7 +125,28 @@ static void poll_cb(uv_poll_t *p, int status, int events) {
   if (!error && !eof && !(events & (UV_READABLE | UV_WRITABLE))) {
     return;
   }
-  us_internal_dispatch_ready_poll((struct us_poll_t *)p->data, error, eof, events);
+  struct us_poll_t *wp = (struct us_poll_t *)p->data;
+  wp->poll_cb_depth++;
+  us_internal_dispatch_ready_poll(wp, error, eof, events);
+  /* The dispatch may have relocated the poll (us_poll_resize); the counter
+   * was copied along, so finish on the block the handle points to now. */
+  wp = (struct us_poll_t *)p->data;
+  /* uv_run is not reentrant, and a handler that waits for a promise runs it
+   * anyway. If such an inner run had closed this handle, it would also have
+   * run the handle's endgame while libuv's outer uv__fast_poll_process_poll_req
+   * frame (the caller of this function) was still using the handle; that
+   * frame then queues the endgame a second time. So us_poll_stop only disarms
+   * the handle while a poll_cb frame is on the stack, and the outermost frame
+   * closes it here, on its way back into libuv: a close from inside the
+   * callback is what libuv supports, and the endgame runs in the outer run.
+   * The socket itself is closed after the handle, as on the direct path in
+   * us_poll_stop (see close_fd). */
+  if (--wp->poll_cb_depth == 0 && wp->stopped) {
+    uv_close((uv_handle_t *)p, close_cb_free_poll);
+    if (wp->close_fd) {
+      bsd_close_socket(wp->fd);
+    }
+  }
 }
 
 static void prepare_cb(uv_prepare_t *p) {
@@ -139,13 +163,21 @@ static void check_cb(uv_check_t *p) {
 /* Not used for polls, since polls need two frees */
 static void close_cb_free(uv_handle_t *h) { us_free(h->data); }
 
-/* This one is different for polls, since we need two frees here */
+/* Polls have two blocks; whichever of us_poll_free and this callback runs
+ * second frees both (see us_poll_t). This one usually runs second: uv_close
+ * cancels the in-flight AFD request, us_poll_free runs from loop_post in the
+ * same iteration, and the cancellation is processed on a later one. It runs
+ * first for a socket closed during a nested tick (a handler that waits for a
+ * promise): the inner run completes the close, but loop_post leaves the closed
+ * list alone until the outermost tick (tick_depth), since the outer dispatch
+ * may still hold sockets on it. */
 static void close_cb_free_poll(uv_handle_t *h) {
-  /* It is only in case we called us_poll_stop then quickly us_poll_free that we
-   * enter this. Most of the time, actual freeing is done by us_poll_free. */
-  if (h->data) {
-    us_free(h->data);
+  struct us_poll_t *p = h->data;
+  if (p->released) {
     us_free(h);
+    us_free(p);
+  } else {
+    p->uv_closed = 1;
   }
 }
 
@@ -168,39 +200,74 @@ void us_poll_init(struct us_poll_t *p, LIBUS_SOCKET_DESCRIPTOR fd,
 }
 
 void us_poll_free(struct us_poll_t *p, struct us_loop_t *loop) {
-  // poll was resized and dont own uv_poll_t anymore
-  if(!p->uv_p) {
+  uv_poll_t *h = p->uv_p;
+  /* us_poll_resize moved the handle to the replacement block, or
+   * us_poll_start_rc already freed it. */
+  if (!h) {
     us_free(p);
     return;
   }
-  /* The idea here is like so; in us_poll_stop we call uv_close after setting
-   * data of uv-poll to 0. This means that in close_cb_free we call free on 0
-   * with does nothing, since us_poll_stop should not really free the poll.
-   * HOWEVER, if we then call us_poll_free while still closing the uv-poll, we
-   * simply change back the data to point to our structure so that we actually
-   * do free it like we should. */
-  if (uv_is_closing((uv_handle_t *)p->uv_p)) {
-    p->uv_p->data = p;
-  } else {
-    us_free(p->uv_p);
+  /* Never started (us_create_poll zeroes the handle): libuv has not seen it.
+   * Closed: libuv is done with it and close_cb_free_poll left it to us. */
+  if (h->type != UV_POLL || p->uv_closed) {
+    us_free(h);
     us_free(p);
+    return;
+  }
+  /* Stopped, or still polling if the caller skipped us_poll_stop: an AFD
+   * request that completes into h may still be in flight, so h has to live
+   * until close_cb_free_poll, which now frees both blocks. */
+  us_poll_stop(p, loop);
+  p->released = 1;
+}
+
+/* One-way: on this backend a stopped poll is a closed (or closing) handle and
+ * cannot be started again. The blocks are freed later, see close_cb_free_poll.
+ * Callers close the socket afterwards through us_internal_poll_close_fd, which
+ * keeps it open for as long as the uv_close below is deferred. */
+void us_poll_stop(struct us_poll_t *p, struct us_loop_t *loop) {
+  uv_poll_t *h = p->uv_p;
+  if (!h || h->type != UV_POLL || p->stopped) return;
+  p->stopped = 1;
+  /* Disarm first: a completion this or an inner run has already dequeued must
+   * not reach poll_cb for a socket that is now on the closed list. */
+  uv_poll_stop(h);
+  /* Inside this poll's own callback the outermost poll_cb frame closes the
+   * handle instead (see poll_cb). */
+  if (p->poll_cb_depth == 0) {
+    uv_close((uv_handle_t *)h, close_cb_free_poll);
   }
 }
 
+void us_internal_poll_close_fd(struct us_poll_t *p) {
+  /* The uv_close is still pending on the outermost poll_cb frame; it has to
+   * see the socket open (see close_fd in us_poll_t), so the frame closes both. */
+  if (p->stopped && p->poll_cb_depth > 0) {
+    p->close_fd = 1;
+    return;
+  }
+  bsd_close_socket(p->fd);
+}
+
 int us_poll_start_rc(struct us_poll_t *p, struct us_loop_t *loop, int events) {
-  if(!p->uv_p) return 0;
+  uv_poll_t *h = p->uv_p;
+  if (!h) return 0;
   p->poll_type = us_internal_poll_type(p) |
                  ((events & LIBUS_SOCKET_READABLE) ? POLL_TYPE_POLLING_IN : 0) |
                  ((events & LIBUS_SOCKET_WRITABLE) ? POLL_TYPE_POLLING_OUT : 0);
 
-  /* uv_poll_init_socket (win/poll.c) can fail either before uv__handle_init
-   * (ioctlsocket FIONBIO) or after it (getsockopt SO_PROTOCOL_INFOW). The
-   * latter leaves the handle linked into loop->handle_queue with
-   * submitted_events_* still unset. Zero first so, on failure, ->type
-   * distinguishes the two states and the fields uv__poll_close reads are 0
-   * rather than garbage. */
-  memset(p->uv_p, 0, sizeof(uv_poll_t));
-  p->uv_p->data = p;
+  if (h->type == UV_POLL) {
+    /* Already registered. Initializing it again would wipe the in-flight AFD
+     * requests and the loop's list links out from under libuv, and the next
+     * completion would land in a handle libuv no longer tracks. A stopped poll
+     * cannot be restarted; a live one only changes its mask. */
+    if (p->stopped) {
+      errno = -UV_EBADF;
+      return UV_EBADF;
+    }
+    uv_poll_start(h, events | UV_DISCONNECT, poll_cb);
+    return 0;
+  }
 
   int rc;
 #if defined(LIBUS_SOCKET_FAULT_INJECTION) && LIBUS_SOCKET_FAULT_INJECTION
@@ -210,20 +277,19 @@ int us_poll_start_rc(struct us_poll_t *p, struct us_loop_t *loop, int events) {
     rc = (int) injected;
   } else
 #endif
-  rc = uv_poll_init_socket(loop->uv_loop, p->uv_p, p->fd);
+  rc = uv_poll_init_socket(loop->uv_loop, h, p->fd);
   if (rc < 0) {
     int saved = LIBUS_ERR;
-    if (p->uv_p->type == UV_POLL) {
-      /* uv__handle_init ran: the handle is in loop->handle_queue. Close it
-       * through libuv so it is unlinked; the caller's us_poll_free sees
-       * uv_is_closing and hands ownership to close_cb_free_poll. */
-      p->uv_p->data = 0;
-      uv_close((uv_handle_t *)p->uv_p, close_cb_free_poll);
+    /* uv_poll_init_socket (win/poll.c) fails either before uv__handle_init
+     * (ioctlsocket FIONBIO) or after it (getsockopt SO_PROTOCOL_INFOW). After
+     * it, the handle is in loop->handle_queue: close it through libuv so it is
+     * unlinked, and the caller's us_poll_free hands it to close_cb_free_poll.
+     * (uv__poll_close reads submitted_events_*, which init did not reach; they
+     * are 0 from us_create_poll.) Before it, the block is still only ours. */
+    if (h->type == UV_POLL) {
+      us_poll_stop(p, loop);
     } else {
-      /* Never reached uv__handle_init: uv_p is still our raw block. Free it
-       * here and null the pointer so the caller's us_poll_free takes the
-       * !uv_p fast path (its uv_is_closing check would read garbage). */
-      us_free(p->uv_p);
+      us_free(h);
       p->uv_p = NULL;
     }
     errno = saved ? saved : -rc;
@@ -232,11 +298,11 @@ int us_poll_start_rc(struct us_poll_t *p, struct us_loop_t *loop, int events) {
   // This unref is okay in the context of Bun's event loop, because sockets have
   // a `Async.KeepAlive` associated with them, which is used instead of the
   // usockets internals. usockets doesnt have a notion of ref-counted handles.
-  uv_unref((uv_handle_t *)p->uv_p);
+  uv_unref((uv_handle_t *)h);
   /* Always ask for UV_DISCONNECT: a peer FIN must fire even when the poll is
    * writable-only at that moment (a half-closed connection whose reads are
    * paused is exactly the state that otherwise hangs; see poll_cb). */
-  uv_poll_start(p->uv_p, events | UV_DISCONNECT, poll_cb);
+  uv_poll_start(h, events | UV_DISCONNECT, poll_cb);
   return 0;
 }
 
@@ -245,7 +311,10 @@ void us_poll_start(struct us_poll_t *p, struct us_loop_t *loop, int events) {
 }
 
 int us_poll_change(struct us_poll_t *p, struct us_loop_t *loop, int events) {
-  if(!p->uv_p) return 0;
+  uv_poll_t *h = p->uv_p;
+  /* A stopped poll belongs to a socket on the closed list; re-arming it would
+   * deliver a poll_cb for that socket. */
+  if (!h || p->stopped) return 0;
   if (us_poll_events(p) != events) {
     p->poll_type =
         us_internal_poll_type(p) |
@@ -254,23 +323,11 @@ int us_poll_change(struct us_poll_t *p, struct us_loop_t *loop, int events) {
     /* The poll stays initialized across changes here (the dispatcher never
      * parks a libuv poll), so this cannot hit the registration failure the
      * epoll re-add can; uv_poll_start on a live poll only rejects bad args. */
-    uv_poll_start(p->uv_p, events | UV_DISCONNECT, poll_cb);
+    uv_poll_start(h, events | UV_DISCONNECT, poll_cb);
   }
   return 0;
 }
 
-void us_poll_stop(struct us_poll_t *p, struct us_loop_t *loop) {
-  if(!p->uv_p) return;
-  uv_poll_stop(p->uv_p);
-
-  /* We normally only want to close the poll here, not free it. But if we stop
-   * it, then quickly "free" it with us_poll_free, we postpone the actual
-   * freeing to close_cb_free_poll whenever it triggers. That's why we set data
-   * to null here, so that us_poll_free can reset it if needed */
-  p->uv_p->data = 0;
-  uv_close((uv_handle_t *)p->uv_p, close_cb_free_poll);
-}
-
 int us_poll_events(struct us_poll_t *p) {
   return ((p->poll_type & POLL_TYPE_POLLING_IN) ? LIBUS_SOCKET_READABLE : 0) |
          ((p->poll_type & POLL_TYPE_POLLING_OUT) ? LIBUS_SOCKET_WRITABLE : 0);
@@ -294,9 +351,11 @@ void us_loop_pump(struct us_loop_t *loop) {
    * timers are never processed. Bun's outer drive loops (wait_for_promise,
    * bun:test) supply their own keep-going predicate, so force exactly one
    * non-blocking iteration; UV_RUN_NOWAIT keeps the poll timeout at 0. */
+  loop->data.tick_depth++;
   loop->uv_loop->active_handles++;
   uv_run(loop->uv_loop, UV_RUN_NOWAIT);
   loop->uv_loop->active_handles--;
+  loop->data.tick_depth--;
 }
 
 struct us_loop_t *us_create_loop(void *hint,
@@ -374,15 +433,28 @@ void us_loop_run(struct us_loop_t *loop) {
     Bun__JSC_onBeforeWait(loop->data.jsc_vm, (uint64_t) uv_now(loop->uv_loop) * 1000000ULL);
   }
 
+  /* check_cb -> us_internal_loop_post frees the closed sockets only at depth
+   * 1: a poll callback that waits for a promise re-enters here, and the outer
+   * dispatch still holds the socket it is dispatching (same as the POSIX
+   * backend's us_loop_run_bun_tick). */
+  loop->data.tick_depth++;
   uv_run(loop->uv_loop, UV_RUN_ONCE);
+  loop->data.tick_depth--;
 }
 
 struct us_poll_t *us_create_poll(struct us_loop_t *loop, int fallthrough,
                                  unsigned int ext_size) {
   struct us_poll_t *p =
       (struct us_poll_t *)us_malloc(sizeof(struct us_poll_t) + ext_size);
-  p->uv_p = us_malloc(sizeof(uv_poll_t));
+  /* Zeroed so that ->type tells us_poll_free and us_poll_stop whether
+   * us_poll_start_rc ever registered the handle (uv__handle_init sets it). */
+  p->uv_p = us_calloc(1, sizeof(uv_poll_t));
   p->uv_p->data = p;
+  p->stopped = 0;
+  p->close_fd = 0;
+  p->uv_closed = 0;
+  p->released = 0;
+  p->poll_cb_depth = 0;
   return p;
 }
 

packages/bun-usockets/src/internal/eventing/libuv.h

@@ -45,6 +45,25 @@ struct us_poll_t {
   uv_poll_t *uv_p;
   LIBUS_SOCKET_DESCRIPTOR fd;
   unsigned char poll_type;
+  /* us_poll_stop ran. uv_p is closing, or closes when poll_cb_depth drops to
+   * 0 (see poll_cb). Nothing re-arms the poll after this. */
+  unsigned char stopped : 1;
+  /* us_internal_poll_close_fd ran while the close of uv_p was still deferred;
+   * fd is closed right after that close is issued. The order matters: uv_close
+   * cancels the in-flight request with an ioctl on the socket, and a process
+   * with strict handle checks (every AppContainer) dies on a closed handle. */
+  unsigned char close_fd : 1;
+  /* Once us_poll_start_rc has registered uv_p, libuv keeps pointers into it
+   * (the in-flight AFD requests live inside the handle, and it sits in the
+   * loop's handle and endgame lists) until it runs the close callback. So the
+   * two blocks are freed by whichever of us_poll_free and close_cb_free_poll
+   * runs second; these record which one has already run. */
+  unsigned char uv_closed : 1;
+  unsigned char released : 1;
+  /* Number of poll_cb frames for this poll on the stack. More than one means
+   * a handler re-entered the event loop (it waited for a promise) and the
+   * inner run dispatched this poll again. */
+  unsigned int poll_cb_depth;
 };
 
 #endif // LIBUV_H

packages/bun-usockets/src/internal/internal.h

@@ -208,6 +208,12 @@ void us_internal_async_wakeup(struct us_internal_async *a);
 size_t us_internal_accept_poll_event(struct us_poll_t *p);
 int us_internal_poll_type(struct us_poll_t *p);
 void us_internal_poll_set_type(struct us_poll_t *p, int poll_type);
+/* Closes the descriptor of a poll that is being closed (after us_poll_stop, or
+ * the kqueue equivalent). On epoll/kqueue this is bsd_close_socket. On libuv
+ * the descriptor has to stay open until the handle's uv_close is issued, which
+ * a stop from inside the poll's own callback defers, so it is closed at that
+ * point instead (see us_poll_stop in eventing/libuv.c). */
+void us_internal_poll_close_fd(struct us_poll_t *p);
 
 /* SSL loop data */
 void us_internal_init_loop_ssl_data(us_loop_r loop);

packages/bun-usockets/src/internal/loop_data.h

@@ -88,10 +88,10 @@ struct us_internal_loop_data_t {
     /* We do not care if this flips or not, it doesn't matter */
     size_t iteration_nr;
     void* jsc_vm;
-    /* Reentrancy depth of us_loop_run_bun_tick. When >1, we are inside a
-     * nested tick (e.g. waitForPromise from a poll callback). Freeing closed
-     * sockets must be deferred to the outermost tick so the outer dispatch
-     * doesn't read a freed poll. */
+    /* Reentrancy depth of us_loop_run / us_loop_run_bun_tick (and us_loop_pump
+     * on libuv). When >1, we are inside a nested tick (e.g. waitForPromise from
+     * a poll callback). Freeing closed sockets must be deferred to the
+     * outermost tick so the outer dispatch doesn't read a freed poll. */
     int tick_depth;
 };
 

packages/bun-usockets/src/socket.c

@@ -199,7 +199,7 @@ void us_connecting_socket_close(struct us_connecting_socket_t *c) {
         us_internal_socket_group_unlink_socket(s->group, s);
 
         us_poll_stop((struct us_poll_t *) s, s->group->loop);
-        bsd_close_socket(us_poll_fd((struct us_poll_t *) s));
+        us_internal_poll_close_fd((struct us_poll_t *) s);
 
         /* Link this socket to the close-list and let it be deleted after this iteration */
         s->next = s->group->loop->data.closed_head;
@@ -310,7 +310,7 @@ struct us_socket_t *us_internal_socket_close_raw(struct us_socket_t *s, int code
             setsockopt(us_poll_fd((struct us_poll_t *)s), SOL_SOCKET, SO_LINGER, (const char*)&l, sizeof(l));
         }
 
-        bsd_close_socket(us_poll_fd((struct us_poll_t *) s));
+        us_internal_poll_close_fd((struct us_poll_t *) s);
 
         /* Mark the socket as closed */
         s->flags.is_closed = 1;

packages/bun-usockets/src/udp.c

@@ -116,7 +116,7 @@ void us_udp_socket_close(struct us_udp_socket_t *s) {
     struct us_loop_t *loop = s->loop;
     struct us_poll_t *p = (struct us_poll_t *) s;
     us_poll_stop(p, loop);
-    bsd_close_socket(us_poll_fd(p));
+    us_internal_poll_close_fd(p);
     s->closed = 1;
     s->next = loop->data.closed_udp_head;
     loop->data.closed_udp_head = s;