Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 30 additions & 139 deletions packages/bun-usockets/src/eventing/epoll_kqueue.c
Original file line number Diff line number Diff line change
Expand Up @@ -101,18 +101,6 @@ void us_internal_poll_set_type(struct us_poll_t *p, int poll_type) {
p->state.poll_type = poll_type | (p->state.poll_type & POLL_TYPE_POLLING_MASK);
}

/* Timer */
void *us_timer_ext(struct us_timer_t *timer) {
return ((struct us_internal_callback_t *) timer) + 1;
}

struct us_loop_t *us_timer_loop(struct us_timer_t *t) {
struct us_internal_callback_t *internal_cb = (struct us_internal_callback_t *) t;

return internal_cb->loop;
}


#if defined(LIBUS_USE_EPOLL)

#include <sys/syscall.h>
Expand Down Expand Up @@ -243,9 +231,9 @@ static void us_internal_dispatch_ready_polls(struct us_loop_t *loop) {
const uint16_t flags = loop->ready_polls[i].flags;
struct kevent_flags bits = {
#if defined(__APPLE__)
.readable = (filter == EVFILT_READ || filter == EVFILT_TIMER || filter == EVFILT_MACHPORT),
.readable = (filter == EVFILT_READ || filter == EVFILT_MACHPORT),
#else
.readable = (filter == EVFILT_READ || filter == EVFILT_TIMER || filter == EVFILT_USER),
.readable = (filter == EVFILT_READ || filter == EVFILT_USER),
#endif
.writable = (filter == EVFILT_WRITE),
.error = !!(flags & EV_ERROR),
Expand Down Expand Up @@ -320,26 +308,45 @@ static void us_internal_drain_ready_polls(struct us_loop_t *loop) {
}
}

void us_loop_run(struct us_loop_t *loop) {
us_loop_integrate(loop);
/* Bound `timeout` by the socket-timeout sweep deadline (NULL == forever). */
static const struct timespec *us_internal_clamp_to_sweep(struct us_loop_t *loop, const struct timespec *timeout, struct timespec *storage) {
long long ns = us_internal_sweep_timeout_ns(loop);
if (ns < 0) {
return timeout;
}
long long sweep_sec = ns / 1000000000LL;
long long sweep_nsec = ns % 1000000000LL;
if (timeout && (timeout->tv_sec < sweep_sec ||
(timeout->tv_sec == sweep_sec && timeout->tv_nsec <= sweep_nsec))) {
return timeout;
}
storage->tv_sec = (time_t) sweep_sec;
storage->tv_nsec = (long) sweep_nsec;
return storage;
}

void us_loop_run(struct us_loop_t *loop) {
/* While we have non-fallthrough polls we shouldn't fall through */
while (loop->num_polls) {
loop->data.tick_depth++;
/* Emit pre callback */
us_internal_loop_pre(loop);

struct timespec sweep_ts;
const struct timespec *timeout = us_internal_clamp_to_sweep(loop, NULL, &sweep_ts);

/* Fetch ready polls */
#ifdef LIBUS_USE_EPOLL
loop->num_ready_polls = bun_epoll_pwait2(loop->fd, loop->ready_polls, LIBUS_MAX_READY_POLLS, NULL);
loop->num_ready_polls = bun_epoll_pwait2(loop->fd, loop->ready_polls, LIBUS_MAX_READY_POLLS, timeout);
#else
do {
loop->num_ready_polls = kevent64(loop->fd, NULL, 0, loop->ready_polls, LIBUS_MAX_READY_POLLS, 0, NULL);
loop->num_ready_polls = kevent64(loop->fd, NULL, 0, loop->ready_polls, LIBUS_MAX_READY_POLLS, 0, timeout);
} while (IS_EINTR(loop->num_ready_polls));
#endif

us_internal_dispatch_ready_polls(loop);
us_internal_drain_ready_polls(loop);
us_internal_sweep_if_due(loop);

/* Emit post callback */
us_internal_loop_post(loop);
Expand All @@ -355,14 +362,6 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout

loop->data.tick_depth++;

struct us_internal_callback_t *timer_callback = (struct us_internal_callback_t*)loop->data.sweep_timer;

// Only integrate the loop if we haven't already.
// Otherwise we will keep restarting the timer.
if(!timer_callback->cb) {
us_loop_integrate(loop);
}

/* Emit pre callback */
us_internal_loop_pre(loop);

Expand All @@ -381,6 +380,9 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout
}
}

struct timespec sweep_ts;
timeout = us_internal_clamp_to_sweep(loop, timeout, &sweep_ts);

const unsigned int had_wakeups = __atomic_exchange_n(&loop->pending_wakeups, 0, __ATOMIC_ACQUIRE);
const int will_idle_inside_event_loop = had_wakeups == 0 && (!timeout || (timeout->tv_nsec != 0 || timeout->tv_sec != 0));
if (will_idle_inside_event_loop && loop->data.jsc_vm)
Expand Down Expand Up @@ -408,6 +410,7 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout

us_internal_dispatch_ready_polls(loop);
us_internal_drain_ready_polls(loop);
us_internal_sweep_if_due(loop);

/* Emit post callback */
us_internal_loop_post(loop);
Expand Down Expand Up @@ -595,123 +598,11 @@ size_t us_internal_accept_poll_event(struct us_poll_t *p) {
} while (IS_EINTR(read_length));
return buf;
#else
/* Kqueue has no underlying FD for timers or user events */
/* Kqueue has no underlying FD for user events */
return 0;
#endif
}

/* Timer */
#ifdef LIBUS_USE_EPOLL
struct us_timer_t *us_create_timer(struct us_loop_t *loop, int fallthrough, unsigned int ext_size) {
struct us_poll_t *p = us_create_poll(loop, fallthrough, sizeof(struct us_internal_callback_t) + ext_size);
memset(p, 0, sizeof(struct us_internal_callback_t) + ext_size);
int timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC);
if (timerfd == -1) {
return NULL;
}
us_poll_init(p, timerfd, POLL_TYPE_CALLBACK);

struct us_internal_callback_t *cb = (struct us_internal_callback_t *) p;
cb->loop = loop;
cb->cb_expects_the_loop = 0;
cb->leave_poll_ready = 0;
cb->has_added_timer_to_event_loop = 0;

return (struct us_timer_t *) cb;
}
#else
struct us_timer_t *us_create_timer(struct us_loop_t *loop, int fallthrough, unsigned int ext_size) {
struct us_internal_callback_t *cb = us_calloc(1, sizeof(struct us_internal_callback_t) + ext_size);

cb->loop = loop;
cb->cb_expects_the_loop = 0;
cb->leave_poll_ready = 0;

/* Bug: us_internal_poll_set_type does not SET the type, it only CHANGES it */
cb->p.state.poll_type = POLL_TYPE_POLLING_IN;
us_internal_poll_set_type((struct us_poll_t *) cb, POLL_TYPE_CALLBACK);

if (!fallthrough) {
loop->num_polls++;
}

return (struct us_timer_t *) cb;
}
#endif

#ifdef LIBUS_USE_EPOLL
void us_timer_close(struct us_timer_t *timer, int fallthrough) {
struct us_internal_callback_t *cb = (struct us_internal_callback_t *) timer;

us_poll_stop(&cb->p, cb->loop);
close(us_poll_fd(&cb->p));

/* (regular) sockets are the only polls which are not freed immediately */
if(fallthrough){
us_free(timer);
}else {
us_poll_free((struct us_poll_t *) timer, cb->loop);
}
}

void us_timer_set(struct us_timer_t *t, void (*cb)(struct us_timer_t *t), int ms, int repeat_ms) {
struct us_internal_callback_t *internal_cb = (struct us_internal_callback_t *) t;

internal_cb->cb = (void (*)(struct us_internal_callback_t *)) cb;

struct itimerspec timer_spec = {
{repeat_ms / 1000, (long) (repeat_ms % 1000) * (long) 1000000},
{ms / 1000, (long) (ms % 1000) * (long) 1000000}
};

timerfd_settime(us_poll_fd((struct us_poll_t *) t), 0, &timer_spec, NULL);

// Avoid the system call overhead of re-adding this timer to the event loop only to receive EEXIST
if (internal_cb->loop->data.sweep_timer == t) {
if (internal_cb->has_added_timer_to_event_loop) {
return;
}
internal_cb->has_added_timer_to_event_loop = 1;
}
us_poll_start((struct us_poll_t *) t, internal_cb->loop, LIBUS_SOCKET_READABLE);
}
#else
void us_timer_close(struct us_timer_t *timer, int fallthrough) {
struct us_internal_callback_t *internal_cb = (struct us_internal_callback_t *) timer;

struct kevent64_s event;
EV_SET64(&event, (uint64_t) (void*) internal_cb, EVFILT_TIMER, EV_DELETE, 0, 0, (uint64_t)internal_cb, 0, 0);
int ret;
do {
ret = kevent64(internal_cb->loop->fd, &event, 1, &event, 1, KEVENT_FLAG_ERROR_EVENTS, NULL);
} while (IS_EINTR(ret));


/* (regular) sockets are the only polls which are not freed immediately */
if(fallthrough){
us_free(timer);
}else {
us_poll_free((struct us_poll_t *) timer, internal_cb->loop);
}
}

void us_timer_set(struct us_timer_t *t, void (*cb)(struct us_timer_t *t), int ms, int repeat_ms) {
struct us_internal_callback_t *internal_cb = (struct us_internal_callback_t *) t;

internal_cb->cb = (void (*)(struct us_internal_callback_t *)) cb;

/* Bug: repeat_ms must be the same as ms, or 0 */
struct kevent64_s event;
uint64_t ptr = (uint64_t)(void*)internal_cb;
EV_SET64(&event, ptr, EVFILT_TIMER, EV_ADD | (repeat_ms ? 0 : EV_ONESHOT), 0, ms, (uint64_t)internal_cb, 0, 0);

int ret;
do {
ret = kevent64(internal_cb->loop->fd, &event, 1, &event, 1, KEVENT_FLAG_ERROR_EVENTS, NULL);
} while (IS_EINTR(ret));
}
#endif

/* Async (internal helper for loop's wakeup feature) */
#ifdef LIBUS_USE_EPOLL
struct us_internal_async *us_internal_create_async(struct us_loop_t *loop, int fallthrough, unsigned int ext_size) {
Expand Down
1 change: 0 additions & 1 deletion packages/bun-usockets/src/internal/eventing/epoll_kqueue.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@

#ifdef LIBUS_USE_EPOLL
#include <sys/epoll.h>
#include <sys/timerfd.h>
#include <sys/eventfd.h>
#define LIBUS_SOCKET_READABLE EPOLLIN
#define LIBUS_SOCKET_WRITABLE EPOLLOUT
Expand Down
6 changes: 6 additions & 0 deletions packages/bun-usockets/src/internal/internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,10 @@ void us_internal_dispatch_ready_poll(struct us_poll_t *p, int error, int eof, in
void us_internal_timer_sweep(us_loop_r loop);
void us_internal_enable_sweep_timer(struct us_loop_t *loop);
void us_internal_disable_sweep_timer(struct us_loop_t *loop);
#ifndef LIBUS_USE_LIBUV
long long us_internal_sweep_timeout_ns(struct us_loop_t *loop);
void us_internal_sweep_if_due(struct us_loop_t *loop);
#endif
void us_internal_free_closed_sockets(us_loop_r loop);
void us_internal_loop_link_group(struct us_loop_t *loop, struct us_socket_group_t *group);
void us_internal_loop_unlink_group(struct us_loop_t *loop, struct us_socket_group_t *group);
Expand Down Expand Up @@ -373,7 +377,9 @@ struct us_internal_callback_t {
int cb_expects_the_loop;
int leave_poll_ready;
void (*cb)(struct us_internal_callback_t *cb);
#ifdef LIBUS_USE_LIBUV
unsigned has_added_timer_to_event_loop;
#endif
};

#endif
Expand Down
15 changes: 11 additions & 4 deletions packages/bun-usockets/src/internal/loop_data.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,13 @@ typedef void* zig_mutex_t;
struct us_quic_socket_context_s;

struct us_internal_loop_data_t {
#ifdef LIBUS_USE_LIBUV
struct us_timer_t *sweep_timer;
#else
/* Absolute monotonic ns of the next sweep, or -1. Folded into the poll
* timeout — no timerfd, no EVFILT_TIMER. */
long long sweep_next_tick_ns;
#endif
int sweep_timer_count;
struct us_internal_async *wakeup_async;
struct us_socket_group_t *head;
Expand All @@ -52,11 +58,12 @@ struct us_internal_loop_data_t {
* the gap between loop_post and getTimeout is sub-µs so storing the
* relative diff is precise enough. */
long long quic_next_tick_us;
/* libuv only: a fallthrough us_timer_t armed to quic_next_tick_us so the
* uv loop wakes for lsquic's time-driven state. POSIX folds the deadline
* into the epoll_pwait2 timeout via getTimeout() instead, so this stays
* NULL there. */
#ifdef LIBUS_USE_LIBUV
/* A fallthrough us_timer_t armed to quic_next_tick_us so the uv loop wakes
* for lsquic's time-driven state. POSIX folds the deadline into the
* epoll_pwait2 timeout via getTimeout() instead. */
struct us_timer_t *quic_timer;
#endif
struct us_socket_group_t *iterator;
char *recv_buf;
char *send_buf;
Expand Down
6 changes: 5 additions & 1 deletion packages/bun-usockets/src/libusockets.h
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,9 @@ void *us_udp_socket_user(struct us_udp_socket_t *s);
/* Binds the UDP socket to an interface and port */
int us_udp_socket_bind(struct us_udp_socket_t *s, const char *hostname, unsigned int port);

/* Public interfaces for timers */
/* Public interfaces for timers. libuv (Windows) only — epoll/kqueue schedules
* on bun.JSC.EventLoopTimer, no file descriptor or syscall. */
#ifdef _WIN32

/* Create a new high precision, low performance timer. May fail and return null */
struct us_timer_t *us_create_timer(us_loop_r loop, int fallthrough, unsigned int ext_size);
Expand All @@ -221,6 +223,8 @@ void us_timer_set(struct us_timer_t *timer, void (*cb)(struct us_timer_t *t), in
/* Returns the loop for this timer */
struct us_loop_t *us_timer_loop(struct us_timer_t *t);

#endif

/* ──────────────────────────────────────────────────────────────────────────
* Socket groups & dispatch
*
Expand Down
Loading
Loading