From 32bcf3dc15b56e42e84b35eebe2fe3478dc38e2b Mon Sep 17 00:00:00 2001 From: Betafer Date: Wed, 19 Aug 2026 08:31:09 +0200 Subject: [PATCH 1/2] feat(receiver): add opt-in zero-copy native Metal path --- TargetBridge-Receiver/TBReceiverC/Makefile | 28 +- .../TBReceiverC/src/decoder.c | 18 +- .../TBReceiverC/src/decoder.h | 9 + .../TBReceiverC/src/display.c | 137 ++++- .../TBReceiverC/src/display.h | 7 + TargetBridge-Receiver/TBReceiverC/src/main.c | 22 +- .../src/tb_native_metal_renderer.h | 52 ++ .../src/tb_native_metal_renderer.m | 572 ++++++++++++++++++ .../tests/test_native_metal_renderer.m | 26 + 9 files changed, 859 insertions(+), 12 deletions(-) create mode 100644 TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.h create mode 100644 TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.m create mode 100644 TargetBridge-Receiver/TBReceiverC/tests/test_native_metal_renderer.m diff --git a/TargetBridge-Receiver/TBReceiverC/Makefile b/TargetBridge-Receiver/TBReceiverC/Makefile index 26ca207..c29eceb 100644 --- a/TargetBridge-Receiver/TBReceiverC/Makefile +++ b/TargetBridge-Receiver/TBReceiverC/Makefile @@ -31,12 +31,16 @@ LDFLAGS += $(shell $(PKG) --libs libavcodec libavutil libswscale sdl2) UNAME_S := $(shell uname -s) ifeq ($(UNAME_S),Darwin) -LDFLAGS += -framework ApplicationServices -framework AppKit -framework CoreFoundation -framework CoreGraphics -framework CoreText -framework VideoToolbox -framework CoreMedia -framework CoreVideo -framework IOSurface -framework CoreServices -framework CoreAudio -framework SystemConfiguration +LDFLAGS += -framework ApplicationServices -framework AppKit -framework CoreFoundation -framework CoreGraphics -framework CoreText -framework VideoToolbox -framework CoreMedia -framework CoreVideo -framework IOSurface -framework CoreServices -framework CoreAudio -framework SystemConfiguration -framework Metal -framework QuartzCore -lc++ endif C_SRC = src/main.c src/net.c src/decoder.c src/display.c src/input_queue.c src/receiver_profile.c src/tb_i18n.c OBJC_SRC = src/tb_gesture_bridge.m src/tb_display_tweaks.m -OBJ = $(C_SRC:.c=.o) $(OBJC_SRC:.m=.o) +OBJCXX_SRC = +ifeq ($(UNAME_S),Darwin) +OBJCXX_SRC += src/tb_native_metal_renderer.m +endif +OBJ = $(C_SRC:.c=.o) $(OBJC_SRC:.m=.o) $(OBJCXX_SRC:.m=.o) BIN = tbreceiver all: $(BIN) @@ -50,8 +54,11 @@ $(BIN): $(OBJ) %.o: %.m $(CC) $(CFLAGS) -fobjc-arc -c -o $@ $< +src/tb_native_metal_renderer.o: src/tb_native_metal_renderer.m src/tb_native_metal_renderer.h + $(CC) $(filter-out -std=c11,$(CFLAGS)) -std=c++17 -x objective-c++ -fobjc-arc -c -o $@ $< + clean: - rm -f $(OBJ) $(BIN) $(TEST_BIN) $(TEST_INPUT_QUEUE_BIN) $(TEST_RECEIVER_PROFILE_BIN) + rm -f $(OBJ) $(BIN) $(TEST_BIN) $(TEST_INPUT_QUEUE_BIN) $(TEST_RECEIVER_PROFILE_BIN) $(TEST_NATIVE_METAL_BIN) # Unit tests for the packet parser. net.c is pure POSIX, so this needs no # ffmpeg/SDL/pkgconf — it runs anywhere, including CI, with no hardware. @@ -63,6 +70,11 @@ endif TEST_BIN = test_net_parser TEST_INPUT_QUEUE_BIN = test_input_queue TEST_RECEIVER_PROFILE_BIN = test_receiver_profile +TEST_NATIVE_METAL_BIN = test_native_metal_renderer +TEST_PLATFORM_BINS = +ifeq ($(UNAME_S),Darwin) +TEST_PLATFORM_BINS += $(TEST_NATIVE_METAL_BIN) +endif test_net_parser: tests/test_net_parser.c src/net.c src/net.h src/proto.h $(CC) $(TEST_CFLAGS) -Isrc tests/test_net_parser.c src/net.c $(TEST_LDFLAGS) -o $@ @@ -73,9 +85,17 @@ $(TEST_INPUT_QUEUE_BIN): tests/test_input_queue.c src/input_queue.c src/input_qu $(TEST_RECEIVER_PROFILE_BIN): tests/test_receiver_profile.c src/receiver_profile.c src/receiver_profile.h $(CC) $(TEST_CFLAGS) -Isrc tests/test_receiver_profile.c src/receiver_profile.c -o $@ -test: $(TEST_BIN) $(TEST_INPUT_QUEUE_BIN) $(TEST_RECEIVER_PROFILE_BIN) +$(TEST_NATIVE_METAL_BIN): tests/test_native_metal_renderer.m src/tb_native_metal_renderer.m src/tb_native_metal_renderer.h + $(CC) -O2 -g -Wall -Wextra -Isrc -std=c++17 -x objective-c++ -fobjc-arc \ + tests/test_native_metal_renderer.m src/tb_native_metal_renderer.m \ + -framework AppKit -framework CoreVideo -framework Metal -framework QuartzCore -lc++ -o $@ + +test: $(TEST_BIN) $(TEST_INPUT_QUEUE_BIN) $(TEST_RECEIVER_PROFILE_BIN) $(TEST_PLATFORM_BINS) ./$(TEST_BIN) ./$(TEST_INPUT_QUEUE_BIN) ./$(TEST_RECEIVER_PROFILE_BIN) +ifeq ($(UNAME_S),Darwin) + ./$(TEST_NATIVE_METAL_BIN) +endif .PHONY: all clean test diff --git a/TargetBridge-Receiver/TBReceiverC/src/decoder.c b/TargetBridge-Receiver/TBReceiverC/src/decoder.c index dd078d0..3d3027b 100644 --- a/TargetBridge-Receiver/TBReceiverC/src/decoder.c +++ b/TargetBridge-Receiver/TBReceiverC/src/decoder.c @@ -46,6 +46,8 @@ struct tb_decoder { tb_frame_cb cb; void *ud; + tb_native_frame_cb native_cb; + void *native_ud; int opened; }; @@ -101,6 +103,14 @@ struct tb_decoder *tb_dec_create(tb_frame_cb cb, void *ud) { return d; } +void tb_dec_set_native_frame_cb(struct tb_decoder *d, + tb_native_frame_cb cb, + void *ud) { + if (!d) return; + d->native_cb = cb; + d->native_ud = ud; +} + int tb_dec_supports_hevc_hwdecode(void) { const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_HEVC); if (!codec) return 0; @@ -368,13 +378,17 @@ int tb_dec_feed_frame(struct tb_decoder *d, const uint8_t *avcc, size_t len) { * Major win on Intel iMac + Radeon (~6× faster in practice). */ if (d->hw_frame->format == d->hw_pix_fmt && d->hw_frame->data[3]) { CVPixelBufferRef pb = (CVPixelBufferRef)d->hw_frame->data[3]; + int w = (int)CVPixelBufferGetWidth(pb); + int h = (int)CVPixelBufferGetHeight(pb); + if (d->native_cb && d->native_cb((void *)pb, w, h, d->native_ud)) { + av_frame_unref(d->hw_frame); + continue; + } if (CVPixelBufferLockBaseAddress(pb, kCVPixelBufferLock_ReadOnly) == 0) { uint8_t *y = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pb, 0); size_t ys = CVPixelBufferGetBytesPerRowOfPlane(pb, 0); uint8_t *uv = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pb, 1); size_t uvs = CVPixelBufferGetBytesPerRowOfPlane(pb, 1); - int w = (int)CVPixelBufferGetWidth(pb); - int h = (int)CVPixelBufferGetHeight(pb); d->cb(y, (int)ys, uv, (int)uvs, w, h, d->ud); CVPixelBufferUnlockBaseAddress(pb, kCVPixelBufferLock_ReadOnly); } diff --git a/TargetBridge-Receiver/TBReceiverC/src/decoder.h b/TargetBridge-Receiver/TBReceiverC/src/decoder.h index b7c2f12..4db4dc0 100644 --- a/TargetBridge-Receiver/TBReceiverC/src/decoder.h +++ b/TargetBridge-Receiver/TBReceiverC/src/decoder.h @@ -16,9 +16,18 @@ typedef void (*tb_frame_cb)(const uint8_t *y, int y_stride, const uint8_t *uv, int uv_stride, int width, int height, void *ud); +/* macOS fast path. The callback receives the VideoToolbox CVPixelBuffer as an + * opaque pointer and returns non-zero when it consumed the frame. Returning + * zero asks the decoder to use its portable CPU-plane callback instead. */ +typedef int (*tb_native_frame_cb)(void *pixel_buffer, + int width, int height, void *ud); + struct tb_decoder; struct tb_decoder *tb_dec_create(tb_frame_cb cb, void *ud); +void tb_dec_set_native_frame_cb(struct tb_decoder *d, + tb_native_frame_cb cb, + void *ud); void tb_dec_destroy(struct tb_decoder *d); int tb_dec_supports_hevc_hwdecode(void); diff --git a/TargetBridge-Receiver/TBReceiverC/src/display.c b/TargetBridge-Receiver/TBReceiverC/src/display.c index 80bdf1d..adf4aba 100644 --- a/TargetBridge-Receiver/TBReceiverC/src/display.c +++ b/TargetBridge-Receiver/TBReceiverC/src/display.c @@ -9,6 +9,7 @@ #include "display.h" #include "tb_i18n.h" #include "tb_gesture_bridge.h" +#include "tb_native_metal_renderer.h" #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #ifndef TB_RECEIVER_VERSION #define TB_RECEIVER_VERSION "3.2.0" @@ -51,6 +53,13 @@ struct tb_display { int cursor_type; int cursor_large; uint32_t last_video_frame_time; +#if defined(__APPLE__) + int metal_native_enabled; + int metal_native_failed_logged; + void *native_metal_renderer; + uint32_t native_stats_tick; + struct tb_native_metal_stats native_stats_previous; +#endif int system_cursor_hidden; char last_ip[64]; @@ -537,6 +546,12 @@ static SDL_Renderer *tb_disp_create_accelerated_renderer(SDL_Window *win) { const char *forced_driver = getenv("TB_RECEIVER_RENDER_DRIVER"); if (forced_driver && forced_driver[0] != '\0') { fprintf(stderr, "[disp] renderer override = %s\n", forced_driver); + /* The native path owns only the video overlay. SDL/OpenGL remains + * underneath for the waiting screen, controls and input loop. */ + if (strcasecmp(forced_driver, "metal-native") == 0 || + strcasecmp(forced_driver, "native-metal") == 0) { + return tb_disp_try_renderer(win, "opengl"); + } return tb_disp_try_renderer(win, forced_driver); } @@ -586,11 +601,32 @@ struct tb_display *tb_disp_create(int fullscreen) { SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709); SDL_RenderSetLogicalSize(d->ren, 0, 0); - /* report which backend SDL picked */ + /* Report which backend SDL picked and enable the experimental direct + * CAMetalLayer path only when explicitly requested. */ SDL_RendererInfo info; if (SDL_GetRendererInfo(d->ren, &info) == 0) { fprintf(stderr, "[disp] renderer = %s\n", info.name); } +#if defined(__APPLE__) + const char *forced_driver = getenv("TB_RECEIVER_RENDER_DRIVER"); + const int native_requested = forced_driver && + (strcasecmp(forced_driver, "metal-native") == 0 || + strcasecmp(forced_driver, "native-metal") == 0); + if (native_requested) { + const char *zero_copy = getenv("TB_RECEIVER_METAL_ZERO_COPY"); + const int disabled = zero_copy && + (strcmp(zero_copy, "0") == 0 || + strcasecmp(zero_copy, "false") == 0 || + strcasecmp(zero_copy, "off") == 0); + if (!disabled) d->native_metal_renderer = tb_native_metal_create(); + if (!disabled && d->native_metal_renderer) { + d->metal_native_enabled = 1; + fprintf(stderr, "[disp] native Metal CAMetalLayer path enabled\n"); + } else if (!disabled) { + fprintf(stderr, "[disp] native Metal unavailable; using OpenGL NV12 upload\n"); + } + } +#endif int win_w = 0, win_h = 0, out_w = 0, out_h = 0; SDL_GetWindowSize(d->win, &win_w, &win_h); @@ -629,6 +665,12 @@ void tb_disp_destroy(struct tb_display *d) { CGDisplayShowCursor(CGMainDisplayID()); d->system_cursor_hidden = 0; } +#if defined(__APPLE__) + if (d->native_metal_renderer) { + tb_native_metal_destroy(d->native_metal_renderer); + d->native_metal_renderer = NULL; + } +#endif tb_disp_destroy_status_texture(d); if (d->tex) SDL_DestroyTexture(d->tex); if (d->ren) SDL_DestroyRenderer(d->ren); @@ -1126,6 +1168,14 @@ void tb_disp_render_nv12(struct tb_display *d, const uint8_t *y, int y_stride, const uint8_t *uv, int uv_stride, int w, int h) { +#if defined(__APPLE__) + if (d && d->native_metal_renderer) { + /* RAW/software-decoded frames have no IOSurface to wrap. Expose the + * proven SDL/OpenGL fallback instead of leaving a stale Metal frame + * above it. */ + tb_native_metal_set_visible(d->native_metal_renderer, 0); + } +#endif if (tb_disp_ensure_texture(d, w, h) < 0) return; tb_disp_set_connection_state(d, 1); @@ -1139,6 +1189,72 @@ void tb_disp_render_nv12(struct tb_display *d, tb_disp_render_current(d); } +#if defined(__APPLE__) +static void tb_disp_log_native_stats(struct tb_display *d) { + const uint32_t now = SDL_GetTicks(); + if (d->native_stats_tick != 0 && now - d->native_stats_tick < 1000) return; + + struct tb_native_metal_stats stats; + tb_native_metal_get_stats(d->native_metal_renderer, &stats); + const uint64_t completed = + stats.completed_frames - d->native_stats_previous.completed_frames; + const uint64_t submitted = + stats.submitted_frames - d->native_stats_previous.submitted_frames; + const uint64_t dropped = + stats.dropped_frames - d->native_stats_previous.dropped_frames; + const double gpuTotal = + stats.gpu_time_ms_total - d->native_stats_previous.gpu_time_ms_total; + fprintf(stderr, + "[metal-native-perf] submitted=%llu completed=%llu dropped=%llu " + "gpuAvg=%.3fms gpuMax=%.3fms\n", + (unsigned long long)submitted, + (unsigned long long)completed, + (unsigned long long)dropped, + completed ? gpuTotal / (double)completed : 0.0, + stats.gpu_time_ms_max); + d->native_stats_previous = stats; + d->native_stats_tick = now; +} +#endif + +int tb_disp_render_native_nv12(struct tb_display *d, + void *pixel_buffer, + int w, int h) { +#if defined(__APPLE__) + if (!d || !pixel_buffer || !d->metal_native_enabled || + !d->native_metal_renderer) return 0; + + tb_disp_set_connection_state(d, 1); + const int result = tb_native_metal_render_nv12( + d->native_metal_renderer, pixel_buffer, + d->cursor_x, d->cursor_y, + d->cursor_source_w, d->cursor_source_h, + d->cursor_visible, d->cursor_type, + d->cursor_large); + if (result >= 0) { + d->last_video_frame_time = SDL_GetTicks(); + tb_disp_log_native_stats(d); + (void)w; + (void)h; + return 1; + } + + d->metal_native_enabled = 0; + tb_native_metal_set_visible(d->native_metal_renderer, 0); + if (!d->metal_native_failed_logged) { + d->metal_native_failed_logged = 1; + fprintf(stderr, + "[disp] native Metal frame failed; falling back to OpenGL NV12 upload\n"); + } +#else + (void)d; + (void)pixel_buffer; + (void)w; + (void)h; +#endif + return 0; +} + void tb_disp_set_cursor(struct tb_display *d, int x, int y, int source_w, int source_h, @@ -1156,6 +1272,20 @@ void tb_disp_set_cursor(struct tb_display *d, uint32_t now = SDL_GetTicks(); if (now - d->last_video_frame_time > 40) { +#if defined(__APPLE__) + if (d->is_connected && d->metal_native_enabled && + d->native_metal_renderer) { + int result = tb_native_metal_render_cursor( + d->native_metal_renderer, + d->cursor_x, d->cursor_y, + d->cursor_source_w, d->cursor_source_h, + d->cursor_visible, d->cursor_type, + d->cursor_large); + if (result >= 0) return; + d->metal_native_enabled = 0; + tb_native_metal_set_visible(d->native_metal_renderer, 0); + } +#endif if (d->is_connected && d->tex) { tb_disp_render_current(d); } @@ -1364,6 +1494,11 @@ static void tb_disp_set_stream_state(struct tb_display *d, int connected, int co if (d->is_connected == connected && d->is_connecting == connecting) return; d->is_connected = connected; d->is_connecting = connecting; +#if defined(__APPLE__) + if (!connected && d->native_metal_renderer) { + tb_native_metal_set_visible(d->native_metal_renderer, 0); + } +#endif tb_disp_refresh_window_mode(d); } diff --git a/TargetBridge-Receiver/TBReceiverC/src/display.h b/TargetBridge-Receiver/TBReceiverC/src/display.h index 901a1e0..3682df2 100644 --- a/TargetBridge-Receiver/TBReceiverC/src/display.h +++ b/TargetBridge-Receiver/TBReceiverC/src/display.h @@ -48,6 +48,13 @@ void tb_disp_render_nv12(struct tb_display *d, const uint8_t *uv, int uv_stride, int w, int h); +/* Present a VideoToolbox CVPixelBuffer through the optional native Metal + * path. Returns non-zero when the frame was handled; portable callers keep + * using tb_disp_render_nv12. */ +int tb_disp_render_native_nv12(struct tb_display *d, + void *pixel_buffer, + int w, int h); + /* Update low-latency local cursor overlay in source-frame coordinates. */ void tb_disp_set_cursor(struct tb_display *d, int x, int y, diff --git a/TargetBridge-Receiver/TBReceiverC/src/main.c b/TargetBridge-Receiver/TBReceiverC/src/main.c index f334413..582804f 100644 --- a/TargetBridge-Receiver/TBReceiverC/src/main.c +++ b/TargetBridge-Receiver/TBReceiverC/src/main.c @@ -895,10 +895,7 @@ static void tb_receiver_apply_input_control_mode(struct app *a, const uint8_t *p /* ---- Callbacks: decoder → display ------------------------------------ */ -static void on_frame(const uint8_t *y, int y_stride, - const uint8_t *uv, int uv_stride, - int w, int h, void *ud) { - struct app *a = (struct app *)ud; +static void on_frame_received(struct app *a, int w, int h) { a->have_video_frame = 1; tb_copy_i18n(a->status_text, sizeof(a->status_text), "receiver.status.stream_active"); { @@ -912,10 +909,24 @@ static void on_frame(const uint8_t *y, int y_stride, snprintf(height_text, sizeof(height_text), "%d", h); tb_format_i18n(a->mode_text, sizeof(a->mode_text), "receiver.mode.receiving", pairs, 2); } - tb_disp_render_nv12(a->disp, y, y_stride, uv, uv_stride, w, h); a->frames++; } +static void on_frame(const uint8_t *y, int y_stride, + const uint8_t *uv, int uv_stride, + int w, int h, void *ud) { + struct app *a = (struct app *)ud; + on_frame_received(a, w, h); + tb_disp_render_nv12(a->disp, y, y_stride, uv, uv_stride, w, h); +} + +static int on_native_frame(void *pixel_buffer, int w, int h, void *ud) { + struct app *a = (struct app *)ud; + if (!tb_disp_render_native_nv12(a->disp, pixel_buffer, w, h)) return 0; + on_frame_received(a, w, h); + return 1; +} + /* Raw passthrough: render received NV12 planes directly, bypassing the decoder. * Payload: [1: format=1(NV12)][BE32 w][BE32 h][BE32 yStride][BE32 uvStride] * [Y plane: yStride*h][CbCr plane: uvStride*(h/2)] */ @@ -1960,6 +1971,7 @@ int main(int argc, char **argv) { a.dec = tb_dec_create(on_frame, &a); if (!a.dec) { fprintf(stderr, "tb_dec_create failed\n"); tb_disp_destroy(a.disp); return 1; } + tb_dec_set_native_frame_cb(a.dec, on_native_frame, &a); tb_parser_init(&a.parser, on_packet, &a); diff --git a/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.h b/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.h new file mode 100644 index 0000000..4593e13 --- /dev/null +++ b/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.h @@ -0,0 +1,52 @@ +/* Direct macOS Metal presentation for VideoToolbox NV12 pixel buffers. */ + +#ifndef TB_NATIVE_METAL_RENDERER_H +#define TB_NATIVE_METAL_RENDERER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct tb_native_metal_stats { + uint64_t submitted_frames; + uint64_t completed_frames; + uint64_t dropped_frames; + double gpu_time_ms_total; + double gpu_time_ms_max; +}; + +void *tb_native_metal_create(void); +void tb_native_metal_destroy(void *renderer); +void tb_native_metal_set_visible(void *renderer, int visible); + +/* Returns 1 when submitted, 0 for a temporary queue/drawable drop and -1 + * when the native renderer cannot handle the frame. */ +int tb_native_metal_render_nv12(void *renderer, + void *pixel_buffer, + int cursor_x, + int cursor_y, + int cursor_source_w, + int cursor_source_h, + int cursor_visible, + int cursor_type, + int cursor_large); + +int tb_native_metal_render_cursor(void *renderer, + int cursor_x, + int cursor_y, + int cursor_source_w, + int cursor_source_h, + int cursor_visible, + int cursor_type, + int cursor_large); + +void tb_native_metal_get_stats(void *renderer, + struct tb_native_metal_stats *stats); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.m b/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.m new file mode 100644 index 0000000..c095ac0 --- /dev/null +++ b/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.m @@ -0,0 +1,572 @@ +#import "tb_native_metal_renderer.h" + +#import +#import +#import +#import +#import +#import + +#include +#include + +@interface TBNativeMetalView : NSView +@end + +@implementation TBNativeMetalView + +- (CALayer *)makeBackingLayer { + return [CAMetalLayer layer]; +} + +- (NSView *)hitTest:(NSPoint)point { + (void)point; + /* SDL's content view remains the sole input target. */ + return nil; +} + +- (BOOL)acceptsFirstResponder { + return NO; +} + +@end + +typedef struct { + vector_float2 drawableSize; + vector_float2 cursorPosition; + vector_float2 cursorSize; + uint32_t cursorVisible; + uint32_t cursorType; + uint32_t fullRange; + uint32_t cursorLarge; +} TBNativeMetalUniforms; + +static const char TBNativeMetalShader[] = R"METAL( +#include +using namespace metal; + +struct RasterData { + float4 position [[position]]; + float2 texCoord; +}; + +struct Uniforms { + float2 drawableSize; + float2 cursorPosition; + float2 cursorSize; + uint cursorVisible; + uint cursorType; + uint fullRange; + uint cursorLarge; +}; + +vertex RasterData tbVideoVertex(uint vertexID [[vertex_id]]) { + constexpr float2 positions[4] = { + float2(-1.0, -1.0), + float2( 1.0, -1.0), + float2(-1.0, 1.0), + float2( 1.0, 1.0) + }; + constexpr float2 texCoords[4] = { + float2(0.0, 1.0), + float2(1.0, 1.0), + float2(0.0, 0.0), + float2(1.0, 0.0) + }; + RasterData out; + out.position = float4(positions[vertexID], 0.0, 1.0); + out.texCoord = texCoords[vertexID]; + return out; +} + +static bool tbArrowMask(float2 p, float expansion) { + const bool head = p.x >= -expansion && + p.y >= -expansion && + p.y <= 31.0 + expansion && + p.x <= p.y * 0.61 + 2.0 + expansion; + const bool stem = p.x >= 7.0 - expansion && + p.x <= 14.0 + expansion && + p.y >= 21.0 - expansion && + p.y <= 41.0 + expansion; + return head || stem; +} + +fragment float4 tbVideoFragment(RasterData in [[stage_in]], + texture2d luma [[texture(0)]], + texture2d chroma [[texture(1)]], + constant Uniforms &uniforms [[buffer(0)]]) { + constexpr sampler linearSampler(coord::normalized, + address::clamp_to_edge, + filter::linear); + float y = luma.sample(linearSampler, in.texCoord).r; + float2 uv = chroma.sample(linearSampler, in.texCoord).rg; + + if (uniforms.fullRange == 0) { + y = max(0.0, (y - (16.0 / 255.0)) * (255.0 / 219.0)); + uv = (uv - 0.5) * (255.0 / 224.0); + } else { + uv -= 0.5; + } + + float3 rgb; + rgb.r = y + 1.5748 * uv.y; + rgb.g = y - 0.1873 * uv.x - 0.4681 * uv.y; + rgb.b = y + 1.8556 * uv.x; + rgb = clamp(rgb, 0.0, 1.0); + + if (uniforms.cursorVisible != 0) { + float2 scale = max(uniforms.cursorSize / float2(24.0, 42.0), float2(0.01)); + float2 cursorPoint = (in.position.xy - uniforms.cursorPosition) / scale; + /* Match the normal macOS-style overlay used by the OpenGL path: + * white outline with a dark body. */ + if (tbArrowMask(cursorPoint, 1.7)) rgb = float3(0.98); + if (tbArrowMask(cursorPoint, 0.0)) rgb = float3(0.03); + } + + return float4(rgb, 1.0); +} +)METAL"; + +@interface TBNativeMetalRenderer : NSObject +- (instancetype)initRenderer; +- (void)setVisible:(BOOL)visible; +- (int)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer + cursorX:(int)cursorX + cursorY:(int)cursorY + cursorWidth:(int)cursorWidth + cursorHeight:(int)cursorHeight + cursorVisible:(BOOL)cursorVisible + cursorType:(int)cursorType + cursorLarge:(BOOL)cursorLarge + rememberFrame:(BOOL)rememberFrame; +- (int)renderCursorX:(int)cursorX + cursorY:(int)cursorY + cursorWidth:(int)cursorWidth + cursorHeight:(int)cursorHeight + cursorVisible:(BOOL)cursorVisible + cursorType:(int)cursorType + cursorLarge:(BOOL)cursorLarge; +- (void)copyStats:(struct tb_native_metal_stats *)stats; +@end + +@implementation TBNativeMetalRenderer { + id _device; + id _commandQueue; + id _pipeline; + CVMetalTextureCacheRef _textureCache; + TBNativeMetalView *_view; + CAMetalLayer *_metalLayer; + dispatch_semaphore_t _inflightSemaphore; + CVPixelBufferRef _latestFrame; + os_unfair_lock _statsLock; + struct tb_native_metal_stats _stats; + BOOL _loggedFirstFrame; +} + +- (instancetype)initRenderer { + self = [super init]; + if (!self) return nil; + + _statsLock = OS_UNFAIR_LOCK_INIT; + _device = MTLCreateSystemDefaultDevice(); + if (!_device) return nil; + _commandQueue = [_device newCommandQueue]; + if (!_commandQueue) return nil; + + NSError *libraryError = nil; + NSString *shaderSource = [NSString stringWithUTF8String:TBNativeMetalShader]; + id library = [_device newLibraryWithSource:shaderSource + options:nil + error:&libraryError]; + if (!library) { + fprintf(stderr, "[metal-native] shader compile failed: %s\n", + libraryError.localizedDescription.UTF8String ?: "unknown error"); + return nil; + } + + MTLRenderPipelineDescriptor *descriptor = [[MTLRenderPipelineDescriptor alloc] init]; + descriptor.label = @"TargetBridge NV12 Pipeline"; + descriptor.vertexFunction = [library newFunctionWithName:@"tbVideoVertex"]; + descriptor.fragmentFunction = [library newFunctionWithName:@"tbVideoFragment"]; + descriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + + NSError *pipelineError = nil; + _pipeline = [_device newRenderPipelineStateWithDescriptor:descriptor error:&pipelineError]; + if (!_pipeline) { + fprintf(stderr, "[metal-native] pipeline creation failed: %s\n", + pipelineError.localizedDescription.UTF8String ?: "unknown error"); + return nil; + } + + CVReturn cacheStatus = CVMetalTextureCacheCreate(kCFAllocatorDefault, + NULL, + _device, + NULL, + &_textureCache); + if (cacheStatus != kCVReturnSuccess || !_textureCache) { + fprintf(stderr, "[metal-native] CVMetalTextureCacheCreate failed: %d\n", + (int)cacheStatus); + return nil; + } + + _inflightSemaphore = dispatch_semaphore_create(3); + fprintf(stderr, "[metal-native] device=%s\n", _device.name.UTF8String ?: "unknown"); + return self; +} + +- (void)dealloc { + [self setVisible:NO]; + if (_textureCache) { + CVMetalTextureCacheFlush(_textureCache, 0); + CFRelease(_textureCache); + _textureCache = NULL; + } +} + +- (NSWindow *)receiverWindow { + NSWindow *best = nil; + CGFloat bestArea = 0.0; + for (NSWindow *window in NSApp.windows) { + if (!window.isVisible) continue; + const CGFloat area = window.frame.size.width * window.frame.size.height; + if (area > bestArea) { + bestArea = area; + best = window; + } + } + return best; +} + +- (BOOL)attachViewIfNeeded { + if (![NSThread isMainThread]) { + fprintf(stderr, "[metal-native] UI attachment requested off main thread\n"); + return NO; + } + + NSWindow *window = [self receiverWindow]; + if (!window) return NO; + NSView *contentView = window.contentView; + if (!contentView) return NO; + if (_view.window == window && _view.superview == contentView && _metalLayer) return YES; + + /* SDL may replace its Cocoa window while entering its fullscreen Space. + * Follow the largest visible receiver window instead of retaining the + * initial 980x620 window forever. */ + if (_view) { + [_view removeFromSuperview]; + _view = nil; + _metalLayer = nil; + } + + _view = [[TBNativeMetalView alloc] initWithFrame:contentView.bounds]; + _view.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable; + _view.wantsLayer = YES; + _view.hidden = YES; + [contentView addSubview:_view positioned:NSWindowAbove relativeTo:nil]; + + _metalLayer = (CAMetalLayer *)_view.layer; + if (![_metalLayer isKindOfClass:CAMetalLayer.class]) return NO; + _metalLayer.device = _device; + _metalLayer.pixelFormat = MTLPixelFormatBGRA8Unorm; + CGColorSpaceRef srgb = CGColorSpaceCreateWithName(kCGColorSpaceSRGB); + if (srgb) { + _metalLayer.colorspace = srgb; + CGColorSpaceRelease(srgb); + } + _metalLayer.framebufferOnly = YES; + _metalLayer.opaque = YES; + _metalLayer.presentsWithTransaction = NO; + _metalLayer.allowsNextDrawableTimeout = YES; + if ([_metalLayer respondsToSelector:@selector(setMaximumDrawableCount:)]) { + _metalLayer.maximumDrawableCount = 3; + } + if ([_metalLayer respondsToSelector:@selector(setDisplaySyncEnabled:)]) { + _metalLayer.displaySyncEnabled = NO; + } + fprintf(stderr, + "[metal-native] attached window=%ld points=%.0fx%.0f scale=%.1f title=%s\n", + (long)window.windowNumber, + contentView.bounds.size.width, + contentView.bounds.size.height, + window.backingScaleFactor, + window.title.UTF8String ?: ""); + return YES; +} + +- (void)updateDrawableSize { + if (!_view || !_metalLayer) return; + NSRect backingBounds = [_view convertRectToBacking:_view.bounds]; + const CGSize size = CGSizeMake(MAX(1.0, backingBounds.size.width), + MAX(1.0, backingBounds.size.height)); + _metalLayer.contentsScale = _view.window.backingScaleFactor; + if (!CGSizeEqualToSize(_metalLayer.drawableSize, size)) { + _metalLayer.drawableSize = size; + fprintf(stderr, "[metal-native] drawable %.0fx%.0f\n", size.width, size.height); + } +} + +- (void)setVisible:(BOOL)visible { + @autoreleasepool { + if (visible) { + if ([self attachViewIfNeeded]) { + [self updateDrawableSize]; + _view.hidden = NO; + } + return; + } + + if (_view) _view.hidden = YES; + if (_latestFrame) { + CVPixelBufferRelease(_latestFrame); + _latestFrame = NULL; + } + } +} + +- (void)recordDrop { + os_unfair_lock_lock(&_statsLock); + _stats.dropped_frames++; + os_unfair_lock_unlock(&_statsLock); +} + +- (int)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer + cursorX:(int)cursorX + cursorY:(int)cursorY + cursorWidth:(int)cursorWidth + cursorHeight:(int)cursorHeight + cursorVisible:(BOOL)cursorVisible + cursorType:(int)cursorType + cursorLarge:(BOOL)cursorLarge + rememberFrame:(BOOL)rememberFrame { + @autoreleasepool { + if (!pixelBuffer || ![self attachViewIfNeeded]) return -1; + [self updateDrawableSize]; + _view.hidden = NO; + + if (rememberFrame && pixelBuffer != _latestFrame) { + CVPixelBufferRetain(pixelBuffer); + if (_latestFrame) CVPixelBufferRelease(_latestFrame); + _latestFrame = pixelBuffer; + } + + if (dispatch_semaphore_wait(_inflightSemaphore, DISPATCH_TIME_NOW) != 0) { + [self recordDrop]; + return 0; + } + + const size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0); + const size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0); + const size_t chromaWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1); + const size_t chromaHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1); + + CVMetalTextureRef lumaRef = NULL; + CVMetalTextureRef chromaRef = NULL; + CVReturn lumaStatus = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, NULL, + MTLPixelFormatR8Unorm, width, height, 0, &lumaRef); + CVReturn chromaStatus = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, NULL, + MTLPixelFormatRG8Unorm, chromaWidth, chromaHeight, 1, &chromaRef); + id lumaTexture = lumaRef ? CVMetalTextureGetTexture(lumaRef) : nil; + id chromaTexture = chromaRef ? CVMetalTextureGetTexture(chromaRef) : nil; + + if (lumaStatus != kCVReturnSuccess || chromaStatus != kCVReturnSuccess || + !lumaTexture || !chromaTexture) { + if (lumaRef) CFRelease(lumaRef); + if (chromaRef) CFRelease(chromaRef); + dispatch_semaphore_signal(_inflightSemaphore); + fprintf(stderr, "[metal-native] CVMetalTexture creation failed: %d/%d\n", + (int)lumaStatus, (int)chromaStatus); + return -1; + } + + id drawable = [_metalLayer nextDrawable]; + if (!drawable) { + CFRelease(lumaRef); + CFRelease(chromaRef); + dispatch_semaphore_signal(_inflightSemaphore); + [self recordDrop]; + return 0; + } + + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = drawable.texture; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 1); + + TBNativeMetalUniforms uniforms; + memset(&uniforms, 0, sizeof(uniforms)); + const CGSize drawableSize = _metalLayer.drawableSize; + uniforms.drawableSize = (vector_float2){(float)drawableSize.width, + (float)drawableSize.height}; + const float sourceWidth = (float)MAX(1, cursorWidth); + const float sourceHeight = (float)MAX(1, cursorHeight); + uniforms.cursorPosition = (vector_float2){ + (float)cursorX * (float)drawableSize.width / sourceWidth, + (float)cursorY * (float)drawableSize.height / sourceHeight + }; + /* Use drawable pixels, not backingScale. Multiplying by Retina scale + * made the Metal cursor twice the size of the author's OpenGL fix. */ + const float cursorBase = drawableSize.width >= 5000.0 + ? (cursorLarge ? 58.0f : 32.0f) + : (cursorLarge ? 44.0f : 24.0f); + uniforms.cursorSize = (vector_float2){cursorBase * 0.75f, + cursorBase * 1.1875f}; + uniforms.cursorVisible = cursorVisible ? 1u : 0u; + uniforms.cursorType = (uint32_t)MAX(0, cursorType); + uniforms.cursorLarge = cursorLarge ? 1u : 0u; + const OSType format = CVPixelBufferGetPixelFormatType(pixelBuffer); + uniforms.fullRange = + format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ? 1u : 0u; + + id commandBuffer = [_commandQueue commandBuffer]; + commandBuffer.label = @"TargetBridge NV12 Frame"; + id encoder = + [commandBuffer renderCommandEncoderWithDescriptor:pass]; + [encoder setRenderPipelineState:_pipeline]; + [encoder setFragmentTexture:lumaTexture atIndex:0]; + [encoder setFragmentTexture:chromaTexture atIndex:1]; + [encoder setFragmentBytes:&uniforms length:sizeof(uniforms) atIndex:0]; + [encoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; + [encoder endEncoding]; + [commandBuffer presentDrawable:drawable]; + + CVPixelBufferRetain(pixelBuffer); + [commandBuffer addCompletedHandler:^(id completed) { + double gpuMS = 0.0; + if (completed.GPUEndTime > completed.GPUStartTime) { + gpuMS = (completed.GPUEndTime - completed.GPUStartTime) * 1000.0; + } + os_unfair_lock_lock(&self->_statsLock); + self->_stats.completed_frames++; + self->_stats.gpu_time_ms_total += gpuMS; + if (gpuMS > self->_stats.gpu_time_ms_max) { + self->_stats.gpu_time_ms_max = gpuMS; + } + os_unfair_lock_unlock(&self->_statsLock); + CVPixelBufferRelease(pixelBuffer); + CFRelease(lumaRef); + CFRelease(chromaRef); + dispatch_semaphore_signal(self->_inflightSemaphore); + }]; + + os_unfair_lock_lock(&_statsLock); + _stats.submitted_frames++; + os_unfair_lock_unlock(&_statsLock); + [commandBuffer commit]; + + if (!_loggedFirstFrame) { + _loggedFirstFrame = YES; + fprintf(stderr, + "[metal-native] first NV12 frame %zux%zu submitted without CPU upload\n", + width, height); + } + return 1; + } +} + +- (int)renderCursorX:(int)cursorX + cursorY:(int)cursorY + cursorWidth:(int)cursorWidth + cursorHeight:(int)cursorHeight + cursorVisible:(BOOL)cursorVisible + cursorType:(int)cursorType + cursorLarge:(BOOL)cursorLarge { + if (!_latestFrame) return 0; + return [self renderPixelBuffer:_latestFrame + cursorX:cursorX + cursorY:cursorY + cursorWidth:cursorWidth + cursorHeight:cursorHeight + cursorVisible:cursorVisible + cursorType:cursorType + cursorLarge:cursorLarge + rememberFrame:NO]; +} + +- (void)copyStats:(struct tb_native_metal_stats *)stats { + if (!stats) return; + os_unfair_lock_lock(&_statsLock); + *stats = _stats; + os_unfair_lock_unlock(&_statsLock); +} + +@end + +void *tb_native_metal_create(void) { + @autoreleasepool { + TBNativeMetalRenderer *renderer = [[TBNativeMetalRenderer alloc] initRenderer]; + return renderer ? (__bridge_retained void *)renderer : NULL; + } +} + +void tb_native_metal_destroy(void *renderer) { + if (!renderer) return; + @autoreleasepool { + TBNativeMetalRenderer *object = CFBridgingRelease(renderer); + [object setVisible:NO]; + } +} + +void tb_native_metal_set_visible(void *renderer, int visible) { + if (!renderer) return; + @autoreleasepool { + [(__bridge TBNativeMetalRenderer *)renderer setVisible:visible ? YES : NO]; + } +} + +int tb_native_metal_render_nv12(void *renderer, + void *pixel_buffer, + int cursor_x, + int cursor_y, + int cursor_source_w, + int cursor_source_h, + int cursor_visible, + int cursor_type, + int cursor_large) { + if (!renderer || !pixel_buffer) return -1; + @autoreleasepool { + return [(__bridge TBNativeMetalRenderer *)renderer + renderPixelBuffer:(CVPixelBufferRef)pixel_buffer + cursorX:cursor_x + cursorY:cursor_y + cursorWidth:cursor_source_w + cursorHeight:cursor_source_h + cursorVisible:cursor_visible ? YES : NO + cursorType:cursor_type + cursorLarge:cursor_large ? YES : NO + rememberFrame:YES]; + } +} + +int tb_native_metal_render_cursor(void *renderer, + int cursor_x, + int cursor_y, + int cursor_source_w, + int cursor_source_h, + int cursor_visible, + int cursor_type, + int cursor_large) { + if (!renderer) return -1; + @autoreleasepool { + return [(__bridge TBNativeMetalRenderer *)renderer + renderCursorX:cursor_x + cursorY:cursor_y + cursorWidth:cursor_source_w + cursorHeight:cursor_source_h + cursorVisible:cursor_visible ? YES : NO + cursorType:cursor_type + cursorLarge:cursor_large ? YES : NO]; + } +} + +void tb_native_metal_get_stats(void *renderer, + struct tb_native_metal_stats *stats) { + if (!stats) return; + memset(stats, 0, sizeof(*stats)); + if (!renderer) return; + @autoreleasepool { + [(__bridge TBNativeMetalRenderer *)renderer copyStats:stats]; + } +} diff --git a/TargetBridge-Receiver/TBReceiverC/tests/test_native_metal_renderer.m b/TargetBridge-Receiver/TBReceiverC/tests/test_native_metal_renderer.m new file mode 100644 index 0000000..8931209 --- /dev/null +++ b/TargetBridge-Receiver/TBReceiverC/tests/test_native_metal_renderer.m @@ -0,0 +1,26 @@ +#import "tb_native_metal_renderer.h" + +#import + +#include + +int main(void) { + @autoreleasepool { + id device = MTLCreateSystemDefaultDevice(); + if (!device) { + printf("native Metal renderer test: skipped (no Metal device)\n"); + return 0; + } + + void *renderer = tb_native_metal_create(); + if (!renderer) { + fprintf(stderr, + "native Metal renderer test: device exists but renderer/shader creation failed\n"); + return 1; + } + tb_native_metal_destroy(renderer); + printf("native Metal renderer test: device=%s shader/pipeline passed\n", + device.name.UTF8String ?: "unknown"); + return 0; + } +} From 1d2d62cdbda3b6eccf24adfa9336a8bb53b07b77 Mon Sep 17 00:00:00 2001 From: Betafer Date: Fri, 21 Aug 2026 01:31:00 +0200 Subject: [PATCH 2/2] feat(receiver): validate native Metal and preserve Display P3 --- TargetBridge-Receiver/TBReceiverC/Makefile | 32 +++- .../benchmark_native_metal_renderer.m | 174 ++++++++++++++++++ .../benchmark_sdl_opengl_renderer.c | 164 +++++++++++++++++ .../TBReceiverC/src/display.c | 82 ++++++++- .../TBReceiverC/src/renderer_policy.c | 37 ++++ .../TBReceiverC/src/renderer_policy.h | 34 ++++ .../src/tb_native_metal_renderer.h | 6 + .../src/tb_native_metal_renderer.m | 85 ++++++++- .../tests/test_native_metal_renderer.m | 38 ++++ .../TBReceiverC/tests/test_renderer_policy.c | 88 +++++++++ 10 files changed, 726 insertions(+), 14 deletions(-) create mode 100644 TargetBridge-Receiver/TBReceiverC/benchmarks/benchmark_native_metal_renderer.m create mode 100644 TargetBridge-Receiver/TBReceiverC/benchmarks/benchmark_sdl_opengl_renderer.c create mode 100644 TargetBridge-Receiver/TBReceiverC/src/renderer_policy.c create mode 100644 TargetBridge-Receiver/TBReceiverC/src/renderer_policy.h create mode 100644 TargetBridge-Receiver/TBReceiverC/tests/test_renderer_policy.c diff --git a/TargetBridge-Receiver/TBReceiverC/Makefile b/TargetBridge-Receiver/TBReceiverC/Makefile index c29eceb..97f8f12 100644 --- a/TargetBridge-Receiver/TBReceiverC/Makefile +++ b/TargetBridge-Receiver/TBReceiverC/Makefile @@ -34,7 +34,7 @@ ifeq ($(UNAME_S),Darwin) LDFLAGS += -framework ApplicationServices -framework AppKit -framework CoreFoundation -framework CoreGraphics -framework CoreText -framework VideoToolbox -framework CoreMedia -framework CoreVideo -framework IOSurface -framework CoreServices -framework CoreAudio -framework SystemConfiguration -framework Metal -framework QuartzCore -lc++ endif -C_SRC = src/main.c src/net.c src/decoder.c src/display.c src/input_queue.c src/receiver_profile.c src/tb_i18n.c +C_SRC = src/main.c src/net.c src/decoder.c src/display.c src/input_queue.c src/receiver_profile.c src/renderer_policy.c src/tb_i18n.c OBJC_SRC = src/tb_gesture_bridge.m src/tb_display_tweaks.m OBJCXX_SRC = ifeq ($(UNAME_S),Darwin) @@ -58,7 +58,7 @@ src/tb_native_metal_renderer.o: src/tb_native_metal_renderer.m src/tb_native_met $(CC) $(filter-out -std=c11,$(CFLAGS)) -std=c++17 -x objective-c++ -fobjc-arc -c -o $@ $< clean: - rm -f $(OBJ) $(BIN) $(TEST_BIN) $(TEST_INPUT_QUEUE_BIN) $(TEST_RECEIVER_PROFILE_BIN) $(TEST_NATIVE_METAL_BIN) + rm -f $(OBJ) $(BIN) $(TEST_BIN) $(TEST_INPUT_QUEUE_BIN) $(TEST_RECEIVER_PROFILE_BIN) $(TEST_RENDERER_POLICY_BIN) $(TEST_NATIVE_METAL_BIN) $(BENCH_NATIVE_METAL_BIN) $(BENCH_OPENGL_BIN) # Unit tests for the packet parser. net.c is pure POSIX, so this needs no # ffmpeg/SDL/pkgconf — it runs anywhere, including CI, with no hardware. @@ -70,7 +70,10 @@ endif TEST_BIN = test_net_parser TEST_INPUT_QUEUE_BIN = test_input_queue TEST_RECEIVER_PROFILE_BIN = test_receiver_profile +TEST_RENDERER_POLICY_BIN = test_renderer_policy TEST_NATIVE_METAL_BIN = test_native_metal_renderer +BENCH_NATIVE_METAL_BIN = benchmark_native_metal_renderer +BENCH_OPENGL_BIN = benchmark_sdl_opengl_renderer TEST_PLATFORM_BINS = ifeq ($(UNAME_S),Darwin) TEST_PLATFORM_BINS += $(TEST_NATIVE_METAL_BIN) @@ -85,17 +88,36 @@ $(TEST_INPUT_QUEUE_BIN): tests/test_input_queue.c src/input_queue.c src/input_qu $(TEST_RECEIVER_PROFILE_BIN): tests/test_receiver_profile.c src/receiver_profile.c src/receiver_profile.h $(CC) $(TEST_CFLAGS) -Isrc tests/test_receiver_profile.c src/receiver_profile.c -o $@ +$(TEST_RENDERER_POLICY_BIN): tests/test_renderer_policy.c src/renderer_policy.c src/renderer_policy.h + $(CC) $(TEST_CFLAGS) -Isrc tests/test_renderer_policy.c src/renderer_policy.c -lm -o $@ + $(TEST_NATIVE_METAL_BIN): tests/test_native_metal_renderer.m src/tb_native_metal_renderer.m src/tb_native_metal_renderer.h $(CC) -O2 -g -Wall -Wextra -Isrc -std=c++17 -x objective-c++ -fobjc-arc \ tests/test_native_metal_renderer.m src/tb_native_metal_renderer.m \ - -framework AppKit -framework CoreVideo -framework Metal -framework QuartzCore -lc++ -o $@ + -framework AppKit -framework CoreVideo -framework IOSurface -framework Metal -framework QuartzCore -lc++ -o $@ + +$(BENCH_NATIVE_METAL_BIN): benchmarks/benchmark_native_metal_renderer.m src/tb_native_metal_renderer.m src/tb_native_metal_renderer.h src/renderer_policy.c src/renderer_policy.h + $(CC) -O2 -g -Wall -Wextra -Isrc -std=c++17 -x objective-c++ -fobjc-arc \ + benchmarks/benchmark_native_metal_renderer.m src/tb_native_metal_renderer.m src/renderer_policy.c \ + -framework AppKit -framework CoreVideo -framework IOSurface -framework Metal -framework QuartzCore -lc++ -o $@ + +benchmark-metal: $(BENCH_NATIVE_METAL_BIN) + ./$(BENCH_NATIVE_METAL_BIN) + +$(BENCH_OPENGL_BIN): benchmarks/benchmark_sdl_opengl_renderer.c + $(CC) -O2 -g -Wall -Wextra $(shell $(PKG) --cflags sdl2) \ + benchmarks/benchmark_sdl_opengl_renderer.c $(shell $(PKG) --libs sdl2) -o $@ + +benchmark-opengl: $(BENCH_OPENGL_BIN) + ./$(BENCH_OPENGL_BIN) -test: $(TEST_BIN) $(TEST_INPUT_QUEUE_BIN) $(TEST_RECEIVER_PROFILE_BIN) $(TEST_PLATFORM_BINS) +test: $(TEST_BIN) $(TEST_INPUT_QUEUE_BIN) $(TEST_RECEIVER_PROFILE_BIN) $(TEST_RENDERER_POLICY_BIN) $(TEST_PLATFORM_BINS) ./$(TEST_BIN) ./$(TEST_INPUT_QUEUE_BIN) ./$(TEST_RECEIVER_PROFILE_BIN) + ./$(TEST_RENDERER_POLICY_BIN) ifeq ($(UNAME_S),Darwin) ./$(TEST_NATIVE_METAL_BIN) endif -.PHONY: all clean test +.PHONY: all clean test benchmark-metal benchmark-opengl diff --git a/TargetBridge-Receiver/TBReceiverC/benchmarks/benchmark_native_metal_renderer.m b/TargetBridge-Receiver/TBReceiverC/benchmarks/benchmark_native_metal_renderer.m new file mode 100644 index 0000000..8dac784 --- /dev/null +++ b/TargetBridge-Receiver/TBReceiverC/benchmarks/benchmark_native_metal_renderer.m @@ -0,0 +1,174 @@ +#import "renderer_policy.h" +#import "tb_native_metal_renderer.h" + +#import +#import +#import + +#include +#include +#include + +static CVPixelBufferRef make_frame(size_t width, size_t height) { + CVPixelBufferRef pixelBuffer = NULL; + CFDictionaryRef attributes = (__bridge CFDictionaryRef)@{ + (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, + (__bridge NSString *)kCVPixelBufferMetalCompatibilityKey: @YES + }; + CVReturn status = CVPixelBufferCreate( + kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + attributes, + &pixelBuffer); + if (status != kCVReturnSuccess || !pixelBuffer) return NULL; + + if (CVPixelBufferLockBaseAddress(pixelBuffer, 0) == kCVReturnSuccess) { + memset(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0), + 128, + CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0) * + CVPixelBufferGetHeightOfPlane(pixelBuffer, 0)); + memset(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1), + 128, + CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1) * + CVPixelBufferGetHeightOfPlane(pixelBuffer, 1)); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + } + CVBufferSetAttachment(pixelBuffer, + kCVImageBufferColorPrimariesKey, + kCVImageBufferColorPrimaries_P3_D65, + kCVAttachmentMode_ShouldPropagate); + CVBufferSetAttachment(pixelBuffer, + kCVImageBufferTransferFunctionKey, + kCVImageBufferTransferFunction_ITU_R_709_2, + kCVAttachmentMode_ShouldPropagate); + CVBufferSetAttachment(pixelBuffer, + kCVImageBufferYCbCrMatrixKey, + kCVImageBufferYCbCrMatrix_ITU_R_709_2, + kCVAttachmentMode_ShouldPropagate); + return pixelBuffer; +} + +int main(int argc, const char *argv[]) { + @autoreleasepool { + const int width = argc > 1 ? atoi(argv[1]) : 4096; + const int height = argc > 2 ? atoi(argv[2]) : 2304; + const int frameCount = argc > 3 ? atoi(argv[3]) : 180; + const int targetFPS = argc > 4 ? atoi(argv[4]) : 60; + if (width <= 0 || height <= 0 || frameCount < 120 || targetFPS <= 0) { + fprintf(stderr, "usage: benchmark_native_metal_renderer [width height frames>=120 fps]\n"); + return 64; + } + if (!MTLCreateSystemDefaultDevice()) { + fprintf(stderr, "TB_METAL_BENCHMARK result=unavailable reason=no-metal-device\n"); + return 69; + } + + [NSApplication sharedApplication]; + [NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory]; + const CGFloat windowWidth = MAX(1.0, (CGFloat)width / 2.0); + const CGFloat windowHeight = MAX(1.0, (CGFloat)height / 2.0); + NSWindow *window = [[NSWindow alloc] + initWithContentRect:NSMakeRect(0, 0, windowWidth, windowHeight) + styleMask:NSWindowStyleMaskBorderless + backing:NSBackingStoreBuffered + defer:NO]; + window.title = @"TargetBridge Metal Benchmark"; + /* Command-line AppKit harnesses own this window through ARC. Avoid the + * legacy close-time self-release, which would otherwise release it a + * second time when the autorelease pool drains. */ + window.releasedWhenClosed = NO; + window.alphaValue = 0.02; + [window orderFront:nil]; + [[NSRunLoop currentRunLoop] + runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; + + CVPixelBufferRef pixelBuffer = make_frame((size_t)width, (size_t)height); + void *renderer = tb_native_metal_create(); + if (!pixelBuffer || !renderer) { + fprintf(stderr, "TB_METAL_BENCHMARK result=failed reason=setup\n"); + if (pixelBuffer) CVPixelBufferRelease(pixelBuffer); + if (renderer) tb_native_metal_destroy(renderer); + [window close]; + return 70; + } + + const CFTimeInterval started = CACurrentMediaTime(); + double submitTimeMsTotal = 0.0; + double submitTimeMsMax = 0.0; + for (int frame = 0; frame < frameCount; frame++) { + const CFTimeInterval submitStarted = CACurrentMediaTime(); + (void)tb_native_metal_render_nv12( + renderer, + pixelBuffer, + (frame * 13) % width, + (frame * 7) % height, + width, + height, + 1, + 0, + 0); + const double submitTimeMs = + (CACurrentMediaTime() - submitStarted) * 1000.0; + submitTimeMsTotal += submitTimeMs; + if (submitTimeMs > submitTimeMsMax) submitTimeMsMax = submitTimeMs; + const CFTimeInterval target = started + + (CFTimeInterval)(frame + 1) / (CFTimeInterval)targetFPS; + const CFTimeInterval delay = target - CACurrentMediaTime(); + if (delay > 0) { + [[NSRunLoop currentRunLoop] + runUntilDate:[NSDate dateWithTimeIntervalSinceNow:delay]]; + } + } + + struct tb_native_metal_stats stats; + const CFTimeInterval completionDeadline = CACurrentMediaTime() + 3.0; + do { + tb_native_metal_get_stats(renderer, &stats); + if (stats.completed_frames >= stats.submitted_frames) break; + [[NSRunLoop currentRunLoop] + runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.01]]; + } while (CACurrentMediaTime() < completionDeadline); + + tb_native_metal_get_stats(renderer, &stats); + const struct tb_renderer_health_sample sample = { + stats.submitted_frames, + stats.completed_frames, + stats.dropped_frames, + stats.gpu_time_ms_total + }; + const struct tb_renderer_health_result health = + tb_renderer_evaluate_health(&sample); + const double elapsed = CACurrentMediaTime() - started; + const char *decision = health.decision == TB_RENDERER_HEALTH_KEEP_METAL + ? "metal" + : health.decision == TB_RENDERER_HEALTH_FALLBACK_OPENGL + ? "opengl" + : "insufficient-sample"; + printf( + "TB_METAL_BENCHMARK result=%s size=%dx%d requested=%d targetFPS=%d " + "elapsed=%.3fs submitted=%llu completed=%llu dropped=%llu " + "submitAvg=%.3fms submitMax=%.3fms gpuAvg=%.3fms gpuMax=%.3fms " + "color=%s\n", + decision, + width, + height, + frameCount, + targetFPS, + elapsed, + (unsigned long long)stats.submitted_frames, + (unsigned long long)stats.completed_frames, + (unsigned long long)stats.dropped_frames, + submitTimeMsTotal / (double)frameCount, + submitTimeMsMax, + health.gpu_average_ms, + stats.gpu_time_ms_max, + tb_native_metal_color_space_name(renderer)); + + CVPixelBufferRelease(pixelBuffer); + tb_native_metal_destroy(renderer); + [window close]; + return health.decision == TB_RENDERER_HEALTH_KEEP_METAL ? 0 : 2; + } +} diff --git a/TargetBridge-Receiver/TBReceiverC/benchmarks/benchmark_sdl_opengl_renderer.c b/TargetBridge-Receiver/TBReceiverC/benchmarks/benchmark_sdl_opengl_renderer.c new file mode 100644 index 0000000..8a57ea9 --- /dev/null +++ b/TargetBridge-Receiver/TBReceiverC/benchmarks/benchmark_sdl_opengl_renderer.c @@ -0,0 +1,164 @@ +#include + +#include +#include +#include +#include + +static double elapsed_ms(uint64_t started, uint64_t frequency) { + return (double)(SDL_GetPerformanceCounter() - started) * 1000.0 / + (double)frequency; +} + +int main(int argc, char **argv) { + const int width = argc > 1 ? atoi(argv[1]) : 4096; + const int height = argc > 2 ? atoi(argv[2]) : 2304; + const int frame_count = argc > 3 ? atoi(argv[3]) : 180; + const int target_fps = argc > 4 ? atoi(argv[4]) : 60; + if (width <= 0 || height <= 0 || frame_count < 120 || target_fps <= 0) { + fprintf(stderr, + "usage: benchmark_sdl_opengl_renderer " + "[width height frames>=120 fps]\n"); + return 64; + } + + SDL_SetHint(SDL_HINT_RENDER_DRIVER, "opengl"); + SDL_SetHint(SDL_HINT_RENDER_VSYNC, "0"); + if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_TIMER) != 0) { + fprintf(stderr, "TB_OPENGL_BENCHMARK result=failed reason=init error=%s\n", + SDL_GetError()); + return 70; + } + + SDL_Window *window = SDL_CreateWindow( + "TargetBridge OpenGL Benchmark", + SDL_WINDOWPOS_UNDEFINED, + SDL_WINDOWPOS_UNDEFINED, + (width + 1) / 2, + (height + 1) / 2, + SDL_WINDOW_BORDERLESS | SDL_WINDOW_ALLOW_HIGHDPI); + if (!window) { + fprintf(stderr, "TB_OPENGL_BENCHMARK result=failed reason=window error=%s\n", + SDL_GetError()); + SDL_Quit(); + return 70; + } + (void)SDL_SetWindowOpacity(window, 0.02f); + + SDL_Renderer *renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED); + SDL_RendererInfo renderer_info; + memset(&renderer_info, 0, sizeof(renderer_info)); + int output_width = 0; + int output_height = 0; + if (!renderer || SDL_GetRendererInfo(renderer, &renderer_info) != 0 || + !renderer_info.name || strcmp(renderer_info.name, "opengl") != 0) { + fprintf(stderr, + "TB_OPENGL_BENCHMARK result=failed reason=renderer " + "selected=%s error=%s\n", + renderer_info.name ? renderer_info.name : "unavailable", + SDL_GetError()); + if (renderer) SDL_DestroyRenderer(renderer); + SDL_DestroyWindow(window); + SDL_Quit(); + return 70; + } + if (SDL_GetRendererOutputSize(renderer, &output_width, &output_height) != 0) { + fprintf(stderr, + "TB_OPENGL_BENCHMARK result=failed reason=output-size error=%s\n", + SDL_GetError()); + SDL_DestroyRenderer(renderer); + SDL_DestroyWindow(window); + SDL_Quit(); + return 70; + } + + SDL_Texture *texture = SDL_CreateTexture( + renderer, + SDL_PIXELFORMAT_NV12, + SDL_TEXTUREACCESS_STREAMING, + width, + height); + const size_t y_size = (size_t)width * (size_t)height; + const size_t uv_size = y_size / 2; + uint8_t *y_plane = malloc(y_size); + uint8_t *uv_plane = malloc(uv_size); + if (!texture || !y_plane || !uv_plane) { + fprintf(stderr, + "TB_OPENGL_BENCHMARK result=failed reason=frame-setup error=%s\n", + SDL_GetError()); + free(y_plane); + free(uv_plane); + if (texture) SDL_DestroyTexture(texture); + SDL_DestroyRenderer(renderer); + SDL_DestroyWindow(window); + SDL_Quit(); + return 70; + } + memset(y_plane, 128, y_size); + memset(uv_plane, 128, uv_size); + + const uint64_t frequency = SDL_GetPerformanceFrequency(); + const uint64_t started = SDL_GetPerformanceCounter(); + double frame_time_ms_total = 0.0; + double frame_time_ms_max = 0.0; + int completed = 0; + for (int frame = 0; frame < frame_count; frame++) { + const uint64_t frame_started = SDL_GetPerformanceCounter(); + const int update_result = SDL_UpdateNVTexture( + texture, + NULL, + y_plane, + width, + uv_plane, + width); + const int clear_result = SDL_RenderClear(renderer); + const int copy_result = SDL_RenderCopy(renderer, texture, NULL, NULL); + SDL_RenderPresent(renderer); + const double frame_time_ms = elapsed_ms(frame_started, frequency); + frame_time_ms_total += frame_time_ms; + if (frame_time_ms > frame_time_ms_max) frame_time_ms_max = frame_time_ms; + if (update_result != 0 || clear_result != 0 || copy_result != 0) { + fprintf(stderr, + "TB_OPENGL_BENCHMARK result=failed reason=frame frame=%d " + "error=%s\n", + frame, + SDL_GetError()); + break; + } + completed++; + SDL_PumpEvents(); + + const double target_ms = + (double)(frame + 1) * 1000.0 / (double)target_fps; + const double wait_ms = target_ms - elapsed_ms(started, frequency); + if (wait_ms >= 1.0) SDL_Delay((uint32_t)wait_ms); + } + const double total_elapsed_ms = elapsed_ms(started, frequency); + const double uploaded_mib = + (double)(y_size + uv_size) * (double)completed / (1024.0 * 1024.0); + printf( + "TB_OPENGL_BENCHMARK result=%s driver=%s size=%dx%d output=%dx%d requested=%d " + "targetFPS=%d elapsed=%.3fs completed=%d frameCallAvg=%.3fms " + "frameCallMax=%.3fms cpuUpload=%.1fMiB\n", + completed == frame_count ? "opengl" : "failed", + renderer_info.name, + width, + height, + output_width, + output_height, + frame_count, + target_fps, + total_elapsed_ms / 1000.0, + completed, + completed ? frame_time_ms_total / (double)completed : 0.0, + frame_time_ms_max, + uploaded_mib); + + free(y_plane); + free(uv_plane); + SDL_DestroyTexture(texture); + SDL_DestroyRenderer(renderer); + SDL_DestroyWindow(window); + SDL_Quit(); + return completed == frame_count ? 0 : 2; +} diff --git a/TargetBridge-Receiver/TBReceiverC/src/display.c b/TargetBridge-Receiver/TBReceiverC/src/display.c index adf4aba..9e8835e 100644 --- a/TargetBridge-Receiver/TBReceiverC/src/display.c +++ b/TargetBridge-Receiver/TBReceiverC/src/display.c @@ -7,6 +7,7 @@ */ #include "display.h" +#include "renderer_policy.h" #include "tb_i18n.h" #include "tb_gesture_bridge.h" #include "tb_native_metal_renderer.h" @@ -55,10 +56,13 @@ struct tb_display { uint32_t last_video_frame_time; #if defined(__APPLE__) int metal_native_enabled; + int metal_native_auto; + int metal_native_auto_decided; int metal_native_failed_logged; void *native_metal_renderer; uint32_t native_stats_tick; struct tb_native_metal_stats native_stats_previous; + struct tb_native_metal_stats native_auto_baseline; #endif int system_cursor_hidden; @@ -549,7 +553,9 @@ static SDL_Renderer *tb_disp_create_accelerated_renderer(SDL_Window *win) { /* The native path owns only the video overlay. SDL/OpenGL remains * underneath for the waiting screen, controls and input loop. */ if (strcasecmp(forced_driver, "metal-native") == 0 || - strcasecmp(forced_driver, "native-metal") == 0) { + strcasecmp(forced_driver, "native-metal") == 0 || + strcasecmp(forced_driver, "auto") == 0 || + strcasecmp(forced_driver, "adaptive") == 0) { return tb_disp_try_renderer(win, "opengl"); } return tb_disp_try_renderer(win, forced_driver); @@ -601,17 +607,22 @@ struct tb_display *tb_disp_create(int fullscreen) { SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_BT709); SDL_RenderSetLogicalSize(d->ren, 0, 0); - /* Report which backend SDL picked and enable the experimental direct - * CAMetalLayer path only when explicitly requested. */ + /* Report which backend SDL picked. Adaptive mode validates the direct + * CAMetalLayer path with real frames and retains OpenGL underneath as an + * immediate fallback. */ SDL_RendererInfo info; if (SDL_GetRendererInfo(d->ren, &info) == 0) { fprintf(stderr, "[disp] renderer = %s\n", info.name); } #if defined(__APPLE__) const char *forced_driver = getenv("TB_RECEIVER_RENDER_DRIVER"); - const int native_requested = forced_driver && + const int native_forced = forced_driver && (strcasecmp(forced_driver, "metal-native") == 0 || strcasecmp(forced_driver, "native-metal") == 0); + const int native_auto = forced_driver && + (strcasecmp(forced_driver, "auto") == 0 || + strcasecmp(forced_driver, "adaptive") == 0); + const int native_requested = native_forced || native_auto; if (native_requested) { const char *zero_copy = getenv("TB_RECEIVER_METAL_ZERO_COPY"); const int disabled = zero_copy && @@ -621,7 +632,11 @@ struct tb_display *tb_disp_create(int fullscreen) { if (!disabled) d->native_metal_renderer = tb_native_metal_create(); if (!disabled && d->native_metal_renderer) { d->metal_native_enabled = 1; - fprintf(stderr, "[disp] native Metal CAMetalLayer path enabled\n"); + d->metal_native_auto = native_auto; + fprintf(stderr, + native_auto + ? "[disp] adaptive renderer: validating native Metal, OpenGL fallback armed\n" + : "[disp] native Metal CAMetalLayer path enabled\n"); } else if (!disabled) { fprintf(stderr, "[disp] native Metal unavailable; using OpenGL NV12 upload\n"); } @@ -1215,6 +1230,62 @@ static void tb_disp_log_native_stats(struct tb_display *d) { d->native_stats_previous = stats; d->native_stats_tick = now; } + +static uint64_t tb_disp_counter_delta(uint64_t value, uint64_t baseline) { + return value >= baseline ? value - baseline : 0; +} + +static void tb_disp_evaluate_native_auto(struct tb_display *d) { + if (!d || !d->metal_native_auto || d->metal_native_auto_decided || + !d->metal_native_enabled || !d->native_metal_renderer) return; + + struct tb_native_metal_stats stats; + tb_native_metal_get_stats(d->native_metal_renderer, &stats); + if (d->native_auto_baseline.submitted_frames == 0 && + d->native_auto_baseline.completed_frames == 0 && + d->native_auto_baseline.dropped_frames == 0) { + d->native_auto_baseline = stats; + return; + } + + const struct tb_renderer_health_sample sample = { + tb_disp_counter_delta(stats.submitted_frames, + d->native_auto_baseline.submitted_frames), + tb_disp_counter_delta(stats.completed_frames, + d->native_auto_baseline.completed_frames), + tb_disp_counter_delta(stats.dropped_frames, + d->native_auto_baseline.dropped_frames), + stats.gpu_time_ms_total >= d->native_auto_baseline.gpu_time_ms_total + ? stats.gpu_time_ms_total - d->native_auto_baseline.gpu_time_ms_total + : 0.0 + }; + const struct tb_renderer_health_result result = + tb_renderer_evaluate_health(&sample); + if (result.decision == TB_RENDERER_HEALTH_WAIT) return; + + d->metal_native_auto_decided = 1; + if (result.decision == TB_RENDERER_HEALTH_KEEP_METAL) { + fprintf(stderr, + "[disp] adaptive renderer selected native Metal: " + "completed=%llu gpuAvg=%.3fms drops=%.1f%%\n", + (unsigned long long)sample.completed_frames, + result.gpu_average_ms, + result.drop_ratio * 100.0); + return; + } + + d->metal_native_enabled = 0; + tb_native_metal_set_visible(d->native_metal_renderer, 0); + fprintf(stderr, + "[disp] adaptive renderer selected OpenGL fallback: " + "submitted=%llu completed=%llu outstanding=%llu " + "gpuAvg=%.3fms drops=%.1f%%\n", + (unsigned long long)sample.submitted_frames, + (unsigned long long)sample.completed_frames, + (unsigned long long)result.outstanding_frames, + result.gpu_average_ms, + result.drop_ratio * 100.0); +} #endif int tb_disp_render_native_nv12(struct tb_display *d, @@ -1234,6 +1305,7 @@ int tb_disp_render_native_nv12(struct tb_display *d, if (result >= 0) { d->last_video_frame_time = SDL_GetTicks(); tb_disp_log_native_stats(d); + tb_disp_evaluate_native_auto(d); (void)w; (void)h; return 1; diff --git a/TargetBridge-Receiver/TBReceiverC/src/renderer_policy.c b/TargetBridge-Receiver/TBReceiverC/src/renderer_policy.c new file mode 100644 index 0000000..04d1d1c --- /dev/null +++ b/TargetBridge-Receiver/TBReceiverC/src/renderer_policy.c @@ -0,0 +1,37 @@ +#include "renderer_policy.h" + +#include + +struct tb_renderer_health_result tb_renderer_evaluate_health( + const struct tb_renderer_health_sample *sample) { + struct tb_renderer_health_result result; + memset(&result, 0, sizeof(result)); + result.decision = TB_RENDERER_HEALTH_WAIT; + if (!sample) return result; + + result.attempted_frames = sample->submitted_frames + sample->dropped_frames; + result.outstanding_frames = sample->submitted_frames > sample->completed_frames + ? sample->submitted_frames - sample->completed_frames + : 0; + result.drop_ratio = result.attempted_frames + ? (double)sample->dropped_frames / (double)result.attempted_frames + : 0.0; + result.gpu_average_ms = sample->completed_frames + ? sample->gpu_time_ms_total / (double)sample->completed_frames + : 0.0; + + if (sample->completed_frames < 120 && result.attempted_frames < 150) { + return result; + } + + const int gpu_healthy = result.gpu_average_ms <= 0.0 || + result.gpu_average_ms <= 12.0; + const int healthy = sample->completed_frames >= 120 && + gpu_healthy && + result.drop_ratio <= 0.08 && + result.outstanding_frames <= 6; + result.decision = healthy + ? TB_RENDERER_HEALTH_KEEP_METAL + : TB_RENDERER_HEALTH_FALLBACK_OPENGL; + return result; +} diff --git a/TargetBridge-Receiver/TBReceiverC/src/renderer_policy.h b/TargetBridge-Receiver/TBReceiverC/src/renderer_policy.h new file mode 100644 index 0000000..9c88fd1 --- /dev/null +++ b/TargetBridge-Receiver/TBReceiverC/src/renderer_policy.h @@ -0,0 +1,34 @@ +#ifndef TB_RENDERER_POLICY_H +#define TB_RENDERER_POLICY_H + +#include + +enum tb_renderer_health_decision { + TB_RENDERER_HEALTH_WAIT = 0, + TB_RENDERER_HEALTH_KEEP_METAL = 1, + TB_RENDERER_HEALTH_FALLBACK_OPENGL = -1 +}; + +struct tb_renderer_health_sample { + uint64_t submitted_frames; + uint64_t completed_frames; + uint64_t dropped_frames; + double gpu_time_ms_total; +}; + +struct tb_renderer_health_result { + enum tb_renderer_health_decision decision; + uint64_t attempted_frames; + uint64_t outstanding_frames; + double drop_ratio; + double gpu_average_ms; +}; + +/* Decide only after a meaningful real-frame sample. Metal must complete at + * least 120 frames, average no more than 12 ms of GPU time, drop no more than + * 8% of attempts and leave no more than 6 command buffers outstanding. A zero + * GPU duration is treated as unavailable because some drivers omit timing. */ +struct tb_renderer_health_result tb_renderer_evaluate_health( + const struct tb_renderer_health_sample *sample); + +#endif /* TB_RENDERER_POLICY_H */ diff --git a/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.h b/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.h index 4593e13..976cdd1 100644 --- a/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.h +++ b/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.h @@ -45,6 +45,12 @@ int tb_native_metal_render_cursor(void *renderer, void tb_native_metal_get_stats(void *renderer, struct tb_native_metal_stats *stats); +/* Color diagnostics. The renderer tags its CAMetalLayer as Display P3 only + * when the decoded CVPixelBuffer carries P3-D65 primaries; all other and + * untagged frames use the conservative sRGB path. */ +const char *tb_native_metal_pixel_buffer_color_space(void *pixel_buffer); +const char *tb_native_metal_color_space_name(void *renderer); + #ifdef __cplusplus } #endif diff --git a/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.m b/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.m index c095ac0..4c41377 100644 --- a/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.m +++ b/TargetBridge-Receiver/TBReceiverC/src/tb_native_metal_renderer.m @@ -127,6 +127,14 @@ constexpr sampler linearSampler(coord::normalized, } )METAL"; +static BOOL tb_pixel_buffer_uses_display_p3(CVPixelBufferRef pixelBuffer) { + if (!pixelBuffer) return NO; + CFTypeRef primaries = CVBufferGetAttachment( + pixelBuffer, kCVImageBufferColorPrimariesKey, NULL); + return primaries && CFGetTypeID(primaries) == CFStringGetTypeID() && + CFEqual(primaries, kCVImageBufferColorPrimaries_P3_D65); +} + @interface TBNativeMetalRenderer : NSObject - (instancetype)initRenderer; - (void)setVisible:(BOOL)visible; @@ -147,6 +155,8 @@ - (int)renderCursorX:(int)cursorX cursorType:(int)cursorType cursorLarge:(BOOL)cursorLarge; - (void)copyStats:(struct tb_native_metal_stats *)stats; +- (const char *)colorSpaceName; +- (void)waitUntilIdle; @end @implementation TBNativeMetalRenderer { @@ -156,11 +166,13 @@ @implementation TBNativeMetalRenderer { CVMetalTextureCacheRef _textureCache; TBNativeMetalView *_view; CAMetalLayer *_metalLayer; + CGColorSpaceRef _layerColorSpace; dispatch_semaphore_t _inflightSemaphore; CVPixelBufferRef _latestFrame; os_unfair_lock _statsLock; struct tb_native_metal_stats _stats; BOOL _loggedFirstFrame; + BOOL _displayP3; } - (instancetype)initRenderer { @@ -216,6 +228,11 @@ - (instancetype)initRenderer { - (void)dealloc { [self setVisible:NO]; + if (_metalLayer) _metalLayer.colorspace = nil; + if (_layerColorSpace) { + CGColorSpaceRelease(_layerColorSpace); + _layerColorSpace = NULL; + } if (_textureCache) { CVMetalTextureCacheFlush(_textureCache, 0); CFRelease(_textureCache); @@ -253,6 +270,11 @@ - (BOOL)attachViewIfNeeded { * Follow the largest visible receiver window instead of retaining the * initial 980x620 window forever. */ if (_view) { + if (_metalLayer) _metalLayer.colorspace = nil; + if (_layerColorSpace) { + CGColorSpaceRelease(_layerColorSpace); + _layerColorSpace = NULL; + } [_view removeFromSuperview]; _view = nil; _metalLayer = nil; @@ -271,8 +293,12 @@ - (BOOL)attachViewIfNeeded { CGColorSpaceRef srgb = CGColorSpaceCreateWithName(kCGColorSpaceSRGB); if (srgb) { _metalLayer.colorspace = srgb; - CGColorSpaceRelease(srgb); + /* CAMetalLayer's CF-typed property does not declare ownership in the + * Objective-C header. Retain our own reference until the layer is + * detached so older macOS releases cannot observe a dangling space. */ + _layerColorSpace = srgb; } + _displayP3 = NO; _metalLayer.framebufferOnly = YES; _metalLayer.opaque = YES; _metalLayer.presentsWithTransaction = NO; @@ -293,6 +319,23 @@ - (BOOL)attachViewIfNeeded { return YES; } +- (void)updateColorSpaceForPixelBuffer:(CVPixelBufferRef)pixelBuffer { + if (!_metalLayer) return; + const BOOL displayP3 = tb_pixel_buffer_uses_display_p3(pixelBuffer); + if (_metalLayer.colorspace && displayP3 == _displayP3) return; + + CGColorSpaceRef colorSpace = CGColorSpaceCreateWithName( + displayP3 ? kCGColorSpaceDisplayP3 : kCGColorSpaceSRGB); + if (!colorSpace) return; + CGColorSpaceRef previousColorSpace = _layerColorSpace; + _layerColorSpace = colorSpace; + _metalLayer.colorspace = colorSpace; + if (previousColorSpace) CGColorSpaceRelease(previousColorSpace); + _displayP3 = displayP3; + fprintf(stderr, "[metal-native] output color space = %s\n", + displayP3 ? "Display P3" : "sRGB"); +} + - (void)updateDrawableSize { if (!_view || !_metalLayer) return; NSRect backingBounds = [_view convertRectToBacking:_view.bounds]; @@ -341,6 +384,7 @@ - (int)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer @autoreleasepool { if (!pixelBuffer || ![self attachViewIfNeeded]) return -1; [self updateDrawableSize]; + [self updateColorSpaceForPixelBuffer:pixelBuffer]; _view.hidden = NO; if (rememberFrame && pixelBuffer != _latestFrame) { @@ -438,6 +482,13 @@ - (int)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer if (completed.GPUEndTime > completed.GPUStartTime) { gpuMS = (completed.GPUEndTime - completed.GPUStartTime) * 1000.0; } + CVPixelBufferRelease(pixelBuffer); + CFRelease(lumaRef); + CFRelease(chromaRef); + /* Publish completion only after all retained frame resources are + * released. Callers use this counter as a destruction barrier in + * benchmarks and diagnostics, so incrementing it earlier exposed + * a narrow use-after-release race at shutdown. */ os_unfair_lock_lock(&self->_statsLock); self->_stats.completed_frames++; self->_stats.gpu_time_ms_total += gpuMS; @@ -445,9 +496,6 @@ - (int)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer self->_stats.gpu_time_ms_max = gpuMS; } os_unfair_lock_unlock(&self->_statsLock); - CVPixelBufferRelease(pixelBuffer); - CFRelease(lumaRef); - CFRelease(chromaRef); dispatch_semaphore_signal(self->_inflightSemaphore); }]; @@ -492,6 +540,21 @@ - (void)copyStats:(struct tb_native_metal_stats *)stats { os_unfair_lock_unlock(&_statsLock); } +- (const char *)colorSpaceName { + return _displayP3 ? "Display P3" : "sRGB"; +} + +- (void)waitUntilIdle { + /* Acquiring every in-flight slot waits for all command-buffer completion + * handlers, then restores the semaphore for an orderly deallocation. */ + for (int slot = 0; slot < 3; slot++) { + dispatch_semaphore_wait(_inflightSemaphore, DISPATCH_TIME_FOREVER); + } + for (int slot = 0; slot < 3; slot++) { + dispatch_semaphore_signal(_inflightSemaphore); + } +} + @end void *tb_native_metal_create(void) { @@ -505,6 +568,7 @@ void tb_native_metal_destroy(void *renderer) { if (!renderer) return; @autoreleasepool { TBNativeMetalRenderer *object = CFBridgingRelease(renderer); + [object waitUntilIdle]; [object setVisible:NO]; } } @@ -570,3 +634,16 @@ void tb_native_metal_get_stats(void *renderer, [(__bridge TBNativeMetalRenderer *)renderer copyStats:stats]; } } + +const char *tb_native_metal_pixel_buffer_color_space(void *pixel_buffer) { + return tb_pixel_buffer_uses_display_p3((CVPixelBufferRef)pixel_buffer) + ? "Display P3" + : "sRGB"; +} + +const char *tb_native_metal_color_space_name(void *renderer) { + if (!renderer) return "unavailable"; + @autoreleasepool { + return [(__bridge TBNativeMetalRenderer *)renderer colorSpaceName]; + } +} diff --git a/TargetBridge-Receiver/TBReceiverC/tests/test_native_metal_renderer.m b/TargetBridge-Receiver/TBReceiverC/tests/test_native_metal_renderer.m index 8931209..d812b87 100644 --- a/TargetBridge-Receiver/TBReceiverC/tests/test_native_metal_renderer.m +++ b/TargetBridge-Receiver/TBReceiverC/tests/test_native_metal_renderer.m @@ -1,8 +1,10 @@ #import "tb_native_metal_renderer.h" +#import #import #include +#include int main(void) { @autoreleasepool { @@ -18,6 +20,42 @@ int main(void) { "native Metal renderer test: device exists but renderer/shader creation failed\n"); return 1; } + CVPixelBufferRef pixelBuffer = NULL; + CFDictionaryRef attributes = (__bridge CFDictionaryRef)@{ + (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{} + }; + CVReturn status = CVPixelBufferCreate( + kCFAllocatorDefault, + 640, + 360, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + attributes, + &pixelBuffer); + if (status != kCVReturnSuccess || !pixelBuffer) { + fprintf(stderr, "native Metal renderer test: pixel buffer creation failed\n"); + tb_native_metal_destroy(renderer); + return 1; + } + + if (strcmp(tb_native_metal_pixel_buffer_color_space(pixelBuffer), "sRGB") != 0) { + fprintf(stderr, "native Metal renderer test: untagged buffer must use sRGB\n"); + CVPixelBufferRelease(pixelBuffer); + tb_native_metal_destroy(renderer); + return 1; + } + CVBufferSetAttachment( + pixelBuffer, + kCVImageBufferColorPrimariesKey, + kCVImageBufferColorPrimaries_P3_D65, + kCVAttachmentMode_ShouldPropagate); + if (strcmp(tb_native_metal_pixel_buffer_color_space(pixelBuffer), "Display P3") != 0) { + fprintf(stderr, "native Metal renderer test: P3 metadata not recognized\n"); + CVPixelBufferRelease(pixelBuffer); + tb_native_metal_destroy(renderer); + return 1; + } + + CVPixelBufferRelease(pixelBuffer); tb_native_metal_destroy(renderer); printf("native Metal renderer test: device=%s shader/pipeline passed\n", device.name.UTF8String ?: "unknown"); diff --git a/TargetBridge-Receiver/TBReceiverC/tests/test_renderer_policy.c b/TargetBridge-Receiver/TBReceiverC/tests/test_renderer_policy.c new file mode 100644 index 0000000..2985581 --- /dev/null +++ b/TargetBridge-Receiver/TBReceiverC/tests/test_renderer_policy.c @@ -0,0 +1,88 @@ +#include "renderer_policy.h" + +#include +#include + +static int checks; +static int failures; + +#define CHECK(condition) do { \ + checks++; \ + if (!(condition)) { \ + failures++; \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #condition); \ + } \ +} while (0) + +static struct tb_renderer_health_result evaluate( + uint64_t submitted, + uint64_t completed, + uint64_t dropped, + double gpu_total) { + const struct tb_renderer_health_sample sample = { + submitted, completed, dropped, gpu_total + }; + return tb_renderer_evaluate_health(&sample); +} + +int main(void) { + struct tb_renderer_health_result result = + tb_renderer_evaluate_health(NULL); + CHECK(result.decision == TB_RENDERER_HEALTH_WAIT); + + result = evaluate(119, 119, 0, 595.0); + CHECK(result.decision == TB_RENDERER_HEALTH_WAIT); + + result = evaluate(124, 122, 3, 610.0); + CHECK(result.decision == TB_RENDERER_HEALTH_KEEP_METAL); + CHECK(result.attempted_frames == 127); + CHECK(result.outstanding_frames == 2); + CHECK(fabs(result.gpu_average_ms - 5.0) < 0.001); + CHECK(result.drop_ratio < 0.03); + + result = evaluate(122, 120, 0, 0.0); + CHECK(result.decision == TB_RENDERER_HEALTH_KEEP_METAL); + + /* Every healthy boundary is inclusive. */ + result = evaluate(138, 132, 12, 132.0 * 12.0); + CHECK(result.outstanding_frames == 6); + CHECK(fabs(result.drop_ratio - 0.08) < 0.000001); + CHECK(fabs(result.gpu_average_ms - 12.0) < 0.000001); + CHECK(result.decision == TB_RENDERER_HEALTH_KEEP_METAL); + + result = evaluate(138, 132, 13, 132.0 * 12.0); + CHECK(result.decision == TB_RENDERER_HEALTH_FALLBACK_OPENGL); + + result = evaluate(139, 132, 0, 132.0 * 12.0); + CHECK(result.decision == TB_RENDERER_HEALTH_FALLBACK_OPENGL); + + result = evaluate(138, 132, 0, 132.0 * 12.001); + CHECK(result.decision == TB_RENDERER_HEALTH_FALLBACK_OPENGL); + + result = evaluate(125, 120, 20, 600.0); + CHECK(result.decision == TB_RENDERER_HEALTH_FALLBACK_OPENGL); + + result = evaluate(135, 120, 0, 600.0); + CHECK(result.decision == TB_RENDERER_HEALTH_FALLBACK_OPENGL); + + result = evaluate(121, 120, 0, 1560.0); + CHECK(result.decision == TB_RENDERER_HEALTH_FALLBACK_OPENGL); + + result = evaluate(150, 0, 0, 0.0); + CHECK(result.decision == TB_RENDERER_HEALTH_FALLBACK_OPENGL); + + result = evaluate(149, 0, 0, 0.0); + CHECK(result.decision == TB_RENDERER_HEALTH_WAIT); + + /* A reset/out-of-order sample cannot underflow outstanding frames. */ + result = evaluate(120, 121, 0, 500.0); + CHECK(result.outstanding_frames == 0); + CHECK(result.decision == TB_RENDERER_HEALTH_KEEP_METAL); + + if (failures == 0) { + printf("renderer policy tests: %d checks passed\n", checks); + return 0; + } + fprintf(stderr, "renderer policy tests: %d/%d checks failed\n", failures, checks); + return 1; +}