diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2e1486e --- /dev/null +++ b/.env.example @@ -0,0 +1,28 @@ +YOUTUBE_API_KEY=your_youtube_api_key + +# 以下の変数はローカル開発では設定不要です。 +# docker-compose.yml のデフォルト値を上書きしたい場合のみ使用 + +# NS_MARIADB_HOSTNAME=db +# NS_MARIADB_PORT=3306 +# NS_MARIADB_DATABASE=playbacq +# NS_MARIADB_USER=playbacq_user +# NS_MARIADB_PASSWORD=my_secure_password_123 +# MYSQL_ROOT_PASSWORD=super_secret_root + +# REDIS_HOST=redis +# REDIS_PORT=6379 +# REDIS_USER=default +# REDIS_PASSWORD= + +# MINIO_ROOT_USER=admin +# MINIO_ROOT_PASSWORD=super_secret_minio +# MINIO_ENDPOINT=minio:9000 +# S3_ENDPOINT=http://127.0.0.1:9000 + +# FRONTEND_URL=http://localhost:4200 +# BACKEND_URL=http://backend:8080 + +# LOCAL_WORKER_EXECUTABLE=./build/playbacq_worker +# LOCAL_BUILD_JOBS=6 +# LOCAL_START_TIMEOUT=600 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2166edc..fe9f8d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,7 @@ option(BUILD_BACKEND "Build the backend executable" ON) option(BUILD_WORKER "Build the worker executable" ON) option(BUILD_LOCAL_DEV "Build for local development with MinIO" OFF) option(USE_LOCAL_WORKER "Use local worker for encoding" OFF) +option(USE_LOCAL_GPU_ENCODER "Use the NVIDIA GPU encoder for local development" OFF) option(ENABLE_COVERAGE "Enable coverage reporting" OFF) if(ENABLE_COVERAGE) @@ -71,8 +72,12 @@ if(BUILD_WORKER) if(BUILD_LOCAL_DEV) target_compile_definitions(playbacq_worker PRIVATE USE_INTERNAL_S3) endif() - if(USE_LOCAL_WORKER) - target_compile_definitions(playbacq_worker PRIVATE USE_LOCAL_WORKER) + + if(NOT BUILD_LOCAL_DEV OR USE_LOCAL_GPU_ENCODER) + target_compile_definitions(playbacq_worker PRIVATE USE_NVIDIA_ENCODER) + message(STATUS "Worker Encoder: GPU (h264_nvenc)") + else() + message(STATUS "Worker Encoder: CPU (libx264)") endif() message(STATUS "Configured to build: WORKER") endif() @@ -112,6 +117,13 @@ if(BUILD_BACKEND) ${CMAKE_CURRENT_SOURCE_DIR}/plugins ) + if(USE_LOCAL_WORKER) + find_package(Boost REQUIRED COMPONENTS filesystem) + target_link_libraries(playbacq_core PUBLIC Boost::filesystem) + target_compile_definitions(playbacq_core PRIVATE USE_LOCAL_WORKER) + message(STATUS "Worker Dispatch: LOCAL") + endif() + add_executable(${PROJECT_NAME} main.cpp) target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra -Wpedantic -O3 diff --git a/Dockerfile.dev b/Dockerfile.dev index d590d74..b1fb7cf 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,9 +1,11 @@ -FROM unabc/playbacq-base:v2 AS builder +FROM unabc/playbacq-base:v2 AS development # キャッシュバスター ENV FORCE_REBUILD_CACHE_BUST=1 RUN ldconfig -ARG BUILD_JOBS=12 WORKDIR /app + +FROM development AS builder +ARG BUILD_JOBS=12 COPY CMakeLists.txt /app/ COPY *.cpp /app/ @@ -14,7 +16,7 @@ COPY plugins/ /app/plugins/ COPY worker/ /app/worker/ COPY config.json config.yaml /app/ -RUN mkdir build && cd build && cmake .. -DBUILD_LOCAL_DEV=ON && make -j${BUILD_JOBS} +RUN mkdir build && cd build && cmake .. -DBUILD_LOCAL_DEV=ON -DUSE_LOCAL_WORKER=ON && make -j${BUILD_JOBS} FROM gcc:15.2 ENV DEBIAN_FRONTEND=noninteractive @@ -33,6 +35,7 @@ COPY --from=builder /usr/lib/x86_64-linux-gnu/libboost_*.so* /usr/lib/x86_64-lin RUN ldconfig WORKDIR /app +ENV LOCAL_WORKER_EXECUTABLE=/usr/local/bin/playbacq_worker COPY config.json config.yaml /app/ COPY --from=builder /app/build/playbacq /usr/local/bin/playbacq @@ -41,4 +44,4 @@ COPY --from=builder /app/build/playbacq_worker /usr/local/bin/playbacq_worker RUN chmod +x /usr/local/bin/playbacq_worker EXPOSE 8080 -CMD ["/usr/local/bin/playbacq"] \ No newline at end of file +CMD ["/usr/local/bin/playbacq"] diff --git a/GNUmakefile b/GNUmakefile new file mode 100644 index 0000000..38169fb --- /dev/null +++ b/GNUmakefile @@ -0,0 +1,25 @@ +LOCAL_COMPOSE = docker compose -f docker-compose.yml -f docker-compose.local.yml +LOCAL_GPU_COMPOSE = $(LOCAL_COMPOSE) -f docker-compose.local-gpu.yml + +.PHONY: local-dev local-dev-gpu local-dev-status local-dev-logs local-dev-shell local-dev-down + +local-dev: + $(LOCAL_COMPOSE) up -d --build --force-recreate backend + $(LOCAL_COMPOSE) exec -T backend /bin/sh /app/docker/dev/wait-ready.sh + +local-dev-gpu: + $(LOCAL_GPU_COMPOSE) up -d --build --force-recreate backend + $(LOCAL_GPU_COMPOSE) exec -T backend /bin/sh /app/docker/dev/wait-ready.sh + +local-dev-status: + $(LOCAL_COMPOSE) ps -a + @$(LOCAL_COMPOSE) exec -T backend /bin/sh /app/docker/dev/status.sh + +local-dev-logs: + $(LOCAL_COMPOSE) logs -f backend + +local-dev-shell: + $(LOCAL_COMPOSE) exec backend bash + +local-dev-down: + $(LOCAL_COMPOSE) down diff --git a/README.md b/README.md index 95a53b1..9cb0fd7 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,11 @@ * **Others**: AWS SDK for C++ (S3 Plugin) ## 環境構築 -` +``` docker compose up -d -` +``` + +`.env`は`.env.example`を参考に設定 ## 実行 ``` @@ -30,7 +32,99 @@ cmake .. make ./playbacq ``` - または VSCodeでCMake Toolsを使用してビルド+実行。 + +## ローカル + +### `make local-dev` + +次のコマンドで、ローカル環境のビルドから起動までまとめて実行可能です。 + +```bash +make local-dev +``` + +内部では次の処理を行います。 + +1. MySQL、Redis、MinIO、バックエンドコンテナを起動 +2. `BUILD_LOCAL_DEV=ON`と`USE_LOCAL_WORKER=ON`でCMake configure +3. `build`が未作成なら通常ビルド、作成済みならインクリメンタルビルド +4. `./build/playbacq`を起動 +5. バックエンドの8080番ポートが利用可能になるまで待機 + +エンコードにはCPUを使用します。GPUを使用する場合は補助コマンドを利用する。 + +### 手動 + +開発用コンテナを起動して手動でビルド・実行することもできます。 + +```bash +docker compose up -d +docker compose exec backend bash +``` + +コンテナ内で実行 + +```bash +cmake -S . -B build -DBUILD_LOCAL_DEV=ON -DUSE_LOCAL_WORKER=ON +cmake --build build -j$(nproc) +./build/playbacq +``` + +#### VSCode CMake Tools + +VSCodeのCmake Toolsでもビルド・実行可能です。 + +CMake Toolsには次を設定します。 + +```json +{ + "cmake.buildDirectory": "${workspaceFolder}/build", + "cmake.configureSettings": { + "CMAKE_BUILD_TYPE": "Debug", + "BUILD_LOCAL_DEV": "ON", + "USE_LOCAL_WORKER": "ON" + }, + "cmake.buildBeforeRun": true, + "cmake.debugConfig": { + "cwd": "${workspaceFolder}" + } +} +``` + +ビルドターゲットは`all`、実行・デバッグ対象は`playbacq`を選択します。 +CMake Toolsから実行する場合は`make local-dev`を実行せず`docker compose up -d`を実行。 + +### 補助コマンド + +サービスの稼働状況、ローカルCPU/GPUモード、エンコーダー設定を表示します。 + +```bash +make local-dev-status +``` + +バックエンドのログを表示します。 + +```bash +make local-dev-logs +``` + +開発用コンテナのシェルを開きます。 + +```bash +make local-dev-shell +``` + +ローカル環境を停止します。 + +```bash +make local-dev-down +``` + +ローカルでもNVIDIA GPUを使用してエンコードする場合は、次のコマンドを使用します。 + +```bash +make local-dev-gpu +``` diff --git a/controllers/webhooks_minio.cpp b/controllers/webhooks_minio.cpp index b1b5a8c..d535674 100644 --- a/controllers/webhooks_minio.cpp +++ b/controllers/webhooks_minio.cpp @@ -5,6 +5,9 @@ #include #include #include +#include +#include +#include #include #include "../models/Videos.h" #include "Status.h" @@ -56,11 +59,27 @@ drogon::Task minio::asyncHandleHttpRequest(HttpRequestP auto video = co_await mapper.findByPrimaryKey(videoId); video.setStatus((uint8_t)Status::processing); co_await mapper.update(video); - std::cout << "Pushing video ID " << videoId << " to Modal" << std::endl; + std::cout << "Dispatching video ID " << videoId << " for encoding" << std::endl; #ifdef USE_LOCAL_WORKER std::thread([videoId]() { try { - boost::process::child c("./build/playbacq_worker", videoId); + const char* workerExecutableEnv = std::getenv("LOCAL_WORKER_EXECUTABLE"); + const std::string workerExecutable = workerExecutableEnv + ? workerExecutableEnv + : "./build/playbacq_worker"; + std::string resolvedWorkerExecutable = workerExecutable; + if (!std::filesystem::exists(resolvedWorkerExecutable)) { + const auto workerPath = boost::process::search_path(workerExecutable); + if (!workerPath.empty()) { + resolvedWorkerExecutable = workerPath.string(); + } + } + if (!std::filesystem::exists(resolvedWorkerExecutable)) { + throw std::runtime_error("Local worker executable not found: " + workerExecutable); + } + std::cout << "Starting local worker " << resolvedWorkerExecutable + << " for video ID " << videoId << std::endl; + boost::process::child c(resolvedWorkerExecutable, videoId); c.detach(); } catch (const std::exception& e) { @@ -154,4 +173,4 @@ drogon::Task minio::receiveEncodeResult(HttpRequestPtr auto resp = drogon::HttpResponse::newHttpResponse(); resp->setStatusCode(drogon::HttpStatusCode::k200OK); co_return resp; -} \ No newline at end of file +} diff --git a/docker-compose.local-gpu.yml b/docker-compose.local-gpu.yml new file mode 100644 index 0000000..48ccb4f --- /dev/null +++ b/docker-compose.local-gpu.yml @@ -0,0 +1,6 @@ +services: + backend: + gpus: all + environment: + LOCAL_GPU_ENCODER: "ON" + NVIDIA_DRIVER_CAPABILITIES: compute,video,utility diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 0000000..23fd9f2 --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,8 @@ +services: + backend: + build: + target: development + command: ["/bin/sh", "/app/docker/dev/start.sh"] + environment: + LOCAL_BUILD_JOBS: ${LOCAL_BUILD_JOBS:-6} + LOCAL_START_TIMEOUT: ${LOCAL_START_TIMEOUT:-600} diff --git a/docker-compose.yml b/docker-compose.yml index ff95725..15f446e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,86 +1,101 @@ -services: - backend: - build: - context: . - args: - BUILD_JOBS: 6 - target: builder - dockerfile: Dockerfile.dev - volumes: - - .:/app - command: sleep infinity - ports: - - "8080:8080" - environment: - - DB_HOST=db - - REDIS_HOST=${REDIS_HOST} - - REDIS_PORT=${REDIS_PORT} - - REDIS_PASSWORD=${REDIS_PASSWORD} - - REDIS_USER=${REDIS_USER} - - NS_MARIADB_USER=${NS_MARIADB_USER} - - NS_MARIADB_PASSWORD=${NS_MARIADB_PASSWORD} - - NS_MARIADB_DATABASE=${NS_MARIADB_DATABASE} - - MINIO_ROOT_USER=${MINIO_ROOT_USER} - - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - - MINIO_ENDPOINT=${MINIO_ENDPOINT} - - S3_ENDPOINT=${S3_ENDPOINT} - - FRONTEND_URL=${FRONTEND_URL} - - BACKEND_URL=${BACKEND_URL} - - YOUTUBE_API_KEY=${YOUTUBE_API_KEY} - depends_on: - - db - - redis - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: 1 - capabilities: [gpu, video] - db: - image: mysql:8.0 - environment: - - MYSQL_DATABASE=${NS_MARIADB_DATABASE} - - MYSQL_USER=${NS_MARIADB_USER} - - MYSQL_PASSWORD=${NS_MARIADB_PASSWORD} - - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} - volumes: - - db_data:/var/lib/mysql - - ./docker/init.sql:/docker-entrypoint-initdb.d/init.sql - ports: - - "3306:3306" - redis: - image: redis:7-alpine - ports: - - "6379:6379" - volumes: - - redis_data:/data - minio: - image: minio/minio:latest - command: server /data --console-address ":9001" - environment: - - MINIO_ROOT_USER=${MINIO_ROOT_USER} - - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - - MINIO_NOTIFY_WEBHOOK_ENABLE_1=on - - MINIO_NOTIFY_WEBHOOK_ENDPOINT_1=http://backend:8080/webhooks/minio - - MINIO_API_CORS_ALLOW_ORIGIN=http://localhost:4200 - ports: - - "9000:9000" # API用 - - "9001:9001" # コンソール用 - volumes: - - minio_data:/data - minio-init: - image: minio/mc:latest - container_name: minio-init - depends_on: - - minio - environment: - - MINIO_ROOT_USER=${MINIO_ROOT_USER} - - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - volumes: - - ./docker/minio/init.sh:/init.sh - entrypoint: /bin/sh -c "chmod +x /init.sh && /init.sh" -volumes: - db_data: - redis_data: - minio_data: +services: + backend: + build: + context: . + args: + BUILD_JOBS: 6 + target: builder + dockerfile: Dockerfile.dev + volumes: + - .:/app + command: sleep infinity + ports: + - "8080:8080" + environment: + NS_MARIADB_HOSTNAME: ${NS_MARIADB_HOSTNAME:-db} + NS_MARIADB_PORT: ${NS_MARIADB_PORT:-3306} + NS_MARIADB_DATABASE: ${NS_MARIADB_DATABASE:-playbacq} + NS_MARIADB_USER: ${NS_MARIADB_USER:-playbacq_user} + NS_MARIADB_PASSWORD: ${NS_MARIADB_PASSWORD:-my_secure_password_123} + REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + REDIS_USER: ${REDIS_USER:-default} + REDIS_PASSWORD: ${REDIS_PASSWORD:-} + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-super_secret_minio} + MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000} + S3_ENDPOINT: ${S3_ENDPOINT:-http://127.0.0.1:9000} + FRONTEND_URL: ${FRONTEND_URL:-http://localhost:4200} + BACKEND_URL: ${BACKEND_URL:-http://backend:8080} + LOCAL_WORKER_EXECUTABLE: ${LOCAL_WORKER_EXECUTABLE:-./build/playbacq_worker} + YOUTUBE_API_KEY: ${YOUTUBE_API_KEY:-} + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + minio-init: + condition: service_completed_successfully + db: + image: mysql:8.0 + environment: + MYSQL_DATABASE: ${NS_MARIADB_DATABASE:-playbacq} + MYSQL_USER: ${NS_MARIADB_USER:-playbacq_user} + MYSQL_PASSWORD: ${NS_MARIADB_PASSWORD:-my_secure_password_123} + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-super_secret_root} + volumes: + - db_data:/var/lib/mysql + - ./docker/init.sql:/docker-entrypoint-initdb.d/init.sql + ports: + - "3306:3306" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h localhost -u$${MYSQL_USER} -p$${MYSQL_PASSWORD} --silent"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 10s + redis: + image: redis:7-alpine + ports: + - "6379:6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-super_secret_minio} + MINIO_NOTIFY_WEBHOOK_ENABLE_1: "on" + MINIO_NOTIFY_WEBHOOK_ENDPOINT_1: http://backend:8080/webhooks/minio + MINIO_API_CORS_ALLOW_ORIGIN: ${FRONTEND_URL:-http://localhost:4200} + ports: + - "9000:9000" # API用 + - "9001:9001" # コンソール用 + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 10 + minio-init: + image: minio/mc:latest + depends_on: + minio: + condition: service_healthy + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-super_secret_minio} + volumes: + - ./docker/minio/init.sh:/init.sh:ro + entrypoint: ["/bin/sh", "/init.sh"] +volumes: + db_data: + redis_data: + minio_data: diff --git a/docker/dev/start.sh b/docker/dev/start.sh new file mode 100644 index 0000000..cdce7bf --- /dev/null +++ b/docker/dev/start.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +build_dir=${LOCAL_BUILD_DIR:-/app/build} +build_jobs=${LOCAL_BUILD_JOBS:-6} +gpu_encoder=${LOCAL_GPU_ENCODER:-OFF} +backend_pid="" +pid_file=/tmp/playbacq-backend.pid +failure_file=/tmp/playbacq-local-start-failed + +keep_container_running() { + echo "The development container will remain running for investigation and debugging." + exec sleep infinity +} + +record_failure() { + echo "$1" | tee "$failure_file" >&2 +} + +stop_backend() { + if [ -n "$backend_pid" ] && kill -0 "$backend_pid" 2>/dev/null; then + kill -TERM "$backend_pid" + wait "$backend_pid" + fi + rm -f "$pid_file" + exit 0 +} + +trap stop_backend INT TERM +rm -f "$pid_file" "$failure_file" + +if ! cmake -S /app -B "$build_dir" -DBUILD_LOCAL_DEV=ON -DUSE_LOCAL_WORKER=ON -DUSE_LOCAL_GPU_ENCODER="$gpu_encoder"; then + record_failure "Local configure failed." + keep_container_running +fi + +if ! cmake --build "$build_dir" --parallel "$build_jobs"; then + record_failure "Local build failed." + keep_container_running +fi + +"$build_dir/playbacq" & +backend_pid=$! +echo "$backend_pid" > "$pid_file" +echo "Local backend started with PID $backend_pid." + +wait "$backend_pid" +backend_status=$? +rm -f "$pid_file" +record_failure "Local backend exited with status $backend_status." +keep_container_running diff --git a/docker/dev/status.sh b/docker/dev/status.sh new file mode 100644 index 0000000..39d2829 --- /dev/null +++ b/docker/dev/status.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +build_dir=${LOCAL_BUILD_DIR:-/app/build} +cache_file="$build_dir/CMakeCache.txt" +worker="$build_dir/playbacq_worker" + +if [ ! -f "$cache_file" ]; then + echo "Local build configuration: unavailable ($cache_file was not found)" + exit 1 +fi + +build_local_dev=$(sed -n 's/^BUILD_LOCAL_DEV:BOOL=//p' "$cache_file" | tail -n 1) +use_local_worker=$(sed -n 's/^USE_LOCAL_WORKER:BOOL=//p' "$cache_file" | tail -n 1) +use_local_gpu=$(sed -n 's/^USE_LOCAL_GPU_ENCODER:BOOL=//p' "$cache_file" | tail -n 1) + +case "$build_local_dev:$use_local_gpu" in + ON:ON) + mode="LOCAL GPU" + ;; + ON:*) + mode="LOCAL CPU" + ;; + *) + mode="NON-LOCAL" + ;; +esac + +printf '\nLocal build configuration:\n' +printf ' Mode: %s\n' "$mode" +printf ' BUILD_LOCAL_DEV=%s\n' "${build_local_dev:-unknown}" +printf ' USE_LOCAL_WORKER=%s\n' "${use_local_worker:-unknown}" +printf ' USE_LOCAL_GPU_ENCODER=%s\n' "${use_local_gpu:-unknown}" + +printf '\nWorker encoder:\n' +if [ -x "$worker" ]; then + "$worker" --show-encoder-config +else + echo " unavailable ($worker was not found)" +fi + +printf '\nGPU:\n' +if gpu_info=$(nvidia-smi -L 2>/dev/null); then + printf '%s\n' "$gpu_info" +else + echo " not exposed to the backend container" +fi diff --git a/docker/dev/wait-ready.sh b/docker/dev/wait-ready.sh new file mode 100644 index 0000000..0270ef3 --- /dev/null +++ b/docker/dev/wait-ready.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +timeout=${LOCAL_START_TIMEOUT:-600} +elapsed=0 +pid_file=/tmp/playbacq-backend.pid +failure_file=/tmp/playbacq-local-start-failed + +if ! command -v bash >/dev/null 2>&1; then + echo "bash is required to check whether the local backend is ready." >&2 + exit 1 +fi + +while [ "$elapsed" -lt "$timeout" ]; do + if [ -f "$failure_file" ]; then + cat "$failure_file" >&2 + exit 1 + fi + + if [ -f "$pid_file" ]; then + backend_pid=$(cat "$pid_file") + if kill -0 "$backend_pid" 2>/dev/null \ + && bash -c '/dev/null; then + echo "Local backend is ready on http://localhost:8080." + exit 0 + fi + fi + + sleep 1 + elapsed=$((elapsed + 1)) +done + +echo "Timed out waiting for the local backend after ${timeout} seconds." >&2 +exit 1 diff --git a/docker/minio/init.sh b/docker/minio/init.sh index ac8e9a3..1626325 100644 --- a/docker/minio/init.sh +++ b/docker/minio/init.sh @@ -1,7 +1,9 @@ #!/bin/sh +set -eu + echo "Waiting for MinIO to start..." -until mc alias set myminio http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}; do +until mc alias set myminio http://minio:9000 "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}"; do echo "MinIO is not available yet. Retrying..." sleep 3 done @@ -15,6 +17,6 @@ mc mb myminio/videofiles --ignore-existing mc anonymous set download myminio/videofiles # バケットのイベント通知を設定 (ARNのIDはMINIO_NOTIFY_WEBHOOK_ENABLE_1の"1"と一致させる) -mc event add myminio/videofiles arn:minio:sqs::1:webhook --event put +mc event add myminio/videofiles arn:minio:sqs::1:webhook --event put --ignore-existing -echo "Bucket created and webhook notification set up successfully." \ No newline at end of file +echo "Bucket created and webhook notification set up successfully." diff --git a/worker/worker.cpp b/worker/worker.cpp index 5622825..0790eda 100644 --- a/worker/worker.cpp +++ b/worker/worker.cpp @@ -4,6 +4,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -13,7 +16,35 @@ #include #include -#define USE_NVIDIA_ENCODER +namespace { +#ifdef USE_NVIDIA_ENCODER +constexpr bool USE_NVIDIA_VIDEO_ENCODER = true; +constexpr const char* VIDEO_ENCODER = "h264_nvenc"; +constexpr const char* VIDEO_PRESET = "p4"; +constexpr const char* VIDEO_BITRATE = "2M"; +#else +constexpr bool USE_NVIDIA_VIDEO_ENCODER = false; +constexpr const char* VIDEO_ENCODER = "libx264"; +constexpr const char* VIDEO_PRESET = "veryfast"; +constexpr const char* VIDEO_BITRATE = ""; +#endif + +class ThreadJoinGuard { +public: + explicit ThreadJoinGuard(std::thread& thread) noexcept : thread_(thread) {} + ThreadJoinGuard(const ThreadJoinGuard&) = delete; + ThreadJoinGuard& operator=(const ThreadJoinGuard&) = delete; + + ~ThreadJoinGuard() { + if (thread_.joinable()) { + thread_.join(); + } + } + +private: + std::thread& thread_; +}; +} bool upload2MinIO(const std::string& local_file_path, const std::string& bucket_name, const std::string& object_key) { const char* envUser = std::getenv("MINIO_ROOT_USER"); @@ -168,11 +199,20 @@ std::string formatTime(int total_seconds) { int main(int argc, char* argv[]) { if (argc != 2) { - std::cerr << "Usage: " << argv[0] << " " << std::endl; + std::cerr << "Usage: " << argv[0] << " |--show-encoder-config" << std::endl; return 1; } std::string video_id = argv[1]; + std::cout << "FFmpeg video encoder: " << VIDEO_ENCODER + << ", preset: " << VIDEO_PRESET; + if (VIDEO_BITRATE[0] != '\0') { + std::cout << ", bitrate: " << VIDEO_BITRATE; + } + std::cout << std::endl; + if (video_id == "--show-encoder-config") { + return 0; + } Aws::SDKOptions options; Aws::InitAPI(options); @@ -209,6 +249,29 @@ int main(int argc, char* argv[]) { std::cout << "Using MinIO endpoint: " << minioEndpoint << std::endl; std::cout << "\n[JOB RECEIVED] Video ID: " << video_id << std::endl; + int last_attempted_percent = -1; + auto updateProgress = [&](int progress) { + progress = std::clamp(progress, 0, 100); + if (progress <= last_attempted_percent) { + return; + } + last_attempted_percent = progress; + try { + redis.set("video:progress:" + video_id, std::to_string(progress), std::chrono::hours(24)); + std::cout << "Progress updated: " << progress << "% for video ID: " << video_id << std::endl; + } + catch (const std::exception& e) { + // 進捗通知の失敗だけでエンコード処理を失敗させない。 + std::cerr << "Progress update failed for video ID " << video_id + << ": " << e.what() << std::endl; + } + catch (...) { + std::cerr << "Progress update failed for video ID " << video_id + << ": unknown error" << std::endl; + } + }; + updateProgress(0); + const char* envMinIOUser = std::getenv("MINIO_ROOT_USER"); const char* envMinIOPassword = std::getenv("MINIO_ROOT_PASSWORD"); std::string accessKey = envMinIOUser ? envMinIOUser : ""; @@ -248,6 +311,9 @@ int main(int argc, char* argv[]) { total_duration_sec = std::stod(duration_str); } probe_c.wait(); + if (probe_c.exit_code() != 0) { + throw std::runtime_error("ffprobe exited with code " + std::to_string(probe_c.exit_code())); + } if (total_duration_sec <= 0.0) { std::cerr << "Invalid video duration: " << total_duration_sec << " seconds for video ID: " << video_id << std::endl; @@ -261,7 +327,6 @@ int main(int argc, char* argv[]) { return 1; } - int last_notified_percent = -1; std::string base_dir = "/tmp/playbacq_encode/" + video_id + "/"; int interval = 10; // サムネイルを10秒ごとに生成 if (total_duration_sec < 600) { @@ -278,37 +343,38 @@ int main(int argc, char* argv[]) { std::filesystem::create_directories(base_dir); boost::process::ipstream output_stream; - // サムネ画像の時間 - int thumb_time = std::min(4, static_cast(total_duration_sec / 2)); - std::string filter_complex = std::format( - "[0:v]split=3[v_hls_in][v_seek_in][v_thumb_in];" + const int thumb_time = std::min(4, static_cast(total_duration_sec / 2)); + const std::string filter_complex = std::format( + "[0:v]split=4[v_hls_in][v_seek_in][v_thumb_in][v_progress_in];" "[v_hls_in]scale='trunc(min(1920,iw)/2)*2':'trunc(min(1080,ih)/2)*2':force_original_aspect_ratio=decrease,pad='ceil(max(iw,ih*(16/9))/2)*2':'ceil(max(ih,iw*(9/16))/2)*2':(ow-iw)/2:(oh-ih)/2:black,format=yuv420p[v_hls_out];" "[v_seek_in]fps=1/{0},scale=160:90:force_original_aspect_ratio=decrease,pad=160:90:(ow-iw)/2:(oh-ih)/2:black,tile=10x10[v_seek_out];" - "[v_thumb_in]select='gte(t\\,{1})',scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2:black[v_thumb_out]", + "[v_thumb_in]select='gte(t\\,{1})',scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2:black[v_thumb_out];" + "[v_progress_in]fps=1,scale=2:2,metadata=mode=add:key=playbacq_progress:value=1," + "metadata=mode=print:key=playbacq_progress:file=/dev/stdout:direct=1[v_progress_out]", interval, thumb_time ); - std::vector args = { - #ifdef USE_NVIDIA_ENCODER - "-hwaccel", "cuda", - #endif + std::vector args; + if (USE_NVIDIA_VIDEO_ENCODER) { + args.insert(args.end(), { "-hwaccel", "cuda" }); + } + args.insert(args.end(), { "-reconnect", "1", "-reconnect_at_eof", "1", "-reconnect_streamed", "1", "-reconnect_delay_max", "5", "-i", video_url, - "-progress", "pipe:1", "-filter_complex", filter_complex, // HLS出力設定 "-map", "[v_hls_out]", "-map", "0:a?", - #ifdef USE_NVIDIA_ENCODER - "-c:v", "h264_nvenc", - "-preset", "p4", - "-b:v", "2M", - #else - "-c:v", "libx264", - #endif + "-c:v", VIDEO_ENCODER, + "-preset", VIDEO_PRESET + }); + if (VIDEO_BITRATE[0] != '\0') { + args.insert(args.end(), { "-b:v", VIDEO_BITRATE }); + } + args.insert(args.end(), { "-c:a", "aac", "-g", "60", "-sc_threshold", "0", @@ -330,46 +396,57 @@ int main(int argc, char* argv[]) { "-frames:v", "1", "-c:v", "mjpeg", "-q:v", "2", - base_dir + "thumbnail.jpg" - }; + base_dir + "thumbnail.jpg", + // 1秒ごとの進捗時刻を生成する軽量なnull出力 + "-map", "[v_progress_out]", + "-f", "null", + "/dev/null" + }); std::cout << "Starting ffmpeg process for video ID: " << video_id << std::endl; + boost::process::ipstream error_stream; boost::process::child ffmpeg_process(ffmpeg_path, boost::process::args(args), boost::process::std_in.close(), boost::process::std_out > output_stream, - boost::process::std_err.close()); + boost::process::std_err > error_stream); + std::thread error_reader([&error_stream]() { + std::string error_line; + while (std::getline(error_stream, error_line)) { + std::cerr << "[FFmpeg stderr] " << error_line << std::endl; + } + }); + ThreadJoinGuard error_reader_join_guard(error_reader); std::string line; while (std::getline(output_stream, line)) { - std::cout << "[FFmpeg] " << line << std::endl; - if (line.starts_with("out_time_us=")) { - try { - // "out_time_us="以降の数値を取得 - long long micro_seconds = std::stoll(line.substr(12)); - double current_sec = micro_seconds / 1000000.0; - int current_percent = std::min(static_cast((current_sec / total_duration_sec) * 100), 100); - - if (current_percent > last_notified_percent) { - // SET video:progress:{id} {percent} (有効期限24時間) - redis.set("video:progress:" + video_id, std::to_string(current_percent), std::chrono::hours(24)); - last_notified_percent = current_percent; - std::cout << "Progress updated: " << current_percent << "% for video ID: " << video_id << std::endl; - } - } - catch (const std::exception& e) { - // パース失敗時 (out_time_us=N/A などが来た場合) は無視して続行 - } + const std::size_t pts_position = line.find("pts_time:"); + if (pts_position == std::string::npos) { + continue; + } + try { + const std::size_t value_start = pts_position + sizeof("pts_time:") - 1; + const std::size_t value_end = line.find_first_of(" \t", value_start); + const double current_sec = std::stod(line.substr(value_start, value_end - value_start)); + const int current_percent = std::min( + static_cast((current_sec / total_duration_sec) * 100.0), + 99 + ); + updateProgress(current_percent); + } + catch (const std::exception& e) { + std::cerr << "Invalid FFmpeg progress line: " << line + << " (" << e.what() << ")" << std::endl; } } ffmpeg_process.wait(); - int exit_code = ffmpeg_process.exit_code(); - if (exit_code == 0) { - std::cout << "Encoding completed successfully for video ID: " << video_id << std::endl; - } else { + error_reader.join(); + const int exit_code = ffmpeg_process.exit_code(); + if (exit_code != 0) { std::cerr << "ffmpeg exited with code " << exit_code << " for video ID: " << video_id << std::endl; - // エラーが発生した場合はDrogonに失敗を知らせる (Pub/Sub) postEncodeResult(video_id, "failed", "ffmpeg exited with code " + std::to_string(exit_code)); return 1; } + std::cout << "Encoding and thumbnail generation completed successfully for video ID: " << video_id << std::endl; + updateProgress(100); } catch (const std::exception& e) { std::cerr << "Encoding Error: " << e.what() << std::endl; @@ -426,4 +503,4 @@ int main(int argc, char* argv[]) { curl_global_cleanup(); Aws::ShutdownAPI(options); return 0; -} \ No newline at end of file +}