From 55e446a41a8014572475a67237748beb3f4e6546 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 20 Jun 2025 17:33:01 -0700 Subject: [PATCH 01/84] initial commit --- brozzler/job_schema.yaml | 6 ++++++ brozzler/model.py | 7 ++++++- brozzler/ydl.py | 33 +++++++++++++++++++++++++++------ pyproject.toml | 1 + 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/brozzler/job_schema.yaml b/brozzler/job_schema.yaml index 59b831f2..4f7afe26 100644 --- a/brozzler/job_schema.yaml +++ b/brozzler/job_schema.yaml @@ -4,6 +4,12 @@ id: - integer required: false +account_id: + type: + - string + - integer + required: false + <<: &multi_level_options time_limit: type: number diff --git a/brozzler/model.py b/brozzler/model.py index eab4b1c1..6d666d57 100644 --- a/brozzler/model.py +++ b/brozzler/model.py @@ -2,7 +2,7 @@ brozzler/models.py - model classes representing jobs, sites, and pages, with related logic -Copyright (C) 2014-2024 Internet Archive +Copyright (C) 2014-2025 Internet Archive Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -100,6 +100,10 @@ def new_job(frontier, job_conf): ) if "id" in job_conf: job.id = job_conf["id"] + if "account_id" in job_conf: + job.account_id = job_conf["account_id"] + else: + job.account_id = None if "max_claimed_sites" in job_conf: job.max_claimed_sites = job_conf["max_claimed_sites"] if "pdfs_only" in job_conf: @@ -115,6 +119,7 @@ def new_job(frontier, job_conf): merged_conf["seed"] = merged_conf.pop("url") site = brozzler.Site(frontier.rr, merged_conf) site.id = str(uuid.uuid4()) + site.account_id = job.account_id sites.append(site) pages.append(new_seed_page(frontier, site)) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index b3e19fe5..3edeee67 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -418,6 +418,23 @@ def _try_youtube_dl(worker, ydl, site, page): return ie_result +def get_video_captures(site, source=None): + import psycopg + # todo: read pg_url from environment var + pg_url = "postgresql://ait_crawling:archive-it-crawling@db.qa-archive-it.org/ait_crawling" + account_id = site.account_id if site.account_id else None + seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None + if account_id and seed and source: + pg_query = ("SELECT containing_page_url from video where account_id = %s and seed = %s and containing_page_url like '%'+%s+'%'", (account_id, seed, source,)) + elif seed: + pg_query = ("SELECT containing_page_url from video where seed = %s and containing_page_url like '%'+%s+'%'", (seed, source)) + else: + return None + with psycopg.connect(pg_url) as conn: + with conn.cursor(row_factory=psycopg.rows.scalar_row) as cur: + cur.execute(pg_query) + return cur.fetchall() + @metrics.brozzler_ytdlp_duration_seconds.time() @metrics.brozzler_in_progress_ytdlps.track_inprogress() def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): @@ -444,10 +461,14 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): ie_result.get("extractor") == "youtube:playlist" or ie_result.get("extractor") == "youtube:tab" ): - # youtube watch pages as outlinks - outlinks = { - "https://www.youtube.com/watch?v=%s" % e["id"] - for e in ie_result.get("entries_no_dl", []) - } - # any outlinks for other cases? soundcloud, maybe? + captured_youtube_watch_pages = get_video_captures(site, source="youtube") + uncaptured_youtube_watch_pages = [] + for e in ie_result.get("entries_no_dl", []): + youtube_watch_url = f"https://www.youtube.com/watch?v={e["id"]}" + if youtube_watch_url in captured_youtube_watch_pages: + continue + uncaptured_youtube_watch_pages.append(youtube_watch_url) + if uncaptured_youtube_watch_pages: + outlinks.add(uncaptured_youtube_watch_pages) + # todo: handle outlinks for instagram and soundcloud here (if anywhere) return outlinks diff --git a/pyproject.toml b/pyproject.toml index c0fc947f..8e40fd54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "python-magic>=0.4.15", "prometheus-client>=0.20.0", "structlog>=25.1.0", + "psycopg[binary]>=3.2.6", ] license = "Apache-2.0" From 03b329cd2a4ae4090cbd73fa27b3909dbd6e7a67 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 20 Jun 2025 17:37:40 -0700 Subject: [PATCH 02/84] formatting fix --- brozzler/ydl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 3edeee67..4c3bf126 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -464,7 +464,7 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): captured_youtube_watch_pages = get_video_captures(site, source="youtube") uncaptured_youtube_watch_pages = [] for e in ie_result.get("entries_no_dl", []): - youtube_watch_url = f"https://www.youtube.com/watch?v={e["id"]}" + youtube_watch_url = f"https://www.youtube.com/watch?v={e['id']}" if youtube_watch_url in captured_youtube_watch_pages: continue uncaptured_youtube_watch_pages.append(youtube_watch_url) From 92a6cacb5f29b80f3c9532a069dc0fda7bc15491 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 20 Jun 2025 17:49:07 -0700 Subject: [PATCH 03/84] ruff format updates --- brozzler/ydl.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 4c3bf126..b4063116 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -420,14 +420,25 @@ def _try_youtube_dl(worker, ydl, site, page): def get_video_captures(site, source=None): import psycopg + # todo: read pg_url from environment var pg_url = "postgresql://ait_crawling:archive-it-crawling@db.qa-archive-it.org/ait_crawling" account_id = site.account_id if site.account_id else None seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None if account_id and seed and source: - pg_query = ("SELECT containing_page_url from video where account_id = %s and seed = %s and containing_page_url like '%'+%s+'%'", (account_id, seed, source,)) + pg_query = ( + "SELECT containing_page_url from video where account_id = %s and seed = %s and containing_page_url like '%'+%s+'%'", + ( + account_id, + seed, + source, + ), + ) elif seed: - pg_query = ("SELECT containing_page_url from video where seed = %s and containing_page_url like '%'+%s+'%'", (seed, source)) + pg_query = ( + "SELECT containing_page_url from video where seed = %s and containing_page_url like '%'+%s+'%'", + (seed, source), + ) else: return None with psycopg.connect(pg_url) as conn: @@ -435,6 +446,7 @@ def get_video_captures(site, source=None): cur.execute(pg_query) return cur.fetchall() + @metrics.brozzler_ytdlp_duration_seconds.time() @metrics.brozzler_in_progress_ytdlps.track_inprogress() def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): From c0db5b94030dd36d379b7a7fb10db57df3f78042 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 20 Jun 2025 17:57:03 -0700 Subject: [PATCH 04/84] CONCAT --- brozzler/ydl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index b4063116..728b3951 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -427,7 +427,7 @@ def get_video_captures(site, source=None): seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None if account_id and seed and source: pg_query = ( - "SELECT containing_page_url from video where account_id = %s and seed = %s and containing_page_url like '%'+%s+'%'", + "SELECT containing_page_url from video where account_id = %s and seed = %s and containing_page_url like CONCAT('%',%s,'%')", ( account_id, seed, @@ -436,7 +436,7 @@ def get_video_captures(site, source=None): ) elif seed: pg_query = ( - "SELECT containing_page_url from video where seed = %s and containing_page_url like '%'+%s+'%'", + "SELECT containing_page_url from video where seed = %s and containing_page_url like CONCAT('%',%s,'%')", (seed, source), ) else: From fd0e0d3f30c473e665893151a6e36b1b149de4b6 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Sun, 22 Jun 2025 20:28:48 -0700 Subject: [PATCH 05/84] variable VIDEO_DATA --- brozzler/ydl.py | 58 +++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 728b3951..01454a3a 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -42,6 +42,8 @@ YTDLP_WAIT = 10 YTDLP_MAX_REDIRECTS = 5 +VIDEO_DATA = "" + logger = structlog.get_logger(logger_name=__name__) @@ -418,33 +420,37 @@ def _try_youtube_dl(worker, ydl, site, page): return ie_result -def get_video_captures(site, source=None): - import psycopg - - # todo: read pg_url from environment var - pg_url = "postgresql://ait_crawling:archive-it-crawling@db.qa-archive-it.org/ait_crawling" - account_id = site.account_id if site.account_id else None - seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None - if account_id and seed and source: - pg_query = ( - "SELECT containing_page_url from video where account_id = %s and seed = %s and containing_page_url like CONCAT('%',%s,'%')", - ( - account_id, - seed, - source, - ), - ) - elif seed: - pg_query = ( - "SELECT containing_page_url from video where seed = %s and containing_page_url like CONCAT('%',%s,'%')", - (seed, source), - ) - else: +def get_video_captures(site, source="youtube"): + if not VIDEO_DATA: return None - with psycopg.connect(pg_url) as conn: - with conn.cursor(row_factory=psycopg.rows.scalar_row) as cur: - cur.execute(pg_query) - return cur.fetchall() + + if VIDEO_DATA and VIDEO_DATA.startswith("postgresql"): + import psycopg + + pg_url = VIDEO_DATA + account_id = site.account_id if site.account_id else None + seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None + containing_page_url_pattern = "http://youtube.com/watch" if source == "youtube" + if account_id and seed and source: + pg_query = ( + "SELECT containing_page_url from video where account_id = %s and seed = %s and containing_page_url like %s", + ( + account_id, + seed, + containing_page_url_pattern, + ), + ) + elif seed and source: + pg_query = ( + "SELECT containing_page_url from video where seed = %s and containing_page_url like %s", + (seed, containing_page_url_pattern), + ) + else: + return None + with psycopg.connect(pg_url) as conn: + with conn.cursor(row_factory=psycopg.rows.scalar_row) as cur: + cur.execute(pg_query) + return cur.fetchall() @metrics.brozzler_ytdlp_duration_seconds.time() From f925660eb40a1e5122d940c83879d828c51047eb Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Sun, 22 Jun 2025 20:46:33 -0700 Subject: [PATCH 06/84] skip ternary op for now --- brozzler/ydl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 01454a3a..92972b51 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -430,7 +430,11 @@ def get_video_captures(site, source="youtube"): pg_url = VIDEO_DATA account_id = site.account_id if site.account_id else None seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None - containing_page_url_pattern = "http://youtube.com/watch" if source == "youtube" + if source == "youtube": + containing_page_url_pattern = "http://youtube.com/watch" + # support other sources here + else: + containing_page_url_pattern = None if account_id and seed and source: pg_query = ( "SELECT containing_page_url from video where account_id = %s and seed = %s and containing_page_url like %s", From fe5ad0c31d7a1b06d78ce74517b29c7cdd5875a4 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 23 Jun 2025 16:46:37 -0700 Subject: [PATCH 07/84] VIDEO_DATA_SOURCE --- brozzler/ydl.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 92972b51..b52ebf7d 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -42,7 +42,7 @@ YTDLP_WAIT = 10 YTDLP_MAX_REDIRECTS = 5 -VIDEO_DATA = "" +VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") logger = structlog.get_logger(logger_name=__name__) @@ -421,23 +421,22 @@ def _try_youtube_dl(worker, ydl, site, page): def get_video_captures(site, source="youtube"): - if not VIDEO_DATA: + if not VIDEO_DATA_SOURCE: return None - if VIDEO_DATA and VIDEO_DATA.startswith("postgresql"): + if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): import psycopg - pg_url = VIDEO_DATA account_id = site.account_id if site.account_id else None seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None if source == "youtube": - containing_page_url_pattern = "http://youtube.com/watch" + containing_page_url_pattern = "http://youtube.com/watch" # yes, video data canonicalization uses "http" # support other sources here else: containing_page_url_pattern = None if account_id and seed and source: pg_query = ( - "SELECT containing_page_url from video where account_id = %s and seed = %s and containing_page_url like %s", + "SELECT distinct(containing_page_url) from video where account_id = %s and seed = %s and containing_page_url like %s", ( account_id, seed, @@ -451,10 +450,11 @@ def get_video_captures(site, source="youtube"): ) else: return None - with psycopg.connect(pg_url) as conn: + with psycopg.connect(VIDEO_DATA_SOURCE) as conn: with conn.cursor(row_factory=psycopg.rows.scalar_row) as cur: cur.execute(pg_query) return cur.fetchall() + return None @metrics.brozzler_ytdlp_duration_seconds.time() From 203d86f40289a878ee9eaa478b063675992cda11 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 23 Jun 2025 17:13:45 -0700 Subject: [PATCH 08/84] use job_conf.get() --- brozzler/model.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/brozzler/model.py b/brozzler/model.py index 6d666d57..d7c1ac2c 100644 --- a/brozzler/model.py +++ b/brozzler/model.py @@ -98,16 +98,10 @@ def new_job(frontier, job_conf): frontier.rr, {"conf": job_conf, "status": "ACTIVE", "started": doublethink.utcnow()}, ) - if "id" in job_conf: - job.id = job_conf["id"] - if "account_id" in job_conf: - job.account_id = job_conf["account_id"] - else: - job.account_id = None - if "max_claimed_sites" in job_conf: - job.max_claimed_sites = job_conf["max_claimed_sites"] - if "pdfs_only" in job_conf: - job.pdfs_only = job_conf["pdfs_only"] + job.id = job_conf.get("id") + job.account_id = job_conf.get("account_id") + job.max_claimed_sites = job_conf.get("max_claimed_sites") + job.pdfs_only = job_conf.get("pdfs_only") job.save() sites = [] From 0526eb816c864bc4255b4242020f66fffa282483 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 23 Jun 2025 17:18:59 -0700 Subject: [PATCH 09/84] make psycopg dependency optional --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8e40fd54..e8ac6afc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,12 +35,12 @@ dependencies = [ "python-magic>=0.4.15", "prometheus-client>=0.20.0", "structlog>=25.1.0", - "psycopg[binary]>=3.2.6", ] license = "Apache-2.0" [project.optional-dependencies] yt-dlp = ["yt-dlp[default,curl-cffi]>=2024.7.25"] +psycopg = ["psycopg[binary]>=3.2.6"] dashboard = ["flask>=1.0", "gunicorn>=19.8.1"] warcprox = ["warcprox>=2.4.31"] rethinkdb = [ From af1aaeee34a2aee3a915d9478933cea786588156 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 23 Jun 2025 17:24:43 -0700 Subject: [PATCH 10/84] containing_page_url_pattern update --- brozzler/ydl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index b52ebf7d..e4743351 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -430,7 +430,7 @@ def get_video_captures(site, source="youtube"): account_id = site.account_id if site.account_id else None seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None if source == "youtube": - containing_page_url_pattern = "http://youtube.com/watch" # yes, video data canonicalization uses "http" + containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" # support other sources here else: containing_page_url_pattern = None From 8dcac47ae8df3b07d893703e581735eb4020ae2a Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 23 Jun 2025 17:39:07 -0700 Subject: [PATCH 11/84] type hint get_video_captures --- brozzler/ydl.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index e4743351..2d0b0cbf 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -27,6 +27,7 @@ import doublethink import structlog +from typing import List import urlcanon import yt_dlp from yt_dlp.utils import ExtractorError, match_filter_func @@ -420,9 +421,9 @@ def _try_youtube_dl(worker, ydl, site, page): return ie_result -def get_video_captures(site, source="youtube"): +def get_video_captures(site, source="youtube") -> List[str]: if not VIDEO_DATA_SOURCE: - return None + return [] if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): import psycopg @@ -449,12 +450,12 @@ def get_video_captures(site, source="youtube"): (seed, containing_page_url_pattern), ) else: - return None + return [] with psycopg.connect(VIDEO_DATA_SOURCE) as conn: with conn.cursor(row_factory=psycopg.rows.scalar_row) as cur: cur.execute(pg_query) return cur.fetchall() - return None + return [] @metrics.brozzler_ytdlp_duration_seconds.time() @@ -483,7 +484,8 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): ie_result.get("extractor") == "youtube:playlist" or ie_result.get("extractor") == "youtube:tab" ): - captured_youtube_watch_pages = get_video_captures(site, source="youtube") + captured_youtube_watch_pages = set() + captured_youtube_watch_pages.add(get_video_captures(site, source="youtube")) uncaptured_youtube_watch_pages = [] for e in ie_result.get("entries_no_dl", []): youtube_watch_url = f"https://www.youtube.com/watch?v={e['id']}" From 667feae5592146f17c8a8fae17c1724796e5bdfe Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 23 Jun 2025 17:50:43 -0700 Subject: [PATCH 12/84] ruff import block fix --- brozzler/ydl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 2d0b0cbf..e514e60f 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -24,10 +24,10 @@ import threading import time import urllib.request +from typing import List import doublethink import structlog -from typing import List import urlcanon import yt_dlp from yt_dlp.utils import ExtractorError, match_filter_func From f21d312ca99e5b6b235f4f4137eed7c5a0ec1f1c Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 23 Jun 2025 19:43:38 -0700 Subject: [PATCH 13/84] initial interface update --- brozzler/ydl.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index e514e60f..f60c5856 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -421,41 +421,41 @@ def _try_youtube_dl(worker, ydl, site, page): return ie_result -def get_video_captures(site, source="youtube") -> List[str]: - if not VIDEO_DATA_SOURCE: - return [] +class VideoDataClient: + import psycopg + from psycopg_pool import ConnectionPool - if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): - import psycopg + def __init__(self, site=None): + if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): + self.pool = ConnectionPool(VIDEO_DATA_SOURCE, min_size=1, max_size=9) + self.account_id = site.account_id if site.account_id else None + self.seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None - account_id = site.account_id if site.account_id else None - seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None + def get_video_captures_from_db(self, source="youtube") -> List[str]: if source == "youtube": containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" # support other sources here else: containing_page_url_pattern = None - if account_id and seed and source: + if self.account_id and self.seed and source: pg_query = ( "SELECT distinct(containing_page_url) from video where account_id = %s and seed = %s and containing_page_url like %s", ( - account_id, - seed, + self.account_id, + self.seed, containing_page_url_pattern, ), ) - elif seed and source: + elif self.seed and source: pg_query = ( "SELECT containing_page_url from video where seed = %s and containing_page_url like %s", - (seed, containing_page_url_pattern), + (self.seed, containing_page_url_pattern), ) - else: - return [] - with psycopg.connect(VIDEO_DATA_SOURCE) as conn: + + with self.pool.connection() as conn: with conn.cursor(row_factory=psycopg.rows.scalar_row) as cur: cur.execute(pg_query) return cur.fetchall() - return [] @metrics.brozzler_ytdlp_duration_seconds.time() @@ -485,7 +485,9 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): or ie_result.get("extractor") == "youtube:tab" ): captured_youtube_watch_pages = set() - captured_youtube_watch_pages.add(get_video_captures(site, source="youtube")) + captured_youtube_watch_pages.add( + VideoDataClient.get_video_captures(site, source="youtube") + ) uncaptured_youtube_watch_pages = [] for e in ie_result.get("entries_no_dl", []): youtube_watch_url = f"https://www.youtube.com/watch?v={e['id']}" From de4e7e0c0845257a1ea5d0c5da69cd9f3d807096 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 25 Jun 2025 18:17:56 -0700 Subject: [PATCH 14/84] VideoDataWrapper refined --- brozzler/ydl.py | 73 ++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index f60c5856..57e5553f 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -421,41 +421,56 @@ def _try_youtube_dl(worker, ydl, site, page): return ie_result -class VideoDataClient: +class VideoDataWrapper: import psycopg from psycopg_pool import ConnectionPool - def __init__(self, site=None): + def __init__(self): if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): - self.pool = ConnectionPool(VIDEO_DATA_SOURCE, min_size=1, max_size=9) - self.account_id = site.account_id if site.account_id else None - self.seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None + pool = ConnectionPool(VIDEO_DATA_SOURCE, min_size=1, max_size=9) + pool.wait() + logger.info("pg pool ready") + self.pool = pool + atexit.register(pool.close) + + def _execute_query( + self, query: str, row_factory=None, fetchone=False, fetchall=False + ) -> Optional[Any]: + with self.pool.connection() as conn: + with conn.cursor(row_factory=row_factory) as cur: + cur.execute(pg_query) + if fetchone: + return cur.fetchone() + if fetchall: + return cur.fetchall() + + def get_pg_video_captures(self, site=None, source="youtube") -> List[str]: + account_id = site.account_id if site.account_id else None + seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None - def get_video_captures_from_db(self, source="youtube") -> List[str]: if source == "youtube": containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" # support other sources here else: containing_page_url_pattern = None - if self.account_id and self.seed and source: + if account_id and seed and source: pg_query = ( "SELECT distinct(containing_page_url) from video where account_id = %s and seed = %s and containing_page_url like %s", ( - self.account_id, - self.seed, + account_id, + seed, containing_page_url_pattern, ), ) - elif self.seed and source: + elif seed and source: pg_query = ( "SELECT containing_page_url from video where seed = %s and containing_page_url like %s", - (self.seed, containing_page_url_pattern), + (seed, containing_page_url_pattern), ) - - with self.pool.connection() as conn: - with conn.cursor(row_factory=psycopg.rows.scalar_row) as cur: - cur.execute(pg_query) - return cur.fetchall() + results = self._execute_query( + pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True + ) + return results @metrics.brozzler_ytdlp_duration_seconds.time() @@ -484,17 +499,19 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): ie_result.get("extractor") == "youtube:playlist" or ie_result.get("extractor") == "youtube:tab" ): - captured_youtube_watch_pages = set() - captured_youtube_watch_pages.add( - VideoDataClient.get_video_captures(site, source="youtube") - ) - uncaptured_youtube_watch_pages = [] - for e in ie_result.get("entries_no_dl", []): - youtube_watch_url = f"https://www.youtube.com/watch?v={e['id']}" - if youtube_watch_url in captured_youtube_watch_pages: - continue - uncaptured_youtube_watch_pages.append(youtube_watch_url) - if uncaptured_youtube_watch_pages: - outlinks.add(uncaptured_youtube_watch_pages) + if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): + video_data = VideoDataWrapper() + captured_youtube_watch_pages = set() + captured_youtube_watch_pages.add( + video_data.get_pg_video_captures(site, source="youtube") + ) + uncaptured_youtube_watch_pages = [] + for e in ie_result.get("entries_no_dl", []): + youtube_watch_url = f"https://www.youtube.com/watch?v={e['id']}" + if youtube_watch_url in captured_youtube_watch_pages: + continue + uncaptured_youtube_watch_pages.append(youtube_watch_url) + if uncaptured_youtube_watch_pages: + outlinks.add(uncaptured_youtube_watch_pages) # todo: handle outlinks for instagram and soundcloud here (if anywhere) return outlinks From 046db4b6cc739b9b1ddcf76258aa4f2b4212570a Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Sun, 29 Jun 2025 19:01:20 -0700 Subject: [PATCH 15/84] VideoDataClient, generalized --- brozzler/worker.py | 1 + brozzler/ydl.py | 133 ++++++++++++++++++++++++++------------------- 2 files changed, 79 insertions(+), 55 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 7e2b254c..983f89d3 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -39,6 +39,7 @@ import brozzler import brozzler.browser from brozzler.model import VideoCaptureOptions +from brozzler.ydl import VideoDataClient from . import metrics diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 57e5553f..423dfe83 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -49,6 +49,81 @@ logger = structlog.get_logger(logger_name=__name__) +class VideoDataClient: + def __init__(self): + if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): + import psycopg + from psycopg_pool import ConnectionPool + + pool = ConnectionPool(VIDEO_DATA_SOURCE, min_size=1, max_size=9) + pool.wait() + logger.info("pg pool ready") + atexit.register(pool.close) + + self.pool = pool + + def _execute_pg_query( + self, query: str, row_factory=None, fetchone=False, fetchall=False + ) -> Optional[Any]: + with self.pool.connection() as conn: + with conn.cursor(row_factory=row_factory) as cur: + cur.execute(query) + if fetchone: + return cur.fetchone() + if fetchall: + return cur.fetchall() + + def get_pg_video_captures(self, site=None, source=None) -> List[str]: + account_id = site.account_id if site.account_id else None + seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None + + # TODO: generalize, maybe make variable? + containing_page_timestamp_pattern = "2025%" + + if source == "youtube": + containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" + # support other media sources here + + if account_id and seed and source: + pg_query = ( + "SELECT distinct(containing_page_url) from video where account_id = %s and seed = %s and containing_page_url like %s", + ( + account_id, + seed, + containing_page_url_pattern, + ), + ) + elif account_id and seed: + pg_query = ( + "SELECT distinct(containing_page_url) from video where account_id = %s and seed = %s and containing_page_timestamp like %s", + ( + account_id, + seed, + containing_page_timestamp_pattern, + ), + ) + elif seed and source: + pg_query = ( + "SELECT distinct(containing_page_url) from video where seed = %s and containing_page_url like %s", + (seed, containing_page_url_pattern), + ) + elif seed: + pg_query = ( + "SELECT distinct(containing_page_url) from video where seed = %s and containing_page_timestamp like %s", + ( + seed, + containing_page_timestamp_pattern, + ), + ) + try: + results = self._execute_query( + pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True + ) + except Exception as e: + logger.warn("postgres query failed: %s", e) + return results + + def isyoutubehost(url): # split 1 splits scheme from url, split 2 splits path from hostname, split 3 splits query string on hostname return "youtube.com" in url.split("//")[-1].split("/")[0].split("?")[0] @@ -421,58 +496,6 @@ def _try_youtube_dl(worker, ydl, site, page): return ie_result -class VideoDataWrapper: - import psycopg - from psycopg_pool import ConnectionPool - - def __init__(self): - if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): - pool = ConnectionPool(VIDEO_DATA_SOURCE, min_size=1, max_size=9) - pool.wait() - logger.info("pg pool ready") - self.pool = pool - atexit.register(pool.close) - - def _execute_query( - self, query: str, row_factory=None, fetchone=False, fetchall=False - ) -> Optional[Any]: - with self.pool.connection() as conn: - with conn.cursor(row_factory=row_factory) as cur: - cur.execute(pg_query) - if fetchone: - return cur.fetchone() - if fetchall: - return cur.fetchall() - - def get_pg_video_captures(self, site=None, source="youtube") -> List[str]: - account_id = site.account_id if site.account_id else None - seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None - - if source == "youtube": - containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" - # support other sources here - else: - containing_page_url_pattern = None - if account_id and seed and source: - pg_query = ( - "SELECT distinct(containing_page_url) from video where account_id = %s and seed = %s and containing_page_url like %s", - ( - account_id, - seed, - containing_page_url_pattern, - ), - ) - elif seed and source: - pg_query = ( - "SELECT containing_page_url from video where seed = %s and containing_page_url like %s", - (seed, containing_page_url_pattern), - ) - results = self._execute_query( - pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True - ) - return results - - @metrics.brozzler_ytdlp_duration_seconds.time() @metrics.brozzler_in_progress_ytdlps.track_inprogress() def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): @@ -500,10 +523,10 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): or ie_result.get("extractor") == "youtube:tab" ): if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): - video_data = VideoDataWrapper() + video_data = VideoDataClient() captured_youtube_watch_pages = set() captured_youtube_watch_pages.add( - video_data.get_pg_video_captures(site, source="youtube") + video_data.get_pg_video_captures_by_source(site, source="youtube") ) uncaptured_youtube_watch_pages = [] for e in ie_result.get("entries_no_dl", []): @@ -513,5 +536,5 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): uncaptured_youtube_watch_pages.append(youtube_watch_url) if uncaptured_youtube_watch_pages: outlinks.add(uncaptured_youtube_watch_pages) - # todo: handle outlinks for instagram and soundcloud here (if anywhere) + # todo: handle outlinks for instagram and soundcloud, other media source, here (if anywhere) return outlinks From 7d58a9ae3b7708dc42c874a0ef72f9cec216e4fc Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 30 Jun 2025 14:38:21 -0700 Subject: [PATCH 16/84] keep it simple for now --- brozzler/ydl.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 423dfe83..a5541f05 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -72,13 +72,14 @@ def _execute_pg_query( return cur.fetchone() if fetchall: return cur.fetchall() + return None def get_pg_video_captures(self, site=None, source=None) -> List[str]: account_id = site.account_id if site.account_id else None seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None # TODO: generalize, maybe make variable? - containing_page_timestamp_pattern = "2025%" + containing_page_timestamp_pattern = "2025%" # for future pre-dup additions if source == "youtube": containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" @@ -93,34 +94,18 @@ def get_pg_video_captures(self, site=None, source=None) -> List[str]: containing_page_url_pattern, ), ) - elif account_id and seed: - pg_query = ( - "SELECT distinct(containing_page_url) from video where account_id = %s and seed = %s and containing_page_timestamp like %s", - ( - account_id, - seed, - containing_page_timestamp_pattern, - ), - ) - elif seed and source: + elif seed and source: # account_id should usually be present pg_query = ( "SELECT distinct(containing_page_url) from video where seed = %s and containing_page_url like %s", (seed, containing_page_url_pattern), ) - elif seed: - pg_query = ( - "SELECT distinct(containing_page_url) from video where seed = %s and containing_page_timestamp like %s", - ( - seed, - containing_page_timestamp_pattern, - ), - ) try: results = self._execute_query( pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True ) except Exception as e: logger.warn("postgres query failed: %s", e) + results = [] return results From db17335ffb7fd3268a69da5ae84564d092f71e6d Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 30 Jun 2025 15:10:14 -0700 Subject: [PATCH 17/84] fix github ruff issues --- .github/workflows/setup/action.yml | 4 +++- brozzler/worker.py | 1 - brozzler/ydl.py | 32 +++++++++++++++++------------- pyproject.toml | 4 ++-- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/setup/action.yml b/.github/workflows/setup/action.yml index a0203f0f..58b2d1f3 100644 --- a/.github/workflows/setup/action.yml +++ b/.github/workflows/setup/action.yml @@ -25,5 +25,7 @@ runs: - name: Install pip dependencies run: | - uv sync --python ${{ inputs.python-version }} --extra rethinkdb --extra warcprox --extra yt-dlp + pip install .[rethinkdb,warcprox,yt-dlp,psycopg] + # setuptools required by rethinkdb==2.4.9 + pip install pytest setuptools shell: bash diff --git a/brozzler/worker.py b/brozzler/worker.py index 983f89d3..7e2b254c 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -39,7 +39,6 @@ import brozzler import brozzler.browser from brozzler.model import VideoCaptureOptions -from brozzler.ydl import VideoDataClient from . import metrics diff --git a/brozzler/ydl.py b/brozzler/ydl.py index a5541f05..685e1e59 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -24,12 +24,14 @@ import threading import time import urllib.request -from typing import List +from typing import Any, List, Optional import doublethink +import psycopg import structlog import urlcanon import yt_dlp +from psycopg_pool import ConnectionPool, PoolTimeout from yt_dlp.utils import ExtractorError, match_filter_func import brozzler @@ -38,7 +40,6 @@ thread_local = threading.local() - PROXY_ATTEMPTS = 4 YTDLP_WAIT = 10 YTDLP_MAX_REDIRECTS = 5 @@ -52,26 +53,29 @@ class VideoDataClient: def __init__(self): if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): - import psycopg - from psycopg_pool import ConnectionPool - pool = ConnectionPool(VIDEO_DATA_SOURCE, min_size=1, max_size=9) pool.wait() logger.info("pg pool ready") - atexit.register(pool.close) + # atexit.register(pool.close) self.pool = pool def _execute_pg_query( self, query: str, row_factory=None, fetchone=False, fetchall=False ) -> Optional[Any]: - with self.pool.connection() as conn: - with conn.cursor(row_factory=row_factory) as cur: - cur.execute(query) - if fetchone: - return cur.fetchone() - if fetchall: - return cur.fetchall() + try: + with self.pool.connection() as conn: + with conn.cursor(row_factory=row_factory) as cur: + cur.execute(query) + if fetchone: + return cur.fetchone() + if fetchall: + return cur.fetchall() + except PoolTimeout as e: + logger.warn("hit PoolTimeout: %s", e) + self.pool.check() + except Exception as e: + logger.warn("postgres query failed: %s", e) return None def get_pg_video_captures(self, site=None, source=None) -> List[str]: @@ -79,7 +83,7 @@ def get_pg_video_captures(self, site=None, source=None) -> List[str]: seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None # TODO: generalize, maybe make variable? - containing_page_timestamp_pattern = "2025%" # for future pre-dup additions + # containing_page_timestamp_pattern = "2025%" # for future pre-dup additions if source == "youtube": containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" diff --git a/pyproject.toml b/pyproject.toml index e8ac6afc..4d7e0c1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "brozzler" -version = "1.7.0" +version = "1.7.1" authors = [ { name="Noah Levitt", email="nlevitt@archive.org" }, ] @@ -40,7 +40,7 @@ license = "Apache-2.0" [project.optional-dependencies] yt-dlp = ["yt-dlp[default,curl-cffi]>=2024.7.25"] -psycopg = ["psycopg[binary]>=3.2.6"] +psycopg = ["psycopg[binary,pool]>=3.2.6"] dashboard = ["flask>=1.0", "gunicorn>=19.8.1"] warcprox = ["warcprox>=2.4.31"] rethinkdb = [ From b4b950c0fca1b015a580de543e505d0adfd51dab Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 30 Jun 2025 17:27:14 -0700 Subject: [PATCH 18/84] self._video_data in worker --- brozzler/worker.py | 5 +++++ brozzler/ydl.py | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 7e2b254c..fa4e7e2f 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -21,6 +21,7 @@ import datetime import io import json +import os import socket import threading import time @@ -39,6 +40,7 @@ import brozzler import brozzler.browser from brozzler.model import VideoCaptureOptions +from brozzler.ydl import VideoDataClient from . import metrics @@ -56,6 +58,7 @@ class BrozzlerWorker: SITE_SESSION_MINUTES = 15 HEADER_REQUEST_TIMEOUT = 30 FETCH_URL_TIMEOUT = 60 + VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") def __init__( self, @@ -88,6 +91,8 @@ def __init__( self._service_registry = service_registry self._ytdlp_proxy_endpoints = ytdlp_proxy_endpoints self._max_browsers = max_browsers + if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): + self._video_data = VideoDataClient() self._warcprox_auto = warcprox_auto self._proxy = proxy diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 685e1e59..f2568313 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -51,14 +51,16 @@ class VideoDataClient: + import psycopg + from psycopg_pool import ConnectionPool, PoolTimeout + def __init__(self): - if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): - pool = ConnectionPool(VIDEO_DATA_SOURCE, min_size=1, max_size=9) - pool.wait() - logger.info("pg pool ready") - # atexit.register(pool.close) + pool = ConnectionPool(VIDEO_DATA_SOURCE, min_size=1, max_size=9) + pool.wait() + logger.info("pg pool ready") + # atexit.register(pool.close) - self.pool = pool + self.pool = pool def _execute_pg_query( self, query: str, row_factory=None, fetchone=False, fetchall=False @@ -512,10 +514,11 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): or ie_result.get("extractor") == "youtube:tab" ): if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): - video_data = VideoDataClient() captured_youtube_watch_pages = set() captured_youtube_watch_pages.add( - video_data.get_pg_video_captures_by_source(site, source="youtube") + worker._video_data.get_pg_video_captures_by_source( + site, source="youtube" + ) ) uncaptured_youtube_watch_pages = [] for e in ie_result.get("entries_no_dl", []): From 43151027f3fb25aaf0543af8dd85b089bf3038e6 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 30 Jun 2025 20:26:48 -0700 Subject: [PATCH 19/84] dataclass VideoDataRecord --- brozzler/ydl.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index f2568313..e1043a56 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -31,6 +31,7 @@ import structlog import urlcanon import yt_dlp +from dataclasses import dataclass from psycopg_pool import ConnectionPool, PoolTimeout from yt_dlp.utils import ExtractorError, match_filter_func @@ -49,6 +50,26 @@ logger = structlog.get_logger(logger_name=__name__) +@dataclass(frozen=True) +class VideoDataRecord: + crawl_job_id: int + is_test_crawl: bool + seed_id: int + collection_id: int + containing_page_timestamp: str + containing_page_digest: str + containing_page_media_index: int + containing_page_media_count: int + video_digest: str + video_timestamp: str + video_mimetype: str + video_http_status: int + video_size: int + containing_page_url: str + video_url: str + video_title: str + video_source_id: str + class VideoDataClient: import psycopg @@ -114,6 +135,9 @@ def get_pg_video_captures(self, site=None, source=None) -> List[str]: results = [] return results + def create_video_capture_record(self, video_capture_record): + + def isyoutubehost(url): # split 1 splits scheme from url, split 2 splits path from hostname, split 3 splits query string on hostname From 2fc30b029c27e7f2d7490099f23598cb77c79c86 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 30 Jun 2025 20:29:28 -0700 Subject: [PATCH 20/84] dataclass VideoCaptureRecord instead --- brozzler/ydl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index e1043a56..a1963776 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -51,7 +51,7 @@ logger = structlog.get_logger(logger_name=__name__) @dataclass(frozen=True) -class VideoDataRecord: +class VideoCaptureRecord: crawl_job_id: int is_test_crawl: bool seed_id: int From 2422e9da0414a607efb25e3bd71d121ddc52066d Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 30 Jun 2025 20:49:37 -0700 Subject: [PATCH 21/84] def create_video_capture_record minimally --- brozzler/ydl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index a1963776..83ff378b 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -50,6 +50,7 @@ logger = structlog.get_logger(logger_name=__name__) +# video_title and video_source_id are new fields, from yt-dlp metadata @dataclass(frozen=True) class VideoCaptureRecord: crawl_job_id: int @@ -136,7 +137,8 @@ def get_pg_video_captures(self, site=None, source=None) -> List[str]: return results def create_video_capture_record(self, video_capture_record): - + # to be implemented + pass def isyoutubehost(url): From d4e0aa67ec25af756bac823e79faab92d445d38c Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 1 Jul 2025 14:53:59 -0700 Subject: [PATCH 22/84] more new fields --- brozzler/ydl.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 83ff378b..c330c848 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -50,7 +50,8 @@ logger = structlog.get_logger(logger_name=__name__) -# video_title and video_source_id are new fields, from yt-dlp metadata + +# video_title, video_display_id, video_resolution, video_capture_status are new fields, mostly from yt-dlp metadata @dataclass(frozen=True) class VideoCaptureRecord: crawl_job_id: int @@ -69,7 +70,11 @@ class VideoCaptureRecord: containing_page_url: str video_url: str video_title: str - video_source_id: str + video_display_id: ( + str # aka yt-dlp metadata as display_id, e.g., youtube watch page v param + ) + video_resolution: str + video_capture_status: str # recrawl? what else? class VideoDataClient: From 825de5f728ec772a21cf015456d5eb15749fac45 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 1 Jul 2025 18:23:48 -0700 Subject: [PATCH 23/84] worker._video_data and seed_id mostly --- brozzler/ydl.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index c330c848..ca45c407 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -45,8 +45,6 @@ YTDLP_WAIT = 10 YTDLP_MAX_REDIRECTS = 5 -VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") - logger = structlog.get_logger(logger_name=__name__) @@ -107,9 +105,9 @@ def _execute_pg_query( logger.warn("postgres query failed: %s", e) return None - def get_pg_video_captures(self, site=None, source=None) -> List[str]: + def get_video_captures(self, site=None, source=None) -> List[str]: account_id = site.account_id if site.account_id else None - seed = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None + seed_id = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None # TODO: generalize, maybe make variable? # containing_page_timestamp_pattern = "2025%" # for future pre-dup additions @@ -118,20 +116,18 @@ def get_pg_video_captures(self, site=None, source=None) -> List[str]: containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" # support other media sources here - if account_id and seed and source: + if account_id and seed_id and source: pg_query = ( - "SELECT distinct(containing_page_url) from video where account_id = %s and seed = %s and containing_page_url like %s", + "SELECT containing_page_url from video where account_id = %s and seed_id = %s and containing_page_url like %s", ( account_id, - seed, + seed_id, containing_page_url_pattern, ), ) - elif seed and source: # account_id should usually be present - pg_query = ( - "SELECT distinct(containing_page_url) from video where seed = %s and containing_page_url like %s", - (seed, containing_page_url_pattern), - ) + else: + logger.warn("missing account_id, seed_id, or source") + results = [] try: results = self._execute_query( pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True @@ -142,7 +138,7 @@ def get_pg_video_captures(self, site=None, source=None) -> List[str]: return results def create_video_capture_record(self, video_capture_record): - # to be implemented + # note: brozzler postcrawl step 72 includes info from crawl-log pass @@ -539,15 +535,16 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): logger.info("tempdir for yt-dlp", tempdir=tempdir) ydl = _build_youtube_dl(worker, tempdir, site, page, ytdlp_proxy_endpoints) ie_result = _try_youtube_dl(worker, ydl, site, page) + # print(ie_result) outlinks = set() if ie_result and ( ie_result.get("extractor") == "youtube:playlist" or ie_result.get("extractor") == "youtube:tab" ): - if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): + if worker._video_data: captured_youtube_watch_pages = set() captured_youtube_watch_pages.add( - worker._video_data.get_pg_video_captures_by_source( + worker._video_data.get_video_captures_by_source( site, source="youtube" ) ) From 701707c7ceb8b792afaa7ae5da2cc1af3496879e Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 1 Jul 2025 23:47:51 -0700 Subject: [PATCH 24/84] save video_record here? --- brozzler/ydl.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index ca45c407..4130f13d 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -408,9 +408,9 @@ def ydl_postprocess_hook(d): return ydl -def _remember_videos(page, pushed_videos=None): +def _remember_videos(page, worker, info_json=None, pushed_videos=None): """ - Saves info about videos captured by yt-dlp in `page.videos`. + Saves info about videos captured by yt-dlp in `page.videos` and postgres. """ if "videos" not in page: page.videos = [] @@ -422,6 +422,29 @@ def _remember_videos(page, pushed_videos=None): "content-type": pushed_video["content-type"], "content-length": pushed_video["content-length"], } + # grab info from info_json if it's available + video_record = worker._video_data.VideoCaptureRecord() + video_record.crawl_job_id = None + video_record.is_test_crawl = None + video_record.seed_id = None + video_record.collection_id = None + video_record.containing_page_timestamp = None + video_record.containing_page_digest = None + video_record.containing_page_media_index = None + video_record.containing_page_media_count = None + video_record.video_digest = None + video_record.video_timestamp = None + video_record.video_mimetype = pushed_video["content-type"] + video_record.video_http_status = pushed_video["response_code"] + video_record.video_size = None + video_record.containing_page_url = None + video_record.video_url = pushed_video["url"] + video_record.video_title = None + video_record.video_display_id = None + video_record.video_resolution = None + video_record.video_capture_status = None # "recrawl" + worker._video_data.save_video_capture_record(video_record) + logger.debug("embedded video", video=video) page.videos.append(video) @@ -496,9 +519,9 @@ def _try_youtube_dl(worker, ydl, site, page): logger.info("ytdlp completed successfully") - _remember_videos(page, ydl.pushed_videos) + info_json = json.dumps(ie_result, sort_keys=True, indent=4) + _remember_videos(page, info_json, ydl.pushed_videos) if worker._using_warcprox(site): - info_json = json.dumps(ie_result, sort_keys=True, indent=4) logger.info( "sending WARCPROX_WRITE_RECORD request to warcprox with yt-dlp json", url=ydl.url, From 3c1328ff533da2a1d4971ee5044ac9aa9f02c9ea Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 2 Jul 2025 17:06:41 -0700 Subject: [PATCH 25/84] save_video_capture_record --- brozzler/ydl.py | 46 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 4130f13d..6e2f3e9f 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -138,8 +138,20 @@ def get_video_captures(self, site=None, source=None) -> List[str]: return results def create_video_capture_record(self, video_capture_record): - # note: brozzler postcrawl step 72 includes info from crawl-log - pass + # note: brozzler postcrawl step 72 includes info from crawl-log — watch for differences! + # TODO: needs added fields added to postgres table, refinement + pg_query = ( + f"INSERT INTO video ({VideoCaptureRecord - items}) VALUES (%s, %s, ...)", + VideoCaptureRecord - values, + ) + try: + results = self._execute_query( + pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True + ) + except Exception as e: + logger.warn("postgres query failed: %s", e) + results = [] + return results def isyoutubehost(url): @@ -408,7 +420,7 @@ def ydl_postprocess_hook(d): return ydl -def _remember_videos(page, worker, info_json=None, pushed_videos=None): +def _remember_videos(page, site, worker, ydl, ie_result, pushed_videos=None): """ Saves info about videos captured by yt-dlp in `page.videos` and postgres. """ @@ -422,12 +434,14 @@ def _remember_videos(page, worker, info_json=None, pushed_videos=None): "content-type": pushed_video["content-type"], "content-length": pushed_video["content-length"], } - # grab info from info_json if it's available + + warc_prefix_items = site.warcprox_meta["warc-prefix"].split("-") + video_record = worker._video_data.VideoCaptureRecord() - video_record.crawl_job_id = None - video_record.is_test_crawl = None - video_record.seed_id = None - video_record.collection_id = None + video_record.crawl_job_id = site.job_id + video_record.is_test_crawl = True if warc_prefix_items[2] == "TEST" else False + video_record.seed_id = site.ait_seed_id + video_record.collection_id = int(warc_prefix_items[1]) video_record.containing_page_timestamp = None video_record.containing_page_digest = None video_record.containing_page_media_index = None @@ -436,13 +450,17 @@ def _remember_videos(page, worker, info_json=None, pushed_videos=None): video_record.video_timestamp = None video_record.video_mimetype = pushed_video["content-type"] video_record.video_http_status = pushed_video["response_code"] - video_record.video_size = None - video_record.containing_page_url = None + video_record.video_size = pushed_video["content-length"] # probably? + video_record.containing_page_url = str( + urlcanon.aggressive(ydl.url) + ) # probably? video_record.video_url = pushed_video["url"] - video_record.video_title = None - video_record.video_display_id = None - video_record.video_resolution = None - video_record.video_capture_status = None # "recrawl" + # note: ie_result may not be correct when multiple videos present + video_record.video_title = ie_result.get("title") + video_record.video_display_id = ie_result.get("display_id") + video_record.video_resolution = ie_result.get("resolution") + video_record.video_capture_status = None # "recrawl" maybe + worker._video_data.save_video_capture_record(video_record) logger.debug("embedded video", video=video) From bc987b2a2737cd8fdbba17001593a513802528fc Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 14 Jul 2025 17:07:33 -0700 Subject: [PATCH 26/84] add get_recent_video_capture, mostly --- brozzler/ydl.py | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 6e2f3e9f..ee837ada 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -24,7 +24,7 @@ import threading import time import urllib.request -from typing import Any, List, Optional +from typing import Any, Bool, List, Optional import doublethink import psycopg @@ -105,6 +105,33 @@ def _execute_pg_query( logger.warn("postgres query failed: %s", e) return None + def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: + account_id = site.account_id if site.account_id else None + seed_id = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None + + if account_id and seed_id and containing_page_url: + # check for postgres query for most recent record + pg_query = ( + "SELECT * from video where account_id = %s and seed_id = %s and containing_page_url = %s LIMIT 1", + ( + account_id, + seed_id, + str(urlcanon.aggressive(containing_page_url)) + ) + ) + try: + results = self._execute_query( + pg_query, fetchall=True + ) + except Exception as e: + logger.warn("postgres query failed: %s", e) + results = [] + else: + logger.warn("missing account_id, seed_id, or containing_page_url") + results = [] + + return results + def get_video_captures(self, site=None, source=None) -> List[str]: account_id = site.account_id if site.account_id else None seed_id = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None @@ -125,16 +152,17 @@ def get_video_captures(self, site=None, source=None) -> List[str]: containing_page_url_pattern, ), ) + try: + results = self._execute_query( + pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True + ) + except Exception as e: + logger.warn("postgres query failed: %s", e) + results = [] else: logger.warn("missing account_id, seed_id, or source") results = [] - try: - results = self._execute_query( - pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True - ) - except Exception as e: - logger.warn("postgres query failed: %s", e) - results = [] + return results def create_video_capture_record(self, video_capture_record): @@ -591,7 +619,7 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): ) uncaptured_youtube_watch_pages = [] for e in ie_result.get("entries_no_dl", []): - youtube_watch_url = f"https://www.youtube.com/watch?v={e['id']}" + youtube_watch_url = str(urlcanon.aggressive(f"http://www.youtube.com/watch?v={e['id']}")) if youtube_watch_url in captured_youtube_watch_pages: continue uncaptured_youtube_watch_pages.append(youtube_watch_url) From 3af824727771c213ea62df5c42a405990dd8f643 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 15 Jul 2025 14:42:53 -0700 Subject: [PATCH 27/84] updates for QA deploy --- brozzler/ydl.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index ee837ada..ab1ccb27 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -113,16 +113,10 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: # check for postgres query for most recent record pg_query = ( "SELECT * from video where account_id = %s and seed_id = %s and containing_page_url = %s LIMIT 1", - ( - account_id, - seed_id, - str(urlcanon.aggressive(containing_page_url)) - ) + (account_id, seed_id, str(urlcanon.aggressive(containing_page_url))), ) try: - results = self._execute_query( - pg_query, fetchall=True - ) + results = self._execute_query(pg_query, fetchall=True) except Exception as e: logger.warn("postgres query failed: %s", e) results = [] @@ -462,7 +456,8 @@ def _remember_videos(page, site, worker, ydl, ie_result, pushed_videos=None): "content-type": pushed_video["content-type"], "content-length": pushed_video["content-length"], } - + """ + # WIP: add new video record to QA postgres here, or in postcrawl only? warc_prefix_items = site.warcprox_meta["warc-prefix"].split("-") video_record = worker._video_data.VideoCaptureRecord() @@ -490,7 +485,7 @@ def _remember_videos(page, site, worker, ydl, ie_result, pushed_videos=None): video_record.video_capture_status = None # "recrawl" maybe worker._video_data.save_video_capture_record(video_record) - + """ logger.debug("embedded video", video=video) page.videos.append(video) @@ -619,10 +614,15 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): ) uncaptured_youtube_watch_pages = [] for e in ie_result.get("entries_no_dl", []): - youtube_watch_url = str(urlcanon.aggressive(f"http://www.youtube.com/watch?v={e['id']}")) + # note: http needed for match + youtube_watch_url = str( + urlcanon.aggressive(f"http://www.youtube.com/watch?v={e['id']}") + ) if youtube_watch_url in captured_youtube_watch_pages: continue - uncaptured_youtube_watch_pages.append(youtube_watch_url) + uncaptured_youtube_watch_pages.append( + f"https://www.youtube.com/watch?v={e['id']}" + ) if uncaptured_youtube_watch_pages: outlinks.add(uncaptured_youtube_watch_pages) # todo: handle outlinks for instagram and soundcloud, other media source, here (if anywhere) From 5b967b9738a55cb5c91e1ac73c5a0da74c68a1c3 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 15 Jul 2025 17:16:00 -0700 Subject: [PATCH 28/84] log urls we skipped adding to outlinks --- brozzler/ydl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index ab1ccb27..2b20f4c1 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -619,6 +619,7 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): urlcanon.aggressive(f"http://www.youtube.com/watch?v={e['id']}") ) if youtube_watch_url in captured_youtube_watch_pages: + logger.info("skipping adding %s to outlinks", youtube_watch_url) continue uncaptured_youtube_watch_pages.append( f"https://www.youtube.com/watch?v={e['id']}" From 3d377e79438cbf59d297ea6cfd17d2f898590c1b Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 15 Jul 2025 19:31:08 -0700 Subject: [PATCH 29/84] updates from qa deploy --- brozzler/worker.py | 2 +- brozzler/ydl.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index fa4e7e2f..2f1b64b9 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -91,7 +91,7 @@ def __init__( self._service_registry = service_registry self._ytdlp_proxy_endpoints = ytdlp_proxy_endpoints self._max_browsers = max_browsers - if VIDEO_DATA_SOURCE and VIDEO_DATA_SOURCE.startswith("postgresql"): + if self.VIDEO_DATA_SOURCE and self.VIDEO_DATA_SOURCE.startswith("postgresql"): self._video_data = VideoDataClient() self._warcprox_auto = warcprox_auto diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 2b20f4c1..1f60aeb9 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -24,7 +24,7 @@ import threading import time import urllib.request -from typing import Any, Bool, List, Optional +from typing import Any, List, Optional import doublethink import psycopg @@ -79,8 +79,10 @@ class VideoDataClient: import psycopg from psycopg_pool import ConnectionPool, PoolTimeout + VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") + def __init__(self): - pool = ConnectionPool(VIDEO_DATA_SOURCE, min_size=1, max_size=9) + pool = ConnectionPool(self.VIDEO_DATA_SOURCE, min_size=1, max_size=9) pool.wait() logger.info("pg pool ready") # atexit.register(pool.close) From 3adae49be7dba6cca8f1c43dfb590d6b3b98f0fe Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 16 Jul 2025 16:30:05 -0700 Subject: [PATCH 30/84] more updates post-QA-deploy --- brozzler/ydl.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 1f60aeb9..4a0e88b7 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -114,11 +114,11 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: if account_id and seed_id and containing_page_url: # check for postgres query for most recent record pg_query = ( - "SELECT * from video where account_id = %s and seed_id = %s and containing_page_url = %s LIMIT 1", + "SELECT * from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY video_timestamp LIMIT 1", (account_id, seed_id, str(urlcanon.aggressive(containing_page_url))), ) try: - results = self._execute_query(pg_query, fetchall=True) + results = self._execute_pg_query(pg_query, fetchall=True) except Exception as e: logger.warn("postgres query failed: %s", e) results = [] @@ -149,7 +149,7 @@ def get_video_captures(self, site=None, source=None) -> List[str]: ), ) try: - results = self._execute_query( + results = self._execute_pg_query( pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True ) except Exception as e: @@ -162,16 +162,15 @@ def get_video_captures(self, site=None, source=None) -> List[str]: return results def create_video_capture_record(self, video_capture_record): - # note: brozzler postcrawl step 72 includes info from crawl-log — watch for differences! + # NOTE: we want to do this in brozzler postcrawl for now + # WIP # TODO: needs added fields added to postgres table, refinement pg_query = ( f"INSERT INTO video ({VideoCaptureRecord - items}) VALUES (%s, %s, ...)", VideoCaptureRecord - values, ) try: - results = self._execute_query( - pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True - ) + results = self._execute_pg_query(pg_query) except Exception as e: logger.warn("postgres query failed: %s", e) results = [] @@ -444,9 +443,10 @@ def ydl_postprocess_hook(d): return ydl -def _remember_videos(page, site, worker, ydl, ie_result, pushed_videos=None): +# new maybe? def _remember_videos(page, site, worker, ydl, ie_result, pushed_videos=None): +def _remember_videos(page, pushed_videos=None): """ - Saves info about videos captured by yt-dlp in `page.videos` and postgres. + Saves info about videos captured by yt-dlp in `page.videos` """ if "videos" not in page: page.videos = [] @@ -480,13 +480,11 @@ def _remember_videos(page, site, worker, ydl, ie_result, pushed_videos=None): urlcanon.aggressive(ydl.url) ) # probably? video_record.video_url = pushed_video["url"] - # note: ie_result may not be correct when multiple videos present + # note: NEW! ie_result may not be correct when multiple videos present video_record.video_title = ie_result.get("title") video_record.video_display_id = ie_result.get("display_id") video_record.video_resolution = ie_result.get("resolution") video_record.video_capture_status = None # "recrawl" maybe - - worker._video_data.save_video_capture_record(video_record) """ logger.debug("embedded video", video=video) page.videos.append(video) @@ -563,7 +561,7 @@ def _try_youtube_dl(worker, ydl, site, page): logger.info("ytdlp completed successfully") info_json = json.dumps(ie_result, sort_keys=True, indent=4) - _remember_videos(page, info_json, ydl.pushed_videos) + _remember_videos(page, ydl.pushed_videos) if worker._using_warcprox(site): logger.info( "sending WARCPROX_WRITE_RECORD request to warcprox with yt-dlp json", @@ -610,9 +608,7 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): if worker._video_data: captured_youtube_watch_pages = set() captured_youtube_watch_pages.add( - worker._video_data.get_video_captures_by_source( - site, source="youtube" - ) + worker._video_data.get_video_captures(site, source="youtube") ) uncaptured_youtube_watch_pages = [] for e in ie_result.get("entries_no_dl", []): From f979bbf15412b90b442c443470e6e13b582b5e47 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 17 Jul 2025 19:09:58 -0700 Subject: [PATCH 31/84] even more updates post-QA-deploy --- brozzler/ydl.py | 65 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 4a0e88b7..6f457c42 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -108,8 +108,10 @@ def _execute_pg_query( return None def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: - account_id = site.account_id if site.account_id else None - seed_id = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None + account_id = site["account_id"] if site["account_id"] else None + seed_id = ( + site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None + ) if account_id and seed_id and containing_page_url: # check for postgres query for most recent record @@ -129,8 +131,10 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: return results def get_video_captures(self, site=None, source=None) -> List[str]: - account_id = site.account_id if site.account_id else None - seed_id = site.metadata.ait_seed_id if site.metadata.ait_seed_id else None + account_id = site["account_id"] if site["account_id"] else None + seed_id = ( + site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None + ) # TODO: generalize, maybe make variable? # containing_page_timestamp_pattern = "2025%" # for future pre-dup additions @@ -465,7 +469,7 @@ def _remember_videos(page, pushed_videos=None): video_record = worker._video_data.VideoCaptureRecord() video_record.crawl_job_id = site.job_id video_record.is_test_crawl = True if warc_prefix_items[2] == "TEST" else False - video_record.seed_id = site.ait_seed_id + video_record.seed_id = site["metadata"]["ait_seed_id"] video_record.collection_id = int(warc_prefix_items[1]) video_record.containing_page_timestamp = None video_record.containing_page_digest = None @@ -606,23 +610,42 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): or ie_result.get("extractor") == "youtube:tab" ): if worker._video_data: - captured_youtube_watch_pages = set() - captured_youtube_watch_pages.add( - worker._video_data.get_video_captures(site, source="youtube") + logger.info( + "checking for previously captured youtube watch pages for account %s, seed_id %s", + site["account_id"], + site["metadata"]["ait_seed_id"], ) - uncaptured_youtube_watch_pages = [] - for e in ie_result.get("entries_no_dl", []): - # note: http needed for match - youtube_watch_url = str( - urlcanon.aggressive(f"http://www.youtube.com/watch?v={e['id']}") - ) - if youtube_watch_url in captured_youtube_watch_pages: - logger.info("skipping adding %s to outlinks", youtube_watch_url) - continue - uncaptured_youtube_watch_pages.append( - f"https://www.youtube.com/watch?v={e['id']}" + try: + captured_youtube_watch_pages = set() + captured_youtube_watch_pages.update( + worker._video_data.get_video_captures(site, source="youtube") ) - if uncaptured_youtube_watch_pages: - outlinks.add(uncaptured_youtube_watch_pages) + uncaptured_youtube_watch_pages = [] + for e in ie_result.get("entries_no_dl", []): + # note: http needed for match + youtube_watch_url = str( + urlcanon.aggressive( + f"http://www.youtube.com/watch?v={e['id']}" + ) + ) + if youtube_watch_url in captured_youtube_watch_pages: + logger.info( + "skipping adding %s to yt-dlp outlinks", + youtube_watch_url, + ) + continue + uncaptured_youtube_watch_pages.append( + f"https://www.youtube.com/watch?v={e['id']}" + ) + except Exception as e: + logger.warning("hit exception processing worker._video_data: %s", e) + if uncaptured_youtube_watch_pages: + outlinks.update(uncaptured_youtube_watch_pages) + else: + outlinks = { + "https://www.youtube.com/watch?v=%s" % e["id"] + for e in ie_result.get("entries_no_dl", []) + } + # todo: handle outlinks for instagram and soundcloud, other media source, here (if anywhere) return outlinks From ada404712c463d40dc24df25447841a673cfe456 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 18 Jul 2025 16:47:03 -0700 Subject: [PATCH 32/84] add account_id to model's populate defaults --- brozzler/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/brozzler/model.py b/brozzler/model.py index d7c1ac2c..70324bfd 100644 --- a/brozzler/model.py +++ b/brozzler/model.py @@ -201,6 +201,8 @@ class Job(doublethink.Document, ElapsedMixIn): def populate_defaults(self): if "status" not in self: self.status = "ACTIVE" + if "account_id" not in self: + self.account_id = None if "pdfs_only" not in self: self.pdfs_only = False if "starts_and_stops" not in self: From c55b7651e3652fdd7b6f5cbc915311faade89938 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 21 Jul 2025 15:53:36 -0700 Subject: [PATCH 33/84] more populate_defaults maybe --- brozzler/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/brozzler/model.py b/brozzler/model.py index 70324bfd..ac4c1099 100644 --- a/brozzler/model.py +++ b/brozzler/model.py @@ -264,6 +264,8 @@ def populate_defaults(self): self.scope = {} if "video_capture" not in self: self.video_capture = VideoCaptureOptions.ENABLE_VIDEO_CAPTURE.value + if "account_id" not in self: + self.account_id = None # backward compatibility if "surt" in self.scope: From 5f3b3600c25d6d0362eb64f1275b53877c309e60 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 21 Jul 2025 15:55:04 -0700 Subject: [PATCH 34/84] use only last proxyrack endpoint for non-youtube-watch urls --- brozzler/ydl.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 6f457c42..ef741b40 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -182,8 +182,8 @@ def create_video_capture_record(self, video_capture_record): def isyoutubehost(url): - # split 1 splits scheme from url, split 2 splits path from hostname, split 3 splits query string on hostname - return "youtube.com" in url.split("//")[-1].split("/")[0].split("?")[0] + # split 1 splits scheme from url, split 2 splits path from hostname + return "youtube.com" in url.split("//")[-1].split("/")[0] class ExtraHeaderAdder(urllib.request.BaseHandler): @@ -427,7 +427,10 @@ def ydl_postprocess_hook(d): ytdlp_url = page.redirect_url if page.redirect_url else page.url is_youtube_host = isyoutubehost(ytdlp_url) if is_youtube_host and ytdlp_proxy_endpoints: - ydl_opts["proxy"] = random.choice(ytdlp_proxy_endpoints) + if 'com/watch' not in ytdlp_url: + ydl_opts["proxy"] = ytdlp_proxy_endpoints[4] + else: + ydl_opts["proxy"] = random.choice(ytdlp_proxy_endpoints[0:4]) # don't log proxy value secrets ytdlp_proxy_for_logs = ( ydl_opts["proxy"].split("@")[1] if "@" in ydl_opts["proxy"] else "@@@" From 0c4ddb7488cc201376ec540f11a519c8f8c39bf6 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 21 Jul 2025 18:15:36 -0700 Subject: [PATCH 35/84] preserve pre-2025.07.21 mtime handling --- brozzler/ydl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index ef741b40..d5671988 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -422,6 +422,8 @@ def ydl_postprocess_hook(d): # recommended to avoid bot detection "sleep_interval": 7, "max_sleep_interval": 27, + # preserve pre-2025.07.21 mtime handling + "updatetime": True, } ytdlp_url = page.redirect_url if page.redirect_url else page.url From 098551811ce1958888d8493e19b0e7b45cea28e0 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 23 Jul 2025 16:14:38 -0700 Subject: [PATCH 36/84] account_id first (alphabetical order?) --- brozzler/job_schema.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brozzler/job_schema.yaml b/brozzler/job_schema.yaml index 4f7afe26..acaf20af 100644 --- a/brozzler/job_schema.yaml +++ b/brozzler/job_schema.yaml @@ -1,10 +1,10 @@ -id: +account_id: type: - string - integer required: false -account_id: +id: type: - string - integer From 825cf7041241247f1806f628d3bde093067f1c88 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 10:07:40 -0700 Subject: [PATCH 37/84] skip creating video records in brozzler for now --- brozzler/ydl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index d5671988..0ab2accd 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -164,7 +164,7 @@ def get_video_captures(self, site=None, source=None) -> List[str]: results = [] return results - + """ def create_video_capture_record(self, video_capture_record): # NOTE: we want to do this in brozzler postcrawl for now # WIP @@ -179,7 +179,7 @@ def create_video_capture_record(self, video_capture_record): logger.warn("postgres query failed: %s", e) results = [] return results - + """ def isyoutubehost(url): # split 1 splits scheme from url, split 2 splits path from hostname From ffaec9dddabb4bb618b2754c64cb41d964c55867 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 10:10:42 -0700 Subject: [PATCH 38/84] ruff and isort --- brozzler/ydl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 0ab2accd..7ec13103 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -24,6 +24,7 @@ import threading import time import urllib.request +from dataclasses import dataclass from typing import Any, List, Optional import doublethink @@ -31,7 +32,6 @@ import structlog import urlcanon import yt_dlp -from dataclasses import dataclass from psycopg_pool import ConnectionPool, PoolTimeout from yt_dlp.utils import ExtractorError, match_filter_func @@ -164,6 +164,7 @@ def get_video_captures(self, site=None, source=None) -> List[str]: results = [] return results + """ def create_video_capture_record(self, video_capture_record): # NOTE: we want to do this in brozzler postcrawl for now @@ -181,6 +182,7 @@ def create_video_capture_record(self, video_capture_record): return results """ + def isyoutubehost(url): # split 1 splits scheme from url, split 2 splits path from hostname return "youtube.com" in url.split("//")[-1].split("/")[0] @@ -429,7 +431,7 @@ def ydl_postprocess_hook(d): ytdlp_url = page.redirect_url if page.redirect_url else page.url is_youtube_host = isyoutubehost(ytdlp_url) if is_youtube_host and ytdlp_proxy_endpoints: - if 'com/watch' not in ytdlp_url: + if "com/watch" not in ytdlp_url: ydl_opts["proxy"] = ytdlp_proxy_endpoints[4] else: ydl_opts["proxy"] = random.choice(ytdlp_proxy_endpoints[0:4]) From c3e4046f462f96faf19bd3296c0cd67849cb3e8d Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 10:48:10 -0700 Subject: [PATCH 39/84] do use uv, not pip --- .github/workflows/setup/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/setup/action.yml b/.github/workflows/setup/action.yml index 58b2d1f3..bddc120a 100644 --- a/.github/workflows/setup/action.yml +++ b/.github/workflows/setup/action.yml @@ -25,7 +25,5 @@ runs: - name: Install pip dependencies run: | - pip install .[rethinkdb,warcprox,yt-dlp,psycopg] - # setuptools required by rethinkdb==2.4.9 - pip install pytest setuptools + uv sync --python ${{ inputs.python-version }} --extra rethinkdb --extra warcprox --extra yt-dlp --extra psycopg shell: bash From 951ce8c92f8011a2efd2e68deee39b60015c48bd Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 11:04:02 -0700 Subject: [PATCH 40/84] skip 3.14 for now with psycopg; try 3.13 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b48e3c2b..149bbbca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: ['3.9', '3.12', '3.14'] + version: ['3.9', '3.12', '3.13'] steps: - uses: actions/checkout@v4 From 82d4e2f14bd890159b2d5e87a70a338e6dbe0767 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 11:38:13 -0700 Subject: [PATCH 41/84] job schema tweak -- more info: https://github.com/pyeve/cerberus/issues/220 --- brozzler/job_schema.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/brozzler/job_schema.yaml b/brozzler/job_schema.yaml index acaf20af..5da74f84 100644 --- a/brozzler/job_schema.yaml +++ b/brozzler/job_schema.yaml @@ -1,9 +1,3 @@ -account_id: - type: - - string - - integer - required: false - id: type: - string @@ -111,3 +105,9 @@ max_claimed_sites: pdfs_only: type: boolean + +account_id: + type: + - string + - integer + required: false From 0fd859fbc8be5fee22cb4d68889485310a0f535a Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 13:36:47 -0700 Subject: [PATCH 42/84] one more tweak? --- brozzler/job_schema.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/brozzler/job_schema.yaml b/brozzler/job_schema.yaml index 5da74f84..32d42935 100644 --- a/brozzler/job_schema.yaml +++ b/brozzler/job_schema.yaml @@ -103,11 +103,10 @@ seeds: max_claimed_sites: type: integer -pdfs_only: - type: boolean - account_id: type: - string - integer - required: false + +pdfs_only: + type: boolean From 7af892aa268b4b33a8640b7b16ecd3c2de310aa6 Mon Sep 17 00:00:00 2001 From: Barbara Miller <3253863+galgeek@users.noreply.github.com> Date: Fri, 25 Jul 2025 13:47:13 -0700 Subject: [PATCH 43/84] un-update job_schema.yaml does this (still) work??? --- brozzler/job_schema.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/brozzler/job_schema.yaml b/brozzler/job_schema.yaml index 32d42935..59b831f2 100644 --- a/brozzler/job_schema.yaml +++ b/brozzler/job_schema.yaml @@ -103,10 +103,5 @@ seeds: max_claimed_sites: type: integer -account_id: - type: - - string - - integer - pdfs_only: type: boolean From 1f8af16a2ad223bce1ae998206d7f1eaf16d86f9 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 14:11:46 -0700 Subject: [PATCH 44/84] make tests pass, even with account_id --- brozzler/job_schema.yaml | 5 +++++ brozzler/model.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/brozzler/job_schema.yaml b/brozzler/job_schema.yaml index 59b831f2..e28c3291 100644 --- a/brozzler/job_schema.yaml +++ b/brozzler/job_schema.yaml @@ -105,3 +105,8 @@ max_claimed_sites: pdfs_only: type: boolean + +account_id: + type: + - string + - integer diff --git a/brozzler/model.py b/brozzler/model.py index ac4c1099..7f2612a0 100644 --- a/brozzler/model.py +++ b/brozzler/model.py @@ -98,10 +98,14 @@ def new_job(frontier, job_conf): frontier.rr, {"conf": job_conf, "status": "ACTIVE", "started": doublethink.utcnow()}, ) - job.id = job_conf.get("id") - job.account_id = job_conf.get("account_id") - job.max_claimed_sites = job_conf.get("max_claimed_sites") - job.pdfs_only = job_conf.get("pdfs_only") + if "id" in job_conf: + job.id = job_conf["id"] + if "max_claimed_sites" in job_conf: + job.max_claimed_sites = job_conf["max_claimed_sites"] + if "pdfs_only" in job_conf: + job.pdfs_only = job_conf["pdfs_only"] + if "account_id" in job_conf: + job.account_id = job_conf.["account_id"] job.save() sites = [] From 4715aa0e3806f4919c04fa311241156b8a5f7bbb Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 14:14:15 -0700 Subject: [PATCH 45/84] fix typo --- brozzler/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/model.py b/brozzler/model.py index 7f2612a0..624c7df3 100644 --- a/brozzler/model.py +++ b/brozzler/model.py @@ -105,7 +105,7 @@ def new_job(frontier, job_conf): if "pdfs_only" in job_conf: job.pdfs_only = job_conf["pdfs_only"] if "account_id" in job_conf: - job.account_id = job_conf.["account_id"] + job.account_id = job_conf["account_id"] job.save() sites = [] From 6484256dfce062793378a1b33fdb312a72d0ae6e Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 15:19:01 -0700 Subject: [PATCH 46/84] add account_id to frontier tests --- tests/test_frontier.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_frontier.py b/tests/test_frontier.py index e5d5b79c..c74d0c10 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -69,6 +69,7 @@ def test_basics(rethinker): }, "status": "ACTIVE", "starts_and_stops": [{"start": job.starts_and_stops[0]["start"], "stop": None}], + "account_id": None, } sites = sorted(list(frontier.job_sites(job.id)), key=lambda x: x.seed) @@ -88,6 +89,7 @@ def test_basics(rethinker): {"start": sites[0].starts_and_stops[0]["start"], "stop": None} ], "status": "ACTIVE", + "account_id": None, } assert sites[1] == { "claimed": False, @@ -105,6 +107,7 @@ def test_basics(rethinker): }, ], "status": "ACTIVE", + "account_id": None, } pages = list(frontier.site_pages(sites[0].id)) From ba16a9b59d0072f7e7d9daef2b289257d223741a Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 25 Jul 2025 15:19:32 -0700 Subject: [PATCH 47/84] add account_id to job_schema (again) --- brozzler/job_schema.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/brozzler/job_schema.yaml b/brozzler/job_schema.yaml index e28c3291..6cc4b04c 100644 --- a/brozzler/job_schema.yaml +++ b/brozzler/job_schema.yaml @@ -98,6 +98,11 @@ seeds: video_capture: type: string + account_id: + type: + - string + - integer + <<: *multi_level_options max_claimed_sites: From d2ec6de49ea50bd4203826bf51bdb099647030ab Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Sat, 26 Jul 2025 15:47:16 -0700 Subject: [PATCH 48/84] partition_id, not account_id --- brozzler/job_schema.yaml | 4 ++-- brozzler/model.py | 14 +++++++------- brozzler/ydl.py | 22 +++++++++++----------- tests/test_frontier.py | 7 ++++--- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/brozzler/job_schema.yaml b/brozzler/job_schema.yaml index 6cc4b04c..76e9cc73 100644 --- a/brozzler/job_schema.yaml +++ b/brozzler/job_schema.yaml @@ -98,7 +98,7 @@ seeds: video_capture: type: string - account_id: + partition_id: type: - string - integer @@ -111,7 +111,7 @@ max_claimed_sites: pdfs_only: type: boolean -account_id: +partition_id: type: - string - integer diff --git a/brozzler/model.py b/brozzler/model.py index 624c7df3..b696a5ff 100644 --- a/brozzler/model.py +++ b/brozzler/model.py @@ -104,8 +104,8 @@ def new_job(frontier, job_conf): job.max_claimed_sites = job_conf["max_claimed_sites"] if "pdfs_only" in job_conf: job.pdfs_only = job_conf["pdfs_only"] - if "account_id" in job_conf: - job.account_id = job_conf["account_id"] + if "partition_id" in job_conf: + job.partition_id = job_conf["partition_id"] job.save() sites = [] @@ -117,7 +117,7 @@ def new_job(frontier, job_conf): merged_conf["seed"] = merged_conf.pop("url") site = brozzler.Site(frontier.rr, merged_conf) site.id = str(uuid.uuid4()) - site.account_id = job.account_id + site.partition_id = job.partition_id sites.append(site) pages.append(new_seed_page(frontier, site)) @@ -205,8 +205,8 @@ class Job(doublethink.Document, ElapsedMixIn): def populate_defaults(self): if "status" not in self: self.status = "ACTIVE" - if "account_id" not in self: - self.account_id = None + if "partition_id" not in self: + self.partition_id = None if "pdfs_only" not in self: self.pdfs_only = False if "starts_and_stops" not in self: @@ -268,8 +268,8 @@ def populate_defaults(self): self.scope = {} if "video_capture" not in self: self.video_capture = VideoCaptureOptions.ENABLE_VIDEO_CAPTURE.value - if "account_id" not in self: - self.account_id = None + if "partition_id" not in self: + self.partition_id = None # backward compatibility if "surt" in self.scope: diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 7ec13103..972fb52f 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -108,16 +108,16 @@ def _execute_pg_query( return None def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: - account_id = site["account_id"] if site["account_id"] else None + partition_id = site["partition_id"] if site["partition_id"] else None seed_id = ( site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None ) - if account_id and seed_id and containing_page_url: + if partition_id and seed_id and containing_page_url: # check for postgres query for most recent record pg_query = ( - "SELECT * from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY video_timestamp LIMIT 1", - (account_id, seed_id, str(urlcanon.aggressive(containing_page_url))), + "SELECT * from video where partition_id = %s and seed_id = %s and containing_page_url = %s ORDER BY video_timestamp LIMIT 1", + (partition_id, seed_id, str(urlcanon.aggressive(containing_page_url))), ) try: results = self._execute_pg_query(pg_query, fetchall=True) @@ -125,13 +125,13 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: logger.warn("postgres query failed: %s", e) results = [] else: - logger.warn("missing account_id, seed_id, or containing_page_url") + logger.warn("missing partition_id, seed_id, or containing_page_url") results = [] return results def get_video_captures(self, site=None, source=None) -> List[str]: - account_id = site["account_id"] if site["account_id"] else None + partition_id = site["partition_id"] if site["partition_id"] else None seed_id = ( site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None ) @@ -143,11 +143,11 @@ def get_video_captures(self, site=None, source=None) -> List[str]: containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" # support other media sources here - if account_id and seed_id and source: + if partition_id and seed_id and source: pg_query = ( - "SELECT containing_page_url from video where account_id = %s and seed_id = %s and containing_page_url like %s", + "SELECT containing_page_url from video where partition_id = %s and seed_id = %s and containing_page_url like %s", ( - account_id, + partition_id, seed_id, containing_page_url_pattern, ), @@ -160,7 +160,7 @@ def get_video_captures(self, site=None, source=None) -> List[str]: logger.warn("postgres query failed: %s", e) results = [] else: - logger.warn("missing account_id, seed_id, or source") + logger.warn("missing partition_id, seed_id, or source") results = [] return results @@ -619,7 +619,7 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): if worker._video_data: logger.info( "checking for previously captured youtube watch pages for account %s, seed_id %s", - site["account_id"], + site["partition_id"], site["metadata"]["ait_seed_id"], ) try: diff --git a/tests/test_frontier.py b/tests/test_frontier.py index 3836e37e..e14e8230 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -70,7 +70,7 @@ def test_basics(rethinker): "status": "ACTIVE", "pdfs_only": False, "starts_and_stops": [{"start": job.starts_and_stops[0]["start"], "stop": None}], - "account_id": None, + "partition_id": None, } sites = sorted(list(frontier.job_sites(job.id)), key=lambda x: x.seed) @@ -89,8 +89,8 @@ def test_basics(rethinker): {"start": sites[0].starts_and_stops[0]["start"], "stop": None} ], "status": "ACTIVE", - "account_id": None, "video_capture": "ENABLE_VIDEO_CAPTURE", + "partition_id": None, } assert sites[1] == { "claimed": False, @@ -107,8 +107,9 @@ def test_basics(rethinker): }, ], "status": "ACTIVE", - "account_id": None, + "partition_id": None, "video_capture": "ENABLE_VIDEO_CAPTURE", + "partition_id": None, } pages = list(frontier.site_pages(sites[0].id)) From 51d2e57126225e1035109eab7118f1ef2368fc5c Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Sat, 26 Jul 2025 16:33:13 -0700 Subject: [PATCH 49/84] ... except account_id for pg_query --- brozzler/ydl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 972fb52f..ccf5f7a3 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -116,7 +116,7 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: if partition_id and seed_id and containing_page_url: # check for postgres query for most recent record pg_query = ( - "SELECT * from video where partition_id = %s and seed_id = %s and containing_page_url = %s ORDER BY video_timestamp LIMIT 1", + "SELECT * from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY video_timestamp LIMIT 1", (partition_id, seed_id, str(urlcanon.aggressive(containing_page_url))), ) try: @@ -145,7 +145,7 @@ def get_video_captures(self, site=None, source=None) -> List[str]: if partition_id and seed_id and source: pg_query = ( - "SELECT containing_page_url from video where partition_id = %s and seed_id = %s and containing_page_url like %s", + "SELECT containing_page_url from video where account_id = %s and seed_id = %s and containing_page_url like %s", ( partition_id, seed_id, From 353a74546adf9e4be03df623dc8e89f2b3feae23 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Sat, 26 Jul 2025 16:35:53 -0700 Subject: [PATCH 50/84] fix repeated key --- tests/test_frontier.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_frontier.py b/tests/test_frontier.py index e14e8230..2d9525b8 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -107,7 +107,6 @@ def test_basics(rethinker): }, ], "status": "ACTIVE", - "partition_id": None, "video_capture": "ENABLE_VIDEO_CAPTURE", "partition_id": None, } From 4a73b1e8d8542274006cd700964e77bee2c035c6 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Sun, 27 Jul 2025 08:05:48 -0700 Subject: [PATCH 51/84] simplify job conf --- brozzler/job_schema.yaml | 10 ---------- brozzler/model.py | 7 ------- brozzler/ydl.py | 28 ++++++++++++++++++++-------- tests/test_frontier.py | 3 --- 4 files changed, 20 insertions(+), 28 deletions(-) diff --git a/brozzler/job_schema.yaml b/brozzler/job_schema.yaml index 76e9cc73..59b831f2 100644 --- a/brozzler/job_schema.yaml +++ b/brozzler/job_schema.yaml @@ -98,11 +98,6 @@ seeds: video_capture: type: string - partition_id: - type: - - string - - integer - <<: *multi_level_options max_claimed_sites: @@ -110,8 +105,3 @@ max_claimed_sites: pdfs_only: type: boolean - -partition_id: - type: - - string - - integer diff --git a/brozzler/model.py b/brozzler/model.py index b696a5ff..28979bb5 100644 --- a/brozzler/model.py +++ b/brozzler/model.py @@ -104,8 +104,6 @@ def new_job(frontier, job_conf): job.max_claimed_sites = job_conf["max_claimed_sites"] if "pdfs_only" in job_conf: job.pdfs_only = job_conf["pdfs_only"] - if "partition_id" in job_conf: - job.partition_id = job_conf["partition_id"] job.save() sites = [] @@ -117,7 +115,6 @@ def new_job(frontier, job_conf): merged_conf["seed"] = merged_conf.pop("url") site = brozzler.Site(frontier.rr, merged_conf) site.id = str(uuid.uuid4()) - site.partition_id = job.partition_id sites.append(site) pages.append(new_seed_page(frontier, site)) @@ -205,8 +202,6 @@ class Job(doublethink.Document, ElapsedMixIn): def populate_defaults(self): if "status" not in self: self.status = "ACTIVE" - if "partition_id" not in self: - self.partition_id = None if "pdfs_only" not in self: self.pdfs_only = False if "starts_and_stops" not in self: @@ -268,8 +263,6 @@ def populate_defaults(self): self.scope = {} if "video_capture" not in self: self.video_capture = VideoCaptureOptions.ENABLE_VIDEO_CAPTURE.value - if "partition_id" not in self: - self.partition_id = None # backward compatibility if "surt" in self.scope: diff --git a/brozzler/ydl.py b/brozzler/ydl.py index ccf5f7a3..0220c4ce 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -108,11 +108,19 @@ def _execute_pg_query( return None def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: - partition_id = site["partition_id"] if site["partition_id"] else None + # using ait_account_id as postgres partition id + partition_id = ( + site["metadata"]["ait_account_id"] + if site["metadata"]["ait_account_id"] + else None + ) seed_id = ( site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None ) + # TODO: generalize, make variable? + # containing_page_timestamp_pattern = "2025%" # for future pre-dup additions + if partition_id and seed_id and containing_page_url: # check for postgres query for most recent record pg_query = ( @@ -125,20 +133,24 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: logger.warn("postgres query failed: %s", e) results = [] else: - logger.warn("missing partition_id, seed_id, or containing_page_url") + logger.warn( + "missing partition_id/account_id, seed_id, or containing_page_url" + ) results = [] return results def get_video_captures(self, site=None, source=None) -> List[str]: - partition_id = site["partition_id"] if site["partition_id"] else None + # using ait_account_id as postgres partition id + partition_id = ( + site["metadata"]["ait_account_id"] + if site["metadata"]["ait_account_id"] + else None + ) seed_id = ( site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None ) - # TODO: generalize, maybe make variable? - # containing_page_timestamp_pattern = "2025%" # for future pre-dup additions - if source == "youtube": containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" # support other media sources here @@ -160,7 +172,7 @@ def get_video_captures(self, site=None, source=None) -> List[str]: logger.warn("postgres query failed: %s", e) results = [] else: - logger.warn("missing partition_id, seed_id, or source") + logger.warn("missing partition_id/account_id, seed_id, or source") results = [] return results @@ -619,7 +631,7 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): if worker._video_data: logger.info( "checking for previously captured youtube watch pages for account %s, seed_id %s", - site["partition_id"], + site["metadata"]["ait_account_id"], site["metadata"]["ait_seed_id"], ) try: diff --git a/tests/test_frontier.py b/tests/test_frontier.py index 2d9525b8..da6f5bfe 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -70,7 +70,6 @@ def test_basics(rethinker): "status": "ACTIVE", "pdfs_only": False, "starts_and_stops": [{"start": job.starts_and_stops[0]["start"], "stop": None}], - "partition_id": None, } sites = sorted(list(frontier.job_sites(job.id)), key=lambda x: x.seed) @@ -90,7 +89,6 @@ def test_basics(rethinker): ], "status": "ACTIVE", "video_capture": "ENABLE_VIDEO_CAPTURE", - "partition_id": None, } assert sites[1] == { "claimed": False, @@ -108,7 +106,6 @@ def test_basics(rethinker): ], "status": "ACTIVE", "video_capture": "ENABLE_VIDEO_CAPTURE", - "partition_id": None, } pages = list(frontier.site_pages(sites[0].id)) From de56dc3d130869151d981b43b79da1e9542a470c Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 15:41:07 -0700 Subject: [PATCH 52/84] check captured_youtube_watch_pages not None --- brozzler/ydl.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 0220c4ce..bbe0a72f 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -635,27 +635,35 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): site["metadata"]["ait_seed_id"], ) try: - captured_youtube_watch_pages = set() - captured_youtube_watch_pages.update( + captured_youtube_watch_pages = ( worker._video_data.get_video_captures(site, source="youtube") ) - uncaptured_youtube_watch_pages = [] - for e in ie_result.get("entries_no_dl", []): - # note: http needed for match - youtube_watch_url = str( - urlcanon.aggressive( - f"http://www.youtube.com/watch?v={e['id']}" - ) + if captured_youtube_watch_pages: + logger.info( + "found %s previously captured youtube watch pages for account %s, seed_id %s", + len(captured_youtube_watch_pages), + site["metadata"]["ait_account_id"], + site["metadata"]["ait_seed_id"], ) - if youtube_watch_url in captured_youtube_watch_pages: - logger.info( - "skipping adding %s to yt-dlp outlinks", - youtube_watch_url, + captured_watch_pages = set() + captured_watch_pages.update(captured_youtube_watch_pages) + uncaptured_watch_pages = [] + for e in ie_result.get("entries_no_dl", []): + # note: http matches, not https + youtube_watch_url = str( + urlcanon.aggressive( + f"http://www.youtube.com/watch?v={e['id']}" + ) + ) + if youtube_watch_url in captured_watch_pages: + logger.info( + "skipping adding %s to yt-dlp outlinks", + youtube_watch_url, + ) + continue + uncaptured_watch_pages.append( + f"https://www.youtube.com/watch?v={e['id']}" ) - continue - uncaptured_youtube_watch_pages.append( - f"https://www.youtube.com/watch?v={e['id']}" - ) except Exception as e: logger.warning("hit exception processing worker._video_data: %s", e) if uncaptured_youtube_watch_pages: From b07d8125d04a5f311dd4ef4d19fa1e2a6740852e Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 16:23:51 -0700 Subject: [PATCH 53/84] skip psycopg.rows.scalar_row --- brozzler/ydl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index bbe0a72f..c1dd3cb8 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -165,9 +165,9 @@ def get_video_captures(self, site=None, source=None) -> List[str]: ), ) try: - results = self._execute_pg_query( - pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True - ) + results = [row[0] for row in self._execute_pg_query( + pg_query, fetchall=True + )] except Exception as e: logger.warn("postgres query failed: %s", e) results = [] From 923ae1f2a247ed87c624255158fe3da7c65bc686 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 16:31:10 -0700 Subject: [PATCH 54/84] fix logged errors --- brozzler/ydl.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index c1dd3cb8..4291eef9 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -76,9 +76,6 @@ class VideoCaptureRecord: class VideoDataClient: - import psycopg - from psycopg_pool import ConnectionPool, PoolTimeout - VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") def __init__(self): @@ -165,9 +162,9 @@ def get_video_captures(self, site=None, source=None) -> List[str]: ), ) try: - results = [row[0] for row in self._execute_pg_query( - pg_query, fetchall=True - )] + results = [ + row[0] for row in self._execute_pg_query(pg_query, fetchall=True) + ] except Exception as e: logger.warn("postgres query failed: %s", e) results = [] @@ -666,8 +663,8 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): ) except Exception as e: logger.warning("hit exception processing worker._video_data: %s", e) - if uncaptured_youtube_watch_pages: - outlinks.update(uncaptured_youtube_watch_pages) + if uncaptured_watch_pages: + outlinks.update(uncaptured_watch_pages) else: outlinks = { "https://www.youtube.com/watch?v=%s" % e["id"] From 59f7830711aa243f2266c12edce988c7418e659c Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 16:40:42 -0700 Subject: [PATCH 55/84] import psycopg only in class VideoDataClient --- brozzler/ydl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 4291eef9..6f253270 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -28,11 +28,9 @@ from typing import Any, List, Optional import doublethink -import psycopg import structlog import urlcanon import yt_dlp -from psycopg_pool import ConnectionPool, PoolTimeout from yt_dlp.utils import ExtractorError, match_filter_func import brozzler @@ -76,6 +74,9 @@ class VideoCaptureRecord: class VideoDataClient: + import psycopg + from psycopg_pool import ConnectionPool, PoolTimeout + VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") def __init__(self): From e8dc41ca30ea46bd390d37c2b439bdca697b912e Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 16:51:09 -0700 Subject: [PATCH 56/84] import psycopg only at top of file --- brozzler/ydl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 6f253270..4291eef9 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -28,9 +28,11 @@ from typing import Any, List, Optional import doublethink +import psycopg import structlog import urlcanon import yt_dlp +from psycopg_pool import ConnectionPool, PoolTimeout from yt_dlp.utils import ExtractorError, match_filter_func import brozzler @@ -74,9 +76,6 @@ class VideoCaptureRecord: class VideoDataClient: - import psycopg - from psycopg_pool import ConnectionPool, PoolTimeout - VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") def __init__(self): From 0bd13a627ef46f16f54d0596523aebd9c9d55dc6 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 18:31:01 -0700 Subject: [PATCH 57/84] query_tuple, check result for None --- brozzler/ydl.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 4291eef9..c151c3e1 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -87,16 +87,14 @@ def __init__(self): self.pool = pool def _execute_pg_query( - self, query: str, row_factory=None, fetchone=False, fetchall=False + self, query_tuple, row_factory=None, fetchall=False ) -> Optional[Any]: + query_str, params = query_tuple try: with self.pool.connection() as conn: with conn.cursor(row_factory=row_factory) as cur: - cur.execute(query) - if fetchone: - return cur.fetchone() - if fetchall: - return cur.fetchall() + cur.execute(query_str, params) + return cur.fetchall() if fetchall else cur.fetchone() except PoolTimeout as e: logger.warn("hit PoolTimeout: %s", e) self.pool.check() @@ -162,9 +160,14 @@ def get_video_captures(self, site=None, source=None) -> List[str]: ), ) try: - results = [ - row[0] for row in self._execute_pg_query(pg_query, fetchall=True) - ] + result = self._execute_pg_query(pg_query, fetchall=True) + if result: + results = [ + row[0] + for row in self._execute_pg_query(pg_query, fetchall=True) + ] + else: + results = None except Exception as e: logger.warn("postgres query failed: %s", e) results = [] From 782cd646f59a3664e156882685c1dc5961415f58 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 29 Jul 2025 12:13:13 -0700 Subject: [PATCH 58/84] tidying --- brozzler/ydl.py | 63 ++++++++++--------------------------------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index c151c3e1..6fea9400 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -41,6 +41,7 @@ thread_local = threading.local() + PROXY_ATTEMPTS = 4 YTDLP_WAIT = 10 YTDLP_MAX_REDIRECTS = 5 @@ -177,23 +178,6 @@ def get_video_captures(self, site=None, source=None) -> List[str]: return results - """ - def create_video_capture_record(self, video_capture_record): - # NOTE: we want to do this in brozzler postcrawl for now - # WIP - # TODO: needs added fields added to postgres table, refinement - pg_query = ( - f"INSERT INTO video ({VideoCaptureRecord - items}) VALUES (%s, %s, ...)", - VideoCaptureRecord - values, - ) - try: - results = self._execute_pg_query(pg_query) - except Exception as e: - logger.warn("postgres query failed: %s", e) - results = [] - return results - """ - def isyoutubehost(url): # split 1 splits scheme from url, split 2 splits path from hostname @@ -443,6 +427,7 @@ def ydl_postprocess_hook(d): ytdlp_url = page.redirect_url if page.redirect_url else page.url is_youtube_host = isyoutubehost(ytdlp_url) if is_youtube_host and ytdlp_proxy_endpoints: + # use last proxy_endpoint only for youtube user, channel, playlist pages if "com/watch" not in ytdlp_url: ydl_opts["proxy"] = ytdlp_proxy_endpoints[4] else: @@ -466,10 +451,9 @@ def ydl_postprocess_hook(d): return ydl -# new maybe? def _remember_videos(page, site, worker, ydl, ie_result, pushed_videos=None): -def _remember_videos(page, pushed_videos=None): +def _remember_videos(page, ie_result, pushed_videos=None): """ - Saves info about videos captured by yt-dlp in `page.videos` + Saves info about videos captured by yt-dlp in `page.videos`. """ if "videos" not in page: page.videos = [] @@ -481,34 +465,12 @@ def _remember_videos(page, pushed_videos=None): "content-type": pushed_video["content-type"], "content-length": pushed_video["content-length"], } - """ - # WIP: add new video record to QA postgres here, or in postcrawl only? - warc_prefix_items = site.warcprox_meta["warc-prefix"].split("-") - - video_record = worker._video_data.VideoCaptureRecord() - video_record.crawl_job_id = site.job_id - video_record.is_test_crawl = True if warc_prefix_items[2] == "TEST" else False - video_record.seed_id = site["metadata"]["ait_seed_id"] - video_record.collection_id = int(warc_prefix_items[1]) - video_record.containing_page_timestamp = None - video_record.containing_page_digest = None - video_record.containing_page_media_index = None - video_record.containing_page_media_count = None - video_record.video_digest = None - video_record.video_timestamp = None - video_record.video_mimetype = pushed_video["content-type"] - video_record.video_http_status = pushed_video["response_code"] - video_record.video_size = pushed_video["content-length"] # probably? - video_record.containing_page_url = str( - urlcanon.aggressive(ydl.url) - ) # probably? - video_record.video_url = pushed_video["url"] - # note: NEW! ie_result may not be correct when multiple videos present - video_record.video_title = ie_result.get("title") - video_record.video_display_id = ie_result.get("display_id") - video_record.video_resolution = ie_result.get("resolution") - video_record.video_capture_status = None # "recrawl" maybe - """ + # should be only 1 video for youtube watch pages + if len(pushed_videos) == 1: + video["title"] = ie_result.get("title") + video["display_id"] = ie_result.get("display_id") + video["resolution"] = ie_result.get("resolution") + video["capture_status"] = None logger.debug("embedded video", video=video) page.videos.append(video) @@ -583,9 +545,9 @@ def _try_youtube_dl(worker, ydl, site, page): logger.info("ytdlp completed successfully") - info_json = json.dumps(ie_result, sort_keys=True, indent=4) - _remember_videos(page, ydl.pushed_videos) + _remember_videos(page, ie_result, ydl.pushed_videos) if worker._using_warcprox(site): + info_json = json.dumps(ie_result, sort_keys=True, indent=4) logger.info( "sending WARCPROX_WRITE_RECORD request to warcprox with yt-dlp json", url=ydl.url, @@ -622,7 +584,6 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): logger.info("tempdir for yt-dlp", tempdir=tempdir) ydl = _build_youtube_dl(worker, tempdir, site, page, ytdlp_proxy_endpoints) ie_result = _try_youtube_dl(worker, ydl, site, page) - # print(ie_result) outlinks = set() if ie_result and ( ie_result.get("extractor") == "youtube:playlist" From cb3d494be2452bd78c59cf7117fc1c8464b803ce Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 30 Jul 2025 12:26:52 -0700 Subject: [PATCH 59/84] tidy psycopg imports, mostly --- brozzler/ydl.py | 7 +------ pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 6fea9400..35e2bfad 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -28,7 +28,6 @@ from typing import Any, List, Optional import doublethink -import psycopg import structlog import urlcanon import yt_dlp @@ -439,6 +438,7 @@ def ydl_postprocess_hook(d): logger.info("using yt-dlp proxy ...", proxy=ytdlp_proxy_for_logs) # skip warcprox proxying yt-dlp v.2023.07.06: youtube extractor using ranges + # should_proxy_vid_maybe = not ydl.is_youtube_host # if worker._proxy_for(site): # ydl_opts["proxy"] = "http://{}".format(worker._proxy_for(site)) @@ -481,11 +481,6 @@ def _try_youtube_dl(worker, ydl, site, page): while attempt < max_attempts: try: logger.info("trying yt-dlp", url=ydl.url) - # should_download_vid = not ydl.is_youtube_host - # then - # ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid) - # if ydl.is_youtube_host and ie_result: - # download_url = ie_result.get("url") with brozzler.thread_accept_exceptions(): # we do whatwg canonicalization here to avoid "" resulting in ProxyError diff --git a/pyproject.toml b/pyproject.toml index 4d7e0c1c..12fd956b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ license = "Apache-2.0" [project.optional-dependencies] yt-dlp = ["yt-dlp[default,curl-cffi]>=2024.7.25"] -psycopg = ["psycopg[binary,pool]>=3.2.6"] +psycopg = ["psycopg[pool]>=3.2.6"] dashboard = ["flask>=1.0", "gunicorn>=19.8.1"] warcprox = ["warcprox>=2.4.31"] rethinkdb = [ From 73476bb46cc837427e3eaf0f8e9edc8584cbacc6 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 30 Jul 2025 18:23:40 -0700 Subject: [PATCH 60/84] predup query for should_ytdlp --- brozzler/worker.py | 27 +++++++++++++++++++++++++++ brozzler/ydl.py | 21 ++++++++++----------- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 39eafbfd..2cf08472 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -277,6 +277,21 @@ def thumb_jpeg(self, full_jpeg): img.save(out, "jpeg", quality=95) return out.getbuffer() + def _timestamp4datetime(timestamp): + """split `timestamp` into a tuple of 6 integers. + + :param timestamp: full-length timestamp + """ + timestamp = timestamp[:14] + return ( + int(timestamp[:-10]), + int(timestamp[-10:-8]), + int(timestamp[-8:-6]), + int(timestamp[-6:-4]), + int(timestamp[-4:-2]), + int(timestamp[-2:]) + ) + def should_ytdlp(self, logger, site, page, page_status): # called only after we've passed needs_browsing() check @@ -295,6 +310,18 @@ def should_ytdlp(self, logger, site, page, page_status): if "chrome-error:" in ytdlp_url: return False + # predup... + if 'youtube.com/watch' in ytdlp_url: + previous_capture = get_recent_video_capture(site, ytdlp_url) + if previous_capture: + capture_timestamp = datetime.datetime(*_timestamp4datetime(previous_capture["containing_page_timestamp"])) + logging.info("capture_timestamp: %s", capture_timestamp) + time_diff = datetime.datetime.now() - capture_timestamp + # TODO: make variable for timedelta + if time_diff < datetime.timedelta(days=90): + logging.info("skipping ytdlp for %s since there's a recent capture", previous_capture["containing_page_url"]) + return False + return True @metrics.brozzler_page_processing_duration_seconds.time() diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 35e2bfad..2b48f633 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -113,27 +113,26 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None ) - # TODO: generalize, make variable? - # containing_page_timestamp_pattern = "2025%" # for future pre-dup additions - if partition_id and seed_id and containing_page_url: # check for postgres query for most recent record pg_query = ( - "SELECT * from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY video_timestamp LIMIT 1", + "SELECT * from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY containing_page_timestamp DESC LIMIT 1", (partition_id, seed_id, str(urlcanon.aggressive(containing_page_url))), ) try: - results = self._execute_pg_query(pg_query, fetchall=True) + result = self._execute_pg_query(pg_query, row_factory=dict_row) + if not result: + result = None except Exception as e: logger.warn("postgres query failed: %s", e) - results = [] + result = None else: logger.warn( "missing partition_id/account_id, seed_id, or containing_page_url" ) - results = [] + result = None - return results + return result def get_video_captures(self, site=None, source=None) -> List[str]: # using ait_account_id as postgres partition id @@ -167,7 +166,7 @@ def get_video_captures(self, site=None, source=None) -> List[str]: for row in self._execute_pg_query(pg_query, fetchall=True) ] else: - results = None + results = [] except Exception as e: logger.warn("postgres query failed: %s", e) results = [] @@ -622,8 +621,8 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): ) except Exception as e: logger.warning("hit exception processing worker._video_data: %s", e) - if uncaptured_watch_pages: - outlinks.update(uncaptured_watch_pages) + if uncaptured_watch_pages: + outlinks.update(uncaptured_watch_pages) else: outlinks = { "https://www.youtube.com/watch?v=%s" % e["id"] From c0f0037e47e55942e740fb59088114433529237a Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 30 Jul 2025 18:29:04 -0700 Subject: [PATCH 61/84] ruff'd worker.py --- brozzler/worker.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 2cf08472..1b82a079 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -278,10 +278,10 @@ def thumb_jpeg(self, full_jpeg): return out.getbuffer() def _timestamp4datetime(timestamp): - """split `timestamp` into a tuple of 6 integers. + """split `timestamp` into a tuple of 6 integers. - :param timestamp: full-length timestamp - """ + :param timestamp: full-length timestamp + """ timestamp = timestamp[:14] return ( int(timestamp[:-10]), @@ -289,8 +289,8 @@ def _timestamp4datetime(timestamp): int(timestamp[-8:-6]), int(timestamp[-6:-4]), int(timestamp[-4:-2]), - int(timestamp[-2:]) - ) + int(timestamp[-2:]), + ) def should_ytdlp(self, logger, site, page, page_status): # called only after we've passed needs_browsing() check @@ -311,15 +311,20 @@ def should_ytdlp(self, logger, site, page, page_status): return False # predup... - if 'youtube.com/watch' in ytdlp_url: + if "youtube.com/watch" in ytdlp_url: previous_capture = get_recent_video_capture(site, ytdlp_url) if previous_capture: - capture_timestamp = datetime.datetime(*_timestamp4datetime(previous_capture["containing_page_timestamp"])) + capture_timestamp = datetime.datetime( + *_timestamp4datetime(previous_capture["containing_page_timestamp"]) + ) logging.info("capture_timestamp: %s", capture_timestamp) time_diff = datetime.datetime.now() - capture_timestamp # TODO: make variable for timedelta if time_diff < datetime.timedelta(days=90): - logging.info("skipping ytdlp for %s since there's a recent capture", previous_capture["containing_page_url"]) + logging.info( + "skipping ytdlp for %s since there's a recent capture", + previous_capture["containing_page_url"], + ) return False return True From d34d42b84668b2faf3680a007cc372d2ad74f2a3 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 30 Jul 2025 18:47:07 -0700 Subject: [PATCH 62/84] fix formatting errors --- brozzler/worker.py | 10 ++++++---- brozzler/ydl.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 1b82a079..876945ae 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -312,16 +312,18 @@ def should_ytdlp(self, logger, site, page, page_status): # predup... if "youtube.com/watch" in ytdlp_url: - previous_capture = get_recent_video_capture(site, ytdlp_url) + previous_capture = VideoDataClient.get_recent_video_capture(site, ytdlp_url) if previous_capture: capture_timestamp = datetime.datetime( - *_timestamp4datetime(previous_capture["containing_page_timestamp"]) + *self._timestamp4datetime( + previous_capture["containing_page_timestamp"] + ) ) - logging.info("capture_timestamp: %s", capture_timestamp) + logger.info("capture_timestamp: %s", capture_timestamp) time_diff = datetime.datetime.now() - capture_timestamp # TODO: make variable for timedelta if time_diff < datetime.timedelta(days=90): - logging.info( + logger.info( "skipping ytdlp for %s since there's a recent capture", previous_capture["containing_page_url"], ) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 2b48f633..d636f8aa 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -120,7 +120,7 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: (partition_id, seed_id, str(urlcanon.aggressive(containing_page_url))), ) try: - result = self._execute_pg_query(pg_query, row_factory=dict_row) + result = self._execute_pg_query(pg_query, row_factory=dict_row) # noqa if not result: result = None except Exception as e: From ed67168c2798443f16df7c7c2731ca23ba9845dd Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 1 Aug 2025 16:31:33 -0700 Subject: [PATCH 63/84] updates for get_video_capture mostly --- brozzler/worker.py | 37 ++++++++++++++++++++++--------------- brozzler/ydl.py | 30 ++++++++++++------------------ 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 876945ae..2fd6a8b4 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -311,24 +311,31 @@ def should_ytdlp(self, logger, site, page, page_status): return False # predup... + logger.info("checking for recent previous captures of %s", ytdlp_url) if "youtube.com/watch" in ytdlp_url: - previous_capture = VideoDataClient.get_recent_video_capture(site, ytdlp_url) - if previous_capture: - capture_timestamp = datetime.datetime( - *self._timestamp4datetime( - previous_capture["containing_page_timestamp"] - ) + try: + previous_capture = self._video_data.get_recent_video_capture( + site, ytdlp_url ) - logger.info("capture_timestamp: %s", capture_timestamp) - time_diff = datetime.datetime.now() - capture_timestamp - # TODO: make variable for timedelta - if time_diff < datetime.timedelta(days=90): - logger.info( - "skipping ytdlp for %s since there's a recent capture", - previous_capture["containing_page_url"], + if previous_capture: + capture_timestamp = datetime.datetime( + *self._timestamp4datetime(previous_capture[0]) ) - return False - + logger.info("capture_timestamp: %s", capture_timestamp) + time_diff = datetime.datetime.now() - capture_timestamp + # TODO: make variable for timedelta + if time_diff < datetime.timedelta(days=90): + logger.info( + "skipping ytdlp for %s since there's a recent capture", + ytdlp_url, + ) + return False + except Exception as e: + logger.warning( + "exception querying for previous capture for %s: %s", + ytdlp_url, + str(e), + ) return True @metrics.brozzler_page_processing_duration_seconds.time() diff --git a/brozzler/ydl.py b/brozzler/ydl.py index d636f8aa..c2225990 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -86,13 +86,11 @@ def __init__(self): self.pool = pool - def _execute_pg_query( - self, query_tuple, row_factory=None, fetchall=False - ) -> Optional[Any]: + def _execute_pg_query(self, query_tuple, fetchall=False) -> Optional[Any]: query_str, params = query_tuple try: with self.pool.connection() as conn: - with conn.cursor(row_factory=row_factory) as cur: + with conn.cursor() as cur: cur.execute(query_str, params) return cur.fetchall() if fetchall else cur.fetchone() except PoolTimeout as e: @@ -112,25 +110,23 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: seed_id = ( site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None ) + result = None if partition_id and seed_id and containing_page_url: # check for postgres query for most recent record pg_query = ( - "SELECT * from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY containing_page_timestamp DESC LIMIT 1", + "SELECT containing_page_timestamp from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY containing_page_timestamp DESC LIMIT 1", (partition_id, seed_id, str(urlcanon.aggressive(containing_page_url))), ) try: - result = self._execute_pg_query(pg_query, row_factory=dict_row) # noqa - if not result: - result = None + result = self._execute_pg_query(pg_query) + except Exception as e: logger.warn("postgres query failed: %s", e) - result = None else: logger.warn( "missing partition_id/account_id, seed_id, or containing_page_url" ) - result = None return result @@ -144,6 +140,7 @@ def get_video_captures(self, site=None, source=None) -> List[str]: seed_id = ( site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None ) + results = [] if source == "youtube": containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" @@ -161,18 +158,11 @@ def get_video_captures(self, site=None, source=None) -> List[str]: try: result = self._execute_pg_query(pg_query, fetchall=True) if result: - results = [ - row[0] - for row in self._execute_pg_query(pg_query, fetchall=True) - ] - else: - results = [] + results = [row[0] for row in result] except Exception as e: logger.warn("postgres query failed: %s", e) - results = [] else: logger.warn("missing partition_id/account_id, seed_id, or source") - results = [] return results @@ -622,6 +612,10 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): except Exception as e: logger.warning("hit exception processing worker._video_data: %s", e) if uncaptured_watch_pages: + logger.info( + "adding %s uncaptured watch pages to yt-dlp outlinks", + len(uncaptured_watch_pages), + ) outlinks.update(uncaptured_watch_pages) else: outlinks = { From 17587b0f1685ae4ec09eb90f3674ba2b93eb826b Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 4 Aug 2025 16:20:11 -0700 Subject: [PATCH 64/84] get_recent_video_capture tuple --- brozzler/worker.py | 2 +- brozzler/ydl.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 2fd6a8b4..49dd5d41 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -319,7 +319,7 @@ def should_ytdlp(self, logger, site, page, page_status): ) if previous_capture: capture_timestamp = datetime.datetime( - *self._timestamp4datetime(previous_capture[0]) + *self._timestamp4datetime(previous_capture) ) logger.info("capture_timestamp: %s", capture_timestamp) time_diff = datetime.datetime.now() - capture_timestamp diff --git a/brozzler/ydl.py b/brozzler/ydl.py index c2225990..99afa1c1 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -31,7 +31,6 @@ import structlog import urlcanon import yt_dlp -from psycopg_pool import ConnectionPool, PoolTimeout from yt_dlp.utils import ExtractorError, match_filter_func import brozzler @@ -76,6 +75,8 @@ class VideoCaptureRecord: class VideoDataClient: + from psycopg_pool import ConnectionPool, PoolTimeout + VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") def __init__(self): @@ -119,7 +120,10 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: (partition_id, seed_id, str(urlcanon.aggressive(containing_page_url))), ) try: - result = self._execute_pg_query(pg_query) + result_tuple = self._execute_pg_query(pg_query) + if result_tuple: + result = result_tuple[0] + logger.info("found most recent video capture record: %s", result) except Exception as e: logger.warn("postgres query failed: %s", e) From 9f0344c70c817f704b722eab021e5726e7b31370 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 13 Aug 2025 14:14:57 -0700 Subject: [PATCH 65/84] mv video data code to new file --- brozzler/video_data.py | 157 +++++++++++++++++++++++++++++++++++++++++ brozzler/worker.py | 3 +- brozzler/ydl.py | 123 -------------------------------- 3 files changed, 159 insertions(+), 124 deletions(-) create mode 100644 brozzler/video_data.py diff --git a/brozzler/video_data.py b/brozzler/video_data.py new file mode 100644 index 00000000..a3f6df65 --- /dev/null +++ b/brozzler/video_data.py @@ -0,0 +1,157 @@ +""" +brozzler/video_data.py - video data support for brozzler predup + +Copyright (C) 2025 Internet Archive + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# TODO: verify imports +import datetime +import json +import os +import random +import tempfile +import threading +import time +import urllib.request +from dataclasses import dataclass +from typing import Any, List, Optional + +import structlog +import urlcanon + +logger = structlog.get_logger(logger_name=__name__) + + +# video_title, video_display_id, video_resolution, video_capture_status are new fields, mostly from yt-dlp metadata +@dataclass(frozen=True) +class VideoCaptureRecord: + crawl_job_id: int + is_test_crawl: bool + seed_id: int + collection_id: int + containing_page_timestamp: str + containing_page_digest: str + containing_page_media_index: int + containing_page_media_count: int + video_digest: str + video_timestamp: str + video_mimetype: str + video_http_status: int + video_size: int + containing_page_url: str + video_url: str + video_title: str + video_display_id: ( + str # aka yt-dlp metadata as display_id, e.g., youtube watch page v param + ) + video_resolution: str + video_capture_status: str # recrawl? what else? + + +class VideoDataClient: + from psycopg_pool import ConnectionPool, PoolTimeout + + VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") + + def __init__(self): + pool = ConnectionPool(self.VIDEO_DATA_SOURCE, min_size=1, max_size=9) + pool.wait() + logger.info("pg pool ready") + # atexit.register(pool.close) + + self.pool = pool + + def _execute_pg_query(self, query_tuple, fetchall=False) -> Optional[Any]: + query_str, params = query_tuple + try: + with self.pool.connection() as conn: + with conn.cursor() as cur: + cur.execute(query_str, params) + return cur.fetchall() if fetchall else cur.fetchone() + except PoolTimeout as e: + logger.warn("hit PoolTimeout: %s", e) + self.pool.check() + except Exception as e: + logger.warn("postgres query failed: %s", e) + return None + + def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: + # using ait_account_id as postgres partition id + partition_id = ( + site["metadata"]["ait_account_id"] + if site["metadata"]["ait_account_id"] + else None + ) + seed_id = ( + site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None + ) + result = None + + if partition_id and seed_id and containing_page_url: + # check for postgres query for most recent record + pg_query = ( + "SELECT containing_page_timestamp from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY containing_page_timestamp DESC LIMIT 1", + (partition_id, seed_id, str(urlcanon.aggressive(containing_page_url))), + ) + try: + result_tuple = self._execute_pg_query(pg_query) + if result_tuple: + result = result_tuple[0] + logger.info("found most recent video capture record: %s", result) + + except Exception as e: + logger.warn("postgres query failed: %s", e) + else: + logger.warn( + "missing partition_id/account_id, seed_id, or containing_page_url" + ) + + return result + + def get_video_captures(self, site=None, source=None) -> List[str]: + # using ait_account_id as postgres partition id + partition_id = ( + site["metadata"]["ait_account_id"] + if site["metadata"]["ait_account_id"] + else None + ) + seed_id = ( + site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None + ) + results = [] + + if source == "youtube": + containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" + # support other media sources here + + if partition_id and seed_id and source: + pg_query = ( + "SELECT containing_page_url from video where account_id = %s and seed_id = %s and containing_page_url like %s", + ( + partition_id, + seed_id, + containing_page_url_pattern, + ), + ) + try: + result = self._execute_pg_query(pg_query, fetchall=True) + if result: + results = [row[0] for row in result] + except Exception as e: + logger.warn("postgres query failed: %s", e) + else: + logger.warn("missing partition_id/account_id, seed_id, or source") + + return results diff --git a/brozzler/worker.py b/brozzler/worker.py index 49dd5d41..cd7c509f 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -41,7 +41,7 @@ import brozzler import brozzler.browser from brozzler.model import VideoCaptureOptions -from brozzler.ydl import VideoDataClient +from brozzler.video_data import VideoDataClient from . import metrics @@ -92,6 +92,7 @@ def __init__( self._service_registry = service_registry self._ytdlp_proxy_endpoints = ytdlp_proxy_endpoints self._max_browsers = max_browsers + # see video_data.py for more info if self.VIDEO_DATA_SOURCE and self.VIDEO_DATA_SOURCE.startswith("postgresql"): self._video_data = VideoDataClient() diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 99afa1c1..03acb7f3 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -48,129 +48,6 @@ logger = structlog.get_logger(logger_name=__name__) -# video_title, video_display_id, video_resolution, video_capture_status are new fields, mostly from yt-dlp metadata -@dataclass(frozen=True) -class VideoCaptureRecord: - crawl_job_id: int - is_test_crawl: bool - seed_id: int - collection_id: int - containing_page_timestamp: str - containing_page_digest: str - containing_page_media_index: int - containing_page_media_count: int - video_digest: str - video_timestamp: str - video_mimetype: str - video_http_status: int - video_size: int - containing_page_url: str - video_url: str - video_title: str - video_display_id: ( - str # aka yt-dlp metadata as display_id, e.g., youtube watch page v param - ) - video_resolution: str - video_capture_status: str # recrawl? what else? - - -class VideoDataClient: - from psycopg_pool import ConnectionPool, PoolTimeout - - VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") - - def __init__(self): - pool = ConnectionPool(self.VIDEO_DATA_SOURCE, min_size=1, max_size=9) - pool.wait() - logger.info("pg pool ready") - # atexit.register(pool.close) - - self.pool = pool - - def _execute_pg_query(self, query_tuple, fetchall=False) -> Optional[Any]: - query_str, params = query_tuple - try: - with self.pool.connection() as conn: - with conn.cursor() as cur: - cur.execute(query_str, params) - return cur.fetchall() if fetchall else cur.fetchone() - except PoolTimeout as e: - logger.warn("hit PoolTimeout: %s", e) - self.pool.check() - except Exception as e: - logger.warn("postgres query failed: %s", e) - return None - - def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: - # using ait_account_id as postgres partition id - partition_id = ( - site["metadata"]["ait_account_id"] - if site["metadata"]["ait_account_id"] - else None - ) - seed_id = ( - site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None - ) - result = None - - if partition_id and seed_id and containing_page_url: - # check for postgres query for most recent record - pg_query = ( - "SELECT containing_page_timestamp from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY containing_page_timestamp DESC LIMIT 1", - (partition_id, seed_id, str(urlcanon.aggressive(containing_page_url))), - ) - try: - result_tuple = self._execute_pg_query(pg_query) - if result_tuple: - result = result_tuple[0] - logger.info("found most recent video capture record: %s", result) - - except Exception as e: - logger.warn("postgres query failed: %s", e) - else: - logger.warn( - "missing partition_id/account_id, seed_id, or containing_page_url" - ) - - return result - - def get_video_captures(self, site=None, source=None) -> List[str]: - # using ait_account_id as postgres partition id - partition_id = ( - site["metadata"]["ait_account_id"] - if site["metadata"]["ait_account_id"] - else None - ) - seed_id = ( - site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None - ) - results = [] - - if source == "youtube": - containing_page_url_pattern = "http://youtube.com/watch%" # yes, video data canonicalization uses "http" - # support other media sources here - - if partition_id and seed_id and source: - pg_query = ( - "SELECT containing_page_url from video where account_id = %s and seed_id = %s and containing_page_url like %s", - ( - partition_id, - seed_id, - containing_page_url_pattern, - ), - ) - try: - result = self._execute_pg_query(pg_query, fetchall=True) - if result: - results = [row[0] for row in result] - except Exception as e: - logger.warn("postgres query failed: %s", e) - else: - logger.warn("missing partition_id/account_id, seed_id, or source") - - return results - - def isyoutubehost(url): # split 1 splits scheme from url, split 2 splits path from hostname return "youtube.com" in url.split("//")[-1].split("/")[0] From 540a57b4deeeedd81a4f14c6a48f8009ab816ba1 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 13 Aug 2025 17:01:26 -0700 Subject: [PATCH 66/84] fix imports post-video_data move --- brozzler/video_data.py | 10 ++-------- brozzler/ydl.py | 2 -- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index a3f6df65..86949744 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -16,15 +16,7 @@ limitations under the License. """ -# TODO: verify imports -import datetime -import json import os -import random -import tempfile -import threading -import time -import urllib.request from dataclasses import dataclass from typing import Any, List, Optional @@ -66,6 +58,7 @@ class VideoDataClient: VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") def __init__(self): + from psycopg_pool import ConnectionPool pool = ConnectionPool(self.VIDEO_DATA_SOURCE, min_size=1, max_size=9) pool.wait() logger.info("pg pool ready") @@ -74,6 +67,7 @@ def __init__(self): self.pool = pool def _execute_pg_query(self, query_tuple, fetchall=False) -> Optional[Any]: + from psycopg_pool import PoolTimeout query_str, params = query_tuple try: with self.pool.connection() as conn: diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 03acb7f3..057d2d2d 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -24,8 +24,6 @@ import threading import time import urllib.request -from dataclasses import dataclass -from typing import Any, List, Optional import doublethink import structlog From 89496327ca37eef73b3a7c93cf7feb88cb17e50f Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 13 Aug 2025 17:10:32 -0700 Subject: [PATCH 67/84] formatting fix --- brozzler/video_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index 86949744..913eebcc 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -59,6 +59,7 @@ class VideoDataClient: def __init__(self): from psycopg_pool import ConnectionPool + pool = ConnectionPool(self.VIDEO_DATA_SOURCE, min_size=1, max_size=9) pool.wait() logger.info("pg pool ready") @@ -68,6 +69,7 @@ def __init__(self): def _execute_pg_query(self, query_tuple, fetchall=False) -> Optional[Any]: from psycopg_pool import PoolTimeout + query_str, params = query_tuple try: with self.pool.connection() as conn: From ed8aad0f9bcf547e9fa54d5d109a20652de4376d Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 14 Aug 2025 16:21:34 -0700 Subject: [PATCH 68/84] import video_data.py when valid VIDEO_DATA_SOURCE --- brozzler/worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index cd7c509f..5fd7f262 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -41,7 +41,6 @@ import brozzler import brozzler.browser from brozzler.model import VideoCaptureOptions -from brozzler.video_data import VideoDataClient from . import metrics @@ -94,6 +93,8 @@ def __init__( self._max_browsers = max_browsers # see video_data.py for more info if self.VIDEO_DATA_SOURCE and self.VIDEO_DATA_SOURCE.startswith("postgresql"): + from brozzler.video_data import VideoDataClient + self._video_data = VideoDataClient() self._warcprox_auto = warcprox_auto From d7fb5471fbd7b92386af97ba71f9e7d427690a65 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 14 Aug 2025 17:18:56 -0700 Subject: [PATCH 69/84] datetime.now(datetime.timezone.utc) --- brozzler/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 5fd7f262..9557ea55 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -324,7 +324,7 @@ def should_ytdlp(self, logger, site, page, page_status): *self._timestamp4datetime(previous_capture) ) logger.info("capture_timestamp: %s", capture_timestamp) - time_diff = datetime.datetime.now() - capture_timestamp + time_diff = datetime.datetime.now(datetime.timezone.utc)() - capture_timestamp # TODO: make variable for timedelta if time_diff < datetime.timedelta(days=90): logger.info( From 6baaf4a7c04243e2a96b129b6fb62d923d81afc0 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 14 Aug 2025 17:20:24 -0700 Subject: [PATCH 70/84] ruff'd --- brozzler/worker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 9557ea55..b44f620a 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -324,7 +324,10 @@ def should_ytdlp(self, logger, site, page, page_status): *self._timestamp4datetime(previous_capture) ) logger.info("capture_timestamp: %s", capture_timestamp) - time_diff = datetime.datetime.now(datetime.timezone.utc)() - capture_timestamp + time_diff = ( + datetime.datetime.now(datetime.timezone.utc)() + - capture_timestamp + ) # TODO: make variable for timedelta if time_diff < datetime.timedelta(days=90): logger.info( From 6a8c62fc136e26b370b7a570d2d0cdcafb313a00 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 15 Aug 2025 11:47:12 -0700 Subject: [PATCH 71/84] recent_video_capture_exists --- brozzler/video_data.py | 39 +++++++++++++++++++++++++++++++++++---- brozzler/worker.py | 39 +++++++-------------------------------- 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index 913eebcc..5dc94f93 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -16,9 +16,10 @@ limitations under the License. """ +import datetime import os from dataclasses import dataclass -from typing import Any, List, Optional +from typing import Any, Bool, List, Optional import structlog import urlcanon @@ -83,7 +84,24 @@ def _execute_pg_query(self, query_tuple, fetchall=False) -> Optional[Any]: logger.warn("postgres query failed: %s", e) return None - def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: + def _timestamp4datetime(timestamp): + """split `timestamp` into a tuple of 6 integers. + + :param timestamp: full-length timestamp + """ + timestamp = timestamp[:14] + return ( + int(timestamp[:-10]), + int(timestamp[-10:-8]), + int(timestamp[-8:-6]), + int(timestamp[-6:-4]), + int(timestamp[-4:-2]), + int(timestamp[-2:]), + ) + + def recent_video_capture_exists( + self, site=None, containing_page_url=None, recent=30 + ) -> Bool: # using ait_account_id as postgres partition id partition_id = ( site["metadata"]["ait_account_id"] @@ -93,7 +111,7 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: seed_id = ( site["metadata"]["ait_seed_id"] if site["metadata"]["ait_seed_id"] else None ) - result = None + result = False if partition_id and seed_id and containing_page_url: # check for postgres query for most recent record @@ -105,7 +123,20 @@ def get_recent_video_capture(self, site=None, containing_page_url=None) -> List: result_tuple = self._execute_pg_query(pg_query) if result_tuple: result = result_tuple[0] - logger.info("found most recent video capture record: %s", result) + logger.info("found most recent capture timestamp: %s", result) + capture_timestamp = datetime.datetime( + *self._timestamp4datetime(result) + ) + time_diff = ( + datetime.datetime.now(datetime.timezone.utc)() + - capture_timestamp + ) + if time_diff < datetime.timedelta(recent): + logger.info( + "recent video capture from %s exists", + containing_page_url, + ) + result = True except Exception as e: logger.warn("postgres query failed: %s", e) diff --git a/brozzler/worker.py b/brozzler/worker.py index b44f620a..06994986 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -279,21 +279,6 @@ def thumb_jpeg(self, full_jpeg): img.save(out, "jpeg", quality=95) return out.getbuffer() - def _timestamp4datetime(timestamp): - """split `timestamp` into a tuple of 6 integers. - - :param timestamp: full-length timestamp - """ - timestamp = timestamp[:14] - return ( - int(timestamp[:-10]), - int(timestamp[-10:-8]), - int(timestamp[-8:-6]), - int(timestamp[-6:-4]), - int(timestamp[-4:-2]), - int(timestamp[-2:]), - ) - def should_ytdlp(self, logger, site, page, page_status): # called only after we've passed needs_browsing() check @@ -316,25 +301,15 @@ def should_ytdlp(self, logger, site, page, page_status): logger.info("checking for recent previous captures of %s", ytdlp_url) if "youtube.com/watch" in ytdlp_url: try: - previous_capture = self._video_data.get_recent_video_capture( - site, ytdlp_url + recent_capture_exists = self._video_data.recent_video_capture_exists( + site, ytdlp_url, recent=90 ) - if previous_capture: - capture_timestamp = datetime.datetime( - *self._timestamp4datetime(previous_capture) - ) - logger.info("capture_timestamp: %s", capture_timestamp) - time_diff = ( - datetime.datetime.now(datetime.timezone.utc)() - - capture_timestamp + if recent_capture_exists: + logger.info( + "recent previous capture of %s found, skipping ytdlp", + ytdlp_url, ) - # TODO: make variable for timedelta - if time_diff < datetime.timedelta(days=90): - logger.info( - "skipping ytdlp for %s since there's a recent capture", - ytdlp_url, - ) - return False + return False except Exception as e: logger.warning( "exception querying for previous capture for %s: %s", From 8abb9cdd140305c891a6931c643c524a5faf834e Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 15 Aug 2025 12:42:50 -0700 Subject: [PATCH 72/84] predup check for all urls --- brozzler/worker.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 06994986..74221392 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -300,22 +300,26 @@ def should_ytdlp(self, logger, site, page, page_status): # predup... logger.info("checking for recent previous captures of %s", ytdlp_url) if "youtube.com/watch" in ytdlp_url: - try: - recent_capture_exists = self._video_data.recent_video_capture_exists( - site, ytdlp_url, recent=90 - ) - if recent_capture_exists: - logger.info( - "recent previous capture of %s found, skipping ytdlp", - ytdlp_url, - ) - return False - except Exception as e: - logger.warning( - "exception querying for previous capture for %s: %s", + recent = 90 # 90 days + else: + recent = 30 # 30 days, current default + try: + recent_capture_exists = self._video_data.recent_video_capture_exists( + site, ytdlp_url, recent + ) + if recent_capture_exists: + logger.info( + "recent previous capture of %s found, skipping ytdlp", ytdlp_url, - str(e), ) + return False + except Exception as e: + logger.warning( + "exception querying for previous capture for %s: %s", + ytdlp_url, + str(e), + ) + return True @metrics.brozzler_page_processing_duration_seconds.time() From 7eae14641fe61984bbc101e31224e8b245e76b5e Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 15 Aug 2025 14:17:34 -0700 Subject: [PATCH 73/84] skip Bool --- brozzler/video_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index 5dc94f93..cbf6d0f6 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -19,7 +19,7 @@ import datetime import os from dataclasses import dataclass -from typing import Any, Bool, List, Optional +from typing import Any, List, Optional import structlog import urlcanon @@ -101,7 +101,7 @@ def _timestamp4datetime(timestamp): def recent_video_capture_exists( self, site=None, containing_page_url=None, recent=30 - ) -> Bool: + ): # using ait_account_id as postgres partition id partition_id = ( site["metadata"]["ait_account_id"] From e24c8bc8d9478759227eb5ffa41bb7aade50d83d Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 20 Aug 2025 10:51:25 -0700 Subject: [PATCH 74/84] use video_timestamp, mostly --- brozzler/video_data.py | 10 +++++----- brozzler/worker.py | 5 +---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index cbf6d0f6..d2a8d2f2 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -116,20 +116,20 @@ def recent_video_capture_exists( if partition_id and seed_id and containing_page_url: # check for postgres query for most recent record pg_query = ( - "SELECT containing_page_timestamp from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY containing_page_timestamp DESC LIMIT 1", + "SELECT video_timestamp from video where account_id = %s and seed_id = %s and containing_page_url = %s ORDER BY video_timestamp DESC LIMIT 1", (partition_id, seed_id, str(urlcanon.aggressive(containing_page_url))), ) try: result_tuple = self._execute_pg_query(pg_query) if result_tuple: - result = result_tuple[0] + capture_timestamp = result_tuple[0] logger.info("found most recent capture timestamp: %s", result) - capture_timestamp = datetime.datetime( - *self._timestamp4datetime(result) + capture_datetime = datetime.datetime( + *self._timestamp4datetime(capture_timestamp) ) time_diff = ( datetime.datetime.now(datetime.timezone.utc)() - - capture_timestamp + - capture_datetime ) if time_diff < datetime.timedelta(recent): logger.info( diff --git a/brozzler/worker.py b/brozzler/worker.py index 74221392..6dad290c 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -299,10 +299,7 @@ def should_ytdlp(self, logger, site, page, page_status): # predup... logger.info("checking for recent previous captures of %s", ytdlp_url) - if "youtube.com/watch" in ytdlp_url: - recent = 90 # 90 days - else: - recent = 30 # 30 days, current default + recent = 90 if "youtube.com/watch" in ytdlp_url else 30 try: recent_capture_exists = self._video_data.recent_video_capture_exists( site, ytdlp_url, recent From 978858327b16dd9e8fbfbce628fa952b940d5f84 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 20 Aug 2025 13:56:08 -0700 Subject: [PATCH 75/84] if result_tuple[0] --- brozzler/video_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index d2a8d2f2..15a2e570 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -121,9 +121,9 @@ def recent_video_capture_exists( ) try: result_tuple = self._execute_pg_query(pg_query) - if result_tuple: - capture_timestamp = result_tuple[0] - logger.info("found most recent capture timestamp: %s", result) + capture_timestamp = result_tuple[0] if result_tuple[0] else None + if capture_timestamp: + logger.info("found most recent capture timestamp: %s", capture_timestamp) capture_datetime = datetime.datetime( *self._timestamp4datetime(capture_timestamp) ) From 67d0ac98c81ade8c2ecff4d5ff6c1d4e858cc869 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 21 Aug 2025 10:17:20 -0700 Subject: [PATCH 76/84] more better result handling for recent_video_capture_exists --- brozzler/video_data.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index 15a2e570..ee2c0a9d 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -121,23 +121,31 @@ def recent_video_capture_exists( ) try: result_tuple = self._execute_pg_query(pg_query) - capture_timestamp = result_tuple[0] if result_tuple[0] else None - if capture_timestamp: - logger.info("found most recent capture timestamp: %s", capture_timestamp) - capture_datetime = datetime.datetime( - *self._timestamp4datetime(capture_timestamp) - ) - time_diff = ( - datetime.datetime.now(datetime.timezone.utc)() - - capture_datetime - ) - if time_diff < datetime.timedelta(recent): - logger.info( - "recent video capture from %s exists", - containing_page_url, + if result_tuple is None: + logger.info("found no result for query '%s'", pg_query) + else: + if result_tuple[0]: + logger.info("found most recent capture timestamp: %s", result_tuple[0]) + capture_datetime = datetime.datetime( + *self._timestamp4datetime(result_tuple[0]) ) - result = True - + time_diff = ( + datetime.datetime.now(datetime.timezone.utc)() + - capture_datetime + ) + if time_diff < datetime.timedelta(recent): + logger.info( + "recent video capture exists from %s", + containing_page_url, + ) + result = True + else: + logger.info( + "no recent video capture exists from %s, time_diff = %s", + containing_page_url, time_diff + ) + else: + logger.info("no video timestamp in result for query '%s'", pg_query) except Exception as e: logger.warn("postgres query failed: %s", e) else: From ae67bcd4a1d836d66ae2c79a22e5f9384e5d28e5 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 22 Aug 2025 17:02:03 -0700 Subject: [PATCH 77/84] fix buglets --- brozzler/video_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index ee2c0a9d..f1ad8419 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -84,7 +84,7 @@ def _execute_pg_query(self, query_tuple, fetchall=False) -> Optional[Any]: logger.warn("postgres query failed: %s", e) return None - def _timestamp4datetime(timestamp): + def _timestamp4datetime(self, timestamp): """split `timestamp` into a tuple of 6 integers. :param timestamp: full-length timestamp @@ -130,7 +130,7 @@ def recent_video_capture_exists( *self._timestamp4datetime(result_tuple[0]) ) time_diff = ( - datetime.datetime.now(datetime.timezone.utc)() + datetime.datetime.now(datetime.timezone.utc) - capture_datetime ) if time_diff < datetime.timedelta(recent): From b7949b5d604b568eb62bccd9265e5c73941cc591 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 22 Aug 2025 18:03:43 -0700 Subject: [PATCH 78/84] ruff'd --- brozzler/video_data.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index f1ad8419..ea82763d 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -125,7 +125,9 @@ def recent_video_capture_exists( logger.info("found no result for query '%s'", pg_query) else: if result_tuple[0]: - logger.info("found most recent capture timestamp: %s", result_tuple[0]) + logger.info( + "found most recent capture timestamp: %s", result_tuple[0] + ) capture_datetime = datetime.datetime( *self._timestamp4datetime(result_tuple[0]) ) @@ -142,10 +144,13 @@ def recent_video_capture_exists( else: logger.info( "no recent video capture exists from %s, time_diff = %s", - containing_page_url, time_diff + containing_page_url, + time_diff, ) else: - logger.info("no video timestamp in result for query '%s'", pg_query) + logger.info( + "no video timestamp in result for query '%s'", pg_query + ) except Exception as e: logger.warn("postgres query failed: %s", e) else: From fc7fa6d0ae47a23a2ef5f6e1b64589908a13770b Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 25 Aug 2025 10:42:02 -0700 Subject: [PATCH 79/84] fix last buglet & tweak logging --- brozzler/video_data.py | 3 ++- brozzler/worker.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index ea82763d..8abecc90 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -129,7 +129,8 @@ def recent_video_capture_exists( "found most recent capture timestamp: %s", result_tuple[0] ) capture_datetime = datetime.datetime( - *self._timestamp4datetime(result_tuple[0]) + *self._timestamp4datetime(result_tuple[0]), + tzinfo=datetime.timezone.utc ) time_diff = ( datetime.datetime.now(datetime.timezone.utc) diff --git a/brozzler/worker.py b/brozzler/worker.py index 6dad290c..7d00b2e7 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -306,7 +306,7 @@ def should_ytdlp(self, logger, site, page, page_status): ) if recent_capture_exists: logger.info( - "recent previous capture of %s found, skipping ytdlp", + "recent capture of %s found, skipping ytdlp", ytdlp_url, ) return False From bade30b2ff7977456d5f70477ec8f93a22836f85 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 25 Aug 2025 11:34:32 -0700 Subject: [PATCH 80/84] ruff'd (added final comma 8|) --- brozzler/video_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/video_data.py b/brozzler/video_data.py index 8abecc90..187fe7fe 100644 --- a/brozzler/video_data.py +++ b/brozzler/video_data.py @@ -130,7 +130,7 @@ def recent_video_capture_exists( ) capture_datetime = datetime.datetime( *self._timestamp4datetime(result_tuple[0]), - tzinfo=datetime.timezone.utc + tzinfo=datetime.timezone.utc, ) time_diff = ( datetime.datetime.now(datetime.timezone.utc) From 09fd33944755fab92be43fecf9545ea1f18127ab Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 27 Aug 2025 11:27:17 -0700 Subject: [PATCH 81/84] should_ytdlp predup only when worker.video_data --- brozzler/worker.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 7d00b2e7..3f7e1799 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -298,24 +298,25 @@ def should_ytdlp(self, logger, site, page, page_status): return False # predup... - logger.info("checking for recent previous captures of %s", ytdlp_url) - recent = 90 if "youtube.com/watch" in ytdlp_url else 30 - try: - recent_capture_exists = self._video_data.recent_video_capture_exists( - site, ytdlp_url, recent - ) - if recent_capture_exists: - logger.info( - "recent capture of %s found, skipping ytdlp", + if worker._video_data: + logger.info("checking for recent previous captures of %s", ytdlp_url) + recent = 90 if "youtube.com/watch" in ytdlp_url else 30 + try: + recent_capture_exists = self._video_data.recent_video_capture_exists( + site, ytdlp_url, recent + ) + if recent_capture_exists: + logger.info( + "recent capture of %s found, skipping ytdlp", + ytdlp_url, + ) + return False + except Exception as e: + logger.warning( + "exception querying for previous capture for %s: %s", ytdlp_url, + str(e), ) - return False - except Exception as e: - logger.warning( - "exception querying for previous capture for %s: %s", - ytdlp_url, - str(e), - ) return True From 7c93eb7a8d7a8a1b23b44fe698a34d36cd27cca3 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 27 Aug 2025 11:36:40 -0700 Subject: [PATCH 82/84] if self._video_data --- brozzler/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 3f7e1799..169f0021 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -298,7 +298,7 @@ def should_ytdlp(self, logger, site, page, page_status): return False # predup... - if worker._video_data: + if self._video_data: logger.info("checking for recent previous captures of %s", ytdlp_url) recent = 90 if "youtube.com/watch" in ytdlp_url else 30 try: From cdf5db4f47b625f06b69ac458c840f4f7d654631 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 27 Aug 2025 13:37:05 -0700 Subject: [PATCH 83/84] self._video_data = None sometimes --- brozzler/worker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/brozzler/worker.py b/brozzler/worker.py index 169f0021..fd4569a1 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -96,6 +96,8 @@ def __init__( from brozzler.video_data import VideoDataClient self._video_data = VideoDataClient() + else: + self._video_data = None self._warcprox_auto = warcprox_auto self._proxy = proxy From 5898d8604ec0f72954e3b25d6cc87ee78226afeb Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 30 Oct 2025 11:31:49 -0700 Subject: [PATCH 84/84] note places we might check for duplicate display_id --- brozzler/ydl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 057d2d2d..37af206f 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -333,7 +333,7 @@ def _remember_videos(page, ie_result, pushed_videos=None): "content-type": pushed_video["content-type"], "content-length": pushed_video["content-length"], } - # should be only 1 video for youtube watch pages + # should be only 1 video for youtube watch pages, maybe vimeo, too, and other similar... if len(pushed_videos) == 1: video["title"] = ie_result.get("title") video["display_id"] = ie_result.get("display_id") @@ -408,6 +408,9 @@ def _try_youtube_dl(worker, ydl, site, page): logger.info("ytdlp completed successfully") + # NOTE: good place for a check for duplicate display_id = ie_result.get("display_id")? + # see ydl_postprocess_hook... best to check for duplicate display_id before writing video! + _remember_videos(page, ie_result, ydl.pushed_videos) if worker._using_warcprox(site): info_json = json.dumps(ie_result, sort_keys=True, indent=4)