feat: improve feed tracking and gate handling

src/directDownload.ts

@@ -59,6 +59,8 @@ async function writeBodyWithSizeLimit(
 	body: ReadableStream<Uint8Array>,
 	target: string,
 	maxBytes: number,
+	totalBytes: number,
+	onProgress: (receivedBytes: number, totalBytes: number) => void,
 ): Promise<void> {
 	const writer = Bun.file(target).writer();
 	const reader = body.getReader();
@@ -75,6 +77,9 @@ async function writeBodyWithSizeLimit(
 				);
 			}
 			writer.write(value);
+			if (totalBytes > 0) {
+				onProgress(total, totalBytes);
+			}
 		}
 		await writer.end();
 	} catch (error) {
@@ -156,8 +161,8 @@ export class DirectDownloader {
 		}
 
 		const contentLengthHeader = response.headers.get('content-length');
+		const contentLength = Number(contentLengthHeader) || 0;
 		if (contentLengthHeader) {
-			const contentLength = Number(contentLengthHeader);
 			if (
 				Number.isFinite(contentLength) &&
 				contentLength > MAX_DOWNLOAD_BYTES
@@ -188,7 +193,27 @@ export class DirectDownloader {
 		}
 
 		const target = join('./downloads', filename);
-		await writeBodyWithSizeLimit(response.body, target, MAX_DOWNLOAD_BYTES);
+		let lastProgressEmit = 0;
+		await writeBodyWithSizeLimit(
+			response.body,
+			target,
+			MAX_DOWNLOAD_BYTES,
+			contentLength,
+			(receivedBytes, totalBytes) => {
+				const now = Date.now();
+				if (now - lastProgressEmit < 250 && receivedBytes < totalBytes) return;
+				lastProgressEmit = now;
+				this.progressCallback?.(
+					'downloading',
+					`Downloading... ${(receivedBytes / 1024 / 1024).toFixed(1)} / ${(totalBytes / 1024 / 1024).toFixed(1)} MB`,
+					0,
+					{ downloadBytes: receivedBytes, totalBytes, browserless: true },
+				);
+			},
+		);
+		this.progressCallback?.('downloading', 'Download complete', 100, {
+			browserless: true,
+		});
 		console.log(`Saved ${filename}`);
 		return filename;
 	}

src/downloadgater.ts

@@ -140,9 +140,14 @@ export class DownloadgaterDownloader {
 			}
 
 			if (pane === 'spotify') {
-				throw new Error(
-					'This DownloadGater gate requires Spotify OAuth, which is not supported yet.',
+				this.emitProgress(
+					'handling_gates',
+					'Handling DownloadGater Spotify step...',
+					35,
+					{ currentGate: 'sp' },
 				);
+				await this.handleSpotifyStep(page);
+				continue;
 			}
 
 			await timeout(500);
@@ -260,6 +265,77 @@ export class DownloadgaterDownloader {
 		);
 	}
 
+	/**
+	 * Honor-system Spotify gate: open the requested Spotify page, close the new
+	 * tab, then confirm on DownloadGater. No Spotify OAuth is required.
+	 */
+	private async handleSpotifyStep(page: Page) {
+		for (let attempt = 0; attempt < 4; attempt++) {
+			if ((await this.detectPane(page)) !== 'spotify') return;
+
+			const pagesBefore = new Set(await this.browser.pages(true));
+			const opened = await page.evaluate(() => {
+				const btn = Array.from(document.querySelectorAll('button')).find(
+					(el) =>
+						/^(follow|save|open).*spotify$/i.test(
+							(el.textContent || '').trim(),
+						) && !(el as HTMLButtonElement).disabled,
+				) as HTMLButtonElement | undefined;
+				btn?.click();
+				return !!btn;
+			});
+
+			if (!opened) {
+				await timeout(500);
+				continue;
+			}
+
+			let popup: Page | undefined;
+			const popupDeadline = Date.now() + 8_000;
+			while (!popup && Date.now() < popupDeadline) {
+				popup = (await this.browser.pages(true)).find(
+					(candidate) => candidate !== page && !pagesBefore.has(candidate),
+				);
+				if (!popup) await timeout(200);
+			}
+
+			await page
+				.waitForFunction(
+					() =>
+						Array.from(document.querySelectorAll('button')).some(
+							(el) =>
+								/^(i followed|i saved it|i opened it|i['’]?(?:ve)? done it)$/i.test(
+									(el.textContent || '').trim(),
+								) && !(el as HTMLButtonElement).disabled,
+						),
+					{ timeout: 8_000 },
+				)
+				.catch(() => {});
+
+			if (popup && !popup.isClosed()) {
+				await popup.close().catch(() => {});
+			}
+
+			const confirmed = await page.evaluate(() => {
+				const btn = Array.from(document.querySelectorAll('button')).find(
+					(el) =>
+						/^(i followed|i saved it|i opened it|i['’]?(?:ve)? done it)$/i.test(
+							(el.textContent || '').trim(),
+						) && !(el as HTMLButtonElement).disabled,
+				) as HTMLButtonElement | undefined;
+				btn?.click();
+				return !!btn;
+			});
+
+			if (confirmed) await timeout(800);
+			if ((await this.detectPane(page)) !== 'spotify') return;
+		}
+
+		throw new Error(
+			'DownloadGater Spotify step did not advance. Allow popups and retry.',
+		);
+	}
+
 	private async clickUnlockFinishIfPresent(page: Page): Promise<boolean> {
 		return page.evaluate(() => {
 			const btn = Array.from(document.querySelectorAll('button')).find(
@@ -603,7 +679,11 @@ export class DownloadgaterDownloader {
 	}
 
 	private async handleDownload(page: Page) {
-		this.emitProgress('downloading', 'Preparing DownloadGater download...', 75);
+		this.emitProgress(
+			'handling_gates',
+			'Preparing DownloadGater download...',
+			75,
+		);
 
 		const client = await page.createCDPSession();
 		await client.send('Browser.setDownloadBehavior', {

src/droploud.ts

@@ -1877,7 +1877,7 @@ export class DroploudDownloader {
 	}
 
 	private async handleDownload(page: Page) {
-		this.emitProgress('downloading', 'Preparing Droploud download...', 75);
+		this.emitProgress('handling_gates', 'Preparing Droploud download...', 75);
 
 		const client = await page.createCDPSession();
 		await client.send('Browser.setDownloadBehavior', {

src/gaterush.ts

@@ -507,7 +507,7 @@ export class GaterushDownloader {
 	}
 
 	private async handleDownload(page: Page) {
-		this.emitProgress('downloading', 'Preparing GateRush download...', 75);
+		this.emitProgress('handling_gates', 'Preparing GateRush download...', 75);
 
 		const client = await page.createCDPSession();
 		await client.send('Browser.setDownloadBehavior', {

src/hypeddit.ts

@@ -649,7 +649,7 @@ export class HypedditDownloader {
 			throw new Error('Download button not found');
 		}
 		console.log('Download button found, setting up CDP session...');
-		this.emitProgress('downloading', 'Preparing download...', 75);
+		this.emitProgress('handling_gates', 'Preparing download...', 75);
 
 		// configure CDP session to allow monitoring download events
 		const client = await page.createCDPSession();

src/server.ts

@@ -205,7 +205,22 @@ async function runDownloadProcess(jobId: string): Promise<void> {
 			extra?: Partial<Job['progress']>,
 		) => {
 			if (jobStore.isCancelled(jobId)) return;
-			jobStore.updateProgress(jobId, stage, message, percent, extra);
+			let stagePercent = percent;
+			if (stage === 'downloading') {
+				if (
+					extra?.downloadBytes !== undefined &&
+					extra.totalBytes !== undefined &&
+					extra.totalBytes > 0
+				) {
+					stagePercent = Math.min(
+						100,
+						Math.max(0, (extra.downloadBytes / extra.totalBytes) * 100),
+					);
+				} else {
+					stagePercent = /download complete/i.test(message) ? 100 : 0;
+				}
+			}
+			jobStore.updateProgress(jobId, stage, message, stagePercent, extra);
 		};
 
 		const gateConfigBase = {
@@ -235,7 +250,7 @@ async function runDownloadProcess(jobId: string): Promise<void> {
 				jobId,
 				'downloading',
 				`Downloading from ${sourceLabel} via yt-dlp...`,
-				40,
+				0,
 				{ browserless: true },
 			);
 			const ytDlpDownloader = new YtDlpDownloader(sourceLabel);
@@ -275,7 +290,7 @@ async function runDownloadProcess(jobId: string): Promise<void> {
 							jobId,
 							'downloading',
 							`Downloading from ${sourceLabel} via yt-dlp...`,
-							50,
+							0,
 							{ browserless: true },
 						);
 						return selectedUrl;
@@ -357,7 +372,7 @@ async function runDownloadProcess(jobId: string): Promise<void> {
 				jobId,
 				'downloading',
 				'Downloading direct file...',
-				40,
+				0,
 				{ browserless: true },
 			);
 			const directDownloader = new DirectDownloader();