Skip to content

feat: improve feed tracking and gate handling #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
D3SOX merged 9 commits into from
Aug 1, 2026
Merged

feat: improve feed tracking and gate handling #91

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
86769de
feat: add feed position tracking controls
D3SOX Aug 1, 2026
05914b0
feat: separate gate and download progress
D3SOX Aug 1, 2026
6454e22
fix: use track title in feed checkpoint label
D3SOX Aug 1, 2026
5eadcee
feat: track both feed position boundaries
D3SOX Aug 1, 2026
77fcb71
fix: handle DownloadGater Spotify gates
D3SOX Aug 1, 2026
1422855
fix: show one feed scroll cancel button
D3SOX Aug 1, 2026
74e317c
fix: address PR review feedback
D3SOX Aug 1, 2026
879153c
fix: address final PR review feedback
D3SOX Aug 1, 2026
eff0426
fix: harden DownloadGater popup handling
D3SOX Aug 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 28 additions & 3 deletions src/directDownload.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ async function writeBodyWithSizeLimit(
body: ReadableStream<Uint8Array>,
target: string,
maxBytes: number,
totalBytes: number,
onProgress: (receivedBytes: number, totalBytes: number) => void,
): Promise<void> {
const writer = Bun.file(target).writer();
const reader = body.getReader();
Expand All @@ -75,6 +77,9 @@ async function writeBodyWithSizeLimit(
);
}
writer.write(value);
if (totalBytes > 0) {
onProgress(total, totalBytes);
}
}
await writer.end();
} catch (error) {
Expand All @@ -98,7 +103,7 @@ export class DirectDownloader {
async downloadAudio(url: string): Promise<string> {
const downloadUrl = normalizeDirectDownloadUrl(url);
console.log(`Downloading direct file: ${downloadUrl}`);
this.progressCallback?.('downloading', 'Downloading direct file...', 40, {
this.progressCallback?.('downloading', 'Downloading direct file...', 0, {
browserless: true,
});

Expand Down Expand Up @@ -156,8 +161,8 @@ export class DirectDownloader {
}

const contentLengthHeader = response.headers.get('content-length');
const contentLength = Number(contentLengthHeader) || 0;
if (contentLengthHeader) {
const contentLength = Number(contentLengthHeader);
if (
Number.isFinite(contentLength) &&
contentLength > MAX_DOWNLOAD_BYTES
Expand Down Expand Up @@ -188,7 +193,27 @@ export class DirectDownloader {
}

const target = join('./downloads', filename);
await writeBodyWithSizeLimit(response.body, target, MAX_DOWNLOAD_BYTES);
let lastProgressEmit = 0;
await writeBodyWithSizeLimit(
response.body,
target,
MAX_DOWNLOAD_BYTES,
contentLength,
(receivedBytes, totalBytes) => {
const now = Date.now();
if (now - lastProgressEmit < 250 && receivedBytes < totalBytes) return;
lastProgressEmit = now;
this.progressCallback?.(
'downloading',
`Downloading... ${(receivedBytes / 1024 / 1024).toFixed(1)} / ${(totalBytes / 1024 / 1024).toFixed(1)} MB`,
0,
{ downloadBytes: receivedBytes, totalBytes, browserless: true },
);
},
);
this.progressCallback?.('downloading', 'Download complete', 100, {
browserless: true,
});
console.log(`Saved ${filename}`);
return filename;
}
Expand Down
204 changes: 129 additions & 75 deletions src/downloadgater.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,9 +140,14 @@ export class DownloadgaterDownloader {
}

if (pane === 'spotify') {
throw new Error(
'This DownloadGater gate requires Spotify OAuth, which is not supported yet.',
this.emitProgress(
'handling_gates',
'Handling DownloadGater Spotify step...',
35,
{ currentGate: 'sp' },
);
await this.handleSpotifyStep(page);
continue;
}

await timeout(500);
Expand Down Expand Up @@ -260,103 +265,147 @@ export class DownloadgaterDownloader {
);
}

private async clickUnlockFinishIfPresent(page: Page): Promise<boolean> {
return page.evaluate(() => {
const btn = Array.from(document.querySelectorAll('button')).find(
(el) =>
/^finish$/i.test((el.textContent || '').trim()) &&
!(el as HTMLButtonElement).disabled,
) as HTMLButtonElement | undefined;
btn?.click();
return !!btn;
});
}

private async readSoundcloudUrlError(page: Page): Promise<string | null> {
try {
return new URL(page.url()).searchParams.get('soundcloud_error');
} catch {
return null;
}
}

/**
* Honor-system Instagram: open each target (popup), wait for the 5s unlock
* timer, then confirm. Multiple IG targets are served one at a time.
* Shared honor-system gate flow: open the requested social page, wait for
* DownloadGater's confirmation, close the popup, and confirm completion.
*/
private async handleInstagramStep(page: Page) {
for (let attempt = 0; attempt < 8; attempt++) {
if ((await this.detectPane(page)) !== 'instagram') return;
private async handleHonorSystemPopupStep(
page: Page,
options: {
pane: 'instagram' | 'spotify';
actionButtonPattern: string;
confirmButtonPattern: string;
popupUrlPattern: string;
maxAttempts: number;
confirmationTimeoutMs: number;
},
) {
for (let attempt = 0; attempt < options.maxAttempts; attempt++) {
if ((await this.detectPane(page)) !== options.pane) return;
if (await this.hasDownloadButton(page)) return;

const pagesBefore = new Set(await this.browser.pages(true));

const followClicked = await page.evaluate(() => {
const opened = await page.evaluate((pattern) => {
const actionPattern = new RegExp(pattern, 'i');
const btn = Array.from(document.querySelectorAll('button')).find(
(el) =>
/follow on instagram/i.test(el.textContent || '') &&
actionPattern.test((el.textContent || '').trim()) &&
!(el as HTMLButtonElement).disabled,
) as HTMLButtonElement | undefined;
btn?.click();
return !!btn;
});
}, options.actionButtonPattern);

if (followClicked) {
let popup: Page | undefined;
const started = Date.now();
while (!popup && Date.now() - started < 8_000) {
const pages = await this.browser.pages(true);
popup = pages.find(
(candidate) =>
candidate !== page &&
!pagesBefore.has(candidate) &&
candidate.url() !== 'about:blank',
);
if (!popup) {
popup = pages.find(
(candidate) =>
candidate !== page && /instagram\.com/i.test(candidate.url()),
);
}
if (!popup) await timeout(200);
}
if (!opened) {
await timeout(500);
continue;
}

// Site unlocks after ~5s with the popup open.
await timeout(5_500);
let popup: Page | undefined;
const popupDeadline = Date.now() + 8_000;
while (!popup && Date.now() < popupDeadline) {
const pages = await this.browser.pages(true);
popup = pages.find(
(candidate) =>
candidate !== page &&
!pagesBefore.has(candidate) &&
candidate.url() !== 'about:blank',
);
popup ??= pages.find(
(candidate) =>
candidate !== page &&
new RegExp(options.popupUrlPattern, 'i').test(candidate.url()),
);
if (!popup) await timeout(200);
}

if (popup && !popup.isClosed()) {
try {
await popup.close();
} catch {
// ignore
}
}
await page
.waitForFunction(
(pattern) => {
const confirmPattern = new RegExp(pattern, 'i');
return Array.from(document.querySelectorAll('button')).some(
(el) =>
confirmPattern.test((el.textContent || '').trim()) &&
!(el as HTMLButtonElement).disabled,
);
},
{ timeout: options.confirmationTimeoutMs },
options.confirmButtonPattern,
)
.catch(() => {});

if (popup && !popup.isClosed()) {
await popup.close().catch(() => {});
}

const confirmed = await page.evaluate(() => {
const confirmed = await page.evaluate((pattern) => {
const confirmPattern = new RegExp(pattern, 'i');
const btn = Array.from(document.querySelectorAll('button')).find(
(el) =>
/^(i followed|i opened it)$/i.test((el.textContent || '').trim()) &&
confirmPattern.test((el.textContent || '').trim()) &&
!(el as HTMLButtonElement).disabled,
) as HTMLButtonElement | undefined;
btn?.click();
return !!btn;
});
}, options.confirmButtonPattern);

if (confirmed) {
await timeout(800);
} else {
await timeout(500);
}

if ((await this.detectPane(page)) !== 'instagram') return;
await timeout(confirmed ? 800 : 500);
if ((await this.detectPane(page)) !== options.pane) return;
}

throw new Error(
'DownloadGater Instagram step did not advance. Allow popups and retry.',
`DownloadGater ${options.pane} step did not advance. Allow popups and retry.`,
);
}

/** Honor-system Spotify gate; no Spotify OAuth is required. */
private async handleSpotifyStep(page: Page) {
await this.handleHonorSystemPopupStep(page, {
pane: 'spotify',
actionButtonPattern: '^(follow|save|open).*spotify$',
confirmButtonPattern:
"^(i followed|i saved it|i opened it|i['’]?(?:ve)? done it)$",
popupUrlPattern: 'open\\.spotify\\.com',
maxAttempts: 4,
confirmationTimeoutMs: 8_000,
});
}

private async clickUnlockFinishIfPresent(page: Page): Promise<boolean> {
return page.evaluate(() => {
const btn = Array.from(document.querySelectorAll('button')).find(
(el) =>
/^finish$/i.test((el.textContent || '').trim()) &&
!(el as HTMLButtonElement).disabled,
) as HTMLButtonElement | undefined;
btn?.click();
return !!btn;
});
}

private async readSoundcloudUrlError(page: Page): Promise<string | null> {
try {
return new URL(page.url()).searchParams.get('soundcloud_error');
} catch {
return null;
}
}

/**
* Honor-system Instagram: open each target (popup), wait for the 5s unlock
* timer, then confirm. Multiple IG targets are served one at a time.
*/
private async handleInstagramStep(page: Page) {
await this.handleHonorSystemPopupStep(page, {
pane: 'instagram',
actionButtonPattern: 'follow on instagram',
confirmButtonPattern: '^(i followed|i opened it)$',
popupUrlPattern: 'instagram\\.com',
maxAttempts: 8,
confirmationTimeoutMs: 8_000,
});
}

private async handleSoundcloudConnect(page: Page) {
const comment = this.config.comment.trim();
const needsComment = await page.evaluate(() =>
Expand Down Expand Up @@ -603,7 +652,11 @@ export class DownloadgaterDownloader {
}

private async handleDownload(page: Page) {
this.emitProgress('downloading', 'Preparing DownloadGater download...', 75);
this.emitProgress(
'handling_gates',
'Preparing DownloadGater download...',
75,
);

const client = await page.createCDPSession();
await client.send('Browser.setDownloadBehavior', {
Expand Down Expand Up @@ -647,7 +700,7 @@ export class DownloadgaterDownloader {
this.emitProgress(
'downloading',
`Downloading ${this.downloadFilename}...`,
76,
0,
);
});

Expand All @@ -656,7 +709,7 @@ export class DownloadgaterDownloader {
if (event.state === 'completed') {
pBar.stop();
console.log('Download completed');
this.emitProgress('downloading', 'Download complete', 85);
this.emitProgress('downloading', 'Download complete', 100);
downloadCompleteResolve(this.downloadFilename);
} else if (event.state === 'inProgress') {
const { receivedBytes, totalBytes } = event;
Expand All @@ -668,11 +721,12 @@ export class DownloadgaterDownloader {
} else {
pBar.start(totalBytes, receivedBytes, { prefix: 'Downloading' });
}
const downloadPercent = totalBytes > 0 ? receivedBytes / totalBytes : 0;
const downloadPercent =
totalBytes > 0 ? (receivedBytes / totalBytes) * 100 : 0;
this.emitProgress(
'downloading',
`Downloading... ${(receivedBytes / 1024 / 1024).toFixed(1)} / ${(totalBytes / 1024 / 1024).toFixed(1)} MB`,
76 + downloadPercent * 8,
downloadPercent,
{ downloadBytes: receivedBytes, totalBytes },
);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
} else if (event.state === 'canceled') {
Expand Down
11 changes: 6 additions & 5 deletions src/droploud.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1877,7 +1877,7 @@ export class DroploudDownloader {
}

private async handleDownload(page: Page) {
this.emitProgress('downloading', 'Preparing Droploud download...', 75);
this.emitProgress('handling_gates', 'Preparing Droploud download...', 75);

const client = await page.createCDPSession();
await client.send('Browser.setDownloadBehavior', {
Expand Down Expand Up @@ -1921,7 +1921,7 @@ export class DroploudDownloader {
this.emitProgress(
'downloading',
`Downloading ${this.downloadFilename}...`,
76,
0,
);
});

Expand All @@ -1930,7 +1930,7 @@ export class DroploudDownloader {
if (event.state === 'completed') {
pBar.stop();
console.log('Download completed');
this.emitProgress('downloading', 'Download complete', 85);
this.emitProgress('downloading', 'Download complete', 100);
downloadCompleteResolve(this.downloadFilename);
} else if (event.state === 'inProgress') {
const { receivedBytes, totalBytes } = event;
Expand All @@ -1942,11 +1942,12 @@ export class DroploudDownloader {
} else {
pBar.start(totalBytes, receivedBytes, { prefix: 'Downloading' });
}
const downloadPercent = totalBytes > 0 ? receivedBytes / totalBytes : 0;
const downloadPercent =
totalBytes > 0 ? (receivedBytes / totalBytes) * 100 : 0;
this.emitProgress(
'downloading',
`Downloading... ${(receivedBytes / 1024 / 1024).toFixed(1)} / ${(totalBytes / 1024 / 1024).toFixed(1)} MB`,
76 + downloadPercent * 8,
downloadPercent,
{ downloadBytes: receivedBytes, totalBytes },
);
} else if (event.state === 'canceled') {
Expand Down
Loading