From c8758e75c867bf4fe0dfe6df6ad321baf3bf8811 Mon Sep 17 00:00:00 2001 From: D3SOX Date: Sat, 1 Aug 2026 01:42:59 +0200 Subject: [PATCH 1/6] feat: resolve smart-links and support direct download URLs Follow unknown purchase/description links to known gates, and allow pasting Dropbox/Drive/raw file URLs as a browserless download path. --- README.md | 6 +- src/browserLaunch.ts | 7 +- src/directDownload.ts | 124 +++++++++++++++++++ src/index.ts | 30 +++-- src/server.ts | 29 ++++- src/soundcloud.ts | 16 +-- src/utils.test.ts | 53 ++++++++ src/utils.ts | 232 +++++++++++++++++++++++++++++++++-- webui/src/components/App.tsx | 6 +- 9 files changed, 462 insertions(+), 41 deletions(-) create mode 100644 src/directDownload.ts diff --git a/README.md b/README.md index 9b37182..574c4c0 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # sc-gate-dl -Download and tag SoundCloud tracks unlocked via Hypeddit, Droploud, GateRush, DownloadGater, or Bandcamp. +Download and tag SoundCloud tracks unlocked via Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, or a direct download link. ## Features -- 🎡 Automatically download audio from Hypeddit, Droploud, GateRush, DownloadGater, and Bandcamp links +- 🎡 Automatically download audio from Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, and direct file links (e.g. Dropbox `dl=1`) - ⚑ Browserless fast path that skips the browser for gates that don't need real verification (see [How It Works](#how-it-works)) - πŸ”„ Handles multiple gate types (see [How It Works](#how-it-works)) - πŸ“ Fetches metadata from the provided SoundCloud link @@ -192,6 +192,8 @@ Droploud, GateRush, and DownloadGater gates are handled via their own downloader **Bandcamp / yt-dlp**: When a SoundCloud track’s purchase URL (or description) points at Bandcamp instead of a gate, the file is downloaded with [yt-dlp](https://github.com/yt-dlp/yt-dlp) (browserless). For Bandcamp album links, the matching track is selected from the SoundCloud title. Traditional unlock gates still take priority if both are present. If no gate or Bandcamp URL is found, the CLI and Web UI can fall back to downloading the SoundCloud track itself via yt-dlp. +**Direct download**: Paste a file URL (Dropbox with `dl=1`, Google Drive file links, or any http(s) URL ending in a common audio/archive extension). The file is fetched browserlessly β€” useful when a gate is unsupported but you already have the download link. + **File Processing**: - **Lossless (WAV/AIFF/FLAC) files**: Converted to MP3 (320kbps) with metadata and artwork diff --git a/src/browserLaunch.ts b/src/browserLaunch.ts index d73aefb..5d8bb2a 100644 --- a/src/browserLaunch.ts +++ b/src/browserLaunch.ts @@ -4,13 +4,14 @@ import type { Browser } from 'puppeteer'; export type AppBrowserLaunchOptions = { headless?: boolean; userDataDir?: string; + /** Extra Chromium/CloakBrowser flags merged onto the defaults. */ args?: string[]; /** Passed through to Puppeteer via cloakbrowser `launchOptions`. */ defaultViewport?: { width: number; height: number } | null; humanize?: boolean; }; -const DEFAULT_ARGS = [ +export const DEFAULT_BROWSER_ARGS = [ '--no-sandbox', '--disable-setuid-sandbox', '--mute-audio', @@ -38,12 +39,14 @@ export async function launchAppBrowser( const geoip = Boolean(proxy) && process.env.CLOAKBROWSER_GEOIP === 'true'; + const args = [...DEFAULT_BROWSER_ARGS, ...(options.args ?? [])]; + const launchOpts = { headless: options.headless ?? true, humanize: options.humanize ?? true, stealthArgs: true, ...(proxy ? { proxy, ...(geoip ? { geoip: true } : {}) } : {}), - args: options.args ?? DEFAULT_ARGS, + args, launchOptions: { ...(options.defaultViewport !== undefined ? { defaultViewport: options.defaultViewport } diff --git a/src/directDownload.ts b/src/directDownload.ts new file mode 100644 index 0000000..d740f5a --- /dev/null +++ b/src/directDownload.ts @@ -0,0 +1,124 @@ +import { mkdir } from 'node:fs/promises'; +import { basename, join } from 'node:path'; +import type { ProgressCallback } from './hypeddit'; +import { trimExtractedUrl } from './utils'; + +const USER_AGENT = + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'; + +const AUDIO_OR_ARCHIVE_EXT_RE = + /\.(mp3|wav|flac|aiff|aif|m4a|aac|ogg|opus|zip|rar)(\?|$)/i; + +/** + * True for http(s) URLs that look like a direct file download (Dropbox, Drive, + * or a path with a common audio/archive extension) β€” not HTML gate pages. + */ +export function isDirectDownloadUrl(value: string): boolean { + try { + const url = new URL(trimExtractedUrl(value.trim())); + if (!/^https?:$/i.test(url.protocol)) return false; + + const host = url.hostname.toLowerCase(); + if (/(?:^|\.)dropbox\.com$/i.test(host)) return true; + if (/(?:^|\.)dropboxusercontent\.com$/i.test(host)) return true; + if (/(?:^|\.)drive\.google\.com$/i.test(host)) return true; + if (/(?:^|\.)docs\.google\.com$/i.test(host)) return true; + + if (AUDIO_OR_ARCHIVE_EXT_RE.test(url.pathname)) return true; + if (url.searchParams.get('dl') === '1') return true; + if (url.searchParams.has('raw')) return true; + + return false; + } catch { + return false; + } +} + +/** Normalize share links so fetch gets the file bytes (e.g. Dropbox dl=1). */ +export function normalizeDirectDownloadUrl(url: string): string { + const trimmed = trimExtractedUrl(url.trim()); + try { + const parsed = new URL(trimmed); + if (/(?:^|\.)dropbox\.com$/i.test(parsed.hostname)) { + parsed.searchParams.set('dl', '1'); + return parsed.toString(); + } + // Google Drive file view β†’ uc?export=download + const driveFile = parsed.pathname.match(/\/file\/d\/([^/]+)/); + if ( + /(?:^|\.)drive\.google\.com$/i.test(parsed.hostname) && + driveFile?.[1] + ) { + return `https://drive.google.com/uc?export=download&id=${driveFile[1]}`; + } + return parsed.toString(); + } catch { + return trimmed; + } +} + +function filenameFromContentDisposition(value: string | null): string | null { + if (!value) return null; + const star = value.match(/filename\*=(?:UTF-8'')?([^;]+)/i)?.[1]; + if (star) { + return decodeURIComponent(star.replace(/["']/g, '')); + } + const plain = value.match(/filename=["']?([^"';]+)["']?/i)?.[1]; + return plain ? plain.trim() : null; +} + +export class DirectDownloader { + private progressCallback: ProgressCallback | null = null; + + setProgressCallback(callback: ProgressCallback): void { + this.progressCallback = callback; + } + + async downloadAudio(url: string): Promise { + const downloadUrl = normalizeDirectDownloadUrl(url); + console.log(`Downloading direct file: ${downloadUrl}`); + this.progressCallback?.('downloading', 'Downloading direct file...', 40, { + browserless: true, + }); + + await mkdir('./downloads', { recursive: true }); + const response = await fetch(downloadUrl, { + redirect: 'follow', + headers: { + 'user-agent': USER_AGENT, + accept: '*/*', + }, + }); + if (!response.ok) { + throw new Error( + `Direct download failed: HTTP ${response.status} for ${downloadUrl}`, + ); + } + + const contentType = response.headers.get('content-type') ?? ''; + if (/text\/html/i.test(contentType)) { + throw new Error( + 'Direct download returned HTML instead of a file β€” check that the link is a direct download (e.g. Dropbox with dl=1).', + ); + } + + const fromHeader = filenameFromContentDisposition( + response.headers.get('content-disposition'), + ); + const urlName = basename(new URL(response.url).pathname); + const filename = + fromHeader || + (urlName && urlName !== '/' + ? decodeURIComponent(urlName) + : `direct-download-${Date.now()}.bin`); + + const target = join('./downloads', filename); + await Bun.write(target, await response.arrayBuffer()); + console.log(`Saved ${filename}`); + return filename; + } + + async close(): Promise { + // no-op (browserless) + } +} diff --git a/src/index.ts b/src/index.ts index f2c8221..22482dc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,7 @@ import { confirm, input, select } from '@inquirer/prompts'; import { AudioProcessor } from './audioProcessor'; import { loadConfig, saveConfig } from './config'; +import { DirectDownloader } from './directDownload'; import { DownloadgaterDownloader } from './downloadgater'; import { DroploudDownloader } from './droploud'; import { GaterushDownloader } from './gaterush'; @@ -10,7 +11,7 @@ import { SoundcloudClient } from './soundcloud'; import { getFfmpegBin, getFfprobeBin, - resolveGateProviderUrl, + resolveGateUrlOrFollow, validateSoundcloudUrl, } from './utils'; import { YtDlpDownloader } from './ytdlp'; @@ -65,7 +66,7 @@ try { value: 'ytdlp' as const, }, { - name: 'Enter a Hypeddit / Droploud / GateRush / DownloadGater / Bandcamp URL', + name: 'Enter a gate / Bandcamp / direct download / resolvable URL', value: 'manual' as const, }, ], @@ -77,16 +78,16 @@ try { } else { gateUrl = await input({ message: - 'Enter the Hypeddit, Droploud, GateRush, DownloadGater, or Bandcamp URL', - validate: (value) => { - const resolved = resolveGateProviderUrl(value); + 'Enter a Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, direct download, or smart-link URL', + validate: async (value) => { + const resolved = await resolveGateUrlOrFollow(value); if (!resolved || resolved.provider === 'soundcloud') { - return 'A valid Hypeddit, Droploud, GateRush, DownloadGater, or Bandcamp URL is required'; + return 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, direct download, or resolvable gate URL is required'; } return true; }, }); - const resolved = resolveGateProviderUrl(gateUrl); + const resolved = await resolveGateUrlOrFollow(gateUrl); gate = resolved && resolved.provider !== 'soundcloud' ? { @@ -103,14 +104,16 @@ try { if (!gateUrl || !gate) { throw new Error( - 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, or SoundCloud URL is required', + 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, direct download, or SoundCloud URL is required', ); } - const isYtDlp = - gate.provider === 'bandcamp' || gate.provider === 'soundcloud'; + const isBrowserless = + gate.provider === 'bandcamp' || + gate.provider === 'soundcloud' || + gate.provider === 'direct'; - const headless = isYtDlp + const headless = isBrowserless ? true : config ? config.headless @@ -119,7 +122,7 @@ try { default: true, }); - const initializeLogins = isYtDlp + const initializeLogins = isBrowserless ? false : config ? config.initializeLogins @@ -146,6 +149,9 @@ try { downloadFilename = await ytDlpDownloader.downloadAudio(gateUrl, { matchTitle: track.title, }); + } else if (gate.provider === 'direct') { + const directDownloader = new DirectDownloader(); + downloadFilename = await directDownloader.downloadAudio(gateUrl); } else if (gate.provider === 'droploud') { usedBrowser = true; const droploudDownloader = new DroploudDownloader(gateConfig); diff --git a/src/server.ts b/src/server.ts index 3c1b243..7cfeda3 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,6 +3,7 @@ import { cp, mkdir, rm } from 'node:fs/promises'; import { basename, extname, join } from 'node:path'; import type { SoundcloudTrack } from 'soundcloud.ts'; import { AudioProcessor } from './audioProcessor'; +import { DirectDownloader } from './directDownload'; import { DownloadgaterDownloader } from './downloadgater'; import { renameDownloadFileExclusive } from './downloadRename'; import { DroploudDownloader } from './droploud'; @@ -14,12 +15,12 @@ import { SoundcloudClient } from './soundcloud'; import type { Job, Metadata, OutputFormat } from './types'; import { artistTitleFilename, - extractGateUrl, + extractAndResolveGateUrl, getDefaultMetadata, getFfmpegBin, getFfprobeBin, isMp3Format, - resolveGateProviderUrl, + resolveGateUrlOrFollow, validateGateUrl, validateSoundcloudUrl, } from './utils'; @@ -180,12 +181,14 @@ async function runDownloadProcess(jobId: string): Promise { const job = jobStore.get(jobId); if (!job?.hypedditUrl) return; - const gateUrl = job.hypedditUrl; - const resolved = resolveGateProviderUrl(gateUrl); + const resolved = await resolveGateUrlOrFollow(job.hypedditUrl); if (!resolved) { jobStore.setError(jobId, 'Unsupported gate URL'); return; } + if (resolved.url !== job.hypedditUrl) { + jobStore.update(jobId, { hypedditUrl: resolved.url }); + } const { url: downloadSourceUrl, provider } = resolved; const throwIfCancelled = () => { @@ -314,6 +317,20 @@ async function runDownloadProcess(jobId: string): Promise { downloadFilename = await downloadgaterDownloader.downloadAudio(downloadSourceUrl); throwIfCancelled(); + } else if (provider === 'direct') { + jobStore.updateProgress( + jobId, + 'downloading', + 'Downloading direct file...', + 40, + { browserless: true }, + ); + const directDownloader = new DirectDownloader(); + activeDownloaders.set(jobId, directDownloader); + directDownloader.setProgressCallback(emitProgress); + downloadFilename = + await directDownloader.downloadAudio(downloadSourceUrl); + throwIfCancelled(); } else { // Hypeddit: always try plain HTTP first (email + social skip gates). jobStore.updateProgress( @@ -647,7 +664,7 @@ const server = Bun.serve({ const hypedditUrl = skipAutomaticHypedditFetch ? null - : extractGateUrl(track); + : await extractAndResolveGateUrl(track); const defaultMetadata = getDefaultMetadata(track); const updatedJob = jobStore.update(job.id, { @@ -716,7 +733,7 @@ const server = Bun.serve({ return jsonResponse({ error: validation }, { status: 400 }); } - const resolved = resolveGateProviderUrl(hypedditUrl); + const resolved = await resolveGateUrlOrFollow(hypedditUrl); if (!resolved) { return jsonResponse( { error: 'Could not extract a supported URL' }, diff --git a/src/soundcloud.ts b/src/soundcloud.ts index 8ba6ec9..ee42571 100644 --- a/src/soundcloud.ts +++ b/src/soundcloud.ts @@ -8,8 +8,8 @@ import Soundcloud, { type SoundcloudUser, } from 'soundcloud.ts'; import { + extractAndResolveGateUrl, extractCaptchaDeliveryUrl, - extractGateUrl, loadCookies, } from './utils'; @@ -518,7 +518,7 @@ export class SoundcloudClient { } async getGateURL(track: SoundcloudTrack) { - const gate = extractGateUrl(track); + const gate = await extractAndResolveGateUrl(track); if (!gate) { return null; } @@ -529,11 +529,13 @@ export class SoundcloudClient { ? 'GateRush' : gate.provider === 'downloadgater' ? 'DownloadGater' - : gate.provider === 'bandcamp' - ? 'Bandcamp' - : gate.provider === 'soundcloud' - ? 'SoundCloud' - : 'Hypeddit'; + : gate.provider === 'direct' + ? 'direct download' + : gate.provider === 'bandcamp' + ? 'Bandcamp' + : gate.provider === 'soundcloud' + ? 'SoundCloud' + : 'Hypeddit'; const sourceLabel = gate.type === 'purchase_url' ? 'purchase URL' : 'description'; console.log( diff --git a/src/utils.test.ts b/src/utils.test.ts index 21e0403..462345a 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { artistTitleFilename, cookiesToNetscape, + findKnownGateInHtml, previewProcessedFilename, resolveGateProviderUrl, sanitizeFilenamePart, @@ -51,8 +52,60 @@ describe('resolveGateProviderUrl', () => { provider: 'hypeddit', }); }); + + test('matches Dropbox direct download URLs and forces dl=1', () => { + expect( + resolveGateProviderUrl( + 'https://www.dropbox.com/scl/fi/abc/track.wav?rlkey=xyz&e=1&dl=0', + ), + ).toEqual({ + url: 'https://www.dropbox.com/scl/fi/abc/track.wav?rlkey=xyz&e=1&dl=1', + provider: 'direct', + }); + }); + + test('matches raw audio file URLs as direct downloads', () => { + expect( + resolveGateProviderUrl( + 'FREE DL: https://cdn.example.com/files/track.wav!', + ), + ).toEqual({ + url: 'https://cdn.example.com/files/track.wav', + provider: 'direct', + }); + }); }); +describe('findKnownGateInHtml', () => { + test('finds embedded Hypeddit destination in smart-link HTML', () => { + const html = ` + + `; + expect(findKnownGateInHtml(html)).toEqual({ + url: 'https://hypeddit.com/dorey/strobedoreyedit', + provider: 'hypeddit', + }); + }); + + test('follows meta refresh to a known gate', () => { + const html = + ''; + expect(findKnownGateInHtml(html)).toEqual({ + url: 'https://hypeddit.com/artist/track', + provider: 'hypeddit', + }); + }); + + test('prefers Hypeddit when Bandcamp is also present', () => { + const html = 'https://hypeddit.com/a/b and https://x.bandcamp.com/track/y'; + expect(findKnownGateInHtml(html)).toEqual({ + url: 'https://hypeddit.com/a/b', + provider: 'hypeddit', + }); + }); +}); describe('writeSoundcloudNetscapeCookies', () => { test('returns null for malformed JSON', async () => { const dir = await mkdtemp(join(tmpdir(), 'sc-gate-dl-test-')); diff --git a/src/utils.ts b/src/utils.ts index 33a79f8..96a83f5 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -219,6 +219,8 @@ export type GateProvider = | 'droploud' | 'gaterush' | 'downloadgater' + /** Direct HTTP(S) file URL (Dropbox, Drive, raw audio link, …). */ + | 'direct' | 'bandcamp' /** Direct track download via yt-dlp (manual fallback, not auto-extracted). */ | 'soundcloud'; @@ -240,10 +242,16 @@ const BANDCAMP_URL_RE = const BANDCAMP_ALBUM_URL_RE = /https?:\/\/(?:[\w-]+\.)?bandcamp\.com\/album\/[^\s?#]+/i; const SOUNDCLOUD_URL_RE = /https:\/\/soundcloud\.com\/[^\s?#]+/i; +const ANY_HTTP_URL_RE = /https?:\/\/[^\s<>"')\]]+/gi; -/** Strip prose/Markdown delimiters glued to the end of a matched URL. */ -function trimExtractedUrl(url: string): string { - return url.replace(/[)\]}>.,;:!?'"…]+$/g, ''); +const GATE_RESOLVE_USER_AGENT = + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'; + +/** Strip prose/Markdown/JSON delimiters glued to the end of a matched URL. */ +export function trimExtractedUrl(url: string): string { + const cut = url.search(/["'<>\\]/); + const base = cut >= 0 ? url.slice(0, cut) : url; + return base.replace(/[)\]}>.,;:!?'"…]+$/g, ''); } export function isHypedditUrl(value: string): boolean { @@ -272,7 +280,8 @@ export function isBandcampAlbumUrl(value: string): boolean { /** * Extract the canonical provider URL from a string (possibly with surrounding - * text). Prefer traditional gates, then Bandcamp, then SoundCloud. + * text). Prefer traditional gates, then Bandcamp, then direct file links, then + * SoundCloud. */ export function resolveGateProviderUrl( value: string, @@ -283,6 +292,9 @@ export function resolveGateProviderUrl( const bandcamp = matchBandcampUrl(value); if (bandcamp) return bandcamp; + const direct = matchDirectDownloadUrl(value); + if (direct) return direct; + const soundcloudMatch = value.match(SOUNDCLOUD_URL_RE)?.[0]; if (soundcloudMatch) { return { @@ -305,10 +317,14 @@ export function validateHypedditUrl(value: string): true | string { } export function validateGateUrl(value: string): true | string { - if (!resolveGateProviderUrl(value)) { - return 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, or SoundCloud URL is required'; + if (resolveGateProviderUrl(value)) { + return true; } - return true; + // Unknown http(s) URLs may still resolve via redirects / embedded destinations. + if (/^https?:\/\/\S+/i.test(value.trim())) { + return true; + } + return 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, direct download, SoundCloud, or resolvable http(s) URL is required'; } function normalizeGateUrl( @@ -318,7 +334,8 @@ function normalizeGateUrl( if ( provider === 'gaterush' || provider === 'downloadgater' || - provider === 'bandcamp' + provider === 'bandcamp' || + provider === 'direct' ) { return { url: url.replace(/^http:\/\//i, 'https://'), @@ -328,6 +345,17 @@ function normalizeGateUrl( return { url, provider }; } +/** Known download gates / Bandcamp / direct files (excludes SoundCloud). */ +function matchKnownDownloadGateUrl( + value: string, +): { url: string; provider: GateProvider } | null { + const traditional = matchTraditionalGateUrl(value); + if (traditional) return traditional; + const bandcamp = matchBandcampUrl(value); + if (bandcamp) return bandcamp; + return matchDirectDownloadUrl(value); +} + /** Traditional unlock gates (browser / HTTP). Prefer these over Bandcamp. */ function matchTraditionalGateUrl( value: string, @@ -364,9 +392,145 @@ function matchBandcampUrl( return null; } +function matchDirectDownloadUrl( + value: string, +): { url: string; provider: GateProvider } | null { + // Inline host/ext checks (same rules as isDirectDownloadUrl) to avoid a + // circular import with directDownload.ts β†’ trimExtractedUrl. + const httpMatch = value.match(/https?:\/\/[^\s<>"')\]]+/i)?.[0]; + if (!httpMatch) return null; + const trimmed = trimExtractedUrl(httpMatch); + try { + const url = new URL(trimmed); + if (!/^https?:$/i.test(url.protocol)) return null; + const host = url.hostname.toLowerCase(); + const isHost = + /(?:^|\.)dropbox\.com$/i.test(host) || + /(?:^|\.)dropboxusercontent\.com$/i.test(host) || + /(?:^|\.)drive\.google\.com$/i.test(host) || + /(?:^|\.)docs\.google\.com$/i.test(host); + const isExt = + /\.(mp3|wav|flac|aiff|aif|m4a|aac|ogg|opus|zip|rar)(\?|$)/i.test( + url.pathname, + ); + const isFlag = + url.searchParams.get('dl') === '1' || url.searchParams.has('raw'); + if (!isHost && !isExt && !isFlag) return null; + + let normalized = trimmed.replace(/^http:\/\//i, 'https://'); + if (/(?:^|\.)dropbox\.com$/i.test(host)) { + const parsed = new URL(normalized); + parsed.searchParams.set('dl', '1'); + normalized = parsed.toString(); + } else { + const driveFile = url.pathname.match(/\/file\/d\/([^/]+)/); + if (/(?:^|\.)drive\.google\.com$/i.test(host) && driveFile?.[1]) { + normalized = `https://drive.google.com/uc?export=download&id=${driveFile[1]}`; + } + } + return { url: normalized, provider: 'direct' }; + } catch { + return null; + } +} + +/** + * Scan HTML (or any text blob) for a known gate / Bandcamp URL, including + * meta-refresh targets. Used for smart-link pages that embed destinations. + */ +export function findKnownGateInHtml( + html: string, +): { url: string; provider: GateProvider } | null { + const normalized = html.replace(/\\\//g, '/'); + const direct = matchKnownDownloadGateUrl(normalized); + if (direct) return direct; + + const refresh = + normalized.match( + /http-equiv=["']refresh["'][^>]*content=["'][^"']*url=([^"'>\s]+)/i, + ) ?? + normalized.match( + /content=["'][^"']*url=([^"'>\s]+)[^"']*["'][^>]*http-equiv=["']refresh["']/i, + ); + if (refresh?.[1]) { + const target = refresh[1].replace(/&/g, '&'); + return matchKnownDownloadGateUrl(target); + } + return null; +} + +/** + * Resolve an unrecognized (or already-known) URL to a download gate by + * following HTTP redirects and scanning the final HTML for embedded destinations. + */ +export async function resolveUnknownGateUrl( + url: string, +): Promise<{ url: string; provider: GateProvider } | null> { + const trimmed = trimExtractedUrl(url.trim()); + if (!/^https?:\/\//i.test(trimmed)) { + return null; + } + + const direct = matchKnownDownloadGateUrl(trimmed); + if (direct) return direct; + + // SoundCloud pages are not smart-link hops we want to chase. + if (isSoundcloudUrl(trimmed)) { + return null; + } + + try { + const response = await fetch(trimmed, { + redirect: 'follow', + headers: { + 'user-agent': GATE_RESOLVE_USER_AGENT, + accept: + 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8', + }, + }); + const finalUrl = trimExtractedUrl(response.url); + const fromFinal = matchKnownDownloadGateUrl(finalUrl); + if (fromFinal) return fromFinal; + + const html = await response.text(); + return findKnownGateInHtml(html); + } catch { + return null; + } +} + +function collectUnresolvedHttpCandidates( + track: SoundcloudTrack, +): { url: string; type: 'purchase_url' | 'description' }[] { + const candidates: { url: string; type: 'purchase_url' | 'description' }[] = + []; + const seen = new Set(); + + const push = (raw: string, type: 'purchase_url' | 'description') => { + const url = trimExtractedUrl(raw); + if (!/^https?:\/\//i.test(url) || isSoundcloudUrl(url) || seen.has(url)) { + return; + } + // Skip URLs already recognized as download gates / Bandcamp. + if (matchKnownDownloadGateUrl(url)) return; + seen.add(url); + candidates.push({ url, type }); + }; + + if (track.purchase_url) { + push(track.purchase_url, 'purchase_url'); + } + if (track.description) { + for (const match of track.description.match(ANY_HTTP_URL_RE) ?? []) { + push(match, 'description'); + } + } + return candidates; +} + /** * Prefer Hypeddit / Droploud / GateRush / DownloadGater from purchase_url or - * description, then fall back to a Bandcamp purchase/description link. + * description, then Bandcamp, then direct file links (Dropbox, Drive, …). */ export function extractGateUrl(track: SoundcloudTrack): GateUrlMatch | null { const { purchase_url, description } = track; @@ -399,9 +563,59 @@ export function extractGateUrl(track: SoundcloudTrack): GateUrlMatch | null { } } + if (purchase_url) { + const direct = matchDirectDownloadUrl(purchase_url); + if (direct) { + return { ...direct, type: 'purchase_url' }; + } + } + + if (description) { + const direct = matchDirectDownloadUrl(description); + if (direct) { + return { ...direct, type: 'description' }; + } + } + return null; } +/** + * Like extractGateUrl, but also follows redirects / scans smart-link HTML when + * purchase_url or description contain unrecognized http(s) links. + */ +export async function extractAndResolveGateUrl( + track: SoundcloudTrack, +): Promise { + const sync = extractGateUrl(track); + if (sync) return sync; + + for (const candidate of collectUnresolvedHttpCandidates(track)) { + const resolved = await resolveUnknownGateUrl(candidate.url); + if (resolved) { + return { ...resolved, type: candidate.type }; + } + } + return null; +} + +/** + * Resolve a user-supplied or stored gate URL: known providers pass through; + * otherwise try redirect / HTML destination resolution. + */ +export async function resolveGateUrlOrFollow( + value: string, +): Promise<{ url: string; provider: GateProvider } | null> { + const direct = resolveGateProviderUrl(value); + if (direct && direct.provider !== 'soundcloud') { + return direct; + } + if (direct?.provider === 'soundcloud') { + return direct; + } + return resolveUnknownGateUrl(value); +} + /** @deprecated Prefer extractGateUrl */ export function extractHypedditUrl( track: SoundcloudTrack, diff --git a/webui/src/components/App.tsx b/webui/src/components/App.tsx index a2948ea..e4d2e44 100644 --- a/webui/src/components/App.tsx +++ b/webui/src/components/App.tsx @@ -776,7 +776,7 @@ export default function App() {

Download & tag SoundCloud tracks from Hypeddit, Droploud, GateRush, - DownloadGater, or Bandcamp + DownloadGater, Bandcamp, or a direct download link

@@ -933,7 +933,7 @@ export default function App() {

{skipAutomaticHypedditFetch ? 'Automatic gate lookup is disabled. Enter a gate URL manually, or download the SoundCloud track via yt-dlp.' - : 'Gate URL not found in track. Enter a Hypeddit, Droploud, GateRush, DownloadGater, or Bandcamp URL, or download via yt-dlp from SoundCloud.'} + : 'Gate URL not found in track. Enter a Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, or direct download URL, or download via yt-dlp from SoundCloud.'}

@@ -945,7 +945,7 @@ export default function App() { name="hypeddit-url" value={hypedditUrlInput} onChange={(e) => setHypedditUrlInput(e.target.value)} - placeholder="https://hypeddit.com/... / droploud.com/gate/... / gaterush.me/... / downloadgater.com/g/... / artist.bandcamp.com/track/..." + placeholder="https://hypeddit.com/... / droploud.com/gate/... / gaterush.me/... / downloadgater.com/g/... / artist.bandcamp.com/track/... / dropbox.com/...&dl=1" autoComplete="off" required disabled={isLoading} From c4b878da5dbb06b6b551d416e8d860a6b6bb46ae Mon Sep 17 00:00:00 2001 From: D3SOX Date: Sat, 1 Aug 2026 01:50:27 +0200 Subject: [PATCH 2/6] feat: let user pick a Bandcamp album track when auto-match fails --- README.md | 2 +- src/index.ts | 15 ++++ src/jobStore.ts | 33 ++++++++ src/server.ts | 99 +++++++++++++++++++++++ src/types.ts | 13 +++ src/ytdlp.ts | 116 +++++++++++++++++++-------- webui/src/components/App.css | 57 ++++++++++++++ webui/src/components/App.tsx | 148 +++++++++++++++++++++++++++-------- 8 files changed, 419 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 574c4c0..f2f1530 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ Panel size/position is remembered in `localStorage` (`sc-gate-dl-panel-geom`). Droploud, GateRush, and DownloadGater gates are handled via their own downloaders with similar email / social unlock flows. -**Bandcamp / yt-dlp**: When a SoundCloud track’s purchase URL (or description) points at Bandcamp instead of a gate, the file is downloaded with [yt-dlp](https://github.com/yt-dlp/yt-dlp) (browserless). For Bandcamp album links, the matching track is selected from the SoundCloud title. Traditional unlock gates still take priority if both are present. If no gate or Bandcamp URL is found, the CLI and Web UI can fall back to downloading the SoundCloud track itself via yt-dlp. +**Bandcamp / yt-dlp**: When a SoundCloud track’s purchase URL (or description) points at Bandcamp instead of a gate, the file is downloaded with [yt-dlp](https://github.com/yt-dlp/yt-dlp) (browserless). For Bandcamp album links, the matching track is selected from the SoundCloud title; if auto-match fails, the CLI and Web UI let you pick a track from the album. Traditional unlock gates still take priority if both are present. If no gate or Bandcamp URL is found, the CLI and Web UI can fall back to downloading the SoundCloud track itself via yt-dlp. **Direct download**: Paste a file URL (Dropbox with `dl=1`, Google Drive file links, or any http(s) URL ending in a common audio/archive extension). The file is fetched browserlessly β€” useful when a gate is unsupported but you already have the download link. diff --git a/src/index.ts b/src/index.ts index 22482dc..2a7999e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -148,6 +148,21 @@ try { const ytDlpDownloader = new YtDlpDownloader(sourceLabel); downloadFilename = await ytDlpDownloader.downloadAudio(gateUrl, { matchTitle: track.title, + onAlbumMatchFailed: async (error) => { + const selected = await select({ + message: error.matchTitle + ? `Could not match β€œ${error.matchTitle}”. Pick a Bandcamp album track:` + : 'Pick a track from the Bandcamp album:', + choices: error.tracks.map((t) => ({ + name: + t.score > 0 + ? `${t.title} (${Math.round(t.score * 100)}% match)` + : t.title, + value: t.url, + })), + }); + return selected; + }, }); } else if (gate.provider === 'direct') { const directDownloader = new DirectDownloader(); diff --git a/src/jobStore.ts b/src/jobStore.ts index a1a97f9..3284554 100644 --- a/src/jobStore.ts +++ b/src/jobStore.ts @@ -1,6 +1,7 @@ import type { Job, JobProgress, JobStage, OutputFormat } from './types'; type ProgressListener = (progress: JobProgress) => void; +type BandcampTrackResolver = (url: string | null) => void; /** * In-memory job store for single-user Web UI @@ -8,6 +9,9 @@ type ProgressListener = (progress: JobProgress) => void; class JobStore { private jobs: Map = new Map(); private listeners: Map> = new Map(); + /** Pending Bandcamp album-track picks (mid-download pause). */ + private bandcampTrackResolvers: Map = + new Map(); /** * Creates a new job and returns its ID @@ -19,6 +23,7 @@ class JobStore { id, soundcloudUrl, hypedditUrl: null, + bandcampAlbumTracks: null, headless: process.env.BROWSER_HEADLESS !== 'false', outputFormat, cancelled: false, @@ -138,16 +143,43 @@ class JobStore { job.cancelled = true; job.error = null; + job.bandcampAlbumTracks = null; job.progress = { stage: 'cancelled', message, percent: job.progress.percent, }; job.updatedAt = new Date(); + this.resolveBandcampTrackSelection(id, null); this.notifyListeners(id, job.progress); return job; } + /** + * Pause until the user picks a Bandcamp album track (or cancel β†’ null). + */ + waitForBandcampTrackSelection(id: string): Promise { + const existing = this.bandcampTrackResolvers.get(id); + if (existing) { + existing(null); + this.bandcampTrackResolvers.delete(id); + } + return new Promise((resolve) => { + this.bandcampTrackResolvers.set(id, resolve); + }); + } + + /** + * Resolve a pending Bandcamp album-track wait with a track URL, or null. + */ + resolveBandcampTrackSelection(id: string, url: string | null): boolean { + const resolve = this.bandcampTrackResolvers.get(id); + if (!resolve) return false; + this.bandcampTrackResolvers.delete(id); + resolve(url); + return true; + } + /** * Clear cancelled flag when restarting a job (e.g. after error). */ @@ -193,6 +225,7 @@ class JobStore { * Delete a job */ delete(id: string): boolean { + this.resolveBandcampTrackSelection(id, null); this.listeners.delete(id); return this.jobs.delete(id); } diff --git a/src/server.ts b/src/server.ts index 7cfeda3..2a2bb50 100644 --- a/src/server.ts +++ b/src/server.ts @@ -245,6 +245,41 @@ async function runDownloadProcess(jobId: string): Promise { downloadSourceUrl, { matchTitle: job.track?.title, + onAlbumMatchFailed: async (error) => { + throwIfCancelled(); + jobStore.update(jobId, { + bandcampAlbumTracks: error.tracks, + error: null, + }); + const message = error.matchTitle + ? `Could not match β€œ${error.matchTitle}” β€” pick a Bandcamp track` + : 'Pick a track from the Bandcamp album'; + jobStore.updateProgress( + jobId, + 'waiting_bandcamp_track', + message, + 45, + { + browserless: true, + bandcampAlbumTracks: error.tracks, + }, + ); + const selectedUrl = + await jobStore.waitForBandcampTrackSelection(jobId); + throwIfCancelled(); + if (!selectedUrl) { + throw new Error('Download cancelled'); + } + jobStore.update(jobId, { bandcampAlbumTracks: null }); + jobStore.updateProgress( + jobId, + 'downloading', + `Downloading from ${sourceLabel} via yt-dlp...`, + 50, + { browserless: true }, + ); + return selectedUrl; + }, }, ); throwIfCancelled(); @@ -755,6 +790,69 @@ const server = Bun.serve({ }, }, + '/api/job/:id/bandcamp-track': { + POST: async (req) => { + try { + const jobId = req.params.id; + const job = jobStore.get(jobId); + + if (!job) { + return jsonResponse({ error: 'Job not found' }, { status: 404 }); + } + + if (job.progress.stage !== 'waiting_bandcamp_track') { + return jsonResponse( + { error: 'Job is not waiting for a Bandcamp track selection' }, + { status: 400 }, + ); + } + + const body = await req.json(); + const { trackUrl } = body as { trackUrl?: string }; + + if (!trackUrl || typeof trackUrl !== 'string') { + return jsonResponse( + { error: 'trackUrl is required' }, + { status: 400 }, + ); + } + + const allowed = job.bandcampAlbumTracks ?? []; + const match = allowed.find((t) => t.url === trackUrl); + if (!match) { + return jsonResponse( + { error: 'trackUrl is not one of the listed album tracks' }, + { status: 400 }, + ); + } + + const resolved = jobStore.resolveBandcampTrackSelection( + jobId, + match.url, + ); + if (!resolved) { + return jsonResponse( + { error: 'No pending Bandcamp track selection' }, + { status: 409 }, + ); + } + + return jsonResponse({ + success: true, + trackUrl: match.url, + title: match.title, + }); + } catch (error) { + return jsonResponse( + { + error: error instanceof Error ? error.message : 'Unknown error', + }, + { status: 500 }, + ); + } + }, + }, + '/api/job/:id/start': { POST: async (req) => { try { @@ -950,6 +1048,7 @@ const server = Bun.serve({ id: job.id, soundcloudUrl: job.soundcloudUrl, hypedditUrl: job.hypedditUrl, + bandcampAlbumTracks: job.bandcampAlbumTracks, track: job.track, defaultMetadata: job.defaultMetadata, existingMetadata: job.existingMetadata, diff --git a/src/types.ts b/src/types.ts index c14629d..4089491 100644 --- a/src/types.ts +++ b/src/types.ts @@ -34,6 +34,7 @@ export type JobStage = | 'pending' | 'fetching_track' | 'waiting_hypeddit' + | 'waiting_bandcamp_track' | 'initializing_browser' | 'preparing_logins' | 'handling_gates' @@ -43,6 +44,14 @@ export type JobStage = | 'error' | 'cancelled'; +/** Candidate track on a Bandcamp album when auto title-match fails. */ +export interface BandcampAlbumTrackChoice { + title: string; + url: string; + /** 0–1 fuzzy score against the SoundCloud title (0 when no title to match). */ + score: number; +} + export interface JobProgress { stage: JobStage; message: string; @@ -53,12 +62,16 @@ export interface JobProgress { // True when the download was handled without a browser. Such downloads never // touch the SoundCloud account, so the UI can skip the cleanup prompt. browserless?: boolean; + /** Present while stage is `waiting_bandcamp_track`. */ + bandcampAlbumTracks?: BandcampAlbumTrackChoice[]; } export interface Job { id: string; soundcloudUrl: string; hypedditUrl: string | null; + /** Candidates when Bandcamp album auto-match fails (cleared after pick). */ + bandcampAlbumTracks: BandcampAlbumTrackChoice[] | null; /** Whether browser automation runs headless. Defaults to true. */ headless: boolean; outputFormat: OutputFormat; diff --git a/src/ytdlp.ts b/src/ytdlp.ts index 6e47b8c..d5e7b20 100644 --- a/src/ytdlp.ts +++ b/src/ytdlp.ts @@ -2,7 +2,7 @@ import { mkdir, rm } from 'node:fs/promises'; import { basename, join } from 'node:path'; import { execa } from 'execa'; import { lookpath } from 'find-bin'; -import type { JobProgress } from './types'; +import type { BandcampAlbumTrackChoice, JobProgress } from './types'; import { isBandcampAlbumUrl, isSoundcloudUrl, @@ -19,11 +19,17 @@ type ProgressCallback = ( export type YtDlpDownloadOptions = { /** Prefer this title when the URL is a Bandcamp album (or multi-entry). */ matchTitle?: string; + /** + * Called when auto title-match fails (or no title was given). + * Return a track URL from `error.tracks` to continue; throw/reject to abort. + */ + onAlbumMatchFailed?: (error: BandcampAlbumMatchError) => Promise; }; const DOWNLOADS_DIR = './downloads'; const YTDLP_DOWNLOAD_TIMEOUT_MS = 10 * 60_000; const YTDLP_METADATA_TIMEOUT_MS = 60_000; +const ALBUM_MATCH_THRESHOLD = 0.5; type FlatEntry = { id?: string; @@ -32,6 +38,26 @@ type FlatEntry = { webpage_url?: string; }; +/** Thrown when a Bandcamp album cannot be auto-matched to a single track. */ +export class BandcampAlbumMatchError extends Error { + readonly albumUrl: string; + readonly matchTitle: string | undefined; + readonly tracks: BandcampAlbumTrackChoice[]; + + constructor( + message: string, + albumUrl: string, + matchTitle: string | undefined, + tracks: BandcampAlbumTrackChoice[], + ) { + super(message); + this.name = 'BandcampAlbumMatchError'; + this.albumUrl = albumUrl; + this.matchTitle = matchTitle; + this.tracks = tracks; + } +} + export async function getYtDlpBin(): Promise { const ytDlpBin = await lookpath('yt-dlp'); if (!ytDlpBin) { @@ -80,7 +106,8 @@ export function titleMatchScore(candidate: string, wanted: string): number { * * Bandcamp album URLs: when `matchTitle` is set, resolves the best-matching * track on the album and downloads only that entry (avoids grabbing the whole - * album and offering the wrong file). + * album and offering the wrong file). If matching fails, `onAlbumMatchFailed` + * (when provided) can pick a track; otherwise a `BandcampAlbumMatchError` is thrown. */ export class YtDlpDownloader { private progressCallback: ProgressCallback | null = null; @@ -133,15 +160,12 @@ export class YtDlpDownloader { const ytDlpBin = await getYtDlpBin(); let downloadUrl = url; - if (isBandcampAlbumUrl(url) && options.matchTitle) { + if (isBandcampAlbumUrl(url)) { downloadUrl = await this.resolveBandcampAlbumTrack( ytDlpBin, url, options.matchTitle, - ); - } else if (isBandcampAlbumUrl(url) && !options.matchTitle) { - throw new Error( - 'Bandcamp album URL needs a track title to pick the right song. Re-fetch the SoundCloud track and try again.', + options.onAlbumMatchFailed, ); } @@ -244,16 +268,19 @@ export class YtDlpDownloader { private async resolveBandcampAlbumTrack( ytDlpBin: string, albumUrl: string, - matchTitle: string, + matchTitle: string | undefined, + onAlbumMatchFailed?: (error: BandcampAlbumMatchError) => Promise, ): Promise { - this.progressCallback?.( - 'downloading', - `Matching β€œ${matchTitle}” on Bandcamp album...`, - 45, - { browserless: true }, - ); + const matchingLabel = matchTitle + ? `Matching β€œ${matchTitle}” on Bandcamp album...` + : 'Listing Bandcamp album tracks...'; + this.progressCallback?.('downloading', matchingLabel, 45, { + browserless: true, + }); console.log( - `${this.sourceLabel}: album URL β€” matching track β€œ${matchTitle}”…`, + matchTitle + ? `${this.sourceLabel}: album URL β€” matching track β€œ${matchTitle}”…` + : `${this.sourceLabel}: album URL β€” no title to match; listing tracks…`, ); const stdout = await this.runYtDlp( @@ -275,33 +302,60 @@ export class YtDlpDownloader { throw new Error(`No tracks found on Bandcamp album: ${albumUrl}`); } - const ranked = entries - .map((entry) => ({ - entry, - score: titleMatchScore(entry.title, matchTitle), - })) + const tracks: BandcampAlbumTrackChoice[] = entries + .map((entry) => { + const url = entry.url || entry.webpage_url; + if (!url) return null; + return { + title: entry.title, + url, + score: matchTitle ? titleMatchScore(entry.title, matchTitle) : 0, + }; + }) + .filter((t): t is BandcampAlbumTrackChoice => t !== null) .sort((a, b) => b.score - a.score); - const best = ranked[0]; - if (!best || best.score < 0.5) { - const available = entries.map((e) => e.title).join(', '); + if (tracks.length === 0) { throw new Error( - `Could not match SoundCloud title β€œ${matchTitle}” to a track on the Bandcamp album. Available: ${available}`, + `Bandcamp album listed tracks but yt-dlp did not provide track URLs: ${albumUrl}`, ); } - const finalUrl = best.entry.url || best.entry.webpage_url; - if (!finalUrl) { - throw new Error( - `Matched β€œ${best.entry.title}” but yt-dlp did not provide a track URL`, + const best = matchTitle ? tracks[0] : undefined; + if (best && best.score >= ALBUM_MATCH_THRESHOLD) { + console.log( + `${this.sourceLabel}: matched β€œ${best.title}” (score ${best.score.toFixed(2)}) β†’ ${best.url}`, ); + return best.url; } - console.log( - `${this.sourceLabel}: matched β€œ${best.entry.title}” (score ${best.score.toFixed(2)}) β†’ ${finalUrl}`, + const available = tracks.map((t) => t.title).join(', '); + const message = matchTitle + ? `Could not match SoundCloud title β€œ${matchTitle}” to a track on the Bandcamp album. Available: ${available}` + : `Bandcamp album URL needs a track pick (no SoundCloud title to match). Available: ${available}`; + + const matchError = new BandcampAlbumMatchError( + message, + albumUrl, + matchTitle, + tracks, ); - return finalUrl; + if (onAlbumMatchFailed) { + const selectedUrl = await onAlbumMatchFailed(matchError); + if (!selectedUrl) { + throw new Error('No Bandcamp album track selected'); + } + const picked = tracks.find((t) => t.url === selectedUrl); + console.log( + picked + ? `${this.sourceLabel}: user selected β€œ${picked.title}” β†’ ${selectedUrl}` + : `${this.sourceLabel}: user selected track URL β†’ ${selectedUrl}`, + ); + return selectedUrl; + } + + throw matchError; } } diff --git a/webui/src/components/App.css b/webui/src/components/App.css index 959d1c5..d44ba35 100644 --- a/webui/src/components/App.css +++ b/webui/src/components/App.css @@ -516,6 +516,63 @@ margin-top: var(--space-sm); } +.bandcamp-track-picker { + display: flex; + flex-direction: column; + gap: var(--space-md); +} + +.bandcamp-track-list { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: var(--space-sm); + max-height: 320px; + overflow-y: auto; +} + +.bandcamp-track-option { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-md); + width: 100%; + padding: var(--space-md); + text-align: left; + background: var(--bg-elevated); + border: 1px solid var(--border-subtle); + border-radius: 8px; + color: var(--text-primary); + cursor: pointer; + transition: + border-color var(--transition-normal), + background var(--transition-normal); +} + +.bandcamp-track-option:hover:not(:disabled) { + border-color: var(--neon-orange); + background: var(--bg-surface); +} + +.bandcamp-track-option:disabled { + opacity: 0.6; + cursor: not-allowed; +} + +.bandcamp-track-title { + font-weight: 500; + line-height: 1.3; +} + +.bandcamp-track-score { + flex-shrink: 0; + font-family: var(--font-mono); + font-size: 0.75rem; + color: var(--text-secondary); +} + /* Metadata form */ .metadata-form { padding: var(--space-xl); diff --git a/webui/src/components/App.tsx b/webui/src/components/App.tsx index e4d2e44..5aee192 100644 --- a/webui/src/components/App.tsx +++ b/webui/src/components/App.tsx @@ -27,6 +27,13 @@ interface JobProgress { downloadBytes?: number; totalBytes?: number; browserless?: boolean; + bandcampAlbumTracks?: BandcampAlbumTrackChoice[]; +} + +interface BandcampAlbumTrackChoice { + title: string; + url: string; + score: number; } interface JobState { @@ -346,6 +353,12 @@ export default function App() { err instanceof Error ? err.message : 'Failed to load job', })); }); + } else if (progress.stage === 'waiting_bandcamp_track') { + setJob((prev) => ({ + ...prev, + progress, + error: null, + })); } else if (progress.stage === 'error') { eventSource.close(); eventSourceRef.current = null; @@ -609,6 +622,35 @@ export default function App() { } }; + const handleBandcampTrackSelect = async (trackUrl: string) => { + if (!job.jobId) return; + + setIsLoading(true); + setJob((prev) => ({ ...prev, error: null })); + + try { + const response = await fetch( + `${API_BASE}/api/job/${job.jobId}/bandcamp-track`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ trackUrl }), + }, + ); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.error || 'Failed to select Bandcamp track'); + } + } catch (err) { + setJob((prev) => ({ + ...prev, + error: err instanceof Error ? err.message : 'Unknown error', + })); + } finally { + setIsLoading(false); + } + }; + // Process metadata and finalize const processMetadata = async (preserveMetadata = false) => { if (!job.jobId) return; @@ -1000,39 +1042,81 @@ export default function App() { {/* Step 3: Progress */} {step === 'progress' && (
-
- - {job.progress?.message || 'Initializing...'} - - {job.progress?.currentGate && ( - - {job.progress.currentGate.toUpperCase()} - - )} -
-
-
-
-
- {formatPercent(job.progress?.percent)}% - {job.progress?.downloadBytes !== undefined && - job.progress?.totalBytes !== undefined && ( - - {(job.progress.downloadBytes / 1024 / 1024).toFixed(1)} /{' '} - {(job.progress.totalBytes / 1024 / 1024).toFixed(1)} MB + {job.progress?.stage === 'waiting_bandcamp_track' ? ( +
+
+ i +

+ {job.progress.message || + 'Could not auto-match a Bandcamp album track. Pick one to download.'} +

+
+
    + {(job.progress.bandcampAlbumTracks ?? []).map((track) => ( +
  • + +
    • + ))} +
+ +
+ ) : ( + <> +
+ + {job.progress?.message || 'Initializing...'} - )} -
- + {job.progress?.currentGate && ( + + {job.progress.currentGate.toUpperCase()} + + )} +
+
+
+
+
+ {formatPercent(job.progress?.percent)}% + {job.progress?.downloadBytes !== undefined && + job.progress?.totalBytes !== undefined && ( + + {(job.progress.downloadBytes / 1024 / 1024).toFixed(1)}{' '} + /{' '} + {(job.progress.totalBytes / 1024 / 1024).toFixed(1)} MB + + )} +
+ + + )}
)} From af3fee8cc31e4dc0e4fa3b713ccb7885450f3f6b Mon Sep 17 00:00:00 2001 From: D3SOX Date: Sat, 1 Aug 2026 01:58:52 +0200 Subject: [PATCH 3/6] fix: harden direct downloads and cover Bandcamp waiters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sanitize download filenames, bound fetch size/timeout, share link rules, block private resolve hops, rewrite Dropbox dl=0β†’dl=1, and test Bandcamp selection lifecycle. --- src/directDownload.ts | 83 ++++++++++++++++-------------- src/directLinkRules.ts | 45 +++++++++++++++++ src/jobStore.test.ts | 56 +++++++++++++++++++++ src/safeOutboundUrl.ts | 82 ++++++++++++++++++++++++++++++ src/utils.test.ts | 20 ++++++++ src/utils.ts | 97 +++++++++++++++++++----------------- webui/src/components/App.tsx | 2 +- 7 files changed, 301 insertions(+), 84 deletions(-) create mode 100644 src/directLinkRules.ts create mode 100644 src/jobStore.test.ts create mode 100644 src/safeOutboundUrl.ts diff --git a/src/directDownload.ts b/src/directDownload.ts index d740f5a..3a23470 100644 --- a/src/directDownload.ts +++ b/src/directDownload.ts @@ -1,13 +1,19 @@ import { mkdir } from 'node:fs/promises'; import { basename, join } from 'node:path'; +import { + normalizeDirectDownloadParsedUrl, + urlLooksLikeDirectDownload, +} from './directLinkRules'; import type { ProgressCallback } from './hypeddit'; -import { trimExtractedUrl } from './utils'; +import { assertSafeOutboundUrl } from './safeOutboundUrl'; +import { sanitizeFilenamePart, trimExtractedUrl } from './utils'; const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'; -const AUDIO_OR_ARCHIVE_EXT_RE = - /\.(mp3|wav|flac|aiff|aif|m4a|aac|ogg|opus|zip|rar)(\?|$)/i; +const FETCH_TIMEOUT_MS = 60_000; +/** Cap buffered downloads (audio / zip packages). */ +const MAX_DOWNLOAD_BYTES = 512 * 1024 * 1024; /** * True for http(s) URLs that look like a direct file download (Dropbox, Drive, @@ -15,43 +21,17 @@ const AUDIO_OR_ARCHIVE_EXT_RE = */ export function isDirectDownloadUrl(value: string): boolean { try { - const url = new URL(trimExtractedUrl(value.trim())); - if (!/^https?:$/i.test(url.protocol)) return false; - - const host = url.hostname.toLowerCase(); - if (/(?:^|\.)dropbox\.com$/i.test(host)) return true; - if (/(?:^|\.)dropboxusercontent\.com$/i.test(host)) return true; - if (/(?:^|\.)drive\.google\.com$/i.test(host)) return true; - if (/(?:^|\.)docs\.google\.com$/i.test(host)) return true; - - if (AUDIO_OR_ARCHIVE_EXT_RE.test(url.pathname)) return true; - if (url.searchParams.get('dl') === '1') return true; - if (url.searchParams.has('raw')) return true; - - return false; + return urlLooksLikeDirectDownload(new URL(trimExtractedUrl(value.trim()))); } catch { return false; } } -/** Normalize share links so fetch gets the file bytes (e.g. Dropbox dl=1). */ +/** Normalize share links so fetch gets the file bytes (e.g. Dropbox dl=0 β†’ dl=1). */ export function normalizeDirectDownloadUrl(url: string): string { const trimmed = trimExtractedUrl(url.trim()); try { - const parsed = new URL(trimmed); - if (/(?:^|\.)dropbox\.com$/i.test(parsed.hostname)) { - parsed.searchParams.set('dl', '1'); - return parsed.toString(); - } - // Google Drive file view β†’ uc?export=download - const driveFile = parsed.pathname.match(/\/file\/d\/([^/]+)/); - if ( - /(?:^|\.)drive\.google\.com$/i.test(parsed.hostname) && - driveFile?.[1] - ) { - return `https://drive.google.com/uc?export=download&id=${driveFile[1]}`; - } - return parsed.toString(); + return normalizeDirectDownloadParsedUrl(new URL(trimmed)); } catch { return trimmed; } @@ -67,6 +47,13 @@ function filenameFromContentDisposition(value: string | null): string | null { return plain ? plain.trim() : null; } +function safeDownloadFilename(raw: string | null): string | null { + if (!raw) return null; + const base = basename(raw); + const cleaned = sanitizeFilenamePart(base); + return cleaned || null; +} + export class DirectDownloader { private progressCallback: ProgressCallback | null = null; @@ -76,6 +63,7 @@ export class DirectDownloader { async downloadAudio(url: string): Promise { const downloadUrl = normalizeDirectDownloadUrl(url); + await assertSafeOutboundUrl(downloadUrl); console.log(`Downloading direct file: ${downloadUrl}`); this.progressCallback?.('downloading', 'Downloading direct file...', 40, { browserless: true, @@ -84,6 +72,7 @@ export class DirectDownloader { await mkdir('./downloads', { recursive: true }); const response = await fetch(downloadUrl, { redirect: 'follow', + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), headers: { 'user-agent': USER_AGENT, accept: '*/*', @@ -95,6 +84,8 @@ export class DirectDownloader { ); } + await assertSafeOutboundUrl(response.url); + const contentType = response.headers.get('content-type') ?? ''; if (/text\/html/i.test(contentType)) { throw new Error( @@ -102,18 +93,36 @@ export class DirectDownloader { ); } - const fromHeader = filenameFromContentDisposition( - response.headers.get('content-disposition'), + const contentLength = Number(response.headers.get('content-length') ?? ''); + if (Number.isFinite(contentLength) && contentLength > MAX_DOWNLOAD_BYTES) { + throw new Error( + `Direct download too large (${contentLength} bytes; max ${MAX_DOWNLOAD_BYTES})`, + ); + } + + const fromHeader = safeDownloadFilename( + filenameFromContentDisposition( + response.headers.get('content-disposition'), + ), + ); + const urlName = safeDownloadFilename( + decodeURIComponent(basename(new URL(response.url).pathname)), ); - const urlName = basename(new URL(response.url).pathname); const filename = fromHeader || (urlName && urlName !== '/' - ? decodeURIComponent(urlName) + ? urlName : `direct-download-${Date.now()}.bin`); + const body = await response.arrayBuffer(); + if (body.byteLength > MAX_DOWNLOAD_BYTES) { + throw new Error( + `Direct download too large (${body.byteLength} bytes; max ${MAX_DOWNLOAD_BYTES})`, + ); + } + const target = join('./downloads', filename); - await Bun.write(target, await response.arrayBuffer()); + await Bun.write(target, body); console.log(`Saved ${filename}`); return filename; } diff --git a/src/directLinkRules.ts b/src/directLinkRules.ts new file mode 100644 index 0000000..01bbdc0 --- /dev/null +++ b/src/directLinkRules.ts @@ -0,0 +1,45 @@ +/** + * Shared direct-download URL rules (no imports from utils / downloaders) so + * matchers and the downloader cannot drift apart. + */ + +export const AUDIO_OR_ARCHIVE_EXT_RE = + /\.(mp3|wav|flac|aiff|aif|m4a|aac|ogg|opus|zip|rar)(\?|$)/i; + +export function isKnownDirectDownloadHost(hostname: string): boolean { + const host = hostname.toLowerCase(); + return ( + /(?:^|\.)dropbox\.com$/i.test(host) || + /(?:^|\.)dropboxusercontent\.com$/i.test(host) || + /(?:^|\.)drive\.google\.com$/i.test(host) || + /(?:^|\.)docs\.google\.com$/i.test(host) + ); +} + +/** True when a parsed URL looks like a direct file link, not an HTML gate page. */ +export function urlLooksLikeDirectDownload(url: URL): boolean { + if (!/^https?:$/i.test(url.protocol)) return false; + if (isKnownDirectDownloadHost(url.hostname)) return true; + if (AUDIO_OR_ARCHIVE_EXT_RE.test(url.pathname)) return true; + // Dropbox preview links often use dl=0; treat any dl= as a download intent. + if (url.searchParams.has('dl')) return true; + if (url.searchParams.has('raw')) return true; + return false; +} + +/** + * Normalize share links so fetch gets the file bytes. + * Dropbox: force `dl=1` (rewrites `dl=0` preview links and adds `dl` when missing). + */ +export function normalizeDirectDownloadParsedUrl(url: URL): string { + if (/(?:^|\.)dropbox\.com$/i.test(url.hostname)) { + const parsed = new URL(url.toString()); + parsed.searchParams.set('dl', '1'); + return parsed.toString(); + } + const driveFile = url.pathname.match(/\/file\/d\/([^/]+)/); + if (/(?:^|\.)drive\.google\.com$/i.test(url.hostname) && driveFile?.[1]) { + return `https://drive.google.com/uc?export=download&id=${driveFile[1]}`; + } + return url.toString(); +} diff --git a/src/jobStore.test.ts b/src/jobStore.test.ts new file mode 100644 index 0000000..990599f --- /dev/null +++ b/src/jobStore.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from 'bun:test'; +import { jobStore } from './jobStore'; + +describe('jobStore Bandcamp track selection', () => { + test('resolveBandcampTrackSelection delivers URL to waiter', async () => { + const job = jobStore.create('https://soundcloud.com/a/b', 'mp3'); + try { + const wait = jobStore.waitForBandcampTrackSelection(job.id); + expect( + jobStore.resolveBandcampTrackSelection( + job.id, + 'https://x.bandcamp.com/track/y', + ), + ).toBe(true); + expect(await wait).toBe('https://x.bandcamp.com/track/y'); + } finally { + jobStore.delete(job.id); + } + }); + + test('cancel resolves pending wait with null', async () => { + const job = jobStore.create('https://soundcloud.com/a/b', 'mp3'); + try { + const wait = jobStore.waitForBandcampTrackSelection(job.id); + jobStore.cancel(job.id); + expect(await wait).toBeNull(); + } finally { + jobStore.delete(job.id); + } + }); + + test('delete resolves pending wait with null', async () => { + const job = jobStore.create('https://soundcloud.com/a/b', 'mp3'); + const wait = jobStore.waitForBandcampTrackSelection(job.id); + jobStore.delete(job.id); + expect(await wait).toBeNull(); + }); + + test('second wait supersedes first with null', async () => { + const job = jobStore.create('https://soundcloud.com/a/b', 'mp3'); + try { + const first = jobStore.waitForBandcampTrackSelection(job.id); + const second = jobStore.waitForBandcampTrackSelection(job.id); + expect(await first).toBeNull(); + expect( + jobStore.resolveBandcampTrackSelection( + job.id, + 'https://x.bandcamp.com/track/z', + ), + ).toBe(true); + expect(await second).toBe('https://x.bandcamp.com/track/z'); + } finally { + jobStore.delete(job.id); + } + }); +}); diff --git a/src/safeOutboundUrl.ts b/src/safeOutboundUrl.ts new file mode 100644 index 0000000..fb228dc --- /dev/null +++ b/src/safeOutboundUrl.ts @@ -0,0 +1,82 @@ +import { lookup } from 'node:dns/promises'; +import { isIP } from 'node:net'; + +function isPrivateOrLocalIp(ip: string): boolean { + const normalized = ip.toLowerCase().replace(/^\[|\]$/g, ''); + + if (isIP(normalized) === 4) { + const parts = normalized.split('.').map(Number); + const a = parts[0] ?? 0; + const b = parts[1] ?? 0; + if (a === 0 || a === 10 || a === 127) return true; + if (a === 169 && b === 254) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT + return false; + } + + if (isIP(normalized) === 6) { + if (normalized === '::1' || normalized === '::') return true; + if (normalized.startsWith('fe80:')) return true; // link-local + if (normalized.startsWith('fc') || normalized.startsWith('fd')) return true; // ULA + // IPv4-mapped + const v4mapped = normalized.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (v4mapped?.[1]) return isPrivateOrLocalIp(v4mapped[1]); + return false; + } + + return true; +} + +function isBlockedHostname(hostname: string): boolean { + const host = hostname.toLowerCase().replace(/\.$/, ''); + if ( + host === 'localhost' || + host === '0.0.0.0' || + host === '::1' || + host === 'metadata.google.internal' || + host.endsWith('.localhost') || + host.endsWith('.local') || + host.endsWith('.internal') + ) { + return true; + } + return false; +} + +/** + * Reject non-http(s) URLs and destinations that resolve to private / local + * addresses (basic SSRF guard for user-supplied and metadata-derived links). + */ +export async function assertSafeOutboundUrl(urlString: string): Promise { + let url: URL; + try { + url = new URL(urlString); + } catch { + throw new Error(`Invalid URL: ${urlString}`); + } + + if (!/^https?:$/i.test(url.protocol)) { + throw new Error(`Refusing non-http(s) URL: ${urlString}`); + } + + const host = url.hostname; + if (isBlockedHostname(host)) { + throw new Error(`Refusing to fetch non-public host: ${host}`); + } + + if (isIP(host) !== 0) { + if (isPrivateOrLocalIp(host)) { + throw new Error(`Refusing to fetch private/local address: ${host}`); + } + return; + } + + const { address } = await lookup(host); + if (isPrivateOrLocalIp(address)) { + throw new Error( + `Refusing to fetch host ${host} (resolves to private/local address ${address})`, + ); + } +} diff --git a/src/utils.test.ts b/src/utils.test.ts index 462345a..98419e4 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -64,6 +64,26 @@ describe('resolveGateProviderUrl', () => { }); }); + test('rewrites Dropbox dl=0 to dl=1 when dl is the only query param', () => { + expect( + resolveGateProviderUrl('https://www.dropbox.com/s/abc123/track.wav?dl=0'), + ).toEqual({ + url: 'https://www.dropbox.com/s/abc123/track.wav?dl=1', + provider: 'direct', + }); + }); + + test('adds dl=1 to Dropbox share links that omit dl', () => { + expect( + resolveGateProviderUrl( + 'https://www.dropbox.com/scl/fi/abc/track.wav?rlkey=xyz', + ), + ).toEqual({ + url: 'https://www.dropbox.com/scl/fi/abc/track.wav?rlkey=xyz&dl=1', + provider: 'direct', + }); + }); + test('matches raw audio file URLs as direct downloads', () => { expect( resolveGateProviderUrl( diff --git a/src/utils.ts b/src/utils.ts index 96a83f5..5165444 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -5,6 +5,11 @@ import { lookpath } from 'find-bin'; import type { CookieData } from 'puppeteer'; import type { SoundcloudTrack } from 'soundcloud.ts'; import packageJson from '../package.json' with { type: 'json' }; +import { + normalizeDirectDownloadParsedUrl, + urlLooksLikeDirectDownload, +} from './directLinkRules'; +import { assertSafeOutboundUrl } from './safeOutboundUrl'; import type { LocalCookieData, Metadata } from './types'; export const REPO_URL = packageJson.repository.url; @@ -395,40 +400,16 @@ function matchBandcampUrl( function matchDirectDownloadUrl( value: string, ): { url: string; provider: GateProvider } | null { - // Inline host/ext checks (same rules as isDirectDownloadUrl) to avoid a - // circular import with directDownload.ts β†’ trimExtractedUrl. const httpMatch = value.match(/https?:\/\/[^\s<>"')\]]+/i)?.[0]; if (!httpMatch) return null; const trimmed = trimExtractedUrl(httpMatch); try { - const url = new URL(trimmed); - if (!/^https?:$/i.test(url.protocol)) return null; - const host = url.hostname.toLowerCase(); - const isHost = - /(?:^|\.)dropbox\.com$/i.test(host) || - /(?:^|\.)dropboxusercontent\.com$/i.test(host) || - /(?:^|\.)drive\.google\.com$/i.test(host) || - /(?:^|\.)docs\.google\.com$/i.test(host); - const isExt = - /\.(mp3|wav|flac|aiff|aif|m4a|aac|ogg|opus|zip|rar)(\?|$)/i.test( - url.pathname, - ); - const isFlag = - url.searchParams.get('dl') === '1' || url.searchParams.has('raw'); - if (!isHost && !isExt && !isFlag) return null; - - let normalized = trimmed.replace(/^http:\/\//i, 'https://'); - if (/(?:^|\.)dropbox\.com$/i.test(host)) { - const parsed = new URL(normalized); - parsed.searchParams.set('dl', '1'); - normalized = parsed.toString(); - } else { - const driveFile = url.pathname.match(/\/file\/d\/([^/]+)/); - if (/(?:^|\.)drive\.google\.com$/i.test(host) && driveFile?.[1]) { - normalized = `https://drive.google.com/uc?export=download&id=${driveFile[1]}`; - } - } - return { url: normalized, provider: 'direct' }; + const url = new URL(trimmed.replace(/^http:\/\//i, 'https://')); + if (!urlLooksLikeDirectDownload(url)) return null; + return { + url: normalizeDirectDownloadParsedUrl(url), + provider: 'direct', + }; } catch { return null; } @@ -462,6 +443,7 @@ export function findKnownGateInHtml( /** * Resolve an unrecognized (or already-known) URL to a download gate by * following HTTP redirects and scanning the final HTML for embedded destinations. + * Each hop is validated against private/local destinations before fetching. */ export async function resolveUnknownGateUrl( url: string, @@ -471,32 +453,55 @@ export async function resolveUnknownGateUrl( return null; } - const direct = matchKnownDownloadGateUrl(trimmed); - if (direct) return direct; + const knownUpFront = matchKnownDownloadGateUrl(trimmed); + if (knownUpFront) return knownUpFront; // SoundCloud pages are not smart-link hops we want to chase. if (isSoundcloudUrl(trimmed)) { return null; } + const maxHops = 10; + let current = trimmed; + try { - const response = await fetch(trimmed, { - redirect: 'follow', - headers: { - 'user-agent': GATE_RESOLVE_USER_AGENT, - accept: - 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8', - }, - }); - const finalUrl = trimExtractedUrl(response.url); - const fromFinal = matchKnownDownloadGateUrl(finalUrl); - if (fromFinal) return fromFinal; - - const html = await response.text(); - return findKnownGateInHtml(html); + for (let hop = 0; hop < maxHops; hop++) { + await assertSafeOutboundUrl(current); + + const known = matchKnownDownloadGateUrl(current); + if (known) return known; + if (isSoundcloudUrl(current)) return null; + + const response = await fetch(current, { + redirect: 'manual', + signal: AbortSignal.timeout(15_000), + headers: { + 'user-agent': GATE_RESOLVE_USER_AGENT, + accept: + 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8', + }, + }); + + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get('location'); + if (!location) return null; + current = trimExtractedUrl(new URL(location, current).toString()); + continue; + } + + const finalUrl = trimExtractedUrl(response.url || current); + await assertSafeOutboundUrl(finalUrl); + const fromFinal = matchKnownDownloadGateUrl(finalUrl); + if (fromFinal) return fromFinal; + + const html = await response.text(); + return findKnownGateInHtml(html); + } } catch { return null; } + + return null; } function collectUnresolvedHttpCandidates( diff --git a/webui/src/components/App.tsx b/webui/src/components/App.tsx index 5aee192..973dea0 100644 --- a/webui/src/components/App.tsx +++ b/webui/src/components/App.tsx @@ -987,7 +987,7 @@ export default function App() { name="hypeddit-url" value={hypedditUrlInput} onChange={(e) => setHypedditUrlInput(e.target.value)} - placeholder="https://hypeddit.com/... / droploud.com/gate/... / gaterush.me/... / downloadgater.com/g/... / artist.bandcamp.com/track/... / dropbox.com/...&dl=1" + placeholder="https://hypeddit.com/... / droploud.com/gate/... / gaterush.me/... / downloadgater.com/g/... / artist.bandcamp.com/track/... / https://www.dropbox.com/...?dl=1" autoComplete="off" required disabled={isLoading} From 7e10bc877669ed494cba29d912749d1324db796a Mon Sep 17 00:00:00 2001 From: D3SOX Date: Sat, 1 Aug 2026 02:09:08 +0200 Subject: [PATCH 4/6] fix: bind outbound fetches to validated IPs and stream downloads Connect via DNS-validated public addresses with Host/SNI, follow redirects manually, enforce size limits while streaming, and tighten direct-link heuristics. --- src/directDownload.ts | 120 ++++++++++++++++++++++++++++++--------- src/directLinkRules.ts | 6 +- src/safeOutboundUrl.ts | 124 ++++++++++++++++++++++++++++++++++++++--- src/soundcloud.ts | 26 ++++----- src/utils.ts | 16 +++--- 5 files changed, 230 insertions(+), 62 deletions(-) diff --git a/src/directDownload.ts b/src/directDownload.ts index 3a23470..87c3684 100644 --- a/src/directDownload.ts +++ b/src/directDownload.ts @@ -1,18 +1,19 @@ -import { mkdir } from 'node:fs/promises'; +import { mkdir, unlink } from 'node:fs/promises'; import { basename, join } from 'node:path'; import { normalizeDirectDownloadParsedUrl, urlLooksLikeDirectDownload, } from './directLinkRules'; import type { ProgressCallback } from './hypeddit'; -import { assertSafeOutboundUrl } from './safeOutboundUrl'; +import { safeFetch } from './safeOutboundUrl'; import { sanitizeFilenamePart, trimExtractedUrl } from './utils'; const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'; const FETCH_TIMEOUT_MS = 60_000; -/** Cap buffered downloads (audio / zip packages). */ +const MAX_REDIRECTS = 10; +/** Cap downloads (audio / zip packages). */ const MAX_DOWNLOAD_BYTES = 512 * 1024 * 1024; /** @@ -54,6 +55,39 @@ function safeDownloadFilename(raw: string | null): string | null { return cleaned || null; } +async function writeBodyWithSizeLimit( + body: ReadableStream, + target: string, + maxBytes: number, +): Promise { + const writer = Bun.file(target).writer(); + const reader = body.getReader(); + let total = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) { + await reader.cancel(); + throw new Error( + `Direct download too large (${total} bytes; max ${maxBytes})`, + ); + } + writer.write(value); + } + await writer.end(); + } catch (error) { + try { + writer.end(); + } catch { + // ignore + } + await unlink(target).catch(() => {}); + throw error; + } +} + export class DirectDownloader { private progressCallback: ProgressCallback | null = null; @@ -63,41 +97,76 @@ export class DirectDownloader { async downloadAudio(url: string): Promise { const downloadUrl = normalizeDirectDownloadUrl(url); - await assertSafeOutboundUrl(downloadUrl); console.log(`Downloading direct file: ${downloadUrl}`); this.progressCallback?.('downloading', 'Downloading direct file...', 40, { browserless: true, }); await mkdir('./downloads', { recursive: true }); - const response = await fetch(downloadUrl, { - redirect: 'follow', - signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), - headers: { - 'user-agent': USER_AGENT, - accept: '*/*', - }, - }); + + let current = downloadUrl; + let response: Response | null = null; + let finalUrl = current; + + for (let hop = 0; hop < MAX_REDIRECTS; hop++) { + const result = await safeFetch(current, { + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + headers: { + 'user-agent': USER_AGENT, + accept: '*/*', + }, + }); + finalUrl = result.url; + response = result.response; + + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get('location'); + await response.body?.cancel().catch(() => {}); + if (!location) { + throw new Error( + `Direct download redirect missing Location from ${current}`, + ); + } + current = new URL(location, current).toString(); + continue; + } + break; + } + + if (!response) { + throw new Error(`Direct download failed for ${downloadUrl}`); + } + if (response.status >= 300 && response.status < 400) { + throw new Error( + `Direct download exceeded ${MAX_REDIRECTS} redirects for ${downloadUrl}`, + ); + } if (!response.ok) { throw new Error( - `Direct download failed: HTTP ${response.status} for ${downloadUrl}`, + `Direct download failed: HTTP ${response.status} for ${finalUrl}`, ); } - await assertSafeOutboundUrl(response.url); - const contentType = response.headers.get('content-type') ?? ''; if (/text\/html/i.test(contentType)) { + await response.body?.cancel().catch(() => {}); throw new Error( 'Direct download returned HTML instead of a file β€” check that the link is a direct download (e.g. Dropbox with dl=1).', ); } - const contentLength = Number(response.headers.get('content-length') ?? ''); - if (Number.isFinite(contentLength) && contentLength > MAX_DOWNLOAD_BYTES) { - throw new Error( - `Direct download too large (${contentLength} bytes; max ${MAX_DOWNLOAD_BYTES})`, - ); + const contentLengthHeader = response.headers.get('content-length'); + if (contentLengthHeader) { + const contentLength = Number(contentLengthHeader); + if ( + Number.isFinite(contentLength) && + contentLength > MAX_DOWNLOAD_BYTES + ) { + await response.body?.cancel().catch(() => {}); + throw new Error( + `Direct download too large (${contentLength} bytes; max ${MAX_DOWNLOAD_BYTES})`, + ); + } } const fromHeader = safeDownloadFilename( @@ -106,7 +175,7 @@ export class DirectDownloader { ), ); const urlName = safeDownloadFilename( - decodeURIComponent(basename(new URL(response.url).pathname)), + decodeURIComponent(basename(new URL(finalUrl).pathname)), ); const filename = fromHeader || @@ -114,15 +183,12 @@ export class DirectDownloader { ? urlName : `direct-download-${Date.now()}.bin`); - const body = await response.arrayBuffer(); - if (body.byteLength > MAX_DOWNLOAD_BYTES) { - throw new Error( - `Direct download too large (${body.byteLength} bytes; max ${MAX_DOWNLOAD_BYTES})`, - ); + if (!response.body) { + throw new Error(`Direct download returned an empty body for ${finalUrl}`); } const target = join('./downloads', filename); - await Bun.write(target, body); + await writeBodyWithSizeLimit(response.body, target, MAX_DOWNLOAD_BYTES); console.log(`Saved ${filename}`); return filename; } diff --git a/src/directLinkRules.ts b/src/directLinkRules.ts index 01bbdc0..af9cbe3 100644 --- a/src/directLinkRules.ts +++ b/src/directLinkRules.ts @@ -11,8 +11,7 @@ export function isKnownDirectDownloadHost(hostname: string): boolean { return ( /(?:^|\.)dropbox\.com$/i.test(host) || /(?:^|\.)dropboxusercontent\.com$/i.test(host) || - /(?:^|\.)drive\.google\.com$/i.test(host) || - /(?:^|\.)docs\.google\.com$/i.test(host) + /(?:^|\.)drive\.google\.com$/i.test(host) ); } @@ -21,9 +20,6 @@ export function urlLooksLikeDirectDownload(url: URL): boolean { if (!/^https?:$/i.test(url.protocol)) return false; if (isKnownDirectDownloadHost(url.hostname)) return true; if (AUDIO_OR_ARCHIVE_EXT_RE.test(url.pathname)) return true; - // Dropbox preview links often use dl=0; treat any dl= as a download intent. - if (url.searchParams.has('dl')) return true; - if (url.searchParams.has('raw')) return true; return false; } diff --git a/src/safeOutboundUrl.ts b/src/safeOutboundUrl.ts index fb228dc..fdbdde8 100644 --- a/src/safeOutboundUrl.ts +++ b/src/safeOutboundUrl.ts @@ -1,4 +1,6 @@ import { lookup } from 'node:dns/promises'; +import http from 'node:http'; +import https from 'node:https'; import { isIP } from 'node:net'; function isPrivateOrLocalIp(ip: string): boolean { @@ -20,7 +22,6 @@ function isPrivateOrLocalIp(ip: string): boolean { if (normalized === '::1' || normalized === '::') return true; if (normalized.startsWith('fe80:')) return true; // link-local if (normalized.startsWith('fc') || normalized.startsWith('fd')) return true; // ULA - // IPv4-mapped const v4mapped = normalized.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); if (v4mapped?.[1]) return isPrivateOrLocalIp(v4mapped[1]); return false; @@ -45,11 +46,19 @@ function isBlockedHostname(hostname: string): boolean { return false; } +export type SafeConnectTarget = { + url: URL; + /** Validated public address used for the TCP connection. */ + address: string; +}; + /** - * Reject non-http(s) URLs and destinations that resolve to private / local - * addresses (basic SSRF guard for user-supplied and metadata-derived links). + * Resolve a URL to a public connect address. Rejects private/local hosts and + * DNS answers that only resolve to private/local IPs. */ -export async function assertSafeOutboundUrl(urlString: string): Promise { +export async function resolveSafeConnectTarget( + urlString: string, +): Promise { let url: URL; try { url = new URL(urlString); @@ -70,13 +79,112 @@ export async function assertSafeOutboundUrl(urlString: string): Promise { if (isPrivateOrLocalIp(host)) { throw new Error(`Refusing to fetch private/local address: ${host}`); } - return; + return { url, address: host.replace(/^\[|\]$/g, '') }; } - const { address } = await lookup(host); - if (isPrivateOrLocalIp(address)) { + const results = await lookup(host, { all: true }); + const publicAddrs = results.filter((r) => !isPrivateOrLocalIp(r.address)); + const chosen = publicAddrs[0]; + if (!chosen) { + const sample = results[0]?.address ?? 'none'; throw new Error( - `Refusing to fetch host ${host} (resolves to private/local address ${address})`, + `Refusing to fetch host ${host} (resolves to private/local address ${sample})`, ); } + return { url, address: chosen.address }; +} + +/** @deprecated Prefer resolveSafeConnectTarget / safeFetch */ +export async function assertSafeOutboundUrl(urlString: string): Promise { + await resolveSafeConnectTarget(urlString); +} + +export type SafeFetchInit = { + method?: string; + headers?: Record; + signal?: AbortSignal; +}; + +export type SafeFetchResult = { + response: Response; + /** Request URL (redirects are not followed). */ + url: string; +}; + +/** + * Fetch via a validated public IP while keeping Host + TLS SNI as the original + * hostname, so DNS TOCTOU cannot redirect the TCP connection to a private IP. + * Redirects are never followed β€” callers must validate each Location hop. + */ +export async function safeFetch( + urlString: string, + init: SafeFetchInit = {}, +): Promise { + const { url, address } = await resolveSafeConnectTarget(urlString); + const isHttps = url.protocol === 'https:'; + const transport = isHttps ? https : http; + const port = url.port ? Number(url.port) : isHttps ? 443 : 80; + const path = `${url.pathname}${url.search}`; + const headers: Record = { + ...(init.headers ?? {}), + host: url.host, + }; + + return await new Promise((resolve, reject) => { + const req = transport.request( + { + hostname: address, + port, + path, + method: init.method ?? 'GET', + headers, + servername: isHttps ? url.hostname : undefined, + signal: init.signal, + }, + (res) => { + const responseHeaders = new Headers(); + for (const [key, value] of Object.entries(res.headers)) { + if (value === undefined) continue; + if (Array.isArray(value)) { + for (const item of value) responseHeaders.append(key, item); + } else { + responseHeaders.set(key, value); + } + } + + const body = new ReadableStream({ + start(controller) { + res.on('data', (chunk: Buffer | string) => { + const bytes = + typeof chunk === 'string' ? Buffer.from(chunk) : chunk; + controller.enqueue(new Uint8Array(bytes)); + }); + res.on('end', () => { + try { + controller.close(); + } catch { + // already closed + } + }); + res.on('error', (err) => controller.error(err)); + }, + cancel() { + res.destroy(); + }, + }); + + resolve({ + url: url.toString(), + response: new Response(body, { + status: res.statusCode ?? 0, + statusText: res.statusMessage, + headers: responseHeaders, + }), + }); + }, + ); + + req.on('error', reject); + req.end(); + }); } diff --git a/src/soundcloud.ts b/src/soundcloud.ts index ee42571..942c795 100644 --- a/src/soundcloud.ts +++ b/src/soundcloud.ts @@ -10,9 +10,20 @@ import Soundcloud, { import { extractAndResolveGateUrl, extractCaptchaDeliveryUrl, + type GateProvider, loadCookies, } from './utils'; +const GATE_PROVIDER_LABELS: Record = { + droploud: 'Droploud', + gaterush: 'GateRush', + downloadgater: 'DownloadGater', + direct: 'direct download', + bandcamp: 'Bandcamp', + soundcloud: 'SoundCloud', + hypeddit: 'Hypeddit', +}; + interface SoundcloudCredentials { clientId: string; oauthToken: string; @@ -522,20 +533,7 @@ export class SoundcloudClient { if (!gate) { return null; } - const providerLabel = - gate.provider === 'droploud' - ? 'Droploud' - : gate.provider === 'gaterush' - ? 'GateRush' - : gate.provider === 'downloadgater' - ? 'DownloadGater' - : gate.provider === 'direct' - ? 'direct download' - : gate.provider === 'bandcamp' - ? 'Bandcamp' - : gate.provider === 'soundcloud' - ? 'SoundCloud' - : 'Hypeddit'; + const providerLabel = GATE_PROVIDER_LABELS[gate.provider]; const sourceLabel = gate.type === 'purchase_url' ? 'purchase URL' : 'description'; console.log( diff --git a/src/utils.ts b/src/utils.ts index 5165444..181d4b2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -9,7 +9,7 @@ import { normalizeDirectDownloadParsedUrl, urlLooksLikeDirectDownload, } from './directLinkRules'; -import { assertSafeOutboundUrl } from './safeOutboundUrl'; +import { safeFetch } from './safeOutboundUrl'; import type { LocalCookieData, Metadata } from './types'; export const REPO_URL = packageJson.repository.url; @@ -466,14 +466,11 @@ export async function resolveUnknownGateUrl( try { for (let hop = 0; hop < maxHops; hop++) { - await assertSafeOutboundUrl(current); - const known = matchKnownDownloadGateUrl(current); if (known) return known; if (isSoundcloudUrl(current)) return null; - const response = await fetch(current, { - redirect: 'manual', + const { response } = await safeFetch(current, { signal: AbortSignal.timeout(15_000), headers: { 'user-agent': GATE_RESOLVE_USER_AGENT, @@ -484,15 +481,18 @@ export async function resolveUnknownGateUrl( if (response.status >= 300 && response.status < 400) { const location = response.headers.get('location'); + await response.body?.cancel().catch(() => {}); if (!location) return null; current = trimExtractedUrl(new URL(location, current).toString()); continue; } - const finalUrl = trimExtractedUrl(response.url || current); - await assertSafeOutboundUrl(finalUrl); + const finalUrl = trimExtractedUrl(current); const fromFinal = matchKnownDownloadGateUrl(finalUrl); - if (fromFinal) return fromFinal; + if (fromFinal) { + await response.body?.cancel().catch(() => {}); + return fromFinal; + } const html = await response.text(); return findKnownGateInHtml(html); From 726d6c3bb58655674268374752c2632ff2861d74 Mon Sep 17 00:00:00 2001 From: D3SOX Date: Sat, 1 Aug 2026 08:59:43 +0200 Subject: [PATCH 5/6] fix: handle bracketed IPv6 hosts and validate Drive file ids Strip IPv6 brackets before IP/blocklist checks, and only treat Google Drive URLs with a valid file id as direct downloads. --- src/directLinkRules.ts | 35 +++++++++++++++++++++++++++++++---- src/safeOutboundUrl.ts | 11 ++++++++--- src/utils.test.ts | 20 ++++++++++++++++++++ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/directLinkRules.ts b/src/directLinkRules.ts index af9cbe3..5cf3cda 100644 --- a/src/directLinkRules.ts +++ b/src/directLinkRules.ts @@ -6,6 +6,9 @@ export const AUDIO_OR_ARCHIVE_EXT_RE = /\.(mp3|wav|flac|aiff|aif|m4a|aac|ogg|opus|zip|rar)(\?|$)/i; +/** Google Drive file IDs are long URL-safe tokens. */ +const DRIVE_FILE_ID_RE = /^[a-zA-Z0-9_-]{10,}$/; + export function isKnownDirectDownloadHost(hostname: string): boolean { const host = hostname.toLowerCase(); return ( @@ -15,10 +18,28 @@ export function isKnownDirectDownloadHost(hostname: string): boolean { ); } +/** Extract a validated Drive file id from path (`/file/d/ID`) or `id` query. */ +export function extractDriveFileId(url: URL): string | null { + const fromPath = url.pathname.match(/\/file\/d\/([^/]+)/)?.[1]; + if (fromPath && DRIVE_FILE_ID_RE.test(fromPath)) return fromPath; + const fromQuery = url.searchParams.get('id'); + if (fromQuery && DRIVE_FILE_ID_RE.test(fromQuery)) return fromQuery; + return null; +} + /** True when a parsed URL looks like a direct file link, not an HTML gate page. */ export function urlLooksLikeDirectDownload(url: URL): boolean { if (!/^https?:$/i.test(url.protocol)) return false; - if (isKnownDirectDownloadHost(url.hostname)) return true; + const host = url.hostname.toLowerCase(); + if ( + /(?:^|\.)dropbox\.com$/i.test(host) || + /(?:^|\.)dropboxusercontent\.com$/i.test(host) + ) { + return true; + } + if (/(?:^|\.)drive\.google\.com$/i.test(host)) { + return extractDriveFileId(url) !== null; + } if (AUDIO_OR_ARCHIVE_EXT_RE.test(url.pathname)) return true; return false; } @@ -26,6 +47,7 @@ export function urlLooksLikeDirectDownload(url: URL): boolean { /** * Normalize share links so fetch gets the file bytes. * Dropbox: force `dl=1` (rewrites `dl=0` preview links and adds `dl` when missing). + * Drive: only rewrite when a valid file id is present. */ export function normalizeDirectDownloadParsedUrl(url: URL): string { if (/(?:^|\.)dropbox\.com$/i.test(url.hostname)) { @@ -33,9 +55,14 @@ export function normalizeDirectDownloadParsedUrl(url: URL): string { parsed.searchParams.set('dl', '1'); return parsed.toString(); } - const driveFile = url.pathname.match(/\/file\/d\/([^/]+)/); - if (/(?:^|\.)drive\.google\.com$/i.test(url.hostname) && driveFile?.[1]) { - return `https://drive.google.com/uc?export=download&id=${driveFile[1]}`; + if (/(?:^|\.)drive\.google\.com$/i.test(url.hostname)) { + const id = extractDriveFileId(url); + if (!id) { + throw new Error( + `Malformed Google Drive link (missing valid file id): ${url}`, + ); + } + return `https://drive.google.com/uc?export=download&id=${id}`; } return url.toString(); } diff --git a/src/safeOutboundUrl.ts b/src/safeOutboundUrl.ts index fdbdde8..3a52587 100644 --- a/src/safeOutboundUrl.ts +++ b/src/safeOutboundUrl.ts @@ -31,7 +31,10 @@ function isPrivateOrLocalIp(ip: string): boolean { } function isBlockedHostname(hostname: string): boolean { - const host = hostname.toLowerCase().replace(/\.$/, ''); + const host = hostname + .toLowerCase() + .replace(/^\[|\]$/g, '') + .replace(/\.$/, ''); if ( host === 'localhost' || host === '0.0.0.0' || @@ -70,7 +73,9 @@ export async function resolveSafeConnectTarget( throw new Error(`Refusing non-http(s) URL: ${urlString}`); } - const host = url.hostname; + // URL.hostname is usually unbracketed for IPv6, but strip defensively so + // isIP / blocklist / connect address never see "[::1]". + const host = url.hostname.replace(/^\[|\]$/g, ''); if (isBlockedHostname(host)) { throw new Error(`Refusing to fetch non-public host: ${host}`); } @@ -79,7 +84,7 @@ export async function resolveSafeConnectTarget( if (isPrivateOrLocalIp(host)) { throw new Error(`Refusing to fetch private/local address: ${host}`); } - return { url, address: host.replace(/^\[|\]$/g, '') }; + return { url, address: host }; } const results = await lookup(host, { all: true }); diff --git a/src/utils.test.ts b/src/utils.test.ts index 98419e4..4820772 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -84,6 +84,26 @@ describe('resolveGateProviderUrl', () => { }); }); + test('normalizes valid Google Drive file links', () => { + expect( + resolveGateProviderUrl( + 'https://drive.google.com/file/d/abcdefghijklmnopqrstuvwx/view?usp=sharing', + ), + ).toEqual({ + url: 'https://drive.google.com/uc?export=download&id=abcdefghijklmnopqrstuvwx', + provider: 'direct', + }); + }); + + test('does not treat malformed Google Drive URLs as direct downloads', () => { + expect( + resolveGateProviderUrl('https://drive.google.com/drive/my-drive'), + ).toBeNull(); + expect( + resolveGateProviderUrl('https://drive.google.com/file/d/short/view'), + ).toBeNull(); + }); + test('matches raw audio file URLs as direct downloads', () => { expect( resolveGateProviderUrl( From 2b8171432d226a1a6f15095361b62a02dc5dbb39 Mon Sep 17 00:00:00 2001 From: D3SOX Date: Sat, 1 Aug 2026 09:09:25 +0200 Subject: [PATCH 6/6] fix: preserve Google Drive resourcekey on download URLs Link-shared Drive files can require resourcekey; keep it when rewriting to uc?export=download. --- src/directLinkRules.ts | 16 +++++++++++++++- src/utils.test.ts | 11 +++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/directLinkRules.ts b/src/directLinkRules.ts index 5cf3cda..f7ef56e 100644 --- a/src/directLinkRules.ts +++ b/src/directLinkRules.ts @@ -62,7 +62,21 @@ export function normalizeDirectDownloadParsedUrl(url: URL): string { `Malformed Google Drive link (missing valid file id): ${url}`, ); } - return `https://drive.google.com/uc?export=download&id=${id}`; + const download = new URL('https://drive.google.com/uc'); + download.searchParams.set('export', 'download'); + download.searchParams.set('id', id); + // Link-shared files may require resourcekey alongside the file id. + let resourceKey: string | null = null; + for (const [key, value] of url.searchParams) { + if (key.toLowerCase() === 'resourcekey' && value) { + resourceKey = value; + break; + } + } + if (resourceKey) { + download.searchParams.set('resourcekey', resourceKey); + } + return download.toString(); } return url.toString(); } diff --git a/src/utils.test.ts b/src/utils.test.ts index 4820772..7f08d1b 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -95,6 +95,17 @@ describe('resolveGateProviderUrl', () => { }); }); + test('preserves Google Drive resourcekey on normalized download URLs', () => { + expect( + resolveGateProviderUrl( + 'https://drive.google.com/file/d/abcdefghijklmnopqrstuvwx/view?usp=sharing&resourcekey=abc-123', + ), + ).toEqual({ + url: 'https://drive.google.com/uc?export=download&id=abcdefghijklmnopqrstuvwx&resourcekey=abc-123', + provider: 'direct', + }); + }); + test('does not treat malformed Google Drive URLs as direct downloads', () => { expect( resolveGateProviderUrl('https://drive.google.com/drive/my-drive'),