diff --git a/README.md b/README.md index 9b37182..f2f1530 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # sc-gate-dl -Download and tag SoundCloud tracks unlocked via Hypeddit, Droploud, GateRush, DownloadGater, or Bandcamp. +Download and tag SoundCloud tracks unlocked via Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, or a direct download link. ## Features -- 🎡 Automatically download audio from Hypeddit, Droploud, GateRush, DownloadGater, and Bandcamp links +- 🎡 Automatically download audio from Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, and direct file links (e.g. Dropbox `dl=1`) - ⚑ Browserless fast path that skips the browser for gates that don't need real verification (see [How It Works](#how-it-works)) - πŸ”„ Handles multiple gate types (see [How It Works](#how-it-works)) - πŸ“ Fetches metadata from the provided SoundCloud link @@ -190,7 +190,9 @@ Panel size/position is remembered in `localStorage` (`sc-gate-dl-panel-geom`). Droploud, GateRush, and DownloadGater gates are handled via their own downloaders with similar email / social unlock flows. -**Bandcamp / yt-dlp**: When a SoundCloud track’s purchase URL (or description) points at Bandcamp instead of a gate, the file is downloaded with [yt-dlp](https://github.com/yt-dlp/yt-dlp) (browserless). For Bandcamp album links, the matching track is selected from the SoundCloud title. Traditional unlock gates still take priority if both are present. If no gate or Bandcamp URL is found, the CLI and Web UI can fall back to downloading the SoundCloud track itself via yt-dlp. +**Bandcamp / yt-dlp**: When a SoundCloud track’s purchase URL (or description) points at Bandcamp instead of a gate, the file is downloaded with [yt-dlp](https://github.com/yt-dlp/yt-dlp) (browserless). For Bandcamp album links, the matching track is selected from the SoundCloud title; if auto-match fails, the CLI and Web UI let you pick a track from the album. Traditional unlock gates still take priority if both are present. If no gate or Bandcamp URL is found, the CLI and Web UI can fall back to downloading the SoundCloud track itself via yt-dlp. + +**Direct download**: Paste a file URL (Dropbox with `dl=1`, Google Drive file links, or any http(s) URL ending in a common audio/archive extension). The file is fetched browserlessly β€” useful when a gate is unsupported but you already have the download link. **File Processing**: diff --git a/src/browserLaunch.ts b/src/browserLaunch.ts index d73aefb..5d8bb2a 100644 --- a/src/browserLaunch.ts +++ b/src/browserLaunch.ts @@ -4,13 +4,14 @@ import type { Browser } from 'puppeteer'; export type AppBrowserLaunchOptions = { headless?: boolean; userDataDir?: string; + /** Extra Chromium/CloakBrowser flags merged onto the defaults. */ args?: string[]; /** Passed through to Puppeteer via cloakbrowser `launchOptions`. */ defaultViewport?: { width: number; height: number } | null; humanize?: boolean; }; -const DEFAULT_ARGS = [ +export const DEFAULT_BROWSER_ARGS = [ '--no-sandbox', '--disable-setuid-sandbox', '--mute-audio', @@ -38,12 +39,14 @@ export async function launchAppBrowser( const geoip = Boolean(proxy) && process.env.CLOAKBROWSER_GEOIP === 'true'; + const args = [...DEFAULT_BROWSER_ARGS, ...(options.args ?? [])]; + const launchOpts = { headless: options.headless ?? true, humanize: options.humanize ?? true, stealthArgs: true, ...(proxy ? { proxy, ...(geoip ? { geoip: true } : {}) } : {}), - args: options.args ?? DEFAULT_ARGS, + args, launchOptions: { ...(options.defaultViewport !== undefined ? { defaultViewport: options.defaultViewport } diff --git a/src/directDownload.ts b/src/directDownload.ts new file mode 100644 index 0000000..87c3684 --- /dev/null +++ b/src/directDownload.ts @@ -0,0 +1,199 @@ +import { mkdir, unlink } from 'node:fs/promises'; +import { basename, join } from 'node:path'; +import { + normalizeDirectDownloadParsedUrl, + urlLooksLikeDirectDownload, +} from './directLinkRules'; +import type { ProgressCallback } from './hypeddit'; +import { safeFetch } from './safeOutboundUrl'; +import { sanitizeFilenamePart, trimExtractedUrl } from './utils'; + +const USER_AGENT = + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'; + +const FETCH_TIMEOUT_MS = 60_000; +const MAX_REDIRECTS = 10; +/** Cap downloads (audio / zip packages). */ +const MAX_DOWNLOAD_BYTES = 512 * 1024 * 1024; + +/** + * True for http(s) URLs that look like a direct file download (Dropbox, Drive, + * or a path with a common audio/archive extension) β€” not HTML gate pages. + */ +export function isDirectDownloadUrl(value: string): boolean { + try { + return urlLooksLikeDirectDownload(new URL(trimExtractedUrl(value.trim()))); + } catch { + return false; + } +} + +/** Normalize share links so fetch gets the file bytes (e.g. Dropbox dl=0 β†’ dl=1). */ +export function normalizeDirectDownloadUrl(url: string): string { + const trimmed = trimExtractedUrl(url.trim()); + try { + return normalizeDirectDownloadParsedUrl(new URL(trimmed)); + } catch { + return trimmed; + } +} + +function filenameFromContentDisposition(value: string | null): string | null { + if (!value) return null; + const star = value.match(/filename\*=(?:UTF-8'')?([^;]+)/i)?.[1]; + if (star) { + return decodeURIComponent(star.replace(/["']/g, '')); + } + const plain = value.match(/filename=["']?([^"';]+)["']?/i)?.[1]; + return plain ? plain.trim() : null; +} + +function safeDownloadFilename(raw: string | null): string | null { + if (!raw) return null; + const base = basename(raw); + const cleaned = sanitizeFilenamePart(base); + return cleaned || null; +} + +async function writeBodyWithSizeLimit( + body: ReadableStream, + target: string, + maxBytes: number, +): Promise { + const writer = Bun.file(target).writer(); + const reader = body.getReader(); + let total = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) { + await reader.cancel(); + throw new Error( + `Direct download too large (${total} bytes; max ${maxBytes})`, + ); + } + writer.write(value); + } + await writer.end(); + } catch (error) { + try { + writer.end(); + } catch { + // ignore + } + await unlink(target).catch(() => {}); + throw error; + } +} + +export class DirectDownloader { + private progressCallback: ProgressCallback | null = null; + + setProgressCallback(callback: ProgressCallback): void { + this.progressCallback = callback; + } + + async downloadAudio(url: string): Promise { + const downloadUrl = normalizeDirectDownloadUrl(url); + console.log(`Downloading direct file: ${downloadUrl}`); + this.progressCallback?.('downloading', 'Downloading direct file...', 40, { + browserless: true, + }); + + await mkdir('./downloads', { recursive: true }); + + let current = downloadUrl; + let response: Response | null = null; + let finalUrl = current; + + for (let hop = 0; hop < MAX_REDIRECTS; hop++) { + const result = await safeFetch(current, { + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + headers: { + 'user-agent': USER_AGENT, + accept: '*/*', + }, + }); + finalUrl = result.url; + response = result.response; + + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get('location'); + await response.body?.cancel().catch(() => {}); + if (!location) { + throw new Error( + `Direct download redirect missing Location from ${current}`, + ); + } + current = new URL(location, current).toString(); + continue; + } + break; + } + + if (!response) { + throw new Error(`Direct download failed for ${downloadUrl}`); + } + if (response.status >= 300 && response.status < 400) { + throw new Error( + `Direct download exceeded ${MAX_REDIRECTS} redirects for ${downloadUrl}`, + ); + } + if (!response.ok) { + throw new Error( + `Direct download failed: HTTP ${response.status} for ${finalUrl}`, + ); + } + + const contentType = response.headers.get('content-type') ?? ''; + if (/text\/html/i.test(contentType)) { + await response.body?.cancel().catch(() => {}); + throw new Error( + 'Direct download returned HTML instead of a file β€” check that the link is a direct download (e.g. Dropbox with dl=1).', + ); + } + + const contentLengthHeader = response.headers.get('content-length'); + if (contentLengthHeader) { + const contentLength = Number(contentLengthHeader); + if ( + Number.isFinite(contentLength) && + contentLength > MAX_DOWNLOAD_BYTES + ) { + await response.body?.cancel().catch(() => {}); + throw new Error( + `Direct download too large (${contentLength} bytes; max ${MAX_DOWNLOAD_BYTES})`, + ); + } + } + + const fromHeader = safeDownloadFilename( + filenameFromContentDisposition( + response.headers.get('content-disposition'), + ), + ); + const urlName = safeDownloadFilename( + decodeURIComponent(basename(new URL(finalUrl).pathname)), + ); + const filename = + fromHeader || + (urlName && urlName !== '/' + ? urlName + : `direct-download-${Date.now()}.bin`); + + if (!response.body) { + throw new Error(`Direct download returned an empty body for ${finalUrl}`); + } + + const target = join('./downloads', filename); + await writeBodyWithSizeLimit(response.body, target, MAX_DOWNLOAD_BYTES); + console.log(`Saved ${filename}`); + return filename; + } + + async close(): Promise { + // no-op (browserless) + } +} diff --git a/src/directLinkRules.ts b/src/directLinkRules.ts new file mode 100644 index 0000000..f7ef56e --- /dev/null +++ b/src/directLinkRules.ts @@ -0,0 +1,82 @@ +/** + * Shared direct-download URL rules (no imports from utils / downloaders) so + * matchers and the downloader cannot drift apart. + */ + +export const AUDIO_OR_ARCHIVE_EXT_RE = + /\.(mp3|wav|flac|aiff|aif|m4a|aac|ogg|opus|zip|rar)(\?|$)/i; + +/** Google Drive file IDs are long URL-safe tokens. */ +const DRIVE_FILE_ID_RE = /^[a-zA-Z0-9_-]{10,}$/; + +export function isKnownDirectDownloadHost(hostname: string): boolean { + const host = hostname.toLowerCase(); + return ( + /(?:^|\.)dropbox\.com$/i.test(host) || + /(?:^|\.)dropboxusercontent\.com$/i.test(host) || + /(?:^|\.)drive\.google\.com$/i.test(host) + ); +} + +/** Extract a validated Drive file id from path (`/file/d/ID`) or `id` query. */ +export function extractDriveFileId(url: URL): string | null { + const fromPath = url.pathname.match(/\/file\/d\/([^/]+)/)?.[1]; + if (fromPath && DRIVE_FILE_ID_RE.test(fromPath)) return fromPath; + const fromQuery = url.searchParams.get('id'); + if (fromQuery && DRIVE_FILE_ID_RE.test(fromQuery)) return fromQuery; + return null; +} + +/** True when a parsed URL looks like a direct file link, not an HTML gate page. */ +export function urlLooksLikeDirectDownload(url: URL): boolean { + if (!/^https?:$/i.test(url.protocol)) return false; + const host = url.hostname.toLowerCase(); + if ( + /(?:^|\.)dropbox\.com$/i.test(host) || + /(?:^|\.)dropboxusercontent\.com$/i.test(host) + ) { + return true; + } + if (/(?:^|\.)drive\.google\.com$/i.test(host)) { + return extractDriveFileId(url) !== null; + } + if (AUDIO_OR_ARCHIVE_EXT_RE.test(url.pathname)) return true; + return false; +} + +/** + * Normalize share links so fetch gets the file bytes. + * Dropbox: force `dl=1` (rewrites `dl=0` preview links and adds `dl` when missing). + * Drive: only rewrite when a valid file id is present. + */ +export function normalizeDirectDownloadParsedUrl(url: URL): string { + if (/(?:^|\.)dropbox\.com$/i.test(url.hostname)) { + const parsed = new URL(url.toString()); + parsed.searchParams.set('dl', '1'); + return parsed.toString(); + } + if (/(?:^|\.)drive\.google\.com$/i.test(url.hostname)) { + const id = extractDriveFileId(url); + if (!id) { + throw new Error( + `Malformed Google Drive link (missing valid file id): ${url}`, + ); + } + const download = new URL('https://drive.google.com/uc'); + download.searchParams.set('export', 'download'); + download.searchParams.set('id', id); + // Link-shared files may require resourcekey alongside the file id. + let resourceKey: string | null = null; + for (const [key, value] of url.searchParams) { + if (key.toLowerCase() === 'resourcekey' && value) { + resourceKey = value; + break; + } + } + if (resourceKey) { + download.searchParams.set('resourcekey', resourceKey); + } + return download.toString(); + } + return url.toString(); +} diff --git a/src/index.ts b/src/index.ts index f2c8221..2a7999e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,7 @@ import { confirm, input, select } from '@inquirer/prompts'; import { AudioProcessor } from './audioProcessor'; import { loadConfig, saveConfig } from './config'; +import { DirectDownloader } from './directDownload'; import { DownloadgaterDownloader } from './downloadgater'; import { DroploudDownloader } from './droploud'; import { GaterushDownloader } from './gaterush'; @@ -10,7 +11,7 @@ import { SoundcloudClient } from './soundcloud'; import { getFfmpegBin, getFfprobeBin, - resolveGateProviderUrl, + resolveGateUrlOrFollow, validateSoundcloudUrl, } from './utils'; import { YtDlpDownloader } from './ytdlp'; @@ -65,7 +66,7 @@ try { value: 'ytdlp' as const, }, { - name: 'Enter a Hypeddit / Droploud / GateRush / DownloadGater / Bandcamp URL', + name: 'Enter a gate / Bandcamp / direct download / resolvable URL', value: 'manual' as const, }, ], @@ -77,16 +78,16 @@ try { } else { gateUrl = await input({ message: - 'Enter the Hypeddit, Droploud, GateRush, DownloadGater, or Bandcamp URL', - validate: (value) => { - const resolved = resolveGateProviderUrl(value); + 'Enter a Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, direct download, or smart-link URL', + validate: async (value) => { + const resolved = await resolveGateUrlOrFollow(value); if (!resolved || resolved.provider === 'soundcloud') { - return 'A valid Hypeddit, Droploud, GateRush, DownloadGater, or Bandcamp URL is required'; + return 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, direct download, or resolvable gate URL is required'; } return true; }, }); - const resolved = resolveGateProviderUrl(gateUrl); + const resolved = await resolveGateUrlOrFollow(gateUrl); gate = resolved && resolved.provider !== 'soundcloud' ? { @@ -103,14 +104,16 @@ try { if (!gateUrl || !gate) { throw new Error( - 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, or SoundCloud URL is required', + 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, direct download, or SoundCloud URL is required', ); } - const isYtDlp = - gate.provider === 'bandcamp' || gate.provider === 'soundcloud'; + const isBrowserless = + gate.provider === 'bandcamp' || + gate.provider === 'soundcloud' || + gate.provider === 'direct'; - const headless = isYtDlp + const headless = isBrowserless ? true : config ? config.headless @@ -119,7 +122,7 @@ try { default: true, }); - const initializeLogins = isYtDlp + const initializeLogins = isBrowserless ? false : config ? config.initializeLogins @@ -145,7 +148,25 @@ try { const ytDlpDownloader = new YtDlpDownloader(sourceLabel); downloadFilename = await ytDlpDownloader.downloadAudio(gateUrl, { matchTitle: track.title, + onAlbumMatchFailed: async (error) => { + const selected = await select({ + message: error.matchTitle + ? `Could not match β€œ${error.matchTitle}”. Pick a Bandcamp album track:` + : 'Pick a track from the Bandcamp album:', + choices: error.tracks.map((t) => ({ + name: + t.score > 0 + ? `${t.title} (${Math.round(t.score * 100)}% match)` + : t.title, + value: t.url, + })), + }); + return selected; + }, }); + } else if (gate.provider === 'direct') { + const directDownloader = new DirectDownloader(); + downloadFilename = await directDownloader.downloadAudio(gateUrl); } else if (gate.provider === 'droploud') { usedBrowser = true; const droploudDownloader = new DroploudDownloader(gateConfig); diff --git a/src/jobStore.test.ts b/src/jobStore.test.ts new file mode 100644 index 0000000..990599f --- /dev/null +++ b/src/jobStore.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from 'bun:test'; +import { jobStore } from './jobStore'; + +describe('jobStore Bandcamp track selection', () => { + test('resolveBandcampTrackSelection delivers URL to waiter', async () => { + const job = jobStore.create('https://soundcloud.com/a/b', 'mp3'); + try { + const wait = jobStore.waitForBandcampTrackSelection(job.id); + expect( + jobStore.resolveBandcampTrackSelection( + job.id, + 'https://x.bandcamp.com/track/y', + ), + ).toBe(true); + expect(await wait).toBe('https://x.bandcamp.com/track/y'); + } finally { + jobStore.delete(job.id); + } + }); + + test('cancel resolves pending wait with null', async () => { + const job = jobStore.create('https://soundcloud.com/a/b', 'mp3'); + try { + const wait = jobStore.waitForBandcampTrackSelection(job.id); + jobStore.cancel(job.id); + expect(await wait).toBeNull(); + } finally { + jobStore.delete(job.id); + } + }); + + test('delete resolves pending wait with null', async () => { + const job = jobStore.create('https://soundcloud.com/a/b', 'mp3'); + const wait = jobStore.waitForBandcampTrackSelection(job.id); + jobStore.delete(job.id); + expect(await wait).toBeNull(); + }); + + test('second wait supersedes first with null', async () => { + const job = jobStore.create('https://soundcloud.com/a/b', 'mp3'); + try { + const first = jobStore.waitForBandcampTrackSelection(job.id); + const second = jobStore.waitForBandcampTrackSelection(job.id); + expect(await first).toBeNull(); + expect( + jobStore.resolveBandcampTrackSelection( + job.id, + 'https://x.bandcamp.com/track/z', + ), + ).toBe(true); + expect(await second).toBe('https://x.bandcamp.com/track/z'); + } finally { + jobStore.delete(job.id); + } + }); +}); diff --git a/src/jobStore.ts b/src/jobStore.ts index a1a97f9..3284554 100644 --- a/src/jobStore.ts +++ b/src/jobStore.ts @@ -1,6 +1,7 @@ import type { Job, JobProgress, JobStage, OutputFormat } from './types'; type ProgressListener = (progress: JobProgress) => void; +type BandcampTrackResolver = (url: string | null) => void; /** * In-memory job store for single-user Web UI @@ -8,6 +9,9 @@ type ProgressListener = (progress: JobProgress) => void; class JobStore { private jobs: Map = new Map(); private listeners: Map> = new Map(); + /** Pending Bandcamp album-track picks (mid-download pause). */ + private bandcampTrackResolvers: Map = + new Map(); /** * Creates a new job and returns its ID @@ -19,6 +23,7 @@ class JobStore { id, soundcloudUrl, hypedditUrl: null, + bandcampAlbumTracks: null, headless: process.env.BROWSER_HEADLESS !== 'false', outputFormat, cancelled: false, @@ -138,16 +143,43 @@ class JobStore { job.cancelled = true; job.error = null; + job.bandcampAlbumTracks = null; job.progress = { stage: 'cancelled', message, percent: job.progress.percent, }; job.updatedAt = new Date(); + this.resolveBandcampTrackSelection(id, null); this.notifyListeners(id, job.progress); return job; } + /** + * Pause until the user picks a Bandcamp album track (or cancel β†’ null). + */ + waitForBandcampTrackSelection(id: string): Promise { + const existing = this.bandcampTrackResolvers.get(id); + if (existing) { + existing(null); + this.bandcampTrackResolvers.delete(id); + } + return new Promise((resolve) => { + this.bandcampTrackResolvers.set(id, resolve); + }); + } + + /** + * Resolve a pending Bandcamp album-track wait with a track URL, or null. + */ + resolveBandcampTrackSelection(id: string, url: string | null): boolean { + const resolve = this.bandcampTrackResolvers.get(id); + if (!resolve) return false; + this.bandcampTrackResolvers.delete(id); + resolve(url); + return true; + } + /** * Clear cancelled flag when restarting a job (e.g. after error). */ @@ -193,6 +225,7 @@ class JobStore { * Delete a job */ delete(id: string): boolean { + this.resolveBandcampTrackSelection(id, null); this.listeners.delete(id); return this.jobs.delete(id); } diff --git a/src/safeOutboundUrl.ts b/src/safeOutboundUrl.ts new file mode 100644 index 0000000..3a52587 --- /dev/null +++ b/src/safeOutboundUrl.ts @@ -0,0 +1,195 @@ +import { lookup } from 'node:dns/promises'; +import http from 'node:http'; +import https from 'node:https'; +import { isIP } from 'node:net'; + +function isPrivateOrLocalIp(ip: string): boolean { + const normalized = ip.toLowerCase().replace(/^\[|\]$/g, ''); + + if (isIP(normalized) === 4) { + const parts = normalized.split('.').map(Number); + const a = parts[0] ?? 0; + const b = parts[1] ?? 0; + if (a === 0 || a === 10 || a === 127) return true; + if (a === 169 && b === 254) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT + return false; + } + + if (isIP(normalized) === 6) { + if (normalized === '::1' || normalized === '::') return true; + if (normalized.startsWith('fe80:')) return true; // link-local + if (normalized.startsWith('fc') || normalized.startsWith('fd')) return true; // ULA + const v4mapped = normalized.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (v4mapped?.[1]) return isPrivateOrLocalIp(v4mapped[1]); + return false; + } + + return true; +} + +function isBlockedHostname(hostname: string): boolean { + const host = hostname + .toLowerCase() + .replace(/^\[|\]$/g, '') + .replace(/\.$/, ''); + if ( + host === 'localhost' || + host === '0.0.0.0' || + host === '::1' || + host === 'metadata.google.internal' || + host.endsWith('.localhost') || + host.endsWith('.local') || + host.endsWith('.internal') + ) { + return true; + } + return false; +} + +export type SafeConnectTarget = { + url: URL; + /** Validated public address used for the TCP connection. */ + address: string; +}; + +/** + * Resolve a URL to a public connect address. Rejects private/local hosts and + * DNS answers that only resolve to private/local IPs. + */ +export async function resolveSafeConnectTarget( + urlString: string, +): Promise { + let url: URL; + try { + url = new URL(urlString); + } catch { + throw new Error(`Invalid URL: ${urlString}`); + } + + if (!/^https?:$/i.test(url.protocol)) { + throw new Error(`Refusing non-http(s) URL: ${urlString}`); + } + + // URL.hostname is usually unbracketed for IPv6, but strip defensively so + // isIP / blocklist / connect address never see "[::1]". + const host = url.hostname.replace(/^\[|\]$/g, ''); + if (isBlockedHostname(host)) { + throw new Error(`Refusing to fetch non-public host: ${host}`); + } + + if (isIP(host) !== 0) { + if (isPrivateOrLocalIp(host)) { + throw new Error(`Refusing to fetch private/local address: ${host}`); + } + return { url, address: host }; + } + + const results = await lookup(host, { all: true }); + const publicAddrs = results.filter((r) => !isPrivateOrLocalIp(r.address)); + const chosen = publicAddrs[0]; + if (!chosen) { + const sample = results[0]?.address ?? 'none'; + throw new Error( + `Refusing to fetch host ${host} (resolves to private/local address ${sample})`, + ); + } + return { url, address: chosen.address }; +} + +/** @deprecated Prefer resolveSafeConnectTarget / safeFetch */ +export async function assertSafeOutboundUrl(urlString: string): Promise { + await resolveSafeConnectTarget(urlString); +} + +export type SafeFetchInit = { + method?: string; + headers?: Record; + signal?: AbortSignal; +}; + +export type SafeFetchResult = { + response: Response; + /** Request URL (redirects are not followed). */ + url: string; +}; + +/** + * Fetch via a validated public IP while keeping Host + TLS SNI as the original + * hostname, so DNS TOCTOU cannot redirect the TCP connection to a private IP. + * Redirects are never followed β€” callers must validate each Location hop. + */ +export async function safeFetch( + urlString: string, + init: SafeFetchInit = {}, +): Promise { + const { url, address } = await resolveSafeConnectTarget(urlString); + const isHttps = url.protocol === 'https:'; + const transport = isHttps ? https : http; + const port = url.port ? Number(url.port) : isHttps ? 443 : 80; + const path = `${url.pathname}${url.search}`; + const headers: Record = { + ...(init.headers ?? {}), + host: url.host, + }; + + return await new Promise((resolve, reject) => { + const req = transport.request( + { + hostname: address, + port, + path, + method: init.method ?? 'GET', + headers, + servername: isHttps ? url.hostname : undefined, + signal: init.signal, + }, + (res) => { + const responseHeaders = new Headers(); + for (const [key, value] of Object.entries(res.headers)) { + if (value === undefined) continue; + if (Array.isArray(value)) { + for (const item of value) responseHeaders.append(key, item); + } else { + responseHeaders.set(key, value); + } + } + + const body = new ReadableStream({ + start(controller) { + res.on('data', (chunk: Buffer | string) => { + const bytes = + typeof chunk === 'string' ? Buffer.from(chunk) : chunk; + controller.enqueue(new Uint8Array(bytes)); + }); + res.on('end', () => { + try { + controller.close(); + } catch { + // already closed + } + }); + res.on('error', (err) => controller.error(err)); + }, + cancel() { + res.destroy(); + }, + }); + + resolve({ + url: url.toString(), + response: new Response(body, { + status: res.statusCode ?? 0, + statusText: res.statusMessage, + headers: responseHeaders, + }), + }); + }, + ); + + req.on('error', reject); + req.end(); + }); +} diff --git a/src/server.ts b/src/server.ts index 3c1b243..2a2bb50 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,6 +3,7 @@ import { cp, mkdir, rm } from 'node:fs/promises'; import { basename, extname, join } from 'node:path'; import type { SoundcloudTrack } from 'soundcloud.ts'; import { AudioProcessor } from './audioProcessor'; +import { DirectDownloader } from './directDownload'; import { DownloadgaterDownloader } from './downloadgater'; import { renameDownloadFileExclusive } from './downloadRename'; import { DroploudDownloader } from './droploud'; @@ -14,12 +15,12 @@ import { SoundcloudClient } from './soundcloud'; import type { Job, Metadata, OutputFormat } from './types'; import { artistTitleFilename, - extractGateUrl, + extractAndResolveGateUrl, getDefaultMetadata, getFfmpegBin, getFfprobeBin, isMp3Format, - resolveGateProviderUrl, + resolveGateUrlOrFollow, validateGateUrl, validateSoundcloudUrl, } from './utils'; @@ -180,12 +181,14 @@ async function runDownloadProcess(jobId: string): Promise { const job = jobStore.get(jobId); if (!job?.hypedditUrl) return; - const gateUrl = job.hypedditUrl; - const resolved = resolveGateProviderUrl(gateUrl); + const resolved = await resolveGateUrlOrFollow(job.hypedditUrl); if (!resolved) { jobStore.setError(jobId, 'Unsupported gate URL'); return; } + if (resolved.url !== job.hypedditUrl) { + jobStore.update(jobId, { hypedditUrl: resolved.url }); + } const { url: downloadSourceUrl, provider } = resolved; const throwIfCancelled = () => { @@ -242,6 +245,41 @@ async function runDownloadProcess(jobId: string): Promise { downloadSourceUrl, { matchTitle: job.track?.title, + onAlbumMatchFailed: async (error) => { + throwIfCancelled(); + jobStore.update(jobId, { + bandcampAlbumTracks: error.tracks, + error: null, + }); + const message = error.matchTitle + ? `Could not match β€œ${error.matchTitle}” β€” pick a Bandcamp track` + : 'Pick a track from the Bandcamp album'; + jobStore.updateProgress( + jobId, + 'waiting_bandcamp_track', + message, + 45, + { + browserless: true, + bandcampAlbumTracks: error.tracks, + }, + ); + const selectedUrl = + await jobStore.waitForBandcampTrackSelection(jobId); + throwIfCancelled(); + if (!selectedUrl) { + throw new Error('Download cancelled'); + } + jobStore.update(jobId, { bandcampAlbumTracks: null }); + jobStore.updateProgress( + jobId, + 'downloading', + `Downloading from ${sourceLabel} via yt-dlp...`, + 50, + { browserless: true }, + ); + return selectedUrl; + }, }, ); throwIfCancelled(); @@ -314,6 +352,20 @@ async function runDownloadProcess(jobId: string): Promise { downloadFilename = await downloadgaterDownloader.downloadAudio(downloadSourceUrl); throwIfCancelled(); + } else if (provider === 'direct') { + jobStore.updateProgress( + jobId, + 'downloading', + 'Downloading direct file...', + 40, + { browserless: true }, + ); + const directDownloader = new DirectDownloader(); + activeDownloaders.set(jobId, directDownloader); + directDownloader.setProgressCallback(emitProgress); + downloadFilename = + await directDownloader.downloadAudio(downloadSourceUrl); + throwIfCancelled(); } else { // Hypeddit: always try plain HTTP first (email + social skip gates). jobStore.updateProgress( @@ -647,7 +699,7 @@ const server = Bun.serve({ const hypedditUrl = skipAutomaticHypedditFetch ? null - : extractGateUrl(track); + : await extractAndResolveGateUrl(track); const defaultMetadata = getDefaultMetadata(track); const updatedJob = jobStore.update(job.id, { @@ -716,7 +768,7 @@ const server = Bun.serve({ return jsonResponse({ error: validation }, { status: 400 }); } - const resolved = resolveGateProviderUrl(hypedditUrl); + const resolved = await resolveGateUrlOrFollow(hypedditUrl); if (!resolved) { return jsonResponse( { error: 'Could not extract a supported URL' }, @@ -738,6 +790,69 @@ const server = Bun.serve({ }, }, + '/api/job/:id/bandcamp-track': { + POST: async (req) => { + try { + const jobId = req.params.id; + const job = jobStore.get(jobId); + + if (!job) { + return jsonResponse({ error: 'Job not found' }, { status: 404 }); + } + + if (job.progress.stage !== 'waiting_bandcamp_track') { + return jsonResponse( + { error: 'Job is not waiting for a Bandcamp track selection' }, + { status: 400 }, + ); + } + + const body = await req.json(); + const { trackUrl } = body as { trackUrl?: string }; + + if (!trackUrl || typeof trackUrl !== 'string') { + return jsonResponse( + { error: 'trackUrl is required' }, + { status: 400 }, + ); + } + + const allowed = job.bandcampAlbumTracks ?? []; + const match = allowed.find((t) => t.url === trackUrl); + if (!match) { + return jsonResponse( + { error: 'trackUrl is not one of the listed album tracks' }, + { status: 400 }, + ); + } + + const resolved = jobStore.resolveBandcampTrackSelection( + jobId, + match.url, + ); + if (!resolved) { + return jsonResponse( + { error: 'No pending Bandcamp track selection' }, + { status: 409 }, + ); + } + + return jsonResponse({ + success: true, + trackUrl: match.url, + title: match.title, + }); + } catch (error) { + return jsonResponse( + { + error: error instanceof Error ? error.message : 'Unknown error', + }, + { status: 500 }, + ); + } + }, + }, + '/api/job/:id/start': { POST: async (req) => { try { @@ -933,6 +1048,7 @@ const server = Bun.serve({ id: job.id, soundcloudUrl: job.soundcloudUrl, hypedditUrl: job.hypedditUrl, + bandcampAlbumTracks: job.bandcampAlbumTracks, track: job.track, defaultMetadata: job.defaultMetadata, existingMetadata: job.existingMetadata, diff --git a/src/soundcloud.ts b/src/soundcloud.ts index 8ba6ec9..942c795 100644 --- a/src/soundcloud.ts +++ b/src/soundcloud.ts @@ -8,11 +8,22 @@ import Soundcloud, { type SoundcloudUser, } from 'soundcloud.ts'; import { + extractAndResolveGateUrl, extractCaptchaDeliveryUrl, - extractGateUrl, + type GateProvider, loadCookies, } from './utils'; +const GATE_PROVIDER_LABELS: Record = { + droploud: 'Droploud', + gaterush: 'GateRush', + downloadgater: 'DownloadGater', + direct: 'direct download', + bandcamp: 'Bandcamp', + soundcloud: 'SoundCloud', + hypeddit: 'Hypeddit', +}; + interface SoundcloudCredentials { clientId: string; oauthToken: string; @@ -518,22 +529,11 @@ export class SoundcloudClient { } async getGateURL(track: SoundcloudTrack) { - const gate = extractGateUrl(track); + const gate = await extractAndResolveGateUrl(track); if (!gate) { return null; } - const providerLabel = - gate.provider === 'droploud' - ? 'Droploud' - : gate.provider === 'gaterush' - ? 'GateRush' - : gate.provider === 'downloadgater' - ? 'DownloadGater' - : gate.provider === 'bandcamp' - ? 'Bandcamp' - : gate.provider === 'soundcloud' - ? 'SoundCloud' - : 'Hypeddit'; + const providerLabel = GATE_PROVIDER_LABELS[gate.provider]; const sourceLabel = gate.type === 'purchase_url' ? 'purchase URL' : 'description'; console.log( diff --git a/src/types.ts b/src/types.ts index c14629d..4089491 100644 --- a/src/types.ts +++ b/src/types.ts @@ -34,6 +34,7 @@ export type JobStage = | 'pending' | 'fetching_track' | 'waiting_hypeddit' + | 'waiting_bandcamp_track' | 'initializing_browser' | 'preparing_logins' | 'handling_gates' @@ -43,6 +44,14 @@ export type JobStage = | 'error' | 'cancelled'; +/** Candidate track on a Bandcamp album when auto title-match fails. */ +export interface BandcampAlbumTrackChoice { + title: string; + url: string; + /** 0–1 fuzzy score against the SoundCloud title (0 when no title to match). */ + score: number; +} + export interface JobProgress { stage: JobStage; message: string; @@ -53,12 +62,16 @@ export interface JobProgress { // True when the download was handled without a browser. Such downloads never // touch the SoundCloud account, so the UI can skip the cleanup prompt. browserless?: boolean; + /** Present while stage is `waiting_bandcamp_track`. */ + bandcampAlbumTracks?: BandcampAlbumTrackChoice[]; } export interface Job { id: string; soundcloudUrl: string; hypedditUrl: string | null; + /** Candidates when Bandcamp album auto-match fails (cleared after pick). */ + bandcampAlbumTracks: BandcampAlbumTrackChoice[] | null; /** Whether browser automation runs headless. Defaults to true. */ headless: boolean; outputFormat: OutputFormat; diff --git a/src/utils.test.ts b/src/utils.test.ts index 21e0403..7f08d1b 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { artistTitleFilename, cookiesToNetscape, + findKnownGateInHtml, previewProcessedFilename, resolveGateProviderUrl, sanitizeFilenamePart, @@ -51,8 +52,111 @@ describe('resolveGateProviderUrl', () => { provider: 'hypeddit', }); }); + + test('matches Dropbox direct download URLs and forces dl=1', () => { + expect( + resolveGateProviderUrl( + 'https://www.dropbox.com/scl/fi/abc/track.wav?rlkey=xyz&e=1&dl=0', + ), + ).toEqual({ + url: 'https://www.dropbox.com/scl/fi/abc/track.wav?rlkey=xyz&e=1&dl=1', + provider: 'direct', + }); + }); + + test('rewrites Dropbox dl=0 to dl=1 when dl is the only query param', () => { + expect( + resolveGateProviderUrl('https://www.dropbox.com/s/abc123/track.wav?dl=0'), + ).toEqual({ + url: 'https://www.dropbox.com/s/abc123/track.wav?dl=1', + provider: 'direct', + }); + }); + + test('adds dl=1 to Dropbox share links that omit dl', () => { + expect( + resolveGateProviderUrl( + 'https://www.dropbox.com/scl/fi/abc/track.wav?rlkey=xyz', + ), + ).toEqual({ + url: 'https://www.dropbox.com/scl/fi/abc/track.wav?rlkey=xyz&dl=1', + provider: 'direct', + }); + }); + + test('normalizes valid Google Drive file links', () => { + expect( + resolveGateProviderUrl( + 'https://drive.google.com/file/d/abcdefghijklmnopqrstuvwx/view?usp=sharing', + ), + ).toEqual({ + url: 'https://drive.google.com/uc?export=download&id=abcdefghijklmnopqrstuvwx', + provider: 'direct', + }); + }); + + test('preserves Google Drive resourcekey on normalized download URLs', () => { + expect( + resolveGateProviderUrl( + 'https://drive.google.com/file/d/abcdefghijklmnopqrstuvwx/view?usp=sharing&resourcekey=abc-123', + ), + ).toEqual({ + url: 'https://drive.google.com/uc?export=download&id=abcdefghijklmnopqrstuvwx&resourcekey=abc-123', + provider: 'direct', + }); + }); + + test('does not treat malformed Google Drive URLs as direct downloads', () => { + expect( + resolveGateProviderUrl('https://drive.google.com/drive/my-drive'), + ).toBeNull(); + expect( + resolveGateProviderUrl('https://drive.google.com/file/d/short/view'), + ).toBeNull(); + }); + + test('matches raw audio file URLs as direct downloads', () => { + expect( + resolveGateProviderUrl( + 'FREE DL: https://cdn.example.com/files/track.wav!', + ), + ).toEqual({ + url: 'https://cdn.example.com/files/track.wav', + provider: 'direct', + }); + }); }); +describe('findKnownGateInHtml', () => { + test('finds embedded Hypeddit destination in smart-link HTML', () => { + const html = ` + + `; + expect(findKnownGateInHtml(html)).toEqual({ + url: 'https://hypeddit.com/dorey/strobedoreyedit', + provider: 'hypeddit', + }); + }); + + test('follows meta refresh to a known gate', () => { + const html = + ''; + expect(findKnownGateInHtml(html)).toEqual({ + url: 'https://hypeddit.com/artist/track', + provider: 'hypeddit', + }); + }); + + test('prefers Hypeddit when Bandcamp is also present', () => { + const html = 'https://hypeddit.com/a/b and https://x.bandcamp.com/track/y'; + expect(findKnownGateInHtml(html)).toEqual({ + url: 'https://hypeddit.com/a/b', + provider: 'hypeddit', + }); + }); +}); describe('writeSoundcloudNetscapeCookies', () => { test('returns null for malformed JSON', async () => { const dir = await mkdtemp(join(tmpdir(), 'sc-gate-dl-test-')); diff --git a/src/utils.ts b/src/utils.ts index 33a79f8..181d4b2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -5,6 +5,11 @@ import { lookpath } from 'find-bin'; import type { CookieData } from 'puppeteer'; import type { SoundcloudTrack } from 'soundcloud.ts'; import packageJson from '../package.json' with { type: 'json' }; +import { + normalizeDirectDownloadParsedUrl, + urlLooksLikeDirectDownload, +} from './directLinkRules'; +import { safeFetch } from './safeOutboundUrl'; import type { LocalCookieData, Metadata } from './types'; export const REPO_URL = packageJson.repository.url; @@ -219,6 +224,8 @@ export type GateProvider = | 'droploud' | 'gaterush' | 'downloadgater' + /** Direct HTTP(S) file URL (Dropbox, Drive, raw audio link, …). */ + | 'direct' | 'bandcamp' /** Direct track download via yt-dlp (manual fallback, not auto-extracted). */ | 'soundcloud'; @@ -240,10 +247,16 @@ const BANDCAMP_URL_RE = const BANDCAMP_ALBUM_URL_RE = /https?:\/\/(?:[\w-]+\.)?bandcamp\.com\/album\/[^\s?#]+/i; const SOUNDCLOUD_URL_RE = /https:\/\/soundcloud\.com\/[^\s?#]+/i; +const ANY_HTTP_URL_RE = /https?:\/\/[^\s<>"')\]]+/gi; -/** Strip prose/Markdown delimiters glued to the end of a matched URL. */ -function trimExtractedUrl(url: string): string { - return url.replace(/[)\]}>.,;:!?'"…]+$/g, ''); +const GATE_RESOLVE_USER_AGENT = + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'; + +/** Strip prose/Markdown/JSON delimiters glued to the end of a matched URL. */ +export function trimExtractedUrl(url: string): string { + const cut = url.search(/["'<>\\]/); + const base = cut >= 0 ? url.slice(0, cut) : url; + return base.replace(/[)\]}>.,;:!?'"…]+$/g, ''); } export function isHypedditUrl(value: string): boolean { @@ -272,7 +285,8 @@ export function isBandcampAlbumUrl(value: string): boolean { /** * Extract the canonical provider URL from a string (possibly with surrounding - * text). Prefer traditional gates, then Bandcamp, then SoundCloud. + * text). Prefer traditional gates, then Bandcamp, then direct file links, then + * SoundCloud. */ export function resolveGateProviderUrl( value: string, @@ -283,6 +297,9 @@ export function resolveGateProviderUrl( const bandcamp = matchBandcampUrl(value); if (bandcamp) return bandcamp; + const direct = matchDirectDownloadUrl(value); + if (direct) return direct; + const soundcloudMatch = value.match(SOUNDCLOUD_URL_RE)?.[0]; if (soundcloudMatch) { return { @@ -305,10 +322,14 @@ export function validateHypedditUrl(value: string): true | string { } export function validateGateUrl(value: string): true | string { - if (!resolveGateProviderUrl(value)) { - return 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, or SoundCloud URL is required'; + if (resolveGateProviderUrl(value)) { + return true; } - return true; + // Unknown http(s) URLs may still resolve via redirects / embedded destinations. + if (/^https?:\/\/\S+/i.test(value.trim())) { + return true; + } + return 'A valid Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, direct download, SoundCloud, or resolvable http(s) URL is required'; } function normalizeGateUrl( @@ -318,7 +339,8 @@ function normalizeGateUrl( if ( provider === 'gaterush' || provider === 'downloadgater' || - provider === 'bandcamp' + provider === 'bandcamp' || + provider === 'direct' ) { return { url: url.replace(/^http:\/\//i, 'https://'), @@ -328,6 +350,17 @@ function normalizeGateUrl( return { url, provider }; } +/** Known download gates / Bandcamp / direct files (excludes SoundCloud). */ +function matchKnownDownloadGateUrl( + value: string, +): { url: string; provider: GateProvider } | null { + const traditional = matchTraditionalGateUrl(value); + if (traditional) return traditional; + const bandcamp = matchBandcampUrl(value); + if (bandcamp) return bandcamp; + return matchDirectDownloadUrl(value); +} + /** Traditional unlock gates (browser / HTTP). Prefer these over Bandcamp. */ function matchTraditionalGateUrl( value: string, @@ -364,9 +397,145 @@ function matchBandcampUrl( return null; } +function matchDirectDownloadUrl( + value: string, +): { url: string; provider: GateProvider } | null { + const httpMatch = value.match(/https?:\/\/[^\s<>"')\]]+/i)?.[0]; + if (!httpMatch) return null; + const trimmed = trimExtractedUrl(httpMatch); + try { + const url = new URL(trimmed.replace(/^http:\/\//i, 'https://')); + if (!urlLooksLikeDirectDownload(url)) return null; + return { + url: normalizeDirectDownloadParsedUrl(url), + provider: 'direct', + }; + } catch { + return null; + } +} + +/** + * Scan HTML (or any text blob) for a known gate / Bandcamp URL, including + * meta-refresh targets. Used for smart-link pages that embed destinations. + */ +export function findKnownGateInHtml( + html: string, +): { url: string; provider: GateProvider } | null { + const normalized = html.replace(/\\\//g, '/'); + const direct = matchKnownDownloadGateUrl(normalized); + if (direct) return direct; + + const refresh = + normalized.match( + /http-equiv=["']refresh["'][^>]*content=["'][^"']*url=([^"'>\s]+)/i, + ) ?? + normalized.match( + /content=["'][^"']*url=([^"'>\s]+)[^"']*["'][^>]*http-equiv=["']refresh["']/i, + ); + if (refresh?.[1]) { + const target = refresh[1].replace(/&/g, '&'); + return matchKnownDownloadGateUrl(target); + } + return null; +} + +/** + * Resolve an unrecognized (or already-known) URL to a download gate by + * following HTTP redirects and scanning the final HTML for embedded destinations. + * Each hop is validated against private/local destinations before fetching. + */ +export async function resolveUnknownGateUrl( + url: string, +): Promise<{ url: string; provider: GateProvider } | null> { + const trimmed = trimExtractedUrl(url.trim()); + if (!/^https?:\/\//i.test(trimmed)) { + return null; + } + + const knownUpFront = matchKnownDownloadGateUrl(trimmed); + if (knownUpFront) return knownUpFront; + + // SoundCloud pages are not smart-link hops we want to chase. + if (isSoundcloudUrl(trimmed)) { + return null; + } + + const maxHops = 10; + let current = trimmed; + + try { + for (let hop = 0; hop < maxHops; hop++) { + const known = matchKnownDownloadGateUrl(current); + if (known) return known; + if (isSoundcloudUrl(current)) return null; + + const { response } = await safeFetch(current, { + signal: AbortSignal.timeout(15_000), + headers: { + 'user-agent': GATE_RESOLVE_USER_AGENT, + accept: + 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8', + }, + }); + + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get('location'); + await response.body?.cancel().catch(() => {}); + if (!location) return null; + current = trimExtractedUrl(new URL(location, current).toString()); + continue; + } + + const finalUrl = trimExtractedUrl(current); + const fromFinal = matchKnownDownloadGateUrl(finalUrl); + if (fromFinal) { + await response.body?.cancel().catch(() => {}); + return fromFinal; + } + + const html = await response.text(); + return findKnownGateInHtml(html); + } + } catch { + return null; + } + + return null; +} + +function collectUnresolvedHttpCandidates( + track: SoundcloudTrack, +): { url: string; type: 'purchase_url' | 'description' }[] { + const candidates: { url: string; type: 'purchase_url' | 'description' }[] = + []; + const seen = new Set(); + + const push = (raw: string, type: 'purchase_url' | 'description') => { + const url = trimExtractedUrl(raw); + if (!/^https?:\/\//i.test(url) || isSoundcloudUrl(url) || seen.has(url)) { + return; + } + // Skip URLs already recognized as download gates / Bandcamp. + if (matchKnownDownloadGateUrl(url)) return; + seen.add(url); + candidates.push({ url, type }); + }; + + if (track.purchase_url) { + push(track.purchase_url, 'purchase_url'); + } + if (track.description) { + for (const match of track.description.match(ANY_HTTP_URL_RE) ?? []) { + push(match, 'description'); + } + } + return candidates; +} + /** * Prefer Hypeddit / Droploud / GateRush / DownloadGater from purchase_url or - * description, then fall back to a Bandcamp purchase/description link. + * description, then Bandcamp, then direct file links (Dropbox, Drive, …). */ export function extractGateUrl(track: SoundcloudTrack): GateUrlMatch | null { const { purchase_url, description } = track; @@ -399,9 +568,59 @@ export function extractGateUrl(track: SoundcloudTrack): GateUrlMatch | null { } } + if (purchase_url) { + const direct = matchDirectDownloadUrl(purchase_url); + if (direct) { + return { ...direct, type: 'purchase_url' }; + } + } + + if (description) { + const direct = matchDirectDownloadUrl(description); + if (direct) { + return { ...direct, type: 'description' }; + } + } + + return null; +} + +/** + * Like extractGateUrl, but also follows redirects / scans smart-link HTML when + * purchase_url or description contain unrecognized http(s) links. + */ +export async function extractAndResolveGateUrl( + track: SoundcloudTrack, +): Promise { + const sync = extractGateUrl(track); + if (sync) return sync; + + for (const candidate of collectUnresolvedHttpCandidates(track)) { + const resolved = await resolveUnknownGateUrl(candidate.url); + if (resolved) { + return { ...resolved, type: candidate.type }; + } + } return null; } +/** + * Resolve a user-supplied or stored gate URL: known providers pass through; + * otherwise try redirect / HTML destination resolution. + */ +export async function resolveGateUrlOrFollow( + value: string, +): Promise<{ url: string; provider: GateProvider } | null> { + const direct = resolveGateProviderUrl(value); + if (direct && direct.provider !== 'soundcloud') { + return direct; + } + if (direct?.provider === 'soundcloud') { + return direct; + } + return resolveUnknownGateUrl(value); +} + /** @deprecated Prefer extractGateUrl */ export function extractHypedditUrl( track: SoundcloudTrack, diff --git a/src/ytdlp.ts b/src/ytdlp.ts index 6e47b8c..d5e7b20 100644 --- a/src/ytdlp.ts +++ b/src/ytdlp.ts @@ -2,7 +2,7 @@ import { mkdir, rm } from 'node:fs/promises'; import { basename, join } from 'node:path'; import { execa } from 'execa'; import { lookpath } from 'find-bin'; -import type { JobProgress } from './types'; +import type { BandcampAlbumTrackChoice, JobProgress } from './types'; import { isBandcampAlbumUrl, isSoundcloudUrl, @@ -19,11 +19,17 @@ type ProgressCallback = ( export type YtDlpDownloadOptions = { /** Prefer this title when the URL is a Bandcamp album (or multi-entry). */ matchTitle?: string; + /** + * Called when auto title-match fails (or no title was given). + * Return a track URL from `error.tracks` to continue; throw/reject to abort. + */ + onAlbumMatchFailed?: (error: BandcampAlbumMatchError) => Promise; }; const DOWNLOADS_DIR = './downloads'; const YTDLP_DOWNLOAD_TIMEOUT_MS = 10 * 60_000; const YTDLP_METADATA_TIMEOUT_MS = 60_000; +const ALBUM_MATCH_THRESHOLD = 0.5; type FlatEntry = { id?: string; @@ -32,6 +38,26 @@ type FlatEntry = { webpage_url?: string; }; +/** Thrown when a Bandcamp album cannot be auto-matched to a single track. */ +export class BandcampAlbumMatchError extends Error { + readonly albumUrl: string; + readonly matchTitle: string | undefined; + readonly tracks: BandcampAlbumTrackChoice[]; + + constructor( + message: string, + albumUrl: string, + matchTitle: string | undefined, + tracks: BandcampAlbumTrackChoice[], + ) { + super(message); + this.name = 'BandcampAlbumMatchError'; + this.albumUrl = albumUrl; + this.matchTitle = matchTitle; + this.tracks = tracks; + } +} + export async function getYtDlpBin(): Promise { const ytDlpBin = await lookpath('yt-dlp'); if (!ytDlpBin) { @@ -80,7 +106,8 @@ export function titleMatchScore(candidate: string, wanted: string): number { * * Bandcamp album URLs: when `matchTitle` is set, resolves the best-matching * track on the album and downloads only that entry (avoids grabbing the whole - * album and offering the wrong file). + * album and offering the wrong file). If matching fails, `onAlbumMatchFailed` + * (when provided) can pick a track; otherwise a `BandcampAlbumMatchError` is thrown. */ export class YtDlpDownloader { private progressCallback: ProgressCallback | null = null; @@ -133,15 +160,12 @@ export class YtDlpDownloader { const ytDlpBin = await getYtDlpBin(); let downloadUrl = url; - if (isBandcampAlbumUrl(url) && options.matchTitle) { + if (isBandcampAlbumUrl(url)) { downloadUrl = await this.resolveBandcampAlbumTrack( ytDlpBin, url, options.matchTitle, - ); - } else if (isBandcampAlbumUrl(url) && !options.matchTitle) { - throw new Error( - 'Bandcamp album URL needs a track title to pick the right song. Re-fetch the SoundCloud track and try again.', + options.onAlbumMatchFailed, ); } @@ -244,16 +268,19 @@ export class YtDlpDownloader { private async resolveBandcampAlbumTrack( ytDlpBin: string, albumUrl: string, - matchTitle: string, + matchTitle: string | undefined, + onAlbumMatchFailed?: (error: BandcampAlbumMatchError) => Promise, ): Promise { - this.progressCallback?.( - 'downloading', - `Matching β€œ${matchTitle}” on Bandcamp album...`, - 45, - { browserless: true }, - ); + const matchingLabel = matchTitle + ? `Matching β€œ${matchTitle}” on Bandcamp album...` + : 'Listing Bandcamp album tracks...'; + this.progressCallback?.('downloading', matchingLabel, 45, { + browserless: true, + }); console.log( - `${this.sourceLabel}: album URL β€” matching track β€œ${matchTitle}”…`, + matchTitle + ? `${this.sourceLabel}: album URL β€” matching track β€œ${matchTitle}”…` + : `${this.sourceLabel}: album URL β€” no title to match; listing tracks…`, ); const stdout = await this.runYtDlp( @@ -275,33 +302,60 @@ export class YtDlpDownloader { throw new Error(`No tracks found on Bandcamp album: ${albumUrl}`); } - const ranked = entries - .map((entry) => ({ - entry, - score: titleMatchScore(entry.title, matchTitle), - })) + const tracks: BandcampAlbumTrackChoice[] = entries + .map((entry) => { + const url = entry.url || entry.webpage_url; + if (!url) return null; + return { + title: entry.title, + url, + score: matchTitle ? titleMatchScore(entry.title, matchTitle) : 0, + }; + }) + .filter((t): t is BandcampAlbumTrackChoice => t !== null) .sort((a, b) => b.score - a.score); - const best = ranked[0]; - if (!best || best.score < 0.5) { - const available = entries.map((e) => e.title).join(', '); + if (tracks.length === 0) { throw new Error( - `Could not match SoundCloud title β€œ${matchTitle}” to a track on the Bandcamp album. Available: ${available}`, + `Bandcamp album listed tracks but yt-dlp did not provide track URLs: ${albumUrl}`, ); } - const finalUrl = best.entry.url || best.entry.webpage_url; - if (!finalUrl) { - throw new Error( - `Matched β€œ${best.entry.title}” but yt-dlp did not provide a track URL`, + const best = matchTitle ? tracks[0] : undefined; + if (best && best.score >= ALBUM_MATCH_THRESHOLD) { + console.log( + `${this.sourceLabel}: matched β€œ${best.title}” (score ${best.score.toFixed(2)}) β†’ ${best.url}`, ); + return best.url; } - console.log( - `${this.sourceLabel}: matched β€œ${best.entry.title}” (score ${best.score.toFixed(2)}) β†’ ${finalUrl}`, + const available = tracks.map((t) => t.title).join(', '); + const message = matchTitle + ? `Could not match SoundCloud title β€œ${matchTitle}” to a track on the Bandcamp album. Available: ${available}` + : `Bandcamp album URL needs a track pick (no SoundCloud title to match). Available: ${available}`; + + const matchError = new BandcampAlbumMatchError( + message, + albumUrl, + matchTitle, + tracks, ); - return finalUrl; + if (onAlbumMatchFailed) { + const selectedUrl = await onAlbumMatchFailed(matchError); + if (!selectedUrl) { + throw new Error('No Bandcamp album track selected'); + } + const picked = tracks.find((t) => t.url === selectedUrl); + console.log( + picked + ? `${this.sourceLabel}: user selected β€œ${picked.title}” β†’ ${selectedUrl}` + : `${this.sourceLabel}: user selected track URL β†’ ${selectedUrl}`, + ); + return selectedUrl; + } + + throw matchError; } } diff --git a/webui/src/components/App.css b/webui/src/components/App.css index 959d1c5..d44ba35 100644 --- a/webui/src/components/App.css +++ b/webui/src/components/App.css @@ -516,6 +516,63 @@ margin-top: var(--space-sm); } +.bandcamp-track-picker { + display: flex; + flex-direction: column; + gap: var(--space-md); +} + +.bandcamp-track-list { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: var(--space-sm); + max-height: 320px; + overflow-y: auto; +} + +.bandcamp-track-option { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-md); + width: 100%; + padding: var(--space-md); + text-align: left; + background: var(--bg-elevated); + border: 1px solid var(--border-subtle); + border-radius: 8px; + color: var(--text-primary); + cursor: pointer; + transition: + border-color var(--transition-normal), + background var(--transition-normal); +} + +.bandcamp-track-option:hover:not(:disabled) { + border-color: var(--neon-orange); + background: var(--bg-surface); +} + +.bandcamp-track-option:disabled { + opacity: 0.6; + cursor: not-allowed; +} + +.bandcamp-track-title { + font-weight: 500; + line-height: 1.3; +} + +.bandcamp-track-score { + flex-shrink: 0; + font-family: var(--font-mono); + font-size: 0.75rem; + color: var(--text-secondary); +} + /* Metadata form */ .metadata-form { padding: var(--space-xl); diff --git a/webui/src/components/App.tsx b/webui/src/components/App.tsx index a2948ea..973dea0 100644 --- a/webui/src/components/App.tsx +++ b/webui/src/components/App.tsx @@ -27,6 +27,13 @@ interface JobProgress { downloadBytes?: number; totalBytes?: number; browserless?: boolean; + bandcampAlbumTracks?: BandcampAlbumTrackChoice[]; +} + +interface BandcampAlbumTrackChoice { + title: string; + url: string; + score: number; } interface JobState { @@ -346,6 +353,12 @@ export default function App() { err instanceof Error ? err.message : 'Failed to load job', })); }); + } else if (progress.stage === 'waiting_bandcamp_track') { + setJob((prev) => ({ + ...prev, + progress, + error: null, + })); } else if (progress.stage === 'error') { eventSource.close(); eventSourceRef.current = null; @@ -609,6 +622,35 @@ export default function App() { } }; + const handleBandcampTrackSelect = async (trackUrl: string) => { + if (!job.jobId) return; + + setIsLoading(true); + setJob((prev) => ({ ...prev, error: null })); + + try { + const response = await fetch( + `${API_BASE}/api/job/${job.jobId}/bandcamp-track`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ trackUrl }), + }, + ); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.error || 'Failed to select Bandcamp track'); + } + } catch (err) { + setJob((prev) => ({ + ...prev, + error: err instanceof Error ? err.message : 'Unknown error', + })); + } finally { + setIsLoading(false); + } + }; + // Process metadata and finalize const processMetadata = async (preserveMetadata = false) => { if (!job.jobId) return; @@ -776,7 +818,7 @@ export default function App() {

Download & tag SoundCloud tracks from Hypeddit, Droploud, GateRush, - DownloadGater, or Bandcamp + DownloadGater, Bandcamp, or a direct download link

@@ -933,7 +975,7 @@ export default function App() {

{skipAutomaticHypedditFetch ? 'Automatic gate lookup is disabled. Enter a gate URL manually, or download the SoundCloud track via yt-dlp.' - : 'Gate URL not found in track. Enter a Hypeddit, Droploud, GateRush, DownloadGater, or Bandcamp URL, or download via yt-dlp from SoundCloud.'} + : 'Gate URL not found in track. Enter a Hypeddit, Droploud, GateRush, DownloadGater, Bandcamp, or direct download URL, or download via yt-dlp from SoundCloud.'}

@@ -945,7 +987,7 @@ export default function App() { name="hypeddit-url" value={hypedditUrlInput} onChange={(e) => setHypedditUrlInput(e.target.value)} - placeholder="https://hypeddit.com/... / droploud.com/gate/... / gaterush.me/... / downloadgater.com/g/... / artist.bandcamp.com/track/..." + placeholder="https://hypeddit.com/... / droploud.com/gate/... / gaterush.me/... / downloadgater.com/g/... / artist.bandcamp.com/track/... / https://www.dropbox.com/...?dl=1" autoComplete="off" required disabled={isLoading} @@ -1000,39 +1042,81 @@ export default function App() { {/* Step 3: Progress */} {step === 'progress' && (
-
- - {job.progress?.message || 'Initializing...'} - - {job.progress?.currentGate && ( - - {job.progress.currentGate.toUpperCase()} - - )} -
-
-
-
-
- {formatPercent(job.progress?.percent)}% - {job.progress?.downloadBytes !== undefined && - job.progress?.totalBytes !== undefined && ( - - {(job.progress.downloadBytes / 1024 / 1024).toFixed(1)} /{' '} - {(job.progress.totalBytes / 1024 / 1024).toFixed(1)} MB + {job.progress?.stage === 'waiting_bandcamp_track' ? ( +
+
+ i +

+ {job.progress.message || + 'Could not auto-match a Bandcamp album track. Pick one to download.'} +

+
+
    + {(job.progress.bandcampAlbumTracks ?? []).map((track) => ( +
  • + +
  • + ))} +
+ +
+ ) : ( + <> +
+ + {job.progress?.message || 'Initializing...'} - )} -
- + {job.progress?.currentGate && ( + + {job.progress.currentGate.toUpperCase()} + + )} +
+
+
+
+
+ {formatPercent(job.progress?.percent)}% + {job.progress?.downloadBytes !== undefined && + job.progress?.totalBytes !== undefined && ( + + {(job.progress.downloadBytes / 1024 / 1024).toFixed(1)}{' '} + /{' '} + {(job.progress.totalBytes / 1024 / 1024).toFixed(1)} MB + + )} +
+ + + )}
)}