From 970fb7e5987639efde24d20369b1c1dfc5331bab Mon Sep 17 00:00:00 2001 From: Pas <74743263+Pasithea0@users.noreply.github.com> Date: Fri, 1 Aug 2025 14:54:29 -0600 Subject: [PATCH] remove external subtitle scraping --- package.json | 3 +- pnpm-lock.yaml | 8 --- src/runners/individualRunner.ts | 46 +------------ src/runners/runner.ts | 31 --------- src/utils/opensubtitles.ts | 110 -------------------------------- src/utils/wyziesubs.ts | 79 ----------------------- 6 files changed, 2 insertions(+), 275 deletions(-) delete mode 100644 src/utils/opensubtitles.ts delete mode 100644 src/utils/wyziesubs.ts diff --git a/package.json b/package.json index 37a6bc1..2e8f529 100644 --- a/package.json +++ b/package.json @@ -94,8 +94,7 @@ "nanoid": "^3.3.8", "node-fetch": "^3.3.2", "set-cookie-parser": "^2.7.1", - "unpacker": "^1.0.1", - "wyzie-lib": "^2.2.5" + "unpacker": "^1.0.1" }, "packageManager": "pnpm@9.14.4" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 487884d..3c9d2b9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,9 +44,6 @@ importers: unpacker: specifier: ^1.0.1 version: 1.0.1 - wyzie-lib: - specifier: ^2.2.5 - version: 2.2.5 devDependencies: '@nabla/vite-plugin-eslint': specifier: ^2.0.5 @@ -2539,9 +2536,6 @@ packages: utf-8-validate: optional: true - wyzie-lib@2.2.5: - resolution: {integrity: sha512-hOvAqS0VJhqTi2TRFkQ1YSEvX5ltrKC+ZWGiGDZ2Qn6b2t5Ov5DJILFCO2W/1ScK+BKOKHen7tRidatj1W62oA==} - y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -5248,8 +5242,6 @@ snapshots: ws@8.18.0: {} - wyzie-lib@2.2.5: {} - y18n@5.0.8: {} yallist@4.0.0: {} diff --git a/src/runners/individualRunner.ts b/src/runners/individualRunner.ts index 9fb7e42..7c4268b 100644 --- a/src/runners/individualRunner.ts +++ b/src/runners/individualRunner.ts @@ -6,7 +6,6 @@ import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { ProviderList } from '@/providers/get'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles'; import { requiresProxy, setupProxy } from '@/utils/proxy'; import { isValidStream, validatePlayableStreams } from '@/utils/valid'; @@ -18,7 +17,6 @@ export type IndividualSourceRunnerOptions = { id: string; events?: IndividualScraperEvents; proxyStreams?: boolean; // temporary - disableOpensubtitles?: boolean; }; export async function scrapeInvidualSource( @@ -74,15 +72,6 @@ export async function scrapeInvidualSource( return true; }); - // opensubtitles - if (!ops.disableOpensubtitles) - for (const embed of output.embeds) - embed.url = `${embed.url}${btoa('MEDIA=')}${btoa( - `${ops.media.imdbId}${ - ops.media.type === 'show' ? `.${ops.media.season.number}.${ops.media.episode.number}` : '' - }`, - )}`; - if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) throw new NotFoundError('No streams found'); @@ -91,20 +80,6 @@ export async function scrapeInvidualSource( const playableStreams = await validatePlayableStreams(output.stream, ops, sourceScraper.id); if (playableStreams.length === 0) throw new NotFoundError('No playable streams found'); - // opensubtitles - if (!ops.disableOpensubtitles) { - for (const playableStream of playableStreams) { - playableStream.captions = await addOpenSubtitlesCaptions( - playableStream.captions, - ops, - btoa( - `${ops.media.imdbId}${ - ops.media.type === 'show' ? `.${ops.media.season.number}.${ops.media.episode.number}` : '' - }`, - ), - ); - } - } output.stream = playableStreams; } return output; @@ -118,7 +93,6 @@ export type IndividualEmbedRunnerOptions = { id: string; events?: IndividualScraperEvents; proxyStreams?: boolean; // temporary - disableOpensubtitles?: boolean; }; export async function scrapeIndividualEmbed( @@ -128,9 +102,7 @@ export async function scrapeIndividualEmbed( const embedScraper = list.embeds.find((v) => ops.id === v.id); if (!embedScraper) throw new Error('Embed with ID not found'); - let url = ops.url; - let media; - if (ops.url.includes(btoa('MEDIA='))) [url, media] = url.split(btoa('MEDIA=')); + const url = ops.url; const output = await embedScraper.scrape({ fetcher: ops.fetcher, @@ -157,22 +129,6 @@ export async function scrapeIndividualEmbed( const playableStreams = await validatePlayableStreams(output.stream, ops, embedScraper.id); if (playableStreams.length === 0) throw new NotFoundError('No playable streams found'); - if (media && !ops.disableOpensubtitles) { - const [imdbId, season, episode] = atob(media) - .split('.') - .map((x, i) => (i === 0 ? x : Number(x) || null)); - const mediaInfo = { - ...ops, - media: { - type: season && episode ? 'show' : 'movie', - imdbId: imdbId?.toString() || '', - ...(season && episode ? { season: { number: season }, episode: { number: episode } } : {}), - } as ScrapeMedia, - }; - for (const playableStream of playableStreams) - playableStream.captions = await addOpenSubtitlesCaptions(playableStream.captions, mediaInfo, media); - } - output.stream = playableStreams; return output; diff --git a/src/runners/runner.ts b/src/runners/runner.ts index 5f43894..92f65cb 100644 --- a/src/runners/runner.ts +++ b/src/runners/runner.ts @@ -8,7 +8,6 @@ import { Stream } from '@/providers/streams'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { reorderOnIdList } from '@/utils/list'; -import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles'; import { requiresProxy, setupProxy } from '@/utils/proxy'; import { isValidStream, validatePlayableStream } from '@/utils/valid'; @@ -38,7 +37,6 @@ export type ProviderRunnerOptions = { events?: FullScraperEvents; media: ScrapeMedia; proxyStreams?: boolean; // temporary - disableOpensubtitles?: boolean; }; export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOptions): Promise { @@ -115,21 +113,6 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt const playableStream = await validatePlayableStream(output.stream[0], ops, source.id); if (!playableStream) throw new NotFoundError('No streams found'); - // opensubtitles - if (!ops.disableOpensubtitles) { - if (ops.media.imdbId) { - playableStream.captions = await addOpenSubtitlesCaptions( - playableStream.captions, - ops, - btoa( - `${ops.media.imdbId}${ - ops.media.type === 'show' ? `.${ops.media.season.number}.${ops.media.episode.number}` : '' - }`, - ), - ); - } - } - return { sourceId: source.id, stream: playableStream, @@ -181,20 +164,6 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt const playableStream = await validatePlayableStream(embedOutput.stream[0], ops, embed.embedId); if (!playableStream) throw new NotFoundError('No streams found'); - // opensubtitles - if (!ops.disableOpensubtitles) { - if (ops.media.imdbId) { - playableStream.captions = await addOpenSubtitlesCaptions( - playableStream.captions, - ops, - btoa( - `${ops.media.imdbId}${ - ops.media.type === 'show' ? `.${ops.media.season.number}.${ops.media.episode.number}` : '' - }`, - ), - ); - } - } embedOutput.stream = [playableStream]; } catch (error) { const updateParams: UpdateEvent = { diff --git a/src/utils/opensubtitles.ts b/src/utils/opensubtitles.ts deleted file mode 100644 index c821f49..0000000 --- a/src/utils/opensubtitles.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* eslint-disable no-console */ -import { ScrapeMedia } from '@/entrypoint/utils/media'; -import { Caption, labelToLanguageCode } from '@/providers/captions'; -import { IndividualEmbedRunnerOptions } from '@/runners/individualRunner'; -import { ProviderRunnerOptions } from '@/runners/runner'; - -import { addWyzieCaptions } from './wyziesubs'; - -type CaptionOptions = (ProviderRunnerOptions | IndividualEmbedRunnerOptions) & { - media?: ScrapeMedia; -}; - -const timeout = (ms: number, source: string) => - new Promise((resolve) => { - setTimeout(() => { - console.error(`${source} captions request timed out after ${ms}ms`); - resolve(null); - }, ms); - }); - -export async function addOpenSubtitlesCaptions( - captions: Caption[], - ops: CaptionOptions, - media: string, -): Promise { - try { - const [imdbId, season, episode] = atob(media) - .split('.') - .map((x, i) => (i === 0 ? x : Number(x) || null)); - if (!imdbId) return captions; - - const allCaptions = [...captions]; - - // Fetch Wyzie captions with 2 second timeout - const wyziePromise = addWyzieCaptions( - [], - ops.media?.tmdbId?.toString() || '', - imdbId.toString(), - typeof season === 'number' ? season : undefined, - typeof episode === 'number' ? episode : undefined, - ) - .then((wyzieCaptions) => { - if (wyzieCaptions && wyzieCaptions.length > 0) { - return wyzieCaptions.map((caption) => ({ - ...caption, - opensubtitles: true, - })); - } - return []; - }) - .catch((error) => { - console.error('Wyzie subtitles fetch failed:', error); - return []; - }); - - // Fetch OpenSubtitles captions with 5 second timeout - const openSubsPromise = ops - .proxiedFetcher( - `https://rest.opensubtitles.org/search/${ - season && episode ? `episode-${episode}/` : '' - }imdbid-${(imdbId as string).slice(2)}${season && episode ? `/season-${season}` : ''}`, - { - headers: { - 'X-User-Agent': 'VLSub 0.10.2', - }, - }, - ) - .then((Res) => { - const openSubtilesCaptions: Caption[] = []; - for (const caption of Res) { - const url = caption.SubDownloadLink.replace('.gz', '').replace('download/', 'download/subencoding-utf8/'); - const language = labelToLanguageCode(caption.LanguageName); - if (!url || !language) continue; - else - openSubtilesCaptions.push({ - id: url, - opensubtitles: true, - url, - type: caption.SubFormat || 'srt', - hasCorsRestrictions: false, - language, - }); - } - return openSubtilesCaptions; - }) - .catch((error) => { - console.error('OpenSubtitles fetch failed:', error); - return []; - }); - - // Wait for both promises with their respective timeouts - const [wyzieCaptions, openSubsCaptions] = await Promise.all([ - Promise.race([wyziePromise, timeout(2000, 'Wyzie')]), - Promise.race([openSubsPromise, timeout(5000, 'OpenSubtitles')]), - ]); - - // Debug logging - // console.log('Wyzie captions found:', wyzieCaptions?.length || 0); - // console.log('OpenSubtitles captions found:', openSubsCaptions?.length || 0); - - // Add any successful captions to our result - if (wyzieCaptions) allCaptions.push(...wyzieCaptions); - if (openSubsCaptions) allCaptions.push(...openSubsCaptions); - - return allCaptions; - } catch (error) { - console.error('Error in addOpenSubtitlesCaptions:', error); - return captions; - } -} diff --git a/src/utils/wyziesubs.ts b/src/utils/wyziesubs.ts deleted file mode 100644 index d7f789a..0000000 --- a/src/utils/wyziesubs.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* eslint-disable no-console */ -import { type SubtitleData, searchSubtitles } from 'wyzie-lib'; - -import { Caption } from '@/providers/captions'; - -// function isSubdlUrl(url: string) { -// return url.endsWith('.subdl'); -// } - -// export function filterSubtitles(list: Caption[]) { -// const selected: Record = {}; - -// for (const sub of list) { -// const existing = selected[sub.language]; - -// if (!existing) { -// selected[sub.language] = sub; -// continue; -// } - -// const existingIsSubdl = isSubdlUrl(existing.url); -// const currentIsSubdl = isSubdlUrl(sub.url); - -// if (existingIsSubdl && !currentIsSubdl) { -// selected[sub.language] = sub; -// } -// } - -// return Object.values(selected); -// } - -export async function addWyzieCaptions( - captions: Caption[], - tmdbId: string | number, - imdbId: string, - season?: number, - episode?: number, -): Promise { - try { - const searchParams: any = { - encoding: 'utf-8', - source: 'all', - imdb_id: imdbId, - }; - - if (tmdbId && !imdbId) { - searchParams.tmdb_id = typeof tmdbId === 'string' ? parseInt(tmdbId, 10) : tmdbId; - } - - if (season && episode) { - searchParams.season = season; - searchParams.episode = episode; - } - - console.log('Searching Wyzie subtitles with params:', searchParams); - const wyzieSubtitles: SubtitleData[] = await searchSubtitles(searchParams); - // console.log('Found Wyzie subtitles:', wyzieSubtitles); - - const wyzieCaptions: Caption[] = wyzieSubtitles.map((subtitle) => ({ - id: subtitle.id, - url: subtitle.url, - type: subtitle.format === 'srt' || subtitle.format === 'vtt' ? subtitle.format : 'srt', - hasCorsRestrictions: false, - language: subtitle.language, - // Additional metadata from Wyzie - flagUrl: subtitle.flagUrl, - display: subtitle.display, - media: subtitle.media, - isHearingImpaired: subtitle.isHearingImpaired, - source: typeof subtitle.source === 'number' ? subtitle.source.toString() : subtitle.source, - encoding: subtitle.encoding, - })); - - return [...captions, ...wyzieCaptions]; - } catch (error) { - console.error('Error fetching Wyzie subtitles:', error); - return captions; - } -}