From 8f5728ebf0571ab09fff20880fb9f4459ce50d56 Mon Sep 17 00:00:00 2001 From: Pas <74743263+Pasithea0@users.noreply.github.com> Date: Fri, 1 Aug 2025 13:57:45 -0600 Subject: [PATCH] move external subtitle scrape to client --- package.json | 1 + pnpm-lock.yaml | 3 + src/assets/locales/en.json | 2 + .../settings/OpensubtitlesCaptionsView.tsx | 13 + src/stores/player/slices/source.ts | 33 +++ src/utils/externalSubtitles.ts | 256 ++++++++++++++++++ 6 files changed, 308 insertions(+) create mode 100644 src/utils/externalSubtitles.ts diff --git a/package.json b/package.json index 6845f887..f31e3826 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "semver": "^7.7.2", "slugify": "^1.6.6", "subsrt-ts": "^2.1.2", + "wyzie-lib": "^2.2.5", "zustand": "^4.5.7" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4411a8c9..93b4a097 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -156,6 +156,9 @@ importers: subsrt-ts: specifier: ^2.1.2 version: 2.1.2 + wyzie-lib: + specifier: ^2.2.5 + version: 2.2.5 zustand: specifier: ^4.5.7 version: 4.5.7(@types/react@18.3.23)(immer@10.1.1)(react@18.3.1) diff --git a/src/assets/locales/en.json b/src/assets/locales/en.json index f52d37f0..c905b464 100644 --- a/src/assets/locales/en.json +++ b/src/assets/locales/en.json @@ -687,6 +687,8 @@ "unknownLanguage": "Unknown", "dropSubtitleFile": "Drop subtitle file here! >_<", "scrapeButton": "Scrape subtitles", + "refresh": "Refresh External Subtitles", + "refreshing": "Refreshing...", "empty": "There are no provided subtitles for this.", "notFound": "None of the available options match your query", "useNativeSubtitles": "Use native video subtitles", diff --git a/src/components/player/atoms/settings/OpensubtitlesCaptionsView.tsx b/src/components/player/atoms/settings/OpensubtitlesCaptionsView.tsx index a7cfca12..15150654 100644 --- a/src/components/player/atoms/settings/OpensubtitlesCaptionsView.tsx +++ b/src/components/player/atoms/settings/OpensubtitlesCaptionsView.tsx @@ -27,6 +27,7 @@ export function OpenSubtitlesCaptionView({ const { selectCaptionById } = useCaptions(); const captionList = usePlayerStore((s) => s.captionList); const getHlsCaptionList = usePlayerStore((s) => s.display?.getCaptionList); + const addExternalSubtitles = usePlayerStore((s) => s.addExternalSubtitles); const captions = useMemo( () => @@ -48,6 +49,10 @@ export function OpenSubtitlesCaptionView({ [selectCaptionById, setCurrentlyDownloading], ); + const [refreshReq, startRefresh] = useAsyncFn(async () => { + return addExternalSubtitles(); + }, [addExternalSubtitles]); + const content = subtitleList.length ? subtitleList.map((v) => { return ( @@ -98,6 +103,14 @@ export function OpenSubtitlesCaptionView({
{t("player.menus.subtitles.empty")} +
) : ( diff --git a/src/stores/player/slices/source.ts b/src/stores/player/slices/source.ts index 9c2335dd..8c04dc6b 100644 --- a/src/stores/player/slices/source.ts +++ b/src/stores/player/slices/source.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ import { ScrapeMedia } from "@p-stream/providers"; import { MakeSlice } from "@/stores/player/slices/types"; @@ -98,6 +99,7 @@ export interface SourceSlice { enableAutomaticQuality(): void; redisplaySource(startAt: number): void; setCaptionAsTrack(asTrack: boolean): void; + addExternalSubtitles(): Promise; } export function metaToScrapeMedia(meta: PlayerMeta): ScrapeMedia { @@ -184,6 +186,12 @@ export const createSourceSlice: MakeSlice = (set, get) => ({ }); const store = get(); store.redisplaySource(startAt); + + // Trigger external subtitle scraping after stream is loaded + // This runs asynchronously so it doesn't block the stream loading + setTimeout(() => { + store.addExternalSubtitles(); + }, 100); }, redisplaySource(startAt: number) { const store = get(); @@ -235,4 +243,29 @@ export const createSourceSlice: MakeSlice = (set, get) => ({ s.caption.asTrack = asTrack; }); }, + async addExternalSubtitles() { + const store = get(); + if (!store.meta) return; + + try { + const { scrapeExternalSubtitles } = await import( + "@/utils/externalSubtitles" + ); + const externalCaptions = await scrapeExternalSubtitles(store.meta); + + if (externalCaptions.length > 0) { + set((s) => { + // Add external captions to the existing list, avoiding duplicates + const existingIds = new Set(s.captionList.map((c) => c.id)); + const newCaptions = externalCaptions.filter( + (c) => !existingIds.has(c.id), + ); + s.captionList = [...s.captionList, ...newCaptions]; + }); + console.log(`Added ${externalCaptions.length} external captions`); + } + } catch (error) { + console.error("Failed to scrape external subtitles:", error); + } + }, }); diff --git a/src/utils/externalSubtitles.ts b/src/utils/externalSubtitles.ts new file mode 100644 index 00000000..d3b33819 --- /dev/null +++ b/src/utils/externalSubtitles.ts @@ -0,0 +1,256 @@ +/* eslint-disable no-console */ +import { type SubtitleData, searchSubtitles } from "wyzie-lib"; + +import { CaptionListItem, PlayerMeta } from "@/stores/player/slices/source"; + +// Helper function to convert language names to language codes +function labelToLanguageCode(languageName: string): string { + const languageMap: Record = { + English: "en", + Spanish: "es", + French: "fr", + German: "de", + Italian: "it", + Portuguese: "pt", + Russian: "ru", + Japanese: "ja", + Korean: "ko", + Chinese: "zh", + Arabic: "ar", + Hindi: "hi", + Turkish: "tr", + Dutch: "nl", + Polish: "pl", + Swedish: "sv", + Norwegian: "no", + Danish: "da", + Finnish: "fi", + Greek: "el", + Hebrew: "he", + Thai: "th", + Vietnamese: "vi", + Indonesian: "id", + Malay: "ms", + Filipino: "tl", + Ukrainian: "uk", + Romanian: "ro", + Czech: "cs", + Hungarian: "hu", + Bulgarian: "bg", + Croatian: "hr", + Serbian: "sr", + Slovak: "sk", + Slovenian: "sl", + Estonian: "et", + Latvian: "lv", + Lithuanian: "lt", + Icelandic: "is", + Maltese: "mt", + Georgian: "ka", + Armenian: "hy", + Azerbaijani: "az", + Kazakh: "kk", + Kyrgyz: "ky", + Uzbek: "uz", + Tajik: "tg", + Turkmen: "tk", + Mongolian: "mn", + Persian: "fa", + Urdu: "ur", + Bengali: "bn", + Tamil: "ta", + Telugu: "te", + Marathi: "mr", + Gujarati: "gu", + Kannada: "kn", + Malayalam: "ml", + Punjabi: "pa", + Sinhala: "si", + Nepali: "ne", + Burmese: "my", + Khmer: "km", + Lao: "lo", + Tibetan: "bo", + Uyghur: "ug", + Kurdish: "ku", + Pashto: "ps", + Dari: "prs", + Sindhi: "sd", + Kashmiri: "ks", + Dogri: "doi", + Konkani: "kok", + Manipuri: "mni", + Bodo: "brx", + Sanskrit: "sa", + Santhali: "sat", + Maithili: "mai", + Bhojpuri: "bho", + Awadhi: "awa", + Chhattisgarhi: "hne", + Magahi: "mag", + Rajasthani: "raj", + Malvi: "mup", + Bundeli: "bns", + Bagheli: "bfy", + Pahari: "phr", + Kumaoni: "kfy", + Garhwali: "gbm", + Kangri: "xnr", + }; + + return languageMap[languageName] || languageName.toLowerCase(); +} + +const timeout = (ms: number, source: string) => + new Promise((resolve) => { + setTimeout(() => { + console.error(`${source} captions request timed out after ${ms}ms`); + resolve(null); + }, ms); + }); + +export async function scrapeWyzieCaptions( + tmdbId: string | number, + imdbId: string, + season?: number, + episode?: number, +): Promise { + try { + const searchParams: any = { + encoding: "utf-8", + source: "all", + imdb_id: imdbId, + }; + + if (tmdbId && !imdbId) { + searchParams.tmdb_id = + typeof tmdbId === "string" ? parseInt(tmdbId, 10) : tmdbId; + } + + if (season && episode) { + searchParams.season = season; + searchParams.episode = episode; + } + + console.log("Searching Wyzie subtitles with params:", searchParams); + const wyzieSubtitles: SubtitleData[] = await searchSubtitles(searchParams); + + const wyzieCaptions: CaptionListItem[] = wyzieSubtitles.map((subtitle) => ({ + id: subtitle.id, + language: subtitle.language, + url: subtitle.url, + type: + subtitle.format === "srt" || subtitle.format === "vtt" + ? subtitle.format + : "srt", + needsProxy: false, + opensubtitles: true, + // Additional metadata from Wyzie + display: subtitle.display, + media: subtitle.media, + isHearingImpaired: subtitle.isHearingImpaired, + source: + typeof subtitle.source === "number" + ? subtitle.source.toString() + : subtitle.source, + encoding: subtitle.encoding, + })); + + return wyzieCaptions; + } catch (error) { + console.error("Error fetching Wyzie subtitles:", error); + return []; + } +} + +export async function scrapeOpenSubtitlesCaptions( + imdbId: string, + season?: number, + episode?: number, +): Promise { + try { + const url = `https://rest.opensubtitles.org/search/${ + season && episode ? `episode-${episode}/` : "" + }imdbid-${imdbId.slice(2)}${season && episode ? `/season-${season}` : ""}`; + + const response = await fetch(url, { + headers: { + "X-User-Agent": "VLSub 0.10.2", + }, + }); + + if (!response.ok) { + throw new Error(`OpenSubtitles API returned ${response.status}`); + } + + const data = await response.json(); + const openSubtitlesCaptions: CaptionListItem[] = []; + + for (const caption of data) { + const downloadUrl = caption.SubDownloadLink.replace(".gz", "").replace( + "download/", + "download/subencoding-utf8/", + ); + const language = labelToLanguageCode(caption.LanguageName); + + if (!downloadUrl || !language) continue; + + openSubtitlesCaptions.push({ + id: downloadUrl, + language, + url: downloadUrl, + type: caption.SubFormat || "srt", + needsProxy: false, + opensubtitles: true, + }); + } + + return openSubtitlesCaptions; + } catch (error) { + console.error("Error fetching OpenSubtitles:", error); + return []; + } +} + +export async function scrapeExternalSubtitles( + meta: PlayerMeta, +): Promise { + try { + // Extract IMDb ID from meta + const imdbId = meta.imdbId; + if (!imdbId) { + console.log("No IMDb ID available for external subtitle scraping"); + return []; + } + + const season = meta.season?.number; + const episode = meta.episode?.number; + const tmdbId = meta.tmdbId; + + // Fetch both Wyzie and OpenSubtitles captions with timeouts + const [wyzieCaptions, openSubsCaptions] = await Promise.all([ + Promise.race([ + scrapeWyzieCaptions(tmdbId, imdbId, season, episode), + timeout(2000, "Wyzie"), + ]), + Promise.race([ + scrapeOpenSubtitlesCaptions(imdbId, season, episode), + timeout(5000, "OpenSubtitles"), + ]), + ]); + + const allCaptions: CaptionListItem[] = []; + + if (wyzieCaptions) allCaptions.push(...wyzieCaptions); + if (openSubsCaptions) allCaptions.push(...openSubsCaptions); + + console.log( + `Found ${allCaptions.length} external captions (Wyzie: ${wyzieCaptions?.length || 0}, OpenSubtitles: ${openSubsCaptions?.length || 0})`, + ); + + return allCaptions; + } catch (error) { + console.error("Error in scrapeExternalSubtitles:", error); + return []; + } +}