move external subtitle scrape to client

package.json

@@ -75,6 +75,7 @@
     "semver": "^7.7.2",
     "slugify": "^1.6.6",
     "subsrt-ts": "^2.1.2",
+    "wyzie-lib": "^2.2.5",
     "zustand": "^4.5.7"
   },
   "devDependencies": {

pnpm-lock.yaml

@@ -156,6 +156,9 @@ importers:
       subsrt-ts:
         specifier: ^2.1.2
         version: 2.1.2
+      wyzie-lib:
+        specifier: ^2.2.5
+        version: 2.2.5
       zustand:
         specifier: ^4.5.7
         version: 4.5.7(@types/react@18.3.23)(immer@10.1.1)(react@18.3.1)

src/assets/locales/en.json

@@ -687,6 +687,8 @@
         "unknownLanguage": "Unknown",
         "dropSubtitleFile": "Drop subtitle file here! >_<",
         "scrapeButton": "Scrape subtitles",
+        "refresh": "Refresh External Subtitles",
+        "refreshing": "Refreshing...",
         "empty": "There are no provided subtitles for this.",
         "notFound": "None of the available options match your query",
         "useNativeSubtitles": "Use native video subtitles",

src/components/player/atoms/settings/OpensubtitlesCaptionsView.tsx

@@ -27,6 +27,7 @@ export function OpenSubtitlesCaptionView({
   const { selectCaptionById } = useCaptions();
   const captionList = usePlayerStore((s) => s.captionList);
   const getHlsCaptionList = usePlayerStore((s) => s.display?.getCaptionList);
+  const addExternalSubtitles = usePlayerStore((s) => s.addExternalSubtitles);
 
   const captions = useMemo(
     () =>
@@ -48,6 +49,10 @@ export function OpenSubtitlesCaptionView({
     [selectCaptionById, setCurrentlyDownloading],
   );
 
+  const [refreshReq, startRefresh] = useAsyncFn(async () => {
+    return addExternalSubtitles();
+  }, [addExternalSubtitles]);
+
   const content = subtitleList.length
     ? subtitleList.map((v) => {
         return (
@@ -98,6 +103,14 @@ export function OpenSubtitlesCaptionView({
           <div className="p-4 rounded-xl bg-video-context-light bg-opacity-10 font-medium text-center">
             <div className="flex flex-col items-center justify-center gap-3">
               {t("player.menus.subtitles.empty")}
+              <button
+                type="button"
+                onClick={() => startRefresh()}
+                disabled={refreshReq.loading}
+                className="p-1 w-3/4 rounded tabbable duration-200 bg-opacity-10 bg-video-context-light hover:bg-opacity-20"
+              >
+                {t("player.menus.subtitles.scrapeButton")}
+              </button>
             </div>
           </div>
         ) : (

src/stores/player/slices/source.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-console */
 import { ScrapeMedia } from "@p-stream/providers";
 
 import { MakeSlice } from "@/stores/player/slices/types";
@@ -98,6 +99,7 @@ export interface SourceSlice {
   enableAutomaticQuality(): void;
   redisplaySource(startAt: number): void;
   setCaptionAsTrack(asTrack: boolean): void;
+  addExternalSubtitles(): Promise<void>;
 }
 
 export function metaToScrapeMedia(meta: PlayerMeta): ScrapeMedia {
@@ -184,6 +186,12 @@ export const createSourceSlice: MakeSlice<SourceSlice> = (set, get) => ({
     });
     const store = get();
     store.redisplaySource(startAt);
+
+    // Trigger external subtitle scraping after stream is loaded
+    // This runs asynchronously so it doesn't block the stream loading
+    setTimeout(() => {
+      store.addExternalSubtitles();
+    }, 100);
   },
   redisplaySource(startAt: number) {
     const store = get();
@@ -235,4 +243,29 @@ export const createSourceSlice: MakeSlice<SourceSlice> = (set, get) => ({
       s.caption.asTrack = asTrack;
     });
   },
+  async addExternalSubtitles() {
+    const store = get();
+    if (!store.meta) return;
+
+    try {
+      const { scrapeExternalSubtitles } = await import(
+        "@/utils/externalSubtitles"
+      );
+      const externalCaptions = await scrapeExternalSubtitles(store.meta);
+
+      if (externalCaptions.length > 0) {
+        set((s) => {
+          // Add external captions to the existing list, avoiding duplicates
+          const existingIds = new Set(s.captionList.map((c) => c.id));
+          const newCaptions = externalCaptions.filter(
+            (c) => !existingIds.has(c.id),
+          );
+          s.captionList = [...s.captionList, ...newCaptions];
+        });
+        console.log(`Added ${externalCaptions.length} external captions`);
+      }
+    } catch (error) {
+      console.error("Failed to scrape external subtitles:", error);
+    }
+  },
 });

src/utils/externalSubtitles.ts

@@ -0,0 +1,256 @@
+/* eslint-disable no-console */
+import { type SubtitleData, searchSubtitles } from "wyzie-lib";
+
+import { CaptionListItem, PlayerMeta } from "@/stores/player/slices/source";
+
+// Helper function to convert language names to language codes
+function labelToLanguageCode(languageName: string): string {
+  const languageMap: Record<string, string> = {
+    English: "en",
+    Spanish: "es",
+    French: "fr",
+    German: "de",
+    Italian: "it",
+    Portuguese: "pt",
+    Russian: "ru",
+    Japanese: "ja",
+    Korean: "ko",
+    Chinese: "zh",
+    Arabic: "ar",
+    Hindi: "hi",
+    Turkish: "tr",
+    Dutch: "nl",
+    Polish: "pl",
+    Swedish: "sv",
+    Norwegian: "no",
+    Danish: "da",
+    Finnish: "fi",
+    Greek: "el",
+    Hebrew: "he",
+    Thai: "th",
+    Vietnamese: "vi",
+    Indonesian: "id",
+    Malay: "ms",
+    Filipino: "tl",
+    Ukrainian: "uk",
+    Romanian: "ro",
+    Czech: "cs",
+    Hungarian: "hu",
+    Bulgarian: "bg",
+    Croatian: "hr",
+    Serbian: "sr",
+    Slovak: "sk",
+    Slovenian: "sl",
+    Estonian: "et",
+    Latvian: "lv",
+    Lithuanian: "lt",
+    Icelandic: "is",
+    Maltese: "mt",
+    Georgian: "ka",
+    Armenian: "hy",
+    Azerbaijani: "az",
+    Kazakh: "kk",
+    Kyrgyz: "ky",
+    Uzbek: "uz",
+    Tajik: "tg",
+    Turkmen: "tk",
+    Mongolian: "mn",
+    Persian: "fa",
+    Urdu: "ur",
+    Bengali: "bn",
+    Tamil: "ta",
+    Telugu: "te",
+    Marathi: "mr",
+    Gujarati: "gu",
+    Kannada: "kn",
+    Malayalam: "ml",
+    Punjabi: "pa",
+    Sinhala: "si",
+    Nepali: "ne",
+    Burmese: "my",
+    Khmer: "km",
+    Lao: "lo",
+    Tibetan: "bo",
+    Uyghur: "ug",
+    Kurdish: "ku",
+    Pashto: "ps",
+    Dari: "prs",
+    Sindhi: "sd",
+    Kashmiri: "ks",
+    Dogri: "doi",
+    Konkani: "kok",
+    Manipuri: "mni",
+    Bodo: "brx",
+    Sanskrit: "sa",
+    Santhali: "sat",
+    Maithili: "mai",
+    Bhojpuri: "bho",
+    Awadhi: "awa",
+    Chhattisgarhi: "hne",
+    Magahi: "mag",
+    Rajasthani: "raj",
+    Malvi: "mup",
+    Bundeli: "bns",
+    Bagheli: "bfy",
+    Pahari: "phr",
+    Kumaoni: "kfy",
+    Garhwali: "gbm",
+    Kangri: "xnr",
+  };
+
+  return languageMap[languageName] || languageName.toLowerCase();
+}
+
+const timeout = (ms: number, source: string) =>
+  new Promise<null>((resolve) => {
+    setTimeout(() => {
+      console.error(`${source} captions request timed out after ${ms}ms`);
+      resolve(null);
+    }, ms);
+  });
+
+export async function scrapeWyzieCaptions(
+  tmdbId: string | number,
+  imdbId: string,
+  season?: number,
+  episode?: number,
+): Promise<CaptionListItem[]> {
+  try {
+    const searchParams: any = {
+      encoding: "utf-8",
+      source: "all",
+      imdb_id: imdbId,
+    };
+
+    if (tmdbId && !imdbId) {
+      searchParams.tmdb_id =
+        typeof tmdbId === "string" ? parseInt(tmdbId, 10) : tmdbId;
+    }
+
+    if (season && episode) {
+      searchParams.season = season;
+      searchParams.episode = episode;
+    }
+
+    console.log("Searching Wyzie subtitles with params:", searchParams);
+    const wyzieSubtitles: SubtitleData[] = await searchSubtitles(searchParams);
+
+    const wyzieCaptions: CaptionListItem[] = wyzieSubtitles.map((subtitle) => ({
+      id: subtitle.id,
+      language: subtitle.language,
+      url: subtitle.url,
+      type:
+        subtitle.format === "srt" || subtitle.format === "vtt"
+          ? subtitle.format
+          : "srt",
+      needsProxy: false,
+      opensubtitles: true,
+      // Additional metadata from Wyzie
+      display: subtitle.display,
+      media: subtitle.media,
+      isHearingImpaired: subtitle.isHearingImpaired,
+      source:
+        typeof subtitle.source === "number"
+          ? subtitle.source.toString()
+          : subtitle.source,
+      encoding: subtitle.encoding,
+    }));
+
+    return wyzieCaptions;
+  } catch (error) {
+    console.error("Error fetching Wyzie subtitles:", error);
+    return [];
+  }
+}
+
+export async function scrapeOpenSubtitlesCaptions(
+  imdbId: string,
+  season?: number,
+  episode?: number,
+): Promise<CaptionListItem[]> {
+  try {
+    const url = `https://rest.opensubtitles.org/search/${
+      season && episode ? `episode-${episode}/` : ""
+    }imdbid-${imdbId.slice(2)}${season && episode ? `/season-${season}` : ""}`;
+
+    const response = await fetch(url, {
+      headers: {
+        "X-User-Agent": "VLSub 0.10.2",
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`OpenSubtitles API returned ${response.status}`);
+    }
+
+    const data = await response.json();
+    const openSubtitlesCaptions: CaptionListItem[] = [];
+
+    for (const caption of data) {
+      const downloadUrl = caption.SubDownloadLink.replace(".gz", "").replace(
+        "download/",
+        "download/subencoding-utf8/",
+      );
+      const language = labelToLanguageCode(caption.LanguageName);
+
+      if (!downloadUrl || !language) continue;
+
+      openSubtitlesCaptions.push({
+        id: downloadUrl,
+        language,
+        url: downloadUrl,
+        type: caption.SubFormat || "srt",
+        needsProxy: false,
+        opensubtitles: true,
+      });
+    }
+
+    return openSubtitlesCaptions;
+  } catch (error) {
+    console.error("Error fetching OpenSubtitles:", error);
+    return [];
+  }
+}
+
+export async function scrapeExternalSubtitles(
+  meta: PlayerMeta,
+): Promise<CaptionListItem[]> {
+  try {
+    // Extract IMDb ID from meta
+    const imdbId = meta.imdbId;
+    if (!imdbId) {
+      console.log("No IMDb ID available for external subtitle scraping");
+      return [];
+    }
+
+    const season = meta.season?.number;
+    const episode = meta.episode?.number;
+    const tmdbId = meta.tmdbId;
+
+    // Fetch both Wyzie and OpenSubtitles captions with timeouts
+    const [wyzieCaptions, openSubsCaptions] = await Promise.all([
+      Promise.race([
+        scrapeWyzieCaptions(tmdbId, imdbId, season, episode),
+        timeout(2000, "Wyzie"),
+      ]),
+      Promise.race([
+        scrapeOpenSubtitlesCaptions(imdbId, season, episode),
+        timeout(5000, "OpenSubtitles"),
+      ]),
+    ]);
+
+    const allCaptions: CaptionListItem[] = [];
+
+    if (wyzieCaptions) allCaptions.push(...wyzieCaptions);
+    if (openSubsCaptions) allCaptions.push(...openSubsCaptions);
+
+    console.log(
+      `Found ${allCaptions.length} external captions (Wyzie: ${wyzieCaptions?.length || 0}, OpenSubtitles: ${openSubsCaptions?.length || 0})`,
+    );
+
+    return allCaptions;
+  } catch (error) {
+    console.error("Error in scrapeExternalSubtitles:", error);
+    return [];
+  }
+}