Merge branch 'dev' into ee3

package.json

@@ -87,6 +87,7 @@
     "cookie": "^0.6.0",
     "crypto-js": "^4.2.0",
     "form-data": "^4.0.0",
+    "hls-parser": "^0.13.2",
     "iso-639-1": "^3.1.2",
     "nanoid": "^3.3.7",
     "node-fetch": "^3.3.2",

src/entrypoint/controls.ts

@@ -14,6 +14,7 @@ export interface ProviderControlsInput {
   features: FeatureMap;
   sources: Sourcerer[];
   embeds: Embed[];
+  proxyStreams?: boolean; // temporary
 }
 
 export interface RunnerOptions {
@@ -85,6 +86,7 @@ export function makeControls(ops: ProviderControlsInput): ProviderControls {
     features: ops.features,
     fetcher: makeFetcher(ops.fetcher),
     proxiedFetcher: makeFetcher(ops.proxiedFetcher ?? ops.fetcher),
+    proxyStreams: ops.proxyStreams,
   };
 
   return {

src/entrypoint/declare.ts

@@ -18,10 +18,17 @@ export interface ProviderMakerOptions {
   // Set this to true, if the requests will have the same IP as
   // the device that the stream will be played on
   consistentIpForRequests?: boolean;
+
+  // This is temporary
+  proxyStreams?: boolean;
 }
 
 export function makeProviders(ops: ProviderMakerOptions) {
-  const features = getTargetFeatures(ops.target, ops.consistentIpForRequests ?? false);
+  const features = getTargetFeatures(
+    ops.proxyStreams ? 'any' : ops.target,
+    ops.consistentIpForRequests ?? false,
+    ops.proxyStreams,
+  );
   const list = getProviders(features, {
     embeds: getBuiltinEmbeds(),
     sources: getBuiltinSources(),
@@ -33,5 +40,6 @@ export function makeProviders(ops: ProviderMakerOptions) {
     features,
     fetcher: ops.fetcher,
     proxiedFetcher: ops.proxiedFetcher,
+    proxyStreams: ops.proxyStreams,
   });
 }

src/entrypoint/utils/targets.ts

@@ -9,6 +9,10 @@ export const flags = {
   // The source/embed is blocking cloudflare ip's
   // This flag is not compatible with a proxy hosted on cloudflare
   CF_BLOCKED: 'cf-blocked',
+
+  // Streams and sources with this flag wont be proxied
+  // And will be exclusive to the extension
+  PROXY_BLOCKED: 'proxy-blocked',
 } as const;
 
 export type Flags = (typeof flags)[keyof typeof flags];
@@ -53,9 +57,14 @@ export const targetToFeatures: Record<Targets, FeatureMap> = {
   },
 };
 
-export function getTargetFeatures(target: Targets, consistentIpForRequests: boolean): FeatureMap {
+export function getTargetFeatures(
+  target: Targets,
+  consistentIpForRequests: boolean,
+  proxyStreams?: boolean,
+): FeatureMap {
   const features = targetToFeatures[target];
   if (!consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED);
+  if (proxyStreams) features.disallowed.push(flags.PROXY_BLOCKED);
   return features;
 }
 

src/fetchers/common.ts

@@ -15,7 +15,11 @@ export function makeFullUrl(url: string, ops?: FullUrlOptions): string {
   if (rightSide.startsWith('/')) rightSide = rightSide.slice(1);
 
   const fullUrl = leftSide + rightSide;
-  if (!fullUrl.startsWith('http://') && !fullUrl.startsWith('https://'))
+
+  // we need the data scheme for base64 encoded hls playlists
+  // this is for playlists that themselves have cors but not their parts
+  // this allows us to proxy them, encode them into base64 and then fetch the parts normally
+  if (!fullUrl.startsWith('http://') && !fullUrl.startsWith('https://') && !fullUrl.startsWith('data:'))
     throw new Error(`Invald URL -- URL doesn't start with a http scheme: '${fullUrl}'`);
 
   const parsedUrl = new URL(fullUrl);
@@ -35,6 +39,7 @@ export function makeFetcher(fetcher: Fetcher): UseableFetcher {
       baseUrl: ops?.baseUrl ?? '',
       readHeaders: ops?.readHeaders ?? [],
       body: ops?.body,
+      credentials: ops?.credentials,
     });
   };
   const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body;

src/fetchers/fetch.ts

@@ -7,6 +7,7 @@ export type FetchOps = {
   headers: Record<string, string>;
   method: string;
   body: any;
+  credentials?: 'include' | 'same-origin' | 'omit';
 };
 
 export type FetchHeaders = {

src/fetchers/standardFetch.ts

@@ -28,6 +28,7 @@ export function makeStandardFetcher(f: FetchLike): Fetcher {
         ...ops.headers,
       },
       body: seralizedBody.body,
+      credentials: ops.credentials,
     });
 
     let body: any;

src/fetchers/types.ts

@@ -7,6 +7,7 @@ export type FetcherOptions = {
   method?: 'HEAD' | 'GET' | 'POST';
   readHeaders?: string[];
   body?: Record<string, any> | string | FormData | URLSearchParams;
+  credentials?: 'include' | 'same-origin' | 'omit';
 };
 
 // Version of the options that always has the defaults set
@@ -18,6 +19,7 @@ export type DefaultedFetcherOptions = {
   query: Record<string, string>;
   readHeaders: string[];
   method: 'HEAD' | 'GET' | 'POST';
+  credentials?: 'include' | 'same-origin' | 'omit';
 };
 
 export type FetcherResponse<T = any> = {

src/providers/all.ts

@@ -8,12 +8,15 @@ import { mixdropScraper } from '@/providers/embeds/mixdrop';
 import { mp4uploadScraper } from '@/providers/embeds/mp4upload';
 import { streambucketScraper } from '@/providers/embeds/streambucket';
 import { streamsbScraper } from '@/providers/embeds/streamsb';
+import { turbovidScraper } from '@/providers/embeds/turbovid';
 import { upcloudScraper } from '@/providers/embeds/upcloud';
 import { upstreamScraper } from '@/providers/embeds/upstream';
 import { vidsrcembedScraper } from '@/providers/embeds/vidsrc';
 import { vTubeScraper } from '@/providers/embeds/vtube';
+import { astraScraper, novaScraper } from '@/providers/embeds/whvx';
 import { autoembedScraper } from '@/providers/sources/autoembed';
 import { ee3Scraper } from '@/providers/sources/ee3';
+import { catflixScraper } from '@/providers/sources/catflix';
 import { flixhqScraper } from '@/providers/sources/flixhq/index';
 import { goMoviesScraper } from '@/providers/sources/gomovies/index';
 import { insertunitScraper } from '@/providers/sources/insertunit';
@@ -24,6 +27,7 @@ import { remotestreamScraper } from '@/providers/sources/remotestream';
 import { showboxScraper } from '@/providers/sources/showbox/index';
 import { tugaflixScraper } from '@/providers/sources/tugaflix';
 import { vidsrcScraper } from '@/providers/sources/vidsrc/index';
+import { whvxScraper } from '@/providers/sources/whvx';
 import { zoechipScraper } from '@/providers/sources/zoechip';
 
 import {
@@ -66,6 +70,7 @@ import { warezcdnScraper } from './sources/warezcdn';
 export function gatherAllSources(): Array<Sourcerer> {
   // all sources are gathered here
   return [
+    catflixScraper,
     flixhqScraper,
     remotestreamScraper,
     kissAsianScraper,
@@ -90,6 +95,7 @@ export function gatherAllSources(): Array<Sourcerer> {
     autoembedScraper,
     tugaflixScraper,
     ee3Scraper,
+    whvxScraper,
   ];
 }
 
@@ -133,5 +139,8 @@ export function gatherAllEmbeds(): Array<Embed> {
     autoembedBengaliScraper,
     autoembedTamilScraper,
     autoembedTeluguScraper,
+    turbovidScraper,
+    novaScraper,
+    astraScraper,
   ];
 }

src/providers/embeds/filemoon/index.ts

@@ -1,6 +1,8 @@
 import { load } from 'cheerio';
 import { unpack } from 'unpacker';
 
+import { flags } from '@/entrypoint/utils/targets';
+
 import { SubtitleResult } from './types';
 import { makeEmbed } from '../../base';
 import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions';
@@ -51,7 +53,7 @@ export const fileMoonScraper = makeEmbed({
           id: 'primary',
           type: 'hls',
           playlist: file[1],
-          flags: [],
+          flags: [flags.IP_LOCKED],
           captions,
         },
       ],

src/providers/embeds/filemoon/mp4.ts

@@ -1,3 +1,4 @@
+import { flags } from '@/entrypoint/utils/targets';
 import { NotFoundError } from '@/utils/errors';
 
 import { makeEmbed } from '../../base';
@@ -28,7 +29,7 @@ export const fileMoonMp4Scraper = makeEmbed({
               url,
             },
           },
-          flags: [],
+          flags: [flags.IP_LOCKED],
           captions: result.stream[0].captions,
         },
       ],

src/providers/embeds/nsbx.ts

@@ -1,5 +1,4 @@
 import { EmbedOutput, makeEmbed } from '@/providers/base';
-import { baseUrl } from '@/providers/sources/nsbx';
 import { NotFoundError } from '@/utils/errors';
 
 const providers = [
@@ -20,21 +19,30 @@ function embed(provider: { id: string; rank: number }) {
     rank: provider.rank,
     disabled: false,
     async scrape(ctx) {
-      const search = await ctx.proxiedFetcher.full(
-        `${baseUrl}/search?query=${encodeURIComponent(ctx.url)}&provider=${provider.id}`,
-      );
+      const [query, baseUrl] = ctx.url.split('|');
 
-      if (search.statusCode === 429) {
-        throw new Error('Rate limited');
-      } else if (search.statusCode !== 200) {
-        throw new NotFoundError('Failed to search');
-      }
+      const search = await ctx.fetcher.full('/search', {
+        query: {
+          query,
+          provider: provider.id,
+        },
+        credentials: 'include',
+        baseUrl,
+      });
+
+      if (search.statusCode === 429) throw new Error('Rate limited');
+      if (search.statusCode !== 200) throw new NotFoundError('Failed to search');
 
       ctx.progress(50);
 
-      const result = await ctx.proxiedFetcher(
-        `${baseUrl}/provider?resourceId=${encodeURIComponent(search.body.url)}&provider=${provider.id}`,
-      );
+      const result = await ctx.fetcher('/provider', {
+        query: {
+          resourceId: search.body.url,
+          provider: provider.id,
+        },
+        credentials: 'include',
+        baseUrl,
+      });
 
       ctx.progress(100);
 

src/providers/embeds/turbovid.ts

@@ -0,0 +1,79 @@
+import { makeEmbed } from '@/providers/base';
+
+// Thanks to Paradox_77 for helping with the decryption
+function hexToChar(hex: string): string {
+  return String.fromCharCode(parseInt(hex, 16));
+}
+
+function decrypt(data: string, key: string): string {
+  const formatedData = data.match(/../g)?.map(hexToChar).join('') || '';
+  return formatedData
+    .split('')
+    .map((char, i) => String.fromCharCode(char.charCodeAt(0) ^ key.charCodeAt(i % key.length)))
+    .join('');
+}
+
+export const turbovidScraper = makeEmbed({
+  id: 'turbovid',
+  name: 'Turbovid',
+  rank: 122,
+  async scrape(ctx) {
+    const baseUrl = new URL(ctx.url).origin;
+    const embedPage = await ctx.proxiedFetcher(ctx.url);
+
+    ctx.progress(30);
+
+    // the whitespace is for future-proofing the regex a bit
+    const apkey = embedPage.match(/const\s+apkey\s*=\s*"(.*?)";/)?.[1];
+    const xxid = embedPage.match(/const\s+xxid\s*=\s*"(.*?)";/)?.[1];
+
+    if (!apkey || !xxid) throw new Error('Failed to get required values');
+
+    // json isn't parsed by proxiedFetcher due to content-type being text/html
+    const juiceKey = JSON.parse(
+      await ctx.proxiedFetcher('/api/cucked/juice_key', {
+        baseUrl,
+        headers: {
+          referer: ctx.url,
+        },
+      }),
+    ).juice;
+
+    if (!juiceKey) throw new Error('Failed to fetch the key');
+
+    ctx.progress(60);
+
+    const data = JSON.parse(
+      await ctx.proxiedFetcher('/api/cucked/the_juice/', {
+        baseUrl,
+        query: {
+          [apkey]: xxid,
+        },
+        headers: {
+          referer: ctx.url,
+        },
+      }),
+    ).data;
+
+    if (!data) throw new Error('Failed to fetch required data');
+
+    ctx.progress(90);
+
+    const playlist = decrypt(data, juiceKey);
+
+    return {
+      stream: [
+        {
+          type: 'hls',
+          id: 'primary',
+          playlist,
+          headers: {
+            referer: baseUrl,
+          },
+          flags: [],
+          captions: [],
+        },
+      ],
+    };
+  },
+});

src/providers/embeds/vidplay/index.ts

@@ -1,3 +1,4 @@
+import { flags } from '@/entrypoint/utils/targets';
 import { makeEmbed } from '@/providers/base';
 import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
 
@@ -53,7 +54,7 @@ export const vidplayScraper = makeEmbed({
           id: 'primary',
           type: 'hls',
           playlist: source,
-          flags: [],
+          flags: [flags.PROXY_BLOCKED],
           headers: {
             Referer: url.origin,
             Origin: url.origin,

src/providers/embeds/whvx.ts

@@ -0,0 +1,56 @@
+import { EmbedOutput, makeEmbed } from '@/providers/base';
+import { NotFoundError } from '@/utils/errors';
+
+import { baseUrl } from '../sources/whvx';
+
+const providers = [
+  {
+    id: 'nova',
+    name: 'Nova',
+    rank: 701,
+  },
+  {
+    id: 'astra',
+    name: 'Astra',
+    rank: 700,
+  },
+];
+
+function embed(provider: { id: string; name: string; rank: number }) {
+  return makeEmbed({
+    id: provider.id,
+    name: provider.name,
+    rank: provider.rank,
+    disabled: false,
+    async scrape(ctx) {
+      const query = ctx.url;
+
+      const search = await ctx.fetcher.full('/search', {
+        query: {
+          query,
+          provider: provider.id,
+        },
+        baseUrl,
+      });
+
+      if (search.statusCode === 429) throw new Error('Rate limited');
+      if (search.statusCode !== 200) throw new NotFoundError('Failed to search');
+
+      ctx.progress(50);
+
+      const result = await ctx.fetcher('/source', {
+        query: {
+          resourceId: search.body.url,
+          provider: provider.id,
+        },
+        baseUrl,
+      });
+
+      ctx.progress(100);
+
+      return result as EmbedOutput;
+    },
+  });
+}
+
+export const [novaScraper, astraScraper] = providers.map(embed);

src/providers/sources/autoembed.ts

@@ -39,6 +39,7 @@ export const autoembedScraper = makeSourcerer({
   id: 'autoembed',
   name: 'Autoembed',
   rank: 10,
+  disabled: true,
   flags: [flags.CORS_ALLOWED],
   scrapeMovie: comboScraper,
   scrapeShow: comboScraper,

src/providers/sources/catflix.ts

@@ -0,0 +1,74 @@
+import { load } from 'cheerio';
+
+import { SourcererOutput, makeSourcerer } from '@/providers/base';
+import { compareMedia } from '@/utils/compare';
+import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
+import { NotFoundError } from '@/utils/errors';
+
+const baseUrl = 'https://catflix.su';
+
+async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> {
+  const searchPage = await ctx.proxiedFetcher('/', {
+    baseUrl,
+    query: {
+      s: ctx.media.title,
+    },
+  });
+
+  ctx.progress(40);
+
+  const $search = load(searchPage);
+  const searchResults: { title: string; year?: number | undefined; url: string }[] = [];
+
+  $search('li').each((_, element) => {
+    const title = $search(element).find('h2').first().text().trim();
+    // the year is always present, but I sitll decided to make it nullable since the impl isn't as future-proof
+    const year = Number($search(element).find('.text-xs > span').eq(1).text().trim()) || undefined;
+    const url = $search(element).find('a').attr('href');
+
+    if (!title || !url) return;
+
+    searchResults.push({ title, year, url });
+  });
+
+  let watchPageUrl = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url;
+  if (!watchPageUrl) throw new NotFoundError('No watchable item found');
+
+  ctx.progress(60);
+
+  if (ctx.media.type === 'show') {
+    const match = watchPageUrl.match(/\/series\/([^/]+)\/?/);
+    if (!match) throw new Error('Failed to parse watch page url');
+    watchPageUrl = watchPageUrl.replace(
+      `/series/${match[1]}`,
+      `/episode/${match[1]}-${ctx.media.season.number}x${ctx.media.episode.number}`,
+    );
+  }
+
+  const watchPage = load(await ctx.proxiedFetcher(watchPageUrl));
+
+  ctx.progress(80);
+
+  const url = watchPage('iframe').first().attr('src'); // I couldn't think of a better way
+  if (!url) throw new Error('Failed to find embed url');
+
+  ctx.progress(90);
+
+  return {
+    embeds: [
+      {
+        embedId: 'turbovid',
+        url,
+      },
+    ],
+  };
+}
+
+export const catflixScraper = makeSourcerer({
+  id: 'catflix',
+  name: 'Catflix',
+  rank: 122,
+  flags: [],
+  scrapeMovie: comboScraper,
+  scrapeShow: comboScraper,
+});

src/providers/sources/nepu/index.ts

@@ -1,5 +1,6 @@
 import { load } from 'cheerio';
 
+import { flags } from '@/entrypoint/utils/targets';
 import { SourcererOutput, makeSourcerer } from '@/providers/base';
 import { compareTitle } from '@/utils/compare';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
@@ -7,8 +8,8 @@ import { NotFoundError } from '@/utils/errors';
 
 import { SearchResults } from './types';
 
-const nepuBase = 'https://nepu.to';
-const nepuReferer = `${nepuBase}/`;
+const nepuBase = 'https://nepu.io';
+const nepuReferer = 'https://nepu.to';
 
 const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => {
   const searchResultRequest = await ctx.proxiedFetcher<string>('/ajax/posts', {
@@ -63,11 +64,11 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) =>
         captions: [],
         playlist: streamUrl[1],
         type: 'hls',
-        flags: [],
         headers: {
-          Origin: nepuBase,
-          Referer: nepuReferer,
+          Origin: nepuReferer,
+          Referer: `${nepuReferer}/`,
         },
+        flags: [],
       },
     ],
   } as SourcererOutput;
@@ -77,8 +78,8 @@ export const nepuScraper = makeSourcerer({
   id: 'nepu',
   name: 'Nepu',
   rank: 80,
-  flags: [],
   disabled: true,
+  flags: [],
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,
 });

src/providers/sources/nsbx.ts

@@ -3,8 +3,6 @@ import { SourcererOutput, makeSourcerer } from '@/providers/base';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
 
-export const baseUrl = 'https://api.nsbx.ru';
-
 async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> {
   const query = {
     title: ctx.media.title,
@@ -12,25 +10,21 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
     tmdbId: ctx.media.tmdbId,
     imdbId: ctx.media.imdbId,
     type: ctx.media.type,
-    season: '',
-    episode: '',
+    ...(ctx.media.type === 'show' && {
+      season: ctx.media.season.number.toString(),
+      episode: ctx.media.episode.number.toString(),
+    }),
   };
 
-  if (ctx.media.type === 'show') {
-    query.season = ctx.media.season.number.toString();
-    query.episode = ctx.media.episode.number.toString();
-  }
+  const res: { providers: string[]; endpoint: string } = await ctx.fetcher('https://api.nsbx.ru/status');
 
-  const res = await ctx.proxiedFetcher(`${baseUrl}/status`);
-
-  if (res.providers?.length === 0) {
-    throw new NotFoundError('No providers available');
-  }
+  if (res.providers?.length === 0) throw new NotFoundError('No providers available');
+  if (!res.endpoint) throw new Error('No endpoint returned');
 
   const embeds = res.providers.map((provider: string) => {
     return {
       embedId: provider,
-      url: JSON.stringify(query),
+      url: `${JSON.stringify(query)}|${res.endpoint}`,
     };
   });
 
@@ -42,7 +36,7 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
 export const nsbxScraper = makeSourcerer({
   id: 'nsbx',
   name: 'NSBX',
-  rank: 150,
+  rank: 129,
   flags: [flags.CORS_ALLOWED],
   disabled: false,
   scrapeMovie: comboScraper,

src/providers/sources/smashystream/index.ts

@@ -11,17 +11,17 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr
   const query =
     ctx.media.type === 'movie'
       ? `?tmdb=${ctx.media.tmdbId}`
-      : `?tmdbId=${ctx.media.tmdbId}&season=${ctx.media.season.number}&episode=${ctx.media.episode.number}`;
+      : `?tmdb=${ctx.media.tmdbId}&season=${ctx.media.season.number}&episode=${ctx.media.episode.number}`;
 
   return {
     embeds: [
       {
         embedId: smashyStreamFScraper.id,
-        url: `https://embed.smashystream.com/video1dn.php${query}`,
+        url: `https://embed.smashystream.com/videofeee.php${query}`,
       },
       {
         embedId: smashyStreamOScraper.id,
-        url: `https://embed.smashystream.com/videoop.php${query}`,
+        url: `https://embed.smashystream.com/shortmoviec.php${query}`,
       },
     ],
   };
@@ -31,6 +31,7 @@ export const smashyStreamScraper = makeSourcerer({
   id: 'smashystream',
   name: 'SmashyStream',
   rank: 30,
+  disabled: true,
   flags: [flags.CORS_ALLOWED],
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,

src/providers/sources/soapertv/index.ts

@@ -1,7 +1,7 @@
 import { load } from 'cheerio';
 
-import { flags } from '@/entrypoint/utils/targets';
 import { Caption, labelToLanguageCode } from '@/providers/captions';
+import { Stream } from '@/providers/streams';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
 
@@ -41,13 +41,11 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Pr
   const contentPage$ = load(contentPage);
 
   const pass = contentPage$('#hId').attr('value');
-  const param = contentPage$('#divU').text();
 
-  if (!pass || !param) throw new NotFoundError('Content not found');
+  if (!pass) throw new NotFoundError('Content not found');
 
   const formData = new URLSearchParams();
   formData.append('pass', pass);
-  formData.append('param', param);
   formData.append('e2', '0');
   formData.append('server', '0');
 
@@ -90,20 +88,22 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Pr
     stream: [
       {
         id: 'primary',
-        playlist: streamResJson.val,
+        playlist: `${baseUrl}/${streamResJson.val}`,
         type: 'hls',
-        flags: [flags.IP_LOCKED],
+        proxyDepth: 2,
+        flags: [],
         captions,
       },
       ...(streamResJson.val_bak
         ? [
             {
               id: 'backup',
-              playlist: streamResJson.val_bak,
-              type: 'hls' as const,
-              flags: [flags.IP_LOCKED],
+              playlist: `${baseUrl}/${streamResJson.val_bak}`,
+              type: 'hls',
+              proxyDepth: 2,
+              flags: [],
               captions,
-            },
+            } as Stream,
           ]
         : []),
     ],
@@ -113,8 +113,8 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Pr
 export const soaperTvScraper = makeSourcerer({
   id: 'soapertv',
   name: 'SoaperTV',
-  rank: 115,
-  flags: [flags.IP_LOCKED],
+  rank: 126,
+  flags: [],
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,
 });

src/providers/sources/vidsrcto/index.ts

@@ -1,5 +1,6 @@
 import { load } from 'cheerio';
 
+import { flags } from '@/entrypoint/utils/targets';
 import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
 
@@ -83,6 +84,6 @@ export const vidSrcToScraper = makeSourcerer({
   name: 'VidSrcTo',
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,
-  flags: [],
+  flags: [flags.PROXY_BLOCKED],
   rank: 130,
 });

src/providers/sources/whvx.ts

@@ -0,0 +1,44 @@
+import { flags } from '@/entrypoint/utils/targets';
+import { SourcererOutput, makeSourcerer } from '@/providers/base';
+import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
+import { NotFoundError } from '@/utils/errors';
+
+export const baseUrl = 'https://api.whvx.net';
+
+async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> {
+  const query = {
+    title: ctx.media.title,
+    releaseYear: ctx.media.releaseYear,
+    tmdbId: ctx.media.tmdbId,
+    imdbId: ctx.media.imdbId,
+    type: ctx.media.type,
+    ...(ctx.media.type === 'show' && {
+      season: ctx.media.season.number.toString(),
+      episode: ctx.media.episode.number.toString(),
+    }),
+  };
+
+  const res: { providers: string[] } = await ctx.fetcher('/status', { baseUrl });
+
+  if (res.providers?.length === 0) throw new NotFoundError('No providers available');
+
+  const embeds = res.providers.map((provider: string) => {
+    return {
+      embedId: provider,
+      url: JSON.stringify(query),
+    };
+  });
+
+  return {
+    embeds,
+  };
+}
+
+export const whvxScraper = makeSourcerer({
+  id: 'whvx',
+  name: 'VidBinge',
+  rank: 128,
+  flags: [flags.CORS_ALLOWED],
+  scrapeMovie: comboScraper,
+  scrapeShow: comboScraper,
+});

src/providers/streams.ts

@@ -30,6 +30,7 @@ export type FileBasedStream = StreamCommon & {
 export type HlsBasedStream = StreamCommon & {
   type: 'hls';
   playlist: string;
+  proxyDepth?: 0 | 1 | 2;
 };
 
 export type Stream = FileBasedStream | HlsBasedStream;

src/runners/individualRunner.ts

@@ -7,6 +7,7 @@ import { ProviderList } from '@/providers/get';
 import { ScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
 import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles';
+import { requiresProxy, setupProxy } from '@/utils/proxy';
 import { isValidStream, validatePlayableStreams } from '@/utils/valid';
 
 export type IndividualSourceRunnerOptions = {
@@ -16,6 +17,7 @@ export type IndividualSourceRunnerOptions = {
   media: ScrapeMedia;
   id: string;
   events?: IndividualScraperEvents;
+  proxyStreams?: boolean; // temporary
 };
 
 export async function scrapeInvidualSource(
@@ -56,6 +58,10 @@ export async function scrapeInvidualSource(
     output.stream = output.stream
       .filter((stream) => isValidStream(stream))
       .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
+
+    output.stream = output.stream.map((stream) =>
+      requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
+    );
   }
 
   if (!output) throw new Error('output is null');
@@ -107,6 +113,7 @@ export type IndividualEmbedRunnerOptions = {
   url: string;
   id: string;
   events?: IndividualScraperEvents;
+  proxyStreams?: boolean; // temporary
 };
 
 export async function scrapeIndividualEmbed(
@@ -138,6 +145,10 @@ export async function scrapeIndividualEmbed(
     .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
   if (output.stream.length === 0) throw new NotFoundError('No streams found');
 
+  output.stream = output.stream.map((stream) =>
+    requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
+  );
+
   const playableStreams = await validatePlayableStreams(output.stream, ops, embedScraper.id);
   if (playableStreams.length === 0) throw new NotFoundError('No playable streams found');
 

src/runners/runner.ts

@@ -9,6 +9,7 @@ import { ScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
 import { reorderOnIdList } from '@/utils/list';
 import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles';
+import { requiresProxy, setupProxy } from '@/utils/proxy';
 import { isValidStream, validatePlayableStream } from '@/utils/valid';
 
 export type RunOutput = {
@@ -36,6 +37,7 @@ export type ProviderRunnerOptions = {
   embedOrder?: string[];
   events?: FullScraperEvents;
   media: ScrapeMedia;
+  proxyStreams?: boolean; // temporary
 };
 
 export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOptions): Promise<RunOutput | null> {
@@ -85,6 +87,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
         output.stream = (output.stream ?? [])
           .filter(isValidStream)
           .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
+
+        output.stream = output.stream.map((stream) =>
+          requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
+        );
       }
       if (!output || (!output.stream?.length && !output.embeds.length)) {
         throw new NotFoundError('No streams found');
@@ -161,6 +167,9 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
         embedOutput.stream = embedOutput.stream
           .filter(isValidStream)
           .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
+        embedOutput.stream = embedOutput.stream.map((stream) =>
+          requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
+        );
         if (embedOutput.stream.length === 0) {
           throw new NotFoundError('No streams found');
         }

src/utils/playlist.ts

@@ -0,0 +1,25 @@
+import { parse, stringify } from 'hls-parser';
+import { MasterPlaylist } from 'hls-parser/types';
+
+import { UseableFetcher } from '@/fetchers/types';
+
+export async function convertPlaylistsToDataUrls(
+  fetcher: UseableFetcher,
+  playlistUrl: string,
+  headers?: Record<string, string>,
+) {
+  const playlistData = await fetcher(playlistUrl, { headers });
+  const playlist = parse(playlistData);
+
+  if (playlist.isMasterPlaylist) {
+    await Promise.all(
+      (playlist as MasterPlaylist).variants.map(async (variant) => {
+        const variantPlaylistData = await fetcher(variant.uri, { headers });
+        const variantPlaylist = parse(variantPlaylistData);
+        variant.uri = `data:application/vnd.apple.mpegurl;base64,${btoa(stringify(variantPlaylist))}`;
+      }),
+    );
+  }
+
+  return `data:application/vnd.apple.mpegurl;base64,${btoa(stringify(playlist))}`;
+}

src/utils/proxy.ts

@@ -0,0 +1,44 @@
+import { flags } from '@/entrypoint/utils/targets';
+import { Stream } from '@/providers/streams';
+
+export function requiresProxy(stream: Stream): boolean {
+  if (!stream.flags.includes(flags.CORS_ALLOWED) && !!(stream.headers && Object.keys(stream.headers).length > 0))
+    return true;
+  return false;
+}
+
+export function setupProxy(stream: Stream): Stream {
+  const headers = stream.headers && Object.keys(stream.headers).length > 0 ? stream.headers : undefined;
+
+  const options = {
+    ...(stream.type === 'hls' && { depth: stream.proxyDepth ?? 0 }),
+  };
+
+  const payload: {
+    type?: 'hls' | 'mp4';
+    url?: string;
+    headers?: Record<string, string>;
+    options?: { depth?: 0 | 1 | 2 };
+  } = {
+    headers,
+    options,
+  };
+
+  if (stream.type === 'hls') {
+    payload.type = 'hls';
+    payload.url = stream.playlist;
+    stream.playlist = `https://proxy.nsbx.ru/proxy?${new URLSearchParams({ payload: Buffer.from(JSON.stringify(payload)).toString('base64url') })}`;
+  }
+
+  if (stream.type === 'file') {
+    payload.type = 'mp4';
+    Object.entries(stream.qualities).forEach((entry) => {
+      payload.url = entry[1].url;
+      entry[1].url = `https://proxy.nsbx.ru/proxy?${new URLSearchParams({ payload: Buffer.from(JSON.stringify(payload)).toString('base64url') })}`;
+    });
+  }
+
+  stream.headers = {};
+  stream.flags = [flags.CORS_ALLOWED];
+  return stream;
+}

src/utils/valid.ts

@@ -1,10 +1,17 @@
 import { alphaScraper, deltaScraper } from '@/providers/embeds/nsbx';
 import { warezcdnembedMp4Scraper } from '@/providers/embeds/warezcdn/mp4';
+import { astraScraper, novaScraper } from '@/providers/embeds/whvx';
 import { Stream } from '@/providers/streams';
 import { IndividualEmbedRunnerOptions } from '@/runners/individualRunner';
 import { ProviderRunnerOptions } from '@/runners/runner';
 
-const SKIP_VALIDATION_CHECK_IDS = [warezcdnembedMp4Scraper.id, deltaScraper.id, alphaScraper.id];
+const SKIP_VALIDATION_CHECK_IDS = [
+  warezcdnembedMp4Scraper.id,
+  deltaScraper.id,
+  alphaScraper.id,
+  novaScraper.id,
+  astraScraper.id,
+];
 
 export function isValidStream(stream: Stream | undefined): boolean {
   if (!stream) return false;
@@ -30,6 +37,9 @@ export async function validatePlayableStream(
   if (SKIP_VALIDATION_CHECK_IDS.includes(sourcererId)) return stream;
 
   if (stream.type === 'hls') {
+    // dirty temp fix for base64 urls to prep for fmhy poll
+    if (stream.playlist.startsWith('data:')) return stream;
+
     const result = await ops.proxiedFetcher.full(stream.playlist, {
       method: 'GET',
       headers: {