mirror of
https://github.com/p-stream/providers.git
synced 2026-01-11 20:10:33 +00:00
fix docs
This commit is contained in:
parent
562ee54e1c
commit
add7dc5b3f
30 changed files with 2306 additions and 3672 deletions
|
|
@ -3,6 +3,6 @@ module.exports = {
|
||||||
extends: '@nuxt/eslint-config',
|
extends: '@nuxt/eslint-config',
|
||||||
rules: {
|
rules: {
|
||||||
'vue/max-attributes-per-line': 'off',
|
'vue/max-attributes-per-line': 'off',
|
||||||
'vue/multi-word-component-names': 'off',
|
'vue/multi-word-component-names': 'off'
|
||||||
},
|
}
|
||||||
};
|
}
|
||||||
|
|
@ -1,20 +1,18 @@
|
||||||
---
|
---
|
||||||
title: '@p-stream/providers | For all your media scraping needs'
|
title: "@p-stream/providers | For all your media scraping needs"
|
||||||
navigation: false
|
navigation: false
|
||||||
layout: page
|
layout: page
|
||||||
---
|
---
|
||||||
|
|
||||||
## ::block-hero
|
::block-hero
|
||||||
|
---
|
||||||
cta:
|
cta:
|
||||||
|
- Get Started
|
||||||
- Get Started
|
- /get-started/introduction
|
||||||
- /get-started/introduction
|
secondary:
|
||||||
secondary:
|
- Open on GitHub →
|
||||||
- Open on GitHub →
|
- https://github.com/p-stream/providers
|
||||||
- https://github.com/p-stream/providers
|
snippet: npm i @p-stream/providers@github:p-stream/providers
|
||||||
snippet: npm i @p-stream/providers@github:p-stream/providers
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
#title
|
#title
|
||||||
|
|
@ -32,22 +30,22 @@ What's included
|
||||||
:ellipsis
|
:ellipsis
|
||||||
|
|
||||||
#default
|
#default
|
||||||
::card{icon="vscode-icons:file-type-light-json"}
|
::card{icon="vscode-icons:file-type-light-json"}
|
||||||
#title
|
#title
|
||||||
Scrape popular streaming websites.
|
Scrape popular streaming websites.
|
||||||
#description
|
#description
|
||||||
Don't settle for just one media site for you content, use everything that's available.
|
Don't settle for just one media site for you content, use everything that's available.
|
||||||
::
|
::
|
||||||
::card{icon="codicon:source-control"}
|
::card{icon="codicon:source-control"}
|
||||||
#title
|
#title
|
||||||
Multi-platform.
|
Multi-platform.
|
||||||
#description
|
#description
|
||||||
Scrape from browser or server, whichever you prefer.
|
Scrape from browser or server, whichever you prefer.
|
||||||
::
|
::
|
||||||
::card{icon="logos:typescript-icon-round"}
|
::card{icon="logos:typescript-icon-round"}
|
||||||
#title
|
#title
|
||||||
Easy to use.
|
Easy to use.
|
||||||
#description
|
#description
|
||||||
Get started with scraping your favourite media sites with just 5 lines of code. Fully typed of course.
|
Get started with scraping your favourite media sites with just 5 lines of code. Fully typed of course.
|
||||||
::
|
::
|
||||||
::
|
::
|
||||||
|
|
|
||||||
|
|
@ -3,35 +3,29 @@ title: 'Changelog'
|
||||||
---
|
---
|
||||||
|
|
||||||
# Version 2.3.0
|
# Version 2.3.0
|
||||||
|
|
||||||
- Fixed RidoMovies search results
|
- Fixed RidoMovies search results
|
||||||
- Added Insertunit, SoaperTV, and WarezCDN providers
|
- Added Insertunit, SoaperTV, and WarezCDN providers
|
||||||
- Disabled Showbox and VidSrc
|
- Disabled Showbox and VidSrc
|
||||||
|
|
||||||
# Version 2.2.9
|
# Version 2.2.9
|
||||||
|
|
||||||
- Fixed VidSrcTo (both Vidplay and Filemoon embeds)
|
- Fixed VidSrcTo (both Vidplay and Filemoon embeds)
|
||||||
- Added dropload, filelions and vtube embeds to Primewire
|
- Added dropload, filelions and vtube embeds to Primewire
|
||||||
- Fixed and enabled Smashystream
|
- Fixed and enabled Smashystream
|
||||||
- Improved RidoMovies search results
|
- Improved RidoMovies search results
|
||||||
|
|
||||||
# Version 2.2.8
|
# Version 2.2.8
|
||||||
|
|
||||||
- Fix package exports for CJS and ESM
|
- Fix package exports for CJS and ESM
|
||||||
- Fixed Mixdrop embed
|
- Fixed Mixdrop embed
|
||||||
- Added thumbnailTrack to Vidplay embed
|
- Added thumbnailTrack to Vidplay embed
|
||||||
|
|
||||||
# Version 2.2.7
|
# Version 2.2.7
|
||||||
|
|
||||||
- Fix showbox
|
- Fix showbox
|
||||||
|
|
||||||
# Version 2.2.6
|
# Version 2.2.6
|
||||||
|
|
||||||
- Fix febbox
|
- Fix febbox
|
||||||
- Validate if a stream is actually playable. Streams that are not responding are no longer returned.
|
- Validate if a stream is actually playable. Streams that are not responding are no longer returned.
|
||||||
|
|
||||||
# Version 2.2.5
|
# Version 2.2.5
|
||||||
|
|
||||||
- Add Primewire provider
|
- Add Primewire provider
|
||||||
- Improve VidSrcTo search results
|
- Improve VidSrcTo search results
|
||||||
- Fixed Filemoon embeds
|
- Fixed Filemoon embeds
|
||||||
|
|
@ -40,11 +34,9 @@ title: 'Changelog'
|
||||||
- Reordered providers in ranking
|
- Reordered providers in ranking
|
||||||
|
|
||||||
# Version 2.2.4
|
# Version 2.2.4
|
||||||
|
|
||||||
- Hotfix for HDRezka provider
|
- Hotfix for HDRezka provider
|
||||||
|
|
||||||
# Version 2.2.3
|
# Version 2.2.3
|
||||||
|
|
||||||
- Fix VidSrcTo
|
- Fix VidSrcTo
|
||||||
- Add HDRezka provider
|
- Add HDRezka provider
|
||||||
- Fix Goojara causing a crash
|
- Fix Goojara causing a crash
|
||||||
|
|
@ -52,18 +44,15 @@ title: 'Changelog'
|
||||||
- Cover an edge case where the title contains 'the movie' or 'the show'
|
- Cover an edge case where the title contains 'the movie' or 'the show'
|
||||||
|
|
||||||
# Version 2.2.2
|
# Version 2.2.2
|
||||||
|
|
||||||
- Fix subtitles not appearing if the name of the subtitle is in its native tongue.
|
- Fix subtitles not appearing if the name of the subtitle is in its native tongue.
|
||||||
- Remove references to the old domain
|
- Remove references to the old domain
|
||||||
- Fixed ridomovies not working for some shows and movies
|
- Fixed ridomovies not working for some shows and movies
|
||||||
- Fixed Showbox not working in react-native.
|
- Fixed Showbox not working in react-native.
|
||||||
|
|
||||||
# Version 2.2.1
|
# Version 2.2.1
|
||||||
|
|
||||||
- Fixed Closeload scraper
|
- Fixed Closeload scraper
|
||||||
|
|
||||||
# Version 2.2.0
|
# Version 2.2.0
|
||||||
|
|
||||||
- Fixed vidsrc.me URL decoding.
|
- Fixed vidsrc.me URL decoding.
|
||||||
- Added ridomovies with Ridoo and Closeload embed.
|
- Added ridomovies with Ridoo and Closeload embed.
|
||||||
- Added Goojara.to source.
|
- Added Goojara.to source.
|
||||||
|
|
@ -74,23 +63,19 @@ title: 'Changelog'
|
||||||
- Disabled Lookmovie and swapped Showbox and VidSrcTo in ranking.
|
- Disabled Lookmovie and swapped Showbox and VidSrcTo in ranking.
|
||||||
|
|
||||||
# Version 2.1.1
|
# Version 2.1.1
|
||||||
|
- Fixed vidplay decryption keys being wrong and switched the domain to one that works
|
||||||
- Fixed vidplay decryption keys being wrong and switched the domain to one that works
|
|
||||||
|
|
||||||
# Version 2.1.0
|
# Version 2.1.0
|
||||||
|
- Add preferedHeaders to most sources
|
||||||
- Add preferedHeaders to most sources
|
- Add CF_BLOCKED flag to sources that have blocked cloudflare API's
|
||||||
- Add CF_BLOCKED flag to sources that have blocked cloudflare API's
|
- Fix vidsrc sometimes having an equal sign where it shouldnt
|
||||||
- Fix vidsrc sometimes having an equal sign where it shouldnt
|
- Increase ranking of lookmovie
|
||||||
- Increase ranking of lookmovie
|
- Re-enabled subtitles for febbox-mp4
|
||||||
- Re-enabled subtitles for febbox-mp4
|
|
||||||
|
|
||||||
# Version 2.0.5
|
# Version 2.0.5
|
||||||
|
|
||||||
- Disable subtitles for febbox-mp4. As their endpoint doesn't work anymore.
|
- Disable subtitles for febbox-mp4. As their endpoint doesn't work anymore.
|
||||||
|
|
||||||
# Version 2.0.4
|
# Version 2.0.4
|
||||||
|
|
||||||
- Added providers:
|
- Added providers:
|
||||||
- Add VidSrcTo provider with Vidplay and Filemoon embeds
|
- Add VidSrcTo provider with Vidplay and Filemoon embeds
|
||||||
- Add VidSrc provider with StreamBucket embeds
|
- Add VidSrc provider with StreamBucket embeds
|
||||||
|
|
@ -104,17 +89,14 @@ title: 'Changelog'
|
||||||
- Added utility to not return multiple subs for the same language - Applies to Lookmovie and Showbox
|
- Added utility to not return multiple subs for the same language - Applies to Lookmovie and Showbox
|
||||||
|
|
||||||
# Version 2.0.3
|
# Version 2.0.3
|
||||||
|
|
||||||
- Actually remove Febbox HLS
|
- Actually remove Febbox HLS
|
||||||
|
|
||||||
# Version 2.0.2
|
# Version 2.0.2
|
||||||
|
|
||||||
- Added Lookmovie caption support
|
- Added Lookmovie caption support
|
||||||
- Fix Febbox duplicate subtitle languages
|
- Fix Febbox duplicate subtitle languages
|
||||||
- Remove Febbox HLS
|
- Remove Febbox HLS
|
||||||
|
|
||||||
# Version 2.0.1
|
# Version 2.0.1
|
||||||
|
|
||||||
- Fixed issue where febbox-mp4 would not show all qualities
|
- Fixed issue where febbox-mp4 would not show all qualities
|
||||||
- Fixed issue where discoverEmbeds event would not show the embeds in the right order
|
- Fixed issue where discoverEmbeds event would not show the embeds in the right order
|
||||||
|
|
||||||
|
|
@ -125,7 +107,6 @@ There are breaking changes in this list, make sure to read them thoroughly if yo
|
||||||
::
|
::
|
||||||
|
|
||||||
**Development tooling:**
|
**Development tooling:**
|
||||||
|
|
||||||
- Added integration test for browser. To make sure the package keeps working in the browser
|
- Added integration test for browser. To make sure the package keeps working in the browser
|
||||||
- Add type checking when building, previously it ignored them
|
- Add type checking when building, previously it ignored them
|
||||||
- Refactored the main folder, now called entrypoint.
|
- Refactored the main folder, now called entrypoint.
|
||||||
|
|
@ -135,13 +116,11 @@ There are breaking changes in this list, make sure to read them thoroughly if yo
|
||||||
- Fetchers can now return a full response with headers and everything
|
- Fetchers can now return a full response with headers and everything
|
||||||
|
|
||||||
**New features:**
|
**New features:**
|
||||||
|
|
||||||
- Added system to allow scraping IP locked sources through the consistentIpforRequests option.
|
- Added system to allow scraping IP locked sources through the consistentIpforRequests option.
|
||||||
- There is now a `buildProviders()` function that gives a builder for the `ProviderControls`. It's an alternative to `makeProviders()`.
|
- There is now a `buildProviders()` function that gives a builder for the `ProviderControls`. It's an alternative to `makeProviders()`.
|
||||||
- Streams can now return a headers object and a `preferredHeaders` object. which is required and optional headers for when using the stream.
|
- Streams can now return a headers object and a `preferredHeaders` object. which is required and optional headers for when using the stream.
|
||||||
|
|
||||||
**Notable changes:**
|
**Notable changes:**
|
||||||
|
|
||||||
- Renamed the NO_CORS flag to CORS_ALLOWED (meaning that resource sharing is allowed)
|
- Renamed the NO_CORS flag to CORS_ALLOWED (meaning that resource sharing is allowed)
|
||||||
- Export Fetcher and Stream types with all types related to it
|
- Export Fetcher and Stream types with all types related to it
|
||||||
- Providers can now return a list of streams instead of just one.
|
- Providers can now return a list of streams instead of just one.
|
||||||
|
|
|
||||||
|
|
@ -7,9 +7,8 @@
|
||||||
## What can I use this on?
|
## What can I use this on?
|
||||||
|
|
||||||
We support many different environments, here are a few examples:
|
We support many different environments, here are a few examples:
|
||||||
|
- In browser, watch streams without needing a server to scrape (does need a proxy)
|
||||||
- In browser, watch streams without needing a server to scrape (does need a proxy)
|
- In a native app, scrape in the app itself
|
||||||
- In a native app, scrape in the app itself
|
- In a backend server, scrape on the server and give the streams to the client to watch.
|
||||||
- In a backend server, scrape on the server and give the streams to the client to watch.
|
|
||||||
|
|
||||||
To find out how to configure the library for your environment, You can read [How to use on X](../2.essentials/0.usage-on-x.md).
|
To find out how to configure the library for your environment, You can read [How to use on X](../2.essentials/0.usage-on-x.md).
|
||||||
|
|
|
||||||
|
|
@ -9,19 +9,15 @@ Let's get started with `@p-stream/providers`. First lets install the package.
|
||||||
::
|
::
|
||||||
|
|
||||||
::code-group
|
::code-group
|
||||||
|
```bash [NPM]
|
||||||
```bash [NPM]
|
npm install @p-stream/providers@github:p-stream/providers#production
|
||||||
npm install @p-stream/providers@github:p-stream/providers#production
|
```
|
||||||
```
|
```bash [Yarn]
|
||||||
|
yarn add @p-stream/providers@github:p-stream/providers#production
|
||||||
```bash [Yarn]
|
```
|
||||||
yarn add @p-stream/providers@github:p-stream/providers#production
|
```bash [PNPM]
|
||||||
```
|
pnpm add @p-stream/providers@github:p-stream/providers#production
|
||||||
|
```
|
||||||
```bash [PNPM]
|
|
||||||
pnpm add @p-stream/providers@github:p-stream/providers#production
|
|
||||||
```
|
|
||||||
|
|
||||||
::
|
::
|
||||||
|
|
||||||
## Scrape your first item
|
## Scrape your first item
|
||||||
|
|
@ -43,8 +39,8 @@ const providers = makeProviders({
|
||||||
fetcher: myFetcher,
|
fetcher: myFetcher,
|
||||||
|
|
||||||
// will be played on a native video player
|
// will be played on a native video player
|
||||||
target: targets.NATIVE,
|
target: targets.NATIVE
|
||||||
});
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
Perfect. You now have an instance of the providers you can reuse everywhere.
|
Perfect. You now have an instance of the providers you can reuse everywhere.
|
||||||
|
|
@ -54,14 +50,14 @@ Now let's scrape an item:
|
||||||
// fetch some data from TMDB
|
// fetch some data from TMDB
|
||||||
const media = {
|
const media = {
|
||||||
type: 'movie',
|
type: 'movie',
|
||||||
title: 'Hamilton',
|
title: "Hamilton",
|
||||||
releaseYear: 2020,
|
releaseYear: 2020,
|
||||||
tmdbId: '556574',
|
tmdbId: "556574"
|
||||||
};
|
}
|
||||||
|
|
||||||
const output = await providers.runAll({
|
const output = await providers.runAll({
|
||||||
media: media,
|
media: media
|
||||||
});
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
Now we have our stream in the output variable. (If the output is `null` then nothing could be found.)
|
Now we have our stream in the output variable. (If the output is `null` then nothing could be found.)
|
||||||
|
|
|
||||||
|
|
@ -3,25 +3,22 @@
|
||||||
The library can run in many environments, so it can be tricky to figure out how to set it up.
|
The library can run in many environments, so it can be tricky to figure out how to set it up.
|
||||||
|
|
||||||
Here is a checklist. For more specific environments, keep reading below:
|
Here is a checklist. For more specific environments, keep reading below:
|
||||||
|
- When requests are very restricted (like browser client-side). Configure a proxied fetcher.
|
||||||
- When requests are very restricted (like browser client-side). Configure a proxied fetcher.
|
- When your requests come from the same device on which it will be streamed (not compatible with proxied fetcher). Set `consistentIpForRequests: true`.
|
||||||
- When your requests come from the same device on which it will be streamed (not compatible with proxied fetcher). Set `consistentIpForRequests: true`.
|
- To set a target. Consult [Targets](./1.targets.md).
|
||||||
- To set a target. Consult [Targets](./1.targets.md).
|
|
||||||
|
|
||||||
To make use of the examples below, check out the following pages:
|
To make use of the examples below, check out the following pages:
|
||||||
|
- [Quick start](../1.get-started/1.quick-start.md)
|
||||||
- [Quick start](../1.get-started/1.quick-start.md)
|
- [Using streams](../2.essentials/4.using-streams.md)
|
||||||
- [Using streams](../2.essentials/4.using-streams.md)
|
|
||||||
|
|
||||||
## NodeJs server
|
## NodeJs server
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { makeProviders, makeStandardFetcher, targets } from '@p-stream/providers';
|
import { makeProviders, makeStandardFetcher, targets } from '@p-stream/providers';
|
||||||
|
|
||||||
const providers = makeProviders({
|
const providers = makeProviders({
|
||||||
fetcher: makeStandardFetcher(fetch),
|
fetcher: makeStandardFetcher(fetch),
|
||||||
target: chooseYourself, // check out https://p-stream.github.io/providers/essentials/targets
|
target: chooseYourself, // check out https://p-stream.github.io/providers/essentials/targets
|
||||||
});
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
## Browser client-side
|
## Browser client-side
|
||||||
|
|
@ -32,27 +29,24 @@ Read more [about proxy fetchers](./2.fetchers.md#using-fetchers-on-the-browser).
|
||||||
```ts
|
```ts
|
||||||
import { makeProviders, makeStandardFetcher, targets } from '@p-stream/providers';
|
import { makeProviders, makeStandardFetcher, targets } from '@p-stream/providers';
|
||||||
|
|
||||||
const proxyUrl = 'https://your.proxy.workers.dev/';
|
const proxyUrl = "https://your.proxy.workers.dev/";
|
||||||
|
|
||||||
const providers = makeProviders({
|
const providers = makeProviders({
|
||||||
fetcher: makeStandardFetcher(fetch),
|
fetcher: makeStandardFetcher(fetch),
|
||||||
proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch),
|
proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch),
|
||||||
target: target.BROWSER,
|
target: target.BROWSER,
|
||||||
});
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
## React native
|
## React native
|
||||||
|
|
||||||
To use the library in a react native app, you would also need a couple of polyfills to polyfill crypto and base64.
|
To use the library in a react native app, you would also need a couple of polyfills to polyfill crypto and base64.
|
||||||
|
|
||||||
1. First install the polyfills:
|
1. First install the polyfills:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install @react-native-anywhere/polyfill-base64 react-native-quick-crypto
|
npm install @react-native-anywhere/polyfill-base64 react-native-quick-crypto
|
||||||
```
|
```
|
||||||
|
|
||||||
2. Add the polyfills to your app:
|
2. Add the polyfills to your app:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
// Import in your entry file
|
// Import in your entry file
|
||||||
import '@react-native-anywhere/polyfill-base64';
|
import '@react-native-anywhere/polyfill-base64';
|
||||||
|
|
@ -69,5 +63,5 @@ const providers = makeProviders({
|
||||||
fetcher: makeStandardFetcher(fetch),
|
fetcher: makeStandardFetcher(fetch),
|
||||||
target: target.NATIVE,
|
target: target.NATIVE,
|
||||||
consistentIpForRequests: true,
|
consistentIpForRequests: true,
|
||||||
});
|
})
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,6 @@ A target is the device on which the stream will be played.
|
||||||
::
|
::
|
||||||
|
|
||||||
#### Possible targets
|
#### Possible targets
|
||||||
|
|
||||||
- **`targets.BROWSER`** Stream will be played in a browser with CORS
|
- **`targets.BROWSER`** Stream will be played in a browser with CORS
|
||||||
- **`targets.BROWSER_EXTENSION`** Stream will be played in a browser using the p-stream extension (WIP)
|
- **`targets.BROWSER_EXTENSION`** Stream will be played in a browser using the p-stream extension (WIP)
|
||||||
- **`targets.NATIVE`** Stream will be played on a native video player
|
- **`targets.NATIVE`** Stream will be played on a native video player
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,6 @@ When creating provider controls, a fetcher will need to be configured.
|
||||||
Depending on your environment, this can come with some considerations:
|
Depending on your environment, this can come with some considerations:
|
||||||
|
|
||||||
## Using `fetch()`
|
## Using `fetch()`
|
||||||
|
|
||||||
In most cases, you can use the `fetch()` API. This will work in newer versions of Node.js (18 and above) and on the browser.
|
In most cases, you can use the `fetch()` API. This will work in newer versions of Node.js (18 and above) and on the browser.
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
|
|
@ -14,19 +13,18 @@ const fetcher = makeStandardFetcher(fetch);
|
||||||
If you using older version of Node.js. You can use the npm package `node-fetch` to polyfill fetch:
|
If you using older version of Node.js. You can use the npm package `node-fetch` to polyfill fetch:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import fetch from 'node-fetch';
|
import fetch from "node-fetch";
|
||||||
|
|
||||||
const fetcher = makeStandardFetcher(fetch);
|
const fetcher = makeStandardFetcher(fetch);
|
||||||
```
|
```
|
||||||
|
|
||||||
## Using fetchers on the browser
|
## Using fetchers on the browser
|
||||||
|
|
||||||
When using this library on a browser, you will need a proxy. Browsers restrict when a web request can be made. To bypass those restrictions, you will need a CORS proxy.
|
When using this library on a browser, you will need a proxy. Browsers restrict when a web request can be made. To bypass those restrictions, you will need a CORS proxy.
|
||||||
|
|
||||||
The p-stream team has a proxy pre-made and pre-configured for you to use. For more information, check out [p-stream/simple-proxy](https://github.com/p-stream/simple-proxy). After installing, you can use this proxy like so:
|
The p-stream team has a proxy pre-made and pre-configured for you to use. For more information, check out [p-stream/simple-proxy](https://github.com/p-stream/simple-proxy). After installing, you can use this proxy like so:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
const fetcher = makeSimpleProxyFetcher('https://your.proxy.workers.dev/', fetch);
|
const fetcher = makeSimpleProxyFetcher("https://your.proxy.workers.dev/", fetch);
|
||||||
```
|
```
|
||||||
|
|
||||||
If you aren't able to use this specific proxy and need to use a different one, you can make your own fetcher in the next section.
|
If you aren't able to use this specific proxy and need to use a different one, you can make your own fetcher in the next section.
|
||||||
|
|
@ -49,15 +47,15 @@ export function makeCustomFetcher(): Fetcher {
|
||||||
|
|
||||||
If you need to make your own fetcher for a proxy, ensure you make it compatible with the following headers: `Set-Cookie`, `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write/read those headers when making a request.
|
If you need to make your own fetcher for a proxy, ensure you make it compatible with the following headers: `Set-Cookie`, `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write/read those headers when making a request.
|
||||||
|
|
||||||
|
|
||||||
## Making a fetcher from scratch
|
## Making a fetcher from scratch
|
||||||
|
|
||||||
In some rare cases, you need to make a fetcher from scratch.
|
In some rare cases, you need to make a fetcher from scratch.
|
||||||
This is the list of features it needs:
|
This is the list of features it needs:
|
||||||
|
- Send/read every header
|
||||||
- Send/read every header
|
- Parse JSON, otherwise parse as text
|
||||||
- Parse JSON, otherwise parse as text
|
- Send JSON, Formdata or normal strings
|
||||||
- Send JSON, Formdata or normal strings
|
- get final destination URL
|
||||||
- get final destination URL
|
|
||||||
|
|
||||||
It's not recommended to do this at all. If you have to, you can base your code on the original implementation of `makeStandardFetcher`. Check out the [source code for it here](https://github.com/p-stream/providers/blob/dev/src/fetchers/standardFetch.ts).
|
It's not recommended to do this at all. If you have to, you can base your code on the original implementation of `makeStandardFetcher`. Check out the [source code for it here](https://github.com/p-stream/providers/blob/dev/src/fetchers/standardFetch.ts).
|
||||||
|
|
||||||
|
|
@ -72,5 +70,5 @@ const myFetcher: Fetcher = (url, ops) => {
|
||||||
headers: new Headers(), // should only contain headers from ops.readHeaders
|
headers: new Headers(), // should only contain headers from ops.readHeaders
|
||||||
statusCode: 200,
|
statusCode: 200,
|
||||||
};
|
};
|
||||||
};
|
}
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,7 @@ const providers = buildProviders()
|
||||||
.build();
|
.build();
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
### Adding your own scrapers to the providers
|
### Adding your own scrapers to the providers
|
||||||
|
|
||||||
If you have your own scraper and still want to use the nice utilities of the provider library or just want to add on to the built-in providers, you can add your own custom source.
|
If you have your own scraper and still want to use the nice utilities of the provider library or just want to add on to the built-in providers, you can add your own custom source.
|
||||||
|
|
@ -60,15 +61,14 @@ If you have your own scraper and still want to use the nice utilities of the pro
|
||||||
const providers = buildProviders()
|
const providers = buildProviders()
|
||||||
.setTarget(targets.NATIVE) // target of where the streams will be used
|
.setTarget(targets.NATIVE) // target of where the streams will be used
|
||||||
.setFetcher(makeStandardFetcher(fetch)) // fetcher, every web request gets called through here
|
.setFetcher(makeStandardFetcher(fetch)) // fetcher, every web request gets called through here
|
||||||
.addSource({
|
.addSource({ // add your own source
|
||||||
// add your own source
|
|
||||||
id: 'my-scraper',
|
id: 'my-scraper',
|
||||||
name: 'My scraper',
|
name: 'My scraper',
|
||||||
rank: 800,
|
rank: 800,
|
||||||
flags: [],
|
flags: [],
|
||||||
scrapeMovie(ctx) {
|
scrapeMovie(ctx) {
|
||||||
throw new Error('Not implemented');
|
throw new Error('Not implemented');
|
||||||
},
|
}
|
||||||
})
|
})
|
||||||
.build();
|
.build();
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -5,13 +5,12 @@ Streams can sometimes be quite picky on how they can be used. So here is a guide
|
||||||
## Essentials
|
## Essentials
|
||||||
|
|
||||||
All streams have the same common parameters:
|
All streams have the same common parameters:
|
||||||
|
- `Stream.type`: The type of stream. Either `hls` or `file`
|
||||||
- `Stream.type`: The type of stream. Either `hls` or `file`
|
- `Stream.id`: The id of this stream, unique per scraper output.
|
||||||
- `Stream.id`: The id of this stream, unique per scraper output.
|
- `Stream.flags`: A list of flags that apply to this stream. Most people won't need to use it.
|
||||||
- `Stream.flags`: A list of flags that apply to this stream. Most people won't need to use it.
|
- `Stream.captions`: A list of captions/subtitles for this stream.
|
||||||
- `Stream.captions`: A list of captions/subtitles for this stream.
|
- `Stream.headers`: Either undefined or a key value object of headers you must set to use the stream.
|
||||||
- `Stream.headers`: Either undefined or a key value object of headers you must set to use the stream.
|
- `Stream.preferredHeaders`: Either undefined or a key value object of headers you may want to set if you want optimal playback - but not required.
|
||||||
- `Stream.preferredHeaders`: Either undefined or a key value object of headers you may want to set if you want optimal playback - but not required.
|
|
||||||
|
|
||||||
Now let's delve deeper into how to watch these streams!
|
Now let's delve deeper into how to watch these streams!
|
||||||
|
|
||||||
|
|
@ -47,9 +46,8 @@ The possibly qualities are: `unknown`, `360`, `480`, `720`, `1080`, `4k`.
|
||||||
File based streams are always guaranteed to have one quality.
|
File based streams are always guaranteed to have one quality.
|
||||||
|
|
||||||
Once you get a streamfile, you have the following parameters:
|
Once you get a streamfile, you have the following parameters:
|
||||||
|
- `StreamFile.type`: Right now it can only be `mp4`.
|
||||||
- `StreamFile.type`: Right now it can only be `mp4`.
|
- `StreamFile.url`: The URL linking to the video file.
|
||||||
- `StreamFile.url`: The URL linking to the video file.
|
|
||||||
|
|
||||||
Here is a code sample of how to watch a file based stream in a browser:
|
Here is a code sample of how to watch a file based stream in a browser:
|
||||||
|
|
||||||
|
|
@ -75,7 +73,6 @@ If your target is set to `BROWSER`, headers will never be required, as it's not
|
||||||
## Using captions/subtitles
|
## Using captions/subtitles
|
||||||
|
|
||||||
All streams have a list of captions at `Stream.captions`. The structure looks like this:
|
All streams have a list of captions at `Stream.captions`. The structure looks like this:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
type Caption = {
|
type Caption = {
|
||||||
type: CaptionType; // Language type, either "srt" or "vtt"
|
type: CaptionType; // Language type, either "srt" or "vtt"
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,3 @@
|
||||||
icon: ph:info-fill
|
icon: ph:info-fill
|
||||||
navigation.redirect: /essentials/usage
|
navigation.redirect: /essentials/usage
|
||||||
navigation.title: 'Get started'
|
navigation.title: "Get started"
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,6 @@
|
||||||
This guide covers everything you need to start contributing.
|
This guide covers everything you need to start contributing.
|
||||||
|
|
||||||
## Get Started
|
## Get Started
|
||||||
|
|
||||||
- **[Setup and Prerequisites](/in-depth/setup-and-prerequisites)** - Start here!
|
- **[Setup and Prerequisites](/in-depth/setup-and-prerequisites)** - Start here!
|
||||||
|
|
||||||
## In-Depth Guides
|
## In-Depth Guides
|
||||||
|
|
|
||||||
|
|
@ -14,14 +14,12 @@ MOVIE_WEB_PROXY_URL = "https://your-proxy-url.com" # Optional
|
||||||
```
|
```
|
||||||
|
|
||||||
**Getting a TMDB API Key:**
|
**Getting a TMDB API Key:**
|
||||||
|
|
||||||
1. Create an account at [TheMovieDB](https://www.themoviedb.org/)
|
1. Create an account at [TheMovieDB](https://www.themoviedb.org/)
|
||||||
2. Go to Settings > API
|
2. Go to Settings > API
|
||||||
3. Request an API key (choose "Developer" for free usage)
|
3. Request an API key (choose "Developer" for free usage)
|
||||||
4. Use the provided key in your `.env` file
|
4. Use the provided key in your `.env` file
|
||||||
|
|
||||||
**Proxy URL (Optional):**
|
**Proxy URL (Optional):**
|
||||||
|
|
||||||
- Useful for testing scrapers that require proxy access
|
- Useful for testing scrapers that require proxy access
|
||||||
- Can help bypass geographical restrictions during development
|
- Can help bypass geographical restrictions during development
|
||||||
- If not provided, the library will use default proxy services
|
- If not provided, the library will use default proxy services
|
||||||
|
|
@ -47,7 +45,6 @@ pnpm cli
|
||||||
```
|
```
|
||||||
|
|
||||||
This will prompt you for:
|
This will prompt you for:
|
||||||
|
|
||||||
- **Fetcher mode** (native, node-fetch, browser)
|
- **Fetcher mode** (native, node-fetch, browser)
|
||||||
- **Scraper ID** (source or embed)
|
- **Scraper ID** (source or embed)
|
||||||
- **TMDB ID** for the content (for sources)
|
- **TMDB ID** for the content (for sources)
|
||||||
|
|
@ -100,41 +97,37 @@ The browser fetcher requires running `pnpm build` first, otherwise you'll get ou
|
||||||
### Understanding CLI Output
|
### Understanding CLI Output
|
||||||
|
|
||||||
#### Source Scraper Output (Returns Embeds)
|
#### Source Scraper Output (Returns Embeds)
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
pnpm cli --source-id catflix --tmdb-id 11527
|
pnpm cli --source-id catflix --tmdb-id 11527
|
||||||
```
|
```
|
||||||
|
|
||||||
Example output:
|
Example output:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"embeds": [
|
embeds: [
|
||||||
{
|
{
|
||||||
"embedId": "turbovid",
|
embedId: 'turbovid',
|
||||||
"url": "https://turbovid.eu/embed/DjncbDBEmbLW"
|
url: 'https://turbovid.eu/embed/DjncbDBEmbLW'
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Embed Scraper Output (Returns Streams)
|
#### Embed Scraper Output (Returns Streams)
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
pnpm cli --source-id turbovid --url "https://turbovid.eu/embed/DjncbDBEmbLW"
|
pnpm cli --source-id turbovid --url "https://turbovid.eu/embed/DjncbDBEmbLW"
|
||||||
```
|
```
|
||||||
|
|
||||||
Example output:
|
Example output:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"stream": [
|
stream: [
|
||||||
{
|
{
|
||||||
"type": "hls",
|
type: 'hls',
|
||||||
"id": "primary",
|
id: 'primary',
|
||||||
"playlist": "https://proxy.fifthwit.net/m3u8-proxy?url=https%3A%2F%2Fqueenselti.pro%2Fwrofm%2Fuwu.m3u8&headers=%7B%22referer%22%3A%22https%3A%2F%2Fturbovid.eu%2F%22%2C%22origin%22%3A%22https%3A%2F%2Fturbovid.eu%22%7D",
|
playlist: 'https://proxy.fifthwit.net/m3u8-proxy?url=https%3A%2F%2Fqueenselti.pro%2Fwrofm%2Fuwu.m3u8&headers=%7B%22referer%22%3A%22https%3A%2F%2Fturbovid.eu%2F%22%2C%22origin%22%3A%22https%3A%2F%2Fturbovid.eu%22%7D',
|
||||||
"flags": [],
|
flags: [],
|
||||||
"captions": []
|
captions: []
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
@ -143,7 +136,6 @@ Example output:
|
||||||
**Notice the proxied URL**: The `createM3U8ProxyUrl()` function creates URLs like `https://proxy.fifthwit.net/m3u8-proxy?url=...&headers=...` to handle protected streams. Read more about this in [Advanced Concepts](/in-depth/advanced-concepts).
|
**Notice the proxied URL**: The `createM3U8ProxyUrl()` function creates URLs like `https://proxy.fifthwit.net/m3u8-proxy?url=...&headers=...` to handle protected streams. Read more about this in [Advanced Concepts](/in-depth/advanced-concepts).
|
||||||
|
|
||||||
#### Interactive Mode Flow
|
#### Interactive Mode Flow
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
pnpm cli
|
pnpm cli
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ export function gatherAllSources(): Array<Sourcerer> {
|
||||||
return [
|
return [
|
||||||
cuevana3Scraper,
|
cuevana3Scraper,
|
||||||
catflixScraper,
|
catflixScraper,
|
||||||
embedsuScraper, // Your source scraper goes here
|
embedsuScraper, // Your source scraper goes here
|
||||||
// ... more sources
|
// ... more sources
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
@ -24,14 +24,13 @@ export function gatherAllSources(): Array<Sourcerer> {
|
||||||
export function gatherAllEmbeds(): Array<Embed> {
|
export function gatherAllEmbeds(): Array<Embed> {
|
||||||
return [
|
return [
|
||||||
upcloudScraper,
|
upcloudScraper,
|
||||||
turbovidScraper, // Your embed scraper goes here
|
turbovidScraper, // Your embed scraper goes here
|
||||||
// ... more embeds
|
// ... more embeds
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**Why this matters:**
|
**Why this matters:**
|
||||||
|
|
||||||
- Only registered scrapers are available to the library
|
- Only registered scrapers are available to the library
|
||||||
- The order in these arrays doesn't matter (ranking determines priority)
|
- The order in these arrays doesn't matter (ranking determines priority)
|
||||||
- You must import your scraper and add it to the appropriate function
|
- You must import your scraper and add it to the appropriate function
|
||||||
|
|
@ -41,43 +40,35 @@ export function gatherAllEmbeds(): Array<Embed> {
|
||||||
There are two distinct types of providers in the system:
|
There are two distinct types of providers in the system:
|
||||||
|
|
||||||
### Sources (Primary Scrapers)
|
### Sources (Primary Scrapers)
|
||||||
|
|
||||||
**Sources** find content on websites and return either:
|
**Sources** find content on websites and return either:
|
||||||
|
|
||||||
- **Direct video streams** (ready to play immediately)
|
- **Direct video streams** (ready to play immediately)
|
||||||
- **Embed URLs** that need further processing by embed scrapers
|
- **Embed URLs** that need further processing by embed scrapers
|
||||||
|
|
||||||
**Characteristics:**
|
**Characteristics:**
|
||||||
|
|
||||||
- Handle website navigation and search
|
- Handle website navigation and search
|
||||||
- Process TMDB IDs to find content
|
- Process TMDB IDs to find content
|
||||||
- Can return multiple server options
|
- Can return multiple server options
|
||||||
- Located in `src/providers/sources/`
|
- Located in `src/providers/sources/`
|
||||||
|
|
||||||
**Example source workflow:**
|
**Example source workflow:**
|
||||||
|
|
||||||
1. Receive movie/show request with TMDB ID
|
1. Receive movie/show request with TMDB ID
|
||||||
2. Search the target website for that content
|
2. Search the target website for that content
|
||||||
3. Extract embed player URLs or direct streams
|
3. Extract embed player URLs or direct streams
|
||||||
4. Return results for further processing
|
4. Return results for further processing
|
||||||
|
|
||||||
### Embeds (Secondary Scrapers)
|
### Embeds (Secondary Scrapers)
|
||||||
|
|
||||||
**Embeds** extract playable video streams from embed players:
|
**Embeds** extract playable video streams from embed players:
|
||||||
|
|
||||||
- Take URLs from sources as input
|
- Take URLs from sources as input
|
||||||
- Handle player-specific extraction and decryption
|
- Handle player-specific extraction and decryption
|
||||||
- Always return direct streams (never more embeds)
|
- Always return direct streams (never more embeds)
|
||||||
|
|
||||||
**Characteristics:**
|
**Characteristics:**
|
||||||
|
|
||||||
- Focus on one player type (turbovid, mixdrop, etc.)
|
- Focus on one player type (turbovid, mixdrop, etc.)
|
||||||
- Handle complex decryption/obfuscation
|
- Handle complex decryption/obfuscation
|
||||||
- Specialized for specific player technologies
|
- Specialized for specific player technologies
|
||||||
- Located in `src/providers/embeds/`
|
- Located in `src/providers/embeds/`
|
||||||
|
|
||||||
**Example embed workflow:**
|
**Example embed workflow:**
|
||||||
|
|
||||||
1. Receive embed player URL from a source
|
1. Receive embed player URL from a source
|
||||||
2. Fetch and parse the embed page
|
2. Fetch and parse the embed page
|
||||||
3. Extract/decrypt the video stream URLs
|
3. Extract/decrypt the video stream URLs
|
||||||
|
|
@ -88,31 +79,28 @@ There are two distinct types of providers in the system:
|
||||||
Every scraper has a **rank** that determines its priority in the execution queue:
|
Every scraper has a **rank** that determines its priority in the execution queue:
|
||||||
|
|
||||||
### How Ranking Works
|
### How Ranking Works
|
||||||
|
|
||||||
- **Higher numbers = Higher priority** (processed first)
|
- **Higher numbers = Higher priority** (processed first)
|
||||||
- **Each rank must be unique** across all providers
|
- **Each rank must be unique** across all providers
|
||||||
- Sources and embeds have separate ranking spaces
|
- Sources and embeds have separate ranking spaces
|
||||||
- Failed scrapers are skipped, next rank is tried
|
- Failed scrapers are skipped, next rank is tried
|
||||||
|
|
||||||
### Rank Ranges
|
### Rank Ranges
|
||||||
|
|
||||||
Usually ranks should be on 10s: 110, 120, 130...
|
Usually ranks should be on 10s: 110, 120, 130...
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Typical rank ranges (not enforced, but conventional)
|
// Typical rank ranges (not enforced, but conventional)
|
||||||
Sources: 1 - 300;
|
Sources: 1-300
|
||||||
Embeds: 1 - 250;
|
Embeds: 1-250
|
||||||
|
|
||||||
// Example rankings
|
// Example rankings
|
||||||
export const embedsuScraper = makeSourcerer({
|
export const embedsuScraper = makeSourcerer({
|
||||||
id: 'embedsu',
|
id: 'embedsu',
|
||||||
rank: 165, // Medium priority source
|
rank: 165, // Medium priority source
|
||||||
// ...
|
// ...
|
||||||
});
|
});
|
||||||
|
|
||||||
export const turbovidScraper = makeEmbed({
|
export const turbovidScraper = makeEmbed({
|
||||||
id: 'turbovid',
|
id: 'turbovid',
|
||||||
rank: 122, // Medium priority embed
|
rank: 122, // Medium priority embed
|
||||||
// ...
|
// ...
|
||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
@ -120,13 +108,11 @@ export const turbovidScraper = makeEmbed({
|
||||||
### Choosing a Rank
|
### Choosing a Rank
|
||||||
|
|
||||||
**For Sources:**
|
**For Sources:**
|
||||||
|
|
||||||
- **200+**: High-quality, reliable sources (fast APIs, good uptime)
|
- **200+**: High-quality, reliable sources (fast APIs, good uptime)
|
||||||
- **100-199**: Medium reliability sources (most scrapers fall here)
|
- **100-199**: Medium reliability sources (most scrapers fall here)
|
||||||
- **1-99**: Lower priority or experimental sources
|
- **1-99**: Lower priority or experimental sources
|
||||||
|
|
||||||
**For Embeds:**
|
**For Embeds:**
|
||||||
|
|
||||||
- **200+**: Fast, reliable embeds (direct URLs, minimal processing)
|
- **200+**: Fast, reliable embeds (direct URLs, minimal processing)
|
||||||
- **100-199**: Standard embeds (typical decryption/extraction)
|
- **100-199**: Standard embeds (typical decryption/extraction)
|
||||||
- **1-99**: Slow or unreliable embeds (complex decryption, poor uptime)
|
- **1-99**: Slow or unreliable embeds (complex decryption, poor uptime)
|
||||||
|
|
@ -151,21 +137,19 @@ Or check the cli to see the ranks.
|
||||||
Each provider is configured using `makeSourcerer()` or `makeEmbed()`:
|
Each provider is configured using `makeSourcerer()` or `makeEmbed()`:
|
||||||
|
|
||||||
### Source Configuration
|
### Source Configuration
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
export const mySourceScraper = makeSourcerer({
|
export const mySourceScraper = makeSourcerer({
|
||||||
id: 'my-source', // Unique identifier (kebab-case)
|
id: 'my-source', // Unique identifier (kebab-case)
|
||||||
name: 'My Source', // Display name (human-readable)
|
name: 'My Source', // Display name (human-readable)
|
||||||
rank: 150, // Priority rank (must be unique)
|
rank: 150, // Priority rank (must be unique)
|
||||||
disabled: false, // Whether scraper is disabled
|
disabled: false, // Whether scraper is disabled
|
||||||
flags: [], // Feature flags (see Advanced Concepts)
|
flags: [], // Feature flags (see Advanced Concepts)
|
||||||
scrapeMovie: comboScraper, // Function for movies
|
scrapeMovie: comboScraper, // Function for movies
|
||||||
scrapeShow: comboScraper, // Function for TV shows
|
scrapeShow: comboScraper, // Function for TV shows
|
||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
### Embed Configuration
|
### Embed Configuration
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
export const myEmbedScraper = makeEmbed({
|
export const myEmbedScraper = makeEmbed({
|
||||||
id: 'my-embed', // Unique identifier (kebab-case)
|
id: 'my-embed', // Unique identifier (kebab-case)
|
||||||
|
|
@ -184,13 +168,11 @@ export const myEmbedScraper = makeEmbed({
|
||||||
The provider system creates a powerful pipeline:
|
The provider system creates a powerful pipeline:
|
||||||
|
|
||||||
### 1. Source → Embed Chain
|
### 1. Source → Embed Chain
|
||||||
|
|
||||||
```
|
```
|
||||||
User Request → Source Scraper → Embed URLs → Embed Scraper → Video Stream → Player
|
User Request → Source Scraper → Embed URLs → Embed Scraper → Video Stream → Player
|
||||||
```
|
```
|
||||||
|
|
||||||
**Pipeline Steps:**
|
**Pipeline Steps:**
|
||||||
|
|
||||||
1. **User Request** - User wants to watch content
|
1. **User Request** - User wants to watch content
|
||||||
2. **Source Scraper** - Finds content on websites
|
2. **Source Scraper** - Finds content on websites
|
||||||
3. **Embed URLs** - Returns player URLs that need processing
|
3. **Embed URLs** - Returns player URLs that need processing
|
||||||
|
|
@ -199,24 +181,20 @@ User Request → Source Scraper → Embed URLs → Embed Scraper → Video Strea
|
||||||
6. **Player** - User watches the content
|
6. **Player** - User watches the content
|
||||||
|
|
||||||
### 2. Multiple Server Options
|
### 2. Multiple Server Options
|
||||||
|
|
||||||
Sources can provide multiple backup servers:
|
Sources can provide multiple backup servers:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Source returns multiple embed options
|
// Source returns multiple embed options
|
||||||
return {
|
return {
|
||||||
embeds: [
|
embeds: [
|
||||||
{ embedId: 'turbovid', url: 'https://turbovid.com/abc' },
|
{ embedId: 'turbovid', url: 'https://turbovid.com/abc' },
|
||||||
{ embedId: 'mixdrop', url: 'https://mixdrop.co/def' },
|
{ embedId: 'mixdrop', url: 'https://mixdrop.co/def' },
|
||||||
{ embedId: 'dood', url: 'https://dood.watch/ghi' },
|
{ embedId: 'dood', url: 'https://dood.watch/ghi' }
|
||||||
],
|
]
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. Fallback System
|
### 3. Fallback System
|
||||||
|
|
||||||
If one embed fails, the system tries the next:
|
If one embed fails, the system tries the next:
|
||||||
|
|
||||||
1. Try turbovid embed (rank 122)
|
1. Try turbovid embed (rank 122)
|
||||||
2. If fails, try mixdrop embed (rank 198)
|
2. If fails, try mixdrop embed (rank 198)
|
||||||
3. If fails, try dood embed (rank 173)
|
3. If fails, try dood embed (rank 173)
|
||||||
|
|
@ -225,21 +203,17 @@ If one embed fails, the system tries the next:
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
### Naming Conventions
|
### Naming Conventions
|
||||||
|
|
||||||
- **IDs**: Use kebab-case (`my-scraper`, not `myMyscraper` or `My_Scraper`)
|
- **IDs**: Use kebab-case (`my-scraper`, not `myMyscraper` or `My_Scraper`)
|
||||||
- **Names**: Use proper capitalization (`VidCloud`, not `vidcloud` or `VIDCLOUD`)
|
- **Names**: Use proper capitalization (`VidCloud`, not `vidcloud` or `VIDCLOUD`)
|
||||||
- **Files**: Match the ID (`my-scraper.ts` for ID `my-scraper`)
|
- **Files**: Match the ID (`my-scraper.ts` for ID `my-scraper`)
|
||||||
|
|
||||||
### Registration Order
|
### Registration Order
|
||||||
|
|
||||||
- The order in `all.ts` arrays doesn't affect execution (rank does)
|
- The order in `all.ts` arrays doesn't affect execution (rank does)
|
||||||
- Group similar scrapers together for maintainability
|
- Group similar scrapers together for maintainability
|
||||||
- Add imports at the top, organized logically
|
- Add imports at the top, organized logically
|
||||||
|
|
||||||
### Testing Integration
|
### Testing Integration
|
||||||
|
|
||||||
Always test that your registration works:
|
Always test that your registration works:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
# Verify your scraper appears in the list (interactive mode shows all available)
|
# Verify your scraper appears in the list (interactive mode shows all available)
|
||||||
pnpm cli
|
pnpm cli
|
||||||
|
|
|
||||||
|
|
@ -26,8 +26,7 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
||||||
const embedPage = await ctx.proxiedFetcher<string>(embedUrl, {
|
const embedPage = await ctx.proxiedFetcher<string>(embedUrl, {
|
||||||
headers: {
|
headers: {
|
||||||
Referer: 'https://embed.su/',
|
Referer: 'https://embed.su/',
|
||||||
'User-Agent':
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -45,7 +44,7 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
||||||
|
|
||||||
// 6. Build the final result
|
// 6. Build the final result
|
||||||
const embeds: SourcererEmbed[] = secondDecode.map((server) => ({
|
const embeds: SourcererEmbed[] = secondDecode.map((server) => ({
|
||||||
embedId: 'viper', // ID of the embed scraper to handle this URL
|
embedId: 'viper', // ID of the embed scraper to handle this URL
|
||||||
url: `https://embed.su/api/e/${server.hash}`,
|
url: `https://embed.su/api/e/${server.hash}`,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
|
@ -56,13 +55,13 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
||||||
|
|
||||||
// Export the scraper configuration
|
// Export the scraper configuration
|
||||||
export const embedsuScraper = makeSourcerer({
|
export const embedsuScraper = makeSourcerer({
|
||||||
id: 'embedsu', // Unique identifier
|
id: 'embedsu', // Unique identifier
|
||||||
name: 'embed.su', // Display name
|
name: 'embed.su', // Display name
|
||||||
rank: 165, // Priority rank (must be unique)
|
rank: 165, // Priority rank (must be unique)
|
||||||
disabled: false, // Whether the scraper is disabled
|
disabled: false, // Whether the scraper is disabled
|
||||||
flags: [], // Feature flags (see Advanced Concepts)
|
flags: [], // Feature flags (see Advanced Concepts)
|
||||||
scrapeMovie: comboScraper, // Function for movies
|
scrapeMovie: comboScraper, // Function for movies
|
||||||
scrapeShow: comboScraper, // Function for TV shows
|
scrapeShow: comboScraper, // Function for TV shows
|
||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -89,7 +88,7 @@ export const myScraper = makeSourcerer({
|
||||||
rank: 150,
|
rank: 150,
|
||||||
disabled: false,
|
disabled: false,
|
||||||
flags: [],
|
flags: [],
|
||||||
scrapeMovie: scrapeMovie, // Separate functions
|
scrapeMovie: scrapeMovie, // Separate functions
|
||||||
scrapeShow: scrapeShow,
|
scrapeShow: scrapeShow,
|
||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
@ -106,19 +105,18 @@ Use when your scraper finds embed players that need further processing:
|
||||||
return {
|
return {
|
||||||
embeds: [
|
embeds: [
|
||||||
{
|
{
|
||||||
embedId: 'turbovid', // Must match an existing embed scraper ID
|
embedId: 'turbovid', // Must match an existing embed scraper ID
|
||||||
url: 'https://turbovid.com/embed/abc123',
|
url: 'https://turbovid.com/embed/abc123'
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
embedId: 'mixdrop', // Backup option
|
embedId: 'mixdrop', // Backup option
|
||||||
url: 'https://mixdrop.co/embed/def456',
|
url: 'https://mixdrop.co/embed/def456'
|
||||||
},
|
}
|
||||||
],
|
]
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
**When to use:**
|
**When to use:**
|
||||||
|
|
||||||
- Your scraper finds embed player URLs
|
- Your scraper finds embed player URLs
|
||||||
- You want to leverage existing embed scrapers
|
- You want to leverage existing embed scrapers
|
||||||
- The site uses common players (turbovid, mixdrop, etc.)
|
- The site uses common players (turbovid, mixdrop, etc.)
|
||||||
|
|
@ -141,116 +139,115 @@ return {
|
||||||
playlist: streamUrl,
|
playlist: streamUrl,
|
||||||
flags: [flags.CORS_ALLOWED],
|
flags: [flags.CORS_ALLOWED],
|
||||||
captions: [], // Subtitle tracks (optional)
|
captions: [], // Subtitle tracks (optional)
|
||||||
},
|
}
|
||||||
],
|
]
|
||||||
};
|
};
|
||||||
|
|
||||||
// For MP4 files with a single quality
|
// For MP4 files with a single quality
|
||||||
return {
|
return {
|
||||||
embeds: [],
|
embeds: [],
|
||||||
stream: [
|
stream: [
|
||||||
{
|
{
|
||||||
id: 'primary',
|
id: 'primary',
|
||||||
captions,
|
captions,
|
||||||
qualities: {
|
qualities: {
|
||||||
unknown: {
|
|
||||||
type: 'mp4',
|
|
||||||
url: streamUrl,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
type: 'file',
|
|
||||||
flags: [flags.CORS_ALLOWED],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
|
|
||||||
// For MP4 files with multiple qualities:
|
|
||||||
// It's recommended to return it using a function similar to this:
|
|
||||||
|
|
||||||
const streams = Object.entries(data.streams).reduce((acc: Record<string, string>, [quality, url]) => {
|
|
||||||
let qualityKey: number;
|
|
||||||
if (quality === 'ORG') {
|
|
||||||
// Only add unknown quality if it's an mp4 (handle URLs with query parameters)
|
|
||||||
const urlPath = url.split('?')[0]; // Remove query parameters
|
|
||||||
if (urlPath.toLowerCase().endsWith('.mp4')) {
|
|
||||||
acc.unknown = url;
|
|
||||||
}
|
|
||||||
return acc;
|
|
||||||
}
|
|
||||||
if (quality === '4K') {
|
|
||||||
qualityKey = 2160;
|
|
||||||
} else {
|
|
||||||
qualityKey = parseInt(quality.replace('P', ''), 10);
|
|
||||||
}
|
|
||||||
if (Number.isNaN(qualityKey) || acc[qualityKey]) return acc;
|
|
||||||
acc[qualityKey] = url;
|
|
||||||
return acc;
|
|
||||||
}, {});
|
|
||||||
|
|
||||||
// Filter qualities based on provider type
|
|
||||||
const filteredStreams = Object.entries(streams).reduce((acc: Record<string, string>, [quality, url]) => {
|
|
||||||
// Skip unknown for cached provider
|
|
||||||
if (provider.useCacheUrl && quality === 'unknown') {
|
|
||||||
return acc;
|
|
||||||
}
|
|
||||||
|
|
||||||
acc[quality] = url;
|
|
||||||
return acc;
|
|
||||||
}, {});
|
|
||||||
|
|
||||||
// Returning each quality like so
|
|
||||||
return {
|
|
||||||
stream: [
|
|
||||||
{
|
|
||||||
id: 'primary',
|
|
||||||
captions: [],
|
|
||||||
qualities: {
|
|
||||||
...(filteredStreams[2160] && {
|
|
||||||
'4k': {
|
|
||||||
type: 'mp4',
|
|
||||||
url: filteredStreams[2160],
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
...(filteredStreams[1080] && {
|
|
||||||
1080: {
|
|
||||||
type: 'mp4',
|
|
||||||
url: filteredStreams[1080],
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
...(filteredStreams[720] && {
|
|
||||||
720: {
|
|
||||||
type: 'mp4',
|
|
||||||
url: filteredStreams[720],
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
...(filteredStreams[480] && {
|
|
||||||
480: {
|
|
||||||
type: 'mp4',
|
|
||||||
url: filteredStreams[480],
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
...(filteredStreams[360] && {
|
|
||||||
360: {
|
|
||||||
type: 'mp4',
|
|
||||||
url: filteredStreams[360],
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
...(filteredStreams.unknown && {
|
|
||||||
unknown: {
|
unknown: {
|
||||||
type: 'mp4',
|
type: 'mp4',
|
||||||
url: filteredStreams.unknown,
|
url: streamUrl,
|
||||||
},
|
},
|
||||||
}),
|
},
|
||||||
|
type: 'file',
|
||||||
|
flags: [flags.CORS_ALLOWED],
|
||||||
},
|
},
|
||||||
type: 'file',
|
],
|
||||||
flags: [flags.CORS_ALLOWED],
|
};
|
||||||
},
|
|
||||||
],
|
// For MP4 files with multiple qualities:
|
||||||
};
|
// It's recommended to return it using a function similar to this:
|
||||||
|
|
||||||
|
const streams = Object.entries(data.streams).reduce((acc: Record<string, string>, [quality, url]) => {
|
||||||
|
let qualityKey: number;
|
||||||
|
if (quality === 'ORG') {
|
||||||
|
// Only add unknown quality if it's an mp4 (handle URLs with query parameters)
|
||||||
|
const urlPath = url.split('?')[0]; // Remove query parameters
|
||||||
|
if (urlPath.toLowerCase().endsWith('.mp4')) {
|
||||||
|
acc.unknown = url;
|
||||||
|
}
|
||||||
|
return acc;
|
||||||
|
}
|
||||||
|
if (quality === '4K') {
|
||||||
|
qualityKey = 2160;
|
||||||
|
} else {
|
||||||
|
qualityKey = parseInt(quality.replace('P', ''), 10);
|
||||||
|
}
|
||||||
|
if (Number.isNaN(qualityKey) || acc[qualityKey]) return acc;
|
||||||
|
acc[qualityKey] = url;
|
||||||
|
return acc;
|
||||||
|
}, {});
|
||||||
|
|
||||||
|
// Filter qualities based on provider type
|
||||||
|
const filteredStreams = Object.entries(streams).reduce((acc: Record<string, string>, [quality, url]) => {
|
||||||
|
// Skip unknown for cached provider
|
||||||
|
if (provider.useCacheUrl && quality === 'unknown') {
|
||||||
|
return acc;
|
||||||
|
}
|
||||||
|
|
||||||
|
acc[quality] = url;
|
||||||
|
return acc;
|
||||||
|
}, {});
|
||||||
|
|
||||||
|
// Returning each quality like so
|
||||||
|
return {
|
||||||
|
stream: [
|
||||||
|
{
|
||||||
|
id: 'primary',
|
||||||
|
captions: [],
|
||||||
|
qualities: {
|
||||||
|
...(filteredStreams[2160] && {
|
||||||
|
'4k': {
|
||||||
|
type: 'mp4',
|
||||||
|
url: filteredStreams[2160],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
...(filteredStreams[1080] && {
|
||||||
|
1080: {
|
||||||
|
type: 'mp4',
|
||||||
|
url: filteredStreams[1080],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
...(filteredStreams[720] && {
|
||||||
|
720: {
|
||||||
|
type: 'mp4',
|
||||||
|
url: filteredStreams[720],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
...(filteredStreams[480] && {
|
||||||
|
480: {
|
||||||
|
type: 'mp4',
|
||||||
|
url: filteredStreams[480],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
...(filteredStreams[360] && {
|
||||||
|
360: {
|
||||||
|
type: 'mp4',
|
||||||
|
url: filteredStreams[360],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
...(filteredStreams.unknown && {
|
||||||
|
unknown: {
|
||||||
|
type: 'mp4',
|
||||||
|
url: filteredStreams.unknown,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
type: 'file',
|
||||||
|
flags: [flags.CORS_ALLOWED],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
**When to use:**
|
**When to use:**
|
||||||
|
|
||||||
- Your scraper can extract direct video URLs
|
- Your scraper can extract direct video URLs
|
||||||
- The site provides its own player technology
|
- The site provides its own player technology
|
||||||
- You need fine control over stream handling
|
- You need fine control over stream handling
|
||||||
|
|
@ -261,71 +258,65 @@ return {
|
||||||
The scraper context (`ctx`) provides everything you need for implementation:
|
The scraper context (`ctx`) provides everything you need for implementation:
|
||||||
|
|
||||||
### Media Information
|
### Media Information
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Basic media info (always available)
|
// Basic media info (always available)
|
||||||
ctx.media.title; // "Spirited Away"
|
ctx.media.title // "Spirited Away"
|
||||||
ctx.media.type; // "movie" | "show"
|
ctx.media.type // "movie" | "show"
|
||||||
ctx.media.tmdbId; // 129
|
ctx.media.tmdbId // 129
|
||||||
ctx.media.releaseYear; // 2001
|
ctx.media.releaseYear // 2001
|
||||||
ctx.media.imdbId; // "tt0245429" (when available)
|
ctx.media.imdbId // "tt0245429" (when available)
|
||||||
|
|
||||||
// For TV shows only (check ctx.media.type === 'show')
|
// For TV shows only (check ctx.media.type === 'show')
|
||||||
ctx.media.season.number; // 1
|
ctx.media.season.number // 1
|
||||||
ctx.media.season.tmdbId; // Season TMDB ID
|
ctx.media.season.tmdbId // Season TMDB ID
|
||||||
ctx.media.episode.number; // 5
|
ctx.media.episode.number // 5
|
||||||
ctx.media.episode.tmdbId; // Episode TMDB ID
|
ctx.media.episode.tmdbId // Episode TMDB ID
|
||||||
```
|
```
|
||||||
|
|
||||||
### HTTP Client
|
### HTTP Client
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Always use proxiedFetcher for external requests to avoid CORS
|
// Always use proxiedFetcher for external requests to avoid CORS
|
||||||
const response = await ctx.proxiedFetcher<string>('https://example.com/api', {
|
const response = await ctx.proxiedFetcher<string>('https://example.com/api', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'User-Agent': 'Mozilla/5.0...',
|
'User-Agent': 'Mozilla/5.0...',
|
||||||
Referer: 'https://example.com',
|
'Referer': 'https://example.com'
|
||||||
},
|
},
|
||||||
body: JSON.stringify({ key: 'value' }),
|
body: JSON.stringify({ key: 'value' })
|
||||||
});
|
});
|
||||||
|
|
||||||
// For API calls with base URL
|
// For API calls with base URL
|
||||||
const data = await ctx.proxiedFetcher('/search', {
|
const data = await ctx.proxiedFetcher('/search', {
|
||||||
baseUrl: 'https://api.example.com',
|
baseUrl: 'https://api.example.com',
|
||||||
query: { q: ctx.media.title, year: ctx.media.releaseYear },
|
query: { q: ctx.media.title, year: ctx.media.releaseYear }
|
||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
### Progress Updates
|
### Progress Updates
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Update the loading indicator (0-100)
|
// Update the loading indicator (0-100)
|
||||||
ctx.progress(25); // Found media page
|
ctx.progress(25); // Found media page
|
||||||
// ... processing ...
|
// ... processing ...
|
||||||
ctx.progress(50); // Extracted embed links
|
ctx.progress(50); // Extracted embed links
|
||||||
// ... more processing ...
|
// ... more processing ...
|
||||||
ctx.progress(90); // Almost done
|
ctx.progress(90); // Almost done
|
||||||
```
|
```
|
||||||
|
|
||||||
## Common Patterns
|
## Common Patterns
|
||||||
|
|
||||||
### 1. URL Building
|
### 1. URL Building
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Handle different media types
|
// Handle different media types
|
||||||
const buildUrl = (ctx: ShowScrapeContext | MovieScrapeContext) => {
|
const buildUrl = (ctx: ShowScrapeContext | MovieScrapeContext) => {
|
||||||
const apiUrl =
|
const apiUrl = ctx.media.type === 'movie'
|
||||||
ctx.media.type === 'movie'
|
? `${baseUrl}/movie/${ctx.media.tmdbId}`
|
||||||
? `${baseUrl}/movie/${ctx.media.tmdbId}`
|
: `${baseUrl}/tv/${ctx.media.tmdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`;
|
||||||
: `${baseUrl}/tv/${ctx.media.tmdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`;
|
|
||||||
|
|
||||||
return apiUrl;
|
return apiUrl;
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2. Data Extraction
|
### 2. Data Extraction
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { load } from 'cheerio';
|
import { load } from 'cheerio';
|
||||||
|
|
||||||
|
|
@ -345,7 +336,6 @@ if (configMatch) {
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. Error Handling
|
### 3. Error Handling
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { NotFoundError } from '@/utils/errors';
|
import { NotFoundError } from '@/utils/errors';
|
||||||
|
|
||||||
|
|
@ -361,56 +351,49 @@ if (!apiResponse.success) {
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4. Protected Streams
|
### 4. Protected Streams
|
||||||
|
|
||||||
There are several ways to bypass protections on streams.
|
There are several ways to bypass protections on streams.
|
||||||
|
|
||||||
Using the M3U8 proxy:
|
Using the M3U8 proxy:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { createM3U8ProxyUrl } from '@/utils/proxy';
|
import { createM3U8ProxyUrl } from '@/utils/proxy';
|
||||||
|
|
||||||
// For streams that require special headers
|
// For streams that require special headers
|
||||||
const streamHeaders = {
|
const streamHeaders = {
|
||||||
Referer: 'https://player.example.com/',
|
'Referer': 'https://player.example.com/',
|
||||||
Origin: 'https://player.example.com',
|
'Origin': 'https://player.example.com',
|
||||||
'User-Agent': 'Mozilla/5.0...',
|
'User-Agent': 'Mozilla/5.0...'
|
||||||
};
|
};
|
||||||
|
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: createM3U8ProxyUrl(originalPlaylist, ctx.features, streamHeaders),
|
||||||
playlist: createM3U8ProxyUrl(originalPlaylist, ctx.features, streamHeaders),
|
headers: streamHeaders, // Include headers in the createM3U8ProxyUrl function and here for native and extension targets
|
||||||
headers: streamHeaders, // Include headers in the createM3U8ProxyUrl function and here for native and extension targets
|
flags: [flags.CORS_ALLOWED], // createM3U8ProxyUrl (or the extension) bypasses cors so we say it's allowed to play in a browser
|
||||||
flags: [flags.CORS_ALLOWED], // createM3U8ProxyUrl (or the extension) bypasses cors so we say it's allowed to play in a browser
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
Using the browser extension:
|
Using the browser extension:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// For streams that require special headers
|
// For streams that require special headers
|
||||||
const streamHeaders = {
|
const streamHeaders = {
|
||||||
Referer: 'https://player.example.com/',
|
'Referer': 'https://player.example.com/',
|
||||||
Origin: 'https://player.example.com',
|
'Origin': 'https://player.example.com',
|
||||||
'User-Agent': 'Mozilla/5.0...',
|
'User-Agent': 'Mozilla/5.0...'
|
||||||
};
|
};
|
||||||
|
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: originalPlaylist,
|
||||||
playlist: originalPlaylist,
|
headers: streamHeaders,
|
||||||
headers: streamHeaders,
|
flags: [], // Use the extension becuase it can pass headers, include no flag for extension or native
|
||||||
flags: [], // Use the extension becuase it can pass headers, include no flag for extension or native
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -439,15 +422,13 @@ export const myEmbedScraper = makeEmbed({
|
||||||
|
|
||||||
// 3. Return the stream
|
// 3. Return the stream
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: streamMatch[1],
|
||||||
playlist: streamMatch[1],
|
flags: [flags.CORS_ALLOWED],
|
||||||
flags: [flags.CORS_ALLOWED],
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
@ -456,7 +437,6 @@ export const myEmbedScraper = makeEmbed({
|
||||||
## Testing Your Scrapers
|
## Testing Your Scrapers
|
||||||
|
|
||||||
### 1. Basic Testing
|
### 1. Basic Testing
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
# Test your scraper with CLI
|
# Test your scraper with CLI
|
||||||
pnpm cli --source-id my-scraper --tmdb-id 11527
|
pnpm cli --source-id my-scraper --tmdb-id 11527
|
||||||
|
|
@ -468,28 +448,24 @@ pnpm cli --source-id my-scraper --tmdb-id 94605 --season 1 --episode 1 # TV sho
|
||||||
### 2. Real CLI Output Examples
|
### 2. Real CLI Output Examples
|
||||||
|
|
||||||
**Testing a source that returns embeds:**
|
**Testing a source that returns embeds:**
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
pnpm cli --source-id catflix --tmdb-id 11527
|
pnpm cli --source-id catflix --tmdb-id 11527
|
||||||
```
|
```
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"embeds": [
|
embeds: [
|
||||||
{
|
{
|
||||||
"embedId": "turbovid",
|
embedId: 'turbovid',
|
||||||
"url": "https://turbovid.eu/embed/DjncbDBEmbLW"
|
url: 'https://turbovid.eu/embed/DjncbDBEmbLW'
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**Testing an embed that returns streams:**
|
**Testing an embed that returns streams:**
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
pnpm cli --source-id turbovid --url "https://turbovid.eu/embed/DjncbDBEmbLW"
|
pnpm cli --source-id turbovid --url "https://turbovid.eu/embed/DjncbDBEmbLW"
|
||||||
```
|
```
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
stream: [
|
stream: [
|
||||||
|
|
@ -507,9 +483,7 @@ pnpm cli --source-id turbovid --url "https://turbovid.eu/embed/DjncbDBEmbLW"
|
||||||
**Notice**: The playlist URL shows how `createM3U8ProxyUrl()` creates proxied URLs to handle protected streams.
|
**Notice**: The playlist URL shows how `createM3U8ProxyUrl()` creates proxied URLs to handle protected streams.
|
||||||
|
|
||||||
### 3. Comprehensive Testing
|
### 3. Comprehensive Testing
|
||||||
|
|
||||||
Test with various content:
|
Test with various content:
|
||||||
|
|
||||||
- Popular movies (The Shining: 11527, Spirited Away: 129, Avatar: 19995)
|
- Popular movies (The Shining: 11527, Spirited Away: 129, Avatar: 19995)
|
||||||
- Recent releases (check current popular movies)
|
- Recent releases (check current popular movies)
|
||||||
- TV shows with multiple seasons
|
- TV shows with multiple seasons
|
||||||
|
|
@ -517,7 +491,6 @@ Test with various content:
|
||||||
- Different languages/regions
|
- Different languages/regions
|
||||||
|
|
||||||
### 4. Debug Mode
|
### 4. Debug Mode
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
# Add debug logging to your scraper
|
# Add debug logging to your scraper
|
||||||
console.log('Fetching URL:', embedUrl);
|
console.log('Fetching URL:', embedUrl);
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,6 @@ User Request → Source Scraper → What did source find?
|
||||||
```
|
```
|
||||||
|
|
||||||
**Flow Breakdown:**
|
**Flow Breakdown:**
|
||||||
|
|
||||||
1. **User requests** content (movie/TV show)
|
1. **User requests** content (movie/TV show)
|
||||||
2. **Source scraper** searches the target website
|
2. **Source scraper** searches the target website
|
||||||
3. **Source returns** either:
|
3. **Source returns** either:
|
||||||
|
|
@ -31,7 +30,6 @@ User Request → Source Scraper → What did source find?
|
||||||
## Sources: The Content Finders
|
## Sources: The Content Finders
|
||||||
|
|
||||||
**Sources** are the first stage - they find content on websites and return either:
|
**Sources** are the first stage - they find content on websites and return either:
|
||||||
|
|
||||||
1. **Direct video streams** (ready to play)
|
1. **Direct video streams** (ready to play)
|
||||||
2. **Embed URLs** that need further processing
|
2. **Embed URLs** that need further processing
|
||||||
|
|
||||||
|
|
@ -43,23 +41,20 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
||||||
// 1. Call an API to find video sources
|
// 1. Call an API to find video sources
|
||||||
const data = await ctx.proxiedFetcher(`/api/getVideoSource`, {
|
const data = await ctx.proxiedFetcher(`/api/getVideoSource`, {
|
||||||
baseUrl: 'https://tom.autoembed.cc',
|
baseUrl: 'https://tom.autoembed.cc',
|
||||||
query: { type: mediaType, id },
|
query: { type: mediaType, id }
|
||||||
});
|
});
|
||||||
|
|
||||||
// 2. Return embed URLs for further processing
|
// 2. Return embed URLs for further processing
|
||||||
return {
|
return {
|
||||||
embeds: [
|
embeds: [{
|
||||||
{
|
embedId: 'autoembed-english', // Points to an embed scraper
|
||||||
embedId: 'autoembed-english', // Points to an embed scraper
|
url: data.videoSource // URL that embed will process
|
||||||
url: data.videoSource, // URL that embed will process
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**What this source does:**
|
**What this source does:**
|
||||||
|
|
||||||
- Queries an API with TMDB ID
|
- Queries an API with TMDB ID
|
||||||
- Gets back a video source URL
|
- Gets back a video source URL
|
||||||
- Returns it as an embed for the `autoembed-english` embed scraper to handle
|
- Returns it as an embed for the `autoembed-english` embed scraper to handle
|
||||||
|
|
@ -82,18 +77,15 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
||||||
|
|
||||||
// 4. Return embed URL for turbovid embed to process
|
// 4. Return embed URL for turbovid embed to process
|
||||||
return {
|
return {
|
||||||
embeds: [
|
embeds: [{
|
||||||
{
|
embedId: 'turbovid', // Points to turbovid embed scraper
|
||||||
embedId: 'turbovid', // Points to turbovid embed scraper
|
url: decodedUrl // Turbovid player URL
|
||||||
url: decodedUrl, // Turbovid player URL
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**What this source does:**
|
**What this source does:**
|
||||||
|
|
||||||
- Scrapes a streaming website
|
- Scrapes a streaming website
|
||||||
- Finds encoded embed player URLs in the page source
|
- Finds encoded embed player URLs in the page source
|
||||||
- Decodes the URL and returns it for the `turbovid` embed scraper
|
- Decodes the URL and returns it for the `turbovid` embed scraper
|
||||||
|
|
@ -106,140 +98,114 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// From src/providers/embeds/autoembed.ts
|
// From src/providers/embeds/autoembed.ts
|
||||||
function embed(provider: { id: string; rank: number }) {
|
async scrape(ctx) {
|
||||||
return makeEmbed({
|
// The URL from the source is already a direct HLS playlist
|
||||||
id: provider.id,
|
return {
|
||||||
name: provider.id.split('-').map(word => word[0].toUpperCase() + word.slice(1)).join(' '),
|
stream: [{
|
||||||
rank: provider.rank,
|
id: 'primary',
|
||||||
flags: [flags.CORS_ALLOWED], // Embed flags match stream flags
|
type: 'hls',
|
||||||
async scrape(ctx) {
|
playlist: ctx.url, // Use the URL directly as HLS playlist
|
||||||
// The URL from the source is already a direct HLS playlist
|
flags: [flags.CORS_ALLOWED],
|
||||||
return {
|
captions: []
|
||||||
stream: [{
|
}]
|
||||||
id: 'primary',
|
};
|
||||||
type: 'hls',
|
|
||||||
playlist: ctx.url, // Use the URL directly as HLS playlist
|
|
||||||
flags: [flags.CORS_ALLOWED], // Stream flags
|
|
||||||
captions: []
|
|
||||||
}]
|
|
||||||
};
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**What this embed does:**
|
**What this embed does:**
|
||||||
|
|
||||||
- Takes the URL from autoembed source
|
- Takes the URL from autoembed source
|
||||||
- Treats it as a direct HLS playlist (no further processing needed)
|
- Treats it as a direct HLS playlist (no further processing needed)
|
||||||
- Returns it as a playable stream
|
- Returns it as a playable stream
|
||||||
- **Note:** Embed flags now match stream flags for consistent filtering
|
|
||||||
|
|
||||||
### Example: Turbovid Embed (Complex)
|
### Example: Turbovid Embed (Complex)
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// From src/providers/embeds/turbovid.ts
|
// From src/providers/embeds/turbovid.ts
|
||||||
export const turbovidScraper = makeEmbed({
|
async scrape(ctx) {
|
||||||
id: 'turbovid',
|
// 1. Fetch the turbovid player page
|
||||||
name: 'Turbovid',
|
const embedPage = await ctx.proxiedFetcher(ctx.url);
|
||||||
rank: 180,
|
|
||||||
flags: [flags.CORS_ALLOWED], // Embed flags match stream flags
|
|
||||||
async scrape(ctx) {
|
|
||||||
// 1. Fetch the turbovid player page
|
|
||||||
const embedPage = await ctx.proxiedFetcher(ctx.url);
|
|
||||||
|
|
||||||
// 2. Extract encryption keys from the page
|
// 2. Extract encryption keys from the page
|
||||||
const apkey = embedPage.match(/const\s+apkey\s*=\s*"(.*?)";/)?.[1];
|
const apkey = embedPage.match(/const\s+apkey\s*=\s*"(.*?)";/)?.[1];
|
||||||
const xxid = embedPage.match(/const\s+xxid\s*=\s*"(.*?)";/)?.[1];
|
const xxid = embedPage.match(/const\s+xxid\s*=\s*"(.*?)";/)?.[1];
|
||||||
|
|
||||||
// 3. Get decryption key from API
|
// 3. Get decryption key from API
|
||||||
const encodedJuiceKey = JSON.parse(
|
const encodedJuiceKey = JSON.parse(
|
||||||
await ctx.proxiedFetcher('/api/cucked/juice_key', { baseUrl })
|
await ctx.proxiedFetcher('/api/cucked/juice_key', { baseUrl })
|
||||||
).juice;
|
).juice;
|
||||||
|
|
||||||
// 4. Get encrypted playlist data
|
// 4. Get encrypted playlist data
|
||||||
const data = JSON.parse(
|
const data = JSON.parse(
|
||||||
await ctx.proxiedFetcher('/api/cucked/the_juice_v2/', {
|
await ctx.proxiedFetcher('/api/cucked/the_juice_v2/', {
|
||||||
baseUrl, query: { [apkey]: xxid }
|
baseUrl, query: { [apkey]: xxid }
|
||||||
})
|
})
|
||||||
).data;
|
).data;
|
||||||
|
|
||||||
// 5. Decrypt the playlist URL
|
// 5. Decrypt the playlist URL
|
||||||
const playlist = decrypt(data, atob(encodedJuiceKey));
|
const playlist = decrypt(data, atob(encodedJuiceKey));
|
||||||
|
|
||||||
// 6. Return proxied stream (handles CORS/headers)
|
// 6. Return proxied stream (handles CORS/headers)
|
||||||
return {
|
return {
|
||||||
stream: [{
|
stream: [{
|
||||||
type: 'hls',
|
type: 'hls',
|
||||||
id: 'primary',
|
id: 'primary',
|
||||||
playlist: createM3U8ProxyUrl(playlist, ctx.features, streamHeaders),
|
playlist: createM3U8ProxyUrl(playlist, ctx.features, streamHeaders),
|
||||||
headers: streamHeaders,
|
headers: streamHeaders,
|
||||||
flags: [flags.CORS_ALLOWED], // Stream flags match embed flags
|
flags: [], captions: []
|
||||||
captions: []
|
}]
|
||||||
}]
|
};
|
||||||
};
|
}
|
||||||
},
|
|
||||||
});
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**What this embed does:**
|
**What this embed does:**
|
||||||
|
|
||||||
- Takes turbovid player URL from catflix source
|
- Takes turbovid player URL from catflix source
|
||||||
- Performs complex extraction: fetches page → gets keys → decrypts data
|
- Performs complex extraction: fetches page → gets keys → decrypts data
|
||||||
- Returns the final HLS playlist with proper proxy handling
|
- Returns the final HLS playlist with proper proxy handling
|
||||||
- **Note:** Embed flags now match stream flags for consistent filtering
|
|
||||||
|
|
||||||
## Key Differences
|
## Key Differences
|
||||||
|
|
||||||
| Sources | Embeds |
|
| Sources | Embeds |
|
||||||
| ----------------------------------- | ---------------------------------------------- |
|
|---------|--------|
|
||||||
| **Find content** on websites | **Extract streams** from players |
|
| **Find content** on websites | **Extract streams** from players |
|
||||||
| Return embed URLs OR direct streams | Always return direct streams |
|
| Return embed URLs OR direct streams | Always return direct streams |
|
||||||
| Handle website navigation/search | Handle player-specific extraction |
|
| Handle website navigation/search | Handle player-specific extraction |
|
||||||
| Can return multiple server options | Process one specific player type |
|
| Can return multiple server options | Process one specific player type |
|
||||||
| Example: "Find Avengers on Catflix" | Example: "Extract stream from Turbovid player" |
|
| Example: "Find Avengers on Catflix" | Example: "Extract stream from Turbovid player" |
|
||||||
|
|
||||||
## Why This Separation?
|
## Why This Separation?
|
||||||
|
|
||||||
### 1. **Reusability**
|
### 1. **Reusability**
|
||||||
|
|
||||||
Multiple sources can use the same embed:
|
Multiple sources can use the same embed:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Both catflix and other sources can return turbovid embeds
|
// Both catflix and other sources can return turbovid embeds
|
||||||
{ embedId: 'turbovid', url: 'https://turbovid.com/player123' }
|
{ embedId: 'turbovid', url: 'https://turbovid.com/player123' }
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2. **Multiple Server Options**
|
### 2. **Multiple Server Options**
|
||||||
|
|
||||||
Sources can provide backup servers:
|
Sources can provide backup servers:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
return {
|
return {
|
||||||
embeds: [
|
embeds: [
|
||||||
{ embedId: 'turbovid', url: 'https://turbovid.com/player123' },
|
{ embedId: 'turbovid', url: 'https://turbovid.com/player123' },
|
||||||
{ embedId: 'vidcloud', url: 'https://vidcloud.co/embed456' },
|
{ embedId: 'vidcloud', url: 'https://vidcloud.co/embed456' },
|
||||||
{ embedId: 'dood', url: 'https://dood.watch/789' },
|
{ embedId: 'dood', url: 'https://dood.watch/789' }
|
||||||
],
|
]
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. **Language/Quality Variants**
|
### 3. **Language/Quality Variants**
|
||||||
|
|
||||||
Sources can offer different options:
|
Sources can offer different options:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
return {
|
return {
|
||||||
embeds: [
|
embeds: [
|
||||||
{ embedId: 'autoembed-english', url: streamUrl },
|
{ embedId: 'autoembed-english', url: streamUrl },
|
||||||
{ embedId: 'autoembed-spanish', url: streamUrlEs },
|
{ embedId: 'autoembed-spanish', url: streamUrlEs },
|
||||||
{ embedId: 'autoembed-hindi', url: streamUrlHi },
|
{ embedId: 'autoembed-hindi', url: streamUrlHi }
|
||||||
],
|
]
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4. **Specialization**
|
### 4. **Specialization**
|
||||||
|
|
||||||
- **Sources** specialize in website structures and search
|
- **Sources** specialize in website structures and search
|
||||||
- **Embeds** specialize in player technologies and decryption
|
- **Embeds** specialize in player technologies and decryption
|
||||||
|
|
||||||
|
|
@ -264,15 +230,14 @@ return {
|
||||||
### Error Handling Chain
|
### Error Handling Chain
|
||||||
|
|
||||||
If the embed fails to extract a stream:
|
If the embed fails to extract a stream:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Source provides multiple backup options
|
// Source provides multiple backup options
|
||||||
return {
|
return {
|
||||||
embeds: [
|
embeds: [
|
||||||
{ embedId: 'turbovid', url: url1 }, // Try first
|
{ embedId: 'turbovid', url: url1 }, // Try first
|
||||||
{ embedId: 'mixdrop', url: url2 }, // Fallback 1
|
{ embedId: 'mixdrop', url: url2 }, // Fallback 1
|
||||||
{ embedId: 'dood', url: url3 }, // Fallback 2
|
{ embedId: 'dood', url: url3 } // Fallback 2
|
||||||
],
|
]
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -281,30 +246,26 @@ The system tries each embed in rank order until one succeeds.
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
### For Sources:
|
### For Sources:
|
||||||
|
|
||||||
- Provide multiple embed options when possible
|
- Provide multiple embed options when possible
|
||||||
- Use descriptive embed IDs that match existing embeds
|
- Use descriptive embed IDs that match existing embeds
|
||||||
- Handle both movies and TV shows (combo scraper pattern)
|
- Handle both movies and TV shows (combo scraper pattern)
|
||||||
- Return direct streams when embed processing isn't needed
|
- Return direct streams when embed processing isn't needed
|
||||||
|
|
||||||
### For Embeds:
|
### For Embeds:
|
||||||
|
|
||||||
- Focus on one player type per embed
|
- Focus on one player type per embed
|
||||||
- Handle errors gracefully with clear error messages
|
- Handle errors gracefully with clear error messages
|
||||||
- Use proxy functions for protected streams
|
- Use proxy functions for protected streams
|
||||||
- Include proper headers and flags at both embed and stream levels
|
- Include proper headers and flags
|
||||||
- **Embed flags should match stream flags** for consistent filtering behavior
|
|
||||||
|
|
||||||
### Registration:
|
### Registration:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// In src/providers/all.ts
|
// In src/providers/all.ts
|
||||||
export function gatherAllSources(): Array<Sourcerer> {
|
export function gatherAllSources(): Array<Sourcerer> {
|
||||||
return [catflixScraper, autoembedScraper /* ... */];
|
return [catflixScraper, autoembedScraper, /* ... */];
|
||||||
}
|
}
|
||||||
|
|
||||||
export function gatherAllEmbeds(): Array<Embed> {
|
export function gatherAllEmbeds(): Array<Embed> {
|
||||||
return [turbovidScraper, autoembedEnglishScraper /* ... */];
|
return [turbovidScraper, autoembedEnglishScraper, /* ... */];
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,80 +10,51 @@ Sometimes a source will block netlify or cloudflare. Making self hosted proxies
|
||||||
|
|
||||||
- **`CORS_ALLOWED`**: Headers from the output streams are set to allow any origin.
|
- **`CORS_ALLOWED`**: Headers from the output streams are set to allow any origin.
|
||||||
- **`IP_LOCKED`**: The streams are locked by IP: requester and watcher must be the same.
|
- **`IP_LOCKED`**: The streams are locked by IP: requester and watcher must be the same.
|
||||||
- **`CF_BLOCKED`**: _(Cosmetic)_ Indicates the source/embed blocks Cloudflare IPs. For actual enforcement, remove `CORS_ALLOWED` or add `IP_LOCKED`.
|
- **`CF_BLOCKED`**: *(Cosmetic)* Indicates the source/embed blocks Cloudflare IPs. For actual enforcement, remove `CORS_ALLOWED` or add `IP_LOCKED`.
|
||||||
- **`PROXY_BLOCKED`**: _(Cosmetic)_ Indicates streams shouldn't be proxied. For actual enforcement, remove `CORS_ALLOWED` or add `IP_LOCKED`.
|
- **`PROXY_BLOCKED`**: *(Cosmetic)* Indicates streams shouldn't be proxied. For actual enforcement, remove `CORS_ALLOWED` or add `IP_LOCKED`.
|
||||||
|
|
||||||
## How Flags Affect Target Compatibility
|
## How Flags Affect Target Compatibility
|
||||||
|
|
||||||
### Stream-Level Flags Impact
|
### Stream-Level Flags Impact
|
||||||
|
|
||||||
**With `CORS_ALLOWED`:**
|
**With `CORS_ALLOWED`:**
|
||||||
|
|
||||||
- ✅ Browser targets (can fetch and play streams)
|
- ✅ Browser targets (can fetch and play streams)
|
||||||
- ✅ Extension targets (bypass needed restrictions)
|
- ✅ Extension targets (bypass needed restrictions)
|
||||||
- ✅ Native targets (direct stream access)
|
- ✅ Native targets (direct stream access)
|
||||||
|
|
||||||
**Without `CORS_ALLOWED`:**
|
**Without `CORS_ALLOWED`:**
|
||||||
|
|
||||||
- ❌ Browser targets (CORS restrictions block access)
|
- ❌ Browser targets (CORS restrictions block access)
|
||||||
- ✅ Extension targets (can bypass CORS)
|
- ✅ Extension targets (can bypass CORS)
|
||||||
- ✅ Native targets (no CORS restrictions)
|
- ✅ Native targets (no CORS restrictions)
|
||||||
|
|
||||||
**With `IP_LOCKED`:**
|
**With `IP_LOCKED`:**
|
||||||
|
|
||||||
- ❌ Proxy setups (different IP between request and playback)
|
- ❌ Proxy setups (different IP between request and playback)
|
||||||
- ✅ Direct connections (same IP for request and playback)
|
- ✅ Direct connections (same IP for request and playback)
|
||||||
- ✅ Extension targets (when user has consistent IP)
|
- ✅ Extension targets (when user has consistent IP)
|
||||||
|
|
||||||
**With `CF_BLOCKED` _(cosmetic only)_:**
|
**With `CF_BLOCKED` *(cosmetic only)*:**
|
||||||
|
|
||||||
- 🏷️ Informational label indicating Cloudflare issues
|
- 🏷️ Informational label indicating Cloudflare issues
|
||||||
- ⚠️ **Still requires removing `CORS_ALLOWED` or adding `IP_LOCKED` for actual enforcement**
|
- ⚠️ **Still requires removing `CORS_ALLOWED` or adding `IP_LOCKED` for actual enforcement**
|
||||||
|
|
||||||
**With `PROXY_BLOCKED` _(cosmetic only)_:**
|
**With `PROXY_BLOCKED` *(cosmetic only)*:**
|
||||||
|
|
||||||
- 🏷️ Informational label indicating proxy incompatibility
|
- 🏷️ Informational label indicating proxy incompatibility
|
||||||
- ⚠️ **Still requires removing `CORS_ALLOWED` or adding `IP_LOCKED` for actual enforcement**
|
- ⚠️ **Still requires removing `CORS_ALLOWED` or adding `IP_LOCKED` for actual enforcement**
|
||||||
|
|
||||||
### Provider-Level Flags Impact
|
### Provider-Level Flags Impact
|
||||||
|
|
||||||
**With `CORS_ALLOWED`:**
|
**With `CORS_ALLOWED`:**
|
||||||
|
|
||||||
- Source appears for all target types
|
- Source appears for all target types
|
||||||
- Individual streams still need appropriate flags
|
- Individual streams still need appropriate flags
|
||||||
|
|
||||||
**Without `CORS_ALLOWED`:**
|
**Without `CORS_ALLOWED`:**
|
||||||
|
|
||||||
- Source only appears for extension/native targets
|
- Source only appears for extension/native targets
|
||||||
- Hidden entirely from browser-only users
|
- Hidden entirely from browser-only users
|
||||||
|
|
||||||
### Embed-Level Flags Impact
|
|
||||||
|
|
||||||
**Embed flags work the same way as source flags** - they control embed visibility based on target compatibility. Unlike sources, embeds typically don't provide multiple stream options, so their flags directly determine if the embed is available for a given target.
|
|
||||||
|
|
||||||
**With `CORS_ALLOWED`:**
|
|
||||||
|
|
||||||
- Embed appears for all target types (browser, extension, native)
|
|
||||||
- Works across all playback environments
|
|
||||||
|
|
||||||
**Without `CORS_ALLOWED`:**
|
|
||||||
|
|
||||||
- Embed only appears for extension/native targets
|
|
||||||
- Hidden from browser-only users who can't bypass CORS restrictions
|
|
||||||
|
|
||||||
**With `IP_LOCKED`:**
|
|
||||||
|
|
||||||
- Embed requires consistent IP between request and playback
|
|
||||||
- Only works when `consistentIpForRequests` is true (typically extension setups)
|
|
||||||
|
|
||||||
**Embed flags are automatically derived from stream flags.** When building an embed scraper, the embed should have the same flags as its streams to ensure consistent filtering behavior.
|
|
||||||
|
|
||||||
### Important: Cosmetic vs Enforcement Flags
|
### Important: Cosmetic vs Enforcement Flags
|
||||||
|
|
||||||
**Cosmetic flags** (`CF_BLOCKED`, `PROXY_BLOCKED`) are informational labels only. They don't enforce any behavior.
|
**Cosmetic flags** (`CF_BLOCKED`, `PROXY_BLOCKED`) are informational labels only. They don't enforce any behavior.
|
||||||
|
|
||||||
**Enforcement flags** (`CORS_ALLOWED`, `IP_LOCKED`) actually control stream compatibility:
|
**Enforcement flags** (`CORS_ALLOWED`, `IP_LOCKED`) actually control stream compatibility:
|
||||||
|
|
||||||
- **Remove all flags**: Most common way to make streams extension/native-only (no browser support)
|
- **Remove all flags**: Most common way to make streams extension/native-only (no browser support)
|
||||||
- **Add `IP_LOCKED`**: Prevents proxy usage when `consistentIpForRequests` is false (rarely needed - most extension-only streams just use no flags)
|
- **Add `IP_LOCKED`**: Prevents proxy usage when `consistentIpForRequests` is false (rarely needed - most extension-only streams just use no flags)
|
||||||
|
|
||||||
|
|
@ -96,7 +67,6 @@ Sometimes a source will block netlify or cloudflare. Making self hosted proxies
|
||||||
## Comprehensive Flags Guide
|
## Comprehensive Flags Guide
|
||||||
|
|
||||||
For detailed information about using flags in your scrapers, including:
|
For detailed information about using flags in your scrapers, including:
|
||||||
|
|
||||||
- When and how to use each flag
|
- When and how to use each flag
|
||||||
- Provider-level vs stream-level flags
|
- Provider-level vs stream-level flags
|
||||||
- Best practices and examples
|
- Best practices and examples
|
||||||
|
|
@ -112,121 +82,90 @@ import { createM3U8ProxyUrl } from '@/utils/proxy';
|
||||||
|
|
||||||
// Extension-only streams (MOST COMMON - just remove all flags)
|
// Extension-only streams (MOST COMMON - just remove all flags)
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: createM3U8ProxyUrl(originalUrl, ctx.features, headers),
|
||||||
playlist: createM3U8ProxyUrl(originalUrl, ctx.features, headers),
|
headers,
|
||||||
headers,
|
flags: [], // No flags = extension/native only, but this case doesn't make sense because the stream is getting proxied.
|
||||||
flags: [], // No flags = extension/native only, but this case doesn't make sense because the stream is getting proxied.
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Universal streams with CORS support
|
// Universal streams with CORS support
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: createM3U8ProxyUrl(originalUrl, ctx.features, headers),
|
||||||
playlist: createM3U8ProxyUrl(originalUrl, ctx.features, headers),
|
headers, // again listing headers twice so the extension can use them.
|
||||||
headers, // again listing headers twice so the extension can use them.
|
flags: [flags.CORS_ALLOWED], // Works across all targets
|
||||||
flags: [flags.CORS_ALLOWED], // Works across all targets
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Direct streams (no proxy needed)
|
// Direct streams (no proxy needed)
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: 'https://example.com/playlist.m3u8',
|
||||||
playlist: 'https://example.com/playlist.m3u8',
|
flags: [flags.CORS_ALLOWED], // Stream can be played directly in browsers
|
||||||
flags: [flags.CORS_ALLOWED], // Stream can be played directly in browsers
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Extension-only streams (usual approach - just remove all flags)
|
// Extension-only streams (usual approach - just remove all flags)
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: 'https://example.com/playlist.m3u8',
|
||||||
playlist: 'https://example.com/playlist.m3u8',
|
flags: [], // No flags = extension/native only (most common)
|
||||||
flags: [], // No flags = extension/native only (most common)
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Cloudflare-blocked streams with cosmetic label (if needed)
|
// Cloudflare-blocked streams with cosmetic label (if needed)
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: 'https://example.com/playlist.m3u8',
|
||||||
playlist: 'https://example.com/playlist.m3u8',
|
flags: [flags.CF_BLOCKED], // Cosmetic only - still extension/native only due to no CORS_ALLOWED
|
||||||
flags: [flags.CF_BLOCKED], // Cosmetic only - still extension/native only due to no CORS_ALLOWED
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// IP-locked streams (when you specifically need consistent IP)
|
// IP-locked streams (when you specifically need consistent IP)
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: 'https://example.com/playlist.m3u8',
|
||||||
playlist: 'https://example.com/playlist.m3u8',
|
flags: [flags.IP_LOCKED], // Prevents proxy usage when IP consistency required
|
||||||
flags: [flags.IP_LOCKED], // Prevents proxy usage when IP consistency required
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// IP-locked streams (when you specifically need consistent IP)
|
// IP-locked streams (when you specifically need consistent IP)
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: 'https://example.com/playlist.m3u8',
|
||||||
playlist: 'https://example.com/playlist.m3u8',
|
flags: [flags.IP_LOCKED], // Prevents proxy usage when IP consistency required
|
||||||
flags: [flags.IP_LOCKED], // Prevents proxy usage when IP consistency required
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Provider-level flags affect source visibility
|
// Provider-level flags affect source visibility
|
||||||
export const mySource = makeSourcerer({
|
export const myScraper = makeSourcerer({
|
||||||
id: 'my-source',
|
id: 'my-scraper',
|
||||||
name: 'My Source',
|
name: 'My Scraper',
|
||||||
rank: 150,
|
rank: 150,
|
||||||
flags: [flags.CORS_ALLOWED], // Source shows for all targets
|
flags: [flags.CORS_ALLOWED], // Source shows for all targets
|
||||||
scrapeMovie: comboScraper,
|
scrapeMovie: comboScraper,
|
||||||
scrapeShow: comboScraper,
|
scrapeShow: comboScraper,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Embed-level flags affect embed visibility
|
|
||||||
export const myEmbed = makeEmbed({
|
|
||||||
id: 'my-embed',
|
|
||||||
name: 'My Embed',
|
|
||||||
rank: 150,
|
|
||||||
flags: [flags.CORS_ALLOWED], // Embed shows for all targets
|
|
||||||
scrape: async (ctx) => ({
|
|
||||||
stream: [{
|
|
||||||
id: 'primary',
|
|
||||||
type: 'hls',
|
|
||||||
playlist: createM3U8ProxyUrl(url, ctx.features, headers),
|
|
||||||
flags: [flags.CORS_ALLOWED], // Stream flags should match embed flags
|
|
||||||
captions: [],
|
|
||||||
}],
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -28,44 +28,43 @@ const playlistUrl = 'https://protected-cdn.example.com/playlist.m3u8';
|
||||||
|
|
||||||
// Headers required to access the playlist
|
// Headers required to access the playlist
|
||||||
const headers = {
|
const headers = {
|
||||||
Referer: 'https://player.example.com/',
|
'Referer': 'https://player.example.com/',
|
||||||
Origin: 'https://player.example.com',
|
'Origin': 'https://player.example.com',
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||||
};
|
};
|
||||||
|
|
||||||
// Convert playlist and all variants to data URLs
|
// Convert playlist and all variants to data URLs
|
||||||
const dataUrl = await convertPlaylistsToDataUrls(ctx.proxiedFetcher, playlistUrl, headers);
|
const dataUrl = await convertPlaylistsToDataUrls(
|
||||||
|
ctx.proxiedFetcher,
|
||||||
|
playlistUrl,
|
||||||
|
headers
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: dataUrl, // Self-contained data URL
|
||||||
playlist: dataUrl, // Self-contained data URL
|
flags: [flags.CORS_ALLOWED], // No CORS issues with data URLs
|
||||||
flags: [flags.CORS_ALLOWED], // No CORS issues with data URLs
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
**Why data URLs work for CORS bypass:**
|
**Why data URLs work for CORS bypass:**
|
||||||
|
- **Inital proxied request WITH HEADERS**: A request is sent from the proxy *with* headers allowing for the playlist to load
|
||||||
- **Inital proxied request WITH HEADERS**: A request is sent from the proxy _with_ headers allowing for the playlist to load
|
|
||||||
- **Fewer external requests**: Content is embedded directly in the URL as base64
|
- **Fewer external requests**: Content is embedded directly in the URL as base64
|
||||||
- **Same-origin**: Browsers treat data URLs as same-origin content
|
- **Same-origin**: Browsers treat data URLs as same-origin content
|
||||||
- **Complete isolation**: No network requests means no CORS preflight checks
|
- **Complete isolation**: No network requests means no CORS preflight checks
|
||||||
- **Self-contained**: All playlist data and segments are embedded in the response
|
- **Self-contained**: All playlist data and segments are embedded in the response
|
||||||
|
|
||||||
**How the conversion works:**
|
**How the conversion works:**
|
||||||
|
|
||||||
1. Fetches the master playlist using provided headers
|
1. Fetches the master playlist using provided headers
|
||||||
2. For each quality variant, fetches the variant playlist
|
2. For each quality variant, fetches the variant playlist
|
||||||
3. Converts all playlists to base64-encoded data URLs
|
3. Converts all playlists to base64-encoded data URLs
|
||||||
4. Returns a master data URL containing all embedded variants
|
4. Returns a master data URL containing all embedded variants
|
||||||
|
|
||||||
**When to use data URLs vs M3U8 proxy:**
|
**When to use data URLs vs M3U8 proxy:**
|
||||||
|
|
||||||
- **Use data URLs** when you can fetch all playlist data upfront
|
- **Use data URLs** when you can fetch all playlist data upfront
|
||||||
- **Use M3U8 proxy** when playlists are too large or change frequently
|
- **Use M3U8 proxy** when playlists are too large or change frequently
|
||||||
- **Data URLs are preferred** for most HLS streams due to simplicity and reliability
|
- **Data URLs are preferred** for most HLS streams due to simplicity and reliability
|
||||||
|
|
@ -85,22 +84,20 @@ const originalPlaylist = 'https://protected-cdn.example.com/playlist.m3u8';
|
||||||
|
|
||||||
// Headers required by the streaming service
|
// Headers required by the streaming service
|
||||||
const streamHeaders = {
|
const streamHeaders = {
|
||||||
Referer: 'https://player.example.com/',
|
'Referer': 'https://player.example.com/',
|
||||||
Origin: 'https://player.example.com',
|
'Origin': 'https://player.example.com',
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||||
};
|
};
|
||||||
|
|
||||||
return {
|
return {
|
||||||
stream: [
|
stream: [{
|
||||||
{
|
id: 'primary',
|
||||||
id: 'primary',
|
type: 'hls',
|
||||||
type: 'hls',
|
playlist: createM3U8ProxyUrl(originalPlaylist, ctx.features, streamHeaders), // Include headers for proxy usage
|
||||||
playlist: createM3U8ProxyUrl(originalPlaylist, ctx.features, streamHeaders), // Include headers for proxy usage
|
headers: streamHeaders, // Include headers for extension/native usage
|
||||||
headers: streamHeaders, // Include headers for extension/native usage
|
flags: [flags.CORS_ALLOWED], // Proxy enables CORS for all targets
|
||||||
flags: [flags.CORS_ALLOWED], // Proxy enables CORS for all targets
|
captions: []
|
||||||
captions: [],
|
}]
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -118,13 +115,12 @@ const SKIP_VALIDATION_CHECK_IDS = [
|
||||||
// Sources here are always proxied, so we dont need to validate with a proxy, but we should fetch nativly
|
// Sources here are always proxied, so we dont need to validate with a proxy, but we should fetch nativly
|
||||||
// NOTE: all m3u8 proxy urls are automatically processed using this method, so no need to add them here manually
|
// NOTE: all m3u8 proxy urls are automatically processed using this method, so no need to add them here manually
|
||||||
const UNPROXIED_VALIDATION_CHECK_IDS = [
|
const UNPROXIED_VALIDATION_CHECK_IDS = [
|
||||||
// ... existing IDs
|
// ... existing IDs
|
||||||
'your-scraper-id', // Add your scraper ID here
|
'your-scraper-id', // Add your scraper ID here
|
||||||
];
|
];
|
||||||
```
|
```
|
||||||
|
|
||||||
**Why this is needed:**
|
**Why this is needed:**
|
||||||
|
|
||||||
- By default, all streams are validated by attempting to fetch metadata
|
- By default, all streams are validated by attempting to fetch metadata
|
||||||
- The validation uses `proxiedFetcher` to check if streams are playable
|
- The validation uses `proxiedFetcher` to check if streams are playable
|
||||||
- If the stream blocks the fetcher, validation will fail
|
- If the stream blocks the fetcher, validation will fail
|
||||||
|
|
@ -132,7 +128,6 @@ const UNPROXIED_VALIDATION_CHECK_IDS = [
|
||||||
- Adding to skip list bypasses validation and returns the proxied URL directly without checking it
|
- Adding to skip list bypasses validation and returns the proxied URL directly without checking it
|
||||||
|
|
||||||
**When to skip validation:**
|
**When to skip validation:**
|
||||||
|
|
||||||
- Validation consistently fails but streams work in browsers
|
- Validation consistently fails but streams work in browsers
|
||||||
- The stream may be origin or IP locked
|
- The stream may be origin or IP locked
|
||||||
- The stream blocks the extension or proxy
|
- The stream blocks the extension or proxy
|
||||||
|
|
@ -148,13 +143,13 @@ let stream = {
|
||||||
type: 'file',
|
type: 'file',
|
||||||
flags: [],
|
flags: [],
|
||||||
qualities: {
|
qualities: {
|
||||||
'1080p': { url: 'https://protected-cdn.example.com/video.mp4' },
|
'1080p': { url: 'https://protected-cdn.example.com/video.mp4' }
|
||||||
},
|
},
|
||||||
headers: {
|
headers: {
|
||||||
Referer: 'https://player.example.com/',
|
'Referer': 'https://player.example.com/',
|
||||||
'User-Agent': 'Mozilla/5.0...',
|
'User-Agent': 'Mozilla/5.0...'
|
||||||
},
|
},
|
||||||
captions: [],
|
captions: []
|
||||||
};
|
};
|
||||||
|
|
||||||
// setupProxy will handle proxying if needed
|
// setupProxy will handle proxying if needed
|
||||||
|
|
@ -168,19 +163,15 @@ return { stream: [stream] };
|
||||||
### Efficient Data Extraction
|
### Efficient Data Extraction
|
||||||
|
|
||||||
**Use targeted selectors:**
|
**Use targeted selectors:**
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// ✅ Good - specific selector
|
// ✅ Good - specific selector
|
||||||
const embedUrl = $('iframe[src*="turbovid"]').attr('src');
|
const embedUrl = $('iframe[src*="turbovid"]').attr('src');
|
||||||
|
|
||||||
// ❌ Bad - searches entire document
|
// ❌ Bad - searches entire document
|
||||||
const embedUrl = $('*')
|
const embedUrl = $('*').filter((_, el) => $(el).attr('src')?.includes('turbovid')).attr('src');
|
||||||
.filter((_, el) => $(el).attr('src')?.includes('turbovid'))
|
|
||||||
.attr('src');
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Cache expensive operations:**
|
**Cache expensive operations:**
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Cache parsed data to avoid re-parsing
|
// Cache parsed data to avoid re-parsing
|
||||||
let cachedConfig;
|
let cachedConfig;
|
||||||
|
|
@ -192,7 +183,6 @@ if (!cachedConfig) {
|
||||||
### Minimize HTTP Requests
|
### Minimize HTTP Requests
|
||||||
|
|
||||||
**Combine operations when possible:**
|
**Combine operations when possible:**
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// ✅ Good - single request with full processing
|
// ✅ Good - single request with full processing
|
||||||
const embedPage = await ctx.proxiedFetcher(embedUrl);
|
const embedPage = await ctx.proxiedFetcher(embedUrl);
|
||||||
|
|
@ -210,7 +200,6 @@ const streams = extractStreams(page2);
|
||||||
### Input Validation
|
### Input Validation
|
||||||
|
|
||||||
**Validate external data:**
|
**Validate external data:**
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Validate URLs before using them
|
// Validate URLs before using them
|
||||||
const isValidUrl = (url: string) => {
|
const isValidUrl = (url: string) => {
|
||||||
|
|
@ -228,7 +217,6 @@ if (!isValidUrl(streamUrl)) {
|
||||||
```
|
```
|
||||||
|
|
||||||
**Sanitize regex inputs:**
|
**Sanitize regex inputs:**
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Be careful with dynamic regex
|
// Be careful with dynamic regex
|
||||||
const safeTitle = ctx.media.title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
const safeTitle = ctx.media.title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,3 @@
|
||||||
icon: ph:atom-fill
|
icon: ph:atom-fill
|
||||||
navigation.redirect: /in-depth/new-providers
|
navigation.redirect: /in-depth/new-providers
|
||||||
navigation.title: 'In-depth'
|
navigation.title: "In-depth"
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,3 @@
|
||||||
icon: ph:aperture-fill
|
icon: ph:aperture-fill
|
||||||
navigation.redirect: /extra-topics/development
|
navigation.redirect: /extra-topics/development
|
||||||
navigation.title: 'Extra topics'
|
navigation.title: "Extra topics"
|
||||||
|
|
|
||||||
|
|
@ -4,30 +4,28 @@ Run all providers one by one in order of their built-in ranking.
|
||||||
You can attach events if you need to know what is going on while it is processing.
|
You can attach events if you need to know what is going on while it is processing.
|
||||||
|
|
||||||
## Example
|
## Example
|
||||||
|
|
||||||
(Use the cli to learn more specifics about inputs)
|
(Use the cli to learn more specifics about inputs)
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
// media from TMDB
|
// media from TMDB
|
||||||
const media = {
|
const media = {
|
||||||
type: 'movie', // "movie" | "show"
|
type: 'movie', // "movie" | "show"
|
||||||
title: 'Hamilton',
|
title: 'Hamilton',
|
||||||
tmdbId: '556574',
|
tmdbId: '556574',
|
||||||
// season: '1',
|
// season: '1',
|
||||||
// episode: '1'
|
// episode: '1'
|
||||||
};
|
}
|
||||||
|
|
||||||
// scrape a stream
|
// scrape a stream
|
||||||
const stream = await providers.runAll({
|
const stream = await providers.runAll({
|
||||||
media: media,
|
media: media,
|
||||||
});
|
})
|
||||||
|
|
||||||
// scrape a stream, but prioritize flixhq above all
|
// scrape a stream, but prioritize flixhq above all
|
||||||
// (other scrapers are still run if flixhq fails, it just has priority)
|
// (other scrapers are still run if flixhq fails, it just has priority)
|
||||||
const flixhqStream = await providers.runAll({
|
const flixhqStream = await providers.runAll({
|
||||||
media: media,
|
media: media,
|
||||||
sourceOrder: ['flixhq'],
|
sourceOrder: ['flixhq']
|
||||||
});
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
## Type
|
## Type
|
||||||
|
|
|
||||||
|
|
@ -12,8 +12,8 @@ const media = {
|
||||||
type: 'movie',
|
type: 'movie',
|
||||||
title: 'Hamilton',
|
title: 'Hamilton',
|
||||||
releaseYear: 2020,
|
releaseYear: 2020,
|
||||||
tmdbId: '556574',
|
tmdbId: '556574'
|
||||||
};
|
}
|
||||||
|
|
||||||
// scrape a stream from flixhq
|
// scrape a stream from flixhq
|
||||||
let output: SourcererOutput;
|
let output: SourcererOutput;
|
||||||
|
|
@ -21,12 +21,12 @@ try {
|
||||||
output = await providers.runSourceScraper({
|
output = await providers.runSourceScraper({
|
||||||
id: 'flixhq',
|
id: 'flixhq',
|
||||||
media: media,
|
media: media,
|
||||||
});
|
})
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof NotFoundError) {
|
if (err instanceof NotFoundError) {
|
||||||
console.log('source does not have this media');
|
console.log('source does not have this media');
|
||||||
} else {
|
} else {
|
||||||
console.log('failed to scrape');
|
console.log('failed to scrape')
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -13,9 +13,9 @@ try {
|
||||||
output = await providers.runEmbedScraper({
|
output = await providers.runEmbedScraper({
|
||||||
id: 'upcloud',
|
id: 'upcloud',
|
||||||
url: 'https://example.com/123',
|
url: 'https://example.com/123',
|
||||||
});
|
})
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.log('failed to scrape');
|
console.log('failed to scrape')
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ Make a fetcher to use with [p-stream/simple-proxy](https://github.com/p-stream/s
|
||||||
```ts
|
```ts
|
||||||
import { targets, makeProviders, makeDefaultFetcher, makeSimpleProxyFetcher } from '@p-stream/providers';
|
import { targets, makeProviders, makeDefaultFetcher, makeSimpleProxyFetcher } from '@p-stream/providers';
|
||||||
|
|
||||||
const proxyUrl = 'https://your.proxy.workers.dev/';
|
const proxyUrl = 'https://your.proxy.workers.dev/'
|
||||||
|
|
||||||
const providers = makeProviders({
|
const providers = makeProviders({
|
||||||
fetcher: makeDefaultFetcher(fetch),
|
fetcher: makeDefaultFetcher(fetch),
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,3 @@
|
||||||
icon: ph:code-simple-fill
|
icon: ph:code-simple-fill
|
||||||
navigation.redirect: /api/makeproviders
|
navigation.redirect: /api/makeproviders
|
||||||
navigation.title: 'Api reference'
|
navigation.title: "Api reference"
|
||||||
|
|
|
||||||
|
|
@ -2,16 +2,20 @@ export default defineNuxtConfig({
|
||||||
// https://github.com/nuxt-themes/docus
|
// https://github.com/nuxt-themes/docus
|
||||||
extends: '@nuxt-themes/docus',
|
extends: '@nuxt-themes/docus',
|
||||||
|
|
||||||
css: ['@/assets/css/main.css'],
|
css: [
|
||||||
|
'@/assets/css/main.css',
|
||||||
|
],
|
||||||
|
|
||||||
build: {
|
build: {
|
||||||
transpile: ['chalk'],
|
transpile: [
|
||||||
|
"chalk"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
|
|
||||||
modules: [
|
modules: [
|
||||||
// https://github.com/nuxt-modules/plausible
|
// https://github.com/nuxt-modules/plausible
|
||||||
'@nuxtjs/plausible',
|
'@nuxtjs/plausible',
|
||||||
// https://github.com/nuxt/devtools
|
// https://github.com/nuxt/devtools
|
||||||
'@nuxt/devtools',
|
'@nuxt/devtools'
|
||||||
],
|
]
|
||||||
});
|
})
|
||||||
|
|
|
||||||
4744
.docs/pnpm-lock.yaml
4744
.docs/pnpm-lock.yaml
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,8 @@
|
||||||
{
|
{
|
||||||
"extends": ["@nuxtjs"],
|
"extends": [
|
||||||
"lockFileMaintenance": {
|
"@nuxtjs"
|
||||||
"enabled": true
|
],
|
||||||
}
|
"lockFileMaintenance": {
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,18 @@
|
||||||
import { defineTheme } from 'pinceau';
|
import { defineTheme } from 'pinceau'
|
||||||
|
|
||||||
export default defineTheme({
|
export default defineTheme({
|
||||||
color: {
|
color: {
|
||||||
primary: {
|
primary: {
|
||||||
50: '#F5E5FF',
|
50: "#F5E5FF",
|
||||||
100: '#E7CCFF',
|
100: "#E7CCFF",
|
||||||
200: '#D4A9FF',
|
200: "#D4A9FF",
|
||||||
300: '#BE85FF',
|
300: "#BE85FF",
|
||||||
400: '#A861FF',
|
400: "#A861FF",
|
||||||
500: '#8E3DFF',
|
500: "#8E3DFF",
|
||||||
600: '#7F36D4',
|
600: "#7F36D4",
|
||||||
700: '#662CA6',
|
700: "#662CA6",
|
||||||
800: '#552578',
|
800: "#552578",
|
||||||
900: '#441E49',
|
900: "#441E49"
|
||||||
},
|
}
|
||||||
},
|
}
|
||||||
});
|
})
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue