mirror of
https://github.com/p-stream/providers.git
synced 2026-01-11 12:00:46 +00:00
fix docs
This commit is contained in:
parent
562ee54e1c
commit
add7dc5b3f
30 changed files with 2306 additions and 3672 deletions
|
|
@ -3,6 +3,6 @@ module.exports = {
|
|||
extends: '@nuxt/eslint-config',
|
||||
rules: {
|
||||
'vue/max-attributes-per-line': 'off',
|
||||
'vue/multi-word-component-names': 'off',
|
||||
},
|
||||
};
|
||||
'vue/multi-word-component-names': 'off'
|
||||
}
|
||||
}
|
||||
|
|
@ -1,20 +1,18 @@
|
|||
---
|
||||
title: '@p-stream/providers | For all your media scraping needs'
|
||||
title: "@p-stream/providers | For all your media scraping needs"
|
||||
navigation: false
|
||||
layout: page
|
||||
---
|
||||
|
||||
## ::block-hero
|
||||
|
||||
::block-hero
|
||||
---
|
||||
cta:
|
||||
|
||||
- Get Started
|
||||
- /get-started/introduction
|
||||
secondary:
|
||||
- Open on GitHub →
|
||||
- https://github.com/p-stream/providers
|
||||
snippet: npm i @p-stream/providers@github:p-stream/providers
|
||||
|
||||
- Get Started
|
||||
- /get-started/introduction
|
||||
secondary:
|
||||
- Open on GitHub →
|
||||
- https://github.com/p-stream/providers
|
||||
snippet: npm i @p-stream/providers@github:p-stream/providers
|
||||
---
|
||||
|
||||
#title
|
||||
|
|
@ -32,22 +30,22 @@ What's included
|
|||
:ellipsis
|
||||
|
||||
#default
|
||||
::card{icon="vscode-icons:file-type-light-json"}
|
||||
#title
|
||||
Scrape popular streaming websites.
|
||||
#description
|
||||
Don't settle for just one media site for you content, use everything that's available.
|
||||
::
|
||||
::card{icon="codicon:source-control"}
|
||||
#title
|
||||
Multi-platform.
|
||||
#description
|
||||
Scrape from browser or server, whichever you prefer.
|
||||
::
|
||||
::card{icon="logos:typescript-icon-round"}
|
||||
#title
|
||||
Easy to use.
|
||||
#description
|
||||
Get started with scraping your favourite media sites with just 5 lines of code. Fully typed of course.
|
||||
::
|
||||
::card{icon="vscode-icons:file-type-light-json"}
|
||||
#title
|
||||
Scrape popular streaming websites.
|
||||
#description
|
||||
Don't settle for just one media site for you content, use everything that's available.
|
||||
::
|
||||
::card{icon="codicon:source-control"}
|
||||
#title
|
||||
Multi-platform.
|
||||
#description
|
||||
Scrape from browser or server, whichever you prefer.
|
||||
::
|
||||
::card{icon="logos:typescript-icon-round"}
|
||||
#title
|
||||
Easy to use.
|
||||
#description
|
||||
Get started with scraping your favourite media sites with just 5 lines of code. Fully typed of course.
|
||||
::
|
||||
::
|
||||
|
|
|
|||
|
|
@ -3,35 +3,29 @@ title: 'Changelog'
|
|||
---
|
||||
|
||||
# Version 2.3.0
|
||||
|
||||
- Fixed RidoMovies search results
|
||||
- Added Insertunit, SoaperTV, and WarezCDN providers
|
||||
- Disabled Showbox and VidSrc
|
||||
|
||||
# Version 2.2.9
|
||||
|
||||
- Fixed VidSrcTo (both Vidplay and Filemoon embeds)
|
||||
- Fixed VidSrcTo (both Vidplay and Filemoon embeds)
|
||||
- Added dropload, filelions and vtube embeds to Primewire
|
||||
- Fixed and enabled Smashystream
|
||||
- Improved RidoMovies search results
|
||||
|
||||
# Version 2.2.8
|
||||
|
||||
- Fix package exports for CJS and ESM
|
||||
- Fixed Mixdrop embed
|
||||
- Added thumbnailTrack to Vidplay embed
|
||||
|
||||
# Version 2.2.7
|
||||
|
||||
- Fix showbox
|
||||
|
||||
# Version 2.2.6
|
||||
|
||||
- Fix febbox
|
||||
- Validate if a stream is actually playable. Streams that are not responding are no longer returned.
|
||||
|
||||
# Version 2.2.5
|
||||
|
||||
- Add Primewire provider
|
||||
- Improve VidSrcTo search results
|
||||
- Fixed Filemoon embeds
|
||||
|
|
@ -40,11 +34,9 @@ title: 'Changelog'
|
|||
- Reordered providers in ranking
|
||||
|
||||
# Version 2.2.4
|
||||
|
||||
- Hotfix for HDRezka provider
|
||||
|
||||
# Version 2.2.3
|
||||
|
||||
- Fix VidSrcTo
|
||||
- Add HDRezka provider
|
||||
- Fix Goojara causing a crash
|
||||
|
|
@ -52,18 +44,15 @@ title: 'Changelog'
|
|||
- Cover an edge case where the title contains 'the movie' or 'the show'
|
||||
|
||||
# Version 2.2.2
|
||||
|
||||
- Fix subtitles not appearing if the name of the subtitle is in its native tongue.
|
||||
- Remove references to the old domain
|
||||
- Fixed ridomovies not working for some shows and movies
|
||||
- Fixed Showbox not working in react-native.
|
||||
|
||||
# Version 2.2.1
|
||||
|
||||
- Fixed Closeload scraper
|
||||
|
||||
# Version 2.2.0
|
||||
|
||||
- Fixed vidsrc.me URL decoding.
|
||||
- Added ridomovies with Ridoo and Closeload embed.
|
||||
- Added Goojara.to source.
|
||||
|
|
@ -74,23 +63,19 @@ title: 'Changelog'
|
|||
- Disabled Lookmovie and swapped Showbox and VidSrcTo in ranking.
|
||||
|
||||
# Version 2.1.1
|
||||
|
||||
- Fixed vidplay decryption keys being wrong and switched the domain to one that works
|
||||
- Fixed vidplay decryption keys being wrong and switched the domain to one that works
|
||||
|
||||
# Version 2.1.0
|
||||
|
||||
- Add preferedHeaders to most sources
|
||||
- Add CF_BLOCKED flag to sources that have blocked cloudflare API's
|
||||
- Fix vidsrc sometimes having an equal sign where it shouldnt
|
||||
- Increase ranking of lookmovie
|
||||
- Re-enabled subtitles for febbox-mp4
|
||||
- Add preferedHeaders to most sources
|
||||
- Add CF_BLOCKED flag to sources that have blocked cloudflare API's
|
||||
- Fix vidsrc sometimes having an equal sign where it shouldnt
|
||||
- Increase ranking of lookmovie
|
||||
- Re-enabled subtitles for febbox-mp4
|
||||
|
||||
# Version 2.0.5
|
||||
|
||||
- Disable subtitles for febbox-mp4. As their endpoint doesn't work anymore.
|
||||
|
||||
# Version 2.0.4
|
||||
|
||||
- Added providers:
|
||||
- Add VidSrcTo provider with Vidplay and Filemoon embeds
|
||||
- Add VidSrc provider with StreamBucket embeds
|
||||
|
|
@ -104,17 +89,14 @@ title: 'Changelog'
|
|||
- Added utility to not return multiple subs for the same language - Applies to Lookmovie and Showbox
|
||||
|
||||
# Version 2.0.3
|
||||
|
||||
- Actually remove Febbox HLS
|
||||
- Actually remove Febbox HLS
|
||||
|
||||
# Version 2.0.2
|
||||
|
||||
- Added Lookmovie caption support
|
||||
- Fix Febbox duplicate subtitle languages
|
||||
- Remove Febbox HLS
|
||||
|
||||
# Version 2.0.1
|
||||
|
||||
- Fixed issue where febbox-mp4 would not show all qualities
|
||||
- Fixed issue where discoverEmbeds event would not show the embeds in the right order
|
||||
|
||||
|
|
@ -125,7 +107,6 @@ There are breaking changes in this list, make sure to read them thoroughly if yo
|
|||
::
|
||||
|
||||
**Development tooling:**
|
||||
|
||||
- Added integration test for browser. To make sure the package keeps working in the browser
|
||||
- Add type checking when building, previously it ignored them
|
||||
- Refactored the main folder, now called entrypoint.
|
||||
|
|
@ -135,13 +116,11 @@ There are breaking changes in this list, make sure to read them thoroughly if yo
|
|||
- Fetchers can now return a full response with headers and everything
|
||||
|
||||
**New features:**
|
||||
|
||||
- Added system to allow scraping IP locked sources through the consistentIpforRequests option.
|
||||
- There is now a `buildProviders()` function that gives a builder for the `ProviderControls`. It's an alternative to `makeProviders()`.
|
||||
- Streams can now return a headers object and a `preferredHeaders` object. which is required and optional headers for when using the stream.
|
||||
|
||||
**Notable changes:**
|
||||
|
||||
- Renamed the NO_CORS flag to CORS_ALLOWED (meaning that resource sharing is allowed)
|
||||
- Export Fetcher and Stream types with all types related to it
|
||||
- Providers can now return a list of streams instead of just one.
|
||||
|
|
|
|||
|
|
@ -7,9 +7,8 @@
|
|||
## What can I use this on?
|
||||
|
||||
We support many different environments, here are a few examples:
|
||||
|
||||
- In browser, watch streams without needing a server to scrape (does need a proxy)
|
||||
- In a native app, scrape in the app itself
|
||||
- In a backend server, scrape on the server and give the streams to the client to watch.
|
||||
- In browser, watch streams without needing a server to scrape (does need a proxy)
|
||||
- In a native app, scrape in the app itself
|
||||
- In a backend server, scrape on the server and give the streams to the client to watch.
|
||||
|
||||
To find out how to configure the library for your environment, You can read [How to use on X](../2.essentials/0.usage-on-x.md).
|
||||
|
|
|
|||
|
|
@ -9,19 +9,15 @@ Let's get started with `@p-stream/providers`. First lets install the package.
|
|||
::
|
||||
|
||||
::code-group
|
||||
|
||||
```bash [NPM]
|
||||
npm install @p-stream/providers@github:p-stream/providers#production
|
||||
```
|
||||
|
||||
```bash [Yarn]
|
||||
yarn add @p-stream/providers@github:p-stream/providers#production
|
||||
```
|
||||
|
||||
```bash [PNPM]
|
||||
pnpm add @p-stream/providers@github:p-stream/providers#production
|
||||
```
|
||||
|
||||
```bash [NPM]
|
||||
npm install @p-stream/providers@github:p-stream/providers#production
|
||||
```
|
||||
```bash [Yarn]
|
||||
yarn add @p-stream/providers@github:p-stream/providers#production
|
||||
```
|
||||
```bash [PNPM]
|
||||
pnpm add @p-stream/providers@github:p-stream/providers#production
|
||||
```
|
||||
::
|
||||
|
||||
## Scrape your first item
|
||||
|
|
@ -43,8 +39,8 @@ const providers = makeProviders({
|
|||
fetcher: myFetcher,
|
||||
|
||||
// will be played on a native video player
|
||||
target: targets.NATIVE,
|
||||
});
|
||||
target: targets.NATIVE
|
||||
})
|
||||
```
|
||||
|
||||
Perfect. You now have an instance of the providers you can reuse everywhere.
|
||||
|
|
@ -54,14 +50,14 @@ Now let's scrape an item:
|
|||
// fetch some data from TMDB
|
||||
const media = {
|
||||
type: 'movie',
|
||||
title: 'Hamilton',
|
||||
title: "Hamilton",
|
||||
releaseYear: 2020,
|
||||
tmdbId: '556574',
|
||||
};
|
||||
|
||||
tmdbId: "556574"
|
||||
}
|
||||
|
||||
const output = await providers.runAll({
|
||||
media: media,
|
||||
});
|
||||
media: media
|
||||
})
|
||||
```
|
||||
|
||||
Now we have our stream in the output variable. (If the output is `null` then nothing could be found.)
|
||||
|
|
|
|||
|
|
@ -3,25 +3,22 @@
|
|||
The library can run in many environments, so it can be tricky to figure out how to set it up.
|
||||
|
||||
Here is a checklist. For more specific environments, keep reading below:
|
||||
|
||||
- When requests are very restricted (like browser client-side). Configure a proxied fetcher.
|
||||
- When your requests come from the same device on which it will be streamed (not compatible with proxied fetcher). Set `consistentIpForRequests: true`.
|
||||
- To set a target. Consult [Targets](./1.targets.md).
|
||||
- When requests are very restricted (like browser client-side). Configure a proxied fetcher.
|
||||
- When your requests come from the same device on which it will be streamed (not compatible with proxied fetcher). Set `consistentIpForRequests: true`.
|
||||
- To set a target. Consult [Targets](./1.targets.md).
|
||||
|
||||
To make use of the examples below, check out the following pages:
|
||||
|
||||
- [Quick start](../1.get-started/1.quick-start.md)
|
||||
- [Using streams](../2.essentials/4.using-streams.md)
|
||||
- [Quick start](../1.get-started/1.quick-start.md)
|
||||
- [Using streams](../2.essentials/4.using-streams.md)
|
||||
|
||||
## NodeJs server
|
||||
|
||||
```ts
|
||||
import { makeProviders, makeStandardFetcher, targets } from '@p-stream/providers';
|
||||
|
||||
const providers = makeProviders({
|
||||
fetcher: makeStandardFetcher(fetch),
|
||||
target: chooseYourself, // check out https://p-stream.github.io/providers/essentials/targets
|
||||
});
|
||||
})
|
||||
```
|
||||
|
||||
## Browser client-side
|
||||
|
|
@ -32,27 +29,24 @@ Read more [about proxy fetchers](./2.fetchers.md#using-fetchers-on-the-browser).
|
|||
```ts
|
||||
import { makeProviders, makeStandardFetcher, targets } from '@p-stream/providers';
|
||||
|
||||
const proxyUrl = 'https://your.proxy.workers.dev/';
|
||||
const proxyUrl = "https://your.proxy.workers.dev/";
|
||||
|
||||
const providers = makeProviders({
|
||||
fetcher: makeStandardFetcher(fetch),
|
||||
proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch),
|
||||
target: target.BROWSER,
|
||||
});
|
||||
})
|
||||
```
|
||||
|
||||
## React native
|
||||
|
||||
To use the library in a react native app, you would also need a couple of polyfills to polyfill crypto and base64.
|
||||
|
||||
1. First install the polyfills:
|
||||
|
||||
```bash
|
||||
npm install @react-native-anywhere/polyfill-base64 react-native-quick-crypto
|
||||
```
|
||||
|
||||
2. Add the polyfills to your app:
|
||||
|
||||
```ts
|
||||
// Import in your entry file
|
||||
import '@react-native-anywhere/polyfill-base64';
|
||||
|
|
@ -69,5 +63,5 @@ const providers = makeProviders({
|
|||
fetcher: makeStandardFetcher(fetch),
|
||||
target: target.NATIVE,
|
||||
consistentIpForRequests: true,
|
||||
});
|
||||
})
|
||||
```
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ A target is the device on which the stream will be played.
|
|||
::
|
||||
|
||||
#### Possible targets
|
||||
|
||||
- **`targets.BROWSER`** Stream will be played in a browser with CORS
|
||||
- **`targets.BROWSER_EXTENSION`** Stream will be played in a browser using the p-stream extension (WIP)
|
||||
- **`targets.NATIVE`** Stream will be played on a native video player
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ When creating provider controls, a fetcher will need to be configured.
|
|||
Depending on your environment, this can come with some considerations:
|
||||
|
||||
## Using `fetch()`
|
||||
|
||||
In most cases, you can use the `fetch()` API. This will work in newer versions of Node.js (18 and above) and on the browser.
|
||||
|
||||
```ts
|
||||
|
|
@ -14,19 +13,18 @@ const fetcher = makeStandardFetcher(fetch);
|
|||
If you using older version of Node.js. You can use the npm package `node-fetch` to polyfill fetch:
|
||||
|
||||
```ts
|
||||
import fetch from 'node-fetch';
|
||||
import fetch from "node-fetch";
|
||||
|
||||
const fetcher = makeStandardFetcher(fetch);
|
||||
```
|
||||
|
||||
## Using fetchers on the browser
|
||||
|
||||
When using this library on a browser, you will need a proxy. Browsers restrict when a web request can be made. To bypass those restrictions, you will need a CORS proxy.
|
||||
|
||||
The p-stream team has a proxy pre-made and pre-configured for you to use. For more information, check out [p-stream/simple-proxy](https://github.com/p-stream/simple-proxy). After installing, you can use this proxy like so:
|
||||
|
||||
```ts
|
||||
const fetcher = makeSimpleProxyFetcher('https://your.proxy.workers.dev/', fetch);
|
||||
const fetcher = makeSimpleProxyFetcher("https://your.proxy.workers.dev/", fetch);
|
||||
```
|
||||
|
||||
If you aren't able to use this specific proxy and need to use a different one, you can make your own fetcher in the next section.
|
||||
|
|
@ -49,15 +47,15 @@ export function makeCustomFetcher(): Fetcher {
|
|||
|
||||
If you need to make your own fetcher for a proxy, ensure you make it compatible with the following headers: `Set-Cookie`, `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write/read those headers when making a request.
|
||||
|
||||
|
||||
## Making a fetcher from scratch
|
||||
|
||||
In some rare cases, you need to make a fetcher from scratch.
|
||||
This is the list of features it needs:
|
||||
|
||||
- Send/read every header
|
||||
- Parse JSON, otherwise parse as text
|
||||
- Send JSON, Formdata or normal strings
|
||||
- get final destination URL
|
||||
- Send/read every header
|
||||
- Parse JSON, otherwise parse as text
|
||||
- Send JSON, Formdata or normal strings
|
||||
- get final destination URL
|
||||
|
||||
It's not recommended to do this at all. If you have to, you can base your code on the original implementation of `makeStandardFetcher`. Check out the [source code for it here](https://github.com/p-stream/providers/blob/dev/src/fetchers/standardFetch.ts).
|
||||
|
||||
|
|
@ -72,5 +70,5 @@ const myFetcher: Fetcher = (url, ops) => {
|
|||
headers: new Headers(), // should only contain headers from ops.readHeaders
|
||||
statusCode: 200,
|
||||
};
|
||||
};
|
||||
}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ To know what to set the configuration to, you can read [How to use on X](./0.usa
|
|||
const providers = makeProviders({
|
||||
// fetcher, every web request gets called through here
|
||||
fetcher: makeStandardFetcher(fetch),
|
||||
|
||||
|
||||
// proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead
|
||||
// of the normal fetcher. Defaults to the normal fetcher.
|
||||
proxiedFetcher: undefined;
|
||||
|
|
@ -52,6 +52,7 @@ const providers = buildProviders()
|
|||
.build();
|
||||
```
|
||||
|
||||
|
||||
### Adding your own scrapers to the providers
|
||||
|
||||
If you have your own scraper and still want to use the nice utilities of the provider library or just want to add on to the built-in providers, you can add your own custom source.
|
||||
|
|
@ -60,15 +61,14 @@ If you have your own scraper and still want to use the nice utilities of the pro
|
|||
const providers = buildProviders()
|
||||
.setTarget(targets.NATIVE) // target of where the streams will be used
|
||||
.setFetcher(makeStandardFetcher(fetch)) // fetcher, every web request gets called through here
|
||||
.addSource({
|
||||
// add your own source
|
||||
.addSource({ // add your own source
|
||||
id: 'my-scraper',
|
||||
name: 'My scraper',
|
||||
rank: 800,
|
||||
flags: [],
|
||||
scrapeMovie(ctx) {
|
||||
throw new Error('Not implemented');
|
||||
},
|
||||
}
|
||||
})
|
||||
.build();
|
||||
```
|
||||
|
|
|
|||
|
|
@ -5,13 +5,12 @@ Streams can sometimes be quite picky on how they can be used. So here is a guide
|
|||
## Essentials
|
||||
|
||||
All streams have the same common parameters:
|
||||
|
||||
- `Stream.type`: The type of stream. Either `hls` or `file`
|
||||
- `Stream.id`: The id of this stream, unique per scraper output.
|
||||
- `Stream.flags`: A list of flags that apply to this stream. Most people won't need to use it.
|
||||
- `Stream.captions`: A list of captions/subtitles for this stream.
|
||||
- `Stream.headers`: Either undefined or a key value object of headers you must set to use the stream.
|
||||
- `Stream.preferredHeaders`: Either undefined or a key value object of headers you may want to set if you want optimal playback - but not required.
|
||||
- `Stream.type`: The type of stream. Either `hls` or `file`
|
||||
- `Stream.id`: The id of this stream, unique per scraper output.
|
||||
- `Stream.flags`: A list of flags that apply to this stream. Most people won't need to use it.
|
||||
- `Stream.captions`: A list of captions/subtitles for this stream.
|
||||
- `Stream.headers`: Either undefined or a key value object of headers you must set to use the stream.
|
||||
- `Stream.preferredHeaders`: Either undefined or a key value object of headers you may want to set if you want optimal playback - but not required.
|
||||
|
||||
Now let's delve deeper into how to watch these streams!
|
||||
|
||||
|
|
@ -47,9 +46,8 @@ The possibly qualities are: `unknown`, `360`, `480`, `720`, `1080`, `4k`.
|
|||
File based streams are always guaranteed to have one quality.
|
||||
|
||||
Once you get a streamfile, you have the following parameters:
|
||||
|
||||
- `StreamFile.type`: Right now it can only be `mp4`.
|
||||
- `StreamFile.url`: The URL linking to the video file.
|
||||
- `StreamFile.type`: Right now it can only be `mp4`.
|
||||
- `StreamFile.url`: The URL linking to the video file.
|
||||
|
||||
Here is a code sample of how to watch a file based stream in a browser:
|
||||
|
||||
|
|
@ -75,7 +73,6 @@ If your target is set to `BROWSER`, headers will never be required, as it's not
|
|||
## Using captions/subtitles
|
||||
|
||||
All streams have a list of captions at `Stream.captions`. The structure looks like this:
|
||||
|
||||
```ts
|
||||
type Caption = {
|
||||
type: CaptionType; // Language type, either "srt" or "vtt"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
icon: ph:info-fill
|
||||
navigation.redirect: /essentials/usage
|
||||
navigation.title: 'Get started'
|
||||
navigation.title: "Get started"
|
||||
|
|
|
|||
|
|
@ -3,12 +3,11 @@
|
|||
This guide covers everything you need to start contributing.
|
||||
|
||||
## Get Started
|
||||
|
||||
- **[Setup and Prerequisites](/in-depth/setup-and-prerequisites)** - Start here!
|
||||
|
||||
## In-Depth Guides
|
||||
|
||||
- **[Provider System](/in-depth/provider-system)** - How sources, embeds, and ranking work
|
||||
- **[Building Scrapers](/in-depth/building-scrapers)** - Complete guide to creating scrapers
|
||||
- **[Building Scrapers](/in-depth/building-scrapers)** - Complete guide to creating scrapers
|
||||
- **[Flags System](/in-depth/flags)** - Target compatibility and stream properties
|
||||
- **[Advanced Concepts](/in-depth/advanced-concepts)** - Error handling, proxying, and best practices
|
||||
|
|
|
|||
|
|
@ -14,14 +14,12 @@ MOVIE_WEB_PROXY_URL = "https://your-proxy-url.com" # Optional
|
|||
```
|
||||
|
||||
**Getting a TMDB API Key:**
|
||||
|
||||
1. Create an account at [TheMovieDB](https://www.themoviedb.org/)
|
||||
2. Go to Settings > API
|
||||
3. Request an API key (choose "Developer" for free usage)
|
||||
4. Use the provided key in your `.env` file
|
||||
|
||||
**Proxy URL (Optional):**
|
||||
|
||||
- Useful for testing scrapers that require proxy access
|
||||
- Can help bypass geographical restrictions during development
|
||||
- If not provided, the library will use default proxy services
|
||||
|
|
@ -47,7 +45,6 @@ pnpm cli
|
|||
```
|
||||
|
||||
This will prompt you for:
|
||||
|
||||
- **Fetcher mode** (native, node-fetch, browser)
|
||||
- **Scraper ID** (source or embed)
|
||||
- **TMDB ID** for the content (for sources)
|
||||
|
|
@ -90,7 +87,7 @@ pnpm cli --fetcher browser --source-id catflix --tmdb-id 11527
|
|||
The CLI supports different fetcher modes:
|
||||
|
||||
- **`native`**: Uses Node.js built-in fetch (undici) - fastest
|
||||
- **`node-fetch`**: Uses the node-fetch library
|
||||
- **`node-fetch`**: Uses the node-fetch library
|
||||
- **`browser`**: Starts headless Chrome for browser-like environment
|
||||
|
||||
::alert{type="warning"}
|
||||
|
|
@ -100,41 +97,37 @@ The browser fetcher requires running `pnpm build` first, otherwise you'll get ou
|
|||
### Understanding CLI Output
|
||||
|
||||
#### Source Scraper Output (Returns Embeds)
|
||||
|
||||
```sh
|
||||
pnpm cli --source-id catflix --tmdb-id 11527
|
||||
```
|
||||
|
||||
Example output:
|
||||
|
||||
```json
|
||||
{
|
||||
"embeds": [
|
||||
embeds: [
|
||||
{
|
||||
"embedId": "turbovid",
|
||||
"url": "https://turbovid.eu/embed/DjncbDBEmbLW"
|
||||
embedId: 'turbovid',
|
||||
url: 'https://turbovid.eu/embed/DjncbDBEmbLW'
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Embed Scraper Output (Returns Streams)
|
||||
|
||||
```sh
|
||||
pnpm cli --source-id turbovid --url "https://turbovid.eu/embed/DjncbDBEmbLW"
|
||||
```
|
||||
|
||||
Example output:
|
||||
|
||||
```json
|
||||
{
|
||||
"stream": [
|
||||
stream: [
|
||||
{
|
||||
"type": "hls",
|
||||
"id": "primary",
|
||||
"playlist": "https://proxy.fifthwit.net/m3u8-proxy?url=https%3A%2F%2Fqueenselti.pro%2Fwrofm%2Fuwu.m3u8&headers=%7B%22referer%22%3A%22https%3A%2F%2Fturbovid.eu%2F%22%2C%22origin%22%3A%22https%3A%2F%2Fturbovid.eu%22%7D",
|
||||
"flags": [],
|
||||
"captions": []
|
||||
type: 'hls',
|
||||
id: 'primary',
|
||||
playlist: 'https://proxy.fifthwit.net/m3u8-proxy?url=https%3A%2F%2Fqueenselti.pro%2Fwrofm%2Fuwu.m3u8&headers=%7B%22referer%22%3A%22https%3A%2F%2Fturbovid.eu%2F%22%2C%22origin%22%3A%22https%3A%2F%2Fturbovid.eu%22%7D',
|
||||
flags: [],
|
||||
captions: []
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -143,14 +136,13 @@ Example output:
|
|||
**Notice the proxied URL**: The `createM3U8ProxyUrl()` function creates URLs like `https://proxy.fifthwit.net/m3u8-proxy?url=...&headers=...` to handle protected streams. Read more about this in [Advanced Concepts](/in-depth/advanced-concepts).
|
||||
|
||||
#### Interactive Mode Flow
|
||||
|
||||
```sh
|
||||
pnpm cli
|
||||
```
|
||||
|
||||
```
|
||||
✔ Select a fetcher mode · native
|
||||
✔ Select a source · catflix
|
||||
✔ Select a source · catflix
|
||||
✔ TMDB ID · 11527
|
||||
✔ Media type · movie
|
||||
✓ Done!
|
||||
|
|
@ -184,4 +176,4 @@ Once your environment is set up:
|
|||
|
||||
::alert{type="info"}
|
||||
Always test your scrapers with multiple different movies and TV shows to ensure reliability across different content types.
|
||||
::
|
||||
::
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ export function gatherAllSources(): Array<Sourcerer> {
|
|||
return [
|
||||
cuevana3Scraper,
|
||||
catflixScraper,
|
||||
embedsuScraper, // Your source scraper goes here
|
||||
embedsuScraper, // Your source scraper goes here
|
||||
// ... more sources
|
||||
];
|
||||
}
|
||||
|
|
@ -24,14 +24,13 @@ export function gatherAllSources(): Array<Sourcerer> {
|
|||
export function gatherAllEmbeds(): Array<Embed> {
|
||||
return [
|
||||
upcloudScraper,
|
||||
turbovidScraper, // Your embed scraper goes here
|
||||
turbovidScraper, // Your embed scraper goes here
|
||||
// ... more embeds
|
||||
];
|
||||
}
|
||||
```
|
||||
|
||||
**Why this matters:**
|
||||
|
||||
- Only registered scrapers are available to the library
|
||||
- The order in these arrays doesn't matter (ranking determines priority)
|
||||
- You must import your scraper and add it to the appropriate function
|
||||
|
|
@ -41,43 +40,35 @@ export function gatherAllEmbeds(): Array<Embed> {
|
|||
There are two distinct types of providers in the system:
|
||||
|
||||
### Sources (Primary Scrapers)
|
||||
|
||||
**Sources** find content on websites and return either:
|
||||
|
||||
- **Direct video streams** (ready to play immediately)
|
||||
- **Embed URLs** that need further processing by embed scrapers
|
||||
|
||||
**Characteristics:**
|
||||
|
||||
- Handle website navigation and search
|
||||
- Process TMDB IDs to find content
|
||||
- Can return multiple server options
|
||||
- Located in `src/providers/sources/`
|
||||
|
||||
**Example source workflow:**
|
||||
|
||||
1. Receive movie/show request with TMDB ID
|
||||
2. Search the target website for that content
|
||||
3. Extract embed player URLs or direct streams
|
||||
4. Return results for further processing
|
||||
|
||||
### Embeds (Secondary Scrapers)
|
||||
|
||||
### Embeds (Secondary Scrapers)
|
||||
**Embeds** extract playable video streams from embed players:
|
||||
|
||||
- Take URLs from sources as input
|
||||
- Handle player-specific extraction and decryption
|
||||
- Always return direct streams (never more embeds)
|
||||
|
||||
**Characteristics:**
|
||||
|
||||
- Focus on one player type (turbovid, mixdrop, etc.)
|
||||
- Handle complex decryption/obfuscation
|
||||
- Specialized for specific player technologies
|
||||
- Located in `src/providers/embeds/`
|
||||
|
||||
**Example embed workflow:**
|
||||
|
||||
1. Receive embed player URL from a source
|
||||
2. Fetch and parse the embed page
|
||||
3. Extract/decrypt the video stream URLs
|
||||
|
|
@ -88,31 +79,28 @@ There are two distinct types of providers in the system:
|
|||
Every scraper has a **rank** that determines its priority in the execution queue:
|
||||
|
||||
### How Ranking Works
|
||||
|
||||
- **Higher numbers = Higher priority** (processed first)
|
||||
- **Each rank must be unique** across all providers
|
||||
- Sources and embeds have separate ranking spaces
|
||||
- Failed scrapers are skipped, next rank is tried
|
||||
|
||||
### Rank Ranges
|
||||
|
||||
Usually ranks should be on 10s: 110, 120, 130...
|
||||
|
||||
```typescript
|
||||
// Typical rank ranges (not enforced, but conventional)
|
||||
Sources: 1 - 300;
|
||||
Embeds: 1 - 250;
|
||||
Sources: 1-300
|
||||
Embeds: 1-250
|
||||
|
||||
// Example rankings
|
||||
export const embedsuScraper = makeSourcerer({
|
||||
id: 'embedsu',
|
||||
rank: 165, // Medium priority source
|
||||
rank: 165, // Medium priority source
|
||||
// ...
|
||||
});
|
||||
|
||||
export const turbovidScraper = makeEmbed({
|
||||
id: 'turbovid',
|
||||
rank: 122, // Medium priority embed
|
||||
id: 'turbovid',
|
||||
rank: 122, // Medium priority embed
|
||||
// ...
|
||||
});
|
||||
```
|
||||
|
|
@ -120,13 +108,11 @@ export const turbovidScraper = makeEmbed({
|
|||
### Choosing a Rank
|
||||
|
||||
**For Sources:**
|
||||
|
||||
- **200+**: High-quality, reliable sources (fast APIs, good uptime)
|
||||
- **100-199**: Medium reliability sources (most scrapers fall here)
|
||||
- **1-99**: Lower priority or experimental sources
|
||||
|
||||
**For Embeds:**
|
||||
|
||||
- **200+**: Fast, reliable embeds (direct URLs, minimal processing)
|
||||
- **100-199**: Standard embeds (typical decryption/extraction)
|
||||
- **1-99**: Slow or unreliable embeds (complex decryption, poor uptime)
|
||||
|
|
@ -151,21 +137,19 @@ Or check the cli to see the ranks.
|
|||
Each provider is configured using `makeSourcerer()` or `makeEmbed()`:
|
||||
|
||||
### Source Configuration
|
||||
|
||||
```typescript
|
||||
export const mySourceScraper = makeSourcerer({
|
||||
id: 'my-source', // Unique identifier (kebab-case)
|
||||
name: 'My Source', // Display name (human-readable)
|
||||
rank: 150, // Priority rank (must be unique)
|
||||
disabled: false, // Whether scraper is disabled
|
||||
flags: [], // Feature flags (see Advanced Concepts)
|
||||
id: 'my-source', // Unique identifier (kebab-case)
|
||||
name: 'My Source', // Display name (human-readable)
|
||||
rank: 150, // Priority rank (must be unique)
|
||||
disabled: false, // Whether scraper is disabled
|
||||
flags: [], // Feature flags (see Advanced Concepts)
|
||||
scrapeMovie: comboScraper, // Function for movies
|
||||
scrapeShow: comboScraper, // Function for TV shows
|
||||
scrapeShow: comboScraper, // Function for TV shows
|
||||
});
|
||||
```
|
||||
|
||||
### Embed Configuration
|
||||
|
||||
```typescript
|
||||
export const myEmbedScraper = makeEmbed({
|
||||
id: 'my-embed', // Unique identifier (kebab-case)
|
||||
|
|
@ -184,13 +168,11 @@ export const myEmbedScraper = makeEmbed({
|
|||
The provider system creates a powerful pipeline:
|
||||
|
||||
### 1. Source → Embed Chain
|
||||
|
||||
```
|
||||
User Request → Source Scraper → Embed URLs → Embed Scraper → Video Stream → Player
|
||||
```
|
||||
|
||||
**Pipeline Steps:**
|
||||
|
||||
1. **User Request** - User wants to watch content
|
||||
2. **Source Scraper** - Finds content on websites
|
||||
3. **Embed URLs** - Returns player URLs that need processing
|
||||
|
|
@ -199,24 +181,20 @@ User Request → Source Scraper → Embed URLs → Embed Scraper → Video Strea
|
|||
6. **Player** - User watches the content
|
||||
|
||||
### 2. Multiple Server Options
|
||||
|
||||
Sources can provide multiple backup servers:
|
||||
|
||||
```typescript
|
||||
// Source returns multiple embed options
|
||||
return {
|
||||
embeds: [
|
||||
{ embedId: 'turbovid', url: 'https://turbovid.com/abc' },
|
||||
{ embedId: 'mixdrop', url: 'https://mixdrop.co/def' },
|
||||
{ embedId: 'dood', url: 'https://dood.watch/ghi' },
|
||||
],
|
||||
{ embedId: 'dood', url: 'https://dood.watch/ghi' }
|
||||
]
|
||||
};
|
||||
```
|
||||
|
||||
### 3. Fallback System
|
||||
|
||||
If one embed fails, the system tries the next:
|
||||
|
||||
1. Try turbovid embed (rank 122)
|
||||
2. If fails, try mixdrop embed (rank 198)
|
||||
3. If fails, try dood embed (rank 173)
|
||||
|
|
@ -225,21 +203,17 @@ If one embed fails, the system tries the next:
|
|||
## Best Practices
|
||||
|
||||
### Naming Conventions
|
||||
|
||||
- **IDs**: Use kebab-case (`my-scraper`, not `myMyscraper` or `My_Scraper`)
|
||||
- **Names**: Use proper capitalization (`VidCloud`, not `vidcloud` or `VIDCLOUD`)
|
||||
- **Files**: Match the ID (`my-scraper.ts` for ID `my-scraper`)
|
||||
|
||||
### Registration Order
|
||||
|
||||
- The order in `all.ts` arrays doesn't affect execution (rank does)
|
||||
- Group similar scrapers together for maintainability
|
||||
- Add imports at the top, organized logically
|
||||
|
||||
### Testing Integration
|
||||
|
||||
Always test that your registration works:
|
||||
|
||||
```sh
|
||||
# Verify your scraper appears in the list (interactive mode shows all available)
|
||||
pnpm cli
|
||||
|
|
@ -254,4 +228,4 @@ Now that you understand the provider system:
|
|||
|
||||
1. Learn the details in [Building Scrapers](/in-depth/building-scrapers)
|
||||
2. Study [Advanced Concepts](/in-depth/advanced-concepts) for flags and error handling
|
||||
3. Look at the [Sources vs Embeds](/in-depth/sources-and-embeds) guide for more examples
|
||||
3. Look at the [Sources vs Embeds](/in-depth/sources-and-embeds) guide for more examples
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ import { NotFoundError } from '@/utils/errors';
|
|||
async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> {
|
||||
// 1. Build the appropriate URL based on media type
|
||||
const embedUrl = `https://embed.su/embed/${
|
||||
ctx.media.type === 'movie'
|
||||
? `movie/${ctx.media.tmdbId}`
|
||||
ctx.media.type === 'movie'
|
||||
? `movie/${ctx.media.tmdbId}`
|
||||
: `tv/${ctx.media.tmdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`
|
||||
}`;
|
||||
|
||||
|
|
@ -26,8 +26,7 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
|||
const embedPage = await ctx.proxiedFetcher<string>(embedUrl, {
|
||||
headers: {
|
||||
Referer: 'https://embed.su/',
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -45,7 +44,7 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
|||
|
||||
// 6. Build the final result
|
||||
const embeds: SourcererEmbed[] = secondDecode.map((server) => ({
|
||||
embedId: 'viper', // ID of the embed scraper to handle this URL
|
||||
embedId: 'viper', // ID of the embed scraper to handle this URL
|
||||
url: `https://embed.su/api/e/${server.hash}`,
|
||||
}));
|
||||
|
||||
|
|
@ -56,13 +55,13 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
|||
|
||||
// Export the scraper configuration
|
||||
export const embedsuScraper = makeSourcerer({
|
||||
id: 'embedsu', // Unique identifier
|
||||
name: 'embed.su', // Display name
|
||||
rank: 165, // Priority rank (must be unique)
|
||||
disabled: false, // Whether the scraper is disabled
|
||||
flags: [], // Feature flags (see Advanced Concepts)
|
||||
scrapeMovie: comboScraper, // Function for movies
|
||||
scrapeShow: comboScraper, // Function for TV shows
|
||||
id: 'embedsu', // Unique identifier
|
||||
name: 'embed.su', // Display name
|
||||
rank: 165, // Priority rank (must be unique)
|
||||
disabled: false, // Whether the scraper is disabled
|
||||
flags: [], // Feature flags (see Advanced Concepts)
|
||||
scrapeMovie: comboScraper, // Function for movies
|
||||
scrapeShow: comboScraper, // Function for TV shows
|
||||
});
|
||||
```
|
||||
|
||||
|
|
@ -78,7 +77,7 @@ async function scrapeMovie(ctx: MovieScrapeContext): Promise<SourcererOutput> {
|
|||
}
|
||||
|
||||
async function scrapeShow(ctx: ShowScrapeContext): Promise<SourcererOutput> {
|
||||
// TV show-specific logic
|
||||
// TV show-specific logic
|
||||
const showUrl = `${baseUrl}/tv/${ctx.media.tmdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`;
|
||||
// ... show processing
|
||||
}
|
||||
|
|
@ -89,7 +88,7 @@ export const myScraper = makeSourcerer({
|
|||
rank: 150,
|
||||
disabled: false,
|
||||
flags: [],
|
||||
scrapeMovie: scrapeMovie, // Separate functions
|
||||
scrapeMovie: scrapeMovie, // Separate functions
|
||||
scrapeShow: scrapeShow,
|
||||
});
|
||||
```
|
||||
|
|
@ -103,22 +102,21 @@ A `SourcererOutput` can return two types of data. Understanding when to use each
|
|||
Use when your scraper finds embed players that need further processing:
|
||||
|
||||
```typescript
|
||||
return {
|
||||
return {
|
||||
embeds: [
|
||||
{
|
||||
embedId: 'turbovid', // Must match an existing embed scraper ID
|
||||
url: 'https://turbovid.com/embed/abc123',
|
||||
embedId: 'turbovid', // Must match an existing embed scraper ID
|
||||
url: 'https://turbovid.com/embed/abc123'
|
||||
},
|
||||
{
|
||||
embedId: 'mixdrop', // Backup option
|
||||
url: 'https://mixdrop.co/embed/def456',
|
||||
},
|
||||
],
|
||||
embedId: 'mixdrop', // Backup option
|
||||
url: 'https://mixdrop.co/embed/def456'
|
||||
}
|
||||
]
|
||||
};
|
||||
```
|
||||
|
||||
**When to use:**
|
||||
|
||||
- Your scraper finds embed player URLs
|
||||
- You want to leverage existing embed scrapers
|
||||
- The site uses common players (turbovid, mixdrop, etc.)
|
||||
|
|
@ -141,116 +139,115 @@ return {
|
|||
playlist: streamUrl,
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: [], // Subtitle tracks (optional)
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
// For MP4 files with a single quality
|
||||
return {
|
||||
embeds: [],
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
captions,
|
||||
qualities: {
|
||||
unknown: {
|
||||
type: 'mp4',
|
||||
url: streamUrl,
|
||||
},
|
||||
},
|
||||
type: 'file',
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// For MP4 files with multiple qualities:
|
||||
// It's recommended to return it using a function similar to this:
|
||||
|
||||
const streams = Object.entries(data.streams).reduce((acc: Record<string, string>, [quality, url]) => {
|
||||
let qualityKey: number;
|
||||
if (quality === 'ORG') {
|
||||
// Only add unknown quality if it's an mp4 (handle URLs with query parameters)
|
||||
const urlPath = url.split('?')[0]; // Remove query parameters
|
||||
if (urlPath.toLowerCase().endsWith('.mp4')) {
|
||||
acc.unknown = url;
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
if (quality === '4K') {
|
||||
qualityKey = 2160;
|
||||
} else {
|
||||
qualityKey = parseInt(quality.replace('P', ''), 10);
|
||||
}
|
||||
if (Number.isNaN(qualityKey) || acc[qualityKey]) return acc;
|
||||
acc[qualityKey] = url;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
// Filter qualities based on provider type
|
||||
const filteredStreams = Object.entries(streams).reduce((acc: Record<string, string>, [quality, url]) => {
|
||||
// Skip unknown for cached provider
|
||||
if (provider.useCacheUrl && quality === 'unknown') {
|
||||
return acc;
|
||||
}
|
||||
|
||||
acc[quality] = url;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
// Returning each quality like so
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
captions: [],
|
||||
qualities: {
|
||||
...(filteredStreams[2160] && {
|
||||
'4k': {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[2160],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams[1080] && {
|
||||
1080: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[1080],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams[720] && {
|
||||
720: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[720],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams[480] && {
|
||||
480: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[480],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams[360] && {
|
||||
360: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[360],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams.unknown && {
|
||||
embeds: [],
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
captions,
|
||||
qualities: {
|
||||
unknown: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams.unknown,
|
||||
url: streamUrl,
|
||||
},
|
||||
}),
|
||||
},
|
||||
type: 'file',
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
},
|
||||
type: 'file',
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
},
|
||||
],
|
||||
};
|
||||
],
|
||||
};
|
||||
|
||||
// For MP4 files with multiple qualities:
|
||||
// It's recommended to return it using a function similar to this:
|
||||
|
||||
const streams = Object.entries(data.streams).reduce((acc: Record<string, string>, [quality, url]) => {
|
||||
let qualityKey: number;
|
||||
if (quality === 'ORG') {
|
||||
// Only add unknown quality if it's an mp4 (handle URLs with query parameters)
|
||||
const urlPath = url.split('?')[0]; // Remove query parameters
|
||||
if (urlPath.toLowerCase().endsWith('.mp4')) {
|
||||
acc.unknown = url;
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
if (quality === '4K') {
|
||||
qualityKey = 2160;
|
||||
} else {
|
||||
qualityKey = parseInt(quality.replace('P', ''), 10);
|
||||
}
|
||||
if (Number.isNaN(qualityKey) || acc[qualityKey]) return acc;
|
||||
acc[qualityKey] = url;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
// Filter qualities based on provider type
|
||||
const filteredStreams = Object.entries(streams).reduce((acc: Record<string, string>, [quality, url]) => {
|
||||
// Skip unknown for cached provider
|
||||
if (provider.useCacheUrl && quality === 'unknown') {
|
||||
return acc;
|
||||
}
|
||||
|
||||
acc[quality] = url;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
// Returning each quality like so
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
captions: [],
|
||||
qualities: {
|
||||
...(filteredStreams[2160] && {
|
||||
'4k': {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[2160],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams[1080] && {
|
||||
1080: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[1080],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams[720] && {
|
||||
720: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[720],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams[480] && {
|
||||
480: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[480],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams[360] && {
|
||||
360: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams[360],
|
||||
},
|
||||
}),
|
||||
...(filteredStreams.unknown && {
|
||||
unknown: {
|
||||
type: 'mp4',
|
||||
url: filteredStreams.unknown,
|
||||
},
|
||||
}),
|
||||
},
|
||||
type: 'file',
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
},
|
||||
],
|
||||
};
|
||||
```
|
||||
|
||||
**When to use:**
|
||||
|
||||
- Your scraper can extract direct video URLs
|
||||
- The site provides its own player technology
|
||||
- You need fine control over stream handling
|
||||
|
|
@ -261,71 +258,65 @@ return {
|
|||
The scraper context (`ctx`) provides everything you need for implementation:
|
||||
|
||||
### Media Information
|
||||
|
||||
```typescript
|
||||
// Basic media info (always available)
|
||||
ctx.media.title; // "Spirited Away"
|
||||
ctx.media.type; // "movie" | "show"
|
||||
ctx.media.tmdbId; // 129
|
||||
ctx.media.releaseYear; // 2001
|
||||
ctx.media.imdbId; // "tt0245429" (when available)
|
||||
ctx.media.title // "Spirited Away"
|
||||
ctx.media.type // "movie" | "show"
|
||||
ctx.media.tmdbId // 129
|
||||
ctx.media.releaseYear // 2001
|
||||
ctx.media.imdbId // "tt0245429" (when available)
|
||||
|
||||
// For TV shows only (check ctx.media.type === 'show')
|
||||
ctx.media.season.number; // 1
|
||||
ctx.media.season.tmdbId; // Season TMDB ID
|
||||
ctx.media.episode.number; // 5
|
||||
ctx.media.episode.tmdbId; // Episode TMDB ID
|
||||
ctx.media.season.number // 1
|
||||
ctx.media.season.tmdbId // Season TMDB ID
|
||||
ctx.media.episode.number // 5
|
||||
ctx.media.episode.tmdbId // Episode TMDB ID
|
||||
```
|
||||
|
||||
### HTTP Client
|
||||
|
||||
```typescript
|
||||
// Always use proxiedFetcher for external requests to avoid CORS
|
||||
const response = await ctx.proxiedFetcher<string>('https://example.com/api', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0...',
|
||||
Referer: 'https://example.com',
|
||||
'Referer': 'https://example.com'
|
||||
},
|
||||
body: JSON.stringify({ key: 'value' }),
|
||||
body: JSON.stringify({ key: 'value' })
|
||||
});
|
||||
|
||||
// For API calls with base URL
|
||||
const data = await ctx.proxiedFetcher('/search', {
|
||||
baseUrl: 'https://api.example.com',
|
||||
query: { q: ctx.media.title, year: ctx.media.releaseYear },
|
||||
query: { q: ctx.media.title, year: ctx.media.releaseYear }
|
||||
});
|
||||
```
|
||||
|
||||
### Progress Updates
|
||||
|
||||
```typescript
|
||||
// Update the loading indicator (0-100)
|
||||
ctx.progress(25); // Found media page
|
||||
ctx.progress(25); // Found media page
|
||||
// ... processing ...
|
||||
ctx.progress(50); // Extracted embed links
|
||||
ctx.progress(50); // Extracted embed links
|
||||
// ... more processing ...
|
||||
ctx.progress(90); // Almost done
|
||||
ctx.progress(90); // Almost done
|
||||
```
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### 1. URL Building
|
||||
|
||||
```typescript
|
||||
// Handle different media types
|
||||
const buildUrl = (ctx: ShowScrapeContext | MovieScrapeContext) => {
|
||||
const apiUrl =
|
||||
ctx.media.type === 'movie'
|
||||
? `${baseUrl}/movie/${ctx.media.tmdbId}`
|
||||
: `${baseUrl}/tv/${ctx.media.tmdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`;
|
||||
|
||||
const apiUrl = ctx.media.type === 'movie'
|
||||
? `${baseUrl}/movie/${ctx.media.tmdbId}`
|
||||
: `${baseUrl}/tv/${ctx.media.tmdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`;
|
||||
|
||||
return apiUrl;
|
||||
};
|
||||
```
|
||||
|
||||
### 2. Data Extraction
|
||||
|
||||
```typescript
|
||||
import { load } from 'cheerio';
|
||||
|
||||
|
|
@ -345,7 +336,6 @@ if (configMatch) {
|
|||
```
|
||||
|
||||
### 3. Error Handling
|
||||
|
||||
```typescript
|
||||
import { NotFoundError } from '@/utils/errors';
|
||||
|
||||
|
|
@ -361,56 +351,49 @@ if (!apiResponse.success) {
|
|||
```
|
||||
|
||||
### 4. Protected Streams
|
||||
|
||||
There are several ways to bypass protections on streams.
|
||||
|
||||
Using the M3U8 proxy:
|
||||
|
||||
```typescript
|
||||
import { createM3U8ProxyUrl } from '@/utils/proxy';
|
||||
|
||||
// For streams that require special headers
|
||||
const streamHeaders = {
|
||||
Referer: 'https://player.example.com/',
|
||||
Origin: 'https://player.example.com',
|
||||
'User-Agent': 'Mozilla/5.0...',
|
||||
'Referer': 'https://player.example.com/',
|
||||
'Origin': 'https://player.example.com',
|
||||
'User-Agent': 'Mozilla/5.0...'
|
||||
};
|
||||
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: createM3U8ProxyUrl(originalPlaylist, ctx.features, streamHeaders),
|
||||
headers: streamHeaders, // Include headers in the createM3U8ProxyUrl function and here for native and extension targets
|
||||
flags: [flags.CORS_ALLOWED], // createM3U8ProxyUrl (or the extension) bypasses cors so we say it's allowed to play in a browser
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: createM3U8ProxyUrl(originalPlaylist, ctx.features, streamHeaders),
|
||||
headers: streamHeaders, // Include headers in the createM3U8ProxyUrl function and here for native and extension targets
|
||||
flags: [flags.CORS_ALLOWED], // createM3U8ProxyUrl (or the extension) bypasses cors so we say it's allowed to play in a browser
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
```
|
||||
|
||||
Using the browser extension:
|
||||
|
||||
```typescript
|
||||
// For streams that require special headers
|
||||
const streamHeaders = {
|
||||
Referer: 'https://player.example.com/',
|
||||
Origin: 'https://player.example.com',
|
||||
'User-Agent': 'Mozilla/5.0...',
|
||||
'Referer': 'https://player.example.com/',
|
||||
'Origin': 'https://player.example.com',
|
||||
'User-Agent': 'Mozilla/5.0...'
|
||||
};
|
||||
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: originalPlaylist,
|
||||
headers: streamHeaders,
|
||||
flags: [], // Use the extension becuase it can pass headers, include no flag for extension or native
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: originalPlaylist,
|
||||
headers: streamHeaders,
|
||||
flags: [], // Use the extension becuase it can pass headers, include no flag for extension or native
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
```
|
||||
|
||||
|
|
@ -427,27 +410,25 @@ export const myEmbedScraper = makeEmbed({
|
|||
rank: 120,
|
||||
async scrape(ctx) {
|
||||
// ctx.url contains the embed URL from a source
|
||||
|
||||
|
||||
// 1. Fetch the embed page
|
||||
const embedPage = await ctx.proxiedFetcher(ctx.url);
|
||||
|
||||
|
||||
// 2. Extract the stream URL (example with regex)
|
||||
const streamMatch = embedPage.match(/src:\s*["']([^"']+\.m3u8[^"']*)/);
|
||||
if (!streamMatch) {
|
||||
throw new NotFoundError('No stream found in embed');
|
||||
}
|
||||
|
||||
|
||||
// 3. Return the stream
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: streamMatch[1],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: streamMatch[1],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
@ -456,7 +437,6 @@ export const myEmbedScraper = makeEmbed({
|
|||
## Testing Your Scrapers
|
||||
|
||||
### 1. Basic Testing
|
||||
|
||||
```sh
|
||||
# Test your scraper with CLI
|
||||
pnpm cli --source-id my-scraper --tmdb-id 11527
|
||||
|
|
@ -468,28 +448,24 @@ pnpm cli --source-id my-scraper --tmdb-id 94605 --season 1 --episode 1 # TV sho
|
|||
### 2. Real CLI Output Examples
|
||||
|
||||
**Testing a source that returns embeds:**
|
||||
|
||||
```sh
|
||||
pnpm cli --source-id catflix --tmdb-id 11527
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"embeds": [
|
||||
embeds: [
|
||||
{
|
||||
"embedId": "turbovid",
|
||||
"url": "https://turbovid.eu/embed/DjncbDBEmbLW"
|
||||
embedId: 'turbovid',
|
||||
url: 'https://turbovid.eu/embed/DjncbDBEmbLW'
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Testing an embed that returns streams:**
|
||||
|
||||
```sh
|
||||
pnpm cli --source-id turbovid --url "https://turbovid.eu/embed/DjncbDBEmbLW"
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
stream: [
|
||||
|
|
@ -507,9 +483,7 @@ pnpm cli --source-id turbovid --url "https://turbovid.eu/embed/DjncbDBEmbLW"
|
|||
**Notice**: The playlist URL shows how `createM3U8ProxyUrl()` creates proxied URLs to handle protected streams.
|
||||
|
||||
### 3. Comprehensive Testing
|
||||
|
||||
Test with various content:
|
||||
|
||||
- Popular movies (The Shining: 11527, Spirited Away: 129, Avatar: 19995)
|
||||
- Recent releases (check current popular movies)
|
||||
- TV shows with multiple seasons
|
||||
|
|
@ -517,7 +491,6 @@ Test with various content:
|
|||
- Different languages/regions
|
||||
|
||||
### 4. Debug Mode
|
||||
|
||||
```sh
|
||||
# Add debug logging to your scraper
|
||||
console.log('Fetching URL:', embedUrl);
|
||||
|
|
@ -536,4 +509,4 @@ Once you've built your scraper:
|
|||
|
||||
::alert{type="warning"}
|
||||
Always test your scrapers with both movies and TV shows, and include multiple examples in your pull request description.
|
||||
::
|
||||
::
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ User Request → Source Scraper → What did source find?
|
|||
```
|
||||
|
||||
**Flow Breakdown:**
|
||||
|
||||
1. **User requests** content (movie/TV show)
|
||||
2. **Source scraper** searches the target website
|
||||
3. **Source returns** either:
|
||||
|
|
@ -31,7 +30,6 @@ User Request → Source Scraper → What did source find?
|
|||
## Sources: The Content Finders
|
||||
|
||||
**Sources** are the first stage - they find content on websites and return either:
|
||||
|
||||
1. **Direct video streams** (ready to play)
|
||||
2. **Embed URLs** that need further processing
|
||||
|
||||
|
|
@ -43,58 +41,52 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
|||
// 1. Call an API to find video sources
|
||||
const data = await ctx.proxiedFetcher(`/api/getVideoSource`, {
|
||||
baseUrl: 'https://tom.autoembed.cc',
|
||||
query: { type: mediaType, id },
|
||||
query: { type: mediaType, id }
|
||||
});
|
||||
|
||||
// 2. Return embed URLs for further processing
|
||||
return {
|
||||
embeds: [
|
||||
{
|
||||
embedId: 'autoembed-english', // Points to an embed scraper
|
||||
url: data.videoSource, // URL that embed will process
|
||||
},
|
||||
],
|
||||
embeds: [{
|
||||
embedId: 'autoembed-english', // Points to an embed scraper
|
||||
url: data.videoSource // URL that embed will process
|
||||
}]
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
**What this source does:**
|
||||
|
||||
- Queries an API with TMDB ID
|
||||
- Gets back a video source URL
|
||||
- Gets back a video source URL
|
||||
- Returns it as an embed for the `autoembed-english` embed scraper to handle
|
||||
|
||||
### Example: Catflix Source
|
||||
|
||||
```typescript
|
||||
// From src/providers/sources/catflix.ts
|
||||
// From src/providers/sources/catflix.ts
|
||||
async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> {
|
||||
// 1. Build URL to the movie/show page
|
||||
const watchPageUrl = `${baseUrl}/movie/${mediaTitle}-${movieId}`;
|
||||
|
||||
|
||||
// 2. Scrape the page for embedded player URLs
|
||||
const watchPage = await ctx.proxiedFetcher(watchPageUrl);
|
||||
const $ = load(watchPage);
|
||||
|
||||
|
||||
// 3. Extract and decode the embed URL
|
||||
const mainOriginMatch = scriptData.data.match(/main_origin = "(.*?)";/);
|
||||
const decodedUrl = atob(mainOriginMatch[1]);
|
||||
|
||||
// 4. Return embed URL for turbovid embed to process
|
||||
return {
|
||||
embeds: [
|
||||
{
|
||||
embedId: 'turbovid', // Points to turbovid embed scraper
|
||||
url: decodedUrl, // Turbovid player URL
|
||||
},
|
||||
],
|
||||
embeds: [{
|
||||
embedId: 'turbovid', // Points to turbovid embed scraper
|
||||
url: decodedUrl // Turbovid player URL
|
||||
}]
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
**What this source does:**
|
||||
|
||||
- Scrapes a streaming website
|
||||
- Scrapes a streaming website
|
||||
- Finds encoded embed player URLs in the page source
|
||||
- Decodes the URL and returns it for the `turbovid` embed scraper
|
||||
|
||||
|
|
@ -106,140 +98,114 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
|
|||
|
||||
```typescript
|
||||
// From src/providers/embeds/autoembed.ts
|
||||
function embed(provider: { id: string; rank: number }) {
|
||||
return makeEmbed({
|
||||
id: provider.id,
|
||||
name: provider.id.split('-').map(word => word[0].toUpperCase() + word.slice(1)).join(' '),
|
||||
rank: provider.rank,
|
||||
flags: [flags.CORS_ALLOWED], // Embed flags match stream flags
|
||||
async scrape(ctx) {
|
||||
// The URL from the source is already a direct HLS playlist
|
||||
return {
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: ctx.url, // Use the URL directly as HLS playlist
|
||||
flags: [flags.CORS_ALLOWED], // Stream flags
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
},
|
||||
});
|
||||
async scrape(ctx) {
|
||||
// The URL from the source is already a direct HLS playlist
|
||||
return {
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: ctx.url, // Use the URL directly as HLS playlist
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
**What this embed does:**
|
||||
|
||||
- Takes the URL from autoembed source
|
||||
- Treats it as a direct HLS playlist (no further processing needed)
|
||||
- Returns it as a playable stream
|
||||
- **Note:** Embed flags now match stream flags for consistent filtering
|
||||
|
||||
### Example: Turbovid Embed (Complex)
|
||||
|
||||
```typescript
|
||||
// From src/providers/embeds/turbovid.ts
|
||||
export const turbovidScraper = makeEmbed({
|
||||
id: 'turbovid',
|
||||
name: 'Turbovid',
|
||||
rank: 180,
|
||||
flags: [flags.CORS_ALLOWED], // Embed flags match stream flags
|
||||
async scrape(ctx) {
|
||||
// 1. Fetch the turbovid player page
|
||||
const embedPage = await ctx.proxiedFetcher(ctx.url);
|
||||
|
||||
// 2. Extract encryption keys from the page
|
||||
const apkey = embedPage.match(/const\s+apkey\s*=\s*"(.*?)";/)?.[1];
|
||||
const xxid = embedPage.match(/const\s+xxid\s*=\s*"(.*?)";/)?.[1];
|
||||
|
||||
// 3. Get decryption key from API
|
||||
const encodedJuiceKey = JSON.parse(
|
||||
await ctx.proxiedFetcher('/api/cucked/juice_key', { baseUrl })
|
||||
).juice;
|
||||
|
||||
// 4. Get encrypted playlist data
|
||||
const data = JSON.parse(
|
||||
await ctx.proxiedFetcher('/api/cucked/the_juice_v2/', {
|
||||
baseUrl, query: { [apkey]: xxid }
|
||||
})
|
||||
).data;
|
||||
|
||||
// 5. Decrypt the playlist URL
|
||||
const playlist = decrypt(data, atob(encodedJuiceKey));
|
||||
|
||||
// 6. Return proxied stream (handles CORS/headers)
|
||||
return {
|
||||
stream: [{
|
||||
type: 'hls',
|
||||
id: 'primary',
|
||||
playlist: createM3U8ProxyUrl(playlist, ctx.features, streamHeaders),
|
||||
headers: streamHeaders,
|
||||
flags: [flags.CORS_ALLOWED], // Stream flags match embed flags
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
},
|
||||
});
|
||||
async scrape(ctx) {
|
||||
// 1. Fetch the turbovid player page
|
||||
const embedPage = await ctx.proxiedFetcher(ctx.url);
|
||||
|
||||
// 2. Extract encryption keys from the page
|
||||
const apkey = embedPage.match(/const\s+apkey\s*=\s*"(.*?)";/)?.[1];
|
||||
const xxid = embedPage.match(/const\s+xxid\s*=\s*"(.*?)";/)?.[1];
|
||||
|
||||
// 3. Get decryption key from API
|
||||
const encodedJuiceKey = JSON.parse(
|
||||
await ctx.proxiedFetcher('/api/cucked/juice_key', { baseUrl })
|
||||
).juice;
|
||||
|
||||
// 4. Get encrypted playlist data
|
||||
const data = JSON.parse(
|
||||
await ctx.proxiedFetcher('/api/cucked/the_juice_v2/', {
|
||||
baseUrl, query: { [apkey]: xxid }
|
||||
})
|
||||
).data;
|
||||
|
||||
// 5. Decrypt the playlist URL
|
||||
const playlist = decrypt(data, atob(encodedJuiceKey));
|
||||
|
||||
// 6. Return proxied stream (handles CORS/headers)
|
||||
return {
|
||||
stream: [{
|
||||
type: 'hls',
|
||||
id: 'primary',
|
||||
playlist: createM3U8ProxyUrl(playlist, ctx.features, streamHeaders),
|
||||
headers: streamHeaders,
|
||||
flags: [], captions: []
|
||||
}]
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
**What this embed does:**
|
||||
|
||||
- Takes turbovid player URL from catflix source
|
||||
- Performs complex extraction: fetches page → gets keys → decrypts data
|
||||
- Returns the final HLS playlist with proper proxy handling
|
||||
- **Note:** Embed flags now match stream flags for consistent filtering
|
||||
|
||||
## Key Differences
|
||||
|
||||
| Sources | Embeds |
|
||||
| ----------------------------------- | ---------------------------------------------- |
|
||||
| **Find content** on websites | **Extract streams** from players |
|
||||
| Return embed URLs OR direct streams | Always return direct streams |
|
||||
| Handle website navigation/search | Handle player-specific extraction |
|
||||
| Can return multiple server options | Process one specific player type |
|
||||
| Sources | Embeds |
|
||||
|---------|--------|
|
||||
| **Find content** on websites | **Extract streams** from players |
|
||||
| Return embed URLs OR direct streams | Always return direct streams |
|
||||
| Handle website navigation/search | Handle player-specific extraction |
|
||||
| Can return multiple server options | Process one specific player type |
|
||||
| Example: "Find Avengers on Catflix" | Example: "Extract stream from Turbovid player" |
|
||||
|
||||
## Why This Separation?
|
||||
|
||||
### 1. **Reusability**
|
||||
|
||||
Multiple sources can use the same embed:
|
||||
|
||||
```typescript
|
||||
// Both catflix and other sources can return turbovid embeds
|
||||
{ embedId: 'turbovid', url: 'https://turbovid.com/player123' }
|
||||
```
|
||||
|
||||
### 2. **Multiple Server Options**
|
||||
|
||||
### 2. **Multiple Server Options**
|
||||
Sources can provide backup servers:
|
||||
|
||||
```typescript
|
||||
return {
|
||||
embeds: [
|
||||
{ embedId: 'turbovid', url: 'https://turbovid.com/player123' },
|
||||
{ embedId: 'vidcloud', url: 'https://vidcloud.co/embed456' },
|
||||
{ embedId: 'dood', url: 'https://dood.watch/789' },
|
||||
],
|
||||
{ embedId: 'dood', url: 'https://dood.watch/789' }
|
||||
]
|
||||
};
|
||||
```
|
||||
|
||||
### 3. **Language/Quality Variants**
|
||||
|
||||
Sources can offer different options:
|
||||
|
||||
```typescript
|
||||
return {
|
||||
embeds: [
|
||||
{ embedId: 'autoembed-english', url: streamUrl },
|
||||
{ embedId: 'autoembed-spanish', url: streamUrlEs },
|
||||
{ embedId: 'autoembed-hindi', url: streamUrlHi },
|
||||
],
|
||||
{ embedId: 'autoembed-hindi', url: streamUrlHi }
|
||||
]
|
||||
};
|
||||
```
|
||||
|
||||
### 4. **Specialization**
|
||||
|
||||
- **Sources** specialize in website structures and search
|
||||
- **Embeds** specialize in player technologies and decryption
|
||||
|
||||
|
|
@ -247,7 +213,7 @@ return {
|
|||
|
||||
### Flow Example: Finding "Spirited Away"
|
||||
|
||||
1. **Source (catflix)**:
|
||||
1. **Source (catflix)**:
|
||||
- Searches catflix.su for "Spirited Away"
|
||||
- Finds movie page with embedded player
|
||||
- Extracts turbovid URL: `https://turbovid.com/embed/abc123`
|
||||
|
|
@ -264,15 +230,14 @@ return {
|
|||
### Error Handling Chain
|
||||
|
||||
If the embed fails to extract a stream:
|
||||
|
||||
```typescript
|
||||
// Source provides multiple backup options
|
||||
return {
|
||||
embeds: [
|
||||
{ embedId: 'turbovid', url: url1 }, // Try first
|
||||
{ embedId: 'mixdrop', url: url2 }, // Fallback 1
|
||||
{ embedId: 'dood', url: url3 }, // Fallback 2
|
||||
],
|
||||
{ embedId: 'turbovid', url: url1 }, // Try first
|
||||
{ embedId: 'mixdrop', url: url2 }, // Fallback 1
|
||||
{ embedId: 'dood', url: url3 } // Fallback 2
|
||||
]
|
||||
};
|
||||
```
|
||||
|
||||
|
|
@ -281,30 +246,26 @@ The system tries each embed in rank order until one succeeds.
|
|||
## Best Practices
|
||||
|
||||
### For Sources:
|
||||
|
||||
- Provide multiple embed options when possible
|
||||
- Use descriptive embed IDs that match existing embeds
|
||||
- Handle both movies and TV shows (combo scraper pattern)
|
||||
- Return direct streams when embed processing isn't needed
|
||||
|
||||
### For Embeds:
|
||||
|
||||
### For Embeds:
|
||||
- Focus on one player type per embed
|
||||
- Handle errors gracefully with clear error messages
|
||||
- Use proxy functions for protected streams
|
||||
- Include proper headers and flags at both embed and stream levels
|
||||
- **Embed flags should match stream flags** for consistent filtering behavior
|
||||
- Include proper headers and flags
|
||||
|
||||
### Registration:
|
||||
|
||||
```typescript
|
||||
// In src/providers/all.ts
|
||||
export function gatherAllSources(): Array<Sourcerer> {
|
||||
return [catflixScraper, autoembedScraper /* ... */];
|
||||
return [catflixScraper, autoembedScraper, /* ... */];
|
||||
}
|
||||
|
||||
export function gatherAllEmbeds(): Array<Embed> {
|
||||
return [turbovidScraper, autoembedEnglishScraper /* ... */];
|
||||
return [turbovidScraper, autoembedEnglishScraper, /* ... */];
|
||||
}
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -10,80 +10,51 @@ Sometimes a source will block netlify or cloudflare. Making self hosted proxies
|
|||
|
||||
- **`CORS_ALLOWED`**: Headers from the output streams are set to allow any origin.
|
||||
- **`IP_LOCKED`**: The streams are locked by IP: requester and watcher must be the same.
|
||||
- **`CF_BLOCKED`**: _(Cosmetic)_ Indicates the source/embed blocks Cloudflare IPs. For actual enforcement, remove `CORS_ALLOWED` or add `IP_LOCKED`.
|
||||
- **`PROXY_BLOCKED`**: _(Cosmetic)_ Indicates streams shouldn't be proxied. For actual enforcement, remove `CORS_ALLOWED` or add `IP_LOCKED`.
|
||||
- **`CF_BLOCKED`**: *(Cosmetic)* Indicates the source/embed blocks Cloudflare IPs. For actual enforcement, remove `CORS_ALLOWED` or add `IP_LOCKED`.
|
||||
- **`PROXY_BLOCKED`**: *(Cosmetic)* Indicates streams shouldn't be proxied. For actual enforcement, remove `CORS_ALLOWED` or add `IP_LOCKED`.
|
||||
|
||||
## How Flags Affect Target Compatibility
|
||||
|
||||
### Stream-Level Flags Impact
|
||||
|
||||
**With `CORS_ALLOWED`:**
|
||||
|
||||
- ✅ Browser targets (can fetch and play streams)
|
||||
- ✅ Extension targets (bypass needed restrictions)
|
||||
- ✅ Extension targets (bypass needed restrictions)
|
||||
- ✅ Native targets (direct stream access)
|
||||
|
||||
**Without `CORS_ALLOWED`:**
|
||||
|
||||
- ❌ Browser targets (CORS restrictions block access)
|
||||
- ✅ Extension targets (can bypass CORS)
|
||||
- ✅ Native targets (no CORS restrictions)
|
||||
|
||||
**With `IP_LOCKED`:**
|
||||
|
||||
- ❌ Proxy setups (different IP between request and playback)
|
||||
- ✅ Direct connections (same IP for request and playback)
|
||||
- ✅ Extension targets (when user has consistent IP)
|
||||
|
||||
**With `CF_BLOCKED` _(cosmetic only)_:**
|
||||
|
||||
**With `CF_BLOCKED` *(cosmetic only)*:**
|
||||
- 🏷️ Informational label indicating Cloudflare issues
|
||||
- ⚠️ **Still requires removing `CORS_ALLOWED` or adding `IP_LOCKED` for actual enforcement**
|
||||
|
||||
**With `PROXY_BLOCKED` _(cosmetic only)_:**
|
||||
|
||||
- 🏷️ Informational label indicating proxy incompatibility
|
||||
**With `PROXY_BLOCKED` *(cosmetic only)*:**
|
||||
- 🏷️ Informational label indicating proxy incompatibility
|
||||
- ⚠️ **Still requires removing `CORS_ALLOWED` or adding `IP_LOCKED` for actual enforcement**
|
||||
|
||||
### Provider-Level Flags Impact
|
||||
|
||||
**With `CORS_ALLOWED`:**
|
||||
|
||||
- Source appears for all target types
|
||||
- Individual streams still need appropriate flags
|
||||
|
||||
**Without `CORS_ALLOWED`:**
|
||||
|
||||
- Source only appears for extension/native targets
|
||||
- Hidden entirely from browser-only users
|
||||
|
||||
### Embed-Level Flags Impact
|
||||
|
||||
**Embed flags work the same way as source flags** - they control embed visibility based on target compatibility. Unlike sources, embeds typically don't provide multiple stream options, so their flags directly determine if the embed is available for a given target.
|
||||
|
||||
**With `CORS_ALLOWED`:**
|
||||
|
||||
- Embed appears for all target types (browser, extension, native)
|
||||
- Works across all playback environments
|
||||
|
||||
**Without `CORS_ALLOWED`:**
|
||||
|
||||
- Embed only appears for extension/native targets
|
||||
- Hidden from browser-only users who can't bypass CORS restrictions
|
||||
|
||||
**With `IP_LOCKED`:**
|
||||
|
||||
- Embed requires consistent IP between request and playback
|
||||
- Only works when `consistentIpForRequests` is true (typically extension setups)
|
||||
|
||||
**Embed flags are automatically derived from stream flags.** When building an embed scraper, the embed should have the same flags as its streams to ensure consistent filtering behavior.
|
||||
|
||||
### Important: Cosmetic vs Enforcement Flags
|
||||
|
||||
**Cosmetic flags** (`CF_BLOCKED`, `PROXY_BLOCKED`) are informational labels only. They don't enforce any behavior.
|
||||
|
||||
**Enforcement flags** (`CORS_ALLOWED`, `IP_LOCKED`) actually control stream compatibility:
|
||||
|
||||
- **Remove all flags**: Most common way to make streams extension/native-only (no browser support)
|
||||
- **Add `IP_LOCKED`**: Prevents proxy usage when `consistentIpForRequests` is false (rarely needed - most extension-only streams just use no flags)
|
||||
|
||||
|
|
@ -96,9 +67,8 @@ Sometimes a source will block netlify or cloudflare. Making self hosted proxies
|
|||
## Comprehensive Flags Guide
|
||||
|
||||
For detailed information about using flags in your scrapers, including:
|
||||
|
||||
- When and how to use each flag
|
||||
- Provider-level vs stream-level flags
|
||||
- Provider-level vs stream-level flags
|
||||
- Best practices and examples
|
||||
- How flags affect stream playback
|
||||
|
||||
|
|
@ -112,121 +82,90 @@ import { createM3U8ProxyUrl } from '@/utils/proxy';
|
|||
|
||||
// Extension-only streams (MOST COMMON - just remove all flags)
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: createM3U8ProxyUrl(originalUrl, ctx.features, headers),
|
||||
headers,
|
||||
flags: [], // No flags = extension/native only, but this case doesn't make sense because the stream is getting proxied.
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: createM3U8ProxyUrl(originalUrl, ctx.features, headers),
|
||||
headers,
|
||||
flags: [], // No flags = extension/native only, but this case doesn't make sense because the stream is getting proxied.
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
|
||||
// Universal streams with CORS support
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: createM3U8ProxyUrl(originalUrl, ctx.features, headers),
|
||||
headers, // again listing headers twice so the extension can use them.
|
||||
flags: [flags.CORS_ALLOWED], // Works across all targets
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: createM3U8ProxyUrl(originalUrl, ctx.features, headers),
|
||||
headers, // again listing headers twice so the extension can use them.
|
||||
flags: [flags.CORS_ALLOWED], // Works across all targets
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
|
||||
// Direct streams (no proxy needed)
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [flags.CORS_ALLOWED], // Stream can be played directly in browsers
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [flags.CORS_ALLOWED], // Stream can be played directly in browsers
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
|
||||
// Extension-only streams (usual approach - just remove all flags)
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [], // No flags = extension/native only (most common)
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [], // No flags = extension/native only (most common)
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
|
||||
// Cloudflare-blocked streams with cosmetic label (if needed)
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [flags.CF_BLOCKED], // Cosmetic only - still extension/native only due to no CORS_ALLOWED
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [flags.CF_BLOCKED], // Cosmetic only - still extension/native only due to no CORS_ALLOWED
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
|
||||
// IP-locked streams (when you specifically need consistent IP)
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [flags.IP_LOCKED], // Prevents proxy usage when IP consistency required
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [flags.IP_LOCKED], // Prevents proxy usage when IP consistency required
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
|
||||
// IP-locked streams (when you specifically need consistent IP)
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [flags.IP_LOCKED], // Prevents proxy usage when IP consistency required
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: 'https://example.com/playlist.m3u8',
|
||||
flags: [flags.IP_LOCKED], // Prevents proxy usage when IP consistency required
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
|
||||
// Provider-level flags affect source visibility
|
||||
export const mySource = makeSourcerer({
|
||||
id: 'my-source',
|
||||
name: 'My Source',
|
||||
export const myScraper = makeSourcerer({
|
||||
id: 'my-scraper',
|
||||
name: 'My Scraper',
|
||||
rank: 150,
|
||||
flags: [flags.CORS_ALLOWED], // Source shows for all targets
|
||||
scrapeMovie: comboScraper,
|
||||
scrapeShow: comboScraper,
|
||||
});
|
||||
|
||||
// Embed-level flags affect embed visibility
|
||||
export const myEmbed = makeEmbed({
|
||||
id: 'my-embed',
|
||||
name: 'My Embed',
|
||||
rank: 150,
|
||||
flags: [flags.CORS_ALLOWED], // Embed shows for all targets
|
||||
scrape: async (ctx) => ({
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: createM3U8ProxyUrl(url, ctx.features, headers),
|
||||
flags: [flags.CORS_ALLOWED], // Stream flags should match embed flags
|
||||
captions: [],
|
||||
}],
|
||||
}),
|
||||
});
|
||||
```
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ Modern streaming services use various protection mechanisms.
|
|||
### Common Protections
|
||||
|
||||
1. **Referer Checking** - URLs only work from specific domains
|
||||
2. **CORS Restrictions** - Prevent browser access from unauthorized origins
|
||||
2. **CORS Restrictions** - Prevent browser access from unauthorized origins
|
||||
3. **Geographic Blocking** - IP-based access restrictions
|
||||
4. **Time-Limited Tokens** - URLs expire after short periods
|
||||
5. **User-Agent Filtering** - Only allow specific browsers/clients
|
||||
|
|
@ -28,44 +28,43 @@ const playlistUrl = 'https://protected-cdn.example.com/playlist.m3u8';
|
|||
|
||||
// Headers required to access the playlist
|
||||
const headers = {
|
||||
Referer: 'https://player.example.com/',
|
||||
Origin: 'https://player.example.com',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
'Referer': 'https://player.example.com/',
|
||||
'Origin': 'https://player.example.com',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||
};
|
||||
|
||||
// Convert playlist and all variants to data URLs
|
||||
const dataUrl = await convertPlaylistsToDataUrls(ctx.proxiedFetcher, playlistUrl, headers);
|
||||
const dataUrl = await convertPlaylistsToDataUrls(
|
||||
ctx.proxiedFetcher,
|
||||
playlistUrl,
|
||||
headers
|
||||
);
|
||||
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: dataUrl, // Self-contained data URL
|
||||
flags: [flags.CORS_ALLOWED], // No CORS issues with data URLs
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: dataUrl, // Self-contained data URL
|
||||
flags: [flags.CORS_ALLOWED], // No CORS issues with data URLs
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
```
|
||||
|
||||
**Why data URLs work for CORS bypass:**
|
||||
|
||||
- **Inital proxied request WITH HEADERS**: A request is sent from the proxy _with_ headers allowing for the playlist to load
|
||||
- **Inital proxied request WITH HEADERS**: A request is sent from the proxy *with* headers allowing for the playlist to load
|
||||
- **Fewer external requests**: Content is embedded directly in the URL as base64
|
||||
- **Same-origin**: Browsers treat data URLs as same-origin content
|
||||
- **Complete isolation**: No network requests means no CORS preflight checks
|
||||
- **Self-contained**: All playlist data and segments are embedded in the response
|
||||
|
||||
**How the conversion works:**
|
||||
|
||||
1. Fetches the master playlist using provided headers
|
||||
2. For each quality variant, fetches the variant playlist
|
||||
3. Converts all playlists to base64-encoded data URLs
|
||||
4. Returns a master data URL containing all embedded variants
|
||||
|
||||
**When to use data URLs vs M3U8 proxy:**
|
||||
|
||||
- **Use data URLs** when you can fetch all playlist data upfront
|
||||
- **Use M3U8 proxy** when playlists are too large or change frequently
|
||||
- **Data URLs are preferred** for most HLS streams due to simplicity and reliability
|
||||
|
|
@ -85,22 +84,20 @@ const originalPlaylist = 'https://protected-cdn.example.com/playlist.m3u8';
|
|||
|
||||
// Headers required by the streaming service
|
||||
const streamHeaders = {
|
||||
Referer: 'https://player.example.com/',
|
||||
Origin: 'https://player.example.com',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
'Referer': 'https://player.example.com/',
|
||||
'Origin': 'https://player.example.com',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||
};
|
||||
|
||||
return {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: createM3U8ProxyUrl(originalPlaylist, ctx.features, streamHeaders), // Include headers for proxy usage
|
||||
headers: streamHeaders, // Include headers for extension/native usage
|
||||
flags: [flags.CORS_ALLOWED], // Proxy enables CORS for all targets
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
stream: [{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: createM3U8ProxyUrl(originalPlaylist, ctx.features, streamHeaders), // Include headers for proxy usage
|
||||
headers: streamHeaders, // Include headers for extension/native usage
|
||||
flags: [flags.CORS_ALLOWED], // Proxy enables CORS for all targets
|
||||
captions: []
|
||||
}]
|
||||
};
|
||||
```
|
||||
|
||||
|
|
@ -118,13 +115,12 @@ const SKIP_VALIDATION_CHECK_IDS = [
|
|||
// Sources here are always proxied, so we dont need to validate with a proxy, but we should fetch nativly
|
||||
// NOTE: all m3u8 proxy urls are automatically processed using this method, so no need to add them here manually
|
||||
const UNPROXIED_VALIDATION_CHECK_IDS = [
|
||||
// ... existing IDs
|
||||
// ... existing IDs
|
||||
'your-scraper-id', // Add your scraper ID here
|
||||
];
|
||||
```
|
||||
|
||||
**Why this is needed:**
|
||||
|
||||
- By default, all streams are validated by attempting to fetch metadata
|
||||
- The validation uses `proxiedFetcher` to check if streams are playable
|
||||
- If the stream blocks the fetcher, validation will fail
|
||||
|
|
@ -132,7 +128,6 @@ const UNPROXIED_VALIDATION_CHECK_IDS = [
|
|||
- Adding to skip list bypasses validation and returns the proxied URL directly without checking it
|
||||
|
||||
**When to skip validation:**
|
||||
|
||||
- Validation consistently fails but streams work in browsers
|
||||
- The stream may be origin or IP locked
|
||||
- The stream blocks the extension or proxy
|
||||
|
|
@ -148,13 +143,13 @@ let stream = {
|
|||
type: 'file',
|
||||
flags: [],
|
||||
qualities: {
|
||||
'1080p': { url: 'https://protected-cdn.example.com/video.mp4' },
|
||||
'1080p': { url: 'https://protected-cdn.example.com/video.mp4' }
|
||||
},
|
||||
headers: {
|
||||
Referer: 'https://player.example.com/',
|
||||
'User-Agent': 'Mozilla/5.0...',
|
||||
'Referer': 'https://player.example.com/',
|
||||
'User-Agent': 'Mozilla/5.0...'
|
||||
},
|
||||
captions: [],
|
||||
captions: []
|
||||
};
|
||||
|
||||
// setupProxy will handle proxying if needed
|
||||
|
|
@ -168,19 +163,15 @@ return { stream: [stream] };
|
|||
### Efficient Data Extraction
|
||||
|
||||
**Use targeted selectors:**
|
||||
|
||||
```typescript
|
||||
// ✅ Good - specific selector
|
||||
const embedUrl = $('iframe[src*="turbovid"]').attr('src');
|
||||
|
||||
// ❌ Bad - searches entire document
|
||||
const embedUrl = $('*')
|
||||
.filter((_, el) => $(el).attr('src')?.includes('turbovid'))
|
||||
.attr('src');
|
||||
const embedUrl = $('*').filter((_, el) => $(el).attr('src')?.includes('turbovid')).attr('src');
|
||||
```
|
||||
|
||||
**Cache expensive operations:**
|
||||
|
||||
```typescript
|
||||
// Cache parsed data to avoid re-parsing
|
||||
let cachedConfig;
|
||||
|
|
@ -192,7 +183,6 @@ if (!cachedConfig) {
|
|||
### Minimize HTTP Requests
|
||||
|
||||
**Combine operations when possible:**
|
||||
|
||||
```typescript
|
||||
// ✅ Good - single request with full processing
|
||||
const embedPage = await ctx.proxiedFetcher(embedUrl);
|
||||
|
|
@ -210,7 +200,6 @@ const streams = extractStreams(page2);
|
|||
### Input Validation
|
||||
|
||||
**Validate external data:**
|
||||
|
||||
```typescript
|
||||
// Validate URLs before using them
|
||||
const isValidUrl = (url: string) => {
|
||||
|
|
@ -228,7 +217,6 @@ if (!isValidUrl(streamUrl)) {
|
|||
```
|
||||
|
||||
**Sanitize regex inputs:**
|
||||
|
||||
```typescript
|
||||
// Be careful with dynamic regex
|
||||
const safeTitle = ctx.media.title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
|
@ -297,4 +285,4 @@ With these advanced concepts:
|
|||
1. Review [Sources vs Embeds](/in-depth/sources-and-embeds) for architectural patterns
|
||||
2. Study existing scrapers in `src/providers/` for real examples
|
||||
3. Test your implementation thoroughly
|
||||
4. Submit pull requests with detailed testing documentation
|
||||
4. Submit pull requests with detailed testing documentation
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
icon: ph:atom-fill
|
||||
navigation.redirect: /in-depth/new-providers
|
||||
navigation.title: 'In-depth'
|
||||
navigation.title: "In-depth"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
icon: ph:aperture-fill
|
||||
navigation.redirect: /extra-topics/development
|
||||
navigation.title: 'Extra topics'
|
||||
navigation.title: "Extra topics"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# `makeProviders`
|
||||
|
||||
Make an instance of provider controls with configuration.
|
||||
Make an instance of provider controls with configuration.
|
||||
This is the main entry-point of the library. It is recommended to make one instance globally and reuse it throughout your application.
|
||||
|
||||
## Example
|
||||
|
|
@ -22,7 +22,7 @@ function makeProviders(ops: ProviderBuilderOptions): ProviderControls;
|
|||
interface ProviderBuilderOptions {
|
||||
// instance of a fetcher, all webrequests are made with the fetcher.
|
||||
fetcher: Fetcher;
|
||||
|
||||
|
||||
// instance of a fetcher, in case the request has CORS restrictions.
|
||||
// this fetcher will be called instead of normal fetcher.
|
||||
// if your environment doesn't have CORS restrictions (like Node.JS), there is no need to set this.
|
||||
|
|
|
|||
|
|
@ -4,30 +4,28 @@ Run all providers one by one in order of their built-in ranking.
|
|||
You can attach events if you need to know what is going on while it is processing.
|
||||
|
||||
## Example
|
||||
|
||||
(Use the cli to learn more specifics about inputs)
|
||||
|
||||
```ts
|
||||
// media from TMDB
|
||||
const media = {
|
||||
type: 'movie', // "movie" | "show"
|
||||
type: 'movie', // "movie" | "show"
|
||||
title: 'Hamilton',
|
||||
tmdbId: '556574',
|
||||
// season: '1',
|
||||
// episode: '1'
|
||||
};
|
||||
}
|
||||
|
||||
// scrape a stream
|
||||
const stream = await providers.runAll({
|
||||
media: media,
|
||||
});
|
||||
})
|
||||
|
||||
// scrape a stream, but prioritize flixhq above all
|
||||
// (other scrapers are still run if flixhq fails, it just has priority)
|
||||
const flixhqStream = await providers.runAll({
|
||||
media: media,
|
||||
sourceOrder: ['flixhq'],
|
||||
});
|
||||
sourceOrder: ['flixhq']
|
||||
})
|
||||
```
|
||||
|
||||
## Type
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ const media = {
|
|||
type: 'movie',
|
||||
title: 'Hamilton',
|
||||
releaseYear: 2020,
|
||||
tmdbId: '556574',
|
||||
};
|
||||
tmdbId: '556574'
|
||||
}
|
||||
|
||||
// scrape a stream from flixhq
|
||||
let output: SourcererOutput;
|
||||
|
|
@ -21,12 +21,12 @@ try {
|
|||
output = await providers.runSourceScraper({
|
||||
id: 'flixhq',
|
||||
media: media,
|
||||
});
|
||||
})
|
||||
} catch (err) {
|
||||
if (err instanceof NotFoundError) {
|
||||
console.log('source does not have this media');
|
||||
} else {
|
||||
console.log('failed to scrape');
|
||||
console.log('failed to scrape')
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ try {
|
|||
output = await providers.runEmbedScraper({
|
||||
id: 'upcloud',
|
||||
url: 'https://example.com/123',
|
||||
});
|
||||
})
|
||||
} catch (err) {
|
||||
console.log('failed to scrape');
|
||||
console.log('failed to scrape')
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Make a fetcher to use with [p-stream/simple-proxy](https://github.com/p-stream/s
|
|||
```ts
|
||||
import { targets, makeProviders, makeDefaultFetcher, makeSimpleProxyFetcher } from '@p-stream/providers';
|
||||
|
||||
const proxyUrl = 'https://your.proxy.workers.dev/';
|
||||
const proxyUrl = 'https://your.proxy.workers.dev/'
|
||||
|
||||
const providers = makeProviders({
|
||||
fetcher: makeDefaultFetcher(fetch),
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
icon: ph:code-simple-fill
|
||||
navigation.redirect: /api/makeproviders
|
||||
navigation.title: 'Api reference'
|
||||
navigation.title: "Api reference"
|
||||
|
|
|
|||
|
|
@ -2,16 +2,20 @@ export default defineNuxtConfig({
|
|||
// https://github.com/nuxt-themes/docus
|
||||
extends: '@nuxt-themes/docus',
|
||||
|
||||
css: ['@/assets/css/main.css'],
|
||||
css: [
|
||||
'@/assets/css/main.css',
|
||||
],
|
||||
|
||||
build: {
|
||||
transpile: ['chalk'],
|
||||
transpile: [
|
||||
"chalk"
|
||||
]
|
||||
},
|
||||
|
||||
modules: [
|
||||
// https://github.com/nuxt-modules/plausible
|
||||
'@nuxtjs/plausible',
|
||||
// https://github.com/nuxt/devtools
|
||||
'@nuxt/devtools',
|
||||
],
|
||||
});
|
||||
'@nuxt/devtools'
|
||||
]
|
||||
})
|
||||
|
|
|
|||
4744
.docs/pnpm-lock.yaml
4744
.docs/pnpm-lock.yaml
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,8 @@
|
|||
{
|
||||
"extends": ["@nuxtjs"],
|
||||
"lockFileMaintenance": {
|
||||
"enabled": true
|
||||
}
|
||||
"extends": [
|
||||
"@nuxtjs"
|
||||
],
|
||||
"lockFileMaintenance": {
|
||||
"enabled": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
import { defineTheme } from 'pinceau';
|
||||
import { defineTheme } from 'pinceau'
|
||||
|
||||
export default defineTheme({
|
||||
color: {
|
||||
primary: {
|
||||
50: '#F5E5FF',
|
||||
100: '#E7CCFF',
|
||||
200: '#D4A9FF',
|
||||
300: '#BE85FF',
|
||||
400: '#A861FF',
|
||||
500: '#8E3DFF',
|
||||
600: '#7F36D4',
|
||||
700: '#662CA6',
|
||||
800: '#552578',
|
||||
900: '#441E49',
|
||||
},
|
||||
},
|
||||
});
|
||||
50: "#F5E5FF",
|
||||
100: "#E7CCFF",
|
||||
200: "#D4A9FF",
|
||||
300: "#BE85FF",
|
||||
400: "#A861FF",
|
||||
500: "#8E3DFF",
|
||||
600: "#7F36D4",
|
||||
700: "#662CA6",
|
||||
800: "#552578",
|
||||
900: "#441E49"
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in a new issue