mirror of
https://github.com/TheBeastLT/torrentio-scraper.git
synced 2026-05-16 08:51:44 +00:00
feat: remove catalogs, add create-tables script
This commit is contained in:
Dylan Foster
parent
458bb3346c
commit
6be1936aa1
15 changed files with 180 additions and 3793 deletions
74
MIGRATION.md
Normal file
74
MIGRATION.md
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
# Database Migration Guide
|
||||
|
||||
This guide shows how to create the required database tables for the Torrentio addon.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Docker and Docker Compose** installed
|
||||
2. **Node.js** installed (for running the migration script)
|
||||
|
||||
## Method 1: Using Docker Compose (Recommended)
|
||||
|
||||
### Step 1: Start the PostgreSQL database
|
||||
```bash
|
||||
cd addon
|
||||
docker-compose up postgres -d
|
||||
```
|
||||
|
||||
### Step 2: Run the migration script
|
||||
```bash
|
||||
# From the addon directory
|
||||
npm run create-tables
|
||||
|
||||
# Or directly
|
||||
node create-tables.js
|
||||
```
|
||||
|
||||
## Method 2: Using existing PostgreSQL instance
|
||||
|
||||
### Step 1: Set up your database
|
||||
Create a PostgreSQL database named `torrentio` with user `torrentio` and password `torrentio`.
|
||||
|
||||
### Step 2: Set environment variable (optional)
|
||||
```bash
|
||||
export DATABASE_URI="postgres://torrentio:torrentio@localhost:5432/torrentio"
|
||||
```
|
||||
|
||||
### Step 3: Run the migration
|
||||
```bash
|
||||
cd addon
|
||||
node create-tables.js
|
||||
```
|
||||
|
||||
## What the script creates
|
||||
|
||||
The migration script creates three tables:
|
||||
|
||||
1. **`torrents`** - Stores torrent metadata
|
||||
- `infoHash` (primary key)
|
||||
- `provider`, `title`, `size`, `type`
|
||||
- `uploadDate`, `seeders`, `trackers`
|
||||
- `languages`, `resolution`
|
||||
|
||||
2. **`files`** - Stores individual files within torrents
|
||||
- `id` (auto-increment primary key)
|
||||
- `infoHash` (foreign key to torrents)
|
||||
- `fileIndex`, `title`, `size`
|
||||
- `imdbId`, `imdbSeason`, `imdbEpisode` (for movies/TV)
|
||||
- `kitsuId`, `kitsuEpisode` (for anime)
|
||||
|
||||
3. **`subtitles`** - Stores subtitle file information
|
||||
- `infoHash` (foreign key to torrents)
|
||||
- `fileIndex`, `fileId` (foreign key to files)
|
||||
- `title`, `size`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **Connection refused**: Make sure PostgreSQL is running
|
||||
- **Database doesn't exist**: Create the `torrentio` database first
|
||||
- **Permission denied**: Check database user permissions
|
||||
- **Tables already exist**: The script will skip existing tables (use `force: true` in the script to recreate them)
|
||||
|
||||
## Next Steps
|
||||
|
||||
After creating the tables, you'll need to populate them with torrent data. This addon only queries existing data - it doesn't scrape or ingest torrents. You'll need a separate service to populate the database with torrent metadata.
|
||||
104
addon/create-tables.js
Normal file
104
addon/create-tables.js
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import { Sequelize } from 'sequelize';
|
||||
|
||||
// Database configuration
|
||||
const DATABASE_URI = process.env.DATABASE_URI || 'postgres://torrentio:torrentio@localhost:5432/torrentio';
|
||||
|
||||
const database = new Sequelize(DATABASE_URI, {
|
||||
logging: console.log, // Enable logging to see SQL commands
|
||||
pool: { max: 30, min: 5, idle: 20 * 60 * 1000 }
|
||||
});
|
||||
|
||||
// Define the same models as in repository.js
|
||||
const Torrent = database.define('torrent',
|
||||
{
|
||||
infoHash: { type: Sequelize.STRING(64), primaryKey: true },
|
||||
provider: { type: Sequelize.STRING(32), allowNull: false },
|
||||
torrentId: { type: Sequelize.STRING(128) },
|
||||
title: { type: Sequelize.STRING(256), allowNull: false },
|
||||
size: { type: Sequelize.BIGINT },
|
||||
type: { type: Sequelize.STRING(16), allowNull: false },
|
||||
uploadDate: { type: Sequelize.DATE, allowNull: false },
|
||||
seeders: { type: Sequelize.SMALLINT },
|
||||
trackers: { type: Sequelize.STRING(4096) },
|
||||
languages: { type: Sequelize.STRING(4096) },
|
||||
resolution: { type: Sequelize.STRING(16) }
|
||||
}
|
||||
);
|
||||
|
||||
const File = database.define('file',
|
||||
{
|
||||
id: { type: Sequelize.BIGINT, autoIncrement: true, primaryKey: true },
|
||||
infoHash: {
|
||||
type: Sequelize.STRING(64),
|
||||
allowNull: false,
|
||||
references: { model: Torrent, key: 'infoHash' },
|
||||
onDelete: 'CASCADE'
|
||||
},
|
||||
fileIndex: { type: Sequelize.INTEGER },
|
||||
title: { type: Sequelize.STRING(256), allowNull: false },
|
||||
size: { type: Sequelize.BIGINT },
|
||||
imdbId: { type: Sequelize.STRING(32) },
|
||||
imdbSeason: { type: Sequelize.INTEGER },
|
||||
imdbEpisode: { type: Sequelize.INTEGER },
|
||||
kitsuId: { type: Sequelize.INTEGER },
|
||||
kitsuEpisode: { type: Sequelize.INTEGER }
|
||||
},
|
||||
);
|
||||
|
||||
const Subtitle = database.define('subtitle',
|
||||
{
|
||||
infoHash: {
|
||||
type: Sequelize.STRING(64),
|
||||
allowNull: false,
|
||||
references: { model: Torrent, key: 'infoHash' },
|
||||
onDelete: 'CASCADE'
|
||||
},
|
||||
fileIndex: { type: Sequelize.INTEGER, allowNull: false },
|
||||
fileId: {
|
||||
type: Sequelize.BIGINT,
|
||||
allowNull: true,
|
||||
references: { model: File, key: 'id' },
|
||||
onDelete: 'SET NULL'
|
||||
},
|
||||
title: { type: Sequelize.STRING(512), allowNull: false },
|
||||
size: { type: Sequelize.BIGINT, allowNull: false },
|
||||
},
|
||||
{ timestamps: false }
|
||||
);
|
||||
|
||||
// Set up relationships
|
||||
Torrent.hasMany(File, { foreignKey: 'infoHash', constraints: false });
|
||||
File.belongsTo(Torrent, { foreignKey: 'infoHash', constraints: false });
|
||||
File.hasMany(Subtitle, { foreignKey: 'fileId', constraints: false });
|
||||
Subtitle.belongsTo(File, { foreignKey: 'fileId', constraints: false });
|
||||
|
||||
async function createTables() {
|
||||
try {
|
||||
console.log('🔄 Connecting to database...');
|
||||
await database.authenticate();
|
||||
console.log('✅ Database connection established successfully!');
|
||||
|
||||
console.log('🔄 Creating database tables...');
|
||||
await database.sync({ force: false }); // Set to true if you want to drop existing tables
|
||||
console.log('✅ Database tables created successfully!');
|
||||
|
||||
console.log('📊 Tables created:');
|
||||
console.log(' - torrents (torrent metadata)');
|
||||
console.log(' - files (individual files within torrents)');
|
||||
console.log(' - subtitles (subtitle file information)');
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Error creating tables:', error.message);
|
||||
console.error('Full error:', error);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await database.close();
|
||||
console.log('🔌 Database connection closed.');
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the migration
|
||||
createTables();
|
||||
|
|
@ -5,7 +5,8 @@
|
|||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"dev": "node --insecure-http-parser --watch index.js"
|
||||
"dev": "node --insecure-http-parser --watch index.js",
|
||||
"create-tables": "node create-tables.js"
|
||||
},
|
||||
"author": "TheBeastLT <pauliox@beyond.lt>",
|
||||
"license": "MIT",
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
**/node_modules
|
||||
**/npm-debug.log
|
||||
**/.env
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
FROM node:16-alpine
|
||||
|
||||
RUN apk update && apk upgrade && \
|
||||
apk add --no-cache git
|
||||
|
||||
WORKDIR /home/node/app
|
||||
|
||||
COPY ./catalogs .
|
||||
COPY ./addon ../addon
|
||||
RUN npm ci --only-production
|
||||
|
||||
CMD [ "node", "index.js" ]
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
import Bottleneck from 'bottleneck';
|
||||
import moment from 'moment';
|
||||
import { addonBuilder } from 'stremio-addon-sdk';
|
||||
import { Providers } from '../addon/lib/filter.js';
|
||||
import { createManifest, genres } from './lib/manifest.js';
|
||||
import { getMetas } from './lib/metadata.js';
|
||||
import { cacheWrapCatalog, cacheWrapIds } from './lib/cache.js';
|
||||
import * as repository from './lib/repository.js';
|
||||
|
||||
const CACHE_MAX_AGE = parseInt(process.env.CACHE_MAX_AGE) || 4 * 60 * 60; // 4 hours in seconds
|
||||
const STALE_REVALIDATE_AGE = 4 * 60 * 60; // 4 hours
|
||||
const STALE_ERROR_AGE = 7 * 24 * 60 * 60; // 7 days
|
||||
|
||||
const manifest = createManifest();
|
||||
const builder = new addonBuilder(manifest);
|
||||
const limiter = new Bottleneck({
|
||||
maxConcurrent: process.env.LIMIT_MAX_CONCURRENT || 20,
|
||||
highWater: process.env.LIMIT_QUEUE_SIZE || 50,
|
||||
strategy: Bottleneck.strategy.OVERFLOW
|
||||
});
|
||||
const defaultProviders = Providers.options
|
||||
.filter(provider => !provider.foreign)
|
||||
.map(provider => provider.label)
|
||||
.sort();
|
||||
|
||||
builder.defineCatalogHandler((args) => {
|
||||
const offset = parseInt(args.extra.skip || '0', 10);
|
||||
const genre = args.extra.genre || 'default';
|
||||
const catalog = manifest.catalogs.find(c => c.id === args.id);
|
||||
const providers = defaultProviders;
|
||||
console.log(`Incoming catalog ${args.id} request with genre=${genre} and skip=${offset}`);
|
||||
if (!catalog) {
|
||||
return Promise.reject(`No catalog found for with id: ${args.id}`);
|
||||
}
|
||||
|
||||
const cacheKey = createCacheKey(catalog.id, providers, genre, offset);
|
||||
return limiter.schedule(() => cacheWrapCatalog(cacheKey, () => getCatalog(catalog, providers, genre, offset)))
|
||||
.then(metas => ({
|
||||
metas: metas,
|
||||
cacheMaxAge: CACHE_MAX_AGE,
|
||||
staleRevalidate: STALE_REVALIDATE_AGE,
|
||||
staleError: STALE_ERROR_AGE
|
||||
}))
|
||||
.catch(error => Promise.reject(`Failed retrieving catalog ${args.id}: ${error.message || error}`));
|
||||
})
|
||||
|
||||
async function getCursor(catalog, providers, genre, offset) {
|
||||
if (offset === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const previousOffset = offset - catalog.pageSize;
|
||||
const previousCacheKey = createCacheKey(catalog.id, providers, genre, previousOffset);
|
||||
return cacheWrapCatalog(previousCacheKey, () => Promise.reject("cursor not found"))
|
||||
.then(metas => metas[metas.length - 1])
|
||||
.then(meta => meta.id.replace('kitsu:', ''))
|
||||
}
|
||||
|
||||
async function getCatalog(catalog, providers, genre, offset) {
|
||||
const cursor = await getCursor(catalog, providers, genre, offset)
|
||||
const startDate = getStartDate(genre)?.toISOString();
|
||||
const endDate = getEndDate(genre)?.toISOString();
|
||||
const cacheKey = createCacheKey(catalog.id, providers, genre);
|
||||
|
||||
return cacheWrapIds(cacheKey, () => repository.getIds(providers, catalog.type, startDate, endDate))
|
||||
.then(ids => ids.slice(ids.indexOf(cursor) + 1))
|
||||
.then(ids => getMetas(ids, catalog.type))
|
||||
.then(metas => metas.slice(0, catalog.pageSize));
|
||||
}
|
||||
|
||||
function getStartDate(genre) {
|
||||
switch (genre) {
|
||||
case genres[0]: return moment().utc().subtract(1, 'day').startOf('day');
|
||||
case genres[1]: return moment().utc().startOf('isoWeek');
|
||||
case genres[2]: return moment().utc().subtract(7, 'day').startOf('isoWeek');
|
||||
case genres[3]: return moment().utc().startOf('month');
|
||||
case genres[4]: return moment().utc().subtract(30, 'day').startOf('month');
|
||||
case genres[5]: return undefined;
|
||||
default: return moment().utc().subtract(30, 'day').startOf('day');
|
||||
}
|
||||
}
|
||||
|
||||
function getEndDate(genre) {
|
||||
switch (genre) {
|
||||
case genres[0]: return moment().utc().subtract(1, 'day').endOf('day');
|
||||
case genres[1]: return moment().utc().endOf('isoWeek');
|
||||
case genres[2]: return moment().utc().subtract(7, 'day').endOf('isoWeek');
|
||||
case genres[3]: return moment().utc().endOf('month');
|
||||
case genres[4]: return moment().utc().subtract(30, 'day').endOf('month');
|
||||
case genres[5]: return undefined;
|
||||
default: return moment().utc().subtract(1, 'day').endOf('day');
|
||||
}
|
||||
}
|
||||
|
||||
function createCacheKey(catalogId, providers, genre, offset) {
|
||||
const dateKey = moment().format('YYYY-MM-DD');
|
||||
return [catalogId, providers.join(','), genre, dateKey, offset].filter(x => x !== undefined).join('|');
|
||||
}
|
||||
|
||||
export default builder.getInterface();
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
import express from 'express';
|
||||
import serverless from './serverless.js';
|
||||
|
||||
const app = express();
|
||||
|
||||
app.use((req, res, next) => serverless(req, res, next));
|
||||
app.listen(process.env.PORT || 7000, () => {
|
||||
console.log(`Started addon at: http://localhost:${process.env.PORT || 7000}`);
|
||||
});
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
import KeyvMongo from "@keyv/mongo";
|
||||
|
||||
const CATALOG_TTL = 24 * 60 * 60 * 1000; // 24 hours
|
||||
|
||||
const MONGO_URI = process.env.MONGODB_URI;
|
||||
|
||||
const remoteCache = MONGO_URI && new KeyvMongo(MONGO_URI, { collection: 'torrentio_catalog_collection' });
|
||||
|
||||
async function cacheWrap(cache, key, method, ttl) {
|
||||
if (!cache) {
|
||||
return method();
|
||||
}
|
||||
const value = await cache.get(key);
|
||||
if (value !== undefined) {
|
||||
return value;
|
||||
}
|
||||
const result = await method();
|
||||
await cache.set(key, result, ttl);
|
||||
return result;
|
||||
}
|
||||
|
||||
export function cacheWrapCatalog(key, method) {
|
||||
return cacheWrap(remoteCache, key, method, CATALOG_TTL);
|
||||
}
|
||||
|
||||
export function cacheWrapIds(key, method) {
|
||||
return cacheWrap(remoteCache, `ids|${key}`, method, CATALOG_TTL);
|
||||
}
|
||||
|
|
@ -1,274 +0,0 @@
|
|||
const STYLESHEET = `
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body,
|
||||
html {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
width: 100%;
|
||||
height: 100%
|
||||
}
|
||||
|
||||
html {
|
||||
background-size: auto 100%;
|
||||
background-size: cover;
|
||||
background-position: center center;
|
||||
background-repeat: repeat-y;
|
||||
}
|
||||
|
||||
body {
|
||||
display: flex;
|
||||
background-color: transparent;
|
||||
font-family: 'Open Sans', Arial, sans-serif;
|
||||
color: white;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 4.5vh;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
h2 {
|
||||
font-size: 2.2vh;
|
||||
font-weight: normal;
|
||||
font-style: italic;
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
h3 {
|
||||
font-size: 2.2vh;
|
||||
}
|
||||
|
||||
h1,
|
||||
h2,
|
||||
h3,
|
||||
p,
|
||||
label {
|
||||
margin: 0;
|
||||
text-shadow: 0 0 1vh rgba(0, 0, 0, 0.15);
|
||||
}
|
||||
|
||||
p {
|
||||
font-size: 1.75vh;
|
||||
}
|
||||
|
||||
ul {
|
||||
font-size: 1.75vh;
|
||||
margin: 0;
|
||||
margin-top: 1vh;
|
||||
padding-left: 3vh;
|
||||
}
|
||||
|
||||
a {
|
||||
color: green
|
||||
}
|
||||
|
||||
a.install-link {
|
||||
text-decoration: none
|
||||
}
|
||||
|
||||
button {
|
||||
border: 0;
|
||||
outline: 0;
|
||||
color: white;
|
||||
background: #8A5AAB;
|
||||
padding: 1.2vh 3.5vh;
|
||||
margin: auto;
|
||||
text-align: center;
|
||||
font-family: 'Open Sans', Arial, sans-serif;
|
||||
font-size: 2.2vh;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
display: block;
|
||||
box-shadow: 0 0.5vh 1vh rgba(0, 0, 0, 0.2);
|
||||
transition: box-shadow 0.1s ease-in-out;
|
||||
}
|
||||
|
||||
button:hover {
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
button:active {
|
||||
box-shadow: 0 0 0 0.5vh white inset;
|
||||
}
|
||||
|
||||
#addon {
|
||||
width: 90vh;
|
||||
margin: auto;
|
||||
padding-left: 10%;
|
||||
padding-right: 10%;
|
||||
background: rgba(0, 0, 0, 0.60);
|
||||
}
|
||||
|
||||
.logo {
|
||||
height: 14vh;
|
||||
width: 14vh;
|
||||
margin: auto;
|
||||
margin-bottom: 3vh;
|
||||
}
|
||||
|
||||
.logo img {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.name, .version {
|
||||
display: inline-block;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
.name {
|
||||
line-height: 5vh;
|
||||
}
|
||||
|
||||
.version {
|
||||
position: absolute;
|
||||
line-height: 5vh;
|
||||
margin-left: 1vh;
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
.contact {
|
||||
position: absolute;
|
||||
left: 0;
|
||||
bottom: 4vh;
|
||||
width: 100%;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.contact a {
|
||||
font-size: 1.4vh;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.separator {
|
||||
margin-bottom: 4vh;
|
||||
}
|
||||
|
||||
.label {
|
||||
font-size: 2.2vh;
|
||||
font-weight: 600;
|
||||
padding: 0;
|
||||
line-height: inherit;
|
||||
}
|
||||
|
||||
.btn-group, .multiselect-container {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.btn {
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.multiselect-container {
|
||||
border: 0;
|
||||
border-radius: 0;
|
||||
}
|
||||
|
||||
.input, .btn {
|
||||
height: 3.8vh;
|
||||
width: 100%;
|
||||
margin: auto;
|
||||
margin-bottom: 10px;
|
||||
padding: 6px 12px;
|
||||
border: 0;
|
||||
border-radius: 0;
|
||||
outline: 0;
|
||||
color: #333;
|
||||
background-color: rgb(255, 255, 255);
|
||||
box-shadow: 0 0.5vh 1vh rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
`;
|
||||
import { Providers } from '../../addon/lib/filter.js';
|
||||
|
||||
export default function landingTemplate(manifest, config = {}) {
|
||||
const providers = config.providers || [];
|
||||
|
||||
const background = manifest.background || 'https://dl.strem.io/addon-background.jpg';
|
||||
const logo = manifest.logo || 'https://dl.strem.io/addon-logo.png';
|
||||
const contactHTML = manifest.contactEmail ?
|
||||
`<div class="contact">
|
||||
<p>Contact ${manifest.name} creator:</p>
|
||||
<a href="mailto:${manifest.contactEmail}">${manifest.contactEmail}</a>
|
||||
</div>` : '<div class="separator"></div>';
|
||||
const providersHTML = Providers.options
|
||||
.map(provider => `<option value="${provider.key}">${provider.foreign || ''}${provider.label}</option>`)
|
||||
.join('\n');
|
||||
const stylizedTypes = manifest.types
|
||||
.map(t => t[0].toUpperCase() + t.slice(1) + (t !== 'series' ? 's' : ''));
|
||||
|
||||
return `
|
||||
<!DOCTYPE html>
|
||||
<html style="background-image: url(${background});">
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>${manifest.name} - Stremio Addon</title>
|
||||
<link rel="shortcut icon" href="${logo}" type="image/x-icon">
|
||||
<link href="https://fonts.googleapis.com/css?family=Open+Sans:400,600,700&display=swap" rel="stylesheet">
|
||||
<script src="https://code.jquery.com/jquery-3.5.1.slim.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js"></script>
|
||||
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
|
||||
<link href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css" rel="stylesheet" >
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-multiselect/0.9.15/js/bootstrap-multiselect.min.js"></script>
|
||||
<link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-multiselect/0.9.15/css/bootstrap-multiselect.css" rel="stylesheet"/>
|
||||
<style>${STYLESHEET}</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div id="addon">
|
||||
<div class="logo">
|
||||
<img src="${logo}">
|
||||
</div>
|
||||
<h1 class="name">${manifest.name}</h1>
|
||||
<h2 class="version">${manifest.version || '0.0.0'}</h2>
|
||||
<h2 class="description">${manifest.description || ''}</h2>
|
||||
|
||||
<div class="separator"></div>
|
||||
|
||||
<h3 class="gives">This addon has more :</h3>
|
||||
<ul>
|
||||
${stylizedTypes.map(t => `<li>${t}</li>`).join('')}
|
||||
</ul>
|
||||
|
||||
<div class="separator"></div>
|
||||
|
||||
<label class="label" for="iProviders">Providers:</label>
|
||||
<select id="iProviders" class="input" name="providers[]" multiple="multiple">
|
||||
${providersHTML}
|
||||
</select>
|
||||
|
||||
<div class="separator"></div>
|
||||
|
||||
<a id="installLink" class="install-link" href="#">
|
||||
<button name="Install">INSTALL</button>
|
||||
</a>
|
||||
${contactHTML}
|
||||
</div>
|
||||
<script type="text/javascript">
|
||||
$(document).ready(function() {
|
||||
$('#iProviders').multiselect({
|
||||
nonSelectedText: 'All providers',
|
||||
onChange: () => generateInstallLink()
|
||||
});
|
||||
$('#iProviders').multiselect('select', [${providers.map(provider => '"' + provider + '"')}]);
|
||||
generateInstallLink();
|
||||
});
|
||||
|
||||
function generateInstallLink() {
|
||||
const providersValue = $('#iProviders').val().join(',') || '';
|
||||
const providers = providersValue.length && providersValue;
|
||||
const configurationValue = [
|
||||
['${Providers.key}', providers],
|
||||
]
|
||||
.filter(([_, value]) => value.length)
|
||||
.map(([key, value]) => key + '=' + value).join('|');
|
||||
const configuration = configurationValue && configurationValue.length ? '/' + configurationValue : '';
|
||||
installLink.href = 'stremio://' + window.location.host + configuration + '/manifest.json';
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>`
|
||||
}
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
import { Type } from '../../addon/lib/types.js';
|
||||
|
||||
export const genres = [
|
||||
'Yesterday',
|
||||
'This Week',
|
||||
'Last Week',
|
||||
'This Month',
|
||||
'Last Month',
|
||||
'All Time'
|
||||
]
|
||||
|
||||
export function createManifest() {
|
||||
return {
|
||||
id: 'com.stremio.torrentio.catalog.addon',
|
||||
version: '1.0.2',
|
||||
name: 'Torrent Catalogs',
|
||||
description: 'Provides catalogs for movies/series/anime based on top seeded torrents. Requires Kitsu addon for anime.',
|
||||
logo: `https://i.ibb.co/w4BnkC9/GwxAcDV.png`,
|
||||
background: `https://i.ibb.co/VtSfFP9/t8wVwcg.jpg`,
|
||||
types: [Type.MOVIE, Type.SERIES, Type.ANIME],
|
||||
resources: ['catalog'],
|
||||
catalogs: [
|
||||
{
|
||||
id: 'top-movies',
|
||||
type: Type.MOVIE,
|
||||
name: "Top seeded",
|
||||
pageSize: 20,
|
||||
extra: [{ name: 'genre', options: genres }, { name: 'skip' }],
|
||||
genres: genres
|
||||
},
|
||||
{
|
||||
id: 'top-series',
|
||||
type: Type.SERIES,
|
||||
name: "Top seeded",
|
||||
pageSize: 20,
|
||||
extra: [{ name: 'genre', options: genres }, { name: 'skip' }],
|
||||
genres: genres
|
||||
},
|
||||
{
|
||||
id: 'top-anime',
|
||||
type: Type.ANIME,
|
||||
name: "Top seeded",
|
||||
pageSize: 20,
|
||||
extra: [{ name: 'genre', options: genres }, { name: 'skip' }],
|
||||
genres: genres
|
||||
}
|
||||
],
|
||||
behaviorHints: {
|
||||
// @TODO might enable configuration to configure providers
|
||||
configurable: false,
|
||||
configurationRequired: false
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
import axios from 'axios';
|
||||
import { Type } from '../../addon/lib/types.js';
|
||||
|
||||
const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
|
||||
const KITSU_URL = 'https://anime-kitsu.strem.fun';
|
||||
const TIMEOUT = 30000;
|
||||
const MAX_SIZE = 40;
|
||||
|
||||
export async function getMetas(ids, type) {
|
||||
if (!ids.length || !type) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return _requestMetadata(ids, type)
|
||||
.catch((error) => {
|
||||
throw new Error(`failed metadata ${type} query due: ${error.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
function _requestMetadata(ids, type) {
|
||||
const url = _getUrl(ids, type);
|
||||
return axios.get(url, { timeout: TIMEOUT })
|
||||
.then(response => response?.data?.metas || response?.data?.metasDetailed || [])
|
||||
.then(metas => metas.filter(meta => meta))
|
||||
.then(metas => metas.map(meta => _sanitizeMeta(meta)));
|
||||
}
|
||||
|
||||
function _getUrl(ids, type) {
|
||||
const joinedIds = ids.slice(0, MAX_SIZE).join(',');
|
||||
if (type === Type.ANIME) {
|
||||
return `${KITSU_URL}/catalog/${type}/kitsu-anime-list/lastVideosIds=${joinedIds}.json`;
|
||||
}
|
||||
return `${CINEMETA_URL}/catalog/${type}/last-videos/lastVideosIds=${joinedIds}.json`;
|
||||
}
|
||||
|
||||
function _sanitizeMeta(meta) {
|
||||
delete meta.videos;
|
||||
delete meta.credits_cast;
|
||||
delete meta.credits_crew;
|
||||
return meta;
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
import { Sequelize, QueryTypes } from 'sequelize';
|
||||
import { Type } from '../../addon/lib/types.js';
|
||||
|
||||
const DATABASE_URI = process.env.DATABASE_URI;
|
||||
|
||||
const database = new Sequelize(DATABASE_URI, { logging: false });
|
||||
|
||||
export async function getIds(providers, type, startDate, endDate) {
|
||||
const idName = type === Type.ANIME ? 'kitsuId' : 'imdbId';
|
||||
const episodeCondition = type === Type.SERIES
|
||||
? 'AND files."imdbSeason" IS NOT NULL AND files."imdbEpisode" IS NOT NULL'
|
||||
: '';
|
||||
const dateCondition = startDate && endDate
|
||||
? `AND "uploadDate" BETWEEN '${startDate}' AND '${endDate}'`
|
||||
: '';
|
||||
const providersCondition = providers && providers.length
|
||||
? `AND provider in (${providers.map(it => `'${it}'`).join(',')})`
|
||||
: '';
|
||||
const titleCondition = type === Type.MOVIE
|
||||
? 'AND torrents.title NOT LIKE \'%[Erotic]%\''
|
||||
: '';
|
||||
const sortCondition = type === Type.MOVIE ? 'sum(torrents.seeders)' : 'max(torrents.seeders)';
|
||||
const query = `SELECT files."${idName}"
|
||||
FROM (SELECT torrents."infoHash", torrents.seeders FROM torrents
|
||||
WHERE seeders > 0 AND type = '${type}' ${providersCondition} ${dateCondition} ${titleCondition}
|
||||
) as torrents
|
||||
JOIN files ON torrents."infoHash" = files."infoHash"
|
||||
WHERE files."${idName}" IS NOT NULL ${episodeCondition}
|
||||
GROUP BY files."${idName}"
|
||||
ORDER BY ${sortCondition} DESC
|
||||
LIMIT 5000`
|
||||
const results = await database.query(query, { type: QueryTypes.SELECT });
|
||||
return results.map(result => `${result.imdbId || result.kitsuId}`);
|
||||
}
|
||||
3145
catalogs/package-lock.json
generated
3145
catalogs/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,25 +0,0 @@
|
|||
{
|
||||
"name": "stremio-torrentio-catalogs",
|
||||
"version": "1.0.3",
|
||||
"exports": "./index.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node index.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16.x"
|
||||
},
|
||||
"author": "TheBeastLT <pauliox@beyond.lt>",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@keyv/mongo": "^3.0.1",
|
||||
"axios": "^1.8.4",
|
||||
"bottleneck": "^2.19.5",
|
||||
"moment": "^2.30.1",
|
||||
"pg": "^8.14.1",
|
||||
"pg-hstore": "^2.3.4",
|
||||
"request-ip": "^3.3.0",
|
||||
"sequelize": "^6.37.7",
|
||||
"stremio-addon-sdk": "^1.6.10"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
import getRouter from 'stremio-addon-sdk/src/getRouter.js';
|
||||
import addonInterface from './addon.js';
|
||||
import qs from 'querystring';
|
||||
import { parseConfiguration } from '../addon/lib/configuration.js';
|
||||
import { createManifest } from './lib/manifest.js';
|
||||
|
||||
const router = getRouter(addonInterface);
|
||||
|
||||
// router.get('/', (_, res) => {
|
||||
// res.redirect('/configure')
|
||||
// res.end();
|
||||
// });
|
||||
//
|
||||
// router.get('/:configuration?/configure', (req, res) => {
|
||||
// const configValues = parseConfiguration(req.params.configuration || '');
|
||||
// const landingHTML = landingTemplate(createManifest(configValues), configValues);
|
||||
// res.setHeader('content-type', 'text/html');
|
||||
// res.end(landingHTML);
|
||||
// });
|
||||
|
||||
router.get('/:configuration?/manifest.json', (req, res) => {
|
||||
const configValues = parseConfiguration(req.params.configuration || '');
|
||||
const manifestBuf = JSON.stringify(createManifest(configValues));
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.end(manifestBuf)
|
||||
});
|
||||
|
||||
router.get('/:configuration/:resource/:type/:id/:extra?.json', (req, res, next) => {
|
||||
const { configuration, resource, type, id } = req.params;
|
||||
const extra = req.params.extra ? qs.parse(req.url.split('/').pop().slice(0, -5)) : {}
|
||||
const configValues = { ...extra, ...parseConfiguration(configuration) };
|
||||
addonInterface.get(resource, type, id, configValues)
|
||||
.then(resp => {
|
||||
const cacheHeaders = {
|
||||
cacheMaxAge: 'max-age',
|
||||
staleRevalidate: 'stale-while-revalidate',
|
||||
staleError: 'stale-if-error'
|
||||
};
|
||||
const cacheControl = Object.keys(cacheHeaders)
|
||||
.map(prop => Number.isInteger(resp[prop]) && cacheHeaders[prop] + '=' + resp[prop])
|
||||
.filter(val => !!val).join(', ');
|
||||
|
||||
res.setHeader('Cache-Control', `${cacheControl}, public`);
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.end(JSON.stringify(resp));
|
||||
})
|
||||
.catch(err => {
|
||||
if (err.noHandler) {
|
||||
if (next) {
|
||||
next()
|
||||
} else {
|
||||
res.writeHead(404);
|
||||
res.end(JSON.stringify({ err: 'not found' }));
|
||||
}
|
||||
} else {
|
||||
console.error(err);
|
||||
res.writeHead(500);
|
||||
res.end(JSON.stringify({ err: 'handler error' }));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
export default function (req, res) {
|
||||
router(req, res, function () {
|
||||
res.statusCode = 404;
|
||||
res.end();
|
||||
});
|
||||
};
|
||||
Loading…
Reference in a new issue