diff --git a/package-lock.json b/package-lock.json index ebf8e29..4415b2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1058,9 +1058,9 @@ "integrity": "sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=" }, "lodash.merge": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.1.tgz", - "integrity": "sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ==" + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" }, "lodash.pick": { "version": "4.4.0", @@ -1692,14 +1692,6 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" }, - "rarbg": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/rarbg/-/rarbg-1.3.0.tgz", - "integrity": "sha512-otXBLZYk+02TQkPnxdfHlx5rzbzNZ+9dcZGkDeNKH+cLVw16yO4CCJkc9h60cGSlMlhHqdt33y9UEZc9hk0Uiw==", - "requires": { - "moment": "^2.22.2" - } - }, "rarbg-api": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/rarbg-api/-/rarbg-api-1.1.3.tgz", diff --git a/scraper/index.js b/scraper/index.js index 75091dc..7b17e05 100644 --- a/scraper/index.js +++ b/scraper/index.js @@ -13,7 +13,7 @@ const thepiratebayDumpScraper = require('./scrapers/thepiratebay/thepiratebay_du const thepiratebayUnofficialDumpScraper = require('./scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper'); const PROVIDERS = [ - // horribleSubsScraper, + horribleSubsScraper, rarbgScraper, thepiratebayScraper, kickassScraper, diff --git a/scraper/lib/repository.js b/scraper/lib/repository.js index 24546a0..3cccb62 100644 --- a/scraper/lib/repository.js +++ b/scraper/lib/repository.js @@ -47,6 +47,7 @@ const File = database.define('file', }, { indexes: [ + { unique: true, fields: ['infoHash'], where: { fileIndex: { [Op.eq]: null } } }, { unique: true, fields: ['infoHash', 'fileIndex', 'imdbSeason', 'imdbEpisode'] }, { unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode'] }, { unique: false, fields: ['kitsuId', 'kitsuEpisode'] } diff --git a/scraper/scrapers/rarbg/rarbg_dump_scraper.js b/scraper/scrapers/rarbg/rarbg_dump_scraper.js index 8428021..5bb3ed9 100644 --- a/scraper/scrapers/rarbg/rarbg_dump_scraper.js +++ b/scraper/scrapers/rarbg/rarbg_dump_scraper.js @@ -20,8 +20,8 @@ async function scrape() { const seriesImdbIds = require('./rargb_series_imdb_ids_2020-03-09.json'); const allImdbIds = [].concat(movieImdbIds).concat(seriesImdbIds); - return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId) - .then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t)))))))) + return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId)) + .then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t))))))) .then(() => console.log(`[${moment()}] finished ${NAME} dump scrape`)); } @@ -38,14 +38,7 @@ async function getTorrentsForImdbId(imdbId) { uploadDate: new Date(torrent.pubdate), imdbId: torrent.episode_info && torrent.episode_info.imdb }))) - .then(torrents => { - console.log(`Completed ${imdbId} request`); - return torrents; - }) - .catch(error => { - console.warn(`Failed ${NAME} request for ${imdbId}: `, error); - return []; - }); + .catch((err) => []); } async function processTorrentRecord(record) {