From 78e7eea3cde87e38f8e10c858eada6eeb6a02f34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Mar 2020 07:53:09 +0000 Subject: [PATCH 1/4] Bump lodash.merge from 4.6.1 to 4.6.2 Bumps [lodash.merge](https://github.com/lodash/lodash) from 4.6.1 to 4.6.2. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/commits) Signed-off-by: dependabot[bot] --- package-lock.json | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index ebf8e29..4415b2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1058,9 +1058,9 @@ "integrity": "sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=" }, "lodash.merge": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.1.tgz", - "integrity": "sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ==" + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" }, "lodash.pick": { "version": "4.4.0", @@ -1692,14 +1692,6 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" }, - "rarbg": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/rarbg/-/rarbg-1.3.0.tgz", - "integrity": "sha512-otXBLZYk+02TQkPnxdfHlx5rzbzNZ+9dcZGkDeNKH+cLVw16yO4CCJkc9h60cGSlMlhHqdt33y9UEZc9hk0Uiw==", - "requires": { - "moment": "^2.22.2" - } - }, "rarbg-api": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/rarbg-api/-/rarbg-api-1.1.3.tgz", -- 2.45.2 From 972d4b8a43d89ed9b05dee940b3c4499082c9511 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Fri, 13 Mar 2020 09:48:01 +0100 Subject: [PATCH 2/4] [scraper] updates rarbg dum scraper with better logging --- scraper/scrapers/rarbg/rarbg_dump_scraper.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scraper/scrapers/rarbg/rarbg_dump_scraper.js b/scraper/scrapers/rarbg/rarbg_dump_scraper.js index 5bb3ed9..8428021 100644 --- a/scraper/scrapers/rarbg/rarbg_dump_scraper.js +++ b/scraper/scrapers/rarbg/rarbg_dump_scraper.js @@ -20,8 +20,8 @@ async function scrape() { const seriesImdbIds = require('./rargb_series_imdb_ids_2020-03-09.json'); const allImdbIds = [].concat(movieImdbIds).concat(seriesImdbIds); - return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId)) - .then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t))))))) + return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId) + .then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t)))))))) .then(() => console.log(`[${moment()}] finished ${NAME} dump scrape`)); } @@ -38,7 +38,14 @@ async function getTorrentsForImdbId(imdbId) { uploadDate: new Date(torrent.pubdate), imdbId: torrent.episode_info && torrent.episode_info.imdb }))) - .catch((err) => []); + .then(torrents => { + console.log(`Completed ${imdbId} request`); + return torrents; + }) + .catch(error => { + console.warn(`Failed ${NAME} request for ${imdbId}: `, error); + return []; + }); } async function processTorrentRecord(record) { -- 2.45.2 From 3ec32b718b97ae45f2f0c194c7b5d83ad836d9bf Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Fri, 13 Mar 2020 09:48:22 +0100 Subject: [PATCH 3/4] [scraper] removes unnecessary file index --- scraper/lib/repository.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scraper/lib/repository.js b/scraper/lib/repository.js index 3cccb62..24546a0 100644 --- a/scraper/lib/repository.js +++ b/scraper/lib/repository.js @@ -47,7 +47,6 @@ const File = database.define('file', }, { indexes: [ - { unique: true, fields: ['infoHash'], where: { fileIndex: { [Op.eq]: null } } }, { unique: true, fields: ['infoHash', 'fileIndex', 'imdbSeason', 'imdbEpisode'] }, { unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode'] }, { unique: false, fields: ['kitsuId', 'kitsuEpisode'] } -- 2.45.2 From a2b47d39cf9dbe739cfb1c000b71f05816957098 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Fri, 13 Mar 2020 09:49:01 +0100 Subject: [PATCH 4/4] [scraper] disables horriblesubs scraper --- scraper/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraper/index.js b/scraper/index.js index 7b17e05..75091dc 100644 --- a/scraper/index.js +++ b/scraper/index.js @@ -13,7 +13,7 @@ const thepiratebayDumpScraper = require('./scrapers/thepiratebay/thepiratebay_du const thepiratebayUnofficialDumpScraper = require('./scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper'); const PROVIDERS = [ - horribleSubsScraper, + // horribleSubsScraper, rarbgScraper, thepiratebayScraper, kickassScraper, -- 2.45.2