Creepso/torrentio-scraper
1
0
Fork 0
mirror of https://github.com/TheBeastLT/torrentio-scraper.git synced 2026-03-11 17:15:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[scraper] updates rarbg dum scraper with better logging

Browse source
This commit is contained in:
TheBeastLT 2020-03-13 09:48:01 +01:00
parent bcd3715612
commit 972d4b8a43
1 changed files with 10 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
scraper/scrapers/rarbg/rarbg_dump_scraper.js
View file

@ -20,8 +20,8 @@ async function scrape() {
const seriesImdbIds = require('./rargb_series_imdb_ids_2020-03-09.json');
const allImdbIds = [].concat(movieImdbIds).concat(seriesImdbIds);
return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId))
.then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t)))))))
return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId)
.then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t))))))))
.then(() => console.log(`[${moment()}] finished ${NAME} dump scrape`));
}
@ -38,7 +38,14 @@ async function getTorrentsForImdbId(imdbId) {
uploadDate: new Date(torrent.pubdate),
imdbId: torrent.episode_info && torrent.episode_info.imdb
})))
.catch((err) => []);
.then(torrents => {
console.log(`Completed ${imdbId} request`);
return torrents;
})
.catch(error => {
console.warn(`Failed ${NAME} request for ${imdbId}: `, error);
return [];
});
}
async function processTorrentRecord(record) {