diff --git a/crunchy.ts b/crunchy.ts index 249378b..370e445 100644 --- a/crunchy.ts +++ b/crunchy.ts @@ -25,7 +25,7 @@ import getKeys, { canDecrypt } from './modules/widevine'; // load req import { domain, api } from './modules/module.api-urls'; -import * as reqModule from './modules/module.req'; +import * as reqModule from './modules/module.fetch'; import { CrunchySearch } from './@types/crunchySearch'; import { CrunchyEpisodeList, CrunchyEpisode } from './@types/crunchyEpisodeList'; import { CrunchyDownloadOptions, CrunchyEpMeta, CrunchyMuxOptions, CrunchyMultiDownload, DownloadedMedia, ParseItem, SeriesSearch, SeriesSearchItem } from './@types/crunchyTypes'; @@ -210,9 +210,9 @@ export default class Crunchy implements ServiceClass { console.info(''); } const fontUrl = fontsData.root + f; - const getFont = await this.req.getData(fontUrl, { binary: true }); + const getFont = await this.req.getData(fontUrl); if(getFont.ok && getFont.res){ - fs.writeFileSync(fontLoc, getFont.res.body); + fs.writeFileSync(fontLoc, Buffer.from(await getFont.res.arrayBuffer())); console.info(`Downloaded: ${f}`); } else{ @@ -240,7 +240,7 @@ export default class Crunchy implements ServiceClass { console.error('Authentication failed!'); return { isOk: false, reason: new Error('Authentication failed') }; } - this.token = JSON.parse(authReq.res.body); + this.token = await authReq.res.json(); this.token.expires = new Date(Date.now() + this.token.expires_in); yamlCfg.saveCRToken(this.token); await this.getProfile(); @@ -263,7 +263,7 @@ export default class Crunchy implements ServiceClass { console.error('Authentication failed!'); return; } - this.token = JSON.parse(authReq.res.body); + this.token = await authReq.res.json(); this.token.expires = new Date(Date.now() + this.token.expires_in); yamlCfg.saveCRToken(this.token); } @@ -284,7 +284,7 @@ export default class Crunchy implements ServiceClass { console.error('Get profile failed!'); return false; } - const profile = JSON.parse(profileReq.res.body); + const profile = await profileReq.res.json(); if (!silent) { console.info('USER: %s (%s)', profile.username, profile.email); } @@ -313,7 +313,7 @@ export default class Crunchy implements ServiceClass { console.error('Token Authentication failed!'); return; } - this.token = JSON.parse(authReq.res.body); + this.token = await authReq.res.json(); this.token.expires = new Date(Date.now() + this.token.expires_in); yamlCfg.saveCRToken(this.token); await this.getProfile(false); @@ -347,7 +347,7 @@ export default class Crunchy implements ServiceClass { console.error('Authentication failed!'); return; } - this.token = JSON.parse(authReq.res.body); + this.token = await authReq.res.json(); this.token.expires = new Date(Date.now() + this.token.expires_in); yamlCfg.saveCRToken(this.token); } @@ -382,7 +382,7 @@ export default class Crunchy implements ServiceClass { console.error('Authentication CMS token failed!'); return; } - this.cmsToken = JSON.parse(cmsTokenReq.res.body); + this.cmsToken = await cmsTokenReq.res.json(); console.info('Your Country: %s\n', this.cmsToken.cms?.bucket.split('/')[1]); } @@ -411,7 +411,7 @@ export default class Crunchy implements ServiceClass { console.error('Get CMS index FAILED!'); return; } - console.info(JSON.parse(indexReq.res.body)); + console.info(await indexReq.res.json()); } public async doSearch(data: SearchData): Promise{ @@ -438,7 +438,7 @@ export default class Crunchy implements ServiceClass { console.error('Search FAILED!'); return { isOk: false, reason: new Error('Search failed. No more information provided') }; } - const searchResults = JSON.parse(searchReq.res.body) as CrunchySearch; + const searchResults = await searchReq.res.json() as CrunchySearch; if(searchResults.total < 1){ console.info('Nothing Found!'); return { isOk: true, value: [] }; @@ -699,7 +699,7 @@ export default class Crunchy implements ServiceClass { console.error('Series Request FAILED!'); return; } - const seriesData = JSON.parse(seriesReq.res.body); + const seriesData = await seriesReq.res.json(); await this.logObject(seriesData.data[0], pad, false); } // seasons list @@ -709,7 +709,7 @@ export default class Crunchy implements ServiceClass { return; } // parse data - const seasonsList = JSON.parse(seriesSeasonListReq.res.body) as SeriesSearch; + const seasonsList = await seriesSeasonListReq.res.json() as SeriesSearch; if(seasonsList.total < 1){ console.info('Series is empty!'); return; @@ -740,7 +740,7 @@ export default class Crunchy implements ServiceClass { console.error('Movie Listing Request FAILED!'); return; } - const movieListing = JSON.parse(movieListingReq.res.body); + const movieListing = await movieListingReq.res.json(); if(movieListing.total < 1){ console.info('Movie Listing is empty!'); return; @@ -755,7 +755,7 @@ export default class Crunchy implements ServiceClass { console.error('Movies List Request FAILED!'); return; } - const moviesList = JSON.parse(moviesListReq.res.body); + const moviesList = await moviesListReq.res.json(); for(const item of moviesList.data){ this.logObject(item, pad+2); } @@ -782,7 +782,7 @@ export default class Crunchy implements ServiceClass { console.error('Get newly added FAILED!'); return; } - const newlyAddedResults = JSON.parse(newlyAddedReq.res.body); + const newlyAddedResults = await newlyAddedReq.res.json(); console.info('Newly added:'); for(const i of newlyAddedResults.items){ await this.logObject(i, 2); @@ -814,7 +814,7 @@ export default class Crunchy implements ServiceClass { console.error('Show Request FAILED!'); return { isOk: false, reason: new Error('Show request failed. No more information provided.') }; } - const showInfo = JSON.parse(showInfoReq.res.body); + const showInfo = await showInfoReq.res.json(); this.logObject(showInfo.data[0], 0); let episodeList = { total: 0, data: [], meta: {} } as CrunchyEpisodeList; @@ -840,7 +840,7 @@ export default class Crunchy implements ServiceClass { return { isOk: false, reason: new Error('Episode List request failed. No more information provided.') }; } //CrunchyEpisodeList - const episodeListAndroid = JSON.parse(reqEpsList.res.body) as CrunchyAndroidEpisodes; + const episodeListAndroid = await reqEpsList.res.json() as CrunchyAndroidEpisodes; episodeList = { total: episodeListAndroid.total, data: episodeListAndroid.items, @@ -853,7 +853,7 @@ export default class Crunchy implements ServiceClass { return { isOk: false, reason: new Error('Episode List request failed. No more information provided.') }; } //CrunchyEpisodeList - episodeList = JSON.parse(reqEpsList.res.body) as CrunchyEpisodeList; + episodeList = await reqEpsList.res.json() as CrunchyEpisodeList; } const epNumList: { @@ -1014,7 +1014,7 @@ export default class Crunchy implements ServiceClass { continue; } - const oldObjectInfo = JSON.parse(extIdReq.res.body) as Record; + const oldObjectInfo = await extIdReq.res.json() as Record; for (const object of oldObjectInfo.items) { objectIds.push(object.id); } @@ -1061,14 +1061,14 @@ export default class Crunchy implements ServiceClass { if(!objectReq.ok || !objectReq.res){ console.error('Objects Request FAILED!'); if(objectReq.error && objectReq.error.res && objectReq.error.res.body){ - const objectInfo = JSON.parse(objectReq.error.res.body as string); + const objectInfo = await objectReq.error.res.json(); console.info('Body:', JSON.stringify(objectInfo, null, '\t')); objectInfo.error = true; return objectInfo; } return []; } - const objectInfoAndroid = JSON.parse(objectReq.res.body) as CrunchyAndroidObject; + const objectInfoAndroid = await objectReq.res.json() as CrunchyAndroidObject; objectInfo = { total: objectInfoAndroid.total, data: objectInfoAndroid.items, @@ -1079,14 +1079,14 @@ export default class Crunchy implements ServiceClass { if(!objectReq.ok || !objectReq.res){ console.error('Objects Request FAILED!'); if(objectReq.error && objectReq.error.res && objectReq.error.res.body){ - const objectInfo = JSON.parse(objectReq.error.res.body as string); + const objectInfo = await objectReq.error.res.json(); console.info('Body:', JSON.stringify(objectInfo, null, '\t')); objectInfo.error = true; return objectInfo; } return []; } - objectInfo = JSON.parse(objectReq.res.body) as ObjectInfo; + objectInfo = await objectReq.res.json() as ObjectInfo; } if(earlyReturn){ @@ -1248,7 +1248,7 @@ export default class Crunchy implements ServiceClass { console.warn('Old Chapter API request failed'); } else { console.info('Old Chapter request successful'); - const chapterData = JSON.parse(oldChapterRequest.res.body) as CrunchyOldChapter; + const chapterData = await oldChapterRequest.res.json() as CrunchyOldChapter; //Generate Timestamps const startTime = new Date(0), endTime = new Date(0); @@ -1278,7 +1278,7 @@ export default class Crunchy implements ServiceClass { } else { //Chapter request succeeded, now let's parse them console.info('Chapter request successful'); - const chapterData = JSON.parse(chapterRequest.res.body) as CrunchyChapters; + const chapterData = await chapterRequest.res.json() as CrunchyChapters; const chapters: CrunchyChapter[] = []; //Make a format more usable for the crunchy chapters @@ -1378,7 +1378,7 @@ export default class Crunchy implements ServiceClass { return undefined; } } - const pbDataAndroid = JSON.parse(playbackReq.res.body) as CrunchyAndroidStreams; + const pbDataAndroid = await playbackReq.res.json() as CrunchyAndroidStreams; pbData = { total: 0, data: [pbDataAndroid.streams], @@ -1402,14 +1402,14 @@ export default class Crunchy implements ServiceClass { return undefined; } } - pbData = JSON.parse(playbackReq.res.body) as PlaybackData; + pbData = await playbackReq.res.json() as PlaybackData; } const playbackReq = await this.req.getData(`https://cr-play-service.prd.crunchyrollsvc.com/v1/${currentVersion ? currentVersion.guid : mMeta.mediaId}/console/switch/play`, AuthHeaders); if(!playbackReq.ok || !playbackReq.res){ console.error('Non-DRM Request Stream URLs FAILED!'); } else { - const playStream = JSON.parse(playbackReq.res.body) as CrunchyPlayStream; + const playStream = await playbackReq.res.json() as CrunchyPlayStream; const derivedPlaystreams = {} as CrunchyStreams; for (const hardsub in playStream.hardSubs) { const stream = playStream.hardSubs[hardsub]; @@ -1557,9 +1557,10 @@ export default class Crunchy implements ServiceClass { console.error('CAN\'T FETCH VIDEO PLAYLISTS!'); dlFailed = true; } else { - if (streamPlaylistsReq.res.body.match('MPD')) { + const streamPlaylistBody = await streamPlaylistsReq.res.text(); + if (streamPlaylistBody.match('MPD')) { //Parse MPD Playlists - const streamPlaylists = parse(streamPlaylistsReq.res.body, langsData.findLang(langsData.fixLanguageTag(pbData.meta.audio_locale as string) || ''), curStream.url.match(/.*\.urlset\//)[0]); + const streamPlaylists = parse(streamPlaylistBody, langsData.findLang(langsData.fixLanguageTag(pbData.meta.audio_locale as string) || ''), curStream.url.match(/.*\.urlset\//)[0]); //Get name of CDNs/Servers const streamServers = Object.keys(streamPlaylists); @@ -1744,10 +1745,10 @@ export default class Crunchy implements ServiceClass { }) }); if(!decReq.ok || !decReq.res){ - console.error('Request to DRM Authentication failed:', decReq.error?.code, decReq.error?.message); + console.error('Request to DRM Authentication failed:', decReq.error?.res.status, decReq.error?.message); return undefined; } - const authData = JSON.parse(decReq.res.body) as {'custom_data': string, 'token': string}; + const authData = await decReq.res.json() as {'custom_data': string, 'token': string}; const encryptionKeys = await getKeys(chosenVideoSegments.pssh, 'https://lic.drmtoday.com/license-proxy-widevine/cenc/', { 'dt-custom-data': authData.custom_data, 'x-dt-auth-token': authData.token @@ -1829,7 +1830,7 @@ export default class Crunchy implements ServiceClass { } } } else if (!options.novids) { - const streamPlaylists = m3u8(streamPlaylistsReq.res.body); + const streamPlaylists = m3u8(streamPlaylistBody); const plServerList: string[] = [], plStreams: Record> = {}, plQuality: { @@ -1936,7 +1937,8 @@ export default class Crunchy implements ServiceClass { console.error('CAN\'T FETCH VIDEO PLAYLIST!'); dlFailed = true; } else { - const chunkPlaylist = m3u8(chunkPage.res.body); + const chunkPageBody = await chunkPage.res.text(); + const chunkPlaylist = m3u8(chunkPageBody); const totalParts = chunkPlaylist.segments.length; const mathParts = Math.ceil(totalParts / options.partsize); const mathMsg = `(${mathParts}*${options.partsize})`; @@ -2076,15 +2078,15 @@ export default class Crunchy implements ServiceClass { if(options.dlsubs.includes('all') || options.dlsubs.includes(langItem.locale)){ const subsAssReq = await this.req.getData(subsItem.url); if(subsAssReq.ok && subsAssReq.res){ - let sBody; + let sBody = await subsAssReq.res.text(); if (subsItem.format == 'vtt') { const chosenFontSize = options.originalFontSize ? undefined : options.fontSize; - if (!options.originalFontSize) subsAssReq.res.body = subsAssReq.res.body.replace(/( font-size:.+?;)/g, '').replace(/(font-size:.+?;)/g, ''); - sBody = vtt2ass(undefined, chosenFontSize, subsAssReq.res.body, '', undefined, options.fontName); + if (!options.originalFontSize) sBody = sBody.replace(/( font-size:.+?;)/g, '').replace(/(font-size:.+?;)/g, ''); + sBody = vtt2ass(undefined, chosenFontSize, sBody, '', undefined, options.fontName); sxData.fonts = fontsData.assFonts(sBody) as Font[]; sxData.file = sxData.file.replace('.vtt','.ass'); } else { - sBody = '\ufeff' + subsAssReq.res.body; + sBody = '\ufeff' + sBody; const sBodySplit = sBody.split('\r\n'); sBodySplit.splice(2, 0, 'ScaledBorderAndShadow: yes'); sBody = sBodySplit.join('\r\n'); @@ -2475,7 +2477,7 @@ export default class Crunchy implements ServiceClass { return; } // parse data - const seasonsList = JSON.parse(seriesSeasonListReq.res.body) as SeriesSearch; + const seasonsList = await seriesSeasonListReq.res.json() as SeriesSearch; if(seasonsList.total < 1){ console.info('Series is empty!'); return; @@ -2502,7 +2504,7 @@ export default class Crunchy implements ServiceClass { console.error('Show Request FAILED!'); return; } - const showInfo = JSON.parse(showInfoReq.res.body); + const showInfo = await showInfoReq.res.json(); if (log) this.logObject(showInfo, 0); @@ -2529,7 +2531,7 @@ export default class Crunchy implements ServiceClass { return; } //CrunchyEpisodeList - const episodeListAndroid = JSON.parse(reqEpsList.res.body) as CrunchyAndroidEpisodes; + const episodeListAndroid = await reqEpsList.res.json() as CrunchyAndroidEpisodes; episodeList = { total: episodeListAndroid.total, data: episodeListAndroid.items, @@ -2542,7 +2544,7 @@ export default class Crunchy implements ServiceClass { return; } //CrunchyEpisodeList - episodeList = JSON.parse(reqEpsList.res.body) as CrunchyEpisodeList; + episodeList = await reqEpsList.res.json() as CrunchyEpisodeList; } if(episodeList.total < 1){ diff --git a/modules/module.api-urls.ts b/modules/module.api-urls.ts index 6dffd45..ad6b88a 100644 --- a/modules/module.api-urls.ts +++ b/modules/module.api-urls.ts @@ -1,5 +1,3 @@ -import { Headers } from 'got/dist/source'; - // api domains const domain = { www: 'https://www.crunchyroll.com', @@ -34,8 +32,8 @@ export type APIType = { cms: string beta_browse: string beta_cms: string, - beta_authHeader: Headers, - beta_authHeaderMob: Headers, + beta_authHeader: Record, + beta_authHeaderMob: Record, hd_apikey: string, hd_devName: string, hd_appId: string, diff --git a/modules/module.fetch.ts b/modules/module.fetch.ts new file mode 100644 index 0000000..7e79b53 --- /dev/null +++ b/modules/module.fetch.ts @@ -0,0 +1,144 @@ +import * as yamlCfg from './module.cfg-loader'; +import { console } from './log'; +import { Method } from 'got'; + +export type Params = { + method?: Method, + headers?: Record, + body?: string | Buffer, + binary?: boolean, + followRedirect?: 'follow' | 'error' | 'manual' +} + +// req +export class Req { + private sessCfg: string; + private service: 'cr'|'funi'|'hd'; + private session: Record = {}; + private cfgDir = yamlCfg.cfgDir; + private curl: boolean|string = false; + + constructor(private domain: Record, private debug: boolean, private nosess = false, private type: 'cr'|'funi'|'hd') { + this.sessCfg = yamlCfg.sessCfgFile[type]; + this.service = type; + } + + async getData(durl: string, params?: Params) { + params = params || {}; + // options + const options: RequestInit = { + method: params.method ? params.method : 'GET', + headers: { + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'accept-language': 'en-US,en;q=0.9', + 'cache-control': 'no-cache', + 'pragma': 'no-cache', + 'sec-ch-ua': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'document', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-user': '?1', + 'upgrade-insecure-requests': '1', + }, + }; + // additional params + if(params.headers){ + options.headers = {...options.headers, ...params.headers}; + } + if(options.method == 'POST'){ + if (!(options.headers as Record)['Content-Type']) { + (options.headers as Record)['Content-Type'] = 'application/x-www-form-urlencoded'; + } + } + if(params.body){ + options.body = params.body; + } + if(typeof params.followRedirect == 'string'){ + options.redirect = params.followRedirect; + } + // debug + if(this.debug){ + console.debug('[DEBUG] GOT OPTIONS:'); + console.debug(options); + } + // try do request + try { + const res = await fetch(durl.toString(), options); + if (!res.ok) { + console.error(`${res.status}: ${res.statusText}`); + const body = await res.text(); + const docTitle = body.match(/(.*)<\/title>/); + if(body && docTitle){ + console.error(docTitle[1]); + } + } + return { + ok: res.ok, + res + }; + } + catch(_error){ + const error = _error as { + name: string + } & TypeError & { + res: Response + }; + if (error.res && error.res.status && error.res.statusText) { + console.error(`${error.name} ${error.res.status}: ${error.res.statusText}`); + } else { + console.error(`${error.name}: ${error.res?.statusText || error.message}`); + } + if(error.res) { + const body = await error.res.text(); + const docTitle = body.match(/<title>(.*)<\/title>/); + if(body && docTitle){ + console.error(docTitle[1]); + } + } + return { + ok: false, + error, + }; + } + } +} + +export function buildProxy(proxyBaseUrl: string, proxyAuth: string){ + if(!proxyBaseUrl.match(/^(https?|socks4|socks5):/)){ + proxyBaseUrl = 'http://' + proxyBaseUrl; + } + + const proxyCfg = new URL(proxyBaseUrl); + let proxyStr = `${proxyCfg.protocol}//`; + + if(typeof proxyCfg.hostname != 'string' || proxyCfg.hostname == ''){ + throw new Error('[ERROR] Hostname and port required for proxy!'); + } + + if(proxyAuth && typeof proxyAuth == 'string' && proxyAuth.match(':')){ + proxyCfg.username = proxyAuth.split(':')[0]; + proxyCfg.password = proxyAuth.split(':')[1]; + proxyStr += `${proxyCfg.username}:${proxyCfg.password}@`; + } + + proxyStr += proxyCfg.hostname; + + if(!proxyCfg.port && proxyCfg.protocol == 'http:'){ + proxyStr += ':80'; + } + else if(!proxyCfg.port && proxyCfg.protocol == 'https:'){ + proxyStr += ':443'; + } + + return proxyStr; +} + \ No newline at end of file