import * as yamlCfg from './module.cfg-loader'; import * as yargs from './module.app-args'; import { console } from './log'; import { connect } from 'puppeteer-real-browser'; import { argvC } from './module.app-args'; import { ProxyAgent, fetch, RequestInit } from 'undici'; export type FetchParams = Partial; export type Params = { method?: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE'; headers?: Record; body?: BodyInit | undefined; binary?: boolean; followRedirect?: 'follow' | 'error' | 'manual'; }; type CustomParams = { useProxy: boolean; }; type GetDataResponse = { ok: boolean; res?: Response; headers?: Record; error?: { name: string; } & TypeError & { res?: Response; }; }; function hasDisplay(): boolean { if (process.platform === 'linux') { return !!process.env.DISPLAY || !!process.env.WAYLAND_DISPLAY; } // Win and Mac true by default return true; } // req export class Req { private debug: boolean; public argv: typeof argvC; constructor() { const cfg = yamlCfg.loadCfg(); this.argv = yargs.appArgv(cfg.cli, process.env.isGUI ? true : false); this.debug = this.argv.debug ?? false; } async getData(durl: string, params: Partial = {}): Promise { const options: RequestInit = { method: params.method ? params.method : 'GET' }; if (params.headers) { options.headers = params.headers; } if (params.body) { options.body = params.body; } if (typeof params.redirect == 'string') { options.redirect = params.redirect; } // Proxy Handler let dispatcher: ProxyAgent | undefined; const validProxy = this.argv.proxy ? this.isValidProxyUrl(this.argv.proxy) : false; if ((params.useProxy || this.argv.proxyAll) && this.argv.proxy && validProxy) { dispatcher = new ProxyAgent(this.argv.proxy); } else if ((params.useProxy || this.argv.proxyAll) && this.argv.proxy && !validProxy) { console.warn('[Fetch] Provided invalid Proxy URL, not proxying traffic.'); } // Debug if (this.debug) { console.debug('[DEBUG] FETCH OPTIONS:'); console.debug(options); } try { const res = await fetch(durl, { ...options, dispatcher: dispatcher }); if (!res.ok) { console.error(`${res.status}: ${res.statusText}`); const body = await res.text(); const docTitle = body.match(/(.*)<\/title>/); if (body && docTitle) { if (docTitle[1] === 'Just a moment...' && durl.includes('crunchyroll') && hasDisplay()) { console.warn('Cloudflare triggered, trying to get cookies...'); const { page } = await connect({ headless: false, turnstile: true }); await page.goto('https://www.crunchyroll.com/', { waitUntil: 'networkidle2' }); await page.waitForRequest('https://www.crunchyroll.com/auth/v1/token'); const cookies = await page.cookies(); await page.close(); params.headers = { ...params.headers, Cookie: cookies.map((c) => `${c.name}=${c.value}`).join('; '), 'Set-Cookie': cookies.map((c) => `${c.name}=${c.value}`).join('; ') }; (params as any).headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36'; return await this.getData(durl, params); } else { console.error(docTitle[1]); } } else { console.error(body); } } return { ok: res.ok, res: res as any, headers: params.headers as Record<string, string> }; } catch (_error) { const error = _error as { name: string; } & TypeError & { res: Response; }; if (error.res && error.res.status && error.res.statusText) { console.error(`${error.name} ${error.res.status}: ${error.res.statusText}`); } else { console.error(`${error.name}: ${error.res?.statusText || error.message}`); } if (error.res) { const body = await error.res.text(); const docTitle = body.match(/<title>(.*)<\/title>/); if (body && docTitle) { console.error(docTitle[1]); } } return { ok: false, error }; } } private isValidProxyUrl(proxyUrl: string): boolean { try { if (!proxyUrl.match(/^(https?|socks4|socks5):\/\//)) { return false; } const url = new URL(proxyUrl); if (!url.hostname) return false; if (!['http:', 'https:'].includes(url.protocol)) { return false; } if (url.port && (!/^\d+$/.test(url.port) || Number(url.port) < 1 || Number(url.port) > 65535)) { return false; } return true; } catch { return false; } } }