diff --git a/package.json b/package.json index 1fafa9a..91c49b1 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "eslint-import-resolver-typescript": "^3.10.1", "eslint-plugin-import": "^2.32.0", "eslint-plugin-prettier": "^5.5.4", + "https-proxy-agent": "^7.0.6", "node-fetch": "^3.3.2", "prettier": "^3.6.2", "puppeteer": "^22.15.0", @@ -79,6 +80,7 @@ "tsc-alias": "^1.8.16", "tsconfig-paths": "^4.2.0", "typescript": "^5.9.3", + "undici": "^7.16.0", "vite": "^5.4.21", "vite-node": "^1.6.1", "vite-plugin-dts": "^3.9.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 744f829..6ee11ee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -108,6 +108,9 @@ importers: eslint-plugin-prettier: specifier: ^5.5.4 version: 5.5.4(@types/eslint@9.6.1)(eslint-config-prettier@9.1.2(eslint@8.57.1))(eslint@8.57.1)(prettier@3.6.2) + https-proxy-agent: + specifier: ^7.0.6 + version: 7.0.6 prettier: specifier: ^3.6.2 version: 3.6.2 @@ -132,6 +135,9 @@ importers: typescript: specifier: ^5.9.3 version: 5.9.3 + undici: + specifier: ^7.16.0 + version: 7.16.0 vite: specifier: ^5.4.21 version: 5.4.21(@types/node@24.9.1)(terser@5.44.0) @@ -2955,6 +2961,10 @@ packages: undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + undici@7.16.0: + resolution: {integrity: sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==} + engines: {node: '>=20.18.1'} + universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} @@ -6313,6 +6323,8 @@ snapshots: undici-types@7.16.0: {} + undici@7.16.0: {} + universalify@0.1.2: {} unpacker@1.0.1: {} diff --git a/src/dev-cli/index.ts b/src/dev-cli/index.ts index 4c344e1..0437b2e 100644 --- a/src/dev-cli/index.ts +++ b/src/dev-cli/index.ts @@ -3,6 +3,7 @@ import { program } from 'commander'; import dotenv from 'dotenv'; import { prompt } from 'enquirer'; +import { ProxyAgent, setGlobalDispatcher } from 'undici'; import { runRankManager } from '@/dev-cli/rank'; import { runScraper } from '@/dev-cli/scraper'; @@ -196,6 +197,14 @@ async function runCommandLine() { await runScraper(providerOptions, validatedSource, validatedOps); } +// this takes care of native fetch +// node-fetch needs to be handled separately +if (process.env.HTTPS_PROXY) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + console.warn(`Using HTTPS proxy at ${process.env.HTTPS_PROXY}`); + setGlobalDispatcher(new ProxyAgent({ uri: new URL(process.env.HTTPS_PROXY).toString() })); +} + if (process.argv.length === 2) { runQuestions() .catch(() => console.error('Exited.')) diff --git a/src/dev-cli/validate.ts b/src/dev-cli/validate.ts index 4455c45..3477ee9 100644 --- a/src/dev-cli/validate.ts +++ b/src/dev-cli/validate.ts @@ -1,3 +1,4 @@ +import { HttpsProxyAgent } from 'https-proxy-agent'; import nodeFetch from 'node-fetch'; import { Embed, Sourcerer } from '@/providers/base'; @@ -74,6 +75,11 @@ export async function processOptions(sources: Array, options: if (options.fetcher === 'native') { fetcher = makeStandardFetcher(fetch); + } else if (process.env.HTTPS_PROXY) { + fetcher = makeStandardFetcher(async (url, ops) => { + const proxyAgent = new HttpsProxyAgent(process.env.HTTPS_PROXY!); + return nodeFetch(url, { ...ops, agent: proxyAgent }); + }); } else { fetcher = makeStandardFetcher(nodeFetch); }