Creepso/pstreams-providers
1
0
Fork 0
mirror of https://github.com/p-stream/providers.git synced 2026-01-11 20:10:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #46 from vlOd2/experimental

Browse source 
adds support for native and node-fetch fetchers to use the HTTPS_PROXY env variable to make requests

This only applies to the dev-cli and it helps aid debugging, it also automatically disables TLS verification only when said proxy is specified
This commit is contained in:
Pas 2025-12-25 14:25:20 -07:00 committed by GitHub
commit 8469150ab9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 29 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -70,6 +70,7 @@
"eslint-import-resolver-typescript": "^3.10.1",
"eslint-plugin-import": "^2.32.0",
"eslint-plugin-prettier": "^5.5.4",
"https-proxy-agent": "^7.0.6",
"node-fetch": "^3.3.2",
"prettier": "^3.6.2",
"puppeteer": "^22.15.0",
@ -79,6 +80,7 @@
"tsc-alias": "^1.8.16",
"tsconfig-paths": "^4.2.0",
"typescript": "^5.9.3",
"undici": "^7.16.0",
"vite": "^5.4.21",
"vite-node": "^1.6.1",
"vite-plugin-dts": "^3.9.1",

12
pnpm-lock.yaml
View file

@ -108,6 +108,9 @@ importers:
eslint-plugin-prettier:
specifier: ^5.5.4
version: 5.5.4(@types/eslint@9.6.1)(eslint-config-prettier@9.1.2(eslint@8.57.1))(eslint@8.57.1)(prettier@3.6.2)
https-proxy-agent:
specifier: ^7.0.6
version: 7.0.6
prettier:
specifier: ^3.6.2
version: 3.6.2
@ -132,6 +135,9 @@ importers:
typescript:
specifier: ^5.9.3
version: 5.9.3
undici:
specifier: ^7.16.0
version: 7.16.0
vite:
specifier: ^5.4.21
version: 5.4.21(@types/node@24.9.1)(terser@5.44.0)
@ -2955,6 +2961,10 @@ packages:
undici-types@7.16.0:
resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==}
undici@7.16.0:
resolution: {integrity: sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==}
engines: {node: '>=20.18.1'}
universalify@0.1.2:
resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==}
engines: {node: '>= 4.0.0'}
@ -6313,6 +6323,8 @@ snapshots:
undici-types@7.16.0: {}
undici@7.16.0: {}
universalify@0.1.2: {}
unpacker@1.0.1: {}

9
src/dev-cli/index.ts
View file

@ -3,6 +3,7 @@
import { program } from 'commander';
import dotenv from 'dotenv';
import { prompt } from 'enquirer';
import { ProxyAgent, setGlobalDispatcher } from 'undici';
import { runRankManager } from '@/dev-cli/rank';
import { runScraper } from '@/dev-cli/scraper';
@ -196,6 +197,14 @@ async function runCommandLine() {
await runScraper(providerOptions, validatedSource, validatedOps);
}
// this takes care of native fetch
// node-fetch needs to be handled separately
if (process.env.HTTPS_PROXY) {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
console.warn(`Using HTTPS proxy at ${process.env.HTTPS_PROXY}`);
setGlobalDispatcher(new ProxyAgent({ uri: new URL(process.env.HTTPS_PROXY).toString() }));
}
if (process.argv.length === 2) {
runQuestions()
.catch(() => console.error('Exited.'))

6
src/dev-cli/validate.ts
View file

@ -1,3 +1,4 @@
import { HttpsProxyAgent } from 'https-proxy-agent';
import nodeFetch from 'node-fetch';
import { Embed, Sourcerer } from '@/providers/base';
@ -74,6 +75,11 @@ export async function processOptions(sources: Array<Embed | Sourcerer>, options:
if (options.fetcher === 'native') {
fetcher = makeStandardFetcher(fetch);
} else if (process.env.HTTPS_PROXY) {
fetcher = makeStandardFetcher(async (url, ops) => {
const proxyAgent = new HttpsProxyAgent(process.env.HTTPS_PROXY!);
return nodeFetch(url, { ...ops, agent: proxyAgent });
});
} else {
fetcher = makeStandardFetcher(nodeFetch);
}