fetcher seralization

README.md

@@ -19,14 +19,9 @@ Todos:
    - makeStandardFetcher()
      - do all parameters get passed to real fetch as expected?
      - does serialisation work as expected? (formdata + json + string)
-     - do baseUrl settings work?
      - does json responses get automatically parsed?
-   - makeFullUrl()
-     - do slashes get converted correctly?
  - running individual scrapers
  - finish fetchers:
-   - make baseUrl param work
-   - proper serialization (with content-type headers) for standard fetcher
    - automatically parse json
  - error logging for failed scrapers
  - add all real providers

src/fetchers/body.ts

@@ -0,0 +1,24 @@
+import FormData = require('form-data');
+
+import { FetcherOptions } from '@/fetchers/types';
+
+export interface SeralizedBody {
+  headers: Record<string, string>;
+  body: FormData | URLSearchParams | string | undefined;
+}
+
+export function serializeBody(body: FetcherOptions['body']): SeralizedBody {
+  if (body === undefined || typeof body === 'string' || body instanceof URLSearchParams || body instanceof FormData)
+    return {
+      headers: {},
+      body,
+    };
+
+  // serialize as JSON
+  return {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(body),
+  };
+}

src/fetchers/standardFetch.ts

@@ -1,5 +1,6 @@
 import fetch from 'node-fetch';
 
+import { serializeBody } from '@/fetchers/body';
 import { makeFullUrl } from '@/fetchers/common';
 import { Fetcher } from '@/fetchers/types';
 
@@ -7,9 +8,15 @@ export function makeStandardFetcher(f: typeof fetch): Fetcher {
   const normalFetch: Fetcher = (url, ops) => {
     const fullUrl = makeFullUrl(url, ops);
 
+    const seralizedBody = serializeBody(ops.body);
+
     return f(fullUrl, {
       method: ops.method,
-      body: JSON.stringify(ops.body), // TODO content type headers + proper serialization
+      headers: {
+        ...seralizedBody.headers,
+        ...ops.headers,
+      },
+      body: seralizedBody.body,
     });
   };
 

src/fetchers/types.ts

@@ -5,7 +5,7 @@ export type FetcherOptions = {
   headers?: Record<string, string>;
   query?: Record<string, string>;
   method?: 'GET' | 'POST';
-  body?: Record<string, any> | string | FormData;
+  body?: Record<string, any> | string | FormData | URLSearchParams;
 };
 
 export type DefaultedFetcherOptions = {