feat: support nestedQuerystring as urllib v2 (#462)

closes #461

package.json

@@ -68,6 +68,7 @@
     "formstream": "^1.1.1",
     "mime-types": "^2.1.35",
     "pump": "^3.0.0",
+    "qs": "^6.11.2",
     "undici": "^5.22.1",
     "ylru": "^1.3.2"
   },
@@ -77,6 +78,7 @@
     "@types/mime-types": "^2.1.1",
     "@types/node": "^20.2.1",
     "@types/pump": "^1.1.1",
+    "@types/qs": "^6.9.7",
     "@types/selfsigned": "^2.0.1",
     "@types/tar-stream": "^2.2.2",
     "@vitest/coverage-v8": "^0.32.0",

src/HttpClient.ts

@@ -26,6 +26,7 @@ import { FormData as FormDataNode } from 'formdata-node';
 import { FormDataEncoder } from 'form-data-encoder';
 import createUserAgent from 'default-user-agent';
 import mime from 'mime-types';
+import qs from 'qs';
 import pump from 'pump';
 // Compatible with old style formstream
 import FormStream from 'formstream';
@@ -87,7 +88,7 @@ export type ClientOptions = {
     rejectUnauthorized?: boolean;
 
     /**
-     * sockePath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
+     * socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
      */
     socketPath?: string | null;
   },
@@ -244,14 +245,14 @@ export class HttpClient extends EventEmitter {
       // the response body and trailers have been received
       contentDownload: 0,
     };
-    const orginalOpaque = args.opaque;
+    const originalOpaque = args.opaque;
     // using opaque to diagnostics channel, binding request and socket
     const internalOpaque = {
       [symbols.kRequestId]: requestId,
       [symbols.kRequestStartTime]: requestStartTime,
       [symbols.kEnableRequestTiming]: !!args.timing,
       [symbols.kRequestTiming]: timing,
-      [symbols.kRequestOrginalOpaque]: orginalOpaque,
+      [symbols.kRequestOriginalOpaque]: originalOpaque,
     };
     const reqMeta = {
       requestId,
@@ -452,10 +453,17 @@ export class HttpClient extends EventEmitter {
           || isReadable(args.data);
         if (isGETOrHEAD) {
           if (!isStringOrBufferOrReadable) {
-            for (const field in args.data) {
-              const fieldValue = args.data[field];
-              if (fieldValue === undefined) continue;
-              requestUrl.searchParams.append(field, fieldValue);
+            if (args.nestedQuerystring) {
+              const querystring = qs.stringify(args.data);
+              // reset the requestUrl
+              const href = requestUrl.href;
+              requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
+            } else {
+              for (const field in args.data) {
+                const fieldValue = args.data[field];
+                if (fieldValue === undefined) continue;
+                requestUrl.searchParams.append(field, fieldValue);
+              }
             }
           }
         } else {
@@ -472,7 +480,11 @@ export class HttpClient extends EventEmitter {
               }
             } else {
               headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
-              requestOptions.body = new URLSearchParams(args.data).toString();
+              if (args.nestedQuerystring) {
+                requestOptions.body = qs.stringify(args.data);
+              } else {
+                requestOptions.body = new URLSearchParams(args.data).toString();
+              }
             }
           }
         }
@@ -582,7 +594,7 @@ export class HttpClient extends EventEmitter {
       this.#updateSocketInfo(socketInfo, internalOpaque);
 
       const clientResponse: HttpClientResponse = {
-        opaque: orginalOpaque,
+        opaque: originalOpaque,
         data,
         status: res.status,
         statusCode: res.status,
@@ -637,7 +649,7 @@ export class HttpClient extends EventEmitter {
           return await this.#requestInternal(url, options, requestContext);
         }
       }
-      err.opaque = orginalOpaque;
+      err.opaque = originalOpaque;
       err.status = res.status;
       err.headers = res.headers;
       err.res = res;

src/Request.ts

@@ -46,6 +46,11 @@ export type RequestOptions = {
    * Default is 'buffer'.
    */
   dataType?: 'text' | 'html' | 'json' | 'buffer' | 'stream';
+  /**
+   * urllib default use URLSearchParams to stringify form data which don't support nested object,
+   * will use qs instead of URLSearchParams to support nested object by set this option to true.
+   */
+  nestedQuerystring?: boolean;
   /**
    * @deprecated
    * Only for d.ts keep compatible with urllib@2, don't use it anymore.

src/symbols.ts

@@ -12,5 +12,5 @@ export default {
   kRequestStartTime: Symbol('request start time'),
   kEnableRequestTiming: Symbol('enable request timing or not'),
   kRequestTiming: Symbol('request timing'),
-  kRequestOrginalOpaque: Symbol('request orginal opaque'),
+  kRequestOriginalOpaque: Symbol('request original opaque'),
 };

test/diagnostics_channel.test.ts

@@ -43,7 +43,7 @@ describe('diagnostics_channel.test.ts', () => {
           }
         }
       }
-      const opaque = request[kHandler].opts.opaque[symbols.kRequestOrginalOpaque];
+      const opaque = request[kHandler].opts.opaque[symbols.kRequestOriginalOpaque];
       if (opaque && name === 'undici:client:sendHeaders' && socket) {
         socket[kRequests]++;
         opaque.tracer.socket = {

test/fixtures/server.ts

@@ -6,6 +6,7 @@ import { createReadStream } from 'node:fs';
 import busboy from 'busboy';
 import iconv from 'iconv-lite';
 import selfsigned from 'selfsigned';
+import qs from 'qs';
 import { readableToBytes, sleep } from '../utils';
 
 const requestsPerSocket = Symbol('requestsPerSocket');
@@ -292,10 +293,20 @@ export async function startServer(options?: {
     }
 
     if (req.headers['content-type']?.startsWith('application/x-www-form-urlencoded')) {
-      const searchParams = new URLSearchParams(requestBytes.toString());
-      requestBody = {};
-      for (const [ field, value ] of searchParams.entries()) {
-        requestBody[field] = value;
+      const raw = requestBytes.toString();
+      requestBody = {
+        __raw__: raw,
+      };
+      if (req.headers['x-qs'] === 'true') {
+        requestBody = {
+          ...qs.parse(raw),
+          __raw__: raw,
+        };
+      } else {
+        const searchParams = new URLSearchParams(raw);
+        for (const [ field, value ] of searchParams.entries()) {
+          requestBody[field] = value;
+        }
       }
     } else if (req.headers['content-type']?.startsWith('application/json')) {
       requestBody = JSON.parse(requestBytes.toString());

test/options.data.test.ts

@@ -1,6 +1,7 @@
 import { strict as assert } from 'node:assert';
 import { createReadStream } from 'node:fs';
 import { Readable } from 'node:stream';
+import qs from 'qs';
 import { describe, it, beforeAll, afterAll } from 'vitest';
 import urllib from '../src';
 import { startServer } from './fixtures/server';
@@ -42,6 +43,82 @@ describe('options.data.test.ts', () => {
     assert.equal(url.searchParams.get('data'), '哈哈');
   });
 
+  it('should GET with data work on nestedQuerystring=true', async () => {
+    const response = await urllib.request(_url, {
+      method: 'GET',
+      data: {
+        sql: 'SELECT * from table',
+        data: '哈哈',
+        foo: {
+          bar: 'bar value',
+          array: [ 1, 2, 3 ],
+        },
+      },
+      nestedQuerystring: true,
+      dataType: 'json',
+      headers: {
+        'x-qs': 'true',
+      },
+    });
+    assert.equal(response.status, 200);
+    assert.equal(response.headers['content-type'], 'application/json');
+    assert.equal(response.data.method, 'GET');
+    assert(response.url.startsWith(_url));
+    // console.log(response);
+    assert(!response.redirected);
+    assert.equal(response.data.url, '/?sql=SELECT%20%2A%20from%20table&data=%E5%93%88%E5%93%88&foo%5Bbar%5D=bar%20value&foo%5Barray%5D%5B0%5D=1&foo%5Barray%5D%5B1%5D=2&foo%5Barray%5D%5B2%5D=3');
+    const query = qs.parse(response.data.url.substring(2));
+    const url = new URL(response.data.href);
+    assert.equal(url.searchParams.get('sql'), 'SELECT * from table');
+    assert.equal(url.searchParams.get('data'), '哈哈');
+    assert.equal(url.searchParams.get('foo[bar]'), 'bar value');
+    assert.equal(url.searchParams.get('foo[array][0]'), '1');
+    assert.equal(url.searchParams.get('foo[array][1]'), '2');
+    assert.equal(url.searchParams.get('foo[array][2]'), '3');
+    assert.equal(query.sql, 'SELECT * from table');
+    assert.equal(query.data, '哈哈');
+    assert.deepEqual(query.foo, { bar: 'bar value', array: [ '1', '2', '3' ] });
+  });
+
+  it('should GET /ok?hello=1 with data work on nestedQuerystring=true', async () => {
+    const response = await urllib.request(`${_url}ok?hello=1`, {
+      method: 'GET',
+      data: {
+        sql: 'SELECT * from table',
+        data: '哈哈',
+        foo: {
+          bar: 'bar value',
+          array: [ 1, 2, 3 ],
+        },
+      },
+      nestedQuerystring: true,
+      dataType: 'json',
+      headers: {
+        'x-qs': 'true',
+      },
+    });
+    assert.equal(response.status, 200);
+    assert.equal(response.headers['content-type'], 'application/json');
+    assert.equal(response.data.method, 'GET');
+    assert(response.url.startsWith(_url));
+    // console.log(response);
+    assert(!response.redirected);
+    assert.equal(response.data.url, '/ok?hello=1&sql=SELECT%20%2A%20from%20table&data=%E5%93%88%E5%93%88&foo%5Bbar%5D=bar%20value&foo%5Barray%5D%5B0%5D=1&foo%5Barray%5D%5B1%5D=2&foo%5Barray%5D%5B2%5D=3');
+    const query = qs.parse(response.data.url.substring(4));
+    const url = new URL(response.data.href);
+    assert.equal(url.searchParams.get('hello'), '1');
+    assert.equal(url.searchParams.get('sql'), 'SELECT * from table');
+    assert.equal(url.searchParams.get('data'), '哈哈');
+    assert.equal(url.searchParams.get('foo[bar]'), 'bar value');
+    assert.equal(url.searchParams.get('foo[array][0]'), '1');
+    assert.equal(url.searchParams.get('foo[array][1]'), '2');
+    assert.equal(url.searchParams.get('foo[array][2]'), '3');
+    assert.equal(query.hello, '1');
+    assert.equal(query.sql, 'SELECT * from table');
+    assert.equal(query.data, '哈哈');
+    assert.deepEqual(query.foo, { bar: 'bar value', array: [ '1', '2', '3' ] });
+  });
+
   it('should HEAD with data and auto convert to query string', async () => {
     const response = await urllib.request(_url, {
       method: 'HEAD',
@@ -182,6 +259,7 @@ describe('options.data.test.ts', () => {
     assert.equal(response.data.headers['content-type'], 'application/x-www-form-urlencoded;charset=UTF-8');
     assert.equal(response.data.requestBody.sql, 'SELECT * from table');
     assert.equal(response.data.requestBody.data, '哈哈 PUT');
+    assert.equal(response.data.requestBody.__raw__, 'sql=SELECT+*+from+table&data=%E5%93%88%E5%93%88+PUT');
   });
 
   it('should PATCH with data and auto using application/x-www-form-urlencoded', async () => {
@@ -226,6 +304,40 @@ describe('options.data.test.ts', () => {
     assert.equal(response.data.requestBody.sql, 'SELECT * from table');
     assert.equal(response.data.requestBody.data, '哈哈 POST');
     assert.equal(response.data.requestBody.foo, '[object Object]');
+    assert.equal(response.data.requestBody.__raw__, 'sql=SELECT+*+from+table&data=%E5%93%88%E5%93%88+POST&foo=%5Bobject+Object%5D');
+  });
+
+  it('should POST with application/x-www-form-urlencoded work on nestedQuerystring=true', async () => {
+    const response = await urllib.request(_url, {
+      method: 'POST',
+      data: {
+        sql: 'SELECT * from table',
+        data: '哈哈',
+        foo: {
+          bar: 'bar value',
+          array: [ 1, 2, 3 ],
+        },
+      },
+      nestedQuerystring: true,
+      dataType: 'json',
+      headers: {
+        'x-qs': 'true',
+      },
+    });
+    assert.equal(response.status, 200);
+    assert.equal(response.headers['content-type'], 'application/json');
+    assert.equal(response.data.method, 'POST');
+    assert(response.url.startsWith(_url));
+    assert(!response.redirected);
+    // console.log(response.data);
+    assert.equal(response.data.url, '/');
+    assert.equal(response.data.headers['content-type'], 'application/x-www-form-urlencoded;charset=UTF-8');
+    assert.equal(response.data.requestBody.sql, 'SELECT * from table');
+    assert.equal(response.data.requestBody.data, '哈哈');
+    assert(response.data.requestBody.foo, 'missing requestBody.foo');
+    assert.equal(response.data.requestBody.foo.bar, 'bar value');
+    assert.deepEqual(response.data.requestBody.foo.array, [ '1', '2', '3' ]);
+    assert.equal(response.data.requestBody.__raw__, 'sql=SELECT%20%2A%20from%20table&data=%E5%93%88%E5%93%88&foo%5Bbar%5D=bar%20value&foo%5Barray%5D%5B0%5D=1&foo%5Barray%5D%5B1%5D=2&foo%5Barray%5D%5B2%5D=3');
   });
 
   it('should PUT with data and contentType = json', async () => {