From 0f4abff96e6038abc7d86dd66e87a46a5529b1f2 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Fri, 18 Aug 2023 16:39:40 +0800 Subject: [PATCH] feat: support nestedQuerystring as urllib v2 (#462) closes https://github.com/node-modules/urllib/issues/461 --- package.json | 2 + src/HttpClient.ts | 32 ++++++--- src/Request.ts | 5 ++ src/symbols.ts | 2 +- test/diagnostics_channel.test.ts | 2 +- test/fixtures/server.ts | 19 ++++-- test/options.data.test.ts | 112 +++++++++++++++++++++++++++++++ 7 files changed, 158 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 829b9480..7251238a 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "formstream": "^1.1.1", "mime-types": "^2.1.35", "pump": "^3.0.0", + "qs": "^6.11.2", "undici": "^5.22.1", "ylru": "^1.3.2" }, @@ -77,6 +78,7 @@ "@types/mime-types": "^2.1.1", "@types/node": "^20.2.1", "@types/pump": "^1.1.1", + "@types/qs": "^6.9.7", "@types/selfsigned": "^2.0.1", "@types/tar-stream": "^2.2.2", "@vitest/coverage-v8": "^0.32.0", diff --git a/src/HttpClient.ts b/src/HttpClient.ts index 26f1e2e6..e8a428c3 100644 --- a/src/HttpClient.ts +++ b/src/HttpClient.ts @@ -26,6 +26,7 @@ import { FormData as FormDataNode } from 'formdata-node'; import { FormDataEncoder } from 'form-data-encoder'; import createUserAgent from 'default-user-agent'; import mime from 'mime-types'; +import qs from 'qs'; import pump from 'pump'; // Compatible with old style formstream import FormStream from 'formstream'; @@ -87,7 +88,7 @@ export type ClientOptions = { rejectUnauthorized?: boolean; /** - * sockePath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe + * socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe */ socketPath?: string | null; }, @@ -244,14 +245,14 @@ export class HttpClient extends EventEmitter { // the response body and trailers have been received contentDownload: 0, }; - const orginalOpaque = args.opaque; + const originalOpaque = args.opaque; // using opaque to diagnostics channel, binding request and socket const internalOpaque = { [symbols.kRequestId]: requestId, [symbols.kRequestStartTime]: requestStartTime, [symbols.kEnableRequestTiming]: !!args.timing, [symbols.kRequestTiming]: timing, - [symbols.kRequestOrginalOpaque]: orginalOpaque, + [symbols.kRequestOriginalOpaque]: originalOpaque, }; const reqMeta = { requestId, @@ -452,10 +453,17 @@ export class HttpClient extends EventEmitter { || isReadable(args.data); if (isGETOrHEAD) { if (!isStringOrBufferOrReadable) { - for (const field in args.data) { - const fieldValue = args.data[field]; - if (fieldValue === undefined) continue; - requestUrl.searchParams.append(field, fieldValue); + if (args.nestedQuerystring) { + const querystring = qs.stringify(args.data); + // reset the requestUrl + const href = requestUrl.href; + requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring); + } else { + for (const field in args.data) { + const fieldValue = args.data[field]; + if (fieldValue === undefined) continue; + requestUrl.searchParams.append(field, fieldValue); + } } } } else { @@ -472,7 +480,11 @@ export class HttpClient extends EventEmitter { } } else { headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8'; - requestOptions.body = new URLSearchParams(args.data).toString(); + if (args.nestedQuerystring) { + requestOptions.body = qs.stringify(args.data); + } else { + requestOptions.body = new URLSearchParams(args.data).toString(); + } } } } @@ -582,7 +594,7 @@ export class HttpClient extends EventEmitter { this.#updateSocketInfo(socketInfo, internalOpaque); const clientResponse: HttpClientResponse = { - opaque: orginalOpaque, + opaque: originalOpaque, data, status: res.status, statusCode: res.status, @@ -637,7 +649,7 @@ export class HttpClient extends EventEmitter { return await this.#requestInternal(url, options, requestContext); } } - err.opaque = orginalOpaque; + err.opaque = originalOpaque; err.status = res.status; err.headers = res.headers; err.res = res; diff --git a/src/Request.ts b/src/Request.ts index ece57ff9..26acb914 100644 --- a/src/Request.ts +++ b/src/Request.ts @@ -46,6 +46,11 @@ export type RequestOptions = { * Default is 'buffer'. */ dataType?: 'text' | 'html' | 'json' | 'buffer' | 'stream'; + /** + * urllib default use URLSearchParams to stringify form data which don't support nested object, + * will use qs instead of URLSearchParams to support nested object by set this option to true. + */ + nestedQuerystring?: boolean; /** * @deprecated * Only for d.ts keep compatible with urllib@2, don't use it anymore. diff --git a/src/symbols.ts b/src/symbols.ts index 0eeccfd3..a474faa0 100644 --- a/src/symbols.ts +++ b/src/symbols.ts @@ -12,5 +12,5 @@ export default { kRequestStartTime: Symbol('request start time'), kEnableRequestTiming: Symbol('enable request timing or not'), kRequestTiming: Symbol('request timing'), - kRequestOrginalOpaque: Symbol('request orginal opaque'), + kRequestOriginalOpaque: Symbol('request original opaque'), }; diff --git a/test/diagnostics_channel.test.ts b/test/diagnostics_channel.test.ts index 472bda1e..a31f9f8b 100644 --- a/test/diagnostics_channel.test.ts +++ b/test/diagnostics_channel.test.ts @@ -43,7 +43,7 @@ describe('diagnostics_channel.test.ts', () => { } } } - const opaque = request[kHandler].opts.opaque[symbols.kRequestOrginalOpaque]; + const opaque = request[kHandler].opts.opaque[symbols.kRequestOriginalOpaque]; if (opaque && name === 'undici:client:sendHeaders' && socket) { socket[kRequests]++; opaque.tracer.socket = { diff --git a/test/fixtures/server.ts b/test/fixtures/server.ts index 221ee14c..4b13be57 100644 --- a/test/fixtures/server.ts +++ b/test/fixtures/server.ts @@ -6,6 +6,7 @@ import { createReadStream } from 'node:fs'; import busboy from 'busboy'; import iconv from 'iconv-lite'; import selfsigned from 'selfsigned'; +import qs from 'qs'; import { readableToBytes, sleep } from '../utils'; const requestsPerSocket = Symbol('requestsPerSocket'); @@ -292,10 +293,20 @@ export async function startServer(options?: { } if (req.headers['content-type']?.startsWith('application/x-www-form-urlencoded')) { - const searchParams = new URLSearchParams(requestBytes.toString()); - requestBody = {}; - for (const [ field, value ] of searchParams.entries()) { - requestBody[field] = value; + const raw = requestBytes.toString(); + requestBody = { + __raw__: raw, + }; + if (req.headers['x-qs'] === 'true') { + requestBody = { + ...qs.parse(raw), + __raw__: raw, + }; + } else { + const searchParams = new URLSearchParams(raw); + for (const [ field, value ] of searchParams.entries()) { + requestBody[field] = value; + } } } else if (req.headers['content-type']?.startsWith('application/json')) { requestBody = JSON.parse(requestBytes.toString()); diff --git a/test/options.data.test.ts b/test/options.data.test.ts index 4b773c7a..71b13e6d 100644 --- a/test/options.data.test.ts +++ b/test/options.data.test.ts @@ -1,6 +1,7 @@ import { strict as assert } from 'node:assert'; import { createReadStream } from 'node:fs'; import { Readable } from 'node:stream'; +import qs from 'qs'; import { describe, it, beforeAll, afterAll } from 'vitest'; import urllib from '../src'; import { startServer } from './fixtures/server'; @@ -42,6 +43,82 @@ describe('options.data.test.ts', () => { assert.equal(url.searchParams.get('data'), '哈哈'); }); + it('should GET with data work on nestedQuerystring=true', async () => { + const response = await urllib.request(_url, { + method: 'GET', + data: { + sql: 'SELECT * from table', + data: '哈哈', + foo: { + bar: 'bar value', + array: [ 1, 2, 3 ], + }, + }, + nestedQuerystring: true, + dataType: 'json', + headers: { + 'x-qs': 'true', + }, + }); + assert.equal(response.status, 200); + assert.equal(response.headers['content-type'], 'application/json'); + assert.equal(response.data.method, 'GET'); + assert(response.url.startsWith(_url)); + // console.log(response); + assert(!response.redirected); + assert.equal(response.data.url, '/?sql=SELECT%20%2A%20from%20table&data=%E5%93%88%E5%93%88&foo%5Bbar%5D=bar%20value&foo%5Barray%5D%5B0%5D=1&foo%5Barray%5D%5B1%5D=2&foo%5Barray%5D%5B2%5D=3'); + const query = qs.parse(response.data.url.substring(2)); + const url = new URL(response.data.href); + assert.equal(url.searchParams.get('sql'), 'SELECT * from table'); + assert.equal(url.searchParams.get('data'), '哈哈'); + assert.equal(url.searchParams.get('foo[bar]'), 'bar value'); + assert.equal(url.searchParams.get('foo[array][0]'), '1'); + assert.equal(url.searchParams.get('foo[array][1]'), '2'); + assert.equal(url.searchParams.get('foo[array][2]'), '3'); + assert.equal(query.sql, 'SELECT * from table'); + assert.equal(query.data, '哈哈'); + assert.deepEqual(query.foo, { bar: 'bar value', array: [ '1', '2', '3' ] }); + }); + + it('should GET /ok?hello=1 with data work on nestedQuerystring=true', async () => { + const response = await urllib.request(`${_url}ok?hello=1`, { + method: 'GET', + data: { + sql: 'SELECT * from table', + data: '哈哈', + foo: { + bar: 'bar value', + array: [ 1, 2, 3 ], + }, + }, + nestedQuerystring: true, + dataType: 'json', + headers: { + 'x-qs': 'true', + }, + }); + assert.equal(response.status, 200); + assert.equal(response.headers['content-type'], 'application/json'); + assert.equal(response.data.method, 'GET'); + assert(response.url.startsWith(_url)); + // console.log(response); + assert(!response.redirected); + assert.equal(response.data.url, '/ok?hello=1&sql=SELECT%20%2A%20from%20table&data=%E5%93%88%E5%93%88&foo%5Bbar%5D=bar%20value&foo%5Barray%5D%5B0%5D=1&foo%5Barray%5D%5B1%5D=2&foo%5Barray%5D%5B2%5D=3'); + const query = qs.parse(response.data.url.substring(4)); + const url = new URL(response.data.href); + assert.equal(url.searchParams.get('hello'), '1'); + assert.equal(url.searchParams.get('sql'), 'SELECT * from table'); + assert.equal(url.searchParams.get('data'), '哈哈'); + assert.equal(url.searchParams.get('foo[bar]'), 'bar value'); + assert.equal(url.searchParams.get('foo[array][0]'), '1'); + assert.equal(url.searchParams.get('foo[array][1]'), '2'); + assert.equal(url.searchParams.get('foo[array][2]'), '3'); + assert.equal(query.hello, '1'); + assert.equal(query.sql, 'SELECT * from table'); + assert.equal(query.data, '哈哈'); + assert.deepEqual(query.foo, { bar: 'bar value', array: [ '1', '2', '3' ] }); + }); + it('should HEAD with data and auto convert to query string', async () => { const response = await urllib.request(_url, { method: 'HEAD', @@ -182,6 +259,7 @@ describe('options.data.test.ts', () => { assert.equal(response.data.headers['content-type'], 'application/x-www-form-urlencoded;charset=UTF-8'); assert.equal(response.data.requestBody.sql, 'SELECT * from table'); assert.equal(response.data.requestBody.data, '哈哈 PUT'); + assert.equal(response.data.requestBody.__raw__, 'sql=SELECT+*+from+table&data=%E5%93%88%E5%93%88+PUT'); }); it('should PATCH with data and auto using application/x-www-form-urlencoded', async () => { @@ -226,6 +304,40 @@ describe('options.data.test.ts', () => { assert.equal(response.data.requestBody.sql, 'SELECT * from table'); assert.equal(response.data.requestBody.data, '哈哈 POST'); assert.equal(response.data.requestBody.foo, '[object Object]'); + assert.equal(response.data.requestBody.__raw__, 'sql=SELECT+*+from+table&data=%E5%93%88%E5%93%88+POST&foo=%5Bobject+Object%5D'); + }); + + it('should POST with application/x-www-form-urlencoded work on nestedQuerystring=true', async () => { + const response = await urllib.request(_url, { + method: 'POST', + data: { + sql: 'SELECT * from table', + data: '哈哈', + foo: { + bar: 'bar value', + array: [ 1, 2, 3 ], + }, + }, + nestedQuerystring: true, + dataType: 'json', + headers: { + 'x-qs': 'true', + }, + }); + assert.equal(response.status, 200); + assert.equal(response.headers['content-type'], 'application/json'); + assert.equal(response.data.method, 'POST'); + assert(response.url.startsWith(_url)); + assert(!response.redirected); + // console.log(response.data); + assert.equal(response.data.url, '/'); + assert.equal(response.data.headers['content-type'], 'application/x-www-form-urlencoded;charset=UTF-8'); + assert.equal(response.data.requestBody.sql, 'SELECT * from table'); + assert.equal(response.data.requestBody.data, '哈哈'); + assert(response.data.requestBody.foo, 'missing requestBody.foo'); + assert.equal(response.data.requestBody.foo.bar, 'bar value'); + assert.deepEqual(response.data.requestBody.foo.array, [ '1', '2', '3' ]); + assert.equal(response.data.requestBody.__raw__, 'sql=SELECT%20%2A%20from%20table&data=%E5%93%88%E5%93%88&foo%5Bbar%5D=bar%20value&foo%5Barray%5D%5B0%5D=1&foo%5Barray%5D%5B1%5D=2&foo%5Barray%5D%5B2%5D=3'); }); it('should PUT with data and contentType = json', async () => {