From 42842e80d1c296e1dfcb88535122656c06c2cac0 Mon Sep 17 00:00:00 2001 From: Shawn Kim Date: Wed, 19 Apr 2023 06:36:53 -0700 Subject: [PATCH] feat: nginx-aws-signature lib submodule --- .github/workflows/main.yml | 12 + .gitmodules | 3 + README.md | 3 +- .../00-check-for-required-env.sh | 6 +- common/etc/nginx/include/awscredentials.js | 178 ------------ common/etc/nginx/include/awssig | 1 + common/etc/nginx/include/awssig2.js | 46 --- common/etc/nginx/include/awssig4.js | 262 ------------------ common/etc/nginx/include/s3gateway.js | 233 +--------------- common/etc/nginx/include/utils.js | 159 ----------- common/etc/nginx/nginx.conf | 6 +- .../etc/nginx/templates/default.conf.template | 4 +- docs/development.md | 18 +- docs/getting_started.md | 16 +- settings.example | 6 +- standalone_ubuntu_oss_install.sh | 30 +- submodule.sh | 7 + test.sh | 19 +- test/docker-compose.yaml | 4 +- test/unit/awscredentials_test.js | 191 ------------- test/unit/awssig2_test.js | 91 ------ test/unit/awssig4_test.js | 172 ------------ test/unit/s3gateway_test.js | 11 +- test/unit/utils_test.js | 197 ------------- 24 files changed, 88 insertions(+), 1587 deletions(-) create mode 100644 .gitmodules delete mode 100644 common/etc/nginx/include/awscredentials.js create mode 160000 common/etc/nginx/include/awssig delete mode 100644 common/etc/nginx/include/awssig2.js delete mode 100644 common/etc/nginx/include/awssig4.js delete mode 100644 common/etc/nginx/include/utils.js create mode 100755 submodule.sh delete mode 100644 test/unit/awscredentials_test.js delete mode 100644 test/unit/awssig2_test.js delete mode 100644 test/unit/awssig4_test.js delete mode 100644 test/unit/utils_test.js diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dfd96396..1ae0404f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,12 +20,24 @@ jobs: - name: Install dependencies run: sudo apt-get install -y wait-for-it + + - name: Pull & update nginx-aws-signature submodules + run: | + git submodule update --init common/etc/nginx/include/awssig + git submodule absorbgitdirs + git -C common/etc/nginx/include/awssig config core.sparseCheckout true + echo 'core/*' >>.git/modules/common/etc/nginx/include/awssig/info/sparse-checkout + git submodule update --force --checkout common/etc/nginx/include/awssig + - name: Run tests - latest njs version run: ./test.sh --latest-njs --type oss + - name: Run tests - stable njs version run: ./test.sh --type oss + - name: Run tests - stable njs version - unprivileged process run: ./test.sh --unprivileged --type oss + - name: Run tests - latest njs version - unprivileged process run: ./test.sh --latest-njs --unprivileged --type oss diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..66095b17 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "common/etc/nginx/include/awssig"] + path = common/etc/nginx/include/awssig + url = https://github.com/nginxinc/nginx-aws-signature.git diff --git a/README.md b/README.md index ec66a630..c27047ff 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,11 @@ and run the gateway. ``` common/ contains files used by both NGINX OSS and Plus configurations etc/nginx/include/ + s3gateway.js common library to integrate the s3 storage from NGINX OSS and Plus + etc/nginx/include/awssig/core required `git submodule` command by executing `submodule.sh` awscredentials.js common library to read and write credentials awssig2.js common library to build AWS signature 2 awssig4.js common library to build AWS signature 4 and get a session token - s3gateway.js common library to integrate the s3 storage from NGINX OSS and Plus utils.js common library to be reused by all of NJS codebases deployments/ contains files used for deployment technologies such as CloudFormation diff --git a/common/docker-entrypoint.d/00-check-for-required-env.sh b/common/docker-entrypoint.d/00-check-for-required-env.sh index 2a057ff4..b3406d38 100755 --- a/common/docker-entrypoint.d/00-check-for-required-env.sh +++ b/common/docker-entrypoint.d/00-check-for-required-env.sh @@ -34,7 +34,7 @@ required=("S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO" if [[ -v AWS_CONTAINER_CREDENTIALS_RELATIVE_URI ]]; then echo "Running inside an ECS task, using container credentials" -elif [[ -v S3_SESSION_TOKEN ]]; then +elif [[ -v AWS_SESSION_TOKEN ]]; then echo "S3 Session token specified - not using IMDS for credentials" # b) Using Instance Metadata Service (IMDS) credentials, if IMDS is present at http://169.254.169.254. @@ -52,7 +52,7 @@ elif [[ -v AWS_WEB_IDENTITY_TOKEN_FILE ]]; then # If none of the options above is used, require static credentials. # See https://docs.aws.amazon.com/sdkref/latest/guide/feature-static-credentials.html. else - required+=("S3_ACCESS_KEY_ID" "S3_SECRET_KEY") + required+=("AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY") fi for name in ${required[@]}; do @@ -101,7 +101,7 @@ if [ $failed -gt 0 ]; then fi echo "S3 Backend Environment" -echo "Access Key ID: ${S3_ACCESS_KEY_ID}" +echo "Access Key ID: ${AWS_ACCESS_KEY_ID}" echo "Origin: ${S3_SERVER_PROTO}://${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}" echo "Region: ${S3_REGION}" echo "Addressing Style: ${S3_STYLE}" diff --git a/common/etc/nginx/include/awscredentials.js b/common/etc/nginx/include/awscredentials.js deleted file mode 100644 index 417a04d8..00000000 --- a/common/etc/nginx/include/awscredentials.js +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2023 F5, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import utils from "./utils.js"; - -const fs = require('fs'); - -/** - * Get the current session token from either the instance profile credential - * cache or environment variables. - * - * @param r {Request} HTTP request object (not used, but required for NGINX configuration) - * @returns {string} current session token or empty string - */ -function sessionToken(r) { - const credentials = readCredentials(r); - if (credentials.sessionToken) { - return credentials.sessionToken; - } - return ''; -} - -/** - * Get the instance profile credentials needed to authenticated against S3 from - * a backend cache. If the credentials cannot be found, then return undefined. - * @param r {Request} HTTP request object (not used, but required for NGINX configuration) - * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string|null), expiration: (string|null)}} AWS instance profile credentials or undefined - */ -function readCredentials(r) { - // TODO: Change the generic constants naming for multiple AWS services. - if ('S3_ACCESS_KEY_ID' in process.env && 'S3_SECRET_KEY' in process.env) { - const sessionToken = 'S3_SESSION_TOKEN' in process.env ? - process.env['S3_SESSION_TOKEN'] : null; - return { - accessKeyId: process.env['S3_ACCESS_KEY_ID'], - secretAccessKey: process.env['S3_SECRET_KEY'], - sessionToken: sessionToken, - expiration: null - }; - } - - if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) { - return _readCredentialsFromKeyValStore(r); - } else { - return _readCredentialsFromFile(); - } -} - -/** - * Read credentials from the NGINX Keyval store. If it is not found, then - * return undefined. - * - * @param r {Request} HTTP request object (not used, but required for NGINX configuration) - * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined - * @private - */ -function _readCredentialsFromKeyValStore(r) { - const cached = r.variables.instance_credential_json; - - if (!cached) { - return undefined; - } - - try { - return JSON.parse(cached); - } catch (e) { - utils.debug_log(r, `Error parsing JSON value from r.variables.instance_credential_json: ${e}`); - return undefined; - } -} - -/** - * Read the contents of the credentials file into memory. If it is not - * found, then return undefined. - * - * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined - * @private - */ -function _readCredentialsFromFile() { - const credsFilePath = _credentialsTempFile(); - - try { - const creds = fs.readFileSync(credsFilePath); - return JSON.parse(creds); - } catch (e) { - /* Do not throw an exception in the case of when the - credentials file path is invalid in order to signal to - the caller that such a file has not been created yet. */ - if (e.code === 'ENOENT') { - return undefined; - } - throw e; - } -} - -/** - * Returns the path to the credentials temporary cache file. - * - * @returns {string} path on the file system to credentials cache file - * @private - */ -function _credentialsTempFile() { - if (process.env['S3_CREDENTIALS_TEMP_FILE']) { - return process.env['S3_CREDENTIALS_TEMP_FILE']; - } - if (process.env['TMPDIR']) { - return `${process.env['TMPDIR']}/credentials.json` - } - - return '/tmp/credentials.json'; -} - -/** - * Write the instance profile credentials to a caching backend. - * - * @param r {Request} HTTP request object (not used, but required for NGINX configuration) - * @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials - */ -function writeCredentials(r, credentials) { - /* Do not bother writing credentials if we are running in a mode where we - do not need instance credentials. */ - if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) { - return; - } - - if (!credentials) { - throw `Cannot write invalid credentials: ${JSON.stringify(credentials)}`; - } - - if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) { - _writeCredentialsToKeyValStore(r, credentials); - } else { - _writeCredentialsToFile(credentials); - } -} - -/** - * Write the instance profile credentials to the NGINX Keyval store. - * - * @param r {Request} HTTP request object (not used, but required for NGINX configuration) - * @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials - * @private - */ -function _writeCredentialsToKeyValStore(r, credentials) { - r.variables.instance_credential_json = JSON.stringify(credentials); -} - -/** - * Write the instance profile credentials to a file on the file system. This - * file will be quite small and should end up in the file cache relatively - * quickly if it is repeatedly read. - * - * @param r {Request} HTTP request object (not used, but required for NGINX configuration) - * @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials - * @private - */ -function _writeCredentialsToFile(credentials) { - fs.writeFileSync(_credentialsTempFile(), JSON.stringify(credentials)); -} - -export default { - readCredentials, - sessionToken, - writeCredentials -} diff --git a/common/etc/nginx/include/awssig b/common/etc/nginx/include/awssig new file mode 160000 index 00000000..63b18210 --- /dev/null +++ b/common/etc/nginx/include/awssig @@ -0,0 +1 @@ +Subproject commit 63b182101446ff768a5c1a1b247d291715d8de85 diff --git a/common/etc/nginx/include/awssig2.js b/common/etc/nginx/include/awssig2.js deleted file mode 100644 index 5630b212..00000000 --- a/common/etc/nginx/include/awssig2.js +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2023 F5, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import utils from "./utils.js"; - -const mod_hmac = require('crypto'); - -/** - * Create HTTP Authorization header for authenticating with an AWS compatible - * v2 API. - * - * @param r {Request} HTTP request object - * @param uri {string} The URI-encoded version of the absolute path component URL to create a request - * @param httpDate {string} RFC2616 timestamp used to sign the request - * @param credentials {object} Credential object with AWS credentials in it (AccessKeyId, SecretAccessKey, SessionToken) - * @returns {string} HTTP Authorization header value - */ -function signatureV2(r, uri, httpDate, credentials) { - const method = r.method; - const hmac = mod_hmac.createHmac('sha1', credentials.secretAccessKey); - const stringToSign = method + '\n\n\n' + httpDate + '\n' + uri; - - utils.debug_log(r, 'AWS v2 Auth Signing String: [' + stringToSign + ']'); - - const signature = hmac.update(stringToSign).digest('base64'); - - return `AWS ${credentials.accessKeyId}:${signature}`; -} - - -export default { - signatureV2 -} diff --git a/common/etc/nginx/include/awssig4.js b/common/etc/nginx/include/awssig4.js deleted file mode 100644 index 6206796c..00000000 --- a/common/etc/nginx/include/awssig4.js +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Copyright 2023 F5, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import utils from "./utils.js"; - -const mod_hmac = require('crypto'); - -/** - * Constant checksum for an empty HTTP body. - * @type {string} - */ -const EMPTY_PAYLOAD_HASH = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; - -/** - * Constant defining the headers being signed. - * @type {string} - */ -const DEFAULT_SIGNED_HEADERS = 'host;x-amz-content-sha256;x-amz-date'; - - -/** - * Create HTTP Authorization header for authenticating with an AWS compatible - * v4 API. - * - * @param r {Request} HTTP request object - * @param timestamp {Date} timestamp associated with request (must fall within a skew) - * @param region {string} API region associated with request - * @param service {string} service code (for example, s3, lambda) - * @param uri {string} The URI-encoded version of the absolute path component URL to create a canonical request - * @param queryParams {string} The URL-encoded query string parameters to create a canonical request - * @param host {string} HTTP host header value - * @param credentials {object} Credential object with AWS credentials in it (AccessKeyId, SecretAccessKey, SessionToken) - * @returns {string} HTTP Authorization header value - */ -function signatureV4(r, timestamp, region, service, uri, queryParams, host, credentials) { - const eightDigitDate = utils.getEightDigitDate(timestamp); - const amzDatetime = utils.getAmzDatetime(timestamp, eightDigitDate); - const canonicalRequest = _buildCanonicalRequest( - r.method, uri, queryParams, host, amzDatetime, credentials.sessionToken); - const signature = _buildSignatureV4(r, amzDatetime, eightDigitDate, - credentials, region, service, canonicalRequest); - const authHeader = 'AWS4-HMAC-SHA256 Credential=' - .concat(credentials.accessKeyId, '/', eightDigitDate, '/', region, '/', service, '/aws4_request,', - 'SignedHeaders=', _signedHeaders(credentials.sessionToken), ',Signature=', signature); - - utils.debug_log(r, 'AWS v4 Auth header: [' + authHeader + ']'); - - return authHeader; -} - -/** - * Creates a canonical request that will later be signed - * - * @see {@link https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html | Creating a Canonical Request} - * @param method {string} HTTP method - * @param uri {string} URI associated with request - * @param queryParams {string} query parameters associated with request - * @param host {string} HTTP Host header value - * @param amzDatetime {string} ISO8601 timestamp string to sign request with - * @returns {string} string with concatenated request parameters - * @private - */ -function _buildCanonicalRequest(method, uri, queryParams, host, amzDatetime, sessionToken) { - let canonicalHeaders = 'host:' + host + '\n' + - 'x-amz-content-sha256:' + EMPTY_PAYLOAD_HASH + '\n' + - 'x-amz-date:' + amzDatetime + '\n'; - - if (sessionToken) { - canonicalHeaders += 'x-amz-security-token:' + sessionToken + '\n' - } - - let canonicalRequest = method + '\n'; - canonicalRequest += uri + '\n'; - canonicalRequest += queryParams + '\n'; - canonicalRequest += canonicalHeaders + '\n'; - canonicalRequest += _signedHeaders(sessionToken) + '\n'; - canonicalRequest += EMPTY_PAYLOAD_HASH; - - return canonicalRequest; -} - -/** - * Creates a signature for use authenticating against an AWS compatible API. - * - * @see {@link https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html | AWS V4 Signing Process} - * @param r {Request} HTTP request object - * @param amzDatetime {string} ISO8601 timestamp string to sign request with - * @param eightDigitDate {string} date in the form of 'YYYYMMDD' - * @param creds {object} AWS credentials - * @param region {string} API region associated with request - * @param service {string} service code (for example, s3, lambda) - * @param canonicalRequest {string} string with concatenated request parameters - * @returns {string} hex encoded hash of signature HMAC value - * @private - */ -function _buildSignatureV4( - r, amzDatetime, eightDigitDate, creds, region, service, canonicalRequest) { - utils.debug_log(r, 'AWS v4 Auth Canonical Request: [' + canonicalRequest + ']'); - - const canonicalRequestHash = mod_hmac.createHash('sha256') - .update(canonicalRequest) - .digest('hex'); - - utils.debug_log(r, 'AWS v4 Auth Canonical Request Hash: [' + canonicalRequestHash + ']'); - - const stringToSign = _buildStringToSign( - amzDatetime, eightDigitDate, region, service, canonicalRequestHash); - - utils.debug_log(r, 'AWS v4 Auth Signing String: [' + stringToSign + ']'); - - let kSigningHash; - - /* If we have a keyval zone and key defined for caching the signing key hash, - * then signing key caching will be enabled. By caching signing keys we can - * accelerate the signing process because we will have four less HMAC - * operations that have to be performed per incoming request. The signing - * key expires every day, so our cache key can persist for 24 hours safely. - */ - if ("variables" in r && r.variables.cache_signing_key_enabled == 1) { - // cached value is in the format: [eightDigitDate]:[signingKeyHash] - const cached = "signing_key_hash" in r.variables ? r.variables.signing_key_hash : ""; - const fields = _splitCachedValues(cached); - const cachedEightDigitDate = fields[0]; - const cacheIsValid = fields.length === 2 && eightDigitDate === cachedEightDigitDate; - - // If true, use cached value - if (cacheIsValid) { - utils.debug_log(r, 'AWS v4 Using cached Signing Key Hash'); - /* We are forced to JSON encode the string returned from the HMAC - * operation because it is in a very specific format that include - * binary data and in order to preserve that data when persisting - * we encode it as JSON. By doing so we can gracefully decode it - * when reading from the cache. */ - kSigningHash = Buffer.from(JSON.parse(fields[1])); - // Otherwise, generate a new signing key hash and store it in the cache - } else { - kSigningHash = _buildSigningKeyHash(creds.secretAccessKey, eightDigitDate, region, service); - utils.debug_log(r, 'Writing key: ' + eightDigitDate + ':' + kSigningHash.toString('hex')); - r.variables.signing_key_hash = eightDigitDate + ':' + JSON.stringify(kSigningHash); - } - // Otherwise, don't use caching at all (like when we are using NGINX OSS) - } else { - kSigningHash = _buildSigningKeyHash(creds.secretAccessKey, eightDigitDate, region, service); - } - - utils.debug_log(r, 'AWS v4 Signing Key Hash: [' + kSigningHash.toString('hex') + ']'); - - const signature = mod_hmac.createHmac('sha256', kSigningHash) - .update(stringToSign).digest('hex'); - - utils.debug_log(r, 'AWS v4 Authorization Header: [' + signature + ']'); - - return signature; -} - -/** - * Creates a string to sign by concatenating together multiple parameters required - * by the signatures algorithm. - * - * @see {@link https://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html | String to Sign} - * @param amzDatetime {string} ISO8601 timestamp string to sign request with - * @param eightDigitDate {string} date in the form of 'YYYYMMDD' - * @param region {string} region associated with server API - * @param service {string} service code (for example, s3, lambda) - * @param canonicalRequestHash {string} hex encoded hash of canonical request string - * @returns {string} a concatenated string of the passed parameters formatted for signatures - * @private - */ -function _buildStringToSign(amzDatetime, eightDigitDate, region, service, canonicalRequestHash) { - return 'AWS4-HMAC-SHA256\n' + - amzDatetime + '\n' + - eightDigitDate + '/' + region + '/' + service + '/aws4_request\n' + - canonicalRequestHash; -} - -/** - * Creates a string containing the headers that need to be signed as part of v4 - * signature authentication. - * - * @param sessionToken {string|undefined} AWS session token if present - * @returns {string} semicolon delimited string of the headers needed for signing - * @private - */ -function _signedHeaders(sessionToken) { - let headers = DEFAULT_SIGNED_HEADERS; - if (sessionToken) { - headers += ';x-amz-security-token'; - } - return headers; -} - -/** - * Creates a signing key HMAC. This value is used to sign the request made to - * the API. - * - * @param kSecret {string} secret access key - * @param eightDigitDate {string} date in the form of 'YYYYMMDD' - * @param region {string} region associated with server API - * @param service {string} name of service that request is for e.g. s3, lambda - * @returns {ArrayBuffer} signing HMAC - * @private - */ -function _buildSigningKeyHash(kSecret, eightDigitDate, region, service) { - const kDate = mod_hmac.createHmac('sha256', 'AWS4'.concat(kSecret)) - .update(eightDigitDate).digest(); - const kRegion = mod_hmac.createHmac('sha256', kDate) - .update(region).digest(); - const kService = mod_hmac.createHmac('sha256', kRegion) - .update(service).digest(); - const kSigning = mod_hmac.createHmac('sha256', kService) - .update('aws4_request').digest(); - - return kSigning; -} - -/** - * Splits the cached values into an array with two elements or returns an - * empty array if the input string is invalid. The first element contains - * the eight digit date string and the second element contains a JSON string - * of the kSigningHash. - * - * @param cached input string to parse - * @returns {string[]|*[]} array containing eight digit date and kSigningHash or empty - * @private - */ -function _splitCachedValues(cached) { - const matchedPos = cached.indexOf(':', 0); - // Do a sanity check on the position returned, if it isn't sane, return - // an empty array and let the caller logic process it. - if (matchedPos < 0 || matchedPos + 1 > cached.length) { - return [] - } - - const eightDigitDate = cached.substring(0, matchedPos); - const kSigningHash = cached.substring(matchedPos + 1); - - return [eightDigitDate, kSigningHash] -} - - -export default { - signatureV4, - // These functions do not need to be exposed, but they are exposed so that - // unit tests can run against them. - _buildCanonicalRequest, - _buildSignatureV4, - _buildSigningKeyHash, - _splitCachedValues -} diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index 6d694749..b688134f 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -14,10 +14,10 @@ * limitations under the License. */ -import awscred from "./awscredentials.js"; -import awssig2 from "./awssig2.js"; -import awssig4 from "./awssig4.js"; -import utils from "./utils.js"; +import awscred from "./awssig/core/awscredentials.js"; +import awssig2 from "./awssig/core/awssig2.js"; +import awssig4 from "./awssig/core/awssig4.js"; +import utils from "./awssig/core/utils.js"; _requireEnvVars('S3_BUCKET_NAME'); _requireEnvVars('S3_SERVER'); @@ -477,234 +477,9 @@ function _requireEnvVars(envVarName) { } } -/** - * Offset to the expiration of credentials, when they should be considered expired and refreshed. The maximum - * time here can be 5 minutes, the IMDS and ECS credentials endpoint will make sure that each returned set of credentials - * is valid for at least another 5 minutes. - * - * To make sure we always refresh the credentials instead of retrieving the same again, keep credentials until 4:30 minutes - * before they really expire. - * - * @type {number} - */ -const maxValidityOffsetMs = 4.5 * 60 * 1000; - -/** - * Get the credentials needed to create AWS signatures in order to authenticate - * to S3. If the gateway is being provided credentials via a instance profile - * credential as provided over the metadata endpoint, this function will: - * 1. Try to read the credentials from cache - * 2. Determine if the credentials are stale - * 3. If the cached credentials are missing or stale, it gets new credentials - * from the metadata endpoint. - * 4. If new credentials were pulled, it writes the credentials back to the - * cache. - * - * If the gateway is not using instance profile credentials, then this function - * quickly exits. - * - * @param r {Request} HTTP request object - * @returns {Promise} - */ -async function fetchCredentials(r) { - /* If we are not using an AWS instance profile to set our credentials we - exit quickly and don't write a credentials file. */ - if (utils.areAllEnvVarsSet(['S3_ACCESS_KEY_ID', 'S3_SECRET_KEY'])) { - r.return(200); - return; - } - - let current; - - try { - current = awscred.readCredentials(r); - } catch (e) { - utils.debug_log(r, `Could not read credentials: ${e}`); - r.return(500); - return; - } - - if (current) { - // If AWS returns a Unix timestamp it will be in seconds, but in Date constructor we should provide timestamp in milliseconds - // In some situations (including EC2 and Fargate) current.expiration will be an RFC 3339 string - see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials - const expireAt = typeof current.expiration == 'number' ? current.expiration * 1000 : current.expiration - const exp = new Date(expireAt).getTime() - maxValidityOffsetMs; - if (NOW.getTime() < exp) { - r.return(200); - return; - } - } - - let credentials; - - utils.debug_log(r, 'Cached credentials are expired or not present, requesting new ones'); - - if (utils.areAllEnvVarsSet('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI')) { - const relative_uri = process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] || ''; - const uri = ECS_CREDENTIAL_BASE_URI + relative_uri; - try { - credentials = await _fetchEcsRoleCredentials(uri); - } catch (e) { - utils.debug_log(r, 'Could not load ECS task role credentials: ' + JSON.stringify(e)); - r.return(500); - return; - } - } - else if (utils.areAllEnvVarsSet('AWS_WEB_IDENTITY_TOKEN_FILE')) { - try { - credentials = await _fetchWebIdentityCredentials(r) - } catch(e) { - utils.debug_log(r, 'Could not assume role using web identity: ' + JSON.stringify(e)); - r.return(500); - return; - } - } else { - try { - credentials = await _fetchEC2RoleCredentials(); - } catch (e) { - utils.debug_log(r, 'Could not load EC2 task role credentials: ' + JSON.stringify(e)); - r.return(500); - return; - } - } - try { - awscred.writeCredentials(r, credentials); - } catch (e) { - utils.debug_log(r, `Could not write credentials: ${e}`); - r.return(500); - return; - } - r.return(200); -} - -/** - * Get the credentials needed to generate AWS signatures from the ECS - * (Elastic Container Service) metadata endpoint. - * - * @param credentialsUri {string} endpoint to get credentials from - * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>} - * @private - */ -async function _fetchEcsRoleCredentials(credentialsUri) { - const resp = await ngx.fetch(credentialsUri); - if (!resp.ok) { - throw 'Credentials endpoint response was not ok.'; - } - const creds = await resp.json(); - - return { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.Token, - expiration: creds.Expiration, - }; -} - -/** - * Get the credentials needed to generate AWS signatures from the EC2 - * metadata endpoint. - * - * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>} - * @private - */ -async function _fetchEC2RoleCredentials() { - const tokenResp = await ngx.fetch(EC2_IMDS_TOKEN_ENDPOINT, { - headers: { - 'x-aws-ec2-metadata-token-ttl-seconds': '21600', - }, - method: 'PUT', - }); - const token = await tokenResp.text(); - let resp = await ngx.fetch(EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT, { - headers: { - 'x-aws-ec2-metadata-token': token, - }, - }); - /* This _might_ get multiple possible roles in other scenarios, however, - EC2 supports attaching one role only.It should therefore be safe to take - the whole output, even given IMDS _might_ (?) be able to return multiple - roles. */ - const credName = await resp.text(); - if (credName === "") { - throw 'No credentials available for EC2 instance'; - } - resp = await ngx.fetch(EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT + credName, { - headers: { - 'x-aws-ec2-metadata-token': token, - }, - }); - const creds = await resp.json(); - - return { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.Token, - expiration: creds.Expiration, - }; -} - -/** - * Get the credentials by assuming calling AssumeRoleWithWebIdentity with the environment variable - * values ROLE_ARN, AWS_WEB_IDENTITY_TOKEN_FILE and HOSTNAME - * - * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>} - * @private - */ -async function _fetchWebIdentityCredentials(r) { - const arn = process.env['AWS_ROLE_ARN']; - const name = process.env['HOSTNAME'] || 'nginx-s3-gateway'; - - let sts_endpoint = process.env['STS_ENDPOINT']; - if (!sts_endpoint) { - /* On EKS, the ServiceAccount can be annotated with - 'eks.amazonaws.com/sts-regional-endpoints' to control - the usage of regional endpoints. We are using the same standard - environment variable here as the AWS SDK. This is with the exception - of replacing the value `legacy` with `global` to match what EKS sets - the variable to. - See: https://docs.aws.amazon.com/sdkref/latest/guide/feature-sts-regionalized-endpoints.html - See: https://docs.aws.amazon.com/eks/latest/userguide/configure-sts-endpoint.html */ - const sts_regional = process.env['AWS_STS_REGIONAL_ENDPOINTS'] || 'global'; - if (sts_regional === 'regional') { - /* STS regional endpoints can be derived from the region's name. - See: https://docs.aws.amazon.com/general/latest/gr/sts.html */ - const region = process.env['AWS_REGION']; - if (region) { - sts_endpoint = `https://sts.${region}.amazonaws.com`; - } else { - throw 'Missing required AWS_REGION env variable'; - } - } else { - // This is the default global endpoint - sts_endpoint = 'https://sts.amazonaws.com'; - } - } - - const token = fs.readFileSync(process.env['AWS_WEB_IDENTITY_TOKEN_FILE']); - - const params = `Version=2011-06-15&Action=AssumeRoleWithWebIdentity&RoleArn=${arn}&RoleSessionName=${name}&WebIdentityToken=${token}`; - - const response = await ngx.fetch(sts_endpoint + "?" + params, { - headers: { - "Accept": "application/json" - }, - method: 'GET', - }); - - const resp = await response.json(); - const creds = resp.AssumeRoleWithWebIdentityResponse.AssumeRoleWithWebIdentityResult.Credentials; - - return { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.SessionToken, - expiration: creds.Expiration, - }; -} export default { awsHeaderDate, - fetchCredentials, s3date, s3auth, s3uri, diff --git a/common/etc/nginx/include/utils.js b/common/etc/nginx/include/utils.js deleted file mode 100644 index 9ab5f4fe..00000000 --- a/common/etc/nginx/include/utils.js +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright 2023 F5, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Flag indicating debug mode operation. If true, additional information - * about signature generation will be logged. - * @type {boolean} - */ -const DEBUG = parseBoolean(process.env['S3_DEBUG']); - -/** - * Checks to see if all of the elements of the passed array are present as keys - * in the running process' environment variables. Alternatively, if a single - * string is passed, it will check for the presence of that string. - * @param envVars {array[string]|string} array of expected keys or single expected key - * @returns {boolean} true if all keys are set as environment variables - */ -function areAllEnvVarsSet(envVars) { - if (envVars instanceof Array) { - const envVarsLen = envVars.length; - for (let i = 0; i < envVarsLen; i++) { - if (!process.env[envVars[i]]) { - return false; - } - } - - return true; - } - - return envVars in process.env; -} - -/** - * Parses a string delimited by semicolons into an array of values - * @param string {string|null} value representing a array of strings - * @returns {Array} a list of values - */ -function parseArray(string) { - if (string == null || !string || string === ';') { - return []; - } - - // Exclude trailing delimiter - if (string.endsWith(';')) { - return string.substr(0, string.length - 1).split(';'); - } - - return string.split(';') -} - -/** - * Parses a string to and returns a boolean value based on its value. If the - * string can't be parsed, this method returns false. - * - * @param string {*} value representing a boolean - * @returns {boolean} boolean value of string - */ -function parseBoolean(string) { - switch(string) { - case "TRUE": - case "true": - case "True": - case "YES": - case "yes": - case "Yes": - case "1": - return true; - default: - return false; - } -} - -/** - * Outputs a log message to the request logger if debug messages are enabled. - * - * @param r {Request} HTTP request object - * @param msg {string} message to log - */ -function debug_log(r, msg) { - if (DEBUG && "log" in r) { - r.log(msg); - } -} - -/** - * Pads the supplied number with leading zeros. - * - * @param num {number|string} number to pad - * @param size number of leading zeros to pad - * @returns {string} a string with leading zeros - * @private - */ -function padWithLeadingZeros(num, size) { - const s = "0" + num; - return s.substr(s.length-size); -} - -/** - * Creates a string in the ISO601 date format (YYYYMMDD'T'HHMMSS'Z') based on - * the supplied timestamp and date. The date is not extracted from the timestamp - * because that operation is already done once during the signing process. - * - * @param timestamp {Date} timestamp to extract date from - * @param eightDigitDate {string} 'YYYYMMDD' format date string that was already extracted from timestamp - * @returns {string} string in the format of YYYYMMDD'T'HHMMSS'Z' - * @private - */ -function getAmzDatetime(timestamp, eightDigitDate) { - const hours = timestamp.getUTCHours(); - const minutes = timestamp.getUTCMinutes(); - const seconds = timestamp.getUTCSeconds(); - - return ''.concat( - eightDigitDate, - 'T', padWithLeadingZeros(hours, 2), - padWithLeadingZeros(minutes, 2), - padWithLeadingZeros(seconds, 2), - 'Z'); -} - -/** - * Formats a timestamp into a date string in the format 'YYYYMMDD'. - * - * @param timestamp {Date} timestamp - * @returns {string} a formatted date string based on the input timestamp - * @private - */ -function getEightDigitDate(timestamp) { - const year = timestamp.getUTCFullYear(); - const month = timestamp.getUTCMonth() + 1; - const day = timestamp.getUTCDate(); - - return ''.concat(padWithLeadingZeros(year, 4), - padWithLeadingZeros(month,2), - padWithLeadingZeros(day,2)); -} - -export default { - debug_log, - getAmzDatetime, - getEightDigitDate, - padWithLeadingZeros, - parseArray, - parseBoolean, - areAllEnvVarsSet -} diff --git a/common/etc/nginx/nginx.conf b/common/etc/nginx/nginx.conf index da3776bf..035f9093 100644 --- a/common/etc/nginx/nginx.conf +++ b/common/etc/nginx/nginx.conf @@ -9,9 +9,9 @@ load_module modules/ngx_http_js_module.so; load_module modules/ngx_http_xslt_filter_module.so; # Preserve S3 environment variables for worker threads -env S3_ACCESS_KEY_ID; -env S3_SECRET_KEY; -env S3_SESSION_TOKEN; +env AWS_ACCESS_KEY_ID; +env AWS_SECRET_ACCESS_KEY; +env AWS_SESSION_TOKEN; env S3_BUCKET_NAME; env S3_SERVER; env S3_SERVER_PORT; diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template index cf4cd605..f5afbefe 100644 --- a/common/etc/nginx/templates/default.conf.template +++ b/common/etc/nginx/templates/default.conf.template @@ -1,4 +1,4 @@ -js_import /etc/nginx/include/awscredentials.js; +js_import /etc/nginx/include/awssig/core/awscredentials.js; js_import /etc/nginx/include/s3gateway.js; # We include only the variables needed for the authentication signatures that @@ -88,7 +88,7 @@ server { location /aws/credentials/retrieve { internal; - js_content s3gateway.fetchCredentials; + js_content awscredentials.fetchCredentials; include /etc/nginx/conf.d/gateway/js_fetch_trusted_certificate.conf; } diff --git a/docs/development.md b/docs/development.md index e690be43..2c98f57f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -2,18 +2,24 @@ ## Integrating with AWS Signature +Run the script of [`submodule.sh`](../submodule.sh) to update the module of `nginx-aws-signature`. +- Common `nginx-aws-signature` lib: [`https://github.com/nginxinc/nginx-aws-signature`](https://github.com/nginxinc/nginx-aws-signature) +- Pull the latest version of `nginx-aws-signature` when updating new lib. + ``` + cd common/etc/nginx/include/awssig + git pull -f + ``` + Update the following files when enhancing `nginx-s3-gateway` to integrate with AWS signature whenever AWS releases a new version of signature or you have a new PR: - NGINX Proxy: [`/etc/nginx/conf.d/default.conf`](/common/etc/nginx/templates/gateway/s3_location.conf.template) -- AWS Credentials Lib: [`/etc/nginx/include/awscredentials.js`](/common/etc/nginx/include/awscredentials.js) - > Note: The `fetchCredentials()` is going to be part of here soon. - +- AWS Credentials Lib: [`/etc/nginx/include/awssig/core/awscredentials.js`](/common/etc/nginx/include/awssig/core/awscredentials.js) - AWS Signature Lib per version: - - [`/etc/nginx/include/awssig2.js`](/common/etc/nginx/include/awssig2.js) - - [`/etc/nginx/include/awssig4.js`](/common/etc/nginx/include/awssig4.js) + - [`/etc/nginx/include/awssig/core/awssig2.js`](/common/etc/nginx/include/awssig/core/awssig2.js) + - [`/etc/nginx/include/awssig/core/awssig4.js`](/common/etc/nginx/include/awssig/core/awssig4.js) - S3 Integration Lib: [`/etc/nginx/include/s3gateway.js`](/common/etc/nginx/include/s3gateway.js) -- Common Lib for all of NJS: [`/etc/nginx/include/utils.js`](/common/etc/nginx/include/utils.js) +- Common Lib for all of NJS: [`/etc/nginx/include/awssig/core/utils.js`](/common/etc/nginx/include/awssig/core/utils.js) ![](./img/nginx-s3-gateway-signature-flow.png) diff --git a/docs/getting_started.md b/docs/getting_started.md index 6704afca..0eba32de 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -17,9 +17,9 @@ running as a Container or as a Systemd service. | ------------------------------------- | --------- | ---------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ALLOW_DIRECTORY_LIST` | Yes | `true`, `false` | `false` | Flag enabling directory listing | | `AWS_SIGS_VERSION` | Yes | 2, 4 | | AWS Signatures API version | -| `S3_ACCESS_KEY_ID` | Yes | | | Access key | -| `S3_SECRET_KEY` | Yes | | | Secret access key | -| `S3_SESSION_TOKEN` | No | | | Session token. | +| `AWS_ACCESS_KEY_ID` | Yes | | | Access key | +| `AWS_SECRET_ACCESS_KEY` | Yes | | | Secret access key | +| `AWS_SESSION_TOKEN` | No | | | Session token. | | `S3_BUCKET_NAME` | Yes | | | Name of S3 bucket to proxy requests to | | `S3_REGION` | Yes | | | Region associated with API | | `S3_SERVER_PORT` | Yes | | | SSL/TLS port to connect to | @@ -40,7 +40,7 @@ running as a Container or as a Systemd service. If you are using [AWS instance profile credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html), -you will need to omit the `S3_ACCESS_KEY_ID`, `S3_SECRET_KEY` and `S3_SESSION_TOKEN` variables from +you will need to omit the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables from the configuration. When running with Docker, the above environment variables can be set in a file @@ -213,8 +213,8 @@ docker run --env-file ./settings --publish 80:80 --name nginx-plus-s3-gateway \ allow you to assign a role to a compute so that other AWS services can trust the instance without having to store authentication keys in the compute instance. This is useful for the gateway because it allows us to run the -gateway without storing an unchanging `S3_ACCESS_KEY_ID`, `S3_SECRET_KEY` and -`S3_SESSION_TOKEN` in a file on disk or in an easily read environment variable. +gateway without storing an unchanging `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and +`AWS_SESSION_TOKEN` in a file on disk or in an easily read environment variable. Instance profiles work by providing credentials to the instance via the [AWS Metadata API](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html). @@ -227,7 +227,7 @@ Following the [AWS documentation](https://docs.aws.amazon.com/AWSEC2/latest/User we can create a IAM role and launch an instance associated with it. On that instance, if we run the gateway as a Systemd service there are no additional steps. We just run the install script without specifying the -`S3_ACCESS_KEY_ID`, `S3_SECRET_KEY` and `S3_SESSION_TOKEN` environment variables. +`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` environment variables. However, if we want to run the gateway as a container instance on that EC2 instance, then we will need to run the following command using the AWS @@ -239,7 +239,7 @@ aws ec2 modify-instance-metadata-options --instance-id \ ``` After that has been run we can start the container normally and omit the -`S3_ACCESS_KEY_ID`, `S3_SECRET_KEY` and `S3_SESSION_TOKEN` environment variables. +`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` environment variables. ### Running in ECS with an IAM Policy diff --git a/settings.example b/settings.example index 9e55a4f8..4cefe719 100644 --- a/settings.example +++ b/settings.example @@ -1,7 +1,7 @@ S3_BUCKET_NAME=my-bucket -S3_ACCESS_KEY_ID=ZZZZZZZZZZZZZZZZZZZZ -S3_SECRET_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -S3_SESSION_TOKEN=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +AWS_ACCESS_KEY_ID=ZZZZZZZZZZZZZZZZZZZZ +AWS_SECRET_ACCESS_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +AWS_SESSION_TOKEN=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb S3_SERVER=s3-us-east-1.amazonaws.com S3_SERVER_PORT=443 S3_SERVER_PROTO=https diff --git a/standalone_ubuntu_oss_install.sh b/standalone_ubuntu_oss_install.sh index 7c4f4703..01929a6a 100644 --- a/standalone_ubuntu_oss_install.sh +++ b/standalone_ubuntu_oss_install.sh @@ -40,11 +40,11 @@ elif curl --output /dev/null --silent --head --fail --connect-timeout 2 "http:// echo "Running inside an EC2 instance, using IMDS for credentials" uses_iam_creds=1 else - required+=("S3_ACCESS_KEY_ID" "S3_SECRET_KEY") + required+=("AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY") uses_iam_creds=0 fi -if [[ -v S3_SESSION_TOKEN ]]; then +if [[ -v AWS_SESSION_TOKEN ]]; then echo "S3 Session token present" fi @@ -78,7 +78,7 @@ echo "Installing using github '${branch}' branch" echo "S3 Backend Environment" -echo "Access Key ID: ${S3_ACCESS_KEY_ID}" +echo "Access Key ID: ${AWS_ACCESS_KEY_ID}" echo "Origin: ${S3_SERVER_PROTO}://${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}" echo "Region: ${S3_REGION}" echo "Addressing Style: ${S3_STYLE}" @@ -187,14 +187,14 @@ EOF if [ $uses_iam_creds -eq 0 ]; then cat >> "/etc/nginx/environment" << EOF # AWS Access key -S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID} +AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} # AWS Secret access key -S3_SECRET_KEY=${S3_SECRET_KEY} +AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} EOF - if [[ -v S3_SESSION_TOKEN ]]; then + if [[ -v AWS_SESSION_TOKEN ]]; then cat >> "/etc/nginx/environment" << EOF # AWS Session Token -S3_SESSION_TOKEN=${S3_SESSION_TOKEN} +AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} EOF fi fi @@ -293,12 +293,12 @@ EOF # to obtain S3 permissions. if [ $uses_iam_creds -eq 0 ]; then cat >> "/etc/nginx/environment" << EOF -env S3_ACCESS_KEY_ID; -env S3_SECRET_KEY; +env AWS_ACCESS_KEY_ID; +env AWS_SECRET_ACCESS_KEY; EOF - if [[ -v S3_SESSION_TOKEN ]]; then + if [[ -v AWS_SESSION_TOKEN ]]; then cat >> "/etc/nginx/environment" << EOF -env S3_SESSION_TOKEN; +env AWS_SESSION_TOKEN; EOF fi fi @@ -349,11 +349,11 @@ http { EOF download "common/etc/nginx/include/listing.xsl" "/etc/nginx/include/listing.xsl" -download "common/etc/nginx/include/awscredentials.js" "/etc/nginx/include/awscredentials.js" -download "common/etc/nginx/include/awssig2.js" "/etc/nginx/include/awssig2.js" -download "common/etc/nginx/include/awssig4.js" "/etc/nginx/include/awssig4.js" +download "common/etc/nginx/include/awssig/core/awscredentials.js" "/etc/nginx/include/awssig/core/awscredentials.js" +download "common/etc/nginx/include/awssig/core/awssig2.js" "/etc/nginx/include/awssig/core/awssig2.js" +download "common/etc/nginx/include/awssig/core/awssig4.js" "/etc/nginx/include/awssig/core/awssig4.js" download "common/etc/nginx/include/s3gateway.js" "/etc/nginx/include/s3gateway.js" -download "common/etc/nginx/include/utils.js" "/etc/nginx/include/utils.js" +download "common/etc/nginx/include/awssig/core/utils.js" "/etc/nginx/include/awssig/core/utils.js" download "common/etc/nginx/templates/default.conf.template" "/etc/nginx/templates/default.conf.template" download "common/etc/nginx/templates/gateway/v2_headers.conf.template" "/etc/nginx/templates/gateway/v2_headers.conf.template" download "common/etc/nginx/templates/gateway/v2_js_vars.conf.template" "/etc/nginx/templates/gateway/v2_js_vars.conf.template" diff --git a/submodule.sh b/submodule.sh new file mode 100755 index 00000000..e2c5e031 --- /dev/null +++ b/submodule.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +git submodule update --init common/etc/nginx/include/awssig +git submodule absorbgitdirs +git -C common/etc/nginx/include/awssig config core.sparseCheckout true +echo 'core/*' >>.git/modules/common/etc/nginx/include/awssig/info/sparse-checkout +git submodule update --force --checkout common/etc/nginx/include/awssig diff --git a/test.sh b/test.sh index ca841707..b2006db6 100755 --- a/test.sh +++ b/test.sh @@ -276,8 +276,8 @@ runUnitTestWithOutSessionToken() { --workdir /var/tmp \ -e "S3_DEBUG=true" \ -e "S3_STYLE=virtual" \ - -e "S3_ACCESS_KEY_ID=unit_test" \ - -e "S3_SECRET_KEY=unit_test" \ + -e "AWS_ACCESS_KEY_ID=unit_test" \ + -e "AWS_SECRET_ACCESS_KEY=unit_test" \ -e "S3_BUCKET_NAME=unit_test" \ -e "S3_SERVER=unit_test" \ -e "S3_SERVER_PROTO=https" \ @@ -299,9 +299,9 @@ runUnitTestWithSessionToken() { --workdir /var/tmp \ -e "S3_DEBUG=true" \ -e "S3_STYLE=virtual" \ - -e "S3_ACCESS_KEY_ID=unit_test" \ - -e "S3_SECRET_KEY=unit_test" \ - -e "S3_SESSION_TOKEN=unit_test" \ + -e "AWS_ACCESS_KEY_ID=unit_test" \ + -e "AWS_SECRET_ACCESS_KEY=unit_test" \ + -e "AWS_SESSION_TOKEN=unit_test" \ -e "S3_BUCKET_NAME=unit_test" \ -e "S3_SERVER=unit_test" \ -e "S3_SERVER_PROTO=https" \ @@ -312,19 +312,10 @@ runUnitTestWithSessionToken() { nginx-s3-gateway -t module -p '/etc/nginx' /var/tmp/"${test_code}" } -p "Running unit tests for utils" -runUnitTestWithSessionToken "utils_test.js" - p "Running unit tests with an access key ID and a secret key in Docker image" -runUnitTestWithOutSessionToken "awscredentials_test.js" -runUnitTestWithOutSessionToken "awssig2_test.js" -runUnitTestWithOutSessionToken "awssig4_test.js" runUnitTestWithOutSessionToken "s3gateway_test.js" p "Running unit tests with an session token in Docker image" -runUnitTestWithSessionToken "awscredentials_test.js" -runUnitTestWithSessionToken "awssig2_test.js" -runUnitTestWithSessionToken "awssig4_test.js" runUnitTestWithSessionToken "s3gateway_test.js" ### INTEGRATION TESTS diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index 8f088013..ffe09fff 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -15,8 +15,8 @@ services: restart: "no" environment: S3_BUCKET_NAME: "bucket-1" - S3_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" - S3_SECRET_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" + AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" S3_SERVER: "minio" S3_SERVER_PORT: "9000" S3_SERVER_PROTO: "http" diff --git a/test/unit/awscredentials_test.js b/test/unit/awscredentials_test.js deleted file mode 100644 index 9a5a6d16..00000000 --- a/test/unit/awscredentials_test.js +++ /dev/null @@ -1,191 +0,0 @@ -#!env njs - -/* - * Copyright 2023 F5, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import awscred from "include/awscredentials.js"; -import fs from "fs"; - - -function testReadCredentialsWithAccessSecretKeyAndSessionTokenSet() { - printHeader('testReadCredentialsWithAccessSecretKeyAndSessionTokenSet'); - let r = {}; - process.env['S3_ACCESS_KEY_ID'] = 'SOME_ACCESS_KEY'; - process.env['S3_SECRET_KEY'] = 'SOME_SECRET_KEY'; - if ('S3_SESSION_TOKEN' in process.env) { - process.env['S3_SESSION_TOKEN'] = 'SOME_SESSION_TOKEN'; - } - - try { - var credentials = awscred.readCredentials(r); - if (credentials.accessKeyId !== process.env['S3_ACCESS_KEY_ID']) { - throw 'static credentials do not match returned value [accessKeyId]'; - } - if (credentials.secretAccessKey !== process.env['S3_SECRET_KEY']) { - throw 'static credentials do not match returned value [secretAccessKey]'; - } - if ('S3_SESSION_TOKEN' in process.env) { - if (credentials.sessionToken !== process.env['S3_SESSION_TOKEN']) { - throw 'static credentials do not match returned value [sessionToken]'; - } - } else { - if (credentials.sessionToken !== null) { - throw 'static credentials do not match returned value [sessionToken]'; - } - } - if (credentials.expiration !== null) { - throw 'static credentials do not match returned value [expiration]'; - } - - } finally { - delete process.env.S3_ACCESS_KEY_ID; - delete process.env.S3_SECRET_KEY; - if ('S3_SESSION_TOKEN' in process.env) { - delete process.env.S3_SESSION_TOKEN; - } - } -} - -function testReadCredentialsFromFilePath() { - printHeader('testReadCredentialsFromFilePath'); - let r = { - variables: { - cache_instance_credentials_enabled: 0 - } - }; - - var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE']; - var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp'); - var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`; - var tempFile = `${tempDir}/credentials-unit-test-${uniqId}.json`; - var testData = '{"accessKeyId":"A","secretAccessKey":"B",' + - '"sessionToken":"C","expiration":"2022-02-15T04:49:08Z"}'; - fs.writeFileSync(tempFile, testData); - - try { - process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile; - var credentials = awscred.readCredentials(r); - var testDataAsJSON = JSON.parse(testData); - if (credentials.accessKeyId !== testDataAsJSON.accessKeyId) { - throw 'JSON test data does not match credentials [accessKeyId]'; - } - if (credentials.secretAccessKey !== testDataAsJSON.secretAccessKey) { - throw 'JSON test data does not match credentials [secretAccessKey]'; - } - if (credentials.sessionToken !== testDataAsJSON.sessionToken) { - throw 'JSON test data does not match credentials [sessionToken]'; - } - if (credentials.expiration !== testDataAsJSON.expiration) { - throw 'JSON test data does not match credentials [expiration]'; - } - } finally { - if (originalCredentialPath) { - process.env['S3_CREDENTIALS_TEMP_FILE'] = originalCredentialPath; - } - if (fs.statSync(tempFile, {throwIfNoEntry: false})) { - fs.unlinkSync(tempFile); - } - } -} - -function testReadCredentialsFromNonexistentPath() { - printHeader('testReadCredentialsFromNonexistentPath'); - let r = { - variables: { - cache_instance_credentials_enabled: 0 - } - }; - var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE']; - var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp'); - var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`; - var tempFile = `${tempDir}/credentials-unit-test-${uniqId}.json`; - - try { - process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile; - var credentials = awscred.readCredentials(r); - if (credentials !== undefined) { - throw 'Credentials returned when no credentials file should be present'; - } - - } finally { - if (originalCredentialPath) { - process.env['S3_CREDENTIALS_TEMP_FILE'] = originalCredentialPath; - } - if (fs.statSync(tempFile, {throwIfNoEntry: false})) { - fs.unlinkSync(tempFile); - } - } -} - -function testReadAndWriteCredentialsFromKeyValStore() { - printHeader('testReadAndWriteCredentialsFromKeyValStore'); - - let accessKeyId = process.env['S3_ACCESS_KEY_ID']; - let secretKey = process.env['S3_SECRET_KEY']; - let sessionToken = null; - if ('S3_SESSION_TOKEN' in process.env) { - sessionToken = process.env['S3_SESSION_TOKEN']; - } - delete process.env.S3_ACCESS_KEY_ID; - delete process.env.S3_SECRET_KEY; - if ('S3_SESSION_TOKEN' in process.env) { - delete process.env.S3_SESSION_TOKEN - } - try { - let r = { - variables: { - cache_instance_credentials_enabled: 1, - instance_credential_json: null - } - }; - let expectedCredentials = { - AccessKeyId: 'AN_ACCESS_KEY_ID', - Expiration: '2017-05-17T15:09:54Z', - RoleArn: 'TASK_ROLE_ARN', - SecretAccessKey: 'A_SECRET_ACCESS_KEY', - Token: 'A_SECURITY_TOKEN', - }; - - awscred.writeCredentials(r, expectedCredentials); - let credentials = JSON.stringify(awscred.readCredentials(r)); - let expectedJson = JSON.stringify(expectedCredentials); - - if (credentials !== expectedJson) { - console.log(`EXPECTED:\n${expectedJson}\nACTUAL:\n${credentials}`); - throw 'Credentials do not match expected value'; - } - } finally { - process.env['S3_ACCESS_KEY_ID'] = accessKeyId; - process.env['S3_SECRET_KEY'] = secretKey; - if ('S3_SESSION_TOKEN' in process.env) { - process.env['S3_SESSION_TOKEN'] = sessionToken - } - } -} - -async function test() { - testReadCredentialsWithAccessSecretKeyAndSessionTokenSet(); - testReadCredentialsFromFilePath(); - testReadCredentialsFromNonexistentPath(); - testReadAndWriteCredentialsFromKeyValStore(); -} - -function printHeader(testName) { - console.log(`\n## ${testName}`); -} - -test(); -console.log('Finished unit tests for awscredentials.js'); diff --git a/test/unit/awssig2_test.js b/test/unit/awssig2_test.js deleted file mode 100644 index 67b69080..00000000 --- a/test/unit/awssig2_test.js +++ /dev/null @@ -1,91 +0,0 @@ -#!env njs - -/* - * Copyright 2023 F5, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import awssig2 from "include/awssig2.js"; -import s3gateway from "include/s3gateway.js"; - - -function _runSignatureV2(r) { - r.log = function(msg) { - console.log(msg); - } - const timestamp = new Date('2020-08-11T19:42:14Z'); - const bucket = 'test-bucket-1'; - const accessKey = 'test-access-key-1'; - const secret = 'pvgoBEA1z7zZKqN9RoKVksKh31AtNou+pspn+iyb' - const creds = { - accessKeyId:accessKey, secretAccessKey: secret, sessionToken: null - }; - - // TODO: Generate request parameters without using s3gateway to only test - // awssig2.js for the purpose of common library. - const httpDate = timestamp.toUTCString(); - const expected = 'AWS test-access-key-1:VviSS4cFhUC6eoB4CYqtRawzDrc='; - let req = s3gateway._s3ReqParamsForSigV2(r, bucket); - let signature = awssig2.signatureV2(r, req.uri, httpDate, creds); - - if (signature !== expected) { - throw 'V2 signature hash was not created correctly.\n' + - 'Actual: [' + signature + ']\n' + - 'Expected: [' + expected + ']'; - } -} - -function testSignatureV2() { - printHeader('testSignatureV2'); - // Note: since this is a read-only gateway, host, query parameters and all - // client headers will be ignored. - var r = { - "remoteAddress" : "172.17.0.1", - "headersIn" : { - "Connection" : "keep-alive", - "Accept-Encoding" : "gzip, deflate", - "Accept-Language" : "en-US,en;q=0.7,ja;q=0.3", - "Host" : "localhost:8999", - "User-Agent" : "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:79.0) Gecko/20100101 Firefox/79.0", - "DNT" : "1", - "Cache-Control" : "max-age=0", - "Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Upgrade-Insecure-Requests" : "1" - }, - "uri" : "/a/c/ramen.jpg", - "method" : "GET", - "httpVersion" : "1.1", - "headersOut" : {}, - "args" : { - "foo" : "bar" - }, - "variables" : { - "uri_path": "/a/c/ramen.jpg" - }, - "status" : 0 - }; - - _runSignatureV2(r); -} - -async function test() { - testSignatureV2(); -} - -function printHeader(testName) { - console.log(`\n## ${testName}`); -} - -test(); -console.log('Finished unit tests for awssig2.js'); diff --git a/test/unit/awssig4_test.js b/test/unit/awssig4_test.js deleted file mode 100644 index 0c1677a1..00000000 --- a/test/unit/awssig4_test.js +++ /dev/null @@ -1,172 +0,0 @@ -#!env njs - -/* - * Copyright 2023 F5, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import awssig4 from "include/awssig4.js"; -import s3gateway from "include/s3gateway.js"; -import utils from "include/utils.js"; - - -function testBuildSigningKeyHashWithReferenceInputs() { - printHeader('testBuildSigningKeyHashWithReferenceInputs'); - var kSecret = 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY'; - var date = '20150830'; - var service = 'iam'; - var region = 'us-east-1'; - var expected = 'c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9'; - var signingKeyHash = awssig4._buildSigningKeyHash(kSecret, date, region, service).toString('hex'); - - if (signingKeyHash !== expected) { - throw 'Signing key hash was not created correctly.\n' + - 'Actual: [' + signingKeyHash + ']\n' + - 'Expected: [' + expected + ']'; - } -} - -function testBuildSigningKeyHashWithTestSuiteInputs() { - printHeader('testBuildSigningKeyHashWithTestSuiteInputs'); - var kSecret = 'pvgoBEA1z7zZKqN9RoKVksKh31AtNou+pspn+iyb'; - var date = '20200811'; - var service = 's3'; - var region = 'us-west-2'; - var expected = 'a48701bfe803103e89051f55af2297dd76783bbceb5eb416dab71e0eadcbc4f6'; - var signingKeyHash = awssig4._buildSigningKeyHash(kSecret, date, region, service).toString('hex'); - - if (signingKeyHash !== expected) { - throw 'Signing key hash was not created correctly.\n' + - 'Actual: [' + signingKeyHash + ']\n' + - 'Expected: [' + expected + ']'; - } -} - -function _runSignatureV4(r) { - r.log = function(msg) { - console.log(msg); - } - var timestamp = new Date('2020-08-11T19:42:14Z'); - var eightDigitDate = utils.getEightDigitDate(timestamp); - var amzDatetime = utils.getAmzDatetime(timestamp, eightDigitDate); - var bucket = 'ez-test-bucket-1' - var secret = 'pvgoBEA1z7zZKqN9RoKVksKh31AtNou+pspn+iyb' - var creds = {secretAccessKey: secret, sessionToken: null}; - var region = 'us-west-2'; - var service = 's3'; - var server = 's3-us-west-2.amazonaws.com'; - - // TODO: Generate request parameters without using s3gateway to only test - // awssig4.js for the purpose of common library. - let req = s3gateway._s3ReqParamsForSigV4(r, bucket, server); - const canonicalRequest = awssig4._buildCanonicalRequest( - r.method, req.uri, req.queryParams, req.host, amzDatetime, creds.sessionToken); - - var expected = 'cf4dd9e1d28c74e2284f938011efc8230d0c20704f56f67e4a3bfc2212026bec'; - var signature = awssig4._buildSignatureV4( - r, amzDatetime, eightDigitDate, creds, region, service, canonicalRequest); - - if (signature !== expected) { - throw 'V4 signature hash was not created correctly.\n' + - 'Actual: [' + signature + ']\n' + - 'Expected: [' + expected + ']'; - } -} - -function testSignatureV4() { - printHeader('testSignatureV4'); - // Note: since this is a read-only gateway, host, query parameters and all - // client headers will be ignored. - var r = { - "remoteAddress" : "172.17.0.1", - "headersIn" : { - "Connection" : "keep-alive", - "Accept-Encoding" : "gzip, deflate", - "Accept-Language" : "en-US,en;q=0.7,ja;q=0.3", - "Host" : "localhost:8999", - "User-Agent" : "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:79.0) Gecko/20100101 Firefox/79.0", - "DNT" : "1", - "Cache-Control" : "max-age=0", - "Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Upgrade-Insecure-Requests" : "1" - }, - "uri" : "/a/c/ramen.jpg", - "method" : "GET", - "httpVersion" : "1.1", - "headersOut" : {}, - "args" : { - "foo" : "bar" - }, - "variables" : { - "uri_path": "/a/c/ramen.jpg" - }, - "status" : 0 - }; - - _runSignatureV4(r); -} - -function testSignatureV4Cache() { - printHeader('testSignatureV4Cache'); - // Note: since this is a read-only gateway, host, query parameters and all - // client headers will be ignored. - var r = { - "remoteAddress" : "172.17.0.1", - "headersIn" : { - "Connection" : "keep-alive", - "Accept-Encoding" : "gzip, deflate", - "Accept-Language" : "en-US,en;q=0.7,ja;q=0.3", - "Host" : "localhost:8999", - "User-Agent" : "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:79.0) Gecko/20100101 Firefox/79.0", - "DNT" : "1", - "Cache-Control" : "max-age=0", - "Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Upgrade-Insecure-Requests" : "1" - }, - "uri" : "/a/c/ramen.jpg", - "method" : "GET", - "httpVersion" : "1.1", - "headersOut" : {}, - "args" : { - "foo" : "bar" - }, - "variables": { - "cache_signing_key_enabled": 1, - "uri_path": "/a/c/ramen.jpg" - }, - "status" : 0 - }; - - _runSignatureV4(r); - - if (!"signing_key_hash" in r.variables) { - throw "Hash key not written to r.variables.signing_key_hash"; - } - - _runSignatureV4(r); -} - -async function test() { - testBuildSigningKeyHashWithReferenceInputs(); - testBuildSigningKeyHashWithTestSuiteInputs(); - testSignatureV4(); - testSignatureV4Cache(); -} - -function printHeader(testName) { - console.log(`\n## ${testName}`); -} - -test(); -console.log('Finished unit tests for awssig4.js'); diff --git a/test/unit/s3gateway_test.js b/test/unit/s3gateway_test.js index 2cd19154..44246f8a 100755 --- a/test/unit/s3gateway_test.js +++ b/test/unit/s3gateway_test.js @@ -16,7 +16,8 @@ * limitations under the License. */ -import awssig4 from "include/awssig4.js"; +import awscred from "include/awssig/core/awscredentials.js" +import awssig4 from "include/awssig/core/awssig4.js"; import s3gateway from "include/s3gateway.js"; globalThis.ngx = {}; @@ -204,7 +205,7 @@ function testEscapeURIPathPreservesDoubleSlashes() { async function testEcsCredentialRetrieval() { printHeader('testEcsCredentialRetrieval'); - delete process.env['S3_ACCESS_KEY_ID']; + delete process.env['AWS_ACCESS_KEY_ID']; process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = '/example'; globalThis.ngx.fetch = function (url) { console.log(' fetching mock credentials'); @@ -243,7 +244,7 @@ async function testEcsCredentialRetrieval() { }, }; - await s3gateway.fetchCredentials(r); + await awscred.fetchCredentials(r); if (globalThis.recordedUrl !== 'http://169.254.170.2/example') { throw `No or wrong ECS credentials fetch URL recorded: ${globalThis.recordedUrl}`; @@ -252,7 +253,7 @@ async function testEcsCredentialRetrieval() { async function testEc2CredentialRetrieval() { printHeader('testEc2CredentialRetrieval'); - delete process.env['S3_ACCESS_KEY_ID']; + delete process.env['AWS_ACCESS_KEY_ID']; delete process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI']; globalThis.ngx.fetch = function (url, options) { if (url === 'http://169.254.169.254/latest/api/token' && options && options.method === 'PUT') { @@ -315,7 +316,7 @@ async function testEc2CredentialRetrieval() { }, }; - await s3gateway.fetchCredentials(r); + await awscred.fetchCredentials(r); if (!globalThis.credentialsIssued) { throw 'Did not reach the point where EC2 credentials were issues.'; diff --git a/test/unit/utils_test.js b/test/unit/utils_test.js deleted file mode 100644 index 709f7149..00000000 --- a/test/unit/utils_test.js +++ /dev/null @@ -1,197 +0,0 @@ -#!env njs - -/* - * Copyright 2023 F5, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import utils from "include/utils.js"; - -function testParseArray() { - printHeader('testParseArray'); - - function testParseNull() { - console.log(' ## testParseNull'); - const actual = utils.parseArray(null); - if (!Array.isArray(actual) || actual.length > 0) { - throw 'Null not parsed into an empty array'; - } - } - function testParseEmptyString() { - console.log(' ## testParseEmptyString'); - const actual = utils.parseArray(''); - if (!Array.isArray(actual) || actual.length > 0) { - throw 'Empty string not parsed into an empty array'; - } - } - function testParseSingleValue() { - console.log(' ## testParseSingleValue'); - const value = 'Single Value'; - const actual = utils.parseArray(value); - if (!Array.isArray(actual) || actual.length !== 1) { - throw 'Single value not parsed into an array with a single element'; - } - if (actual[0] !== value) { - throw `Unexpected array element: ${actual[0]}` - } - } - function testParseMultipleValues() { - console.log(' ## testParseMultipleValues'); - const values = ['string 1', 'something else', 'Yet another value']; - const textValues = values.join(';'); - const actual = utils.parseArray(textValues); - if (!Array.isArray(actual) || actual.length !== values.length) { - throw 'Multiple values not parsed into an array with the expected length'; - } - for (let i = 0; i < values.length; i++) { - if (values[i] !== actual[i]) { - throw `Unexpected array element [${i}]: ${actual[i]}` - } - } - } - - function testParseMultipleValuesTrailingDelimiter() { - console.log(' ## testParseMultipleValuesTrailingDelimiter'); - const values = ['string 1', 'something else', 'Yet another value']; - const textValues = values.join(';'); - const actual = utils.parseArray(textValues + ';'); - if (!Array.isArray(actual) || actual.length !== values.length) { - throw 'Multiple values not parsed into an array with the expected length'; - } - for (let i = 0; i < values.length; i++) { - if (values[i] !== actual[i]) { - throw `Unexpected array element [${i}]: ${actual[i]}` - } - } - } - - testParseNull(); - testParseEmptyString(); - testParseSingleValue(); - testParseMultipleValues(); - testParseMultipleValuesTrailingDelimiter(); -} - -function testAmzDatetime() { - printHeader('testAmzDatetime'); - var timestamp = new Date('2020-08-03T02:01:09.004Z'); - var eightDigitDate = utils.getEightDigitDate(timestamp); - var amzDatetime = utils.getAmzDatetime(timestamp, eightDigitDate); - var expected = '20200803T020109Z'; - - if (amzDatetime !== expected) { - throw 'Amazon date time was not created correctly.\n' + - 'Actual: [' + amzDatetime + ']\n' + - 'Expected: [' + expected + ']'; - } -} - -function testEightDigitDate() { - printHeader('testEightDigitDate'); - var timestamp = new Date('2020-08-03T02:01:09.004Z'); - var eightDigitDate = utils.getEightDigitDate(timestamp); - var expected = '20200803'; - - if (eightDigitDate !== expected) { - throw 'Eight digit date was not created correctly.\n' + - 'Actual: ' + eightDigitDate + '\n' + - 'Expected: ' + expected; - } -} - -function testPad() { - printHeader('testPad'); - var padSingleDigit = utils.padWithLeadingZeros(3, 2); - var expected = '03'; - - if (padSingleDigit !== expected) { - throw 'Single digit 3 was not padded with leading zero.\n' + - 'Actual: ' + padSingleDigit + '\n' + - 'Expected: ' + expected; - } -} - -function testAreAllEnvVarsSet() { - function testAreAllEnvVarsSetStringFound() { - console.log(' ## testAreAllEnvVarsSetStringFound'); - const key = 'TEST_ENV_VAR_KEY'; - process.env[key] = 'some value'; - const actual = utils.areAllEnvVarsSet(key); - if (!actual) { - throw 'Environment variable that was set not indicated as present'; - } - } - - function testAreAllEnvVarsSetStringNotFound() { - console.log(' ## testAreAllEnvVarsSetStringNotFound'); - const actual = utils.areAllEnvVarsSet('UNKNOWN_ENV_VAR_KEY'); - if (actual) { - throw 'Unknown environment variable indicated as being present'; - } - } - - function testAreAllEnvVarsSetStringArrayFound() { - console.log(' ## testAreAllEnvVarsSetStringArrayFound'); - const keys = ['TEST_ENV_VAR_KEY_1', 'TEST_ENV_VAR_KEY_2', 'TEST_ENV_VAR_KEY_3']; - for (let i = 0; i < keys.length; i++) { - process.env[keys[i]] = 'something'; - } - const actual = utils.areAllEnvVarsSet(keys); - if (!actual) { - throw 'Environment variables that were set not indicated as present'; - } - } - - function testAreAllEnvVarsSetStringArrayNotFound() { - console.log(' ## testAreAllEnvVarsSetStringArrayNotFound'); - const keys = ['UNKNOWN_ENV_VAR_KEY_1', 'UNKNOWN_ENV_VAR_KEY_2', 'UNKNOWN_ENV_VAR_KEY_3']; - const actual = utils.areAllEnvVarsSet(keys); - if (actual) { - throw 'Unknown environment variables that were not set indicated as present'; - } - } - - function testAreAllEnvVarsSetStringArrayWithSomeSet() { - console.log(' ## testAreAllEnvVarsSetStringArrayWithSomeSet'); - const keys = ['TEST_ENV_VAR_KEY_1', 'UNKNOWN_ENV_VAR_KEY_2', 'UNKNOWN_ENV_VAR_KEY_3']; - process.env[keys[0]] = 'something'; - - const actual = utils.areAllEnvVarsSet(keys); - if (actual) { - throw 'Unknown environment variables that were not set indicated as present'; - } - } - - printHeader('testAreAllEnvVarsSet'); - testAreAllEnvVarsSetStringFound(); - testAreAllEnvVarsSetStringNotFound(); - testAreAllEnvVarsSetStringArrayFound(); - testAreAllEnvVarsSetStringArrayNotFound(); - testAreAllEnvVarsSetStringArrayWithSomeSet(); -} - -async function test() { - testAmzDatetime(); - testEightDigitDate(); - testPad(); - testParseArray(); - testAreAllEnvVarsSet(); -} - -function printHeader(testName) { - console.log(`\n## ${testName}`); -} - -test(); -console.log('Finished unit tests for utils.js');