From 45159eb5e182fb21ffa77bda5b291d6af1965c55 Mon Sep 17 00:00:00 2001 From: duanlv Date: Thu, 18 Jan 2024 16:32:16 +0800 Subject: [PATCH] refactor: refactor configuration --- .prettierrc | 4 ++ README.md | 37 ++++++++++-- __tests__/aws.spec.ts | 8 +-- package.json | 4 +- src/awos.ts | 134 ------------------------------------------ src/aws.ts | 59 ++++++------------- src/client.ts | 36 +++++++----- src/index.ts | 46 ++++----------- src/oss.ts | 28 ++------- src/types.ts | 8 +++ 10 files changed, 107 insertions(+), 257 deletions(-) create mode 100644 .prettierrc delete mode 100644 src/awos.ts diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..1502887 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "singleQuote": true, + "trailingComma": "es5" +} \ No newline at end of file diff --git a/README.md b/README.md index 08ca9b2..912ab14 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,18 @@ const AWOS = require('awos-js') ### for Aliyun OSS ```javascript +import { build } from 'awos-js' + +// for v3.*.* +const client = build({ + storageType: 'oss', + accessKeyID: "xxx", + accessKeySecret: "xxx", + bucket: "my_bucket", + endpoint: 'endpoint', +}) + +// for v2.*.* const client = new AWOS.Client({ type: 'oss', ossOptions: { @@ -43,11 +55,26 @@ const client = new AWOS.Client({ }) ``` -### for Amazon S3(minio) +### for AWS-S3 / MINIO ```javascript +import { build } from 'awos-js' + +// for v3.*.* +const client = build({ + storageType: 'aws', + accessKeyID: "xxx", + accessKeySecret: "xxx", + // when use aws s3, endpoint is unnecessary and region must be set + endpoint: "https://xxxx.myminio.com", + bucket: "my_bucket", + // when use minio, S3ForcePathStyle must be set true + s3ForcePathStyle: true, +}) + +// For v2.*.* const client = new AWOS.Client({ - type: 'aws', + storageType: 'aws', awsOptions: { accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', @@ -77,6 +104,9 @@ copy(key: string, source: string, options?: ICopyObjectOptions): Promise; ``` ### Change Log +- v3.0.0 / 2024-01-18 + - **[Breaking]** refactor configuration options ⚠️ + - support common prefix configuration - v2.0.0 / 2020-06-18 - Breaking @@ -94,6 +124,3 @@ copy(key: string, source: string, options?: ICopyObjectOptions): Promise; - v1.0.1 / 2019-03-19 - bug fix: oss listObject() should return [] when options.prefix not exist in the bucket; oss listObject() maxKeys not working - - - diff --git a/__tests__/aws.spec.ts b/__tests__/aws.spec.ts index faf0002..f86d420 100644 --- a/__tests__/aws.spec.ts +++ b/__tests__/aws.spec.ts @@ -7,10 +7,10 @@ const _ = require('lodash'); const prefix = 'test-awos-multi'; const client = new AWS({ - accessKeyId: process.env.AWS_ID!, - secretAccessKey: process.env.AWS_SECRET!, + accessKeyID: process.env.AWS_ID!, + accessKeySecret: process.env.AWS_SECRET!, bucket: process.env.AWS_BUCKET!, - endpoint: process.env.ENDPOINT, + endpoint: process.env.ENDPOINT!, s3ForcePathStyle: true, prefix, }); @@ -90,7 +90,7 @@ it('should copy() works fine', async () => { }); it('should get() works fine', async () => { - const res = await client.get(key, ['length']) as IGetObjectResponse; + const res = (await client.get(key, ['length'])) as IGetObjectResponse; expect(res.content).toEqual(content); expect(res.meta.get('length')).toEqual(String(content.length)); expect(res.headers['content-type']).toEqual(contentType); diff --git a/package.json b/package.json index 80925ac..4d4a8a8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "awos-js", - "version": "2.1.4", + "version": "3.0.0", "description": "AWOS: Wrapper For OSS And AWS(MINIO) SDK", "main": "./lib/index.js", "typings": "./lib/index.d.ts", @@ -9,7 +9,7 @@ ], "scripts": { "clean": "rimraf lib", - "format": "prettier --write \"{src,__tests__}/**/*.ts\" --single-quote --trailing-comma es5", + "format": "prettier --write \"{src,__tests__}/**/*.ts\"", "lint": "tslint --force --format verbose \"src/**/*.ts\"", "prebuild": "npm run clean && npm run format && npm run lint && echo Using TypeScript && tsc --version", "prepublish": "npm run build", diff --git a/src/awos.ts b/src/awos.ts deleted file mode 100644 index 753cd20..0000000 --- a/src/awos.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { - IGetObjectResponse, - IListObjectOptions, - IListObjectV2Options, - ISignatureUrlOptions, - IGetBufferedObjectResponse, - IPutObjectOptions, - IListObjectOutput, - IListObjectV2Output, - ICopyObjectOptions, - IHeadOptions, -} from './types'; -import OSS, { IOSSOptions } from './oss'; -import AWS, { IAWSOptions } from './aws'; - -import * as _ from 'lodash'; -import { AbstractClient } from './client'; - -const assert = require('assert'); - -export interface IOptions { - type: 'oss' | 'aws'; - prefix?: string; - ossOptions?: Exclude; - awsOptions?: Exclude; -} - -export default class AWOS { - private client: AbstractClient; - - constructor(options: IOptions) { - assert(options.type, 'options.type is required!'); - - const { prefix } = options; - if (options.type === 'oss' && options.ossOptions) { - this.client = new OSS({ - ...options.ossOptions!, - prefix, - }); - } else if (options.type === 'aws' && options.awsOptions) { - this.client = new AWS({ - ...options.awsOptions, - prefix, - }); - } else { - throw Error('invalid options!'); - } - } - - public async get( - key: string, - metaKeys: string[] = [] - ): Promise { - return this.client.get(key, metaKeys); - } - - public async getAsBuffer( - key: string, - metaKeys: string[] = [] - ): Promise { - return this.client.getAsBuffer(key, metaKeys); - } - - public async put( - key: string, - data: string | Buffer, - options?: IPutObjectOptions - ): Promise { - return this.client.put(key, data, options); - } - - public async copy( - key: string, - source: string, - options?: ICopyObjectOptions - ): Promise { - return this.client.copy(key, source, options); - } - - public async del(key: string): Promise { - return this.client.del(key); - } - - public async delMulti(keys: string[]): Promise { - if (keys.length > 1000) { - throw new Error('Cannot delete more than 1000 keys'); - } - return this.client.delMulti(keys); - } - - // 标准响应头包括 'content-type','content-length','accept-ranges','etag','last-modified' - // 其中 last-modified 统一返回毫秒级时间戳 - public async head( - key: string, - options?: IHeadOptions - ): Promise | null> { - return this.client.head(key, options); - } - - public async signatureUrl( - key: string, - _options?: ISignatureUrlOptions - ): Promise { - return this.client.signatureUrl(key, _options); - } - - public async listObject( - key: string, - options?: IListObjectOptions | undefined - ): Promise { - return this.client.listObject(key, options); - } - - public async listDetails( - key: string, - options?: IListObjectOptions - ): Promise { - return this.client.listDetails(key, options); - } - - public async listObjectV2( - key: string, - options?: IListObjectV2Options | undefined - ): Promise { - return this.client.listObjectV2(key, options); - } - - public async listDetailsV2( - key: string, - options?: IListObjectV2Options - ): Promise { - return this.client.listDetailsV2(key, options); - } -} diff --git a/src/aws.ts b/src/aws.ts index 93894b7..373dc08 100644 --- a/src/aws.ts +++ b/src/aws.ts @@ -10,6 +10,7 @@ import { IListObjectV2Output, ICopyObjectOptions, IHeadOptions, + ICommonClientOptions, } from './types'; import * as _ from 'lodash'; import { AbstractClient } from './client'; @@ -25,17 +26,11 @@ const STANDARD_HEADERS_KEYMAP = { LastModified: 'last-modified', }; -export interface IAWSOptions { - accessKeyId: string; - secretAccessKey: string; - bucket: string; - endpoint?: string; - shards?: string[]; +export interface IAWSOptions extends ICommonClientOptions { s3ForcePathStyle?: boolean; region?: string; - signatureVersion?: string; prefix?: string; - [key: string]: any; + signatureVersion?: string; } const DefaultSignatureVersion = 'v4'; @@ -44,53 +39,35 @@ export default class AWSClient extends AbstractClient { private client: AWS.S3; constructor(options: IAWSOptions) { - const bucket = - options.shards && options.shards.length - ? options.shards - : [options.bucket]; - super({ - prefix: options.prefix || '', - bucket, - }); + super(options); + const awsClientOptions: AWS.S3.Types.ClientConfiguration = { + accessKeyId: options.accessKeyID, + secretAccessKey: options.accessKeySecret, + signatureVersion: options.signatureVersion || DefaultSignatureVersion, + }; const s3ForcePathStyle = !!options.s3ForcePathStyle; - - ['accessKeyId', 'secretAccessKey', 'bucket'].forEach(key => { - assert(options[key], `options.${key} required`); - }); - - // use minio if (s3ForcePathStyle) { + // minio assert( options.endpoint, 'options.endpoint is required when options.s3ForcePathStyle = true' ); - this.client = new AWS.S3({ - accessKeyId: options.accessKeyId, - secretAccessKey: options.secretAccessKey, - endpoint: options.endpoint, - region: options.region || 'cn-north-1', - signatureVersion: options.signatureVersion || DefaultSignatureVersion, - s3ForcePathStyle, - }); - } - // use aws s3 - else { + awsClientOptions.endpoint = options.endpoint; + awsClientOptions.region = options.region || 'cn-north-1'; + awsClientOptions.s3ForcePathStyle = true; + } else { + // aws s3 assert( options.region, 'options.region is required when options.s3ForcePathStyle = false' ); - const s3Options: any = { - accessKeyId: options.accessKeyId, - secretAccessKey: options.secretAccessKey, - region: options.region, - signatureVersion: options.signatureVersion || DefaultSignatureVersion, - }; + awsClientOptions.region = options.region; if (options.endpoint) { - s3Options.endpoint = options.endpoint; + awsClientOptions.endpoint = options.endpoint; } - this.client = new AWS.S3(s3Options); } + this.client = new AWS.S3(awsClientOptions); } protected async _get( diff --git a/src/client.ts b/src/client.ts index e293e2d..dc01542 100644 --- a/src/client.ts +++ b/src/client.ts @@ -11,11 +11,6 @@ import { IHeadOptions, } from './types'; -interface IOptions { - bucket: string[]; - prefix: string; -} - function normalizeKeyPrefix(prefix: string): string { if (prefix.startsWith('/')) { return prefix.slice(1); @@ -23,14 +18,22 @@ function normalizeKeyPrefix(prefix: string): string { return prefix; } +export interface IAbstractClientOptions { + bucket: string; + shards?: string[]; + prefix?: string; +} + export abstract class AbstractClient { protected prefix: string; - protected bucket: string[]; + protected buckets: string[]; + protected shards?: string[]; - constructor(options: IOptions) { - const { bucket, prefix } = options; - this.bucket = bucket; - this.prefix = normalizeKeyPrefix(prefix); + constructor(options: IAbstractClientOptions) { + const { bucket, prefix, shards } = options; + this.shards = shards; + this.buckets = shards ? shards.map(s => `${bucket}-${s}`) : [bucket]; + this.prefix = prefix ? normalizeKeyPrefix(prefix) : ''; } public get( @@ -143,12 +146,15 @@ export abstract class AbstractClient { } protected getBucketName(key: string): string { - if (this.bucket.length === 1) { - return this.bucket[0]; + if (!this.shards) { + return this.buckets[0]; } - - const suffix = key.slice(-1).toLowerCase(); - const bucket = this.bucket.find(b => b.indexOf(suffix)); + const keySuffix = key.slice(-1).toLowerCase(); + const shardIndex = this.shards.findIndex(s => s.indexOf(keySuffix) > -1); + if (!shardIndex) { + throw Error('key not exist in shards bucket!'); + } + const bucket = this.buckets[shardIndex]; if (!bucket) { throw Error('key not exist in shards bucket!'); } diff --git a/src/index.ts b/src/index.ts index 12605c9..e414484 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,44 +1,24 @@ -import AWOS, { IOptions } from './awos'; -import { - IGetObjectResponse, - IGetBufferedObjectResponse, - IListObjectOptions, - ISignatureUrlOptions, -} from './types'; -import OSSClient, { IOSSOptions } from './oss'; -import AWSClient, { IAWSOptions } from './aws'; +import OSSClient from './oss'; +import AWSClient from './aws'; import { AbstractClient } from './client'; +import { ICommonClientOptions } from './types'; -const assert = require('assert'); +export interface IClientOptions extends ICommonClientOptions { + storageType: 'oss' | 'aws'; +} -export function build(options: IOptions): AbstractClient { - const { prefix } = options; +export function build(options: IClientOptions): AbstractClient { + const { storageType, ...commonOptions } = options; - switch (options.type) { + switch (storageType) { case 'oss': - assert(options.ossOptions, 'ossOptions is required when type is "oss"'); - return new OSSClient({ - ...options.ossOptions!, - prefix, - }); + return new OSSClient(commonOptions); case 'aws': - assert(options.awsOptions, 'awsOptions is required when type is "aws"'); - return new AWSClient({ - ...options.awsOptions!, - prefix, - }); + return new AWSClient(commonOptions); default: throw Error('invalid options!'); } } -export { - AWOS, - IOptions, - IGetObjectResponse, - IGetBufferedObjectResponse, - IListObjectOptions, - IOSSOptions, - IAWSOptions, - ISignatureUrlOptions, -}; +export * from './types'; +export * from './aws'; diff --git a/src/oss.ts b/src/oss.ts index f15c958..998cbe1 100644 --- a/src/oss.ts +++ b/src/oss.ts @@ -10,11 +10,11 @@ import { IListObjectV2Output, ICopyObjectOptions, IHeadOptions, + ICommonClientOptions, } from './types'; import { defaults } from 'lodash'; const OSS = require('ali-oss'); -const assert = require('assert'); const retry = require('async-retry'); const STANDARD_HEADERS = [ @@ -25,35 +25,17 @@ const STANDARD_HEADERS = [ 'last-modified', ]; -export interface IOSSOptions { - accessKeyId: string; - accessKeySecret: string; - bucket: string; - endpoint: string; - shards?: string[]; - prefix?: string; - [key: string]: any; -} - export default class OSSClient extends AbstractClient { private clients: Map = new Map(); // private OSS_META_PREFIX = 'x-oss-meta-'; - constructor(options: IOSSOptions) { - const bucket = options.shards - ? options.shards.map(s => `${options.bucket}-${s.toLowerCase()}`) - : [options.bucket]; - super({ bucket, prefix: options.prefix || '' }); - - ['accessKeyId', 'accessKeySecret', 'bucket'].forEach(key => { - assert(options[key], `options.${key} required`); - }); - - bucket.forEach(bucket => { + constructor(options: ICommonClientOptions) { + super(options); + this.buckets.forEach(bucket => { this.clients.set( bucket, new OSS({ - accessKeyId: options.accessKeyId, + accessKeyId: options.accessKeyID, accessKeySecret: options.accessKeySecret, endpoint: options.endpoint, bucket, diff --git a/src/types.ts b/src/types.ts index dc55270..74b96c6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,3 +1,11 @@ +import { IAbstractClientOptions } from './client'; + +export interface ICommonClientOptions extends IAbstractClientOptions { + accessKeyID: string; + accessKeySecret: string; + endpoint: string; +} + export interface IGetObjectResponse { content: string; meta: Map;