From c0d9cd8c6c25a145c02c559d384809acd7ef9eba Mon Sep 17 00:00:00 2001 From: Krisztian Toth Date: Fri, 18 Jan 2019 10:57:26 +0100 Subject: [PATCH 1/4] feat(table-to-file): format options comes from a parameter PKG-0023 Co-authored-by: Tamas Mohos --- src/query-to-file/index.js | 9 +++++---- src/query-to-file/index.spec.js | 10 ++++++++-- src/table-to-file/index.js | 9 +++++---- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/query-to-file/index.js b/src/query-to-file/index.js index 43e1d80..67c77ea 100644 --- a/src/query-to-file/index.js +++ b/src/query-to-file/index.js @@ -8,15 +8,16 @@ const BigQuery = require('../big-query'); class QueryToFile { - static create(baseName) { + static create(baseName, formatOptions = { format: 'JSON', gzip: true }) { const tableName = `tmp_${baseName}_${+new Date}`; - return new QueryToFile(tableName); + return new QueryToFile(tableName, formatOptions); } - constructor(tableName) { + constructor(tableName, formatOptions = { format: 'JSON', gzip: true }) { this._tableName = tableName; + this._formatOptions = formatOptions; } @@ -24,7 +25,7 @@ class QueryToFile { const file = File.create(`tmp/${this._tableName}.json.gz`); yield QueryToTable.create(this._tableName).run(query, options); - yield TableToFile.create(this._tableName, file).run(); + yield TableToFile.create(this._tableName, file, this._formatOptions).run(); yield BigQuery.create().table(this._tableName).delete(); return file; diff --git a/src/query-to-file/index.spec.js b/src/query-to-file/index.spec.js index 6c2afe3..1571204 100644 --- a/src/query-to-file/index.spec.js +++ b/src/query-to-file/index.spec.js @@ -11,12 +11,14 @@ describe('QueryToFile', function() { let clock; let table; let instance; + let formatOptions; beforeEach(function() { clock = this.sandbox.useFakeTimers(+new Date('2016-12-08')); //= 1481155200000 table = { delete: this.sandbox.stub().resolves() }; + formatOptions = { format: 'JSON', gzip: false }; - instance = QueryToFile.create('base_name'); + instance = QueryToFile.create('base_name', formatOptions); }); @@ -71,7 +73,11 @@ describe('QueryToFile', function() { it('instantiates a TableToFile with temp table name and temp file', function() { - expect(TableToFile.create).to.calledWithExactly('tmp_base_name_1481155200000', '[temp storage file]'); + expect(TableToFile.create).to.calledWithExactly( + 'tmp_base_name_1481155200000', + '[temp storage file]', + formatOptions + ); }); diff --git a/src/table-to-file/index.js b/src/table-to-file/index.js index b0fe75a..f9b2697 100644 --- a/src/table-to-file/index.js +++ b/src/table-to-file/index.js @@ -6,21 +6,22 @@ const JobRunner = require('../job-runner'); class TableToFile { - static create(tableName, file) { + static create(tableName, file, formatOptions = { format: 'JSON', gzip: true }) { const table = BigQuery.create().table(tableName); - return new TableToFile(table, file); + return new TableToFile(table, file, formatOptions); } - constructor(table, file) { + constructor(table, file, formatOptions = { format: 'JSON', gzip: true }) { this._table = table; this._file = file; + this._formatOptions = formatOptions; } *run() { - return yield JobRunner.run(this._table.export(this._file, { format: 'JSON', gzip: true })); + return yield JobRunner.run(this._table.export(this._file, this._formatOptions)); } } From 4e8706fd4191b4fe88646bb42256a8955e77c30c Mon Sep 17 00:00:00 2001 From: Krisztian Toth Date: Fri, 18 Jan 2019 13:04:21 +0100 Subject: [PATCH 2/4] feat(query-to-file): filename comes from a parameter PKG-0023 Co-authored-by: Tamas Mohos --- src/query-to-file/index.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/query-to-file/index.js b/src/query-to-file/index.js index 67c77ea..6120ace 100644 --- a/src/query-to-file/index.js +++ b/src/query-to-file/index.js @@ -8,21 +8,22 @@ const BigQuery = require('../big-query'); class QueryToFile { - static create(baseName, formatOptions = { format: 'JSON', gzip: true }) { + static create(baseName, formatOptions = { format: 'JSON', gzip: true }, filename = null) { const tableName = `tmp_${baseName}_${+new Date}`; - return new QueryToFile(tableName, formatOptions); + return new QueryToFile(tableName, formatOptions, filename); } - constructor(tableName, formatOptions = { format: 'JSON', gzip: true }) { + constructor(tableName, formatOptions = { format: 'JSON', gzip: true }, filename = null) { this._tableName = tableName; this._formatOptions = formatOptions; + this._fileName = filename || `tmp/${this._tableName}.json.gz`; } *run(query, options) { - const file = File.create(`tmp/${this._tableName}.json.gz`); + const file = File.create(this._fileName); yield QueryToTable.create(this._tableName).run(query, options); yield TableToFile.create(this._tableName, file, this._formatOptions).run(); From f8accf27c8c3d8cba04e155d13dd3a38d52c1b0c Mon Sep 17 00:00:00 2001 From: Krisztian Toth Date: Mon, 21 Jan 2019 17:44:38 +0100 Subject: [PATCH 3/4] feat(query-to-file): gzip option is inferred from filetype PKG-0023 --- src/query-to-file/index.js | 10 +++++----- src/query-to-file/index.spec.js | 8 ++++---- src/table-to-file/index.js | 14 ++++++++++---- src/table-to-file/index.spec.js | 23 +++++++++++++++++++++++ 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/query-to-file/index.js b/src/query-to-file/index.js index 6120ace..746ef45 100644 --- a/src/query-to-file/index.js +++ b/src/query-to-file/index.js @@ -8,16 +8,16 @@ const BigQuery = require('../big-query'); class QueryToFile { - static create(baseName, formatOptions = { format: 'JSON', gzip: true }, filename = null) { + static create(baseName, format = 'JSON', filename = null) { const tableName = `tmp_${baseName}_${+new Date}`; - return new QueryToFile(tableName, formatOptions, filename); + return new QueryToFile(tableName, format, filename); } - constructor(tableName, formatOptions = { format: 'JSON', gzip: true }, filename = null) { + constructor(tableName, format = 'JSON', filename = null) { this._tableName = tableName; - this._formatOptions = formatOptions; + this._format = format; this._fileName = filename || `tmp/${this._tableName}.json.gz`; } @@ -26,7 +26,7 @@ class QueryToFile { const file = File.create(this._fileName); yield QueryToTable.create(this._tableName).run(query, options); - yield TableToFile.create(this._tableName, file, this._formatOptions).run(); + yield TableToFile.create(this._tableName, file, this._format).run(); yield BigQuery.create().table(this._tableName).delete(); return file; diff --git a/src/query-to-file/index.spec.js b/src/query-to-file/index.spec.js index 1571204..fb709df 100644 --- a/src/query-to-file/index.spec.js +++ b/src/query-to-file/index.spec.js @@ -11,14 +11,14 @@ describe('QueryToFile', function() { let clock; let table; let instance; - let formatOptions; + let format; beforeEach(function() { clock = this.sandbox.useFakeTimers(+new Date('2016-12-08')); //= 1481155200000 table = { delete: this.sandbox.stub().resolves() }; - formatOptions = { format: 'JSON', gzip: false }; + format = 'JSON'; - instance = QueryToFile.create('base_name', formatOptions); + instance = QueryToFile.create('base_name', format); }); @@ -76,7 +76,7 @@ describe('QueryToFile', function() { expect(TableToFile.create).to.calledWithExactly( 'tmp_base_name_1481155200000', '[temp storage file]', - formatOptions + format ); }); diff --git a/src/table-to-file/index.js b/src/table-to-file/index.js index f9b2697..9eb3811 100644 --- a/src/table-to-file/index.js +++ b/src/table-to-file/index.js @@ -6,17 +6,23 @@ const JobRunner = require('../job-runner'); class TableToFile { - static create(tableName, file, formatOptions = { format: 'JSON', gzip: true }) { + static create(tableName, file, format = 'JSON') { const table = BigQuery.create().table(tableName); - return new TableToFile(table, file, formatOptions); + return new TableToFile(table, file, format); } - constructor(table, file, formatOptions = { format: 'JSON', gzip: true }) { + constructor(table, file, format = 'JSON') { this._table = table; this._file = file; - this._formatOptions = formatOptions; + if (format === 'JSON') { + this._formatOptions = { format: 'JSON', gzip: true }; + } else if (format === 'CSV') { + this._formatOptions = { format: 'CSV', gzip: false }; + } else { + throw Error('Unknown file format'); + } } diff --git a/src/table-to-file/index.spec.js b/src/table-to-file/index.spec.js index 8fd6ad0..17392a3 100644 --- a/src/table-to-file/index.spec.js +++ b/src/table-to-file/index.spec.js @@ -120,4 +120,27 @@ describe('TableToFile', function() { }); + describe('#formats', function() { + const job = new EventEmitter; + let apiResponse; + let jobMetadata; + + beforeEach(function* () { + apiResponse = { status: { state: 'IN_PROGRESS' } }; + jobMetadata = { status: { state: 'DONE' } }; + table.export.resolves([job, apiResponse]); + setTimeout(() => job.emit('complete', jobMetadata)); + }); + + it('should pass the proper options to export for CSV', function*() { + yield TableToFile.create(tableName, file, 'CSV').run(); + expect(table.export).to.calledWithExactly('[file]', { format: 'CSV', gzip: false }); + }); + + it('should throw error for unknown file format', function*() { + expect(() => TableToFile.create(tableName, file, 'xls')).to.throw(); + }); + + }); + }); From 93120c1195d2ff5c9d82c520fb5998a92fcec76a Mon Sep 17 00:00:00 2001 From: Krisztian Toth Date: Tue, 22 Jan 2019 10:42:55 +0100 Subject: [PATCH 4/4] feat(query-to-file): file extension is inferred from format PKG-0023 Co-authored-by: Gabor Nemeth --- README.md | 8 ++++---- src/query-to-file/index.js | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9446c19..934870f 100644 --- a/README.md +++ b/README.md @@ -204,9 +204,9 @@ This is a wrapper of the original [createTable()](https://cloud.google.com/nodej ## BigQuery.QueryToFile -#### create(baseName):QueryToFile `static` +#### create(baseName, format, file):QueryToFile `static` -#### constructor(tableName):QueryToFile +#### constructor(tableName, format, file):QueryToFile #### run(query, options):File `generator` @@ -252,9 +252,9 @@ This is a wrapper of the original [createTable()](https://cloud.google.com/nodej ## BigQuery.TableToFile -#### create(tableName, file):TableToFile `static` +#### create(tableName, file, format):TableToFile `static` -#### constructor(table, file):TableToFile +#### constructor(table, file, format):TableToFile #### run():Job `generator` diff --git a/src/query-to-file/index.js b/src/query-to-file/index.js index 746ef45..351568f 100644 --- a/src/query-to-file/index.js +++ b/src/query-to-file/index.js @@ -17,8 +17,9 @@ class QueryToFile { constructor(tableName, format = 'JSON', filename = null) { this._tableName = tableName; - this._format = format; - this._fileName = filename || `tmp/${this._tableName}.json.gz`; + this._format = format === 'CSV' ? 'CSV' : 'JSON'; + const extension = this._format === 'CSV' ? '.csv' : '.json.gz'; + this._fileName = filename || `tmp/${this._tableName}${extension}`; }