Skip to content

Commit

Permalink
feat(query-to-file): exported file format can be JSON or CSV
Browse files Browse the repository at this point in the history
  • Loading branch information
bankyadam authored Jan 22, 2019
2 parents f6255ec + 93120c1 commit d1a4baf
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 15 deletions.
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,9 @@ This is a wrapper of the original [createTable()](https://cloud.google.com/nodej

## BigQuery.QueryToFile

#### create(baseName):QueryToFile `static`
#### create(baseName, format, file):QueryToFile `static`

#### constructor(tableName):QueryToFile
#### constructor(tableName, format, file):QueryToFile

#### run(query, options):File `generator`

Expand Down Expand Up @@ -252,9 +252,9 @@ This is a wrapper of the original [createTable()](https://cloud.google.com/nodej

## BigQuery.TableToFile

#### create(tableName, file):TableToFile `static`
#### create(tableName, file, format):TableToFile `static`

#### constructor(table, file):TableToFile
#### constructor(table, file, format):TableToFile

#### run():Job `generator`

13 changes: 8 additions & 5 deletions src/query-to-file/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,23 +8,26 @@ const BigQuery = require('../big-query');

class QueryToFile {

static create(baseName) {
static create(baseName, format = 'JSON', filename = null) {
const tableName = `tmp_${baseName}_${+new Date}`;

return new QueryToFile(tableName);
return new QueryToFile(tableName, format, filename);
}


constructor(tableName) {
constructor(tableName, format = 'JSON', filename = null) {
this._tableName = tableName;
this._format = format === 'CSV' ? 'CSV' : 'JSON';
const extension = this._format === 'CSV' ? '.csv' : '.json.gz';
this._fileName = filename || `tmp/${this._tableName}${extension}`;
}


*run(query, options) {
const file = File.create(`tmp/${this._tableName}.json.gz`);
const file = File.create(this._fileName);

yield QueryToTable.create(this._tableName).run(query, options);
yield TableToFile.create(this._tableName, file).run();
yield TableToFile.create(this._tableName, file, this._format).run();
yield BigQuery.create().table(this._tableName).delete();

return file;
Expand Down
10 changes: 8 additions & 2 deletions src/query-to-file/index.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@ describe('QueryToFile', function() {
let clock;
let table;
let instance;
let format;

beforeEach(function() {
clock = this.sandbox.useFakeTimers(+new Date('2016-12-08')); //= 1481155200000
table = { delete: this.sandbox.stub().resolves() };
format = 'JSON';

instance = QueryToFile.create('base_name');
instance = QueryToFile.create('base_name', format);
});


Expand Down Expand Up @@ -71,7 +73,11 @@ describe('QueryToFile', function() {


it('instantiates a TableToFile with temp table name and temp file', function() {
expect(TableToFile.create).to.calledWithExactly('tmp_base_name_1481155200000', '[temp storage file]');
expect(TableToFile.create).to.calledWithExactly(
'tmp_base_name_1481155200000',
'[temp storage file]',
format
);
});


Expand Down
15 changes: 11 additions & 4 deletions src/table-to-file/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,28 @@ const JobRunner = require('../job-runner');

class TableToFile {

static create(tableName, file) {
static create(tableName, file, format = 'JSON') {
const table = BigQuery.create().table(tableName);

return new TableToFile(table, file);
return new TableToFile(table, file, format);
}


constructor(table, file) {
constructor(table, file, format = 'JSON') {
this._table = table;
this._file = file;
if (format === 'JSON') {
this._formatOptions = { format: 'JSON', gzip: true };
} else if (format === 'CSV') {
this._formatOptions = { format: 'CSV', gzip: false };
} else {
throw Error('Unknown file format');
}
}


*run() {
return yield JobRunner.run(this._table.export(this._file, { format: 'JSON', gzip: true }));
return yield JobRunner.run(this._table.export(this._file, this._formatOptions));
}

}
Expand Down
23 changes: 23 additions & 0 deletions src/table-to-file/index.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -120,4 +120,27 @@ describe('TableToFile', function() {

});

describe('#formats', function() {
const job = new EventEmitter;
let apiResponse;
let jobMetadata;

beforeEach(function* () {
apiResponse = { status: { state: 'IN_PROGRESS' } };
jobMetadata = { status: { state: 'DONE' } };
table.export.resolves([job, apiResponse]);
setTimeout(() => job.emit('complete', jobMetadata));
});

it('should pass the proper options to export for CSV', function*() {
yield TableToFile.create(tableName, file, 'CSV').run();
expect(table.export).to.calledWithExactly('[file]', { format: 'CSV', gzip: false });
});

it('should throw error for unknown file format', function*() {
expect(() => TableToFile.create(tableName, file, 'xls')).to.throw();
});

});

});

0 comments on commit d1a4baf

Please sign in to comment.