From 154568a0ddaf454a9545e3cdf075094ea6e0321f Mon Sep 17 00:00:00 2001 From: Marcin Wielgoszewski Date: Fri, 22 Jul 2016 15:19:37 -0400 Subject: [PATCH 1/3] Add test to track down reported python3 issues --- tests/test_functional.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_functional.py b/tests/test_functional.py index 83b4fb6..62d96b3 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -1538,3 +1538,31 @@ def test_node_info_not_updated_on_erroneous_data(self, node, testapp): learn_from_result(result, node.to_dict()) assert 'foobar' not in node.node_info + + +class TestCSVExport: + def test_node_csv_download(self, node, testapp): + from doorman.compat import StringIO + import csv + + node.enrolled_on = dt.datetime.utcnow() + node.last_checkin = dt.datetime.utcnow() + node.last_ip = '1.1.1.1' + node.node_info = {'hardware_vendor': "Honest Achmed's Computer Supply"} + node.save() + + resp = testapp.get(url_for('manage.nodes_csv')) + + assert resp.headers['Content-Type'] == 'text/csv' + assert resp.headers['Content-Disposition'] == 'attachment; filename=nodes.csv' + + reader = csv.DictReader(StringIO(resp.body)) + row = next(reader) + + assert row['Display Name'] == node.display_name + assert row['Host Identifier'] == node.host_identifier + assert row['Enrolled On'] == str(node.enrolled_on) + assert row['Last Check-In'] == str(node.last_checkin) + assert row['Last Ip Address'] == node.last_ip + assert row['Is Active'] == 'True' + assert row['Make'] == node.node_info['hardware_vendor'] From 3ea1f0c67bc907637e2ec97e2e598d9cbf65844a Mon Sep 17 00:00:00 2001 From: Marcin Wielgoszewski Date: Fri, 22 Jul 2016 15:34:08 -0400 Subject: [PATCH 2/3] `map()` returns a map object, not a list * Use unicodecsv module to write utf-8 to a csv file. * Use BytesIO instead of StringIO * Use Flask's `send_file` instead of `make_response` --- doorman/compat.py | 6 ------ doorman/manage/views.py | 26 ++++++++++++++++---------- requirements/dev.txt | 1 + requirements/prod.txt | 1 + tests/test_functional.py | 7 +++---- 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/doorman/compat.py b/doorman/compat.py index 4ca0353..845193c 100644 --- a/doorman/compat.py +++ b/doorman/compat.py @@ -18,12 +18,6 @@ basestring = (str, bytes) -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - - def with_metaclass(meta, *bases): """Create a base class with a metaclass.""" class metaclass(meta): diff --git a/doorman/manage/views.py b/doorman/manage/views.py index b0ecd4f..f6f1c70 100644 --- a/doorman/manage/views.py +++ b/doorman/manage/views.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- +from io import BytesIO from operator import itemgetter -import csv import json import datetime as dt +import unicodecsv as csv from flask import ( - Blueprint, current_app, flash, jsonify, make_response, redirect, - render_template, request, url_for + Blueprint, current_app, flash, jsonify, redirect, render_template, + request, send_file, url_for ) from flask_login import login_required from flask_paginate import Pagination @@ -26,7 +27,6 @@ UpdateRuleForm, UpdateNodeForm, ) -from doorman.compat import StringIO from doorman.database import db from doorman.models import ( DistributedQuery, DistributedQueryTask, DistributedQueryResult, @@ -112,10 +112,10 @@ def nodes_csv(): column_names = map(itemgetter(0), current_app.config['DOORMAN_CAPTURE_NODE_INFO']) labels = map(itemgetter(1), current_app.config['DOORMAN_CAPTURE_NODE_INFO']) headers.extend(labels) - headers = map(str.title, headers) + headers = list(map(str.title, headers)) - sio = StringIO() - writer = csv.writer(sio) + bio = BytesIO() + writer = csv.writer(bio) writer.writerow(headers) for node in Node.query: @@ -130,9 +130,15 @@ def nodes_csv(): row.extend([node.node_info.get(column, '') for column in column_names]) writer.writerow(row) - response = make_response(sio.getvalue()) - response.headers["Content-Disposition"] = "attachment; filename=nodes.csv" - response.headers["Content-Type"] = "text/csv" + bio.seek(0) + + response = send_file( + bio, + mimetype='text/csv', + as_attachment=True, + attachment_filename='nodes.csv' + ) + return response diff --git a/requirements/dev.txt b/requirements/dev.txt index a4c7228..704dc5c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -53,6 +53,7 @@ scales==1.0.9 six==1.10.0 SQLAlchemy==1.0.12 waitress==0.9.0 +unicodecsv==0.14.1 webassets==0.11.1 WebOb==1.6.0 WebTest==2.0.21 diff --git a/requirements/prod.txt b/requirements/prod.txt index eb7d61b..fce6157 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -41,6 +41,7 @@ requests==2.10.0 requests-oauthlib==0.6.1 scales==1.0.9 SQLAlchemy==1.0.12 +unicodecsv==0.14.1 webassets==0.11.1 Werkzeug==0.11.8 WTForms==2.1 diff --git a/tests/test_functional.py b/tests/test_functional.py index 62d96b3..293fc32 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -1542,8 +1542,7 @@ def test_node_info_not_updated_on_erroneous_data(self, node, testapp): class TestCSVExport: def test_node_csv_download(self, node, testapp): - from doorman.compat import StringIO - import csv + import unicodecsv as csv node.enrolled_on = dt.datetime.utcnow() node.last_checkin = dt.datetime.utcnow() @@ -1553,10 +1552,10 @@ def test_node_csv_download(self, node, testapp): resp = testapp.get(url_for('manage.nodes_csv')) - assert resp.headers['Content-Type'] == 'text/csv' + assert resp.headers['Content-Type'] == 'text/csv; charset=utf-8' assert resp.headers['Content-Disposition'] == 'attachment; filename=nodes.csv' - reader = csv.DictReader(StringIO(resp.body)) + reader = csv.DictReader(io.BytesIO(resp.body)) row = next(reader) assert row['Display Name'] == node.display_name From 004582cf2723353e218508ae69174b1e0cfc73a7 Mon Sep 17 00:00:00 2001 From: Marcin Wielgoszewski Date: Mon, 25 Jul 2016 09:23:00 -0400 Subject: [PATCH 3/3] up revision to 0.5.1 [ci skip] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ded3050..061dfad 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ name='doorman', description='an osquery fleet manager', url='https://github.com/mwielgoszewski/doorman', - version='0.5', + version='0.5.1', packages=find_packages( exclude=[ 'tests*',