From c25c19576f60fc56b1511d10ab3c1591ed7419a2 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Fri, 20 Apr 2018 12:55:36 -0700 Subject: [PATCH 1/2] Possible fix for notifications bug in #998 - And other annoying bugs with paths and modules not found... --- .travis.yml | 7 +++--- .../celeryconfig.py | 2 +- security_monkey/manage.py | 4 +-- security_monkey/task_scheduler/util.py | 5 +++- .../tests/scheduling/test_celery_scheduler.py | 16 ++++++------ security_monkey/views/user_settings.py | 25 ++++++++++++++----- setup.py | 4 ++- 7 files changed, 40 insertions(+), 23 deletions(-) rename celeryconfig.py => security_monkey/celeryconfig.py (97%) diff --git a/.travis.yml b/.travis.yml index 7eb837b45..7fb8056e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ matrix: env: - UNIT_TEST_JOB=true - PIP_DOWNLOAD_CACHE=".pip_download_cache" + - SECURITY_MONKEY_SETTINGS="$( pwd )/env-config/config.py" addons: postgresql: "9.4" @@ -31,8 +32,8 @@ matrix: - pip install openstacksdk - pip install cloudaux\[gcp\] - pip install cloudaux\[openstack\] - - python setup.py develop - - pip install .[tests] + - pip install -e . + - pip install ."[tests]" - pip install coveralls - monkey db upgrade - monkey amazon_accounts @@ -130,7 +131,7 @@ matrix: - BUILD_DOCKER=True - DOCKER_COMPOSE_VERSION=1.18.0 - PIP_DOWNLOAD_CACHE=".pip_download_cache" - - SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config.py + - SECURITY_MONKEY_SETTINGS="$( pwd )/env-config/config.py" services: - docker diff --git a/celeryconfig.py b/security_monkey/celeryconfig.py similarity index 97% rename from celeryconfig.py rename to security_monkey/celeryconfig.py index 7df9b6aa6..173e3a2af 100644 --- a/celeryconfig.py +++ b/security_monkey/celeryconfig.py @@ -1,5 +1,5 @@ """ -.. module: celeryconfig +.. module: securitymonkey.celeryconfig :platform: Unix :synopsis: Use this file to set up the Celery configuration for task scheduling. diff --git a/security_monkey/manage.py b/security_monkey/manage.py index 4ce25cb56..6bdbea7e9 100644 --- a/security_monkey/manage.py +++ b/security_monkey/manage.py @@ -166,10 +166,8 @@ def amazon_accounts(): """ Pre-populates standard AWS owned accounts """ import json from security_monkey.datastore import Account, AccountType - from os.path import dirname, join - data_file = join(dirname(dirname(__file__)), "data", "aws_accounts.json") - data = json.load(open(data_file, 'r')) + data = json.load(open("data/aws_accounts.json", 'r')) app.logger.info('Adding / updating Amazon owned accounts') try: diff --git a/security_monkey/task_scheduler/util.py b/security_monkey/task_scheduler/util.py index 826aea00b..feb806553 100644 --- a/security_monkey/task_scheduler/util.py +++ b/security_monkey/task_scheduler/util.py @@ -13,13 +13,15 @@ from security_monkey.common.utils import find_modules import os +import importlib from security_monkey.exceptions import InvalidCeleryConfigurationType def get_celery_config_file(): """This gets the Celery configuration file as a module that Celery uses""" - return __import__(os.environ.get("SM_CELERY_CONFIG", "celeryconfig")) + return importlib.import_module("security_monkey.{}".format(os.environ.get("SM_CELERY_CONFIG", "celeryconfig")), + "security_monkey") def make_celery(app): @@ -38,6 +40,7 @@ def make_celery(app): celery.conf.update(app.config) TaskBase = celery.Task + class ContextTask(TaskBase): abstract = True diff --git a/security_monkey/tests/scheduling/test_celery_scheduler.py b/security_monkey/tests/scheduling/test_celery_scheduler.py index 05f7b64d7..8fad76a56 100644 --- a/security_monkey/tests/scheduling/test_celery_scheduler.py +++ b/security_monkey/tests/scheduling/test_celery_scheduler.py @@ -354,8 +354,8 @@ def test_celery_purge(self): assert mock.control.purge.called def test_get_sm_celery_config_value(self): - import celeryconfig - setattr(celeryconfig, "test_value", {"some", "set", "of", "things"}) + import security_monkey.celeryconfig + setattr(security_monkey.celeryconfig, "test_value", {"some", "set", "of", "things"}) # We should get the proper thing back out: from security_monkey.task_scheduler.util import get_sm_celery_config_value, get_celery_config_file c = get_celery_config_file() @@ -382,8 +382,8 @@ def test_get_sm_celery_config_value(self): def test_get_celery_config_file(self): import os from security_monkey.task_scheduler.util import get_celery_config_file - os.environ["SM_CELERY_CONFIG"] = "security_monkey" - assert hasattr(get_celery_config_file(), "app") + os.environ["SM_CELERY_CONFIG"] = "celeryconfig" + assert hasattr(get_celery_config_file(), "broker_url") del os.environ["SM_CELERY_CONFIG"] assert hasattr(get_celery_config_file(), "broker_url") @@ -559,8 +559,8 @@ def test_celery_skipabeat(self, mock_store_exception, mock_expired_exceptions, m @patch("security_monkey.task_scheduler.beat.store_exception") def test_celery_only_tech(self, mock_store_exception, mock_expired_exceptions, mock_account_tech, mock_purge, mock_setup): - import celeryconfig - celeryconfig.security_monkey_only_watch = {"iamrole"} + import security_monkey.celeryconfig + security_monkey.celeryconfig.security_monkey_only_watch = {"iamrole"} from security_monkey.task_scheduler.beat import setup_the_tasks from security_monkey.watchers.iam.iam_role import IAMRole @@ -619,8 +619,8 @@ def test_celery_only_tech(self, mock_store_exception, mock_expired_exceptions, m @patch("security_monkey.task_scheduler.beat.store_exception") def test_celery_ignore_tech(self, mock_store_exception, mock_expired_exceptions, mock_account_tech, mock_purge, mock_setup): - import celeryconfig - celeryconfig.security_monkey_watcher_ignore = {"policy"} + import security_monkey.celeryconfig + security_monkey.celeryconfig.security_monkey_watcher_ignore = {"policy"} from security_monkey.task_scheduler.beat import setup_the_tasks from security_monkey.watchers.iam.iam_role import IAMRole diff --git a/security_monkey/views/user_settings.py b/security_monkey/views/user_settings.py index 2049d79c3..485f95c62 100644 --- a/security_monkey/views/user_settings.py +++ b/security_monkey/views/user_settings.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from flask import request +from marshmallow.validate import OneOf +from werkzeug.exceptions import BadRequest from security_monkey.views import AuthenticatedService from security_monkey.views import USER_SETTINGS_FIELDS @@ -18,8 +21,15 @@ from security_monkey.datastore import User from security_monkey import db, rbac -from flask_restful import marshal, reqparse +from flask_restful import marshal, abort from flask_login import current_user +from marshmallow import Schema, fields, ValidationError + + +class SaveSettingsSchema(Schema): + accounts = fields.List(fields.Integer()) + daily_audit_email = fields.Boolean(allow_none=True, required=True) + change_report_setting = fields.String(allow_none=True, required=True, validate=OneOf(["ISSUES", "ALL", "NONE"])) class UserSettings(AuthenticatedService): @@ -148,11 +158,14 @@ def post(self): :statuscode 200: no error :statuscode 401: Authentication Error. Please Login. """ - - self.reqparse.add_argument('accounts', required=True, type=list, help='Must provide accounts', location='json') - self.reqparse.add_argument('change_report_setting', required=True, type=str, help='Must provide change_report_setting', location='json') - self.reqparse.add_argument('daily_audit_email', required=True, type=bool, help='Must provide daily_audit_email', location='json') - args = self.reqparse.parse_args() + json_data = request.get_json() + if not json_data: + raise BadRequest() + + try: + args = SaveSettingsSchema(strict=True).load(json_data).data + except ValidationError as ve: + abort(400, message=ve.message) current_user.daily_audit_email = args['daily_audit_email'] current_user.change_reports = args['change_report_setting'] diff --git a/setup.py b/setup.py index a3be2d77c..f0c5171c6 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,9 @@ 'pyjwt>=1.01', 'netaddr', 'swag-client>=0.3.7', - 'idna==2.6' + 'idna==2.6', + 'marshmallow==2.15.0', + 'flask-marshmallow==0.8.0' ], extras_require={ 'onelogin': ['python-saml>=2.4.0'], From 3d71b8fc8c4e4bef261ae3a358565dbf058ca52c Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Fri, 20 Apr 2018 15:52:07 -0700 Subject: [PATCH 2/2] Updated the docs to address breaking changes to the celeryconfig.py location. --- docs/autostarting.md | 6 +++--- docs/tuneworkers.md | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/autostarting.md b/docs/autostarting.md index fc8e877b1..8b1402efc 100755 --- a/docs/autostarting.md +++ b/docs/autostarting.md @@ -91,8 +91,8 @@ firewall rules to permit worker instances access. This is documented in the abov If installing on the `localhost` of the scheduler instance, you will need to install Redis on the instance (this is complete if following the quickstart guide). ### Celery Configuration -You will need to modify the `celeryconfig.py` file that is stored in the base Security Monkey location -at `/usr/local/src/security_monkey/celeryconfig.py`(https://github.com/Netflix/security_monkey/blob/develop/celeryconfig.py). +You will need to modify the `security_monkey/celeryconfig.py` file that is stored in the base Security Monkey location +at `/usr/local/src/security_monkey/security_monkey/celeryconfig.py`(https://github.com/Netflix/security_monkey/blob/develop/security_monkey/celeryconfig.py). This file looks like this: ``` @@ -153,7 +153,7 @@ The workers are instances that fetch data from your configured accounts. These a You are able to deploy as many workers as you like for your environment. Security Monkey splits up tasks based on the account and technology pair. -Similar to configuring the Scheduler above, the workers need to have the **same** `celeryconfig.py` as the scheduler. In here, you can optionally configure +Similar to configuring the Scheduler above, the workers need to have the **same** `security_monkey/celeryconfig.py` as the scheduler. In here, you can optionally configure the number of processes that exist within each instance (via the `worker_concurrency` configuration). By default 10 is selected. You can adjust this as necessary. In general, if you would like to scale horizontally, you should deploy more worker instances. This will allow for maximum parallelization. diff --git a/docs/tuneworkers.md b/docs/tuneworkers.md index 7a702b607..178743c3b 100644 --- a/docs/tuneworkers.md +++ b/docs/tuneworkers.md @@ -20,7 +20,7 @@ Create a new Redis cache ([ElastiCache works well](elasticache_directions.md)), **Keep note of the endpoint, you'll need this later**. ## Create a dedicated Celery configuration -For this use case, you would have two different [Celery configuration Python](https://github.com/Netflix/security_monkey/blob/develop/celeryconfig.py) files. +For this use case, you would have two different [Celery configuration Python](https://github.com/Netflix/security_monkey/blob/develop/securitymonkey/celeryconfig.py) files. You will need to make note of the following section: ``` # This specifies a list of technologies that workers for the above Redis broker should IGNORE. @@ -36,8 +36,8 @@ In these variables, you will enter in the index name of the technology. For exam of the technologies as they appear in the UI. For this use case, we are going to have a dedicated stack of workers (called the `iam` stack) for IAM Roles, and another stack for everything else (called the `main` stack). -1. Make a copy of `celeryconfig.py`, and call it `mainceleryconfig.py` -1. In `mainceleryconfig.py`, make a modification to the `security_monkey_watcher_ignore` variable such that its value is: +1. Make a copy of `security_monkey/celeryconfig.py`, and call it `security_monkey/mainceleryconfig.py` +1. In `security_monkey/mainceleryconfig.py`, make a modification to the `security_monkey_watcher_ignore` variable such that its value is: ``` security_monkey_watcher_ignore = set(['iamrole']) ``` @@ -45,6 +45,7 @@ For this use case, we are going to have a dedicated stack of workers (called the Next, you will need to make it so that your scheduler and corresponding set of workers that will load this configuration. There is a new environment variable that Security Monkey will check to properly load this configuration: `SM_CELERY_CONFIG`. For this stack, `SM_CELERY_CONFIG` needs to be set to: `"mainceleryconfig.py"`. +(Do not place `security_monkey` in the variable name...just call it the destination name of the file that resides within the `security_monkey/` python code location -- this is the same place that `manage.py` lives) Because we utilize `supervisor`, you will need to add this to the `environment` section. Here are sample configurations: *MAIN-SCHEDULER*