From 7d6dded143b469d41eedfcf6a6b836ee4c121eec Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Tue, 25 Sep 2018 16:18:47 -0700 Subject: [PATCH 1/5] add validations to the drivermetadata file if a function exists/mentioned in the metadata but does not exist in the driver, an error will be shown with the list of functions. The other way is ok (have a function in the driver.py that do not exist in the metadata). --- .../utilities/shell_package_builder.py | 8 ++ .../utilities/validations/__init__.py | 1 + .../driver_metadata_validations.py | 78 +++++++++++++ .../test_driver_metadata_validations.py | 107 ++++++++++++++++++ 4 files changed, 194 insertions(+) create mode 100644 shellfoundry/utilities/validations/driver_metadata_validations.py create mode 100644 tests/test_utilities/test_validations/test_driver_metadata_validations.py diff --git a/shellfoundry/utilities/shell_package_builder.py b/shellfoundry/utilities/shell_package_builder.py index bd8eb49..1c670c1 100644 --- a/shellfoundry/utilities/shell_package_builder.py +++ b/shellfoundry/utilities/shell_package_builder.py @@ -10,6 +10,8 @@ from shellfoundry.utilities.archive_creator import ArchiveCreator from shellfoundry.utilities.shell_package import ShellPackage from shellfoundry.utilities.temp_dir_context import TempDirContext +from shellfoundry.utilities.validations import DriverMetadataValidations +from shellfoundry.exceptions import FatalError class ShellPackageBuilder(object): @@ -106,6 +108,12 @@ def _remove_all_pyc(package_path): def _create_driver(path, package_path, dir_path, driver_name, mandatory=True): dir_to_zip = os.path.join(path, dir_path) if os.path.exists(dir_to_zip): + try: + driver_validation = DriverMetadataValidations() + driver_validation.validate_driver_metadata(dir_to_zip) + except Exception as ex: + raise FatalError(ex.message) + zip_file_path = os.path.join(package_path, driver_name) ArchiveCreator.make_archive(zip_file_path, "zip", dir_to_zip) elif mandatory: diff --git a/shellfoundry/utilities/validations/__init__.py b/shellfoundry/utilities/validations/__init__.py index bda2c22..12ab2af 100644 --- a/shellfoundry/utilities/validations/__init__.py +++ b/shellfoundry/utilities/validations/__init__.py @@ -1,2 +1,3 @@ from .shell_name_validations import ShellNameValidations from .shell_generation_validation import ShellGenerationValidations +from .driver_metadata_validations import DriverMetadataValidations diff --git a/shellfoundry/utilities/validations/driver_metadata_validations.py b/shellfoundry/utilities/validations/driver_metadata_validations.py new file mode 100644 index 0000000..9cfc4ce --- /dev/null +++ b/shellfoundry/utilities/validations/driver_metadata_validations.py @@ -0,0 +1,78 @@ +import os +import xml.etree.ElementTree as etree +import re +from inspect import getmembers, isfunction, ismethod, isclass +import ast +import _ast +import imp +import sys +from importlib import import_module + +class DriverMetadataValidations(object): + + @staticmethod + def _parse_xml(xml_string): + # type: (str) -> etree.Element + """ + :param xml_string: xml string + :return: etree.Element xml element from the input string + """ + parser = etree.XMLParser(encoding='utf-8') + return etree.fromstring(xml_string, parser) + + @staticmethod + def _get_driver_commands(driver_path, driver_name): + """ + :param driver_path: + :return: dict: public command names dictionary + """ + if os.path.exists(driver_path): + with open(driver_path) as mf: + tree = ast.parse(mf.read()) + + module_classes = [] + for i in tree.body: + if isinstance(i, _ast.ClassDef): + if i.name == driver_name: + module_classes.append(i) + + # module_classes = [_ for _ in tree.body if isinstance(_, _ast.ClassDef)] + commands = {} + for f in module_classes[0].body: + if isinstance(f, _ast.FunctionDef) and not f.name.startswith('_'): + args = f.args.args + if len(args) >= 2: + if args[0].id == 'self' and args[1].id == 'context': + commands[f.name] = [a.id for a in f.args.args] + + return commands + else: + return {} + + def validate_driver_metadata(self, driver_path): + """ Validate driver metadata + :param str driver_path: path to the driver directory + """ + + metadata_path = os.path.join(driver_path, 'drivermetadata.xml') + driver_path = os.path.join(driver_path, 'driver.py') + if os.path.exists(metadata_path): + with open(metadata_path, 'r') as f: + metadata_str = f.read() + + metadata_xml = self._parse_xml(metadata_str) + metadata_commands = metadata_xml.findall('.//Command') + + driver_commands = self._get_driver_commands(driver_path, 'NutShellDriver') + + missing = [] + for mc in metadata_commands: + if mc.attrib['Name'] in driver_commands: + continue + else: + missing.append(mc.attrib['Name']) + + if len(missing) > 0: + err = 'The following commands do not exist in the driver.py but still mentioned in ' \ + 'the DriverMetadata.xml file: {}.\nPlease update the metadata or driver files accordingly.'.format(', '.join(missing)) + raise Exception(err) diff --git a/tests/test_utilities/test_validations/test_driver_metadata_validations.py b/tests/test_utilities/test_validations/test_driver_metadata_validations.py new file mode 100644 index 0000000..20e8aeb --- /dev/null +++ b/tests/test_utilities/test_validations/test_driver_metadata_validations.py @@ -0,0 +1,107 @@ +from mock import patch +from pyfakefs import fake_filesystem_unittest +from tests.asserts import * + +from shellfoundry.utilities.validations import DriverMetadataValidations + +drivermetadataxml = """ + + + + + + + + + """ + + +class TestDriverMetadataValidations(fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + + def test_not_failing_when_more_commands_in_driver_than_in_metadata(self): + # Arrange + self.fs.CreateFile('nut_shell/src/drivermetadata.xml', contents=drivermetadataxml) + self.fs.CreateFile('nut_shell/src/requirements.txt') + self.fs.CreateFile('nut_shell/src/driver.py', contents=""" +from cloudshell.shell.core.resource_driver_interface import ResourceDriverInterface +from cloudshell.shell.core.driver_context import InitCommandContext, ResourceCommandContext, AutoLoadResource, \ + AutoLoadAttribute, AutoLoadDetails, CancellationContext + +class NutShellDriver (ResourceDriverInterface): + + def __init__(self): + pass + + def initialize(self, context): + pass + + def _internal_function(self, context): + pass + + @staticmethod + def mystatic(): + pass + + def cleanup(self): + pass + + # + + def get_inventory(self, context): + return AutoLoadDetails([], []) + + # + + # + def orchestration_save(self, context, cancellation_context, mode, custom_params): + pass + + def orchestration_restore(self, context, cancellation_context, saved_artifact_info, custom_params): + pass + + # + """) + # os.chdir('nut_shell') + + driver_path = 'nut_shell/src' + validations = DriverMetadataValidations() + + # Act + try: + validations.validate_driver_metadata(driver_path) + except Exception as ex: + self.fail('validate_driver_metadata raised an exception when it shouldn\'t: ' + ex.message) + + def test_fails_when_command_in_metadata_but_not_in_driver(self): + # Arrange + self.fs.CreateFile('nut_shell/src/drivermetadata.xml', contents=drivermetadataxml) + self.fs.CreateFile('nut_shell/src/requirements.txt') + self.fs.CreateFile('nut_shell/src/driver.py', contents=""" +from cloudshell.shell.core.resource_driver_interface import ResourceDriverInterface +from cloudshell.shell.core.driver_context import InitCommandContext, ResourceCommandContext, AutoLoadResource, \ + AutoLoadAttribute, AutoLoadDetails, CancellationContext + +class NutShellDriver (ResourceDriverInterface): + + def __init__(self): + pass + + def orchestration_save(self, context, cancellation_context, mode, custom_params): + pass + + """) + # os.chdir('nut_shell') + + driver_path = 'nut_shell/src' + validations = DriverMetadataValidations() + + # Act + with self.assertRaises(Exception) as context: + validations.validate_driver_metadata(driver_path) + + # Assert + self.assertEqual(context.exception.message, """The following commands do not exist in the driver.py but still mentioned in the DriverMetadata.xml file: orchestration_restore. +Please update the metadata or driver files accordingly.""") + From 211d5c41b5fca76a84185e9ecd297dfacb9802b9 Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Tue, 25 Sep 2018 16:20:19 -0700 Subject: [PATCH 2/5] add todo --- .../utilities/validations/driver_metadata_validations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shellfoundry/utilities/validations/driver_metadata_validations.py b/shellfoundry/utilities/validations/driver_metadata_validations.py index 9cfc4ce..c68a9bd 100644 --- a/shellfoundry/utilities/validations/driver_metadata_validations.py +++ b/shellfoundry/utilities/validations/driver_metadata_validations.py @@ -76,3 +76,5 @@ def validate_driver_metadata(self, driver_path): err = 'The following commands do not exist in the driver.py but still mentioned in ' \ 'the DriverMetadata.xml file: {}.\nPlease update the metadata or driver files accordingly.'.format(', '.join(missing)) raise Exception(err) + + #TODO: add validation for command inputs as well From 20fba30e610288b56ee7677a49553cee6eeacda5 Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Tue, 25 Sep 2018 19:47:28 -0700 Subject: [PATCH 3/5] cleanups --- .../utilities/validations/driver_metadata_validations.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/shellfoundry/utilities/validations/driver_metadata_validations.py b/shellfoundry/utilities/validations/driver_metadata_validations.py index c68a9bd..ca24f18 100644 --- a/shellfoundry/utilities/validations/driver_metadata_validations.py +++ b/shellfoundry/utilities/validations/driver_metadata_validations.py @@ -1,12 +1,8 @@ import os import xml.etree.ElementTree as etree -import re -from inspect import getmembers, isfunction, ismethod, isclass import ast import _ast -import imp -import sys -from importlib import import_module + class DriverMetadataValidations(object): @@ -36,7 +32,6 @@ def _get_driver_commands(driver_path, driver_name): if i.name == driver_name: module_classes.append(i) - # module_classes = [_ for _ in tree.body if isinstance(_, _ast.ClassDef)] commands = {} for f in module_classes[0].body: if isinstance(f, _ast.FunctionDef) and not f.name.startswith('_'): @@ -77,4 +72,4 @@ def validate_driver_metadata(self, driver_path): 'the DriverMetadata.xml file: {}.\nPlease update the metadata or driver files accordingly.'.format(', '.join(missing)) raise Exception(err) - #TODO: add validation for command inputs as well + # TODO: add validation for command inputs as well From e114b08d808b1564d6c22cbaa2417d4ba5b67f3a Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Tue, 25 Sep 2018 20:00:18 -0700 Subject: [PATCH 4/5] get specific click version click 7.0 was released, progressbar.next() is not supported anymore. other things might not work as well and should be tested. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5be31dc..0c80f04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests cookiecutter qpm -click +click>=6.7,<6.8 pyyaml terminaltables cloudshell-rest-api>=8.2.1.0 From 15a0077b6f684cc4cfffd9ab121c5291441f6999 Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Wed, 26 Sep 2018 09:15:52 -0700 Subject: [PATCH 5/5] set click to 6.7 as in the master --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0c80f04..875cf9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests cookiecutter qpm -click>=6.7,<6.8 +click==6.7 pyyaml terminaltables cloudshell-rest-api>=8.2.1.0