From d5a0842422df0ef3e48037cf2596f7a1897768ac Mon Sep 17 00:00:00 2001 From: Serhii Koropets <33310880+koropets@users.noreply.github.com> Date: Tue, 30 May 2023 21:00:03 +0300 Subject: [PATCH] Fix readthedocs conf. And additional API docs. (#1322) * Last changes * Last changes * Last changes * Last changes * Last changes * docs requirements fix * Last changes --- .readthedocs.yml | 5 + README.md | 2 + docs/api/builder.rst | 11 +- docs/api/cli.rst | 12 ++ docs/api/index.rst | 2 + docs/api/machine/index.rst | 1 + docs/api/machine/loader.rst | 7 + docs/api/machine/machine.rst | 12 +- docs/api/machine/model/anomaly.rst | 10 +- docs/api/machine/model/index.rst | 5 +- docs/api/machine/model/model-factories.rst | 16 ++- docs/api/machine/model/model.rst | 12 ++ docs/api/machine/model/register.rst | 7 + docs/api/machine/model/transformer-funcs.rst | 2 +- docs/api/machine/model/transformers.rst | 2 +- docs/api/machine/model/utils.rst | 7 + docs/api/machine/validators.rst | 4 +- docs/api/reporters.rst | 22 +++ docs/api/serializer.rst | 12 +- docs/api/server/index.rst | 2 + docs/api/server/prometheus.rst | 17 +++ docs/api/server/properties.rst | 7 + docs/api/util.rst | 21 ++- docs/api/workflow.rst | 17 ++- docs/conf.py | 4 +- gordo/cli/cli.py | 17 ++- gordo/cli/exceptions_reporter.py | 15 +- gordo/machine/loader.py | 12 +- gordo/machine/machine.py | 15 +- gordo/machine/model/anomaly/base.py | 2 +- gordo/machine/model/anomaly/diff.py | 69 ++++------ .../factories/feedforward_autoencoder.py | 59 ++++---- .../model/factories/lstm_autoencoder.py | 59 ++++---- gordo/machine/model/factories/utils.py | 13 +- gordo/machine/model/models.py | 128 +++++++----------- gordo/machine/model/register.py | 31 +++-- .../model/transformer_funcs/general.py | 19 ++- gordo/machine/model/transformers/imputer.py | 8 +- gordo/machine/model/utils.py | 19 ++- gordo/machine/validators.py | 23 ++-- gordo/reporters/base.py | 1 - gordo/reporters/mlflow.py | 54 +++----- gordo/reporters/postgres.py | 3 +- gordo/serializer/from_definition.py | 40 +++--- gordo/serializer/into_definition.py | 8 +- gordo/serializer/serializer.py | 19 +-- gordo/server/blueprints/anomaly.py | 3 +- gordo/server/blueprints/base.py | 1 - gordo/server/model_io.py | 3 +- gordo/server/properties.py | 6 +- gordo/server/server.py | 21 ++- gordo/server/utils.py | 32 ++--- gordo/util/disk_registry.py | 16 +-- gordo/util/utils.py | 3 +- gordo/util/version.py | 3 +- gordo/workflow/workflow_generator/helpers.py | 7 +- .../workflow_generator/workflow_generator.py | 6 +- setup.py | 5 +- 58 files changed, 494 insertions(+), 445 deletions(-) create mode 100644 docs/api/cli.rst create mode 100644 docs/api/machine/loader.rst create mode 100644 docs/api/machine/model/model.rst create mode 100644 docs/api/machine/model/register.rst create mode 100644 docs/api/machine/model/utils.rst create mode 100644 docs/api/reporters.rst create mode 100644 docs/api/server/prometheus.rst create mode 100644 docs/api/server/properties.rst diff --git a/.readthedocs.yml b/.readthedocs.yml index 4e740b1e2..43a208d32 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,6 +4,11 @@ version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.9" + sphinx: builder: html configuration: docs/conf.py diff --git a/README.md b/README.md index 64714a755..d4220acef 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,8 @@ pytest -m 'dockertest' ### Build the documentation +This command will run the local documentation server: + ```console > cd docs/ > make watch diff --git a/docs/api/builder.rst b/docs/api/builder.rst index 502c5e597..d58a60cc2 100644 --- a/docs/api/builder.rst +++ b/docs/api/builder.rst @@ -2,7 +2,7 @@ Builder ------- Model builder -============= +^^^^^^^^^^^^^ .. automodule:: gordo.builder.build_model :members: :undoc-members: @@ -10,7 +10,7 @@ Model builder Local Model builder -=================== +^^^^^^^^^^^^^^^^^^^ This is meant to provide a good way to validate a configuration file as well as to enable creating and testing models locally with little overhead. @@ -20,3 +20,10 @@ as well as to enable creating and testing models locally with little overhead. :undoc-members: :show-inheritance: +Model builder utils +^^^^^^^^^^^^^^^^^^^ + +.. automodule:: gordo.builder.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/cli.rst b/docs/api/cli.rst new file mode 100644 index 000000000..ad85c20a1 --- /dev/null +++ b/docs/api/cli.rst @@ -0,0 +1,12 @@ +CLI +--- + +.. automodule:: gordo.cli.exceptions_reporter + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.cli.custom_types + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/index.rst b/docs/api/index.rst index f8a8a8e5f..3f8b61328 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -7,6 +7,8 @@ API ./builder.rst ./serializer.rst ./util.rst + ./cli.rst + ./reporters.rst ./workflow.rst ./server/index.rst ./machine/index.rst \ No newline at end of file diff --git a/docs/api/machine/index.rst b/docs/api/machine/index.rst index 9e60fbb15..bd8f72934 100644 --- a/docs/api/machine/index.rst +++ b/docs/api/machine/index.rst @@ -7,4 +7,5 @@ Machine ./machine.rst ./metadata.rst ./validators.rst + ./loader.rst ./model/index.rst \ No newline at end of file diff --git a/docs/api/machine/loader.rst b/docs/api/machine/loader.rst new file mode 100644 index 000000000..6861d34c4 --- /dev/null +++ b/docs/api/machine/loader.rst @@ -0,0 +1,7 @@ +Loader +------ + +.. automodule:: gordo.machine.loader + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/machine/machine.rst b/docs/api/machine/machine.rst index a2ddabc8a..a6bfe41ce 100644 --- a/docs/api/machine/machine.rst +++ b/docs/api/machine/machine.rst @@ -1,10 +1,20 @@ Machine ------- -A ``Machine`` is the central unity of a model, dataset, metadata and everything +A :class:`gordo.machine.Machine` is the central unity of a model, dataset, metadata and everything needed to create and build a ML model. .. automodule:: gordo.machine.machine + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.machine.encoders + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.machine.constants :members: :undoc-members: :show-inheritance: \ No newline at end of file diff --git a/docs/api/machine/model/anomaly.rst b/docs/api/machine/model/anomaly.rst index 59fad5206..574fb6924 100644 --- a/docs/api/machine/model/anomaly.rst +++ b/docs/api/machine/model/anomaly.rst @@ -1,12 +1,12 @@ Anomaly Models -------------- -Models which implment a ``.anomaly(X, y)`` and can be served under the -model server ``/anomaly/prediction`` endpoint. +Models which implment a :func:`gordo.machine.model.anomaly.base.AnomalyDetectorBase.anomaly` and can be served under the +model server :ref:`post-prediction` endpoint. AnomalyDetectorBase -=================== +^^^^^^^^^^^^^^^^^^^ The base class for all other anomaly detector models @@ -17,10 +17,10 @@ The base class for all other anomaly detector models DiffBasedAnomalyDetector -======================== +^^^^^^^^^^^^^^^^^^^^^^^^ Calculates the absolute value prediction differences between y and yhat as well -as the absolute difference error between both matrices via ``numpy.linalg.norm(..., axis=1)`` +as the absolute difference error between both matrices via :func:`numpy.linalg.norm` .. automodule:: gordo.machine.model.anomaly.diff :members: diff --git a/docs/api/machine/model/index.rst b/docs/api/machine/model/index.rst index 84229f110..04726084e 100644 --- a/docs/api/machine/model/index.rst +++ b/docs/api/machine/model/index.rst @@ -4,7 +4,10 @@ Model .. toctree:: :maxdepth: 2 + ./model.rst ./anomaly.rst ./model-factories.rst ./transformer-funcs.rst - ./transformers.rst \ No newline at end of file + ./transformers.rst + ./register.rst + ./utils.rst diff --git a/docs/api/machine/model/model-factories.rst b/docs/api/machine/model/model-factories.rst index 51ad4be0a..fcd1db71c 100644 --- a/docs/api/machine/model/model-factories.rst +++ b/docs/api/machine/model/model-factories.rst @@ -5,14 +5,14 @@ Model factories are stand alone functions which take an arbitrary number of primitive parameters (int, float, list, dict, etc) and return a model which can then be used in the ``kind`` parameter of some Scikit-Learn like wrapper model. -An example of this is ``KerasAutoEncoder`` which accepts a ``kind`` argument -(as all custom gordo models do) and can be given `feedforward_model`. Meaning +An example of this is :class:`gordo.machine.model.models.KerasAutoEncoder` which accepts a ``kind`` argument +(as all custom gordo models do) and can be given ``feedforward_model``. Meaning that function will be used to create the underlying Keras model for ``KerasAutoEncoder`` feedforward factories -===================== +^^^^^^^^^^^^^^^^^^^^^ .. automodule:: gordo.machine.model.factories.feedforward_autoencoder :members: @@ -20,9 +20,17 @@ feedforward factories :show-inheritance: lstm factories -============== +^^^^^^^^^^^^^^ .. automodule:: gordo.machine.model.factories.lstm_autoencoder :members: :undoc-members: :show-inheritance: + +Utils +^^^^^ + +.. automodule:: gordo.machine.model.factories.utils + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/api/machine/model/model.rst b/docs/api/machine/model/model.rst new file mode 100644 index 000000000..1cafade51 --- /dev/null +++ b/docs/api/machine/model/model.rst @@ -0,0 +1,12 @@ +Model +----- + +.. automodule:: gordo.machine.model.base + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.machine.model.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/machine/model/register.rst b/docs/api/machine/model/register.rst new file mode 100644 index 000000000..3ab663a54 --- /dev/null +++ b/docs/api/machine/model/register.rst @@ -0,0 +1,7 @@ +Register +-------- + +.. automodule:: gordo.machine.model.register + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/api/machine/model/transformer-funcs.rst b/docs/api/machine/model/transformer-funcs.rst index 50a1c4a36..816df428d 100644 --- a/docs/api/machine/model/transformer-funcs.rst +++ b/docs/api/machine/model/transformer-funcs.rst @@ -2,7 +2,7 @@ Transformer Functions --------------------- A collection of functions which can be referenced within the -``sklearn.preprocessing.FunctionTransformer`` transformer. +:class:``sklearn.preprocessing.FunctionTransformer`` transformer. General ======= diff --git a/docs/api/machine/model/transformers.rst b/docs/api/machine/model/transformers.rst index f207226f5..7f4ce66c3 100644 --- a/docs/api/machine/model/transformers.rst +++ b/docs/api/machine/model/transformers.rst @@ -3,7 +3,7 @@ Transformers Specialized transformers to address Gordo specific problems. This function just like Scikit-Learn's transformers and thus can be -inserted into ``Pipeline`` objects. +inserted into :class:`sklearn.pipeline.Pipeline` objects. Imputers ======== diff --git a/docs/api/machine/model/utils.rst b/docs/api/machine/model/utils.rst new file mode 100644 index 000000000..a7e8c5696 --- /dev/null +++ b/docs/api/machine/model/utils.rst @@ -0,0 +1,7 @@ +Utils +----- + +.. automodule:: gordo.machine.model.utils + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/api/machine/validators.rst b/docs/api/machine/validators.rst index dcb1b7c0d..08b76df73 100644 --- a/docs/api/machine/validators.rst +++ b/docs/api/machine/validators.rst @@ -1,5 +1,5 @@ -Descriptors ------------ +Validators +---------- Collection of descriptors to verify types and conditions of the Machine attributes when loading. diff --git a/docs/api/reporters.rst b/docs/api/reporters.rst new file mode 100644 index 000000000..f024b9af4 --- /dev/null +++ b/docs/api/reporters.rst @@ -0,0 +1,22 @@ +Reporters +--------- + +.. automodule:: gordo.reporters.base + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.reporters.exceptions + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.reporters.mlflow + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.reporters.postgres + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/api/serializer.rst b/docs/api/serializer.rst index 18798165f..f41699e37 100644 --- a/docs/api/serializer.rst +++ b/docs/api/serializer.rst @@ -16,7 +16,7 @@ objects which will be (de)serialized by the serializer to complete this goal. From Definition -=============== +^^^^^^^^^^^^^^^ The ability to take a 'raw' representation of an object in ``dict`` form and load it into a Python object. @@ -28,7 +28,7 @@ and load it into a Python object. Into Definitiion -================ +^^^^^^^^^^^^^^^^ The ability to take a Python object, such as a scikit-learn pipeline and convert it into a primitive ``dict``, which can then be inserted @@ -38,3 +38,11 @@ into a YAML config file. :members: :undoc-members: :show-inheritance: + +Utils +^^^^^ + +.. automodule:: gordo.serializer.utils + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/api/server/index.rst b/docs/api/server/index.rst index f478896c8..bf42ab25b 100644 --- a/docs/api/server/index.rst +++ b/docs/api/server/index.rst @@ -6,5 +6,7 @@ Server ./server.rst ./base.rst + ./properties.rst ./anomaly.rst + ./prometheus.rst ./utils.rst \ No newline at end of file diff --git a/docs/api/server/prometheus.rst b/docs/api/server/prometheus.rst new file mode 100644 index 000000000..93d89192d --- /dev/null +++ b/docs/api/server/prometheus.rst @@ -0,0 +1,17 @@ +Prometheus +---------- + +.. automodule:: gordo.server.prometheus.metrics + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.server.prometheus.server + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.server.prometheus.gunicorn_config + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/server/properties.rst b/docs/api/server/properties.rst new file mode 100644 index 000000000..b10f457a1 --- /dev/null +++ b/docs/api/server/properties.rst @@ -0,0 +1,7 @@ +Properties +---------- + +.. automodule:: gordo.server.properties + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/util.rst b/docs/api/util.rst index 3a80b0245..3d387539a 100644 --- a/docs/api/util.rst +++ b/docs/api/util.rst @@ -3,16 +3,27 @@ Util Project helpers, and associated functionality which have no home. -Disk Registry -============= -.. automodule:: gordo.util.disk_registry +.. automodule:: gordo.utils :members: :undoc-members: :show-inheritance: -Utils -===== .. automodule:: gordo.util.utils :members: :undoc-members: :show-inheritance: + +.. automodule:: gordo.util.text + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.util.version + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: gordo.util.disk_registry + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/workflow.rst b/docs/api/workflow.rst index 998b8fe90..8d6116156 100644 --- a/docs/api/workflow.rst +++ b/docs/api/workflow.rst @@ -7,14 +7,15 @@ and serve the ML models. Normalized Config -================= +^^^^^^^^^^^^^^^^^ + .. automodule:: gordo.workflow.config_elements.normalized_config :members: :undoc-members: :show-inheritance: Workflow Generator -================== +^^^^^^^^^^^^^^^^^^ Workflow loading/processing functionality to help the CLI 'workflow' sub-command. @@ -23,10 +24,18 @@ Workflow loading/processing functionality to help the CLI 'workflow' sub-command :undoc-members: :show-inheritance: +Schemas +^^^^^^^ + +.. automodule:: gordo.workflow.config_elements.schemas + :members: + :undoc-members: + :show-inheritance: + Helpers -======= +^^^^^^^ + .. automodule:: gordo.workflow.workflow_generator.helpers :members: :undoc-members: :show-inheritance: - diff --git a/docs/conf.py b/docs/conf.py index 230fdbde6..4500f4113 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,7 +53,9 @@ _ignore_linkcode_infos = [ # caused "OSError: could not find class definition" - {"module": "gordo_core.utils", "fullname": "PredictionResult"} + {"module": "gordo_core.utils", "fullname": "PredictionResult"}, + {'module': 'gordo.workflow.config_elements.schemas', 'fullname': 'Model.Config.extra'}, + {'module': 'gordo.reporters.postgres', 'fullname': 'Machine.DoesNotExist'} ] diff --git a/gordo/cli/cli.py b/gordo/cli/cli.py index 6fba99bc7..d30c4f155 100644 --- a/gordo/cli/cli.py +++ b/gordo/cli/cli.py @@ -196,10 +196,9 @@ def expand_model(model_config: str, model_parameters: dict): Parameters ---------- - model_config: str + model_config Jinja template which when expanded becomes a valid model config json. - model_parameters: - Parameters for the model config. + model_parameters Parameters for the model config. Raises ------ @@ -208,7 +207,6 @@ def expand_model(model_config: str, model_parameters: dict): Returns ------- - str The model config with variables expanded """ @@ -226,13 +224,13 @@ def expand_model(model_config: str, model_parameters: dict): def get_all_score_strings(machine): """Given metadata from the model builder this function returns a list of strings of the following format: - {metric_name}-{tag_name}_{fold-fold-number} = {score_val}. This computes the score for the given tag and + ``{metric_name}-{tag_name}_{fold-fold-number} = {score_val}``. This computes the score for the given tag and cross validation split. - {metric_name}-{tag_name}_{fold-aggregation} = {score_val}. This computes the score for the given tag and aggregates + ``{metric_name}-{tag_name}_{fold-aggregation} = {score_val}``. This computes the score for the given tag and aggregates the score over all cross validation splits (aggregations currently used are mean, std, min and max) - {metric_name}_{fold-fold-number} = {score_val}. This computes the score aggregate across all tags (uses sklearn's default + ``{metric_name}_{fold-fold-number} = {score_val}``. This computes the score aggregate across all tags (uses sklearn's default aggregation method) for a given cross validation split. - {metric_name}_{fold-aggregation} = {score_val}. This computes the score aggregate across all tags (uses sklearn's default + ``{metric_name}_{fold-aggregation} = {score_val}``. This computes the score aggregate across all tags (uses sklearn's default aggregation method) and cross validation splits (aggregations currently used are mean, std, min and max). for katib to pick up. @@ -244,7 +242,8 @@ def get_all_score_strings(machine): Parameters ---------- - machine : Machine + machine + Machine to score """ all_scores = [] for ( diff --git a/gordo/cli/exceptions_reporter.py b/gordo/cli/exceptions_reporter.py index 38860f4cb..397e0f8fe 100644 --- a/gordo/cli/exceptions_reporter.py +++ b/gordo/cli/exceptions_reporter.py @@ -51,9 +51,9 @@ def __init__( exceptions Exceptions list with preferred exit codes for each of them default_exit_code - Default exit code. It might be used as `sys.exit()` code + Default exit code. It might be used as :func:`sys.exit` code traceback_limit - Limit for `traceback.format_exception()` + Limit for :func:`traceback.format_exception` """ self.exceptions_items = self.sort_exceptions(exceptions) self.default_exit_code = default_exit_code @@ -108,16 +108,13 @@ def found_exception_item(self, exc_type: Type[BaseException]): def exception_exit_code(self, exc_type: Optional[Type[BaseException]]) -> int: """ - Possible `sys.exit()` code for given exception type + Possible :func:`sys.exit` code for given exception type Parameters ---------- exc_type The exception type - Returns - ------- - int """ if exc_type is None: return 0 @@ -135,11 +132,11 @@ def report( ): """ Report exception to the file. - `exc_type`, `exc_value`, `exc_traceback` might be values returned by `sys.exc_info()` + ``exc_type``, ``exc_value``, ``exc_traceback`` might be values returned by :func:`sys.exc_info` Parameters ---------- - level: ReportLevel + level Level of the report verbosity exc_type The exception type @@ -194,7 +191,7 @@ def safe_report( max_message_len: Optional[int] = None, ): """ - Basically this is a wrapper for `ExceptionsReporter.report()` function + Basically this is a wrapper for :func:`~ExceptionsReporter.report` function with additional internal exceptions handling Parameters diff --git a/gordo/machine/loader.py b/gordo/machine/loader.py index 8ab11e637..69fc3dd43 100644 --- a/gordo/machine/loader.py +++ b/gordo/machine/loader.py @@ -58,9 +58,9 @@ def load_globals_config(config: dict, json_path: str = None) -> GlobalsConfig: Parameters ---------- - config: str + config Config to load. - json_path: str + json_path JSON path position of the config. Returns @@ -76,9 +76,9 @@ def load_machine_config(config: dict, json_path: str = None) -> MachineConfig: Parameters ---------- - config: str + config Config to load. - json_path: str + json_path JSON path position of the config. Returns @@ -99,9 +99,9 @@ def load_model_config(config: dict, json_path: str = None) -> ModelConfig: Parameters ---------- - config: str + config Config to load. - json_path: str + json_path JSON path position of the config. Returns diff --git a/gordo/machine/machine.py b/gordo/machine/machine.py index e554ce9e1..f5992e337 100644 --- a/gordo/machine/machine.py +++ b/gordo/machine/machine.py @@ -89,19 +89,17 @@ def from_config( # type: ignore Parameters ---------- - config: dict[str, Any] + config The loaded block of config which represents a 'Machine' in YAML - project_name: str + project_name Name of the project this Machine belongs to. - config_globals: - The block of config within the YAML file within `globals` - back_compatibles: Optional[BackCompatibleLocations] + config_globals The block of config within the YAML file within `globals` + back_compatibles See `gordo_core.import_utils.prepare_back_compatible_locations()` function for reference. - default_data_provider: Optional[str] + default_data_provider Returns ------- - :class:`~Machine` """ if config_globals is None: config_globals = dict() @@ -156,11 +154,10 @@ def normalize_sensor_tags(self, tag_list: TagsList) -> List[SensorTag]: Parameters ---------- - tag_list: TagsList + tag_list Returns ------- - List[SensorTag] """ metadata = self.metadata diff --git a/gordo/machine/model/anomaly/base.py b/gordo/machine/model/anomaly/base.py index 0c064ab55..68e8b024e 100644 --- a/gordo/machine/model/anomaly/base.py +++ b/gordo/machine/model/anomaly/base.py @@ -17,7 +17,7 @@ def anomaly( frequency: Optional[timedelta] = None, ) -> Union[pd.DataFrame, xr.Dataset]: """ - Take X, y and optionally frequency; returning a dataframe containing + Take ``X``, ``y`` and optionally frequency; returning a dataframe containing anomaly score(s) """ ... diff --git a/gordo/machine/model/anomaly/diff.py b/gordo/machine/model/anomaly/diff.py index a3aaaf832..051610d21 100644 --- a/gordo/machine/model/anomaly/diff.py +++ b/gordo/machine/model/anomaly/diff.py @@ -41,27 +41,27 @@ def __init__( Parameters ---------- - base_estimator: sklearn.base.BaseEstimator + base_estimator The model to which normal ``.fit``, ``.predict`` methods will be used. defaults to py:class:`gordo.machine.model.models.KerasAutoEncoder` with ``kind='feedforward_hourglass`` - scaler: sklearn.base.TransformerMixin + scaler Defaults to ``sklearn.preprocessing.RobustScaler`` Used for transforming model output and the original ``y`` to calculate the difference/error in model output vs expected. - require_thresholds: bool + require_thresholds Requires calculating ``thresholds_`` via a call to :func:`~DiffBasedAnomalyDetector.cross_validate`. If this is set (default True), but :func:`~DiffBasedAnomalyDetector.cross_validate` was not called before calling :func:`~DiffBasedAnomalyDetector.anomaly` an ``AttributeError`` will be raised. - shuffle: bool + shuffle Flag to shuffle or not data in ``.fit`` so that the model, if relevant, will be trained on a sample of data accross the time range and not just the last elements according to model arg ``validation_split``. - window: int + window Window size for smoothed thresholds - smoothing_method: str + smoothing_method Method to be used together with ``window`` to smooth metrics. Must be one of: 'smm': simple moving median, 'sma': simple moving average or 'ewma': exponential weighted moving average. @@ -88,10 +88,6 @@ def __getattr__(self, item): def get_metadata(self): """ Generates model metadata. - - Returns - ------- - dict """ metadata = dict() if hasattr(self, "feature_thresholds_"): @@ -156,10 +152,6 @@ def score( def get_params(self, deep=True): """ Get parameters for this estimator. - - Returns - ------- - dict """ params = { "base_estimator": self.base_estimator, @@ -195,17 +187,13 @@ def cross_validate( Parameters ---------- - X: Union[pd.DataFrame, np.ndarray] + X Input data to the model - y: Union[pd.DataFrame, np.ndarray] + y Target data - kwargs: dict + kwargs Any additional kwargs to be passed to :func:`sklearn.model_selection.cross_validate` - - Returns - ------- - dict """ # Depend on having the trained fold models kwargs.update(dict(return_estimator=True, cv=cv)) @@ -287,13 +275,13 @@ def _scaled_mse_per_timestep( Calculate the scaled MSE per timestep/sample Parameters ---------- - model: BaseEstimator + model Instance of a fitted :class:`~DiffBasedAnomalyDetector` - y_true: Union[numpy.ndarray, pd.DataFrame] - y_pred: Union[numpy.ndarray, pd.DataFrame] + y_true + y_pred + Returns ------- - panadas.Series The MSE calculated from the scaled y and y predicted. """ try: @@ -330,14 +318,13 @@ def anomaly( Parameters ---------- - X: pd.DataFrame + X Dataframe representing the data to go into the model. - y: pd.DataFrame + y Dataframe representing the target output of the model. Returns ------- - pd.DataFrame A superset of the original base dataframe with added anomaly specific features """ @@ -495,32 +482,32 @@ def __init__( Parameters ---------- - base_estimator: sklearn.base.BaseEstimator + base_estimator The model to which normal ``.fit``, ``.predict`` methods will be used. defaults to py:class:`gordo.machine.model.models.KerasAutoEncoder` with ``kind='feedforward_hourglass`` - scaler: sklearn.base.TransformerMixin + scaler Defaults to ``sklearn.preprocessing.RobustScaler`` Used for transforming model output and the original ``y`` to calculate the difference/error in model output vs expected. - require_thresholds: bool + require_thresholds Requires calculating ``thresholds_`` via a call to :func:`~DiffBasedAnomalyDetector.cross_validate`. If this is set (default True), but :func:`~DiffBasedAnomalyDetector.cross_validate` was not called before calling :func:`~DiffBasedAnomalyDetector.anomaly` an ``AttributeError`` will be raised. - shuffle: bool + shuffle Flag to shuffle or not data in ``.fit`` so that the model, if relevant, will be trained on a sample of data accross the time range and not just the last elements according to model arg ``validation_split``. - window: int + window Window size for smooth metrics and threshold calculation. - smoothing_method: str + smoothing_method Method to be used together with ``window`` to smooth metrics. Must be one of: 'smm': simple moving median, 'sma': simple moving average or 'ewma': exponential weighted moving average. - threshold_percentile: float + threshold_percentile Percentile of the validation data to be used to calculate the threshold. """ self.base_estimator = base_estimator @@ -537,7 +524,6 @@ def get_params(self, deep=True): Returns ------- - dict """ params = { "base_estimator": self.base_estimator, @@ -555,7 +541,6 @@ def get_metadata(self): Returns ------- - dict """ metadata = dict() @@ -592,17 +577,13 @@ def cross_validate( Parameters ---------- - X: Union[pd.DataFrame, np.ndarray] + X Input data to the model - y: Union[pd.DataFrame, np.ndarray] + y Target data - kwargs: dict + kwargs Any additional kwargs to be passed to :func:`sklearn.model_selection.cross_validate` - - Returns - ------- - dict """ # Depend on having the trained fold models diff --git a/gordo/machine/model/factories/feedforward_autoencoder.py b/gordo/machine/model/factories/feedforward_autoencoder.py index 9ed933a47..1e569dac5 100644 --- a/gordo/machine/model/factories/feedforward_autoencoder.py +++ b/gordo/machine/model/factories/feedforward_autoencoder.py @@ -31,35 +31,34 @@ def feedforward_model( Parameters ---------- - n_features: int + n_features Number of features the dataset X will contain. - n_features_out: Optional[int] + n_features_out Number of features the model will output, default to ``n_features``. - encoding_dim: tuple + encoding_dim Tuple of numbers with the number of neurons in the encoding part. - decoding_dim: tuple + decoding_dim Tuple of numbers with the number of neurons in the decoding part. - encoding_func: tuple + encoding_func Activation functions for the encoder part. - decoding_func: tuple + decoding_func Activation functions for the decoder part. - out_func: str + out_func Activation function for the output layer - optimizer: Union[str, Optimizer] + optimizer If str then the name of the optimizer must be provided (e.x. "Adam"). The arguments of the optimizer can be supplied in optimize_kwargs. If a Keras optimizer call the instance of the respective class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)). If no arguments are provided Keras default values will be set. - optimizer_kwargs: Dict[str, Any] + optimizer_kwargs The arguments for the chosen optimizer. If not provided Keras' default values will be used. - compile_kwargs: Dict[str, Any] + compile_kwargs Parameters to pass to ``keras.Model.compile``. Returns ------- - keras.models.Sequential """ @@ -120,31 +119,26 @@ def feedforward_symmetric( Parameters ---------- - n_features: int + n_features Number of input and output neurons. - n_features_out: Optional[int] + n_features_out Number of features the model will output, default to ``n_features``. - dim: List[int] + dim Number of neurons per layers for the encoder, reversed for the decoder. Must have len > 0. - funcs: List[str] + funcs Activation functions for the internal layers - optimizer: Union[str, Optimizer] + optimizer If str then the name of the optimizer must be provided (e.x. "Adam"). The arguments of the optimizer can be supplied in optimization_kwargs. If a Keras optimizer call the instance of the respective - class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)). If no arguments are + class (e.x. ``Adam(lr=0.01,beta_1=0.9, beta_2=0.999)``). If no arguments are provided Keras default values will be set. - optimizer_kwargs: Dict[str, Any] + optimizer_kwargs The arguments for the chosen optimizer. If not provided Keras' default values will be used. - compile_kwargs: Dict[str, Any] + compile_kwargs Parameters to pass to ``keras.Model.compile``. - - Returns - ------- - keras.models.Sequential - """ if len(dims) == 0: raise ValueError("Parameter dims must have len > 0") @@ -181,30 +175,30 @@ def feedforward_hourglass( Parameters ---------- - n_features: int + n_features Number of input and output neurons. - n_features_out: Optional[int] + n_features_out Number of features the model will output, default to ``n_features``. - encoding_layers: int + encoding_layers Number of layers from the input layer (exclusive) to the narrowest layer (inclusive). Must be > 0. The total nr of layers including input and output layer will be 2*encoding_layers + 1. - compression_factor: float + compression_factor How small the smallest layer is as a ratio of n_features (smallest layer is rounded up to nearest integer). Must satisfy 0 <= compression_factor <= 1. - func: str + func Activation function for the internal layers - optimizer: Union[str, Optimizer] + optimizer If str then the name of the optimizer must be provided (e.x. "Adam"). The arguments of the optimizer can be supplied in optimization_kwargs. If a Keras optimizer call the instance of the respective class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)). If no arguments are provided Keras default values will be set. - optimizer_kwargs: Dict[str, Any] + optimizer_kwargs The arguments for the chosen optimizer. If not provided Keras' default values will be used. - compile_kwargs: Dict[str, Any] + compile_kwargs Parameters to pass to ``keras.Model.compile``. Notes @@ -224,7 +218,6 @@ class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)). If no arguments are Returns ------- - keras.models.Sequential Examples -------- diff --git a/gordo/machine/model/factories/lstm_autoencoder.py b/gordo/machine/model/factories/lstm_autoencoder.py index adfe0c64f..d2240ce20 100644 --- a/gordo/machine/model/factories/lstm_autoencoder.py +++ b/gordo/machine/model/factories/lstm_autoencoder.py @@ -33,40 +33,39 @@ def lstm_model( Parameters ---------- - n_features: int + n_features Number of features the dataset X will contain. - n_features_out: Optional[int] + n_features_out Number of features the model will output, default to ``n_features``. - lookback_window: int + lookback_window Number of timesteps used to train the model. One timestep = current observation in the sample. Two timesteps = current observation + previous observation in the sample. ... - encoding_dim: tuple + encoding_dim Tuple of numbers with the number of neurons in the encoding part. - decoding_dim: tuple + decoding_dim Tuple of numbers with the number of neurons in the decoding part. - encoding_func: tuple + encoding_func Activation functions for the encoder part. - decoding_func: tuple + decoding_func Activation functions for the decoder part. - out_func: str + out_func Activation function for the output Dense layer. - optimizer: Union[str, Optimizer] + optimizer If str then the name of the optimizer must be provided (e.x. "Adam"). The arguments of the optimizer can be supplied in optimize_kwargs. If a Keras optimizer call the instance of the respective class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)). If no arguments are provided Keras default values will be set. - optimizer_kwargs: Dict[str, Any] + optimizer_kwargs The arguments for the chosen optimizer. If not provided Keras' default values will be used. - compile_kwargs: Dict[str, Any] + compile_kwargs Parameters to pass to ``keras.Model.compile``. Returns ------- - keras.models.Sequential Returns Keras sequential model. """ @@ -122,37 +121,36 @@ def lstm_symmetric( Parameters ---------- - n_features: int + n_features Number of input and output neurons. - n_features_out: Optional[int] + n_features_out Number of features the model will output, default to ``n_features``. - lookback_window: int + lookback_window Number of timesteps used to train the model. One timestep = sample contains current observation. Two timesteps = sample contains current and previous observation. ... - dims: Tuple[int,...] + dims Number of neurons per layers for the encoder, reversed for the decoder. Must have len > 0 - funcs: List[str] + funcs Activation functions for the internal layers. - out_func: str + out_func Activation function for the output Dense layer. - optimizer: Union[str, Optimizer] + optimizer If str then the name of the optimizer must be provided (e.x. "Adam"). The arguments of the optimizer can be supplied in optimization_kwargs. If a Keras optimizer call the instance of the respective class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)). If no arguments are provided Keras default values will be set. - optimizer_kwargs: Dict[str, Any] + optimizer_kwargs The arguments for the chosen optimizer. If not provided Keras' default values will be used. - compile_kwargs: Dict[str, Any] + compile_kwargs Parameters to pass to ``keras.Model.compile``. Returns ------- - keras.models.Sequential Returns Keras sequential model. """ @@ -200,11 +198,11 @@ def lstm_hourglass( Parameters ---------- - n_features: int + n_features Number of input and output neurons. - n_features_out: Optional[int] + n_features_out Number of features the model will output, default to ``n_features``. - encoding_layers: int + encoding_layers Number of layers from the input layer (exclusive) to the narrowest layer (inclusive). Must be > 0. The total nr of layers including input and output layer will be 2*encoding_layers + 1. @@ -212,26 +210,25 @@ def lstm_hourglass( How small the smallest layer is as a ratio of n_features (smallest layer is rounded up to nearest integer). Must satisfy 0 <= compression_factor <= 1. - func: str + func Activation function for the internal layers. - out_func: str + out_func Activation function for the output Dense layer. - optimizer: Union[str, Optimizer] + optimizer If str then the name of the optimizer must be provided (e.x. "Adam"). The arguments of the optimizer can be supplied in optimization_kwargs. If a Keras optimizer call the instance of the respective class (e.x. Adam(lr=0.01,beta_1=0.9, beta_2=0.999)). If no arguments are provided Keras default values will be set. - optimizer_kwargs: Dict[str, Any] + optimizer_kwargs The arguments for the chosen optimizer. If not provided Keras' default values will be used. - compile_kwargs: Dict[str, Any] + compile_kwargs Parameters to pass to ``keras.Model.compile``. Returns ------- - keras.models.Sequential Examples -------- diff --git a/gordo/machine/model/factories/utils.py b/gordo/machine/model/factories/utils.py index 687a08bf4..d0b04f3ba 100644 --- a/gordo/machine/model/factories/utils.py +++ b/gordo/machine/model/factories/utils.py @@ -12,20 +12,19 @@ def hourglass_calc_dims( Parameters ---------- - compression_factor: float + compression_factor How small the smallest layer is as a ratio of n_features (smallest layer is rounded up to nearest integer). Must satisfy 0 <= compression_factor <= 1. - encoding_layers: int + encoding_layers Number of layers from the input layer (exclusive) to the narrowest layer (inclusive). Must be > 0. The total nr of layers including input and output layer will be 2*encoding_layers + 1. - n_features_out: Optional[int] + n_features_out Number of features the model will output, default to ``n_features``. Returns ------- - dims: Tuple[int,...] Number of neurons per layers for the encoder, reversed for the decoder. Must have len > 0 """ @@ -48,11 +47,11 @@ def check_dim_func_len(prefix: str, dim: Tuple[int, ...], func: Tuple[str, ...]) Parameters ---------- - prefix: str + prefix Parameter name prefix for error message generation (Options: "encoding" or "decoding"). - dim: tuple of int + dim Tuple of numbers with the number of neurons in the encoding or decoding part. - func: Tuple[str,...] + func Tuple of numbers with the number of neurons in the decoding part. """ if len(dim) != len(func): diff --git a/gordo/machine/model/models.py b/gordo/machine/model/models.py index a639ace18..236b808d7 100644 --- a/gordo/machine/model/models.py +++ b/gordo/machine/model/models.py @@ -61,15 +61,14 @@ def __init__( ) -> None: """ Initialized a Scikit-Learn API compatitble Keras model with a pre-registered - function or a builder function - directly. + function or a builder function directly. Parameters ---------- - kind: Union[callable, str] + kind The structure of the model to build. As designated by any registered builder functions, registered with - `gordo_compontents.model.register.register_model_builder`. + :func:`gordo.machine.model.register.register_model_builder`. Alternatively, one may pass a builder function directly to this argument. Such a function should accept `n_features` as it's first argument, and pass any additional parameters to `**kwargs` @@ -126,10 +125,7 @@ def extract_supported_fit_args(cls, kwargs): Parameters ---------- - kwargs: dict - - Returns - ------- + kwargs """ fit_args = {} @@ -141,14 +137,12 @@ def extract_supported_fit_args(cls, kwargs): @classmethod def from_definition(cls, definition: dict): """ - Handler for ``gordo.serializer.from_definition`` + Handler for :func:`gordo.serializer.from_definition` Parameters ---------- - definition: dict - - Returns - ------- + definition + Model definition """ kind = definition.pop("kind") @@ -161,7 +155,6 @@ def into_definition(self) -> dict: Returns ------- - dict """ definition = copy(self.kwargs) @@ -249,19 +242,14 @@ def fit( Parameters ---------- - X: Union[np.ndarray, pd.DataFrame, xr.Dataset] + X numpy array or pandas dataframe - y: Union[np.ndarray, pd.DataFrame, xr.Dataset] + y numpy array or pandas dataframe - sample_weight: np.ndarray + sample_weight array like - weight to assign to samples kwargs Any additional kwargs to supply to keras fit method. - - Returns - ------- - self - 'KerasAutoEncoder' """ # Reshape y if needed, and set n features of target @@ -291,16 +279,10 @@ def predict(self, X: np.ndarray, **kwargs) -> np.ndarray: Parameters ---------- - X: np.ndarray + X Input data - kwargs: dict + kwargs kwargs which are passed to Kera's ``predict`` method - - - Returns - ------- - results: - np.ndarray """ kwargs.setdefault("verbose", 0) return self.model.predict(X, **kwargs) @@ -316,7 +298,6 @@ def get_params(self, **params): Returns ------- - Dict[str, Any] Parameters used in this estimator """ params = super().get_params(**params) @@ -351,7 +332,6 @@ def get_metadata(self): Returns ------- - Dict Metadata dictionary, including a history object if present """ if hasattr(self, "model") and hasattr(self, "history"): @@ -379,18 +359,17 @@ def score( Parameters ---------- - X: Union[np.ndarray, pd.DataFrame] + X Input data to the model - y: Union[np.ndarray, pd.DataFrame] + y Target - sample_weight: Optional[np.ndarray] + sample_weight sample weights kwargs Additional kwargs for model.predict() Returns ------- - score: float Returns the explained variance score """ if not hasattr(self, "model"): @@ -406,8 +385,9 @@ def score( class KerasRawModelRegressor(KerasAutoEncoder): """ - Create a scikit-learn like model with an underlying tensorflow.keras model + Create a scikit-learn like model with an underlying ``tensorflow.keras`` model from a raw config. + Examples -------- >>> import yaml @@ -465,8 +445,7 @@ def __call__(self): class KerasLSTMBaseEstimator(KerasBaseEstimator, TransformerMixin, metaclass=ABCMeta): """ - Abstract Base Class to allow to train a many-one LSTM autoencoder and an LSTM - 1 step forecast + Abstract Base Class to allow to train a many-one LSTM autoencoder and an LSTM 1 step forecast """ def __init__( @@ -479,24 +458,23 @@ def __init__( """ Parameters ---------- - kind: Union[Callable, str] + kind The structure of the model to build. As designated by any registered builder - functions, registered with - `gordo.machine.model.register.register_model_builder`. + functions, registered with :func:`gordo.machine.model.register.register_model_builder`. Alternatively, one may pass a builder function directly to this argument. - Such a function should accept `n_features` as it's first argument, and pass - any additional parameters to `**kwargs`. - lookback_window: int + Such a function should accept ``n_features`` as it's first argument, and pass + any additional parameters to ``**kwargs``. + lookback_window Number of timestamps (lags) used to train the model. - batch_size: int + batch_size Number of training examples used in one epoch. - epochs: int + epochs Number of epochs to train the model. An epoch is an iteration over the entire data provided. - verbose: int + verbose Verbosity mode. Possible values are 0, 1, or 2 where 0 = silent, 1 = progress bar, 2 = one line per epoch. - kwargs: dict + kwargs Any arguments which are passed to the factory building function and/or any additional args to be passed to the intermediate fit method. """ @@ -541,7 +519,6 @@ def get_metadata(self): Returns ------- - metadata: dict Metadata dictionary, including forecast steps. """ metadata = super().get_metadata() @@ -570,16 +547,15 @@ def fit( # type: ignore Parameters ---------- - X: np.ndarray + X 2D numpy array of dimension n_samples x n_features. Input data to train. - y: np.ndarray + y 2D numpy array representing the target - kwargs: dict - Any additional args to be passed to Keras `fit_generator` method. + kwargs + Any additional args to be passed to Keras ``fit_generator`` method. Returns ------- - class: KerasLSTMForecast """ @@ -628,19 +604,15 @@ def predict(self, X: np.ndarray, **kwargs) -> np.ndarray: """ Parameters ---------- - X: np.ndarray - Data to predict/transform. 2D numpy array of dimension `n_samples x - n_features` where `n_samples` must be > lookback_window. + X + Data to predict/transform. 2D numpy array of dimension ``n_samples x n_features`` where ``n_samples`` must be > lookback_window. Returns ------- - results: np.ndarray - 2D numpy array of dimension `(n_samples - lookback_window) x - 2*n_features`. The first half of the array `(results[:, - :n_features])` corresponds to X offset by `lookback_window+1` (i.e., - `X[lookback_window:,:]`) whereas the second half corresponds to the - predicted values of `X[lookback_window:,:]`. - + 2D numpy array of dimension ``(n_samples - lookback_window) x 2*n_features``. + The first half of the array ``(results[:, :n_features])`` corresponds to X offset + by ``lookback_window+1`` (i.e., ``X[lookback_window:,:]``) whereas the second half corresponds to + the predicted values of ``X[lookback_window:,:]``. Example ------- @@ -681,22 +653,21 @@ def score( ) -> float: """ Returns the explained variance score between 1 step forecasted input and true - input at next time step (note: for LSTM X is offset by `lookback_window`). + input at next time step (note: for LSTM X is offset by ``lookback_window``). Parameters ---------- - X: Union[np.ndarray, pd.DataFrame] + X Input data to the model. - y: Union[np.ndarray, pd.DataFrame] + y Target - sample_weight: Optional[np.ndarray] + sample_weight Sample weights kwargs Additional kwargs for predict Returns ------- - score: float Returns the explained variance score. """ if not hasattr(self, "model"): @@ -732,7 +703,7 @@ def create_keras_timeseriesgenerator( lookahead: int, ) -> tensorflow.keras.preprocessing.sequence.TimeseriesGenerator: """ - Provides a `keras.preprocessing.sequence.TimeseriesGenerator` for use with + Provides a :class:`keras.preprocessing.sequence.TimeseriesGenerator` for use with LSTM's, but with the added ability to specify the lookahead of the target in y. If lookahead==0 then the generated samples in X will have as their last element @@ -743,25 +714,24 @@ def create_keras_timeseriesgenerator( Parameters ---------- - X: np.ndarray + X 2d array of values, each row being one sample. - y: Optional[np.ndarray] + y array representing the target. - batch_size: int + batch_size How big should the generated batches be? - lookback_window: int + lookback_window How far back should each sample see. 1 means that it contains a single measurement - lookahead: int + lookahead How much is Y shifted relative to X Returns ------- - TimeseriesGenerator 3d matrix with a list of batchX-batchY pairs, where batchX is a batch of - X-values, and correspondingly for batchY. A batch consist of `batch_size` nr + X-values, and correspondingly for batchY. A batch consist of ``batch_size`` nr of pairs of samples (or y-values), and each sample is a list of length - `lookback_window`. + ``lookback_window``. Examples ------- diff --git a/gordo/machine/model/register.py b/gordo/machine/model/register.py index b5022c48f..aa85ac5bb 100644 --- a/gordo/machine/model/register.py +++ b/gordo/machine/model/register.py @@ -9,28 +9,29 @@ class register_model_builder: """ - Decorator to register a function as an available 'type' in supporting - factory classes such as gordo_compontents.models._models.KerasAutoEncoder. + Decorator to register a function as an available 'type' in supporting + factory classes such as :class:`gordo.machine.model.models.KerasAutoEncoder`. - When submitting the config file, it's important that the 'kind' is compatible - with 'type'. + When submitting the config file, it's important that the 'kind' is compatible + with 'type'. - ie. 'type': 'KerasAutoEncoder' should support the object returned by a given - decorated function. + ie. 'type': 'KerasAutoEncoder' should support the object returned by a given + decorated function. - Example for KerasAutoEncoder: + .. code:: python - from gordo_compontents.models.register import register_model_builder + from gordo_compontents.models.register import register_model_builder - @register_model_builder(type='KerasAutoEncoder') - def special_keras_model_builder(n_features, ...): - ... + @register_model_builder(type='KerasAutoEncoder') + def special_keras_model_builder(n_features, ...): + ... + + A valid yaml config would be: + model: + gordo.machine.models.KerasAutoEncoder: + kind: special_keras_model_builder - A valid yaml config would be: - model: - gordo.machine.models.KerasAutoEncoder: - kind: special_keras_model_builder """ """ diff --git a/gordo/machine/model/transformer_funcs/general.py b/gordo/machine/model/transformer_funcs/general.py index e98f783f1..941d9d620 100644 --- a/gordo/machine/model/transformer_funcs/general.py +++ b/gordo/machine/model/transformer_funcs/general.py @@ -1,22 +1,21 @@ # -*- coding: utf-8 -*- """ -Functions to be used within sklearn's FunctionTransformer -https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.FunctionTransformer.html +Functions to be used within sklearn's :func:`sklearn.preprocessing.FunctionTransformer`. Each function SHALL take an X, and optionally a y. Functions CAN take additional arguments which should be given during the initialization of the FunctionTransformer -Example: +.. code:: python ->>> from sklearn.preprocessing import FunctionTransformer ->>> import numpy as np ->>> def my_function(X, another_arg): -... # Some fancy X manipulation... -... return X ->>> transformer = FunctionTransformer(func=my_function, kw_args={'another_arg': 'this thing'}) ->>> out = transformer.fit_transform(np.random.random(100).reshape(10, 10)) + >>> from sklearn.preprocessing import FunctionTransformer + >>> import numpy as np + >>> def my_function(X, another_arg): + ... # Some fancy X manipulation... + ... return X + >>> transformer = FunctionTransformer(func=my_function, kw_args={'another_arg': 'this thing'}) + >>> out = transformer.fit_transform(np.random.random(100).reshape(10, 10)) """ diff --git a/gordo/machine/model/transformers/imputer.py b/gordo/machine/model/transformers/imputer.py index 85bb448a8..5c3d6c57b 100644 --- a/gordo/machine/model/transformers/imputer.py +++ b/gordo/machine/model/transformers/imputer.py @@ -24,18 +24,18 @@ def __init__( Parameters ---------- - inf_fill_value: numeric + inf_fill_value Value to fill 'inf' values - neg_inf_fill_value: numeric + neg_inf_fill_value Value to fill '-inf' values - strategy: str + strategy How to fill values, irrelevant if fill value is provided. choices: 'extremes', 'minmax' -'extremes' will use the min and max values for the current datatype. such that 'inf' in a float32 dataset will have float32's largest value inserted. - 'minmax' will look at the min and max values in the feature where the -inf / inf appears and fill with the max/min found in that feature. - delta: float + delta Only applicable if ``strategy='minmax'`` Will add/subtract the max/min value, by feature, by this delta. If the max value in a feature was 10 and ``delta=2`` any inf value will be filled with 12. diff --git a/gordo/machine/model/utils.py b/gordo/machine/model/utils.py index cd22a3cc6..3e17b1865 100644 --- a/gordo/machine/model/utils.py +++ b/gordo/machine/model/utils.py @@ -26,10 +26,10 @@ def metric_wrapper(metric, scaler: Optional[TransformerMixin] = None): ---------- metric Metric which must accept y_true and y_pred of the same length - scaler : Optional[TransformerMixin] + scaler Transformer which will be applied on y and y_pred before the metrics is - calculated. Must have method `transform`, so for most scalers it must already - be fitted on `y`. + calculated. Must have method ``transform``, so for most scalers it must already + be fitted on ``y``. """ @functools.wraps(metric) @@ -61,25 +61,24 @@ def make_base_dataframe( Parameters ---------- - tags: List[Union[str, SensorTag]] + tags Tags which will be assigned to ``model-input`` and/or ``model-output`` if the shapes match. - model_input: np.ndarray + model_input Original input given to the model - model_output: np.ndarray + model_output Raw model output - target_tag_list: Optional[Union[List[SensorTag], List[str]]] + target_tag_list Tags to be assigned to ``model-output`` if not assigned but model output matches model input, ``tags`` will be used. - index: Optional[np.ndarray] + index The index which should be assigned to the resulting dataframe, will be clipped to the length of ``model_output``, should the model output less than its input. - frequency: Optional[datetime.timedelta] + frequency The spacing of the time between points. Returns ------- - pd.DataFrame """ # Set target_tag_list to default to tags if not specified. diff --git a/gordo/machine/validators.py b/gordo/machine/validators.py index d3cd51d04..0711bed74 100644 --- a/gordo/machine/validators.py +++ b/gordo/machine/validators.py @@ -20,7 +20,7 @@ class BaseDescriptor: """ Base descriptor class - New object should override __set__(self, instance, value) method to check + New object should override ``__set__(self, instance, value)`` method to check if 'value' meets required needs. """ @@ -36,7 +36,7 @@ def __set__(self, instance, value): class ValidDataset(BaseDescriptor): """ - Descriptor for attributes requiring type :class:`gordo.workflow.config_elements.Dataset` + Descriptor for attributes requiring type :class:`gordo_core.time_series.TimeSeriesDataset` """ def __set__(self, instance, value): @@ -53,7 +53,7 @@ def __set__(self, instance, value): class ValidDatasetKwargs(BaseDescriptor): """ - Descriptor for attributes requiring type :class:`gordo.workflow.config_elements.Dataset` + Descriptor for attributes requiring type :class:`gordo_core.time_series.TimeSeriesDataset` """ def _verify_resolution(self, resolution: str): @@ -178,7 +178,7 @@ def fix_resource_limits(resources: dict) -> dict: Parameters ---------- - resources: dict + resources Dictionary with possible requests/limits Examples @@ -191,8 +191,7 @@ def fix_resource_limits(resources: dict) -> dict: Returns ------- - dict: - A copy of `resource_dict` with the any limits bumped to the corresponding request if + A copy of ``resource_dict`` with the any limits bumped to the corresponding request if they are both set. """ resources = copy.deepcopy(resources) @@ -269,10 +268,10 @@ def __set__(self, instance, value): class ValidUrlString(BaseDescriptor): """ - Descriptor for use in objects which require valid URL values. - Where 'valid URL values' is Gordo's version: alphanumeric with dashes. + Descriptor for use in objects which require valid URL values. + Where 'valid URL values' is Gordo's version: alphanumeric with dashes. - Use: + Use: .. code-block:: python @@ -307,11 +306,9 @@ def valid_url_string(string: str) -> bool: Parameters ---------- - string: str - String to check + string + String to check - Returns - ------- - bool """ return bool( re.match( diff --git a/gordo/reporters/base.py b/gordo/reporters/base.py index 0405e9f83..d448e22f4 100644 --- a/gordo/reporters/base.py +++ b/gordo/reporters/base.py @@ -21,7 +21,6 @@ def to_dict(self) -> dict: Returns ------- - dict """ return serializer.into_definition(self) diff --git a/gordo/reporters/mlflow.py b/gordo/reporters/mlflow.py index e45ef9a8a..c7c64793c 100644 --- a/gordo/reporters/mlflow.py +++ b/gordo/reporters/mlflow.py @@ -40,9 +40,9 @@ def _validate_dict(d: dict, required_keys: List[str]): Parameters ---------- - d: dict + d Dictionary to validate. - required_keys: List[str] + required_keys Keys that must be present in provided dictionary. """ if any([key not in d for key in required_keys]): @@ -59,7 +59,7 @@ def get_mlflow_client( Parameters ---------- - workspace_kwargs: dict + workspace_kwargs AzureML Workspace configuration to use for remote MLFlow tracking. An empty dict will result in local logging by the MlflowClient. Example:: @@ -82,7 +82,6 @@ def get_mlflow_client( Returns ------- - client: mlflow.tracking.MlflowClient Client with tracking uri set to AzureML if configured. """ logger.info("Creating MLflow tracking client.") @@ -128,16 +127,15 @@ def get_run_id(client: MlflowClient, experiment_name: str, model_key: str) -> st Parameters ---------- - client: mlflow.tracking.MlflowClient + client Client with tracking uri set to AzureML if configured. - experiment_name: str + experiment_name Name of experiment to log to. - model_key: str + model_key Unique ID of model configuration. Returns ------- - run_id: str Unique ID of MLflow run to log to. """ experiment = client.get_experiment_by_name(experiment_name) @@ -156,12 +154,11 @@ def _datetime_to_ms_since_epoch(dt: datetime) -> int: Parameters ---------- - dt: datetime.datetime + dt Timestamp to convert (can be timezone aware or naive). Returns ------- - dt: int Timestamp as milliseconds since Unix epoch Example @@ -181,7 +178,6 @@ def epoch_now() -> int: Returns ------- - now: int Milliseconds since Unix epoch. """ return _datetime_to_ms_since_epoch(datetime.now(tz=UTC)) @@ -191,18 +187,16 @@ def get_machine_log_items(machine: Machine) -> Tuple[List[Metric], List[Param]]: """ Create flat lists of MLflow logging entities from multilevel dictionary - For more information, see the mlflow docs: - https://www.mlflow.org/docs/latest/python_api/mlflow.tracking.html#mlflow.tracking.MlflowClient.log_batch + For more information, see the `mlflow docs `_. Parameters ---------- - machine: Machine + machine + Machine to log. Returns ------- - metrics: List[Metric] List of MLFlow Metric objects to log. - params: List[Param] List of MLFlow Param objects to log. """ @@ -296,18 +290,17 @@ def batch_log_items( Parameters ---------- - metrics: List[Metric] + metrics List of MLFlow Metric objects to log. - params: List[Param] + params List of MLFlow Param objects to log. - n_max_metrics:int + n_max_metrics Limit to number of metrics AzureML allows per batch log request payload. - n_max_params:int + n_max_params Limit to number of params MLFlow allows per batch log request payload. Returns ------- - log_batches: List[Dict[str, Union[Metric, Param]]] List of MlflowClinet.log_batch keyworkd arguments, split to quatnitites that respect limits present for MLFlow and AzureML. """ @@ -345,13 +338,12 @@ def get_kwargs_from_secret(name: str, keys: List[str]) -> dict: Parameters ---------- - name: str + name Name of the environment variable whose content is a colon separated list of secrets. Returns ------- - kwargs: dict Dictionary of keyword arguments parsed from environment variable. """ secret_str = os.getenv(name) @@ -381,7 +373,6 @@ def get_workspace_kwargs() -> dict: Returns ------- - workspace_kwargs: dict AzureML Workspace configuration to use for remote MLFlow tracking. See :func:`gordo.builder.mlflow_utils.get_mlflow_client`. """ @@ -399,7 +390,6 @@ def get_spauth_kwargs() -> dict: Returns ------- - service_principal_kwargs: dict AzureML ServicePrincipalAuthentication keyword arguments. See :func:`gordo.builder.mlflow_utils.get_mlflow_client` """ @@ -421,14 +411,14 @@ def mlflow_context( Parameters ---------- - name: str + name The name of the log group to log to (e.g. a model name). - model_key: str + model_key Unique ID of logging run. - workspace_kwargs: dict + workspace_kwargs AzureML Workspace configuration to use for remote MLFlow tracking. See :func:`gordo.builder.mlflow_utils.get_mlflow_client`. - service_principal_kwargs: dict + service_principal_kwargs AzureML ServicePrincipalAuthentication keyword arguments. See :func:`gordo.builder.mlflow_utils.get_mlflow_client` @@ -457,11 +447,11 @@ def log_machine(mlflow_client: MlflowClient, run_id: str, machine: Machine): Parameters ---------- - mlflow_client: MlflowClient + mlflow_client Client instance to call logging methods from. - run_id: str + run_id Unique ID off MLflow Run to log to. - machine: Machine + machine Machine to log with MlflowClient. """ # Log machine metrics and params diff --git a/gordo/reporters/postgres.py b/gordo/reporters/postgres.py index b37282777..33e121f1e 100644 --- a/gordo/reporters/postgres.py +++ b/gordo/reporters/postgres.py @@ -66,11 +66,10 @@ def report(self, machine: GordoMachine): Parameters ---------- - machine: gordo.machine.Machine + machine Returns ------- - None """ try: with self.db.atomic(): diff --git a/gordo/serializer/from_definition.py b/gordo/serializer/from_definition.py index 2a122af79..4412e7947 100644 --- a/gordo/serializer/from_definition.py +++ b/gordo/serializer/from_definition.py @@ -57,7 +57,6 @@ def from_definition( Returns ------- - sklearn.pipeline.Pipeline pipeline """ # Avoid some mutation @@ -142,21 +141,22 @@ def _build_step( Parameters ---------- - step: dict/str - A dict, with a single key and associated dict - where the associated dict are parameters for the - given step. - - Example: {'sklearn.preprocessing.PCA': - {'n_components': 4} - } - Gives: PCA(n_components=4) - - Alternatively, 'step' can be a single string, in - which case the step will be initiated w/ default - params. - - Example: 'sklearn.preprocessing.PCA' - Gives: PCA() + step + A dict, with a single key and associated dict + where the associated dict are parameters for the + given step. + + Example: {'sklearn.preprocessing.PCA': + {'n_components': 4} + } + Gives: PCA(n_components=4) + + Alternatively, 'step' can be a single string, in + which case the step will be initiated w/ default + params. + + Example: 'sklearn.preprocessing.PCA' + Gives: PCA() Returns ------- Scikit-Learn Transformer or BaseEstimator @@ -252,7 +252,7 @@ def _build_callbacks(definitions: list): """ Parameters ---------- - definitions: List + definitions List of callbacks definitions Examples @@ -263,7 +263,6 @@ def _build_callbacks(definitions: list): Returns ------- - dict """ callbacks = [] for callback in definitions: @@ -282,7 +281,7 @@ def _load_param_classes(params: dict): Parameters ---------- - params: dict + params key value pairs of kwargs, which can have full class paths defined. Examples @@ -304,7 +303,6 @@ def _load_param_classes(params: dict): Returns ------- - dict Updated params which has any possible class paths loaded up as instantiated objects """ @@ -362,7 +360,7 @@ def load_params_from_definition(definition: dict) -> dict: Parameters ---------- - definition: dict + definition """ if not isinstance(definition, dict): raise ValueError( diff --git a/gordo/serializer/into_definition.py b/gordo/serializer/into_definition.py index b379e45dc..ae13588b9 100644 --- a/gordo/serializer/into_definition.py +++ b/gordo/serializer/into_definition.py @@ -19,17 +19,16 @@ def into_definition( Parameters ---------- - pipeline: sklearn.pipeline.Pipeline + pipeline Instance of pipeline to decompose - prune_default_params: bool + prune_default_params Whether to prune the default parameters found in current instance of the transformers vs what their default params are. - tuples_to_list: bool + tuples_to_list Convert all tuples in output to lists Returns ------- - dict definitions for the pipeline, compatible to be reconstructed with :func:`gordo.serializer.from_definition` @@ -84,7 +83,6 @@ def _decompose_node( Returns ------- - dict decomposed node - Where key is the import string for the class and associated value is a dict of parameters for that class. """ diff --git a/gordo/serializer/serializer.py b/gordo/serializer/serializer.py index 996a253ca..306b2e85e 100644 --- a/gordo/serializer/serializer.py +++ b/gordo/serializer/serializer.py @@ -26,12 +26,11 @@ def dumps(model: Union[Pipeline, GordoBase]) -> bytes: Parameters ---------- - model: Union[Pipeline, GordoBase] + model A gordo model/pipeline Returns ------- - bytes Serialized model which supports loading via ``serializer.loads()`` Example @@ -55,12 +54,11 @@ def loads(bytes_object: bytes) -> GordoBase: Parameters ---------- - bytes_object: bytes + bytes_object Bytes to be loaded, should be the result of `serializer.dumps(model)` Returns ------- - Union[GordoBase, Pipeline, BaseEstimator] Custom gordo model, scikit learn pipeline or other scikit learn like object. """ return pickle.loads(bytes_object) @@ -103,13 +101,12 @@ def load_metadata(source_dir: Union[os.PathLike, str]) -> dict: Parameters ---------- - source_dir: Union[os.PathLike, str] + source_dir Directory of the saved model, As with serializer.load(source_dir) this source_dir can be the top level, or the first dir into the serialized model. Returns ------- - dict Raises ------ @@ -137,12 +134,11 @@ def load(source_dir: Union[os.PathLike, str]) -> Any: Parameters ---------- - source_dir: Union[os.PathLike, str] + source_dir Location of the top level dir the pipeline was saved Returns ------- - Union[GordoBase, Pipeline, BaseEstimator] """ # This source dir should have a single pipeline entry directory. # may have been passed a top level dir, containing such an entry: @@ -163,18 +159,17 @@ def dump( ---------- obj The object to dump. Must be pickle-able. - dest_dir: Union[os.PathLike, str] + dest_dir The directory to which to save the model metadata: dict - any additional metadata to be saved alongside this model if it exists, will be returned from the corresponding "load" function - metadata: Optional dict of metadata which will be serialized to a file together + metadata with the model, and loaded again by :func:`load_metadata`. - info: Optional[str] + info Current revision info. For now, only used for storing "checksum" Returns ------- - None Example ------- diff --git a/gordo/server/blueprints/anomaly.py b/gordo/server/blueprints/anomaly.py index 9a1f58a32..91983d955 100644 --- a/gordo/server/blueprints/anomaly.py +++ b/gordo/server/blueprints/anomaly.py @@ -29,13 +29,12 @@ def _create_anomaly_response(start_time: float = None): Parameters ---------- - start_time: Optional[float] + start_time Start time to use when timing the processing time of the request, will construct a new one if not provided. Returns ------- - flask.Response The formatted anomaly representation response object. """ if start_time is None: diff --git a/gordo/server/blueprints/base.py b/gordo/server/blueprints/base.py index c2b02a990..23582ef4d 100644 --- a/gordo/server/blueprints/base.py +++ b/gordo/server/blueprints/base.py @@ -173,7 +173,6 @@ def get_download_model(): Returns ------- - bytes Results from ``gordo.serializer.dumps()`` """ serialized_model = serializer.dumps(g.model) diff --git a/gordo/server/model_io.py b/gordo/server/model_io.py index b9ba5d0c1..b2b5bc088 100644 --- a/gordo/server/model_io.py +++ b/gordo/server/model_io.py @@ -21,12 +21,11 @@ def get_model_output(model: Pipeline, X: np.ndarray) -> np.ndarray: Parameters ---------- - X: np.ndarray + X 2d array of sample(s) Returns ------- - np.ndarray The raw output of the model in numpy array form. """ try: diff --git a/gordo/server/properties.py b/gordo/server/properties.py index 26b28c21e..50bd076d7 100644 --- a/gordo/server/properties.py +++ b/gordo/server/properties.py @@ -21,8 +21,8 @@ def find_path_in_dict(path: list[str], data: dict) -> Any: Parameters ---------- - path: List[str] - data: dict + path + data Returns ------- @@ -73,7 +73,6 @@ def get_tags() -> list[SensorTag]: Returns ------- - list[SensorTag] """ dataset = g.metadata["dataset"] tag_list = dataset["tag_list"] @@ -88,7 +87,6 @@ def get_target_tags() -> list[SensorTag]: Returns ------- - list[SensorTag] """ # TODO refactor this part to have the same tag preparation logic as in TimeSeriesDataset orig_target_tag_list = [] diff --git a/gordo/server/server.py b/gordo/server/server.py index fdaf1d8fb..1399c3ff9 100644 --- a/gordo/server/server.py +++ b/gordo/server/server.py @@ -50,7 +50,7 @@ def adapt_proxy_deployment(wsgi_app: typing.Callable) -> typing.Callable: Parameters ---------- - wsgi_app: typing.Callable + wsgi_app The underlying WSGI application of a flask app, for example Notes @@ -79,7 +79,6 @@ def adapt_proxy_deployment(wsgi_app: typing.Callable) -> typing.Callable: Returns ------- - Callable Example ------- @@ -254,24 +253,24 @@ def run_server( Parameters ---------- - host: str + host The host to run the server on. - port: int + port The port to run the server on. - workers: int + workers The number of worker processes for handling requests. - log_level: str + log_level The log level for the `gunicorn` webserver. Valid log level names can be found in the [gunicorn documentation](http://docs.gunicorn.org/en/stable/settings.html#loglevel). - config_module: str + config_module The config module. Will be passed with `python:` [prefix](https://docs.gunicorn.org/en/stable/settings.html#config). - worker_connections: int + worker_connections The maximum number of simultaneous clients per worker process. - threads: str + threads The number of worker threads for handling requests. - worker_class: str + worker_class The type of workers to use. - server_app: str + server_app The application to run """ diff --git a/gordo/server/utils.py b/gordo/server/utils.py index 36921a968..9cf1a1942 100644 --- a/gordo/server/utils.py +++ b/gordo/server/utils.py @@ -52,14 +52,13 @@ def dataframe_into_parquet_bytes( Parameters ---------- - df: pd.DataFrame + df DataFrame to be compressed - compression: str + compression Compression to use, passed to :func:`pyarrow.parquet.write_table` Returns ------- - bytes """ table = pa.Table.from_pandas(df) buf = pa.BufferOutputStream() @@ -73,13 +72,12 @@ def dataframe_from_parquet_bytes(buf: bytes) -> pd.DataFrame: Parameters ---------- - buf: bytes + buf Bytes representing a parquet table. Can be the direct result from `func`::gordo.server.utils.dataframe_into_parquet_bytes Returns ------- - pandas.DataFrame """ table = pq.read_table(io.BytesIO(buf)) return table.to_pandas() @@ -98,12 +96,11 @@ def dataframe_to_dict(df: pd.DataFrame) -> dict: Parameters ---------- - df: pandas.DataFrame + df Dataframe expected to have columns of type :class:`pandas.MultiIndex` 2 levels deep. Returns ------- - List[dict] List of records representing the dataframe in a 'flattened' form. @@ -153,12 +150,11 @@ def dataframe_from_dict(data: dict) -> pd.DataFrame: Parameters ---------- - data: dict + data Data to be loaded into a MultiIndex column dataframe Returns ------- - pandas.core.DataFrame MultiIndex column dataframe. Examples @@ -218,15 +214,14 @@ def _verify_dataframe( Parameters ---------- - df: pandas.core.DataFrame + df DataFrame to verify. - expected_columns: List[str] + expected_columns List of expected column names to give if the dataframe does not consist of them but the number of columns matches ``len(expected_columns)`` Returns ------- - Union[flask.wrappers.Response, pandas.core.DataFrame] """ if not isinstance(df.columns, pd.MultiIndex): if not all(col in df.columns for col in expected_columns): @@ -266,13 +261,12 @@ def extract_X_y(method): Parameters ---------- - method: Callable + method The flask route to decorate, and will return it's own response object and will want to use ``flask.g.X`` and/or ``flask.g.y`` Returns ------- - flask.Response Will either run a :class:`flask.Response` with status code 400 if it fails to extract the X and optionally the y. Otherwise will run the decorated ``method`` which is also expected to return some sort of :class:`flask.Response` object. @@ -342,15 +336,14 @@ def load_model(directory: str, name: str) -> BaseEstimator: Parameters ---------- - directory: str + directory Directory to look for the model - name: str + name Name of the model to load, this would be the sub directory within the directory parameter. Returns ------- - BaseEstimator """ start_time = timeit.default_timer() model = serializer.load(os.path.join(directory, name)) @@ -374,15 +367,14 @@ def load_metadata(directory: str, name: str) -> dict: Parameters ---------- - directory: str + directory Directory to look for the model's metadata - name: str + name Name of the model to load metadata for, this would be the sub directory within the directory parameter. Returns ------- - dict """ compressed_metadata = _load_compressed_metadata(directory, name) return pickle.loads(zlib.decompress(compressed_metadata)) diff --git a/gordo/util/disk_registry.py b/gordo/util/disk_registry.py index d8d182f94..bb09460e1 100644 --- a/gordo/util/disk_registry.py +++ b/gordo/util/disk_registry.py @@ -20,12 +20,12 @@ def write_key(registry_dir: Union[os.PathLike, str], key: str, val: AnyStr): Parameters ---------- - registry_dir: Union[os.PathLike, str] + registry_dir Path to the registry. If it does not exists it will be created, including any missing folders in the path. - key: str + key Key to use for the key/value. Must be valid as a filename. - val: AnyStr + val Value to write to the registry. Examples @@ -60,14 +60,13 @@ def get_value(registry_dir: Union[os.PathLike, str], key: str) -> Optional[AnySt Parameters ---------- - registry_dir: Union[os.PathLike, str] + registry_dir Path to the registry. If it does not exist we return None - key: str + key Key to look up in the registry. Returns ------- - Optional[AnyStr]: The value of `key` in the registry, None if no value is registered with that key in the registry. """ @@ -95,14 +94,13 @@ def delete_value(registry_dir: Union[os.PathLike, str], key: str) -> bool: Parameters ---------- - registry_dir: Union[os.PathLike, str] + registry_dir Path to the registry. Does not need to exist - key: str + key Key to look up in the registry. Returns ------- - bool: True if the key existed, false otherwise """ key_file_path = Path(registry_dir).joinpath(key) diff --git a/gordo/util/utils.py b/gordo/util/utils.py index 16b956e86..8675dd0ac 100644 --- a/gordo/util/utils.py +++ b/gordo/util/utils.py @@ -11,12 +11,11 @@ def capture_args(method: Callable): Parameters ---------- - method: Callable + method Some method of an object, with 'self' as the first parameter. Returns ------- - Any Returns whatever the original method would return """ diff --git a/gordo/util/version.py b/gordo/util/version.py index 737bc5b4e..a43d3a5dd 100644 --- a/gordo/util/version.py +++ b/gordo/util/version.py @@ -94,7 +94,7 @@ def parse_version( Parameters ---------- - gordo_version: str + gordo_version Example ------- @@ -105,7 +105,6 @@ def parse_version( Returns ------- - Union[GordoRelease, GordoSpecial, GordoPR, GordoSHA] """ special_version = Special.find(gordo_version) diff --git a/gordo/workflow/workflow_generator/helpers.py b/gordo/workflow/workflow_generator/helpers.py index a3243ad28..ec20df6ab 100644 --- a/gordo/workflow/workflow_generator/helpers.py +++ b/gordo/workflow/workflow_generator/helpers.py @@ -20,9 +20,9 @@ def patch_dict(original_dict: dict, patch_dictionary: dict) -> dict: Parameters ---------- - original_dict : dict + original_dict Base dictionary which will get paths added/changed - patch_dictionary: dict + patch_dictionary Dictionary which will be overlaid on top of original_dict Examples @@ -36,7 +36,6 @@ def patch_dict(original_dict: dict, patch_dictionary: dict) -> dict: Returns ------- - dict A new dictionary which is the result of overlaying `patch_dictionary` on top of `original_dict` @@ -52,7 +51,7 @@ def parse_argo_version(argo_version: str) -> Optional[version.Version]: Parameters ---------- - argo_version: str + argo_version Returns ------- diff --git a/gordo/workflow/workflow_generator/workflow_generator.py b/gordo/workflow/workflow_generator/workflow_generator.py index 3acc4efd6..604c59008 100644 --- a/gordo/workflow/workflow_generator/workflow_generator.py +++ b/gordo/workflow/workflow_generator/workflow_generator.py @@ -31,13 +31,12 @@ def _valid_owner_ref(owner_reference_str: str): Parameters ---------- - owner_reference_str: str + owner_reference_str String representation of the list of owner-references, should be parsable as yaml/json Returns ------- - list[dict] The list of owner-references """ @@ -107,12 +106,11 @@ def load_workflow_template(workflow_template: str) -> jinja2.Template: Parameters ---------- - workflow_template: str + workflow_template Path to a workflow template Returns ------- - jinja2.Template Loaded but non-rendered jinja2 template for the workflow """ path_to_workflow_template = os.path.abspath(workflow_template) diff --git a/setup.py b/setup.py index be96436c3..d87a86d94 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,13 @@ def requirements(fp: str): extras_require = { - "docs": requirements("docs_requirements.in"), "mlflow": requirements("mlflow_requirements.in"), "postgres": requirements("postgres_requirements.in"), "tests": requirements("test_requirements.txt"), } -extras_require["full"] = extras_require["mlflow"] + extras_require["postgres"] +full_extras = extras_require["mlflow"] + extras_require["postgres"] +extras_require["full"] = full_extras +extras_require["docs"] = requirements("docs_requirements.in") + full_extras install_requires = requirements("requirements.in") # Allow flexible deps for install