Develop: Rebase for OCP resource cleanup on AWS

cloudwash/cli.py

@@ -100,8 +100,13 @@ def azure(ctx, vms, discs, nics, images, pips, _all, _all_rg):
 @click.option("--images", is_flag=True, help="Remove only images from the provider")
 @click.option("--pips", is_flag=True, help="Remove only Public IPs from the provider")
 @click.option("--stacks", is_flag=True, help="Remove only CloudFormations from the provider")
+@click.option(
+    "--ocps",
+    is_flag=True,
+    help="Remove only unused OCP Cluster occupied resources from the provider",
+)
 @click.pass_context
-def aws(ctx, vms, discs, nics, images, pips, stacks, _all):
+def aws(ctx, vms, discs, nics, images, pips, stacks, ocps, _all):
     # Validate Amazon Settings
     validate_provider(ctx.command.name)
     is_dry_run = ctx.parent.params["dry"]
@@ -112,6 +117,7 @@ def aws(ctx, vms, discs, nics, images, pips, stacks, _all):
         images=images,
         pips=pips,
         stacks=stacks,
+        ocps=ocps,
         _all=_all,
         dry_run=is_dry_run,
     )

cloudwash/constants.py

@@ -1,4 +1,4 @@
-aws_data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES', 'STACKS']
+aws_data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES', 'STACKS', 'OCPS']
 azure_data = ['VMS', 'NICS', 'DISCS', 'IMAGES', 'PIPS', 'RESOURCES']
 gce_data = ['VMS', 'NICS', 'DISCS']
 vmware_data = ['VMS', 'NICS', 'DISCS']

cloudwash/entities/providers.py

@@ -5,6 +5,7 @@
 from cloudwash.entities.resources.images import CleanAzureImages
 from cloudwash.entities.resources.nics import CleanAWSNics
 from cloudwash.entities.resources.nics import CleanAzureNics
+from cloudwash.entities.resources.ocps import CleanAWSOcps
 from cloudwash.entities.resources.pips import CleanAWSPips
 from cloudwash.entities.resources.pips import CleanAzurePips
 from cloudwash.entities.resources.stacks import CleanAWSStacks
@@ -18,6 +19,14 @@ class providerCleanup:
     def __init__(self, client):
         self.client = client
 
+    @property
+    def ocps(self):
+        providerclass = self.__class__.__name__
+        if 'AWS' in providerclass:
+            return CleanAWSOcps(client=self.client)
+        else:
+            raise NotImplementedError(f'The OCPs cleanup on {providerclass} is not implemented')
+
     @property
     def vms(self):
         providerclass = self.__class__.__name__

cloudwash/entities/resources/base.py

@@ -16,6 +16,24 @@ def _set_dry(self):
         pass
 
 
+class OCPsCleanup(ResourceCleanup):
+    @abstractmethod
+    def list(self):
+        pass
+
+    @abstractmethod
+    def cleanup(self):
+        pass
+
+    @abstractmethod
+    def remove(self):
+        pass
+
+    @abstractmethod
+    def _set_dry(self):
+        pass
+
+
 class DiscsCleanup(ResourceCleanup):
     @abstractmethod
     def list(self):

cloudwash/entities/resources/ocps.py

@@ -0,0 +1,58 @@
+from cloudwash.config import settings
+from cloudwash.entities.resources.base import OCPsCleanup
+from cloudwash.utils import calculate_time_threshold
+from cloudwash.utils import dry_data
+from cloudwash.utils import filter_resources_by_time_modified
+from cloudwash.utils import group_ocps_by_cluster
+from cloudwash.utils import OCP_TAG_SUBSTR
+
+
+class CleanOCPs(OCPsCleanup):
+    def __init__(self, client):
+        self.client = client
+        self._delete = []
+        self.list()
+
+    def _set_dry(self):
+        dry_data['OCPS']['delete'] = self._delete
+
+    def list(self):
+        pass
+
+    def remove(self):
+        pass
+
+    def cleanup(self):
+        if not settings.dry_run:
+            self.remove()
+
+
+class CleanAWSOcps(CleanOCPs):
+    def list(self):
+        time_threshold = calculate_time_threshold(time_ref=settings.aws.criteria.ocps.sla)
+
+        query = " ".join([f"tag.key:{OCP_TAG_SUBSTR}*", f"region:{self.client.cleaning_region}"])
+        resources = self.client.list_resources(query=query)
+
+        # Prepare resources to be filtered before deletion
+        cluster_map = group_ocps_by_cluster(resources=resources)
+        for cluster_name in cluster_map.keys():
+            cluster_resources = cluster_map[cluster_name].get("Resources")
+            instances = cluster_map[cluster_name].get("Instances")
+
+            if instances:
+                # For resources with associated EC2 Instances, filter by Instances SLA
+                if not filter_resources_by_time_modified(
+                    time_threshold,
+                    resources=instances,
+                ):
+                    self._delete.extend(cluster_resources)
+            else:
+                # For resources with no associated EC2 Instances, identify as leftovers
+                self._delete.extend(
+                    filter_resources_by_time_modified(time_threshold, resources=cluster_resources)
+                )
+
+        # Sort resources by type
+        self._delete = sorted(self._delete, key=lambda x: x.resource_type)
+        self._set_dry()

cloudwash/providers/aws.py

@@ -12,27 +12,44 @@ def cleanup(**kwargs):
     is_dry_run = kwargs.get("dry_run", False)
     dry_data['PROVIDER'] = "AWS"
     regions = settings.aws.auth.regions
-    if "all" in regions:
-        with compute_client("aws", aws_region="us-west-2") as client:
-            regions = client.list_regions()
-    for region in regions:
-        for items in data:
-            dry_data[items]['delete'] = []
-        with compute_client("aws", aws_region=region) as aws_client:
-            awscleanup = AWSCleanup(client=aws_client)
-            # Actual Cleaning and dry execution
-            logger.info(f"\nResources from the region: {region}")
-            if kwargs["vms"] or kwargs["_all"]:
-                awscleanup.vms.cleanup()
-            if kwargs["nics"] or kwargs["_all"]:
-                awscleanup.nics.cleanup()
-            if kwargs["discs"] or kwargs["_all"]:
-                awscleanup.discs.cleanup()
-            if kwargs["pips"] or kwargs["_all"]:
-                awscleanup.pips.cleanup()
-            if kwargs["images"] or kwargs["_all"]:
-                awscleanup.images.cleanup()
-            if kwargs["stacks"] or kwargs["_all"]:
-                awscleanup.stacks.cleanup()
-            if is_dry_run:
-                echo_dry(dry_data)
+    if kwargs["ocps"]:
+        aws_client_region = settings.aws.criteria.ocps.ocp_client_region
+        with compute_client("aws", aws_region=aws_client_region) as aws_ocp_client:
+            if "all" in regions:
+                regions = aws_ocp_client.list_regions()
+            awscleanup = AWSCleanup(client=aws_ocp_client)
+            for region in regions:
+                aws_ocp_client.cleaning_region = region
+                # Emptying the dry data for previous region everytime
+                for items in data:
+                    dry_data[items]['delete'] = []
+                logger.info(f"\nResources from the region: {region}")
+                awscleanup.ocps.cleanup()
+                if is_dry_run:
+                    echo_dry(dry_data)
+    else:
+        if "all" in regions:
+            with compute_client("aws", aws_region="us-west-2") as client:
+                regions = client.list_regions()
+        for region in regions:
+            # Emptying the dry data for previous region everytime
+            for items in data:
+                dry_data[items]['delete'] = []
+            with compute_client("aws", aws_region=region) as aws_client:
+                awscleanup = AWSCleanup(client=aws_client)
+                # Actual Cleaning and dry execution
+                logger.info(f"\nResources from the region: {region}")
+                if kwargs["vms"] or kwargs["_all"]:
+                    awscleanup.vms.cleanup()
+                if kwargs["nics"] or kwargs["_all"]:
+                    awscleanup.nics.cleanup()
+                if kwargs["discs"] or kwargs["_all"]:
+                    awscleanup.discs.cleanup()
+                if kwargs["pips"] or kwargs["_all"]:
+                    awscleanup.pips.cleanup()
+                if kwargs["images"] or kwargs["_all"]:
+                    awscleanup.images.cleanup()
+                if kwargs["stacks"] or kwargs["_all"]:
+                    awscleanup.stacks.cleanup()
+                if is_dry_run:
+                    echo_dry(dry_data)

cloudwash/utils.py

@@ -3,8 +3,10 @@
 from collections import namedtuple
 from datetime import datetime
 
+import dateparser
 import dominate
 import pytz
+from dominate.tags import br
 from dominate.tags import div
 from dominate.tags import h1
 from dominate.tags import h3
@@ -14,22 +16,27 @@
 from dominate.tags import td
 from dominate.tags import tr
 from dominate.util import raw
+from wrapanapi.systems.ec2 import ResourceExplorerResource
 
 from cloudwash.assets import css
 from cloudwash.logger import logger
 
+OCP_TAG_SUBSTR = "kubernetes.io/cluster/"
+
 _vms_dict = {"VMS": {"delete": [], "stop": [], "skip": []}}
 _containers_dict = {"CONTAINERS": {"delete": [], "stop": [], "skip": []}}
 
 dry_data = {
     "NICS": {"delete": []},
     "DISCS": {"delete": []},
     "PIPS": {"delete": []},
+    "OCPS": {"delete": []},
     "RESOURCES": {"delete": []},
     "STACKS": {"delete": []},
     "IMAGES": {"delete": []},
     "PROVIDER": "",
 }
+
 dry_data.update(_vms_dict)
 dry_data.update(_containers_dict)
 
@@ -41,6 +48,7 @@ def echo_dry(dry_data=None) -> None:
         it follows the format of module scoped `dry_data` variable in this module
     """
     logger.info("\n=========== DRY SUMMARY ============\n")
+
     resource_data = {
         "provider": dry_data.get('PROVIDER'),
         "deletable_vms": dry_data["VMS"]["delete"],
@@ -55,6 +63,12 @@ def echo_dry(dry_data=None) -> None:
         "deletable_pips": dry_data["PIPS"]["delete"] if "PIPS" in dry_data else None,
         "deletable_resources": dry_data["RESOURCES"]["delete"],
         "deletable_stacks": dry_data["STACKS"]["delete"] if "STACKS" in dry_data else None,
+        "deletable_ocps": {
+            ocp.resource_type: [
+                r.name for r in dry_data["OCPS"]["delete"] if r.resource_type == ocp.resource_type
+            ]
+            for ocp in dry_data["OCPS"]["delete"]
+        },
     }
 
     # Group the same resource type under the same section for logging
@@ -99,11 +113,21 @@ def create_html(**kwargs):
                             with tr():
                                 td(table_head.replace("_", " ").title())
                                 bullet = '•'
+                                tab = ' '
                                 if isinstance(kwargs[table_head], list):
                                     component = ''
                                     for resource_name in kwargs[table_head]:
                                         component += bullet + ' ' + resource_name + ' '
                                     td(raw(component))
+                                if isinstance(kwargs[table_head], dict):
+                                    rline = []
+                                    for rtype, resources in kwargs[table_head].items():
+                                        component = ' ' + tab * 2 + ' '
+                                        rtype_line = bullet + rtype + str(br())
+                                        for resource_name in resources:
+                                            component += bullet + ' ' + resource_name + ' '
+                                        rline.append(rtype_line + component)
+                                    td(raw(str(br()).join(rline)))
                                 else:
                                     td(raw(bullet + ' ' + kwargs[table_head]))
     with open('cleanup_resource_{}.html'.format(kwargs.get('provider')), 'w') as file:
@@ -155,3 +179,77 @@ def gce_zones() -> list:
     _zones_combo = {**_bcds, **_abcfs, **_abcs}
     zones = [f"{loc}-{zone}" for loc, zones in _zones_combo.items() for zone in zones]
     return zones
+
+
+def group_ocps_by_cluster(resources: list = None) -> dict:
+    """Group different types of AWS resources under their original OCP clusters
+    :param list resources: AWS resources collected by defined region and sla
+    :return: A dictionary with the clusters as keys and the associated resources as values
+    """
+    if resources is None:
+        resources = []
+    clusters_map = {}
+
+    for resource in resources:
+        for key in resource.get_tags(regex=OCP_TAG_SUBSTR):
+            cluster_name = key.get("Key")
+            if OCP_TAG_SUBSTR in cluster_name:
+                cluster_name = cluster_name.split(OCP_TAG_SUBSTR)[1]
+                if cluster_name not in clusters_map.keys():
+                    clusters_map[cluster_name] = {"Resources": [], "Instances": []}
+
+                # Set cluster's EC2 instances
+                if hasattr(resource, 'ec2_instance'):
+                    clusters_map[cluster_name]["Instances"].append(resource)
+                # Set resource under cluster
+                else:
+                    clusters_map[cluster_name]["Resources"].append(resource)
+    return clusters_map
+
+
+def calculate_time_threshold(time_ref=""):
+    """Parses a time reference for data filtering
+    :param str time_ref: a relative time reference for indicating the filter value
+    of a relative time, given in a {time_value}{time_unit} format; default is "" (no filtering)
+    :return datetime time_threshold
+    """
+    if time_ref is None:
+        time_ref = ""
+
+    if time_ref.isnumeric():
+        # Use default time value as Minutes
+        time_ref += "m"
+
+    # Time Ref is Optional; if empty, time_threshold will be set as "now"
+    time_threshold = dateparser.parse(f"now-{time_ref}-UTC")
+    logger.debug(
+        f"\nAssociated OCP resources are filtered by last creation time of: {time_threshold}"
+    )
+    return time_threshold
+
+
+def filter_resources_by_time_modified(
+    time_threshold,
+    resources: list[ResourceExplorerResource] = None,
+) -> list:
+    """
+    Filter list of AWS resources by checking modification date ("LastReportedAt")
+    :param datetime time_threshold: Time filtering criteria
+    :param list resources: List of resources to be filtered out
+    :return: list of resources that last modified before time threshold
+    :Example:
+        Use the time_ref "1h" to collect resources that exist for more than an hour
+    """
+    filtered_resources = []
+
+    for resource in resources:
+        # Will not collect resources recorded during the SLA time
+        if resource.date_modified > time_threshold:
+            continue
+        filtered_resources.append(resource)
+    return filtered_resources
+
+
+def delete_ocp(ocp):
+    # WIP: add support for deletion
+    pass

settings.yaml.template

@@ -99,6 +99,11 @@ AWS:
             DELETE_STACK: 'test'
             # Number of minutes the deletable CloudFormation should be allowed to live, e.g 120 minutes = 2 Hours
             SLA_MINUTES: 120
+        OCPS:
+            OCP_CLIENT_REGION: "us-east-1"
+            # Specified as {time_value}{time_unit} format, e.g. "7d" = 7 Days
+            # If a time unit is not specified (the value is numeric), it will be considered as Minutes
+            SLA: 7d
     EXCEPTIONS:
         VM:
             # VM names that would be skipped from cleanup