From d736858e40d0874e9b402c4bec21a6959d3db2ca Mon Sep 17 00:00:00 2001 From: jyejare Date: Sat, 3 Aug 2024 20:25:07 +0530 Subject: [PATCH] Develop: Rebase GCE cleanup --- .pre-commit-config.yaml | 2 +- cloudwash/constants.py | 1 + cloudwash/entities/providers.py | 9 +++++ cloudwash/entities/resources/vms.py | 32 +++++++++++++-- cloudwash/providers/gce.py | 62 ++++++++++------------------- settings.yaml.template | 2 + 6 files changed, 61 insertions(+), 47 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e2a0bf29a..440ee222e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,6 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/flake8 - rev: 3.9.2 + rev: 7.1.0 hooks: - id: flake8 diff --git a/cloudwash/constants.py b/cloudwash/constants.py index d60b570bf..a8e58d2c6 100644 --- a/cloudwash/constants.py +++ b/cloudwash/constants.py @@ -1,2 +1,3 @@ aws_data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES', 'STACKS'] azure_data = ['VMS', 'NICS', 'DISCS', 'IMAGES', 'PIPS', 'RESOURCES'] +gce_data = ['VMS', 'NICS', 'DISCS'] diff --git a/cloudwash/entities/providers.py b/cloudwash/entities/providers.py index cf6d0136e..8ae29933f 100644 --- a/cloudwash/entities/providers.py +++ b/cloudwash/entities/providers.py @@ -9,6 +9,7 @@ from cloudwash.entities.resources.stacks import CleanAWSStacks from cloudwash.entities.resources.vms import CleanAWSVms from cloudwash.entities.resources.vms import CleanAzureVMs +from cloudwash.entities.resources.vms import CleanGCEVMs class providerCleanup: @@ -22,6 +23,8 @@ def vms(self): return CleanAzureVMs(client=self.client) elif 'AWS' in providerclass: return CleanAWSVms(client=self.client) + elif 'GCE' in providerclass: + return CleanGCEVMs(client=self.client) @property def discs(self): @@ -72,3 +75,9 @@ class AWSCleanup(providerCleanup): def __init__(self, client): self.client = client super().__init__(client) + + +class GCECleanup(providerCleanup): + def __init__(self, client): + self.client = client + super().__init__(client) diff --git a/cloudwash/entities/resources/vms.py b/cloudwash/entities/resources/vms.py index bb2b7a27f..b919b5126 100644 --- a/cloudwash/entities/resources/vms.py +++ b/cloudwash/entities/resources/vms.py @@ -23,14 +23,14 @@ def _set_dry(self): def list(self): pass - def remove(self): + def remove(self, **kwargs): for vm_name in self._delete: - self.client.get_vm(vm_name).delete() + self.client.get_vm(vm_name, **kwargs).delete() logger.info(f"Removed VMs: \n{self._delete}") - def stop(self): + def stop(self, **kwargs): for vm_name in self._stop: - self.client.get_vm(vm_name).stop() + self.client.get_vm(vm_name, **kwargs).stop() logger.info(f"Stopped VMs: \n{self._stop}") def skip(self): @@ -90,3 +90,27 @@ def list(self): elif vm.name.startswith(settings.azure.criteria.vm.delete_vm): self._delete.append(vm.name) self._set_dry() + + +class CleanGCEVMs(CleanVMs): + def list(self): + + allvms = self.client.list_vms(zones=[self.client.cleaning_zone]) + + for vm in allvms: + if vm.name in settings.gce.exceptions.vm.vm_list: + self._skip.append(vm.name) + continue + elif total_running_time(vm).minutes >= settings.gce.criteria.vm.sla_minutes: + if vm.name in settings.gce.exceptions.vm.stop_list: + self._stop.append(vm.name) + continue + elif vm.name.startswith(settings.gce.criteria.vm.delete_vm): + self._delete.append(vm.name) + self._set_dry() + + def remove(self): + super().remove(zone=self.client.cleaning_zone) + + def stop(self): + super().stop(zone=self.client.cleaning_zone) diff --git a/cloudwash/providers/gce.py b/cloudwash/providers/gce.py index 7e3f257f3..0f8bba8e3 100644 --- a/cloudwash/providers/gce.py +++ b/cloudwash/providers/gce.py @@ -1,53 +1,31 @@ """GCE CR Cleanup Utilities""" from cloudwash.client import compute_client from cloudwash.config import settings +from cloudwash.constants import gce_data as data +from cloudwash.entities.providers import GCECleanup from cloudwash.logger import logger from cloudwash.utils import dry_data from cloudwash.utils import echo_dry from cloudwash.utils import gce_zones -from cloudwash.utils import total_running_time def cleanup(**kwargs): - is_dry_run = kwargs["dry_run"] - + is_dry_run = kwargs.get("dry_run", False) + zones = settings.gce.auth.get('zones', ['all']) + if "all" in zones: + zones = gce_zones() + if kwargs["nics"] or kwargs["_all"]: + logger.warning("Cloudwash does not supports NICs operation for GCE yet!") + if kwargs["discs"] or kwargs["_all"]: + logger.warning("Cloudwash does not supports DISCs operation for GCE yet!") with compute_client("gce") as gce_client: - if kwargs["vms"] or kwargs["_all"]: - allvms = gce_client.list_vms(zones=gce_zones()) - for vm in allvms: - if vm.name in settings.gce.exceptions.vm.vm_list: - dry_data["VMS"]["skip"].append(vm.name) - continue - elif total_running_time(vm).minutes >= settings.gce.criteria.vm.sla_minutes: - if vm.name in settings.gce.exceptions.vm.stop_list: - dry_data["VMS"]["stop"].append(vm.name) - if not is_dry_run: - try: - vm.stop() - except TypeError: - logger.info(f"Stopped VM: {vm.name}") - except Exception: - logger.exception(f"Error stopping VM {vm.name}") - continue - elif vm.name.startswith(settings.gce.criteria.vm.delete_vm): - dry_data["VMS"]["delete"].append(vm.name) - if not is_dry_run: - try: - vm.delete() - # There as an issue with GCE API while deleting/stopping the VM - # That it throws TypeError, hence catching that error here - # Remove it once its fixed - except TypeError: - logger.info(f"Deleted VM: {vm.name}") - except Exception: - logger.exception(f"Error deleting VM {vm.name}") - if kwargs["nics"] or kwargs["_all"]: - logger.warning( - "Cloudwash dependency 'WrapanAPI' does not supports NICs operation for GCE yet!" - ) - if kwargs["discs"] or kwargs["_all"]: - logger.warning( - "Cloudwash dependency 'WrapanAPI' does not supports DISCs operation for GCE yet!" - ) - if is_dry_run: - echo_dry(dry_data) + for zone in zones: + for items in data: + dry_data[items]['delete'] = [] + gce_client.cleaning_zone = zone + gcecleanup = GCECleanup(client=gce_client) + logger.info(f"\nResources from the zone: {zone}") + if kwargs["vms"] or kwargs["_all"]: + gcecleanup.vms.cleanup() + if is_dry_run: + echo_dry(dry_data) diff --git a/settings.yaml.template b/settings.yaml.template index c3ed96f3b..70c65165b 100644 --- a/settings.yaml.template +++ b/settings.yaml.template @@ -6,6 +6,8 @@ GCE: # Multiline json contents from Service Account key SERVICE_ACCOUNT: > {} + # Multiple zones can be added like ["eastus2"] or ["northeurope", "eastus2", "westus"] or ["all"] for all regions + ZONES: ['all'] CRITERIA: VM: # The VM to be deleted with prepend string, e.g VM name that starts with 'test'