Merge pull request #31 from claffin/hetzner

Hetzner added as a provider

README.md

@@ -20,6 +20,7 @@ CloudProxy exposes an API with the IPs and credentials of the provisioned proxie
 ### Providers supported:
 * [DigitalOcean](docs/digitalocean.md)
 * [AWS](docs/aws.md)
+* [Hetzner](docs/hetzner.md)
 
 ### Planned:
 * Google Cloud

cloudproxy/main.py

@@ -66,6 +66,18 @@ def get_ip_list():
                     + ip
                     + ":8899"
                 )
+    if settings.config["providers"]["hetzner"]["ips"]:
+        for ip in settings.config["providers"]["hetzner"]["ips"]:
+            if ip not in delete_queue and ip not in restart_queue:
+                ip_list.append(
+                    "http://"
+                    + settings.config["auth"]["username"]
+                    + ":"
+                    + settings.config["auth"]["password"]
+                    + "@"
+                    + ip
+                    + ":8899"
+                )
     return ip_list
 
 

cloudproxy/providers/hetzner/functions.py

@@ -0,0 +1,37 @@
+import os
+import uuid
+
+from hcloud import Client
+from hcloud.images.domain import Image
+from hcloud.server_types.domain import ServerType
+from hcloud.datacenters.domain import Datacenter
+from hcloud.locations.domain import Location
+
+from cloudproxy.providers import settings
+from cloudproxy.providers.config import set_auth
+
+client = Client(token=settings.config["providers"]["hetzner"]["secrets"]["access_token"])
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+
+def create_proxy():
+    user_data = set_auth(
+        settings.config["auth"]["username"], settings.config["auth"]["password"]
+    )
+    client.servers.create(name=str(uuid.uuid1()),
+                          server_type=ServerType("cx11"),
+                          image=Image(name="ubuntu-20.04"),
+                          location=Location(name=settings.config["providers"]["hetzner"]["location"]),
+                          # datacenter=Datacenter(name=settings.config["providers"]["hetzner"]["datacenter"]),
+                          user_data=user_data, labels={"cloudproxy": "cloudproxy"})
+    return True
+
+
+def delete_proxy(server):
+    deleted = client.servers.delete(server)
+    return deleted
+
+
+def list_proxies():
+    servers = client.servers.get_all(label_selector={"cloudproxy": "cloudproxy"})
+    return servers

cloudproxy/providers/hetzner/main.py

@@ -0,0 +1,71 @@
+import itertools
+import datetime
+
+import dateparser
+from loguru import logger
+
+from cloudproxy.check import check_alive
+from cloudproxy.providers import settings
+from cloudproxy.providers.hetzner.functions import list_proxies, delete_proxy, create_proxy
+from cloudproxy.providers.settings import config, delete_queue, restart_queue
+
+
+def hetzner_deployment(min_scaling):
+    total_proxies = len(list_proxies())
+    if min_scaling < total_proxies:
+        logger.info("Overprovisioned: Hetzner destroying.....")
+        for proxy in itertools.islice(
+                list_proxies(), 0, (total_proxies - min_scaling)
+        ):
+            delete_proxy(proxy)
+            logger.info("Destroyed: Hetzner -> " + str(proxy.public_net.ipv4.ip))
+    if min_scaling - total_proxies < 1:
+        logger.info("Minimum Hetzner proxies met")
+    else:
+        total_deploy = min_scaling - total_proxies
+        logger.info("Deploying: " + str(total_deploy) + " Hetzner proxy")
+        for _ in range(total_deploy):
+            create_proxy()
+            logger.info("Deployed")
+    return len(list_proxies())
+
+
+def hetzner_check_alive():
+    ip_ready = []
+    for proxy in list_proxies():
+        elapsed = datetime.datetime.now(
+            datetime.timezone.utc
+        ) - dateparser.parse(str(proxy.created))
+        if config["age_limit"] > 0:
+            if elapsed > datetime.timedelta(seconds=config["age_limit"]):
+                delete_proxy(proxy)
+                logger.info(
+                    "Recycling proxy, reached age limit -> " + str(proxy.public_net.ipv4.ip)
+                )
+        elif check_alive(proxy.public_net.ipv4.ip):
+            logger.info("Alive: Hetzner -> " + str(proxy.public_net.ipv4.ip))
+            ip_ready.append(proxy.public_net.ipv4.ip)
+        else:
+            if elapsed > datetime.timedelta(minutes=10):
+                delete_proxy(proxy)
+                logger.info(
+                    "Destroyed: Hetzner took too long -> " + str(proxy.public_net.ipv4.ip)
+                )
+            else:
+                logger.info("Waiting: Hetzner -> " + str(proxy.public_net.ipv4.ip))
+    return ip_ready
+
+
+def hetzner_check_delete():
+    for proxy in list_proxies():
+        if proxy.public_net.ipv4.ip in delete_queue or proxy.public_net.ipv4.ip in restart_queue:
+            delete_proxy(proxy)
+            logger.info("Destroyed: not wanted -> " + str(proxy.public_net.ipv4.ip))
+            delete_queue.remove(proxy.public_net.ipv4.ip)
+
+
+def hetzner_start():
+    hetzner_check_delete()
+    hetzner_deployment(settings.config["providers"]["hetzner"]["scaling"]["min_scaling"])
+    ip_ready = hetzner_check_alive()
+    return ip_ready

cloudproxy/providers/manager.py

@@ -3,6 +3,7 @@
 from cloudproxy.providers import settings
 from cloudproxy.providers.aws.main import aws_start
 from cloudproxy.providers.digitalocean.main import do_start
+from cloudproxy.providers.hetzner.main import hetzner_start
 
 
 def do_manager():
@@ -17,6 +18,12 @@ def aws_manager():
     return ip_list
 
 
+def hetzner_manager():
+    ip_list = hetzner_start()
+    settings.config["providers"]["hetzner"]["ips"] = [ip for ip in ip_list]
+    return ip_list
+
+
 def init_schedule():
     sched = BackgroundScheduler()
     sched.start()
@@ -28,3 +35,7 @@ def init_schedule():
         sched.add_job(aws_manager, "interval", seconds=20)
     else:
         logger.info("AWS not enabled")
+    if settings.config["providers"]["hetzner"]["enabled"] == 'True':
+        sched.add_job(hetzner_manager, "interval", seconds=20)
+    else:
+        logger.info("Hetzner not enabled")

cloudproxy/providers/settings.py

@@ -21,6 +21,15 @@
             "region": "",
             "secrets": {"access_key_id": "", "secret_access_key": ""},
         },
+        "hetzner": {
+            "enabled": False,
+            "ips": [],
+            "scaling": {"min_scaling": 0, "max_scaling": 0},
+            "size": "",
+            "location": "",
+            "datacenter": "",
+            "secrets": {"access_token": ""},
+        },
     },
 }
 
@@ -34,7 +43,7 @@
 config["auth"]["password"] = os.environ.get("PASSWORD", "changeme")
 config["age_limit"] = int(os.environ.get('AGE_LIMIT', 0))
 
-# Set DigitalOceana config
+# Set DigitalOcean config
 config["providers"]["digitalocean"]["enabled"] = os.environ.get(
     "DIGITALOCEAN_ENABLED", False
 )
@@ -70,3 +79,26 @@
 )
 config["providers"]["aws"]["size"] = os.environ.get("AWS_SIZE", "t2.micro")
 config["providers"]["aws"]["region"] = os.environ.get("AWS_REGION", "eu-west-2")
+
+# Set Hetzner config
+config["providers"]["hetzner"]["enabled"] = os.environ.get(
+    "HETZNER_ENABLED", False
+)
+config["providers"]["hetzner"]["secrets"]["access_token"] = os.environ.get(
+    "HETZNER_ACCESS_TOKEN"
+)
+config["providers"]["hetzner"]["scaling"]["min_scaling"] = int(
+    os.environ.get("HETZNER_MIN_SCALING", 2)
+)
+config["providers"]["hetzner"]["scaling"]["max_scaling"] = int(
+    os.environ.get("HETZNER_MAX_SCALING", 2)
+)
+config["providers"]["hetzner"]["size"] = os.environ.get(
+    "HETZNER_SIZE", "cx11"
+)
+config["providers"]["hetzner"]["location"] = os.environ.get(
+    "HETZNER_LOCATION", "nbg1"
+)
+# config["providers"]["hetzner"]["datacenter"] = os.environ.get(
+#     "HETZNER_DATACENTER", "dc3"
+# )

docs/hetzner.md

@@ -0,0 +1,30 @@
+# Hetzner Configuration
+
+To use Hetzner as a provider, you’ll first need to generate an API token.
+
+## Steps
+
+1. Login to your Hetzner Cloud account.
+2. Select a project, then on the left sidebar, select 'Security'.
+3. In the Security section, select 'API Tokens' on the top menu bar, click the Generate API Token button. This opens an API token window.
+4. Enter a token name, this can be anything, I recommend 'CloudProxy' so you know what it is being used for.
+5. Select 'Read & Write', write permission is needed so CloudProxy can provision instances.
+6. When you click 'Generate API Token', your token is generated and presented to you. Be sure to record your API token. For security purposes, it will not be shown again.
+
+Now you have your token, you can now use Hetzner as a proxy provider, on this page you can see how to set it is an environment variable. 
+
+## Configuration options
+### Environment variables: 
+#### Required:
+`` HETZNER_ENABLED`` - to enable Hetzner as a provider, set as True. Default value: False
+
+`` HETZNER_ACCESS_TOKEN`` - the token to allow CloudProxy access to your account. 
+
+#### Optional:
+``HETZNER_MIN_SCALING`` - this is the minimal proxies you required to be provisioned. Default value: 2
+
+``HETZNER_MAX_SCALING`` - this is currently unused, however will be when autoscaling is implemented. We recommend you set this as the same as the minimum scaling to avoid future issues for now. Default value: 2
+
+``HETZNER_SIZE``  - this sets the instance size, we recommend the smallest instance as the volume even a small instance can handle is high. Default value: cx11
+
+``HETZNER_LOCATION`` - this sets the location where the instance is deployed. Some websites may redirect to the language of the country your IP is from. Default value: nbg1

requirements.txt

@@ -10,4 +10,5 @@ python-digitalocean==1.16.0
 boto3==1.17.55
 urllib3==1.26.4
 aiofiles==0.6.0
-botocore~=1.20.84
+botocore~=1.20.84
+hcloud==1.12.0