New Output: Elastic ES|QL (#6)

* First prototype of the Elastic output platform

* quickfix to add interval

* refactor: use parameter for license value in ElasticPlatform class

* adding raw export for ESQL

* adding custom raw language selector

* refactor: update is_raw_rule function to handle ESQL platform correctly

* added index parsing with fallback
there needs to be more intelligence here, ideas welcome

* refactor: update ElasticPlatform class to handle ESQL platform correctly

* added custom disabled handling

* improved import mechanism to use json instead of ndjson import, this improves performance and allows for enabling and disabling rules

* adding eql converting capabilities

* improving EQL support

* adding requests as requirement since it's used in elastic.py

* improving error handling

* added prefix

* added error handling for correlation rule failure

* add remove search

* Update src/droid/__main__.py

Co-authored-by: Mathieu <[email protected]>

* removed quickfix

* refactored the buildingblock process

* fixed if no tags are set in sigma rule

* quality fixes

* bugfix to make index information more stable

* adding eql support

* changed encoding to utf8, we are in the EU after all

* added index field into elastic output

* adding search

* added elasticsearch dependency

* added ESQL implementation for search

* add: abstraction and index catching

* added language transitioning for rules

* fix: stop unsupported rule to go further

* refined index selection

* adding warning if defaulting to logs-*

* not self.

* logic fix, always default to index_value logs-*

* adding more stability to index search

* adding EQL and ESQL Search

* removing index from search

* added debugging

* add: improvements and search range config for eql and esql

* added integrity checking

* added checking if update is necessary

* chore: Update default value for 'eql_search_range_gte' to 24 hours

* deduplication function added for ESQL

* fix for older elastic versions

* bracketstory v2 and bugfix of policy integrity check

* added bugfix for index unknown in raw rules

* add: new get_rule abstraction and fix some issues in elastic and defender

* add: building block prefix option

* changed default behaviour for buildingblock

* reworked index matching since it was not perfect. Now it only counts actual sigma related fields

* upd: logger in Elastic

* fix: bug in finding the index from pipeline

* upd: required modules

* add: test for pull request as well

* upd: job name in CI

---------

Co-authored-by: Mathieu <[email protected]>

.github/workflows/test.yml

@@ -3,12 +3,15 @@ name: Test Build
 on:
   push:
     branches:
-      - '**'  # This will match any branch
+      - '**'
     tags:
-      - '*'  # This will match any tag
+      - '*'
+  pull_request:
+    paths:
+      - 'src/**'
 
 jobs:
-  test_build:
+  run_tests:
     runs-on: ubuntu-latest
 
     container:

requirements.txt

@@ -13,3 +13,5 @@ azure-monitor-query==1.3.0
 splunk-sdk==2.0.1
 colorama==0.4.6
 python-json-logger==2.0.7
+requests==2.32.3
+elasticsearch==8.14.0

setup.cfg

@@ -29,6 +29,8 @@ install_requires =
     splunk-sdk==2.0.1
     colorama==0.4.6
     python-json-logger==2.0.7
+    elasticsearch==8.14.0
+    requests==2.32.3
 
 [options.packages.find]
 where=src

src/droid/__main__.py

@@ -39,7 +39,7 @@ def init_argparse() -> argparse.ArgumentParser:
     parser.add_argument("-cf", "--config-file", help="DROID configuration file path")
     parser.add_argument("-d", "--debug", help="Enable debugging", action="store_true")
     parser.add_argument("-e", "--export", help="Export the rules", action="store_true")
-    parser.add_argument("-p", "--platform", help="Platform target", choices=['splunk', 'azure', 'microsoft_defender'])
+    parser.add_argument("-p", "--platform", help="Platform target", choices=['splunk', 'azure', 'microsoft_defender', 'esql', 'eql'])
     parser.add_argument("-sm", "--sentinel-mde", help="Use Sentinel as backend for MDE", action="store_true")
     parser.add_argument("-u", "--update", help="Update from source", choices=['sigmahq-core'])
     parser.add_argument("-l", "--list", help="List items from rules", choices=['unique_fields', 'pipelines'])
@@ -82,14 +82,16 @@ def is_raw_rule(args, base_config):
             exit(1)
     else:
         return False
-
     if (
         (args.platform in ['splunk', 'azure'] or
         (args.platform == 'microsoft_defender' and args.sentinel_mde)) and
         (raw_rule_folder_name in args.rules and args.platform in args.rules)
     ):
         return True
-
+    elif args.platform in ['esql', 'eql'] and raw_rule_folder_name in args.rules:
+        return True
+    elif args.platform in ['esql', 'eql']:
+        return False
     elif (
         args.platform in ['splunk', 'azure'] or
         (args.platform == 'microsoft_defender' and args.sentinel_mde)
@@ -153,8 +155,7 @@ def droid_platform_config(args, config_path):
 
         return config_splunk
 
-    if 'azure' or 'defender' in args.platform:
-
+    if args.platform == 'azure' or args.platform == 'microsoft_defender':
         try:
             with open(config_path) as file_obj:
                 content = file_obj.read()
@@ -225,6 +226,32 @@ def droid_platform_config(args, config_path):
 
         return config
 
+    if args.platform in ['esql', 'eql']:
+
+        try:
+            with open(config_path) as file_obj:
+                content = file_obj.read()
+                config_data = tomllib.loads(content)
+                config_elastic = config_data["platforms"]["elastic"]
+        except Exception as e:
+            raise Exception(f"Something unexpected happened: {e}")
+
+        if config_elastic["auth_method"] == "basic":
+            if args.export or args.search or args.integrity:
+                if environ.get('DROID_ELASTIC_USERNAME'):
+                    username = environ.get('DROID_ELASTIC_USERNAME')
+                    config_elastic["username"] = username
+                else:
+                    raise Exception("Please use: export DROID_ELASTIC_USERNAME=<username>")
+                if environ.get('DROID_ELASTIC_PASSWORD'):
+                    password = environ.get('DROID_ELASTIC_PASSWORD')
+                    config_elastic["password"] = password
+                else:
+                    raise Exception("Please use: export DROID_ELASTIC_PASSWORD=<password>")
+
+        return config_elastic
+
+
 def main(argv=None) -> None:
     """Main function
 
@@ -255,7 +282,6 @@ def main(argv=None) -> None:
         raise Exception(f"Error: configuration file {args.config_file} not found.")
     else:
         config_path = args.config_file
-
     if args.validate:
 
         logger.info(f"Validation mode was selected - path selected: {args.rules}")
@@ -309,7 +335,7 @@ def main(argv=None) -> None:
             search_error, search_warning = search_rule_raw(parameters, droid_platform_config(args, config_path))
         else:
             logger.info(f"Searching Sigma rule for platform {args.platform} selected")
-            search_error, search_warning = convert_rules(parameters, droid_platform_config(args, config_path))
+            search_error, search_warning = convert_rules(parameters, droid_platform_config(args, config_path), base_config)
 
         if search_error and search_warning:
             logger.warning("Hits found while search one or multiple rules")
@@ -338,26 +364,34 @@ def main(argv=None) -> None:
                 logger.info("Splunk raw rule selected")
                 export_error = export_rule_raw(parameters, droid_platform_config(args, config_path))
             else:
-                export_error = convert_rules(parameters, droid_platform_config(args, config_path))
+                export_error = convert_rules(parameters, droid_platform_config(args, config_path), base_config)
 
         elif args.platform == 'azure':
             if is_raw_rule(args, base_config):
                 logger.info("Azure Sentinel raw rule selected")
                 export_error = export_rule_raw(parameters, droid_platform_config(args, config_path))
             else:
-                export_error = convert_rules(parameters, droid_platform_config(args, config_path))
+                export_error = convert_rules(parameters, droid_platform_config(args, config_path), base_config)
 
         elif args.platform == 'microsoft_defender' and args.sentinel_mde:
             if is_raw_rule(args, base_config):
                 logger.info("Microsoft Defender for Endpoint raw rule selected")
                 export_error = export_rule_raw(parameters, droid_platform_config(args, config_path))
             else:
-                export_error = convert_rules(parameters, droid_platform_config(args, config_path))
+                export_error = convert_rules(parameters, droid_platform_config(args, config_path), base_config)
 
         elif args.platform == "microsoft_defender" and not args.sentinel_mde:
             logger.error("Export mode for MDE is only available via Azure Sentinel backend for now.")
             exit(1)
 
+        elif args.platform == 'esql' or args.platform == 'eql':
+            args.platform == 'elastic'
+            if is_raw_rule(args, base_config):
+                logger.info("Elastic Security raw rule selected")
+                export_error = export_rule_raw(parameters, droid_platform_config(args, config_path))
+            else:
+                export_error = convert_rules(parameters, droid_platform_config(args, config_path), base_config)
+
         else:
             logger.error("Please select one platform. See option -p in --help")
             exit(1)
@@ -390,7 +424,7 @@ def main(argv=None) -> None:
             integrity_error = integrity_rule_raw(parameters, droid_platform_config(args, config_path))
         else:
             logger.info(f"Integrity check for platform {args.platform} selected")
-            integrity_error = convert_rules(parameters, droid_platform_config(args, config_path))
+            integrity_error = convert_rules(parameters, droid_platform_config(args, config_path), base_config)
 
         if integrity_error:
             logger.error("Integrity error")

src/droid/abstracts.py

@@ -0,0 +1,35 @@
+from abc import ABC, abstractmethod
+
+
+class AbstractPlatform(ABC):
+    """
+    AbstractRule is an abstract base class that defines the structure for a platform.
+    It has three abstract methods: create_rule, remove_rule
+    """
+
+    def __init__(self, name: str):
+        """
+        Initialize the platform.
+        """
+        self.platform_name = name
+
+    @abstractmethod
+    def create_rule(self):
+        """
+        Create a detection rule. This method should be implemented by subclasses.
+        """
+        raise NotImplemented()
+
+    @abstractmethod
+    def get_rule(self):
+        """
+        Get the parameter from a rule. This method should be implemented by subclasses.
+        """
+        raise NotImplemented()
+
+    @abstractmethod
+    def remove_rule(self):
+        """
+        Remove a rule. This method should be implemented by subclasses.
+        """
+        raise NotImplemented()

src/droid/convert.py

@@ -13,6 +13,7 @@
 from droid.integrity import integrity_rule
 from droid.platforms.splunk import SplunkPlatform
 from droid.platforms.sentinel import SentinelPlatform
+from droid.platforms.elastic import ElasticPlatform
 from droid.color import ColorLogger
 
 class Conversion:
@@ -80,15 +81,15 @@ def init_sigma_rule(self, rule_file) -> None:
         Args:
             rule
         """
-        with open(rule_file, "r") as file:
+        with open(rule_file, "r", encoding="utf-8") as file:
             if self._filters_directory:
                 sigma_rule = self.init_sigma_filters(rule_file)
             else:
                 sigma_rule = SigmaCollection.from_yaml(file)
 
         return sigma_rule
 
-    def convert_rule(self, rule_content, rule_file):
+    def convert_rule(self, rule_content, rule_file, platform):
 
         plugins = InstalledSigmaPlugins.autodiscover()
         backends = plugins.backends
@@ -121,14 +122,17 @@ def convert_rule(self, rule_content, rule_file):
             backend: Backend = backend_class(processing_pipeline=pipeline)
             sigma_rule = self.init_sigma_rule(rule_file)
             rule_converted = backend.convert(sigma_rule, self._format)[0]
+            # For esql and eql backend only
+            if isinstance(platform, ElasticPlatform):
+                platform.get_index_name(pipeline, rule_content)
             self.logger.info(f"Successfully convert the rule {rule_file}", extra={"rule_file": rule_file, "rule_content": rule_content, "rule_format": self._format, "rule_converted": rule_converted})
             return rule_converted
         else:
             self.logger.warning(f"Rule not supported: {rule_file}", extra={"rule_file": rule_file, "rule_content": rule_content})
 
 def load_rule(rule_file):
 
-    with open(rule_file, 'r') as stream:
+    with open(rule_file, 'r', encoding="utf-8") as stream:
         try:
             object = list(yaml.safe_load_all(stream))[0]
             if 'fields' in object:
@@ -179,6 +183,10 @@ def convert_rules(parameters, droid_config, base_config):
         target = Conversion(droid_config, base_config, platform_name, parameters.debug, parameters.json)
         if platform_name == 'splunk':
             platform = SplunkPlatform(droid_config, parameters.debug, parameters.json)
+        elif 'esql' in platform_name:
+            platform = ElasticPlatform(droid_config, parameters.debug, parameters.json, "esql", raw=False)
+        elif 'eql' in platform_name:
+            platform = ElasticPlatform(droid_config, parameters.debug, parameters.json, "eql", raw=False)
         elif 'azure' or 'defender' in platform_name:
             platform = SentinelPlatform(droid_config, parameters.debug, parameters.json)
 
@@ -229,7 +237,7 @@ def convert_rules(parameters, droid_config, base_config):
 def convert_sigma(parameters, logger, rule_content, rule_file, target, platform, error, search_warning, rules):
 
     try:
-        rule_converted = target.convert_rule(rule_content, rule_file)
+        rule_converted = target.convert_rule(rule_content, rule_file, platform)
 
         if parameters.debug:
             logger.debug(f"Rule {rule_file} converted into: {rule_converted}", extra={"rule_file": rule_file, "rule_converted": rule_converted, "rule_content": rule_content})
@@ -243,7 +251,10 @@ def convert_sigma(parameters, logger, rule_content, rule_file, target, platform,
         logger.error(f"Sigma Transformation error: {rule_file} - error: {e}", extra={"rule_file": rule_file, "error": e, "rule_content": rule_content})
         error = True
         return error, search_warning
-
+    except NotImplementedError as e:
+        logger.error(f"Sigma Transformation error: {rule_file} - error: {e}", extra={"rule_file": rule_file, "error": e, "rule_content": rule_content})
+        error = True
+        return error, search_warning
     except Exception as e:
             logger.error(f"Fatal error when compiling the rule {rule_file} - verify the backend {e} is installed")
             error = True

src/droid/export.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from droid.platforms.splunk import SplunkPlatform
 from droid.platforms.sentinel import SentinelPlatform
+from droid.platforms.elastic import ElasticPlatform
 from droid.color import ColorLogger
 
 def post_rule_content(rule_content):
@@ -36,7 +37,10 @@ def load_rule(rule_file):
             error = True
             return error
 
-def export_rule(parameters: dict, rule_content: object, rule_converted: str, platform: object, rule_file: str, error: bool):
+def export_rule(
+        parameters: dict, rule_content: object, rule_converted: str,
+        platform: object, rule_file: str, error: bool):
+
     logger = ColorLogger("droid.export")
 
     rule_content = post_rule_content(rule_content)
@@ -45,9 +49,9 @@ def export_rule(parameters: dict, rule_content: object, rule_converted: str, pla
         logger.enable_json_logging()
     try:
         if rule_content.get('custom', {}).get('removed', False): # If rule is set as removed
-            platform.remove_search(rule_content, rule_converted, rule_file)
+            platform.remove_rule(rule_content, rule_converted, rule_file)
         else:
-            platform.create_search(rule_content, rule_converted, rule_file)
+            platform.create_rule(rule_content, rule_converted, rule_file)
     except Exception as e:
         logger.error(f"Could not export the rule {rule_file}: {e}")
         error = True
@@ -69,6 +73,10 @@ def export_rule_raw(parameters: dict, export_config: dict):
         platform = SentinelPlatform(export_config, parameters.debug, parameters.json)
     elif parameters.platform == 'microsoft_defender' and parameters.sentinel_mde:
         platform = SentinelPlatform(export_config, parameters.debug, parameters.json)
+    elif parameters.platform == 'esql':
+        platform = ElasticPlatform(export_config, parameters.debug, parameters.json, "esql", raw=True)
+    elif parameters.platform == 'eql':
+        platform = ElasticPlatform(export_config, parameters.debug, parameters.json, "eql", raw=True)
 
     if path.is_dir():
         error_i = False
@@ -78,13 +86,13 @@ def export_rule_raw(parameters: dict, export_config: dict):
             rule_converted = rule_content['detection']
             if rule_content.get('custom', {}).get('removed', False): # If rule is set as removed
                 try:
-                    platform.remove_search(rule_content, rule_converted, rule_file)
+                    platform.remove_rule(rule_content, rule_converted, rule_file)
                 except:
                     logger.error(f"Error in removing search for rule {rule_file}")
                     error_i = True
             else:
                 try:
-                    platform.create_search(rule_content, rule_converted, rule_file)
+                    platform.create_rule(rule_content, rule_converted, rule_file)
                 except:
                     logger.error(f"Error in creating search for rule {rule_file}")
                     error_i = True
@@ -99,13 +107,13 @@ def export_rule_raw(parameters: dict, export_config: dict):
         rule_converted = rule_content['detection']
         if rule_content.get('custom', {}).get('removed', False): # If rule is set as removed
             try:
-                platform.remove_search(rule_content, rule_converted, rule_file)
+                platform.remove_rule(rule_content, rule_converted, rule_file)
             except:
                 logger.error(f"Error in removing search for rule {rule_file}")
                 error = True
         else:
             try:
-                platform.create_search(rule_content, rule_converted, rule_file)
+                platform.create_rule(rule_content, rule_converted, rule_file)
             except:
                 logger.error(f"Error in creating search for rule {rule_file}")
                 error = True