Changes to handle multiple system configuations log2timeline#2286

plaso/cli/extraction_tool.py

@@ -183,6 +183,8 @@ def _CreateExtractionProcessingConfiguration(self):
       ProcessingConfiguration: extraction processing configuration.
     """
     configuration = configurations.ProcessingConfiguration()
+    configuration.artifact_definitions_path = self._artifact_definitions_path
+    configuration.custom_artifacts_path = self._custom_artifacts_path
     configuration.data_location = self._data_location
     configuration.extraction.archive_types_string = self._archive_types_string
     configuration.artifact_filters = self._artifact_filters
@@ -412,35 +414,6 @@ def _ParseProcessingOptions(self, options):
       dfvfs_definitions.PREFERRED_GPT_BACK_END = (
           dfvfs_definitions.TYPE_INDICATOR_GPT)
 
-  def _PreprocessSource(self, extraction_engine, storage_writer):
-    """Preprocesses the source.
-
-    Args:
-      extraction_engine (BaseEngine): extraction engine to preprocess
-          the sources.
-      storage_writer (StorageWriter): storage writer.
-
-    Returns:
-      list[SystemConfigurationArtifact]: system configurations found in
-          the source.
-    """
-    logger.debug('Starting preprocessing.')
-
-    try:
-      system_configurations = extraction_engine.PreprocessSource(
-          self._artifact_definitions_path, self._custom_artifacts_path,
-          self._file_system_path_specs, storage_writer,
-          resolver_context=self._resolver_context)
-
-    except IOError as exception:
-      system_configurations = []
-
-      logger.error('Unable to preprocess with error: {0!s}'.format(exception))
-
-    logger.debug('Preprocessing done.')
-
-    return system_configurations
-
   def _ProcessSource(self, session, storage_writer):
     """Processes the source and extract events.
 
@@ -460,19 +433,32 @@ def _ProcessSource(self, session, storage_writer):
 
     extraction_engine = self._CreateExtractionEngine(single_process_mode)
 
+    extraction_engine.BuildArtifactsRegistry(
+        self._artifact_definitions_path, self._custom_artifacts_path)
+
     source_configuration = artifacts.SourceConfigurationArtifact(
         path=self._source_path, source_type=self._source_type)
 
     # TODO: check if the source was processed previously.
     # TODO: add check for modification time of source.
 
-    if self._source_type not in self._SOURCE_TYPES_TO_PREPROCESS:
-      system_configurations = []
-    else:
-      # If the source is a directory or a storage media image
-      # run pre-processing.
-      system_configurations = self._PreprocessSource(
-          extraction_engine, storage_writer)
+    # If the source is a directory or a storage media image run pre-processing.
+
+    system_configurations = []
+    if self._source_type in self._SOURCE_TYPES_TO_PREPROCESS:
+      try:
+        logger.debug('Starting preprocessing.')
+
+        system_configurations = extraction_engine.PreprocessSource(
+            self._file_system_path_specs, storage_writer,
+            resolver_context=self._resolver_context)
+
+        logger.debug('Preprocessing done.')
+
+      except IOError as exception:
+        system_configurations = []
+
+        logger.error('Unable to preprocess with error: {0!s}'.format(exception))
 
       # TODO: check if the source was processed previously and if system
       # configuration differs.
@@ -510,13 +496,15 @@ def _ProcessSource(self, session, storage_writer):
 
       self._extract_winevt_resources = False
 
-    configuration = self._CreateExtractionProcessingConfiguration()
+    processing_configuration = (
+        self._CreateExtractionProcessingConfiguration())
+    processing_configuration.force_parser = force_parser
+
     environment_variables = (
         extraction_engine.knowledge_base.GetEnvironmentVariables())
 
     try:
       extraction_engine.BuildCollectionFilters(
-          self._artifact_definitions_path, self._custom_artifacts_path,
           environment_variables, artifact_filter_names=self._artifact_filters,
           filter_file_path=self._filter_file)
     except errors.InvalidFilter as exception:
@@ -551,18 +539,17 @@ def _ProcessSource(self, session, storage_writer):
         logger.debug('Starting extraction in single process mode.')
 
         processing_status = extraction_engine.ProcessSource(
-            storage_writer, self._resolver_context, configuration,
-            system_configurations, self._file_system_path_specs,
-            force_parser=force_parser)
+            storage_writer, self._resolver_context, processing_configuration,
+            system_configurations, self._file_system_path_specs)
 
       else:
         logger.debug('Starting extraction in multi process mode.')
 
-        # The following overrides are needed because pylint 2.6.0 gets confused
-        # about which ProcessSource to check against.
-        # pylint: disable=no-value-for-parameter,unexpected-keyword-arg
-        processing_status = extraction_engine.ProcessSource(
-            storage_writer, session.identifier, configuration,
+        # The method is named ProcessSourceMulti because pylint 2.6.0 and
+        # later gets confused about keyword arguments when ProcessSource
+        # is used.
+        processing_status = extraction_engine.ProcessSourceMulti(
+            storage_writer, session.identifier, processing_configuration,
             system_configurations, self._file_system_path_specs,
             enable_sigsegv_handler=self._enable_sigsegv_handler,
             storage_file_path=self._storage_file_path)

plaso/cli/image_export_tool.py

@@ -314,34 +314,44 @@ def _Extract(
     """
     extraction_engine = engine.BaseEngine()
 
-    # If the source is a directory or a storage media image
-    # run pre-processing.
+    extraction_engine.BuildArtifactsRegistry(
+        artifact_definitions_path, custom_artifacts_path)
+
+    # If the source is a directory or a storage media image run pre-processing.
+
+    system_configurations = []
     if self._source_type in self._SOURCE_TYPES_TO_PREPROCESS:
-      system_configurations = self._PreprocessSource(extraction_engine)
+      try:
+        logger.debug('Starting preprocessing.')
+
+        # Setting storage writer to None here since we do not want to store
+        # preprocessing information.
+        system_configurations = extraction_engine.PreprocessSource(
+            self._file_system_path_specs, None,
+            resolver_context=self._resolver_context)
+
+        logger.debug('Preprocessing done.')
 
-      # TODO: use system_configurations instead of knowledge base
-      _ = system_configurations
+      except IOError as exception:
+        logger.error('Unable to preprocess with error: {0!s}'.format(exception))
+
+    # TODO: use system_configurations instead of knowledge base
+    _ = system_configurations
 
     environment_variables = (
         extraction_engine.knowledge_base.GetEnvironmentVariables())
 
     try:
       extraction_engine.BuildCollectionFilters(
-          artifact_definitions_path, custom_artifacts_path,
           environment_variables, artifact_filter_names=artifact_filters,
           filter_file_path=filter_file)
     except errors.InvalidFilter as exception:
       raise errors.BadConfigOption(
           'Unable to build collection filters with error: {0!s}'.format(
               exception))
 
-    filters_helper = extraction_engine.collection_filters_helper
-
-    excluded_find_specs = None
-    included_find_specs = None
-    if filters_helper:
-      excluded_find_specs = filters_helper.excluded_file_system_find_specs
-      included_find_specs = filters_helper.included_file_system_find_specs
+    excluded_find_specs = extraction_engine.GetCollectionExcludedFindSpecs()
+    included_find_specs = extraction_engine.GetCollectionIncludedFindSpecs()
 
     output_writer.Write('Extracting file entries.\n')
 
@@ -475,34 +485,6 @@ def _ParseSignatureIdentifiers(self, data_location, signature_identifiers):
         specification_store, signature_identifiers)
     self._filter_collection.AddFilter(file_entry_filter)
 
-  def _PreprocessSource(self, extraction_engine):
-    """Preprocesses the source.
-
-    Args:
-      extraction_engine (BaseEngine): extraction engine to preprocess
-          the sources.
-
-    Returns:
-      list[SystemConfigurationArtifact]: system configurations found in
-          the source.
-    """
-    logger.debug('Starting preprocessing.')
-
-    try:
-      # Setting storage writer to None here since we do not want to store
-      # preprocessing information.
-      system_configurations = extraction_engine.PreprocessSource(
-          self._artifact_definitions_path, self._custom_artifacts_path,
-          self._file_system_path_specs, None,
-          resolver_context=self._resolver_context)
-
-    except IOError as exception:
-      logger.error('Unable to preprocess with error: {0!s}'.format(exception))
-
-    logger.debug('Preprocessing done.')
-
-    return system_configurations
-
   def _ReadSpecificationFile(self, path):
     """Reads the format specification file.
 

plaso/engine/configurations.py

@@ -183,10 +183,14 @@ class ProcessingConfiguration(interface.AttributeContainer):
   """Configuration settings for processing.
 
   Attributes:
+    artifact_definitions_path (str): path to artifact definitions directory
+        or file.
     artifact_filters (Optional list[str]): names of artifact
           definitions that are used for filtering file system and Windows
           Registry key paths.
     credentials (list[CredentialConfiguration]): credential configurations.
+    custom_artifacts_path (str): path to custom artifact definitions
+        directory or file.
     data_location (str): path to the data files.
     debug_output (bool): True if debug output should be enabled.
     dynamic_time (bool): True if date and time values should be represented
@@ -195,6 +199,8 @@ class ProcessingConfiguration(interface.AttributeContainer):
         configuration.
     extraction (ExtractionConfiguration): extraction configuration.
     filter_file (str): path to a file with find specifications.
+    force_parser (bool): True if a specified parser should be forced to be used
+        to extract events.
     log_filename (str): name of the log file.
     parser_filter_expression (str): parser filter expression,
         where None represents all parsers and plugins.
@@ -215,14 +221,17 @@ class ProcessingConfiguration(interface.AttributeContainer):
   def __init__(self):
     """Initializes a process configuration object."""
     super(ProcessingConfiguration, self).__init__()
+    self.artifact_definitions_path = None
     self.artifact_filters = None
     self.credentials = []
+    self.custom_artifacts_path = None
     self.data_location = None
     self.debug_output = False
     self.dynamic_time = False
     self.event_extraction = EventExtractionConfiguration()
     self.extraction = ExtractionConfiguration()
     self.filter_file = None
+    self.force_parser = None
     self.log_filename = None
     self.parser_filter_expression = None
     self.preferred_codepage = None