Changes to handle multiple system configuations log2timeline#2286 (lo…

…g2timeline#4636)

plaso/engine/artifact_filters.py

@@ -3,16 +3,15 @@
 
 from artifacts import definitions as artifact_types
 
-from dfwinreg import registry_searcher
+from dfvfs.helpers import file_system_searcher as dfvfs_file_system_searcher
 
-from dfvfs.helpers import file_system_searcher
+from dfwinreg import registry_searcher as dfwinreg_registry_searcher
 
-from plaso.engine import filters_helper
 from plaso.engine import logger
 from plaso.engine import path_helper
 
 
-class ArtifactDefinitionsFiltersHelper(filters_helper.CollectionFiltersHelper):
+class ArtifactDefinitionsFiltersHelper(object):
   """Helper to create collection filters based on artifact definitions.
 
   Builds collection filters from forensic artifact definitions.
@@ -23,8 +22,12 @@ class ArtifactDefinitionsFiltersHelper(filters_helper.CollectionFiltersHelper):
   Attributes:
     file_system_artifact_names (set[str]): names of artifacts definitions that
         generated file system find specifications.
+    file_system_find_specs (list[dfvfs.FindSpec]): file system find
+        specifications of paths to include in the collection.
     registry_artifact_names (set[str]): names of artifacts definitions that
         generated Windows Registry find specifications.
+    registry_find_specs (list[dfwinreg.FindSpec]): Windows Registry find
+        specifications.
   """
 
   _COMPATIBLE_REGISTRY_KEY_PATH_PREFIXES = frozenset([
@@ -46,7 +49,9 @@ def __init__(self, artifacts_registry):
     self._artifacts_registry = artifacts_registry
 
     self.file_system_artifact_names = set()
+    self.file_system_find_specs = []
     self.registry_artifact_names = set()
+    self.registry_find_specs = []
 
   def _BuildFindSpecsFromArtifact(
       self, definition, environment_variables, user_accounts):
@@ -161,7 +166,8 @@ def _BuildFindSpecsFromRegistrySourceKey(self, key_path):
       elif key_path_glob_upper.startswith('HKEY_USERS\\%%USERS.SID%%'):
         key_path_glob = 'HKEY_CURRENT_USER{0:s}'.format(key_path_glob[26:])
 
-      find_spec = registry_searcher.FindSpec(key_path_glob=key_path_glob)
+      find_spec = dfwinreg_registry_searcher.FindSpec(
+          key_path_glob=key_path_glob)
       find_specs.append(find_spec)
 
     return find_specs
@@ -203,7 +209,7 @@ def _BuildFindSpecsFromFileSourcePath(
           continue
 
         try:
-          find_spec = file_system_searcher.FindSpec(
+          find_spec = dfvfs_file_system_searcher.FindSpec(
               case_sensitive=False, location_glob=path,
               location_separator=path_separator)
         except ValueError as exception:
@@ -244,10 +250,10 @@ def BuildFindSpecs(
       find_specs.extend(artifact_find_specs)
 
     for find_spec in find_specs:
-      if isinstance(find_spec, file_system_searcher.FindSpec):
-        self.included_file_system_find_specs.append(find_spec)
+      if isinstance(find_spec, dfvfs_file_system_searcher.FindSpec):
+        self.file_system_find_specs.append(find_spec)
 
-      elif isinstance(find_spec, registry_searcher.FindSpec):
+      elif isinstance(find_spec, dfwinreg_registry_searcher.FindSpec):
         self.registry_find_specs.append(find_spec)
 
       else:

plaso/engine/engine.py

@@ -42,11 +42,13 @@ def __init__(self):
     self._abort = False
     self._analyzers_profiler = None
     self._artifacts_registry = None
-    self._collection_filters_helper = None
+    self._excluded_file_system_find_specs = None
+    self._included_file_system_find_specs = None
     self._memory_profiler = None
     self._name = 'Main'
     self._processing_status = processing_status.ProcessingStatus()
     self._processing_profiler = None
+    self._registry_find_specs = None
     self._serializers_profiler = None
     # The interval of status updates in number of seconds.
     self._status_update_interval = 0.5
@@ -202,11 +204,15 @@ def BuildCollectionFilters(
             self._WINDOWS_REGISTRY_FILES_ARTIFACT_NAMES,
             environment_variables=environment_variables)
 
-      if not filters_helper.included_file_system_find_specs:
+      if not filters_helper.file_system_find_specs:
         raise errors.InvalidFilter(
             'No valid file system find specifications were built from '
             'artifacts.')
 
+      self._included_file_system_find_specs = (
+          filters_helper.file_system_find_specs)
+      self._registry_find_specs = filters_helper.registry_find_specs
+
     elif filter_file_path:
       logger.debug(
           'building find specification based on filter file: {0:s}'.format(
@@ -232,7 +238,10 @@ def BuildCollectionFilters(
             'No valid file system find specifications were built from filter '
             'file: {0:s}.').format(filter_file_path))
 
-    self._collection_filters_helper = filters_helper
+      self._excluded_file_system_find_specs = (
+          filters_helper.excluded_file_system_find_specs)
+      self._included_file_system_find_specs = (
+          filters_helper.included_file_system_find_specs)
 
   # pylint: disable=too-many-arguments
   @classmethod
@@ -269,19 +278,15 @@ def GetCollectionExcludedFindSpecs(self):
     Returns:
       list[dfvfs.FindSpec]: find specifications to exclude from collection.
     """
-    return getattr(
-        self._collection_filters_helper, 'excluded_file_system_find_specs',
-        None) or []
+    return self._excluded_file_system_find_specs or []
 
   def GetCollectionIncludedFindSpecs(self):
     """Retrieves find specifications to include in collection.
 
     Returns:
       list[dfvfs.FindSpec]: find specifications to include in collection.
     """
-    return getattr(
-        self._collection_filters_helper, 'included_file_system_find_specs',
-        None) or []
+    return self._included_file_system_find_specs or []
 
   def GetSourceFileSystem(self, file_system_path_spec, resolver_context=None):
     """Retrieves the file system of the source.

plaso/engine/path_filters.py

@@ -7,7 +7,6 @@
 
 from dfvfs.helpers import file_system_searcher
 
-from plaso.engine import filters_helper
 from plaso.engine import logger
 from plaso.engine import path_helper
 
@@ -51,8 +50,21 @@ def __init__(
     self.paths = paths or []
 
 
-class PathCollectionFiltersHelper(filters_helper.CollectionFiltersHelper):
-  """Path collection filters helper."""
+class PathCollectionFiltersHelper(object):
+  """Path collection filters helper.
+
+  Attributes:
+    excluded_file_system_find_specs (list[dfvfs.FindSpec]): file system find
+        specifications of paths to exclude from the collection.
+    included_file_system_find_specs (list[dfvfs.FindSpec]): file system find
+        specifications of paths to include in the collection.
+  """
+
+  def __init__(self):
+    """Initializes a collection filters helper."""
+    super(PathCollectionFiltersHelper, self).__init__()
+    self.excluded_file_system_find_specs = []
+    self.included_file_system_find_specs = []
 
   def BuildFindSpecs(self, path_filters, environment_variables=None):
     """Builds find specifications from path filters.

plaso/multi_process/extraction_engine.py

@@ -758,8 +758,8 @@ def _StartWorkerProcess(self, process_name):
         timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
 
     process = extraction_process.ExtractionWorkerProcess(
-        task_queue, self._collection_filters_helper,
-        self._processing_configuration, self._system_configurations,
+        task_queue, self._processing_configuration, self._system_configurations,
+        self._excluded_file_system_find_specs, self._registry_find_specs,
         enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name)
 
     # Remove all possible log handlers to prevent a child process from logging

plaso/multi_process/extraction_process.py

@@ -22,33 +22,36 @@ class ExtractionWorkerProcess(task_process.MultiProcessTaskProcess):
   _FILE_SYSTEM_CACHE_SIZE = 3
 
   def __init__(
-      self, task_queue, collection_filters_helper, processing_configuration,
-      system_configurations, **kwargs):
+      self, task_queue, processing_configuration, system_configurations,
+      excluded_file_system_find_specs, registry_find_specs, **kwargs):
     """Initializes an extraction worker process.
 
     Non-specified keyword arguments (kwargs) are directly passed to
     multiprocessing.Process.
 
     Args:
       task_queue (PlasoQueue): task queue.
-      collection_filters_helper (CollectionFiltersHelper): collection filters
-          helper.
       processing_configuration (ProcessingConfiguration): processing
           configuration.
       system_configurations (list[SystemConfigurationArtifact]): system
           configurations.
+     excluded_file_system_find_specs (list[dfvfs.FindSpec]): file system find
+         specifications of paths to exclude from the collection.
+     registry_find_specs (list[dfwinreg.FindSpec]): Windows Registry find
+         specifications.
       kwargs: keyword arguments to pass to multiprocessing.Process.
     """
     super(ExtractionWorkerProcess, self).__init__(
         processing_configuration, **kwargs)
     self._abort = False
-    self._collection_filters_helper = collection_filters_helper
     self._buffer_size = 0
     self._current_display_name = ''
+    self._excluded_file_system_find_specs = excluded_file_system_find_specs
     self._extraction_worker = None
     self._file_system_cache = []
     self._number_of_consumed_sources = 0
     self._parser_mediator = None
+    self._registry_find_specs = registry_find_specs
     self._resolver_context = None
     self._status = definitions.STATUS_INDICATOR_INITIALIZED
     self._task = None
@@ -95,11 +98,8 @@ def _CreateParserMediator(
     Returns:
       ParserMediator: parser mediator.
     """
-    registry_find_specs = getattr(
-        self._collection_filters_helper, 'registry_find_specs', None)
-
     mediator = parsers_mediator.ParserMediator(
-        registry_find_specs=registry_find_specs,
+        registry_find_specs=self._registry_find_specs,
         resolver_context=resolver_context,
         system_configurations=system_configurations)
 
@@ -275,22 +275,17 @@ def _ProcessPathSpec(self, extraction_worker, parser_mediator, path_spec):
       parser_mediator (ParserMediator): parser mediator.
       path_spec (dfvfs.PathSpec): path specification.
     """
-    excluded_find_specs = None
-    if self._collection_filters_helper:
-      excluded_find_specs = (
-          self._collection_filters_helper.excluded_file_system_find_specs)
-
     self._current_display_name = parser_mediator.GetDisplayNameForPathSpec(
         path_spec)
 
     try:
       self._CacheFileSystem(path_spec)
 
-      if excluded_find_specs:
+      if self._excluded_file_system_find_specs:
         file_system = path_spec_resolver.Resolver.OpenFileSystem(
             path_spec, resolver_context=self._resolver_context)
 
-        for find_spec in excluded_find_specs:
+        for find_spec in self._excluded_file_system_find_specs:
           if find_spec.ComparePathSpecLocation(path_spec, file_system):
             logger.debug('Excluded from extraction: {0:s}.'.format(
                 self._current_display_name))

plaso/single_process/extraction_engine.py

@@ -348,11 +348,8 @@ def _CreateParserMediator(
           'Unable to build collection filters with error: {0!s}'.format(
               exception))
 
-    registry_find_specs = getattr(
-        self._collection_filters_helper, 'registry_find_specs', None)
-
     parser_mediator = parsers_mediator.ParserMediator(
-        registry_find_specs=registry_find_specs,
+        registry_find_specs=self._registry_find_specs,
         resolver_context=resolver_context,
         system_configurations=system_configurations)
 

tests/engine/artifact_filters.py

@@ -87,14 +87,13 @@ def testBuildFindSpecsWithFileSystem(self):
         artifact_filter_names, environment_variables=[environment_variable],
         user_accounts=test_user_accounts)
 
-    self.assertEqual(
-        len(test_filters_helper.included_file_system_find_specs), 16)
+    self.assertEqual(len(test_filters_helper.file_system_find_specs), 16)
     self.assertEqual(len(test_filters_helper.registry_find_specs), 0)
 
     # Last find_spec should contain the testuser2 profile path.
     location_segments = sorted([
         find_spec._location_segments
-        for find_spec in test_filters_helper.included_file_system_find_specs])
+        for find_spec in test_filters_helper.file_system_find_specs])
     path_segments = [
         'Users', 'testuser2', 'Documents', 'WindowsPowerShell', 'profile\\.ps1']
     self.assertEqual(location_segments[2], path_segments)
@@ -106,7 +105,7 @@ def testBuildFindSpecsWithFileSystem(self):
         file_system, path_spec)
 
     path_spec_generator = searcher.Find(
-        find_specs=test_filters_helper.included_file_system_find_specs)
+        find_specs=test_filters_helper.file_system_find_specs)
     self.assertIsNotNone(path_spec_generator)
 
     path_specs = list(path_spec_generator)
@@ -137,8 +136,7 @@ def testBuildFindSpecsWithFileSystemAndGroup(self):
         artifact_filter_names, environment_variables=[environment_variable],
         user_accounts=test_user_accounts)
 
-    self.assertEqual(
-        len(test_filters_helper.included_file_system_find_specs), 16)
+    self.assertEqual(len(test_filters_helper.file_system_find_specs), 16)
     self.assertEqual(len(test_filters_helper.registry_find_specs), 0)
 
     path_spec = path_spec_factory.Factory.NewPathSpec(
@@ -148,14 +146,13 @@ def testBuildFindSpecsWithFileSystemAndGroup(self):
         file_system, path_spec)
 
     path_spec_generator = searcher.Find(
-        find_specs=test_filters_helper.included_file_system_find_specs)
+        find_specs=test_filters_helper.file_system_find_specs)
     self.assertIsNotNone(path_spec_generator)
 
     path_specs = list(path_spec_generator)
 
     # Two evtx, one symbolic link to evtx, one AUTHORS, two filter_*.txt
-    # files,
-    # total 6 path specifications.
+    # files, which total 6 path specifications.
     self.assertEqual(len(path_specs), 6)
 
   def testBuildFindSpecsWithRegistry(self):
@@ -166,8 +163,7 @@ def testBuildFindSpecsWithRegistry(self):
     test_filters_helper.BuildFindSpecs(artifact_filter_names)
 
     # There should be 3 Windows Registry find specifications.
-    self.assertEqual(
-        len(test_filters_helper.included_file_system_find_specs), 0)
+    self.assertEqual(len(test_filters_helper.file_system_find_specs), 0)
     self.assertEqual(len(test_filters_helper.registry_find_specs), 3)
 
     file_entry = self._GetTestFileEntry(['SYSTEM'])