Changes for handling multiple system configurations (#4609)

config/dpkg/changelog

@@ -1,5 +1,5 @@
-plaso (20230311-1) unstable; urgency=low
+plaso (20230402-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Log2Timeline maintainers <[email protected]>  Sat, 11 Mar 2023 13:54:59 +0100
+ -- Log2Timeline maintainers <[email protected]>  Sun, 02 Apr 2023 11:27:54 +0200

docs/sources/api/plaso.parsers.rst

@@ -254,6 +254,14 @@ plaso.parsers.olecf module
    :undoc-members:
    :show-inheritance:
 
+plaso.parsers.onedrive module
+-----------------------------
+
+.. automodule:: plaso.parsers.onedrive
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 plaso.parsers.opera module
 --------------------------
 
@@ -422,6 +430,14 @@ plaso.parsers.winlnk module
    :undoc-members:
    :show-inheritance:
 
+plaso.parsers.winpca module
+---------------------------
+
+.. automodule:: plaso.parsers.winpca
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 plaso.parsers.winprefetch module
 --------------------------------
 

docs/sources/api/plaso.parsers.text_plugins.rst

@@ -156,6 +156,14 @@ plaso.parsers.text\_plugins.postgresql module
    :undoc-members:
    :show-inheritance:
 
+plaso.parsers.text\_plugins.powershell\_transcript module
+---------------------------------------------------------
+
+.. automodule:: plaso.parsers.text_plugins.powershell_transcript
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 plaso.parsers.text\_plugins.santa module
 ----------------------------------------
 

docs/sources/api/plaso.storage.rst

@@ -38,6 +38,14 @@ plaso.storage.reader module
    :undoc-members:
    :show-inheritance:
 
+plaso.storage.serializers module
+--------------------------------
+
+.. automodule:: plaso.storage.serializers
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 plaso.storage.time\_range module
 --------------------------------
 

docs/sources/user/Parsers-and-plugins.md

@@ -29,6 +29,7 @@ mft | Parser for NTFS $MFT metadata files.
 msiecf | Parser for Microsoft Internet Explorer (MSIE) 4 - 9 cache (index.dat) files.
 networkminer_fileinfo | Parser for NetworkMiner .fileinfos files.
 olecf | Parser for OLE Compound File (OLECF) format.
+onedrive_log | Parser for OneDrive Log files.
 opera_global | Parser for Opera global history (global_history.dat) files.
 opera_typed_history | Parser for Opera typed history (typed_history.xml) files.
 pe | Parser for Portable Executable (PE) files.
@@ -52,6 +53,8 @@ windefender_history | Parser for Windows Defender scan DetectionHistory files.
 winevt | Parser for Windows EventLog (EVT) files.
 winevtx | Parser for Windows XML EventLog (EVTX) files.
 winjob | Parser for Windows Scheduled Task job (or at-job) files.
+winpca_db0 | Parser for Windows PCA DB0 log files.
+winpca_dic | Parser for Windows PCA DIC log files.
 winreg | Parser for Windows NT Registry (REGF) files.
 
 ### Parser plugins: bencode
@@ -184,6 +187,7 @@ mac_securityd | Parser for MacOS security daemon (securityd) log files.
 mac_wifi | Parser for MacOS Wi-Fi log (wifi.log) files.
 popularity_contest | Parser for Popularity Contest log files.
 postgresql | Parser for PostgreSQL application log files.
+powershell_transcript | Parser for PowerShell transcript event.
 santa | Parser for Santa log (santa.log) files.
 sccm | Parser for System Center Configuration Manager (SCCM) client log files.
 selinux | Parser for SELinux audit log (audit.log) files.
@@ -253,7 +257,7 @@ linux | bencode, czip/oxml, jsonl/docker_container_config, jsonl/docker_containe
 macos | asl_log, bencode, bsm_log, cups_ipp, czip/oxml, filestat, fseventsd, mac_keychain, olecf, plist, spotlight_storedb, sqlite/appusage, sqlite/google_drive, sqlite/imessage, sqlite/ls_quarantine, sqlite/mac_document_versions, sqlite/mac_notes, sqlite/mackeeper_cache, sqlite/mac_knowledgec, sqlite/skype, text/bash_history, text/gdrive_synclog, text/mac_appfirewall_log, text/mac_securityd, text/mac_wifi, text/syslog, text/syslog_traditional, text/zsh_extended_history, utmpx, webhist
 mactime | bodyfile
 webhist | binary_cookies, chrome_cache, chrome_preferences, esedb/msie_webcache, firefox_cache, java_idx, msiecf, opera_global, opera_typed_history, plist/safari_history, sqlite/chrome_8_history, sqlite/chrome_17_cookies, sqlite/chrome_27_history, sqlite/chrome_66_cookies, sqlite/chrome_autofill, sqlite/chrome_extension_activity, sqlite/firefox_cookies, sqlite/firefox_downloads, sqlite/firefox_history, sqlite/safari_historydb
-win7 | custom_destinations, esedb/file_history, esedb/user_access_logging, olecf/olecf_automatic_destinations, recycle_bin, winevtx, win_gen
+win7 | custom_destinations, esedb/file_history, esedb/user_access_logging, olecf/olecf_automatic_destinations, recycle_bin, text/powershell_transcript, winevtx, win_gen, winpca_db0, winpca_dic
 win7_slow | esedb, mft, win7
 win_gen | bencode, czip/oxml, filestat, lnk, mcafee_protection, olecf, pe, prefetch, sqlite/google_drive, sqlite/skype, symantec_scanlog, text/gdrive_synclog, text/sccm, text/setupapi, text/skydrive_log_v1, text/skydrive_log_v2, text/winfirewall, usnjrnl, webhist, winjob, winreg
 winxp | recycle_bin_info2, rplog, win_gen, winevt

plaso/__init__.py

@@ -6,4 +6,4 @@
 of log2timeline.
 """
 
-__version__ = '20230311'
+__version__ = '20230402'

plaso/cli/extraction_tool.py

@@ -149,6 +149,28 @@ def _CheckStorageFile(self, storage_file_path, warn_about_existing=False):
       raise errors.BadConfigOption(
           'Unable to write to storage file: {0:s}'.format(storage_file_path))
 
+  def _CreateExtractionEngine(self, single_process_mode):
+    """Creates an extraction engine.
+
+    Args:
+      single_process_mode (bool): True if the engine should use single process
+          mode.
+
+    Returns:
+      BaseEngine: extraction engine.
+    """
+    if single_process_mode:
+      extraction_engine = single_extraction_engine.SingleProcessEngine()
+    else:
+      extraction_engine = multi_extraction_engine.ExtractionMultiProcessEngine(
+          number_of_worker_processes=self._number_of_extraction_workers,
+          worker_memory_limit=self._worker_memory_limit,
+          worker_timeout=self._worker_timeout)
+
+    extraction_engine.SetStatusUpdateInterval(self._status_view_interval)
+
+    return extraction_engine
+
   def _CreateExtractionProcessingConfiguration(self):
     """Creates an extraction processing configuration.
 
@@ -215,12 +237,11 @@ def _GenerateStorageFileName(self):
 
     return '{0:s}-{1:s}.plaso'.format(datetime_string, source_name)
 
-  def _GetExpandedParserFilterExpression(self, knowledge_base):
+  def _GetExpandedParserFilterExpression(self, system_configuration):
     """Determines the expanded parser filter expression.
 
     Args:
-      knowledge_base (KnowledgeBase): contains information from the source
-          data needed for parsing.
+      system_configuration (SystemConfigurationArtifact): system configuration.
 
     Returns:
       str: expanded parser filter expression.
@@ -230,16 +251,11 @@ def _GetExpandedParserFilterExpression(self, knowledge_base):
           be expanded or if an invalid parser or plugin name is specified.
     """
     parser_filter_expression = self._parser_filter_expression
-    if not parser_filter_expression:
-      operating_system_family = knowledge_base.GetValue('operating_system')
-      operating_system_product = knowledge_base.GetValue(
-          'operating_system_product')
-      operating_system_version = knowledge_base.GetValue(
-          'operating_system_version')
-
+    if not parser_filter_expression and system_configuration:
       operating_system_artifact = artifacts.OperatingSystemArtifact(
-          family=operating_system_family, product=operating_system_product,
-          version=operating_system_version)
+          family=system_configuration.operating_system,
+          product=system_configuration.operating_system_product,
+          version=system_configuration.operating_system_version)
 
       preset_definitions = self._presets_manager.GetPresetsByOperatingSystem(
           operating_system_artifact)
@@ -391,33 +407,41 @@ def _ParseProcessingOptions(self, options):
       dfvfs_definitions.PREFERRED_GPT_BACK_END = (
           dfvfs_definitions.TYPE_INDICATOR_GPT)
 
-  def _PreprocessSources(self, extraction_engine, storage_writer):
-    """Preprocesses the sources.
+  def _PreprocessSource(self, extraction_engine, storage_writer):
+    """Preprocesses the source.
 
     Args:
       extraction_engine (BaseEngine): extraction engine to preprocess
           the sources.
       storage_writer (StorageWriter): storage writer.
+
+    Returns:
+      list[SystemConfigurationArtifact]: system configurations found in
+          the source.
     """
     logger.debug('Starting preprocessing.')
 
     try:
-      extraction_engine.PreprocessSources(
+      system_configurations = extraction_engine.PreprocessSource(
           self._artifact_definitions_path, self._custom_artifacts_path,
-          self._source_path_specs, storage_writer,
+          self._file_system_path_specs, storage_writer,
           resolver_context=self._resolver_context)
 
     except IOError as exception:
+      system_configurations = []
+
       logger.error('Unable to preprocess with error: {0!s}'.format(exception))
 
     logger.debug('Preprocessing done.')
 
-  def _ProcessSources(self, session, storage_writer):
-    """Processes the sources and extract events.
+    return system_configurations
+
+  def _ProcessSource(self, session, storage_writer):
+    """Processes the source and extract events.
 
     Args:
-      session (Session): session in which the sources are processed.
-      storage_writer (StorageWriter): storage writer for a session storage.
+      session (Session): session in which the source is processed.
+      storage_writer (StorageWriter): storage writer to store extracted events.
 
     Returns:
       ProcessingStatus: processing status.
@@ -429,24 +453,31 @@ def _ProcessSources(self, session, storage_writer):
     if self._source_type == dfvfs_definitions.SOURCE_TYPE_FILE:
       single_process_mode = True
 
-    if single_process_mode:
-      extraction_engine = single_extraction_engine.SingleProcessEngine()
-    else:
-      extraction_engine = multi_extraction_engine.ExtractionMultiProcessEngine(
-          number_of_worker_processes=self._number_of_extraction_workers,
-          worker_memory_limit=self._worker_memory_limit,
-          worker_timeout=self._worker_timeout)
+    extraction_engine = self._CreateExtractionEngine(single_process_mode)
 
-    extraction_engine.SetStatusUpdateInterval(self._status_view_interval)
+    source_configuration = artifacts.SourceConfigurationArtifact(
+        path=self._source_path, source_type=self._source_type)
 
-    # If the source is a directory or a storage media image
-    # run pre-processing.
-    if self._source_type in self._SOURCE_TYPES_TO_PREPROCESS:
-      self._PreprocessSources(extraction_engine, storage_writer)
+    # TODO: check if the source was processed previously.
+    # TODO: add check for modification time of source.
 
+    if self._source_type not in self._SOURCE_TYPES_TO_PREPROCESS:
+      system_configurations = []
+      system_configuration = None
+    else:
+      # If the source is a directory or a storage media image
+      # run pre-processing.
+      system_configurations = self._PreprocessSource(
+          extraction_engine, storage_writer)
+      # TODO: add support for more than 1 system configuration.
+      system_configuration = system_configurations[0]
+
+      # TODO: check if the source was processed previously and if system
+      # configuration differs.
+
+    # TODO: add support for more than 1 system configuration.
     self._expanded_parser_filter_expression = (
-        self._GetExpandedParserFilterExpression(
-            extraction_engine.knowledge_base))
+        self._GetExpandedParserFilterExpression(system_configuration))
 
     enabled_parser_names = self._expanded_parser_filter_expression.split(',')
 
@@ -505,38 +536,33 @@ def _ProcessSources(self, session, storage_writer):
     processing_status = None
 
     try:
-      source_configurations = []
-      for path_spec in self._source_path_specs:
-        source_configuration = artifacts.SourceConfigurationArtifact(
-            path_spec=path_spec)
-        source_configurations.append(source_configuration)
-
-      # TODO: improve to detect more than 1 system configurations.
-      # TODO: improve to add volumes to system configuration.
-      system_configuration = (
-          extraction_engine.knowledge_base.GetSystemConfigurationArtifact())
-      storage_writer.AddAttributeContainer(system_configuration)
+      storage_writer.AddAttributeContainer(source_configuration)
+
+      for system_configuration in system_configurations:
+        storage_writer.AddAttributeContainer(system_configuration)
 
       status_update_callback = (
           self._status_view.GetExtractionStatusUpdateCallback())
 
       if single_process_mode:
         logger.debug('Starting extraction in single process mode.')
 
-        processing_status = extraction_engine.ProcessSources(
-            source_configurations, storage_writer, self._resolver_context,
-            configuration, force_parser=force_parser,
+        processing_status = extraction_engine.ProcessSource(
+            storage_writer, self._resolver_context, configuration,
+            system_configurations, self._file_system_path_specs,
+            force_parser=force_parser,
             status_update_callback=status_update_callback)
 
       else:
         logger.debug('Starting extraction in multi process mode.')
 
         # The following overrides are needed because pylint 2.6.0 gets confused
-        # about which ProcessSources to check against.
+        # about which ProcessSource to check against.
         # pylint: disable=no-value-for-parameter,unexpected-keyword-arg
-        processing_status = extraction_engine.ProcessSources(
-            source_configurations, storage_writer, session.identifier,
-            configuration, enable_sigsegv_handler=self._enable_sigsegv_handler,
+        processing_status = extraction_engine.ProcessSource(
+            storage_writer, session.identifier, configuration,
+            system_configurations, self._file_system_path_specs,
+            enable_sigsegv_handler=self._enable_sigsegv_handler,
             status_update_callback=status_update_callback,
             storage_file_path=self._storage_file_path)
 
@@ -705,9 +731,9 @@ def ExtractEventsFromSources(self):
 
     if self._source_type == dfvfs_definitions.SOURCE_TYPE_FILE:
       archive_path_spec = self._ScanSourceForArchive(
-          self._source_path_specs[0])
+          self._file_system_path_specs[0])
       if archive_path_spec:
-        self._source_path_specs = [archive_path_spec]
+        self._file_system_path_specs = [archive_path_spec]
         self._source_type = definitions.SOURCE_TYPE_ARCHIVE
 
     self._status_view.SetMode(self._status_view_mode)
@@ -744,7 +770,7 @@ def ExtractEventsFromSources(self):
           storage_writer.GetNumberOfAttributeContainers('extraction_warning'))
 
       try:
-        processing_status = self._ProcessSources(session, storage_writer)
+        processing_status = self._ProcessSource(session, storage_writer)
 
       finally:
         number_of_extraction_warnings = (