Refactor meta analyzer to always fetch integrity data

proto/src/main/proto/org/dependencytrack/repometaanalysis/v1/repo_meta_analysis.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-// Public API for Hyades repository meta analysis.
+// Public API for DependencyTrack repository meta analysis.
 package org.dependencytrack.repometaanalysis.v1;
 
 import "google/protobuf/timestamp.proto";
@@ -8,17 +8,11 @@
 option java_multiple_files = true;
 option java_package = "org.dependencytrack.proto.repometaanalysis.v1";
 
 message AnalysisCommand {
   // The component that shall be analyzed.
   Component component = 1;
-  FetchMeta fetch_meta = 2;
-}
 
-enum FetchMeta{
-  FETCH_META_UNSPECIFIED = 0;
-  FETCH_META_INTEGRITY_DATA = 1;
-  FETCH_META_LATEST_VERSION = 2;
-  FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION = 3;
+  reserved 2; // fetch_meta; removed in 5.6.0
 }
 
 message AnalysisResult {
@@ -34,8 +28,33 @@
   // When the latest version was published.
   optional google.protobuf.Timestamp published = 4;
 
-  // Integrity metadata of the component.
   optional IntegrityMeta integrity_meta = 5;
+
+  // When the latest version information was fetched from the repository.
+  optional google.protobuf.Timestamp fetched_at = 6;
+}
+
+message IntegrityMeta {
+  // The MD5 hash of the component.
+  optional string md5 = 1;
+
+  // The SHA1 hash of the component.
+  optional string sha1 = 2;
+
+  // The SHA256 hash of the component.
+  optional string sha256 = 3;
+
+  // The SHA512 hash of the component.
+  optional string sha512 = 4;
+
+  // When the component was published.
+  optional google.protobuf.Timestamp current_version_last_modified = 5;
+
+  // URL from where the information was sourced.
+  string meta_source_url = 6;
+
+  // When the integrity metadata was fetched.
+  google.protobuf.Timestamp fetched_at = 7;
 }
 
 message Component {
@@ -47,14 +66,3 @@
   optional bool internal = 2;
   optional string uuid = 3;
 }
-
-message IntegrityMeta {
-  optional string md5 = 1;
-  optional string sha1 = 2;
-  optional string sha256 = 3;
-  optional string sha512 = 4;
-  // When the component current version last modified.
-  optional google.protobuf.Timestamp current_version_last_modified = 5;
-  // Complete URL to fetch integrity metadata of the component.
-  optional string meta_source_url = 6;
-}

repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/RepositoryMetaAnalyzerTopology.java

@@ -60,6 +60,8 @@ public Topology topology(final RepositoryAnalyzerFactory analyzerFactory,
                         .withName(processorNameConsume(KafkaTopic.REPO_META_ANALYSIS_COMMAND)))
                 .filter((key, scanCommand) -> scanCommand.hasComponent() && isValidPurl(scanCommand.getComponent().getPurl()),
                         Named.as("filter_components_with_valid_purl"))
+                // TODO: This repartition is no longer required as of API server 5.6.0.
+                //  Remove this in Hyades v0.7.0 and consume from REPO_META_ANALYSIS_COMMAND directly instead.
                 // Re-key to PURL coordinates WITHOUT VERSION. As we are fetching data for packages,
                 // but not specific package versions, including the version here would make our caching
                 // largely ineffective. We want events for the same package to be sent to the same partition.

repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/model/IntegrityMeta.java

@@ -35,6 +35,8 @@ public class IntegrityMeta implements Serializable {
 
     private String metaSourceUrl;
 
+    private final Date fetchedAt = new Date();
+
     public String getMd5() {
         return md5;
     }
@@ -82,4 +84,9 @@ public String getMetaSourceUrl() {
     public void setMetaSourceUrl(String metaSourceUrl) {
         this.metaSourceUrl = metaSourceUrl;
     }
+
+    public Date getFetchedAt() {
+        return fetchedAt;
+    }
+
 }

repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/model/MetaModel.java

@@ -28,9 +28,12 @@ public class MetaModel implements Serializable {
     private Component component;
     private String latestVersion;
     private Date publishedTimestamp;
+    private String repositoryIdentifier;
+    private final Date fetchedAt = new Date();
 
-    public MetaModel(){
+    public MetaModel() {
     }
+
     public MetaModel(final Component component) {
         this.component = component;
     }
@@ -54,4 +57,17 @@ public Date getPublishedTimestamp() {
     public void setPublishedTimestamp(final Date publishedTimestamp) {
         this.publishedTimestamp = publishedTimestamp;
     }
+
+    public String getRepositoryIdentifier() {
+        return repositoryIdentifier;
+    }
+
+    public void setRepositoryIdentifier(final String repositoryIdentifier) {
+        this.repositoryIdentifier = repositoryIdentifier;
+    }
+
+    public Date getFetchedAt() {
+        return fetchedAt;
+    }
+
 }