feat: Add organization metadata to broker (#353)

CHANGELOG.md

@@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 #### Minor
 
+- Added an endpoint for syncing organization metadata from the Authority Portal.
+- Added organization information to connectors and data offers.
+
 ### Deployment Migration Notes
 
 #### Compatible Versions

extensions/broker-server-api/api/src/main/java/de/sovity/edc/ext/brokerserver/api/BrokerServerResource.java

@@ -14,13 +14,14 @@
 
 package de.sovity.edc.ext.brokerserver.api;
 
+import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalConnectorInfo;
+import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalOrganizationMetadataRequest;
 import de.sovity.edc.ext.brokerserver.api.model.CatalogPageQuery;
 import de.sovity.edc.ext.brokerserver.api.model.CatalogPageResult;
 import de.sovity.edc.ext.brokerserver.api.model.ConnectorDetailPageQuery;
 import de.sovity.edc.ext.brokerserver.api.model.ConnectorDetailPageResult;
 import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageQuery;
 import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageResult;
-import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalConnectorInfo;
 import de.sovity.edc.ext.brokerserver.api.model.DataOfferCountResult;
 import de.sovity.edc.ext.brokerserver.api.model.DataOfferDetailPageQuery;
 import de.sovity.edc.ext.brokerserver.api.model.DataOfferDetailPageResult;
@@ -96,4 +97,10 @@ public interface BrokerServerResource {
     @Operation(description = "Query the amount of public Data Offers by provided Connector URLs." +
         "This endpoint has been replaced by the Authority Portal Connector Metadata endpoint and will be removed in the near future.")
     DataOfferCountResult dataOfferCount(List<String> endpoints);
+
+    @POST
+    @Path("authority-portal-api/organization-metadata")
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Operation(description = "Update organization metadata. Organizations not contained in the payload will be deleted.")
+    void setOrganizationMetadata(AuthorityPortalOrganizationMetadataRequest organizationMetadataRequest, @QueryParam("adminApiKey") String adminApiKey);
 }

extensions/broker-server-api/api/src/main/java/de/sovity/edc/ext/brokerserver/api/model/AuthorityPortalOrganizationMetadata.java

@@ -0,0 +1,23 @@
+package de.sovity.edc.ext.brokerserver.api.model;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+
+import java.time.OffsetDateTime;
+
+@Getter
+@Setter
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+@Schema(description = "Information about a single organization from the Authority Portal.")
+public class AuthorityPortalOrganizationMetadata {
+    @Schema(description = "MDS-ID from the Authority Portal")
+    private String mdsId;
+    @Schema(description = "Company name")
+    private String name;
+}

extensions/broker-server-api/api/src/main/java/de/sovity/edc/ext/brokerserver/api/model/AuthorityPortalOrganizationMetadataRequest.java

@@ -0,0 +1,21 @@
+package de.sovity.edc.ext.brokerserver.api.model;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+
+import java.util.List;
+
+@Getter
+@Setter
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+@Schema(description = "Information about organizations from the Authority Portal.")
+public class AuthorityPortalOrganizationMetadataRequest {
+    @Schema(description = "Organization metadata")
+    private List<AuthorityPortalOrganizationMetadata> organizations;
+}

extensions/broker-server-api/api/src/main/java/de/sovity/edc/ext/brokerserver/api/model/ConnectorDetailPageResult.java

@@ -36,6 +36,9 @@ public class ConnectorDetailPageResult {
     @Schema(description = "Connector Endpoint", example = "https://my-test.connector/api/dsp", requiredMode = Schema.RequiredMode.REQUIRED)
     private String endpoint;
 
+    @Schema(description = "Name of the responsible organization", requiredMode = Schema.RequiredMode.REQUIRED)
+    private String organizationName;
+
     @Schema(description = "Creation date in Broker", requiredMode = Schema.RequiredMode.REQUIRED)
     private OffsetDateTime createdAt;
 

extensions/broker-server-api/api/src/main/java/de/sovity/edc/ext/brokerserver/api/model/ConnectorListEntry.java

@@ -36,6 +36,9 @@ public class ConnectorListEntry {
     @Schema(description = "Connector Endpoint", example = "https://my-test.connector/api/dsp", requiredMode = Schema.RequiredMode.REQUIRED)
     private String endpoint;
 
+    @Schema(description = "Name of the responsible organization", requiredMode = Schema.RequiredMode.REQUIRED)
+    private String organizationName;
+
     @Schema(description = "Creation date in Broker", requiredMode = Schema.RequiredMode.REQUIRED)
     private OffsetDateTime createdAt;
 

extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V7__Organization_Metadata.sql

@@ -0,0 +1,11 @@
+-- Create table for organization metadata
+create table organization_metadata
+(
+    mds_id              text                             not null primary key,
+    name                text                             not null
+);
+
+-- Add MDS-ID column to organization table
+alter table connector add column mds_id text;
+update connector set mds_id = split_part(participant_id, '.', 1)
+where participant_id ~ '^MDSL[A-Za-z0-9]+\.C[A-Za-z0-9]+$';

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtensionContextBuilder.java

@@ -40,6 +40,7 @@
 import de.sovity.edc.ext.brokerserver.services.OfflineConnectorKiller;
 import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalConnectorMetadataApiService;
 import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalConnectorQueryService;
+import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalOrganizationMetadataApiService;
 import de.sovity.edc.ext.brokerserver.services.api.CatalogApiService;
 import de.sovity.edc.ext.brokerserver.services.api.ConnectorApiService;
 import de.sovity.edc.ext.brokerserver.services.api.ConnectorDetailApiService;
@@ -296,6 +297,7 @@ public static BrokerServerExtensionContext buildContext(
         );
         var connectorDetailApiService = new ConnectorDetailApiService(connectorDetailQueryService, connectorOnlineStatusMapper);
         var connectorListApiService = new ConnectorListApiService(connectorListQueryService, connectorOnlineStatusMapper, paginationMetadataUtils);
+        var authorityPortalOrganizationMetadataApiService = new AuthorityPortalOrganizationMetadataApiService();
         var brokerServerResource = new BrokerServerResourceImpl(
                 dslContextFactory,
                 connectorApiService,
@@ -304,7 +306,8 @@ public static BrokerServerExtensionContext buildContext(
                 catalogApiService,
                 dataOfferDetailApiService,
                 adminApiKeyValidator,
-                dataOfferCountApiService
+                dataOfferCountApiService,
+                authorityPortalOrganizationMetadataApiService
         );
 
         return new BrokerServerExtensionContext(

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerResourceImpl.java

@@ -16,6 +16,7 @@
 
 import de.sovity.edc.ext.brokerserver.api.BrokerServerResource;
 import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalConnectorInfo;
+import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalOrganizationMetadataRequest;
 import de.sovity.edc.ext.brokerserver.api.model.CatalogPageQuery;
 import de.sovity.edc.ext.brokerserver.api.model.CatalogPageResult;
 import de.sovity.edc.ext.brokerserver.api.model.ConnectorDetailPageQuery;
@@ -27,6 +28,7 @@
 import de.sovity.edc.ext.brokerserver.api.model.DataOfferDetailPageResult;
 import de.sovity.edc.ext.brokerserver.db.DslContextFactory;
 import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalConnectorMetadataApiService;
+import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalOrganizationMetadataApiService;
 import de.sovity.edc.ext.brokerserver.services.api.CatalogApiService;
 import de.sovity.edc.ext.brokerserver.services.api.ConnectorApiService;
 import de.sovity.edc.ext.brokerserver.services.api.ConnectorDetailApiService;
@@ -51,6 +53,7 @@ public class BrokerServerResourceImpl implements BrokerServerResource {
     private final DataOfferDetailApiService dataOfferDetailApiService;
     private final AdminApiKeyValidator adminApiKeyValidator;
     private final AuthorityPortalConnectorMetadataApiService authorityPortalConnectorMetadataApiService;
+    private final AuthorityPortalOrganizationMetadataApiService authorityPortalOrganizationMetadataApiService;
 
     @Override
     public CatalogPageResult catalogPage(CatalogPageQuery query) {
@@ -94,4 +97,10 @@ public List<AuthorityPortalConnectorInfo> getConnectorMetadata(List<String> endp
     public DataOfferCountResult dataOfferCount(List<String> endpoints) {
         return dslContextFactory.transactionResult(dsl -> authorityPortalConnectorMetadataApiService.countByEndpoints(dsl, endpoints));
     }
+
+    @Override
+    public void setOrganizationMetadata(AuthorityPortalOrganizationMetadataRequest organizationMetadataRequest, String adminApiKey) {
+        adminApiKeyValidator.validateAdminApiKey(adminApiKey);
+        dslContextFactory.transaction(dsl -> authorityPortalOrganizationMetadataApiService.setOrganizationMetadata(dsl, organizationMetadataRequest.getOrganizations()));
+    }
 }

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryDataOfferFetcher.java

@@ -63,6 +63,7 @@ public Field<List<DataOfferListEntryRs>> queryDataOffers(
                 c.ENDPOINT.as("connectorEndpoint"),
                 c.ONLINE_STATUS.as("connectorOnlineStatus"),
                 c.PARTICIPANT_ID.as("connectorParticipantId"),
+                fields.getOrganizationName().as("organizationName"),
                 fields.getOfflineSinceOrLastUpdatedAt().as("connectorOfflineSinceOrLastUpdatedAt")
         );
 

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryFields.java

@@ -14,6 +14,7 @@
 
 package de.sovity.edc.ext.brokerserver.dao.pages.catalog;
 
+import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
 import de.sovity.edc.ext.brokerserver.db.jooq.tables.Connector;
 import de.sovity.edc.ext.brokerserver.db.jooq.tables.DataOffer;
 import de.sovity.edc.ext.brokerserver.db.jooq.tables.DataOfferViewCount;
@@ -27,6 +28,8 @@
 
 import java.time.OffsetDateTime;
 
+import static org.jooq.impl.DSL.coalesce;
+
 /**
  * Tables and fields used in the catalog page query.
  * <p>
@@ -103,10 +106,24 @@ public Field<Integer> getViewCount() {
         return subquery.asField();
     }
 
+    public Field<String> getOrganizationName() {
+        return organizationName(connectorTable.MDS_ID);
+    }
+
     public static Field<OffsetDateTime> offlineSinceOrLastUpdatedAt(Connector connectorTable) {
         return DSL.coalesce(
                 connectorTable.LAST_SUCCESSFUL_REFRESH_AT,
                 connectorTable.CREATED_AT
         );
     }
+
+    public static Field<String> organizationName(Field<String> mdsId) {
+        var om = Tables.ORGANIZATION_METADATA;
+        var organizationName = DSL.select(om.NAME)
+                .from(om)
+                .where(om.MDS_ID.eq(mdsId))
+                .asField()
+                .cast(String.class);
+        return coalesce(organizationName, "Unknown");
+    }
 }

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/DataOfferListEntryRs.java

@@ -36,5 +36,6 @@ public class DataOfferListEntryRs {
     String connectorEndpoint;
     ConnectorOnlineStatus connectorOnlineStatus;
     String connectorParticipantId;
+    String organizationName;
     OffsetDateTime connectorOfflineSinceOrLastUpdatedAt;
 }

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/ConnectorDetailQueryService.java

@@ -14,6 +14,7 @@
 
 package de.sovity.edc.ext.brokerserver.dao.pages.connector;
 
+import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
 import de.sovity.edc.ext.brokerserver.dao.pages.connector.model.ConnectorDetailsRs;
 import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
 import de.sovity.edc.ext.brokerserver.db.jooq.enums.MeasurementErrorStatus;
@@ -32,6 +33,7 @@ public ConnectorDetailsRs queryConnectorDetailPage(DSLContext dsl, String connec
         return dsl.select(
                 c.ENDPOINT.as("endpoint"),
                 c.PARTICIPANT_ID.as("participantId"),
+                CatalogQueryFields.organizationName(c.MDS_ID).as("organizationName"),
                 c.CREATED_AT.as("createdAt"),
                 c.LAST_SUCCESSFUL_REFRESH_AT.as("lastSuccessfulRefreshAt"),
                 c.LAST_REFRESH_ATTEMPT_AT.as("lastRefreshAttemptAt"),

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/ConnectorListQueryService.java

@@ -15,6 +15,7 @@
 package de.sovity.edc.ext.brokerserver.dao.pages.connector;
 
 import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageSortingType;
+import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
 import de.sovity.edc.ext.brokerserver.dao.pages.connector.model.ConnectorListEntryRs;
 import de.sovity.edc.ext.brokerserver.dao.utils.SearchUtils;
 import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
@@ -32,11 +33,16 @@
 public class ConnectorListQueryService {
     public List<ConnectorListEntryRs> queryConnectorPage(DSLContext dsl, String searchQuery, ConnectorPageSortingType sorting) {
         var c = Tables.CONNECTOR;
-        var filterBySearchQuery = SearchUtils.simpleSearch(searchQuery, List.of(c.ENDPOINT, c.PARTICIPANT_ID));
+        var filterBySearchQuery = SearchUtils.simpleSearch(searchQuery, List.of(
+                c.ENDPOINT,
+                c.PARTICIPANT_ID,
+                CatalogQueryFields.organizationName(c.MDS_ID)
+        ));
 
         return dsl.select(
                 c.ENDPOINT.as("endpoint"),
                 c.PARTICIPANT_ID.as("participantId"),
+                CatalogQueryFields.organizationName(c.MDS_ID).as("organizationName"),
                 c.CREATED_AT.as("createdAt"),
                 c.LAST_SUCCESSFUL_REFRESH_AT.as("lastSuccessfulRefreshAt"),
                 c.LAST_REFRESH_ATTEMPT_AT.as("lastRefreshAttemptAt"),

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/model/ConnectorDetailsRs.java

@@ -28,6 +28,7 @@
 public class ConnectorDetailsRs {
     String endpoint;
     String participantId;
+    String organizationName;
     OffsetDateTime createdAt;
     OffsetDateTime lastSuccessfulRefreshAt;
     OffsetDateTime lastRefreshAttemptAt;

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/model/ConnectorListEntryRs.java

@@ -28,6 +28,7 @@
 public class ConnectorListEntryRs {
     String endpoint;
     String participantId;
+    String organizationName;
     OffsetDateTime createdAt;
     OffsetDateTime lastSuccessfulRefreshAt;
     OffsetDateTime lastRefreshAttemptAt;

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/DataOfferDetailPageQueryService.java

@@ -49,6 +49,7 @@ public DataOfferDetailRs queryDataOfferDetailsPage(DSLContext dsl, String assetI
                         c.ENDPOINT.as("connectorEndpoint"),
                         c.ONLINE_STATUS.as("connectorOnlineStatus"),
                         c.PARTICIPANT_ID.as("connectorParticipantId"),
+                        fields.getOrganizationName().as("organizationName"),
                         fields.getViewCount().as("viewCount"))
                 .from(d)
                 .leftJoin(c).on(c.ENDPOINT.eq(d.CONNECTOR_ENDPOINT))

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/model/DataOfferDetailRs.java

@@ -35,6 +35,7 @@ public class DataOfferDetailRs {
     String connectorEndpoint;
     ConnectorOnlineStatus connectorOnlineStatus;
     String connectorParticipantId;
+    String organizationName;
     OffsetDateTime connectorOfflineSinceOrLastUpdatedAt;
     Integer viewCount;
 }

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalOrganizationMetadataApiService.java

@@ -0,0 +1,44 @@
+/*
+ *  Copyright (c) 2023 sovity GmbH
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.brokerserver.services.api;
+
+import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalOrganizationMetadata;
+import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
+import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.OrganizationMetadataRecord;
+import lombok.RequiredArgsConstructor;
+import org.jetbrains.annotations.NotNull;
+import org.jooq.DSLContext;
+
+import java.util.List;
+
+@RequiredArgsConstructor
+public class AuthorityPortalOrganizationMetadataApiService {
+
+    public void setOrganizationMetadata(DSLContext dsl, List<AuthorityPortalOrganizationMetadata> organizationMetadata) {
+        var records = organizationMetadata.stream().map(this::buildRecord).toList();
+
+        dsl.deleteFrom(Tables.ORGANIZATION_METADATA).execute();
+        dsl.batchInsert(records).execute();
+    }
+
+    @NotNull
+    private OrganizationMetadataRecord buildRecord(AuthorityPortalOrganizationMetadata it) {
+        var record = new OrganizationMetadataRecord();
+        record.setMdsId(it.getMdsId());
+        record.setName(it.getName());
+
+        return record;
+    }
+}

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/CatalogApiService.java

@@ -92,7 +92,8 @@ private CatalogDataOffer buildCatalogDataOffer(DataOfferListEntryRs dataOfferRs)
         var asset = dataOfferMappingUtils.buildUiAsset(
             dataOfferRs.getAssetJsonLd(),
             dataOfferRs.getConnectorEndpoint(),
-            dataOfferRs.getConnectorParticipantId()
+            dataOfferRs.getConnectorParticipantId(),
+            dataOfferRs.getOrganizationName()
         );
 
         var dataOffer = new CatalogDataOffer();

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorDetailApiService.java

@@ -34,6 +34,7 @@ public ConnectorDetailPageResult connectorDetailPage(DSLContext dsl, ConnectorDe
         var dto = new ConnectorDetailPageResult();
         dto.setParticipantId(connectorDbRow.getParticipantId());
         dto.setEndpoint(connectorDbRow.getEndpoint());
+        dto.setOrganizationName(connectorDbRow.getOrganizationName());
         dto.setCreatedAt(connectorDbRow.getCreatedAt());
         dto.setLastRefreshAttemptAt(connectorDbRow.getLastRefreshAttemptAt());
         dto.setLastSuccessfulRefreshAt(connectorDbRow.getLastSuccessfulRefreshAt());

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorListApiService.java

@@ -60,6 +60,7 @@ private ConnectorListEntry buildConnectorListEntry(ConnectorListEntryRs connecto
         var dto = new ConnectorListEntry();
         dto.setParticipantId(connector.getParticipantId());
         dto.setEndpoint(connector.getEndpoint());
+        dto.setOrganizationName(connector.getOrganizationName());
         dto.setCreatedAt(connector.getCreatedAt());
         dto.setLastRefreshAttemptAt(connector.getLastRefreshAttemptAt());
         dto.setLastSuccessfulRefreshAt(connector.getLastSuccessfulRefreshAt());

extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferDetailApiService.java

@@ -41,7 +41,8 @@ public DataOfferDetailPageResult dataOfferDetailPage(DSLContext dsl, DataOfferDe
         var asset = dataOfferMappingUtils.buildUiAsset(
             dataOffer.getAssetJsonLd(),
             dataOffer.getConnectorEndpoint(),
-            dataOffer.getConnectorParticipantId()
+            dataOffer.getConnectorParticipantId(),
+            dataOffer.getOrganizationName()
         );
         viewCountLogger.increaseDataOfferViewCount(dsl, query.getAssetId(), query.getConnectorEndpoint());