-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: import crawler from edc-ce, adjust to ap & cleanup
- Loading branch information
Showing
101 changed files
with
6,026 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| name: "Build EDC Connector Image" | ||
| description: "Builds and deploys the React frontend to AWS S3" | ||
| inputs: | ||
| registry-url: | ||
| required: true | ||
| description: "Docker Registry" | ||
| registry-user: | ||
| required: true | ||
| description: "Docker Registry Login Username" | ||
| registry-password: | ||
| required: true | ||
| description: "Docker Registry Login Password" | ||
| image-base-name: | ||
| required: true | ||
| description: "Docker Image Base Name (Company)" | ||
| image-name: | ||
| required: true | ||
| description: "Docker Image Name (Artifact Name)" | ||
| connector-name: | ||
| required: true | ||
| description: "EDC Connector Name in launchers/connectors/{connector-name}" | ||
| title: | ||
| required: true | ||
| description: "Docker Image Title" | ||
| description: | ||
| required: true | ||
| description: "Docker Image Description" | ||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| - name: "Docker: Log in to the Container registry" | ||
| uses: docker/login-action@v2 | ||
| with: | ||
| registry: ${{ inputs.registry-url }} | ||
| username: ${{ inputs.registry-user }} | ||
| password: ${{ inputs.registry-password }} | ||
| - name: "Docker: Store last commit info and build date" | ||
| id: last-commit-information | ||
| shell: bash | ||
| run: | | ||
| echo "LAST_COMMIT_INFO<<EOF" >> $GITHUB_ENV | ||
| export LAST_COMMIT_INFO=$(git log -1) | ||
| echo "$LAST_COMMIT_INFO" >> $GITHUB_ENV | ||
| echo "EOF" >> $GITHUB_ENV | ||
| echo "BUILD_DATE=$(date --utc +%FT%TZ)" >> $GITHUB_ENV | ||
| - name: "Docker: Extract metadata (tags, labels)" | ||
| id: meta | ||
| uses: docker/metadata-action@v4 | ||
| with: | ||
| images: ${{ inputs.registry-url }}/${{ inputs.image-base-name }}/${{ inputs.image-name }} | ||
| labels: | | ||
| org.opencontainers.image.title=${{ inputs.title }} | ||
| org.opencontainers.image.description=${{ inputs.description }} | ||
| tags: | | ||
| type=schedule | ||
| type=semver,pattern={{version}} | ||
| type=semver,pattern={{major}}.{{minor}} | ||
| type=semver,pattern={{major}} | ||
| type=ref,event=branch | ||
| type=ref,event=pr | ||
| type=sha | ||
| type=raw,value=latest,enable={{is_default_branch}} | ||
| type=raw,value=release,enable=${{ startsWith(github.ref, 'refs/tags/') }} | ||
| - name: "Docker: Build and Push" | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| file: authority-portal-backend/catalog-crawler/Dockerfile | ||
| context: . | ||
| push: true | ||
| tags: ${{ steps.meta.outputs.tags }} | ||
| labels: ${{ steps.meta.outputs.labels }} | ||
| build-args: | | ||
| CONNECTOR_NAME=${{ inputs.connector-name }} | ||
| "EDC_LAST_COMMIT_INFO_ARG=${{ env.LAST_COMMIT_INFO }}" | ||
| EDC_BUILD_DATE_ARG=${{ env.BUILD_DATE }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| FROM eclipse-temurin:17-jre-alpine | ||
|
|
||
| # Install curl for healthcheck, bash for entrypoint | ||
| RUN apk add --no-cache curl bash | ||
| SHELL ["/bin/bash", "-c"] | ||
|
|
||
| # Use a non-root user | ||
| RUN adduser -D -H -s /sbin/nologin edc | ||
| USER edc:edc | ||
|
|
||
| # Which app.jar to include | ||
| ARG CONNECTOR_NAME="catalog-crawler-ce" | ||
|
|
||
| # For last-commit-info extension | ||
| ARG EDC_LAST_COMMIT_INFO_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_LAST_COMMIT_INFO_ARG, so there's no last commit info." | ||
| ARG EDC_BUILD_DATE_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_BUILD_DATE_ARG, so there's no build date." | ||
|
|
||
| WORKDIR /app | ||
| COPY ./authority-portal-backend/catalog-crawler/${CONNECTOR_NAME}-launcher/build/libs/app.jar /app | ||
| COPY ./authority-portal-backend/catalog-crawler/logging.properties /app | ||
| COPY ./authority-portal-backend/catalog-crawler/logging.dev.properties /app | ||
|
|
||
| RUN touch /app/empty-properties-file.properties | ||
|
|
||
| ENV EDC_LAST_COMMIT_INFO=$EDC_LAST_COMMIT_INFO_ARG | ||
| ENV EDC_BUILD_DATE=$EDC_BUILD_DATE_ARG | ||
| ENV JVM_ARGS="" | ||
|
|
||
| COPY ./authority-portal-backend/catalog-crawler/docker-entrypoint.sh /app/entrypoint.sh | ||
| ENTRYPOINT ["/app/entrypoint.sh"] | ||
| CMD ["start"] | ||
|
|
||
| # health status is determined by the availability of the /health endpoint | ||
| HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11001/api/check/health || curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11001/backend/api/check/health |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| <!-- PROJECT LOGO --> | ||
| <br /> | ||
| <div align="center"> | ||
| <a href="https://github.com/sovity/authority-portal"> | ||
| <img src="https://raw.githubusercontent.com/sovity/edc-ui/main/src/assets/images/sovity_logo.svg" alt="Logo" width="300"> | ||
| </a> | ||
|
|
||
| <h3 align="center">EDC-Connector Extension:<br />Catalog Crawler</h3> | ||
|
|
||
| <p align="center"> | ||
| <a href="https://github.com/sovity/authority-portal/issues/new?assignees=&labels=kind%2Fbug&projects=&template=bug_report.yaml">Report Bug</a> | ||
| · | ||
| <a href="https://github.com/sovity/edc-ce/issues/new?template=feature_request.md">Request Feature</a> | ||
| </p> | ||
| </div> | ||
|
|
||
| ## About this Extension | ||
|
|
||
| The Catalog Crawler is an additional deployment unit needed to determine the online status of registered connectors and populate the Data Catalog: | ||
|
|
||
| - It is a modified EDC connector with the task to crawl the other connectors' public data offers. | ||
| - It periodically checks the Authority Portal's connector list for its environment. | ||
| - It crawls the given connectors in regular intervals. | ||
| - It writes the data offers and connector statuses into the Authority Portal DB. | ||
| - Each environment configured in the Authority Portal requires its own Catalog Crawler with credentials for that environment's DAPS. | ||
|
|
||
| ## Why does this component exist? | ||
|
|
||
| The Authority Portal uses a non-EDC stack and thus it cannot read the catalogs of participating connectors directly. | ||
|
|
||
| ## Deployment | ||
|
|
||
| Please see the [Productive Deployment Guide](../../docs/deployment-guide/goals/production/README.md) for more information. | ||
|
|
||
| ## License | ||
|
|
||
| Apache License 2.0 - see [LICENSE](../../LICENSE) | ||
|
|
||
| ## Contact | ||
|
|
||
| sovity GmbH - [email protected] |
26 changes: 26 additions & 0 deletions
26
authority-portal-backend/catalog-crawler/catalog-crawler-ce-launcher/build.gradle.kts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| plugins { | ||
| `java-library` | ||
| id("application") | ||
| alias(libs.plugins.shadow) | ||
| } | ||
|
|
||
| dependencies { | ||
| implementation(project(":catalog-crawler:catalog-crawler-launcher-base")) | ||
|
|
||
| api(libs.edc.monitorJdkLogger) | ||
| api(libs.edc.apiObservability) | ||
|
|
||
| implementation(libs.edc.oauth2Core) | ||
| implementation(libs.edc.vaultFilesystem) | ||
| } | ||
|
|
||
| application { | ||
| mainClass.set("de.sovity.edc.ext.catalog.crawler.Main") | ||
| } | ||
|
|
||
| tasks.withType<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar> { | ||
| mergeServiceFiles() | ||
| archiveFileName.set("app.jar") | ||
| } | ||
|
|
||
| group = libs.versions.sovityEdcGroup.get() |
19 changes: 19 additions & 0 deletions
19
authority-portal-backend/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| plugins { | ||
| `java-library` | ||
| } | ||
|
|
||
| dependencies { | ||
| // A minimal EDC that can request catalogs | ||
| api(libs.edc.controlPlaneCore) | ||
| api(libs.edc.dataPlaneSelectorCore) | ||
| api(libs.edc.configurationFilesystem) | ||
| api(libs.edc.controlPlaneAggregateServices) | ||
| api(libs.edc.http) | ||
| api(libs.edc.dsp) | ||
| api(libs.edc.jsonLd) | ||
|
|
||
| // Data Catalog Crawler | ||
| api(project(":catalog-crawler:catalog-crawler")) | ||
| } | ||
|
|
||
| group = libs.versions.sovityEdcGroup.get() |
41 changes: 41 additions & 0 deletions
41
authority-portal-backend/catalog-crawler/catalog-crawler/build.gradle.kts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| plugins { | ||
| `java-library` | ||
| } | ||
|
|
||
| dependencies { | ||
| annotationProcessor(libs.lombok) | ||
| compileOnly(libs.lombok) | ||
|
|
||
| implementation(libs.edc.controlPlaneSpi) | ||
| implementation(libs.edc.managementApiConfiguration) | ||
|
|
||
| implementation(libs.quartz.quartz) | ||
| implementation(libs.commons.lang3) | ||
| implementation(libs.quarkus.jooq) | ||
|
|
||
| api(libs.sovity.edc.catalogParser) | ||
| api(libs.sovity.edc.jsonAndJsonLdUtils) | ||
| api(libs.sovity.edc.wrapperCommonMappers) | ||
| api(libs.sovity.edc.ext.postgresFlywayCore) | ||
| api(libs.sovity.edc.config) | ||
| api(project(":authority-portal-db")) | ||
|
|
||
| testAnnotationProcessor(libs.lombok) | ||
| testCompileOnly(libs.lombok) | ||
| testImplementation(libs.sovity.edc.ext.testUtils) | ||
| testImplementation(libs.assertj.core) | ||
| testImplementation(libs.mockito.core) | ||
| testImplementation(libs.restAssured.restAssured) | ||
| testImplementation(libs.testcontainers.testcontainers) | ||
| testImplementation(libs.flyway.core) | ||
| testImplementation(libs.testcontainers.junitJupiter) | ||
| testImplementation(libs.testcontainers.postgresql) | ||
| testImplementation(libs.junit.api) | ||
| testImplementation(libs.jsonAssert) | ||
| testRuntimeOnly(libs.junit.engine) | ||
| } | ||
|
|
||
| tasks.getByName<Test>("test") { | ||
| useJUnitPlatform() | ||
| maxParallelForks = 1 | ||
| } |
Oops, something went wrong.