From 483b8e96fbe23848b2cbac469f90e46dee363ec5 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti <111659831+jannotti-glaucio@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:21:51 -0300 Subject: [PATCH] Fix to transfer filters and multiple datasources bugs (#71) * fix: datasink with a non ionos provider datasource * fix: datasink with a non ionos provider datasource * fix: filter in transfer of multiple folders * fix: filter in transfer of multiple folders --- extensions/core-ionos-s3/build.gradle.kts | 11 +- .../data-plane-ionos-s3/build.gradle.kts | 14 +- .../edc/dataplane/ionos/s3/IonosDataSink.java | 62 ++- .../dataplane/ionos/s3/IonosDataSource.java | 14 +- .../ionos/s3/IonosDataSourceTest.java | 417 ++++++++++++++++++ .../ionos/s3/util/FileTransferHelperTest.java | 2 +- .../ionos/s3/IonosDataSinkFractoryTest.java | 38 -- .../ionos/s3/IonosDataSourceFactoryTest.java | 35 -- .../provision-ionos-s3/build.gradle.kts | 8 +- .../s3/IonosProvisionedResourceTest.java | 44 -- gradle.properties | 2 + 11 files changed, 483 insertions(+), 164 deletions(-) create mode 100644 extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java rename extensions/data-plane-ionos-s3/src/test/java/{org/eclipse/edc/connector => com/ionos/edc}/dataplane/ionos/s3/util/FileTransferHelperTest.java (98%) delete mode 100644 extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/IonosDataSinkFractoryTest.java delete mode 100644 extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/IonosDataSourceFactoryTest.java delete mode 100644 extensions/provision-ionos-s3/src/test/java/com/ionos/edc/provision/s3/IonosProvisionedResourceTest.java diff --git a/extensions/core-ionos-s3/build.gradle.kts b/extensions/core-ionos-s3/build.gradle.kts index 24a0aea3..13d735e3 100644 --- a/extensions/core-ionos-s3/build.gradle.kts +++ b/extensions/core-ionos-s3/build.gradle.kts @@ -1,17 +1,12 @@ plugins { `java-library` `maven-publish` - } -val javaVersion: String by project -val faaastVersion: String by project + val edcGroup: String by project val edcVersion: String by project -val okHttpVersion: String by project -val rsApi: String by project val metaModelVersion: String by project val minIOVersion: String by project - val extensionsGroup: String by project val extensionsVersion: String by project @@ -21,14 +16,12 @@ val gitHubUser: String? by project val gitHubToken: String? by project dependencies { - api("${edcGroup}:runtime-metamodel:${metaModelVersion}") implementation("${edcGroup}:transfer-spi:${edcVersion}") implementation("io.minio:minio:${minIOVersion}") - - testImplementation ("${edcGroup}:junit:${edcVersion}") } + java { withJavadocJar() withSourcesJar() diff --git a/extensions/data-plane-ionos-s3/build.gradle.kts b/extensions/data-plane-ionos-s3/build.gradle.kts index 15e13a89..3d166d94 100644 --- a/extensions/data-plane-ionos-s3/build.gradle.kts +++ b/extensions/data-plane-ionos-s3/build.gradle.kts @@ -8,6 +8,8 @@ val edcGroup: String by project val edcVersion: String by project val extensionsGroup: String by project val extensionsVersion: String by project +val junitVersion: String by project +val mockitoVersion: String by project val gitHubPkgsName: String by project val gitHubPkgsUrl: String by project @@ -15,23 +17,19 @@ val gitHubUser: String? by project val gitHubToken: String? by project dependencies { - api("${edcGroup}:data-plane-spi:${edcVersion}") + implementation(project(":extensions:core-ionos-s3")) implementation("${edcGroup}:util:${edcVersion}") implementation("${edcGroup}:transfer-spi:${edcVersion}") implementation("${edcGroup}:data-plane-util:${edcVersion}") implementation("${edcGroup}:data-plane-core:${edcVersion}") implementation("${edcGroup}:http:${edcVersion}") implementation("${edcGroup}:validator-spi:${edcVersion}") - - implementation(project(":extensions:core-ionos-s3")) - - implementation("org.realityforge.org.jetbrains.annotations:org.jetbrains.annotations:1.7.0") - testImplementation("org.junit.jupiter:junit-jupiter-api:5.9.1") - testImplementation("org.junit.jupiter:junit-jupiter-engine:5.9.1") - testImplementation("org.assertj:assertj-core:3.22.0") + testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}") + testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}") + testImplementation("org.mockito:mockito-core:${mockitoVersion}") } java { diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java index 668ef225..377aa164 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java @@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.util.List; import java.util.Objects; @@ -46,27 +47,56 @@ protected StreamResult transferParts(List parts) { blobName = part.name(); } - var streamsOutput = new ByteArrayOutputStream(); - var stream = part.openStream(); - while (stream != null) { - try { - streamsOutput.write(stream.readAllBytes()); - stream.close(); + ByteArrayOutputStream streamsOutput = null; + InputStream stream = null; + try { + streamsOutput = new ByteArrayOutputStream(); + stream = part.openStream(); - } catch (Exception e) { - return uploadFailure(e, blobName); + if (part instanceof IonosDataSource.S3Part) { + // Multiple fetches + while (stream != null) { + try { + streamsOutput.write(stream.readAllBytes()); + stream.close(); + + } catch (Exception e) { + return uploadFailure(e, blobName); + } + + stream = part.openStream(); + } + } else { + // Single fetch + try { + streamsOutput.write(stream.readAllBytes()); + stream.close(); + + } catch (Exception e) { + return uploadFailure(e, blobName); + } } - stream = part.openStream(); - } + var byteArray = streamsOutput.toByteArray(); + try (var streamsInput = new ByteArrayInputStream(byteArray)) { + s3Api.uploadObject(bucketName, blobName, streamsInput); + streamsOutput.close(); - var byteArray = streamsOutput.toByteArray(); - try (var streamsInput = new ByteArrayInputStream(byteArray)) { - s3Api.uploadObject(bucketName, blobName, streamsInput); - streamsOutput.close(); + } catch (Exception e) { + return uploadFailure(e, blobName); + } + } finally { + try { + if (streamsOutput != null) { + streamsOutput.close(); + } + if (stream != null) { + stream.close(); + } - } catch (Exception e) { - return uploadFailure(e, blobName); + } catch (Exception e) { + monitor.severe("Error closing streams", e); + } } } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java index b9e2e106..7a7bd2b8 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSource.java @@ -30,7 +30,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure.Reason.GENERAL_ERROR; +import static org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure.Reason.NOT_FOUND; import static org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult.success; import static org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult.failure; @@ -53,7 +53,7 @@ public StreamResult> openPartStream() { if (objects.isEmpty()) { return failure(new StreamFailure( - List.of("No files found in bucket " + bucketName + " with blobName " + blobName), GENERAL_ERROR) + List.of("No files found in bucket " + bucketName + " with blobName " + blobName), NOT_FOUND) ); } @@ -71,7 +71,7 @@ public StreamResult> openPartStream() { if (objects.isEmpty()) { return failure(new StreamFailure( - List.of("No files found in bucket " + bucketName + " with blobName " + blobName), GENERAL_ERROR) + List.of("No files found in bucket " + bucketName + " with blobName " + blobName), NOT_FOUND) ); } @@ -85,25 +85,25 @@ boolean applyFilterIncludes(S3Object object) { if (object.isRootObject(blobName)) return true; - return filterIncludes.matcher(object.shortObjectName(blobName)).find(); + return filterIncludes.matcher(object.shortObjectName(blobName)).matches(); } boolean applyFilterExcludes(S3Object object) { if (object.isRootObject(blobName)) return true; - return !filterExcludes.matcher(object.shortObjectName(blobName)).find(); + return !filterExcludes.matcher(object.shortObjectName(blobName)).matches(); } @Override public void close() { } - private static class S3Part implements Part { + public static class S3Part implements Part { private final S3ConnectorApi s3Api; private final Monitor monitor; private final String bucketName; private final String blobName; - private boolean isDirectory; + private final boolean isDirectory; private final long fileSize; private boolean isOpened = true; diff --git a/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java new file mode 100644 index 00000000..df97cfb8 --- /dev/null +++ b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSourceTest.java @@ -0,0 +1,417 @@ +/* + * This Java source file was generated by the Gradle 'init' task. + */ +package com.ionos.edc.dataplane.ionos.s3; + +import com.ionos.edc.extension.s3.api.S3ConnectorApi; +import com.ionos.edc.extension.s3.api.S3Object; +import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure; +import org.eclipse.edc.spi.monitor.Monitor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; + +class IonosDataSourceTest { + + private static final String TEST_BUCKET = "bucket1"; + + private static final String TEST_FILE_1_NAME = "device1-data.csv"; + private static final int TEST_FILE_1_SIZE = 1024; + private static final String TEST_FILE_2_NAME = "device2-data.csv"; + private static final int TEST_FILE_2_SIZE = 2048; + private static final String TEST_FILE_3_NAME = "device3-data.csv"; + private static final int TEST_FILE_3_SIZE = 3072; + private static final String TEST_FILE_4_NAME = "device4-data.csv"; + private static final int TEST_FILE_4_SIZE = 4096; + + private static final String TEST_FOLDER_NAME = "devices/"; + private static final String TEST_SUB_FOLDER_1_NAME = "device1/"; + private static final String TEST_SUB_FOLDER_2_NAME = "device2/"; + + @Mock + private S3ConnectorApi s3Api; + @Mock + private Monitor monitor; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + } + + @Test + public void openPartStream_empty() { + + doReturn(List.of()) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FILE_1_NAME) + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.failed()); + assertNull(stream.getContent()); + assertEquals(StreamFailure.Reason.NOT_FOUND, stream.getFailure().getReason()); + } + + @Test + public void openPartStream_singleFile() { + + var s3Objects = List.of(new S3Object(TEST_FILE_1_NAME, TEST_FILE_1_SIZE)); + doReturn(s3Objects) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FILE_1_NAME) + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.succeeded()); + + var parts = stream.getContent().collect(Collectors.toList()); + assertEquals(1, parts.size()); + + var part = parts.get(0); + assertEquals(TEST_FILE_1_NAME, part.name()); + assertEquals(TEST_FILE_1_SIZE, part.size()); + } + + @Test + public void openPartStream_folder() { + + var s3Objects = List.of(new S3Object(TEST_FOLDER_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_1_NAME, TEST_FILE_1_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_2_NAME, TEST_FILE_2_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) + ); + doReturn(s3Objects) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FOLDER_NAME) + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.succeeded()); + + var parts = stream.getContent().collect(Collectors.toList()); + assertEquals(4, parts.size()); + + var partFolder = parts.get(0); + assertEquals(TEST_FOLDER_NAME, partFolder.name()); + assertEquals(0, partFolder.size()); + + var partFile1 = parts.get(1); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_1_NAME, partFile1.name()); + assertEquals(TEST_FILE_1_SIZE, partFile1.size()); + + var partFile2 = parts.get(2); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_2_NAME, partFile2.name()); + assertEquals(TEST_FILE_2_SIZE, partFile2.size()); + + var partFile3 = parts.get(3); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_3_NAME, partFile3.name()); + assertEquals(TEST_FILE_3_SIZE, partFile3.size()); + } + + @Test + public void openPartStream_folder_includeFiles() { + + var s3Objects = List.of(new S3Object(TEST_FOLDER_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_1_NAME, TEST_FILE_1_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_2_NAME, TEST_FILE_2_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) + ); + doReturn(s3Objects) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FOLDER_NAME) + .filterIncludes("device[1-2]-data.csv") + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.succeeded()); + + var parts = stream.getContent().collect(Collectors.toList()); + assertEquals(3, parts.size()); + + var partFolder = parts.get(0); + assertEquals(TEST_FOLDER_NAME, partFolder.name()); + assertEquals(0, partFolder.size()); + + var partFile1 = parts.get(1); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_1_NAME, partFile1.name()); + assertEquals(TEST_FILE_1_SIZE, partFile1.size()); + + var partFile2 = parts.get(2); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_2_NAME, partFile2.name()); + assertEquals(TEST_FILE_2_SIZE, partFile2.size()); + } + + @Test + public void openPartStream_folder_excludeFiles() { + + var s3Objects = List.of(new S3Object(TEST_FOLDER_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_1_NAME, TEST_FILE_1_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_2_NAME, TEST_FILE_2_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) + ); + doReturn(s3Objects) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FOLDER_NAME) + .filterExcludes("device[1-2]-data.csv") + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.succeeded()); + + var parts = stream.getContent().collect(Collectors.toList()); + assertEquals(2, parts.size()); + + var partFolder = parts.get(0); + assertEquals(TEST_FOLDER_NAME, partFolder.name()); + assertEquals(0, partFolder.size()); + + var partFile1 = parts.get(1); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_3_NAME, partFile1.name()); + assertEquals(TEST_FILE_3_SIZE, partFile1.size()); + } + + @Test + public void openPartStream_subFolder() { + + var s3Objects = List.of(new S3Object(TEST_FOLDER_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_1_NAME, TEST_FILE_1_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_2_NAME, TEST_FILE_2_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) + ); + doReturn(s3Objects) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FOLDER_NAME) + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.succeeded()); + + var parts = stream.getContent().collect(Collectors.toList()); + assertEquals(5, parts.size()); + + var partFolder = parts.get(0); + assertEquals(TEST_FOLDER_NAME, partFolder.name()); + assertEquals(0, partFolder.size()); + + var partFile1 = parts.get(1); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_1_NAME, partFile1.name()); + assertEquals(TEST_FILE_1_SIZE, partFile1.size()); + + var partFile2 = parts.get(2); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_2_NAME, partFile2.name()); + assertEquals(TEST_FILE_2_SIZE, partFile2.size()); + + var parSubFolder1 = parts.get(3); + assertEquals(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME, parSubFolder1.name()); + assertEquals(0, parSubFolder1.size()); + + var partFile3 = parts.get(4); + assertEquals(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, partFile3.name()); + assertEquals(TEST_FILE_3_SIZE, partFile3.size()); + } + + @Test + public void openPartStream_subFolder_includeFilesInFolder() { + + var s3Objects = List.of(new S3Object(TEST_FOLDER_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_1_NAME, TEST_FILE_1_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_2_NAME, TEST_FILE_2_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) + ); + doReturn(s3Objects) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FOLDER_NAME) + .filterIncludes("device[1-2]-data.csv") + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.succeeded()); + + var parts = stream.getContent().collect(Collectors.toList()); + assertEquals(3, parts.size()); + + var partFolder = parts.get(0); + assertEquals(TEST_FOLDER_NAME, partFolder.name()); + assertEquals(0, partFolder.size()); + + var partFile1 = parts.get(1); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_1_NAME, partFile1.name()); + assertEquals(TEST_FILE_1_SIZE, partFile1.size()); + + var partFile2 = parts.get(2); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_2_NAME, partFile2.name()); + assertEquals(TEST_FILE_2_SIZE, partFile2.size()); + } + + @Test + public void openPartStream_subFolder_excludeFilesInFolder() { + + var s3Objects = List.of(new S3Object(TEST_FOLDER_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_1_NAME, TEST_FILE_1_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_2_NAME, TEST_FILE_2_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE) + ); + doReturn(s3Objects) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FOLDER_NAME) + .filterExcludes("device1-data.csv") + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.succeeded()); + + var parts = stream.getContent().collect(Collectors.toList()); + assertEquals(4, parts.size()); + + var partFolder = parts.get(0); + assertEquals(TEST_FOLDER_NAME, partFolder.name()); + assertEquals(0, partFolder.size()); + + var partFile2 = parts.get(1); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_2_NAME, partFile2.name()); + assertEquals(TEST_FILE_2_SIZE, partFile2.size()); + + var partSubFolder1 = parts.get(2); + assertEquals(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME, partSubFolder1.name()); + assertEquals(0, partSubFolder1.size()); + + var partFile3 = parts.get(3); + assertEquals(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, partFile3.name()); + assertEquals(TEST_FILE_3_SIZE, partFile3.size()); + } + + @Test + public void openPartStream_subFolder_includeFilesInSubFolder() { + + var s3Objects = List.of(new S3Object(TEST_FOLDER_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_1_NAME, TEST_FILE_1_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_2_NAME, TEST_FILE_2_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_4_NAME, TEST_FILE_4_SIZE) + ); + doReturn(s3Objects) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FOLDER_NAME) + .filterIncludes("device1/device[3-4]-data.csv") + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.succeeded()); + + var parts = stream.getContent().collect(Collectors.toList()); + assertEquals(3, parts.size()); + + var partFolder = parts.get(0); + assertEquals(TEST_FOLDER_NAME, partFolder.name()); + assertEquals(0, partFolder.size()); + + var partFile3 = parts.get(1); + assertEquals(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, partFile3.name()); + assertEquals(TEST_FILE_3_SIZE, partFile3.size()); + + var partFile4 = parts.get(2); + assertEquals(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_4_NAME, partFile4.name()); + assertEquals(TEST_FILE_4_SIZE, partFile4.size()); + } + + @Test + public void openPartStream_subFolder_excludeFilesInSubFolder() { + + var s3Objects = List.of(new S3Object(TEST_FOLDER_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_FILE_1_NAME, TEST_FILE_1_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME, 0), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, TEST_FILE_3_SIZE), + new S3Object(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_4_NAME, TEST_FILE_4_SIZE) + ); + doReturn(s3Objects) + .when(s3Api).listObjects(any(String.class), any(String.class)); + + var dataSource = IonosDataSource.Builder.newInstance() + .client(s3Api) + .monitor(monitor) + .bucketName(TEST_BUCKET) + .blobName(TEST_FOLDER_NAME) + .filterExcludes("device1/device4-data.csv") + .build(); + + var stream = dataSource.openPartStream(); + assertTrue(stream.succeeded()); + + var parts = stream.getContent().collect(Collectors.toList()); + assertEquals(4, parts.size()); + + var partFolder = parts.get(0); + assertEquals(TEST_FOLDER_NAME, partFolder.name()); + assertEquals(0, partFolder.size()); + + var partFile1 = parts.get(1); + assertEquals(TEST_FOLDER_NAME + TEST_FILE_1_NAME, partFile1.name()); + assertEquals(TEST_FILE_1_SIZE, partFile1.size()); + + var parSubFolder1 = parts.get(2); + assertEquals(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME, parSubFolder1.name()); + assertEquals(0, parSubFolder1.size()); + + var partFile3 = parts.get(3); + assertEquals(TEST_FOLDER_NAME + TEST_SUB_FOLDER_1_NAME + TEST_FILE_3_NAME, partFile3.name()); + assertEquals(TEST_FILE_3_SIZE, partFile3.size()); + } + +} diff --git a/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/util/FileTransferHelperTest.java b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelperTest.java similarity index 98% rename from extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/util/FileTransferHelperTest.java rename to extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelperTest.java index d9baa788..0d46d80c 100644 --- a/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/util/FileTransferHelperTest.java +++ b/extensions/data-plane-ionos-s3/src/test/java/com/ionos/edc/dataplane/ionos/s3/util/FileTransferHelperTest.java @@ -1,4 +1,4 @@ -package org.eclipse.edc.connector.dataplane.ionos.s3.util; +package com.ionos.edc.dataplane.ionos.s3.util; import com.ionos.edc.dataplane.ionos.s3.util.FileTransferHelper; import org.eclipse.edc.spi.EdcException; diff --git a/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/IonosDataSinkFractoryTest.java b/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/IonosDataSinkFractoryTest.java deleted file mode 100644 index 74012868..00000000 --- a/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/IonosDataSinkFractoryTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * This Java source file was generated by the Gradle 'init' task. - */ -package org.eclipse.edc.connector.dataplane.ionos.s3; - -import com.ionos.edc.extension.s3.schema.IonosBucketSchema; -import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.junit.jupiter.api.Test; - - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.UUID; - - - -class IonosDataSinkFractoryTest { - - @Test - void shouldProvidePipelineServices(PipelineService pipelineService) { - var request = DataFlowRequest.Builder.newInstance().processId(UUID.randomUUID().toString()) - .sourceDataAddress(getDataAddress("company-1")).destinationDataAddress(getDataAddress("company-2")) - .build(); - System.out.println("IonosDataSinkFractoryTest"); - var result = pipelineService.validate(request); - - assertThat(result.succeeded()).isTrue(); - } - - private DataAddress getDataAddress(String bucketName) { - return DataAddress.Builder.newInstance().type(IonosBucketSchema.TYPE).keyName("111") - .property(IonosBucketSchema.BUCKET_NAME, bucketName) - .property(IonosBucketSchema.STORAGE_NAME, "s3-eu-central-1.ionos.com").build(); - } - -} diff --git a/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/IonosDataSourceFactoryTest.java b/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/IonosDataSourceFactoryTest.java deleted file mode 100644 index 01557756..00000000 --- a/extensions/data-plane-ionos-s3/src/test/java/org/eclipse/edc/connector/dataplane/ionos/s3/IonosDataSourceFactoryTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * This Java source file was generated by the Gradle 'init' task. - */ -package org.eclipse.edc.connector.dataplane.ionos.s3; - - -import com.ionos.edc.extension.s3.schema.IonosBucketSchema; -import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.junit.jupiter.api.Test; - -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; - -class IonosDataSourceFactoryTest { - - @Test - void shouldProvidePipelineServices(PipelineService pipelineService) { - var request = DataFlowRequest.Builder.newInstance().processId(UUID.randomUUID().toString()) - .sourceDataAddress(getDataAddress("company-1")).destinationDataAddress(getDataAddress("company-2")) - .build(); - var result = pipelineService.validate(request); - - assertThat(result.succeeded()).isTrue(); - } - - private DataAddress getDataAddress(String bucketName) { - return DataAddress.Builder.newInstance().type(IonosBucketSchema.TYPE).keyName("111") - .property(IonosBucketSchema.BUCKET_NAME, bucketName) - .property(IonosBucketSchema.STORAGE_NAME, "s3-eu-central-1.ionos.com").build(); - } - -} diff --git a/extensions/provision-ionos-s3/build.gradle.kts b/extensions/provision-ionos-s3/build.gradle.kts index 286b9ebf..38478452 100644 --- a/extensions/provision-ionos-s3/build.gradle.kts +++ b/extensions/provision-ionos-s3/build.gradle.kts @@ -9,6 +9,7 @@ val edcVersion: String by project val metaModelVersion: String by project val extensionsGroup: String by project val extensionsVersion: String by project +val junitVersion: String by project val gitHubPkgsName: String by project val gitHubPkgsUrl: String by project @@ -18,13 +19,8 @@ val gitHubToken: String? by project dependencies { api("${edcGroup}:runtime-metamodel:${metaModelVersion}") - implementation("${edcGroup}:transfer-spi:${edcVersion}") - implementation(project(":extensions:core-ionos-s3")) - - implementation("dev.failsafe:failsafe:3.2.4") - - testImplementation("org.junit.jupiter:junit-jupiter-api:5.9.1") + implementation("${edcGroup}:transfer-spi:${edcVersion}") } java { diff --git a/extensions/provision-ionos-s3/src/test/java/com/ionos/edc/provision/s3/IonosProvisionedResourceTest.java b/extensions/provision-ionos-s3/src/test/java/com/ionos/edc/provision/s3/IonosProvisionedResourceTest.java deleted file mode 100644 index de690d1d..00000000 --- a/extensions/provision-ionos-s3/src/test/java/com/ionos/edc/provision/s3/IonosProvisionedResourceTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * This Java source file was generated by the Gradle 'init' task. - */ -package com.ionos.edc.provision.s3; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.ionos.edc.provision.s3.bucket.IonosS3ProvisionedResource; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.StringWriter; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - - - -class IonosProvisionedResourceTest { - private IonosS3ProvisionedResource proviResource; - - @BeforeEach - void setUp() { - proviResource = IonosS3ProvisionedResource.Builder.newInstance().id("test").transferProcessId("111") - .resourceDefinitionId("test-resource").resourceName("resource-name") - .bucketName("bucket").build(); - - } - - @Test - void verifyDeserialization() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - - StringWriter writer = new StringWriter(); - mapper.writeValue(writer, proviResource); - - IonosS3ProvisionedResource deserialized = mapper.readValue(writer.toString(), IonosS3ProvisionedResource.class); - - assertNotNull(deserialized); - - assertEquals("bucket", deserialized.getBucketName()); - } - -} diff --git a/gradle.properties b/gradle.properties index 847b7f3b..103f5177 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,6 +9,8 @@ metaModelVersion=0.0.1-SNAPSHOT postgresVersion=42.6.0 rsApi=3.1.0 minIOVersion=8.5.8 +junitVersion=5.9.1 +mockitoVersion=5.2.0 ionosDevelopersName=Paulo Lory, Paulo Cabrita gitHubPkgsName=GitHubPackages gitHubRpName=edc-ionos-s3