diff --git a/docs/images/reactor-graph.png b/docs/images/reactor-graph.png index b0e2940..d9e9193 100644 Binary files a/docs/images/reactor-graph.png and b/docs/images/reactor-graph.png differ diff --git a/domain/src/main/java/net/kemitix/thorp/domain/StorageEvent.java b/domain/src/main/java/net/kemitix/thorp/domain/StorageEvent.java index 88620c5..a4c4235 100644 --- a/domain/src/main/java/net/kemitix/thorp/domain/StorageEvent.java +++ b/domain/src/main/java/net/kemitix/thorp/domain/StorageEvent.java @@ -1,6 +1,7 @@ package net.kemitix.thorp.domain; import lombok.AccessLevel; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; @@ -27,6 +28,7 @@ public class StorageEvent { public static class DoNothingEvent extends StorageEvent { public final RemoteKey remoteKey; } + @EqualsAndHashCode(callSuper = false) @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public static class CopyEvent extends StorageEvent { public final RemoteKey sourceKey; @@ -37,6 +39,7 @@ public class StorageEvent { public final RemoteKey remoteKey; public final MD5Hash md5Hash; } + @EqualsAndHashCode(callSuper = false) @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public static class DeleteEvent extends StorageEvent { public final RemoteKey remoteKey; diff --git a/filesystem/src/main/java/net/kemitix/thorp/filesystem/MD5HashGenerator.java b/filesystem/src/main/java/net/kemitix/thorp/filesystem/MD5HashGenerator.java index b67e1e2..6c8c150 100644 --- a/filesystem/src/main/java/net/kemitix/thorp/filesystem/MD5HashGenerator.java +++ b/filesystem/src/main/java/net/kemitix/thorp/filesystem/MD5HashGenerator.java @@ -13,7 +13,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; public class MD5HashGenerator implements HashGenerator { - private static final int maxBufferSize = 8048; + private static final int maxBufferSize = 8192; private static final byte[] defaultBuffer = new byte[maxBufferSize]; public static String hex(byte[] in) throws NoSuchAlgorithmException { MessageDigest md5 = MessageDigest.getInstance("MD5"); diff --git a/lib/src/main/scala/net/kemitix/thorp/lib/UnversionedMirrorArchive.scala b/lib/src/main/scala/net/kemitix/thorp/lib/UnversionedMirrorArchive.scala index 9edab94..121511b 100644 --- a/lib/src/main/scala/net/kemitix/thorp/lib/UnversionedMirrorArchive.scala +++ b/lib/src/main/scala/net/kemitix/thorp/lib/UnversionedMirrorArchive.scala @@ -2,6 +2,7 @@ package net.kemitix.thorp.lib import net.kemitix.thorp.config.Configuration import net.kemitix.thorp.domain.Action.{ToCopy, ToDelete, ToUpload} +import net.kemitix.thorp.domain.StorageEvent.ActionSummary import net.kemitix.thorp.domain._ import net.kemitix.thorp.storage.Storage import net.kemitix.thorp.uishell.{UIEvent, UploadEventListener} @@ -35,7 +36,16 @@ trait UnversionedMirrorArchive extends ThorpArchive { .copy(bucket, sourceKey, hash, targetKey) case toDelete: ToDelete => val remoteKey = toDelete.remoteKey - Storage.getInstance().delete(bucket, remoteKey) + try { + Storage.getInstance().delete(bucket, remoteKey) + } catch { + case e: Throwable => + StorageEvent.errorEvent( + ActionSummary.delete(remoteKey.key()), + remoteKey, + e + ); + } case doNothing: Action.DoNothing => val remoteKey = doNothing.remoteKey StorageEvent.doNothingEvent(remoteKey) diff --git a/storage-aws/pom.xml b/storage-aws/pom.xml index 49e1b3c..50c2db4 100644 --- a/storage-aws/pom.xml +++ b/storage-aws/pom.xml @@ -48,11 +48,10 @@ assertj-core test - - - org.scala-lang - scala-library + org.mockito + mockito-junit-jupiter + test @@ -95,27 +94,5 @@ commons-logging 1.2 - - - - org.scalatest - scalatest_2.13 - test - - - org.scalamock - scalamock_2.13 - test - - - - - - net.alchim31.maven - scala-maven-plugin - - - - diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Copier.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Copier.java index bcf1b6c..b0b1018 100644 --- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Copier.java +++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Copier.java @@ -1,5 +1,6 @@ package net.kemitix.thorp.storage.aws; +import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.model.CopyObjectRequest; import net.kemitix.thorp.domain.Bucket; import net.kemitix.thorp.domain.MD5Hash; @@ -24,16 +25,19 @@ public interface S3Copier { return request -> { RemoteKey sourceKey = RemoteKey.create(request.getSourceKey()); RemoteKey targetKey = RemoteKey.create(request.getDestinationKey()); - return client.copyObject(request) - .map(success -> StorageEvent.copyEvent(sourceKey, targetKey)) - .orElseGet(() -> errorEvent(sourceKey, targetKey)); + try { + return client.copyObject(request) + .map(success -> StorageEvent.copyEvent(sourceKey, targetKey)) + .orElseGet(() -> StorageEvent.errorEvent( + actionSummary(sourceKey, targetKey), + targetKey, S3Exception.hashError())); + } catch (SdkClientException e) { + return StorageEvent.errorEvent( + actionSummary(sourceKey, targetKey), targetKey, e); + } }; } - static StorageEvent.ErrorEvent errorEvent(RemoteKey sourceKey, RemoteKey targetKey) { - return StorageEvent.errorEvent(actionSummary(sourceKey, targetKey), targetKey, S3Exception.hashError()); - } - static StorageEvent.ActionSummary.Copy actionSummary(RemoteKey sourceKey, RemoteKey targetKey) { return StorageEvent.ActionSummary.copy( String.format("%s => %s", sourceKey.key(), targetKey.key())); diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Deleter.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Deleter.java index e6ff768..b011584 100644 --- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Deleter.java +++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Deleter.java @@ -1,5 +1,6 @@ package net.kemitix.thorp.storage.aws; +import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.model.DeleteObjectRequest; import net.kemitix.thorp.domain.Bucket; import net.kemitix.thorp.domain.RemoteKey; @@ -13,9 +14,15 @@ public interface S3Deleter { } static Function deleter(AmazonS3Client client) { return request -> { - client.deleteObject(request); RemoteKey remoteKey = RemoteKey.create(request.getKey()); - return StorageEvent.deleteEvent(remoteKey); + try { + client.deleteObject(request); + return StorageEvent.deleteEvent(remoteKey); + } catch (SdkClientException e) { + return StorageEvent.errorEvent( + StorageEvent.ActionSummary.delete(remoteKey.key()), + remoteKey, e); + } }; } } diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3ETagGenerator.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3ETagGenerator.java index 68eff55..9debcfa 100644 --- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3ETagGenerator.java +++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3ETagGenerator.java @@ -7,15 +7,18 @@ import net.kemitix.thorp.domain.HashGenerator; import net.kemitix.thorp.domain.HashType; import net.kemitix.thorp.domain.Hashes; import net.kemitix.thorp.domain.MD5Hash; +import net.kemitix.thorp.filesystem.MD5HashGenerator; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Path; -import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.stream.Collectors; import java.util.stream.LongStream; +import static net.kemitix.thorp.filesystem.MD5HashGenerator.md5FileChunk; + public class S3ETagGenerator implements HashGenerator { @Deprecated // Use hashFile public String eTag(Path path) throws IOException, NoSuchAlgorithmException { @@ -43,7 +46,7 @@ public class S3ETagGenerator implements HashGenerator { public List offsets(long totalFileSizeBytes, long optimalPartSize) { return LongStream - .range(0, totalFileSizeBytes / optimalPartSize) + .rangeClosed(0, totalFileSizeBytes / optimalPartSize) .mapToObj(part -> part * optimalPartSize) .collect(Collectors.toList()); } @@ -63,12 +66,11 @@ public class S3ETagGenerator implements HashGenerator { } private String eTagHex(Path path, long partSize, long parts) throws IOException, NoSuchAlgorithmException { - HashGenerator hashGenerator = HashGenerator.generatorFor("MD5"); - MessageDigest md5 = MessageDigest.getInstance("MD5"); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (long i = 0; i < parts ; i++ ){ - md5.update(hashGenerator.hashChunk(path, i, partSize).digest()); + bos.write(md5FileChunk(path, i * partSize, partSize).digest()); } - return MD5Hash.digestAsString(md5.digest()); + return MD5HashGenerator.hex(bos.toByteArray()); } @Override public HashType hashType() { diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Lister.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Lister.java index 3a7b566..4bd8034 100644 --- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Lister.java +++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Lister.java @@ -1,5 +1,7 @@ package net.kemitix.thorp.storage.aws; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; import com.amazonaws.services.s3.model.S3ObjectSummary; @@ -71,10 +73,14 @@ public interface S3Lister { AmazonS3Client client, ListObjectsV2Request request ) { - Batch batch = fetchBatch(client, request); - List more = fetchMore(client, request, batch.more); - batch.summaries.addAll(more); - return batch.summaries; + try { + Batch batch = fetchBatch(client, request); + List more = fetchMore(client, request, batch.more); + batch.summaries.addAll(more); + return batch.summaries; + } catch (SdkClientException e) { + return Collections.emptyList(); + } }; static Optional moreToken(ListObjectsV2Result result) { diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3TransferManager.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3TransferManager.java index c66b9fd..760f091 100644 --- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3TransferManager.java +++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3TransferManager.java @@ -2,7 +2,6 @@ package net.kemitix.thorp.storage.aws; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.Upload; import java.util.function.Function; @@ -17,14 +16,9 @@ public interface S3TransferManager { } @Override public Function uploader() { - return request -> { - Upload upload = transferManager.upload(request); - try { - return S3Upload.inProgress(upload); - } catch (S3Exception.UploadError error) { - return S3Upload.errored(error); - } - }; + return request -> + S3Upload.inProgress( + transferManager.upload(request)); } }; } diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Uploader.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Uploader.java index 377308e..48c4cc5 100644 --- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Uploader.java +++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Uploader.java @@ -1,5 +1,6 @@ package net.kemitix.thorp.storage.aws; +import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.transfer.model.UploadResult; @@ -29,13 +30,20 @@ public interface S3Uploader { S3TransferManager transferManager ) { return request -> { - UploadResult uploadResult = - transferManager.uploader() - .apply(request) - .waitForUploadResult(); - return StorageEvent.uploadEvent( - RemoteKey.create(uploadResult.getKey()), - MD5Hash.create(uploadResult.getETag())); + try { + UploadResult uploadResult = + transferManager.uploader() + .apply(request) + .waitForUploadResult(); + return StorageEvent.uploadEvent( + RemoteKey.create(uploadResult.getKey()), + MD5Hash.create(uploadResult.getETag())); + } catch (SdkClientException e) { + String key = request.getKey(); + return StorageEvent.errorEvent( + StorageEvent.ActionSummary.upload(key), + RemoteKey.create(key), e); + } }; } } diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/CopierTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/CopierTest.java new file mode 100644 index 0000000..952d0e5 --- /dev/null +++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/CopierTest.java @@ -0,0 +1,65 @@ +package net.kemitix.thorp.storage.aws; + +import com.amazonaws.SdkClientException; +import com.amazonaws.services.s3.model.CopyObjectRequest; +import com.amazonaws.services.s3.model.CopyObjectResult; +import net.kemitix.thorp.domain.Bucket; +import net.kemitix.thorp.domain.MD5Hash; +import net.kemitix.thorp.domain.RemoteKey; +import net.kemitix.thorp.domain.StorageEvent; +import org.assertj.core.api.WithAssertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.doThrow; + +@ExtendWith(MockitoExtension.class) +public class CopierTest + implements WithAssertions { + + private final Bucket bucket = Bucket.named("aBucket"); + private final RemoteKey sourceKey = RemoteKey.create("sourceKey"); + private final MD5Hash hash = MD5Hash.create("aHash"); + private final RemoteKey targetKey = RemoteKey.create("targetKey"); + private final CopyObjectRequest request = S3Copier.request(bucket, sourceKey, hash, targetKey); + + private final AmazonS3Client amazonS3Client; + + public CopierTest(@Mock AmazonS3Client amazonS3Client) { + this.amazonS3Client = amazonS3Client; + } + + @Test + @DisplayName("copy produces copy event") + public void copy_thenCopyEvent() { + //given + StorageEvent event = StorageEvent.copyEvent(sourceKey, targetKey); + given(amazonS3Client.copyObject(request)) + .willReturn(Optional.of(new CopyObjectResult())); + //when + StorageEvent result = S3Copier.copier(amazonS3Client).apply(request); + //then + assertThat(result).isEqualTo(event); + } + + @Test + @DisplayName("when error copying then return an error storage event") + public void whenCopyErrors_thenErrorEvent() { + //given + doThrow(SdkClientException.class) + .when(amazonS3Client) + .copyObject(request); + //when + StorageEvent result = S3Copier.copier(amazonS3Client).apply(request); + //then + assertThat(result).isInstanceOf(StorageEvent.ErrorEvent.class); + } + +} diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/DeleterTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/DeleterTest.java new file mode 100644 index 0000000..aa22d99 --- /dev/null +++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/DeleterTest.java @@ -0,0 +1,52 @@ +package net.kemitix.thorp.storage.aws; + +import com.amazonaws.SdkClientException; +import com.amazonaws.services.s3.model.DeleteObjectRequest; +import net.kemitix.thorp.domain.Bucket; +import net.kemitix.thorp.domain.RemoteKey; +import net.kemitix.thorp.domain.StorageEvent; +import org.assertj.core.api.WithAssertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.mockito.Mockito.doThrow; + +@ExtendWith(MockitoExtension.class) +public class DeleterTest + implements WithAssertions { + + private final Bucket bucket = Bucket.named("aBucket"); + private final RemoteKey remoteKey = RemoteKey.create("aRemoteKey"); + private final DeleteObjectRequest request = S3Deleter.request(bucket, remoteKey); + + private final AmazonS3Client amazonS3Client; + + public DeleterTest(@Mock AmazonS3Client amazonS3Client) { + this.amazonS3Client = amazonS3Client; + } + + @Test + @DisplayName("when delete then return delete storage event") + public void whenDelete_thenDeleteEvent() { + //when + StorageEvent result = S3Deleter.deleter(amazonS3Client).apply(request); + //then + assertThat(result).isEqualTo(StorageEvent.deleteEvent(remoteKey)); + } + + @Test + @DisplayName("when error deleting then return an error storage event") + public void whenDeleteErrors_thenErrorEvent() { + //given + doThrow(SdkClientException.class) + .when(amazonS3Client) + .deleteObject(request); + //when + StorageEvent result = S3Deleter.deleter(amazonS3Client).apply(request); + //then + assertThat(result).isInstanceOf(StorageEvent.ErrorEvent.class); + } +} diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ETagGeneratorTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ETagGeneratorTest.java new file mode 100644 index 0000000..eb87722 --- /dev/null +++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ETagGeneratorTest.java @@ -0,0 +1,54 @@ +package net.kemitix.thorp.storage.aws; + +import net.kemitix.thorp.domain.MD5Hash; +import net.kemitix.thorp.domain.Tuple; +import net.kemitix.thorp.filesystem.Resource; +import org.assertj.core.api.WithAssertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.List; + +public class ETagGeneratorTest + implements WithAssertions { + + public static final String BIG_FILE_ETAG = "f14327c90ad105244c446c498bfe9a7d-2"; + private final Resource bigFile = Resource.select(this, "big-file"); + private final long chunkSize = 1200000; + private final S3ETagGenerator generator = new S3ETagGenerator(); + + @Test + @DisplayName("creates offsets") + public void createsOffsets() { + List offsets = generator.offsets(bigFile.length(), chunkSize); + assertThat(offsets).containsExactly( + 0L, chunkSize, chunkSize * 2, chunkSize * 3, chunkSize * 4); + } + + @Test + @DisplayName("generate valid hashes") + public void generatesValidHashes() throws IOException, NoSuchAlgorithmException { + List> md5Hashes = Arrays.asList( + Tuple.create(0, "68b7d37e6578297621e06f01800204f1"), + Tuple.create(1, "973475b14a7bda6ad8864a7f9913a947"), + Tuple.create(2, "b9adcfc5b103fe2dd5924a5e5e6817f0"), + Tuple.create(3, "5bd6e10a99fef100fe7bf5eaa0a42384"), + Tuple.create(4, "8a0c1d0778ac8fcf4ca2010eba4711eb")); + for (Tuple t : md5Hashes) { + long offset = t.a * chunkSize; + MD5Hash md5Hash = + generator.hashChunk(bigFile.toPath(), offset, chunkSize); + assertThat(md5Hash.getValue()).isEqualTo(t.b); + } + } + + @Test + @DisplayName("create eTag for whole file") + public void createTagForWholeFile() throws IOException, NoSuchAlgorithmException { + String result = generator.hashFile(bigFile.toPath()); + assertThat(result).isEqualTo(BIG_FILE_ETAG); + } +} diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/HashGeneratorTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/HashGeneratorTest.java index 7ea82d3..c53d7f6 100644 --- a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/HashGeneratorTest.java +++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/HashGeneratorTest.java @@ -42,7 +42,8 @@ public class HashGeneratorTest //when Hashes result = HashGenerator.hashObject(path); //then - assertThat(result.get(HashType.MD5)).contains(MD5HashData.rootHash()); + assertThat(result.get(HashType.MD5)) + .contains(MD5Hash.create("a3a6ac11a0eb577b81b3bb5c95cc8a6e")); } @Test @DisplayName("leaf-file") @@ -52,7 +53,8 @@ public class HashGeneratorTest //when Hashes result = HashGenerator.hashObject(path); //then - assertThat(result.get(HashType.MD5)).contains(MD5HashData.leafHash()); + assertThat(result.get(HashType.MD5)) + .contains(MD5Hash.create("208386a650bdec61cfcd7bd8dcb6b542")); } private Path getResource(String s) { diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ListerTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ListerTest.java new file mode 100644 index 0000000..cb2592a --- /dev/null +++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ListerTest.java @@ -0,0 +1,136 @@ +package net.kemitix.thorp.storage.aws; + +import com.amazonaws.SdkClientException; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import net.kemitix.thorp.domain.*; +import org.assertj.core.api.WithAssertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.doThrow; + +@ExtendWith(MockitoExtension.class) +public class ListerTest + implements WithAssertions { + + private final Bucket bucket = Bucket.named("aBucket"); + private final RemoteKey prefix = RemoteKey.create("aRemoteKey"); + private final ListObjectsV2Request request = S3Lister.request(bucket, prefix); + + private final AmazonS3Client amazonS3Client; + + public ListerTest(@Mock AmazonS3Client amazonS3Client) { + this.amazonS3Client = amazonS3Client; + } + + @Test + @DisplayName("single fetch") + public void singleFetch() { + //given + Date nowDate = new Date(); + String key = "key"; + String etag = "etag"; + Map expectedHashMap = + Collections.singletonMap( + MD5Hash.create(etag), + RemoteKey.create(key) + ); + Map expectedKeyMap = + Collections.singletonMap( + RemoteKey.create(key), + MD5Hash.create(etag) + ); + given(amazonS3Client.listObjects(any())) + .willReturn(objectResults(nowDate, key, etag, false)); + //when + RemoteObjects result = S3Lister.lister(amazonS3Client).apply(request); + //then + assertThat(result.byHash.asMap()).isEqualTo(expectedHashMap); + assertThat(result.byKey.asMap()).isEqualTo(expectedKeyMap); + } + + @Test + @DisplayName("two fetches") + public void twoFetches() { + //given + Date nowDate = new Date(); + String key1 = "key1"; + String etag1 = "etag1"; + String key2 = "key2"; + String etag2 = "etag2"; + Map expectedHashMap = new HashMap<>(); + expectedHashMap.put( + MD5Hash.create(etag1), + RemoteKey.create(key1)); + expectedHashMap.put( + MD5Hash.create(etag2), + RemoteKey.create(key2)); + Map expectedKeyMap = new HashMap<>(); + expectedKeyMap.put( + RemoteKey.create(key1), + MD5Hash.create(etag1)); + expectedKeyMap.put( + RemoteKey.create(key2), + MD5Hash.create(etag2)); + given(amazonS3Client.listObjects(any())) + .willReturn(objectResults(nowDate, key1, etag1, true)) + .willReturn(objectResults(nowDate, key2, etag2, false)); + //when + RemoteObjects result = S3Lister.lister(amazonS3Client).apply(request); + //then + assertThat(result.byHash.asMap()).isEqualTo(expectedHashMap); + assertThat(result.byKey.asMap()).isEqualTo(expectedKeyMap); + } + + @Test + @DisplayName("when error listing then return an error storage event") + public void whenListErrors_thenErrorEvent() { + //given + doThrow(SdkClientException.class) + .when(amazonS3Client) + .listObjects(request); + //when + RemoteObjects result = S3Lister.lister(amazonS3Client).apply(request); + //then + assertThat(result.byKey.asMap()).isEmpty(); + assertThat(result.byHash.asMap()).isEmpty(); + } + + private ListObjectsV2Result objectResults( + Date nowDate, + String key, + String etag, + boolean truncated + ) { + ListObjectsV2Result result = new ListObjectsV2Result(); + result.getObjectSummaries().add(objectSummary(key, etag, nowDate)); + result.setNextContinuationToken("next token"); + result.setTruncated(truncated); + return result; + } + + private S3ObjectSummary objectSummary( + String key, + String etag, + Date lastModified + ) { + S3ObjectSummary summary = new S3ObjectSummary(); + summary.setKey(key); + summary.setETag(etag); + summary.setLastModified(lastModified); + return summary; + } + +} diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ObjectsByHashTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ObjectsByHashTest.java new file mode 100644 index 0000000..0b4f2a2 --- /dev/null +++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ObjectsByHashTest.java @@ -0,0 +1,44 @@ +package net.kemitix.thorp.storage.aws; + +import com.amazonaws.services.s3.model.S3ObjectSummary; +import lombok.val; +import net.kemitix.thorp.domain.MD5Hash; +import net.kemitix.thorp.domain.RemoteKey; +import org.assertj.core.api.WithAssertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ObjectsByHashTest + implements WithAssertions { + private S3ObjectSummary s3object(MD5Hash md5Hash, + RemoteKey remoteKey) { + S3ObjectSummary summary = new S3ObjectSummary(); + summary.setETag(md5Hash.hash()); + summary.setKey(remoteKey.key()); + return summary; + } + + @Test + @DisplayName("grouping s3 objects together by their hash value") + public void groupS3ObjectsByHashValue() { + //given + MD5Hash hash = MD5Hash.create("hash"); + RemoteKey key1 = RemoteKey.create("key-1"); + RemoteKey key2 = RemoteKey.create("key-2"); + S3ObjectSummary o1 = s3object(hash, key1); + S3ObjectSummary o2 = s3object(hash, key2); + List os = Arrays.asList(o1, o2); + Map expected = Collections.singletonMap( + hash, key2); + //when + Map result = S3Lister.byHash(os); + //then + assertThat(result).isEqualTo(expected); + } + +} diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/UploaderTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/UploaderTest.java new file mode 100644 index 0000000..6b14937 --- /dev/null +++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/UploaderTest.java @@ -0,0 +1,81 @@ +package net.kemitix.thorp.storage.aws; + +import com.amazonaws.SdkClientException; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.Upload; +import com.amazonaws.services.s3.transfer.model.UploadResult; +import net.kemitix.thorp.domain.HashType; +import net.kemitix.thorp.domain.*; +import net.kemitix.thorp.filesystem.Resource; +import org.assertj.core.api.WithAssertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.File; + +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.doThrow; + +@ExtendWith(MockitoExtension.class) +public class UploaderTest + implements WithAssertions { + + private final File aSource = Resource.select(this, "").toFile(); + private final File aFile = Resource.select(this, "small-file").toFile(); + private final MD5Hash aHash = MD5Hash.create("aHash"); + private final Hashes hashes = Hashes.create(HashType.MD5, aHash); + private final RemoteKey remoteKey = RemoteKey.create("aRemoteKey"); + private final LocalFile localFile = + LocalFile.create(aFile, aSource, hashes, remoteKey, aFile.length()); + private final Bucket bucket = Bucket.named("aBucket"); + private final PutObjectRequest request = + S3Uploader.request(localFile, bucket); + + private final TransferManager transferManager; + + private final S3TransferManager s3TransferManager; + + private final UploadResult uploadResult; + private final Upload upload; + + public UploaderTest(@Mock TransferManager transferManager, + @Mock Upload upload) { + this.transferManager = transferManager; + this.upload = upload; + s3TransferManager = S3TransferManager.create(transferManager); + uploadResult = new UploadResult(); + uploadResult.setKey(remoteKey.key()); + uploadResult.setETag(aHash.hash()); + } + + @Test + @DisplayName("when upload then return upload event") + public void whenUpload_thenUploadEvent() throws InterruptedException { + //given + given(transferManager.upload(request)) + .willReturn(upload); + given(upload.waitForUploadResult()).willReturn(uploadResult); + //when + StorageEvent result = S3Uploader.uploader(s3TransferManager).apply(request); + //then + assertThat(result).isInstanceOf(StorageEvent.UploadEvent.class); + } + + @Test + @DisplayName("when error uploading then return an error storage event") + public void whenUploadErrors_thenErrorEvent() { + //given + doThrow(SdkClientException.class) + .when(transferManager) + .upload(request); + //when + StorageEvent result = S3Uploader.uploader(s3TransferManager).apply(request); + //then + assertThat(result).isInstanceOf(StorageEvent.ErrorEvent.class); + } + +} diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/AmazonS3ClientTestFixture.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/AmazonS3ClientTestFixture.scala deleted file mode 100644 index c96c5cc..0000000 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/AmazonS3ClientTestFixture.scala +++ /dev/null @@ -1,60 +0,0 @@ -package net.kemitix.thorp.storage.aws - -import net.kemitix.thorp.storage.Storage -import org.scalamock.scalatest.MockFactory - -trait AmazonS3ClientTestFixture extends MockFactory { - - @SuppressWarnings(Array("org.wartremover.warts.PublicInference")) - private val manager = stub[S3TransferManager] - @SuppressWarnings(Array("org.wartremover.warts.PublicInference")) - private val client = stub[AmazonS3Client] - val fixture: Fixture = Fixture(client, manager) - - case class Fixture(amazonS3Client: AmazonS3Client, - amazonS3TransferManager: S3TransferManager, - ) { - lazy val storageService: Storage = Storage.getInstance() -// new Storage.Service { -// -// private val client = amazonS3Client -// private val transferManager = amazonS3TransferManager -// -// override def listObjects( -// bucket: Bucket, -// prefix: RemoteKey -// ): RIO[Storage, RemoteObjects] = -// UIO { -// S3Lister.lister(client)(S3Lister.request(bucket, prefix)) -// } -// -// override def upload(localFile: LocalFile, -// bucket: Bucket, -// listenerSettings: UploadEventListener.Settings, -// ): UIO[StorageEvent] = -// UIO( -// S3Uploader -// .uploader(transferManager)(S3Uploader.request(localFile, bucket)) -// ) -// -// override def copy(bucket: Bucket, -// sourceKey: RemoteKey, -// hash: MD5Hash, -// targetKey: RemoteKey): UIO[StorageEvent] = -// UIO { -// val request = S3Copier.request(bucket, sourceKey, hash, targetKey) -// S3Copier.copier(client)(request) -// } -// -// override def delete(bucket: Bucket, -// remoteKey: RemoteKey): UIO[StorageEvent] = -// UIO(S3Deleter.deleter(client)(S3Deleter.request(bucket, remoteKey))) -// -// override def shutdown: UIO[StorageEvent] = { -// UIO(transferManager.shutdownNow(true)) *> UIO(client.shutdown()) -// .map(_ => StorageEvent.shutdownEvent()) -// } -// } - } - -} diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/CopierTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/CopierTest.scala deleted file mode 100644 index 7cc02ac..0000000 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/CopierTest.scala +++ /dev/null @@ -1,88 +0,0 @@ -package net.kemitix.thorp.storage.aws - -import org.scalatest.FreeSpec - -class CopierTest extends FreeSpec { - -// private val runtime = Runtime(Console.Live, PlatformLive.Default) -// -// "copier" - { -// val bucket = Bucket.named("aBucket") -// val sourceKey = RemoteKey.create("sourceKey") -// val hash = MD5Hash.create("aHash") -// val targetKey = RemoteKey.create("targetKey") -// "when source exists" - { -// "when source hash matches" - { -// "copies from source to target" in { -// val event = StorageEvent.copyEvent(sourceKey, targetKey) -// val expected = Right(event) -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3Client.copyObject) -// .when() -// .returns(_ => Task.succeed(Some(new CopyObjectResult))) -// private val result = -// invoke(bucket, sourceKey, hash, targetKey, fixture.amazonS3Client) -// assertResult(expected)(result) -// } -// } -// } -// "when source hash does not match" - { -// "skip the file with an error" in { -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3Client.copyObject) -// .when() -// .returns(_ => Task.succeed(None)) -// private val result = -// invoke(bucket, sourceKey, hash, targetKey, fixture.amazonS3Client) -// result match { -// case right: Right[Throwable, StorageEvent] => { -// val e = right.value.asInstanceOf[ErrorEvent].e -// e match { -// case HashError => assert(true) -// case _ => fail(s"Not a HashError: ${e.getMessage}") -// } -// } -// case e => fail(s"Not an ErrorQueueEvent: $e") -// } -// } -// } -// } -// "when client throws an exception" - { -// "skip the file with an error" in { -// new AmazonS3ClientTestFixture { -// private val expectedMessage = "The specified key does not exist" -// (() => fixture.amazonS3Client.copyObject) -// .when() -// .returns(_ => Task.fail(new AmazonS3Exception(expectedMessage))) -// private val result = -// invoke(bucket, sourceKey, hash, targetKey, fixture.amazonS3Client) -// val key = RemoteKey.create("targetKey") -// result match { -// case right: Right[Throwable, StorageEvent] => { -// val e = right.value.asInstanceOf[ErrorEvent].e -// e match { -// case CopyError(cause) => -// assert(cause.getMessage.startsWith(expectedMessage)) -// case _ => fail(s"Not a CopyError: ${e.getMessage}") -// } -// } -// case e => fail(s"Not an ErrorQueueEvent: ${e}") -// } -// } -// } -// } -// } -// def invoke( -// bucket: Bucket, -// sourceKey: RemoteKey, -// hash: MD5Hash, -// targetKey: RemoteKey, -// amazonS3Client: AmazonS3Client -// ) = -// runtime.unsafeRunSync { -// Copier.copy(amazonS3Client)( -// Copier.Request(bucket, sourceKey, hash, targetKey)) -// }.toEither -// } - -} diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/DeleterTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/DeleterTest.scala deleted file mode 100644 index be8a7b3..0000000 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/DeleterTest.scala +++ /dev/null @@ -1,59 +0,0 @@ -package net.kemitix.thorp.storage.aws - -import org.scalatest.FreeSpec - -class DeleterTest extends FreeSpec { - -// private val runtime = Runtime(Console.Live, PlatformLive.Default) -// -// "delete" - { -// val bucket = Bucket.named("aBucket") -// val remoteKey = RemoteKey.create("aRemoteKey") -// "when no errors" in { -// val expected = Right(StorageEvent.deleteEvent(remoteKey)) -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3Client.deleteObject) -// .when() -// .returns(_ => UIO.succeed(())) -// private val result = invoke(fixture.amazonS3Client)(bucket, remoteKey) -// assertResult(expected)(result) -// } -// } -// "when Amazon Service Exception" in { -// val exception = new AmazonS3Exception("message") -// val expected = -// Right( -// StorageEvent.errorEvent(ActionSummary.delete(remoteKey.key), -// remoteKey, -// exception)) -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3Client.deleteObject) -// .when() -// .returns(_ => Task.fail(exception)) -// private val result = invoke(fixture.amazonS3Client)(bucket, remoteKey) -// assertResult(expected)(result) -// } -// } -// "when Amazon SDK Client Exception" in { -// val exception = new SdkClientException("message") -// val expected = -// Right( -// StorageEvent.errorEvent(ActionSummary.delete(remoteKey.key), -// remoteKey, -// exception)) -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3Client.deleteObject) -// .when() -// .returns(_ => Task.fail(exception)) -// private val result = invoke(fixture.amazonS3Client)(bucket, remoteKey) -// assertResult(expected)(result) -// } -// } -// def invoke(amazonS3Client: AmazonS3Client.Client)(bucket: Bucket, -// remoteKey: RemoteKey) = -// runtime.unsafeRunSync { -// Deleter.delete(amazonS3Client)(bucket, remoteKey) -// }.toEither -// -// } -} diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala deleted file mode 100644 index f279b1e..0000000 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala +++ /dev/null @@ -1,119 +0,0 @@ -package net.kemitix.thorp.storage.aws - -import org.scalatest.FreeSpec - -class ListerTest extends FreeSpec { - -// "list" - { -// val bucket = Bucket.named("aBucket") -// val prefix = RemoteKey.create("aRemoteKey") -// "when no errors" - { -// "when single fetch required" in { -// val nowDate = new Date -// val key = "key" -// val etag = "etag" -// val expectedHashMap = Map(MD5Hash.create(etag) -> RemoteKey.create(key)) -// val expectedKeyMap = Map(RemoteKey.create(key) -> MD5Hash.create(etag)) -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3Client.listObjectsV2) -// .when() -// .returns(_ => { -// UIO.succeed(objectResults(nowDate, key, etag, truncated = false)) -// }) -// private val result = invoke(fixture.amazonS3Client)(bucket, prefix) -// private val hashMap = -// result.map(_.byHash).map(m => Map.from(m.asMap.asScala)) -// private val keyMap = -// result.map(_.byKey).map(m => Map.from(m.asMap.asScala)) -// hashMap should be(Right(expectedHashMap)) -// keyMap should be(Right(expectedKeyMap)) -// } -// } -// -// "when second fetch required" in { -// val nowDate = new Date -// val key1 = "key1" -// val etag1 = "etag1" -// val key2 = "key2" -// val etag2 = "etag2" -// val expectedHashMap = Map( -// MD5Hash.create(etag1) -> RemoteKey.create(key1), -// MD5Hash.create(etag2) -> RemoteKey.create(key2) -// ) -// val expectedKeyMap = Map( -// RemoteKey.create(key1) -> MD5Hash.create(etag1), -// RemoteKey.create(key2) -> MD5Hash.create(etag2) -// ) -// new AmazonS3ClientTestFixture { -// -// (() => fixture.amazonS3Client.listObjectsV2) -// .when() -// .returns(_ => -// UIO(objectResults(nowDate, key1, etag1, truncated = true))) -// .noMoreThanOnce() -// -// (() => fixture.amazonS3Client.listObjectsV2) -// .when() -// .returns(_ => -// UIO(objectResults(nowDate, key2, etag2, truncated = false))) -// private val result = invoke(fixture.amazonS3Client)(bucket, prefix) -// private val hashMap = -// result.map(_.byHash).map(m => Map.from(m.asMap.asScala)) -// private val keyMap = -// result.map(_.byKey).map(m => Map.from(m.asMap.asScala)) -// hashMap should be(Right(expectedHashMap)) -// keyMap should be(Right(expectedKeyMap)) -// } -// } -// -// def objectSummary(key: String, etag: String, lastModified: Date) = { -// val objectSummary = new S3ObjectSummary -// objectSummary.setKey(key) -// objectSummary.setETag(etag) -// objectSummary.setLastModified(lastModified) -// objectSummary -// } -// -// def objectResults(nowDate: Date, -// key: String, -// etag: String, -// truncated: Boolean) = { -// val result = new ListObjectsV2Result -// result.getObjectSummaries.add(objectSummary(key, etag, nowDate)) -// result.setTruncated(truncated) -// result -// } -// -// } -// "when Amazon Service Exception" in { -// val exception = new AmazonS3Exception("message") -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3Client.listObjectsV2) -// .when() -// .returns(_ => Task.fail(exception)) -// private val result = invoke(fixture.amazonS3Client)(bucket, prefix) -// assert(result.isLeft) -// } -// } -// "when Amazon SDK Client Exception" in { -// val exception = new SdkClientException("message") -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3Client.listObjectsV2) -// .when() -// .returns(_ => Task.fail(exception)) -// private val result = invoke(fixture.amazonS3Client)(bucket, prefix) -// assert(result.isLeft) -// } -// } -// def invoke(amazonS3Client: AmazonS3Client.Client)(bucket: Bucket, -// prefix: RemoteKey) = { -// object TestEnv extends Storage.Test -// val program: RIO[Storage, RemoteObjects] = Lister -// .listObjects(amazonS3Client)(bucket, prefix) -// val runtime = new DefaultRuntime {} -// runtime.unsafeRunSync(program.provide(TestEnv)).toEither -// } -// -// } - -} diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/MD5HashData.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/MD5HashData.scala deleted file mode 100644 index ffb6047..0000000 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/MD5HashData.scala +++ /dev/null @@ -1,11 +0,0 @@ -package net.kemitix.thorp.storage.aws - -import net.kemitix.thorp.domain.MD5Hash - -object MD5HashData { - - val rootHash = MD5Hash.create("a3a6ac11a0eb577b81b3bb5c95cc8a6e") - - val leafHash = MD5Hash.create("208386a650bdec61cfcd7bd8dcb6b542") - -} diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala deleted file mode 100644 index 08c2285..0000000 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala +++ /dev/null @@ -1,35 +0,0 @@ -package net.kemitix.thorp.storage.aws - -import scala.jdk.CollectionConverters._ - -import com.amazonaws.services.s3.model.S3ObjectSummary -import net.kemitix.thorp.domain.{MD5Hash, RemoteKey} -import org.scalatest.FunSpec - -class S3ObjectsByHashSuite extends FunSpec { - - describe("grouping s3 object together by their hash values") { - val hash = MD5Hash.create("hash") - val key1 = RemoteKey.create("key-1") - val key2 = RemoteKey.create("key-2") - val o1 = s3object(hash, key1) - val o2 = s3object(hash, key2) - val os = List(o1, o2) - it("should group by the hash value") { - val expected: Map[MD5Hash, RemoteKey] = Map( - hash -> key2 - ) - val result = Map.from(S3Lister.byHash(os.asJava).asScala) - assertResult(expected)(result) - } - } - - private def s3object(md5Hash: MD5Hash, - remoteKey: RemoteKey): S3ObjectSummary = { - val summary = new S3ObjectSummary() - summary.setETag(md5Hash.hash()) - summary.setKey(remoteKey.key) - summary - } - -} diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala deleted file mode 100644 index a6f00a0..0000000 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala +++ /dev/null @@ -1,109 +0,0 @@ -package net.kemitix.thorp.storage.aws - -import org.scalamock.scalatest.MockFactory -import org.scalatest.FreeSpec - -class UploaderTest extends FreeSpec with MockFactory { - -// val uiChannel: UChannel[Any, UIEvent] = zioMessage => () -// -// "upload" - { -// val aSource: File = Resource(this, "").toFile -// val aFile: File = Resource(this, "small-file").toFile -// val aHash = MD5Hash.create("aHash") -// val hashes = Hashes.create(MD5, aHash) -// val remoteKey = RemoteKey.create("aRemoteKey") -// val localFile = -// LocalFile.create(aFile, aSource, hashes, remoteKey, aFile.length) -// val bucket = Bucket.named("aBucket") -// val uploadResult = new UploadResult -// uploadResult.setKey(remoteKey.key) -// uploadResult.setETag(aHash.hash()) -// val listenerSettings = -// UploadEventListener.Settings(uiChannel, localFile, 0, 0, batchMode = true) -// "when no error" in { -// val expected = -// Right(StorageEvent.uploadEvent(remoteKey, aHash)) -// val inProgress = new AmazonUpload.InProgress { -// override def waitForUploadResult: Task[UploadResult] = -// Task(uploadResult) -// } -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3TransferManager.uploader) -// .when() -// .returns(_ => UIO.succeed(inProgress)) -// private val result = -// invoke(fixture.amazonS3TransferManager)( -// localFile, -// bucket, -// listenerSettings -// ) -// assertResult(expected)(result) -// } -// } -// "when Amazon Service Exception" in { -// val exception = new AmazonS3Exception("message") -// val expected = -// Right( -// StorageEvent.errorEvent(ActionSummary.upload(remoteKey.key), -// remoteKey, -// exception)) -// val inProgress = new AmazonUpload.InProgress { -// override def waitForUploadResult: Task[UploadResult] = -// Task.fail(exception) -// } -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3TransferManager.upload) -// .when() -// .returns(_ => UIO.succeed(inProgress)) -// private val result = -// invoke(fixture.amazonS3TransferManager)( -// localFile, -// bucket, -// listenerSettings -// ) -// assertResult(expected)(result) -// } -// } -// "when Amazon SDK Client Exception" in { -// val exception = new SdkClientException("message") -// val expected = -// Right( -// StorageEvent.errorEvent(ActionSummary.upload(remoteKey.key), -// remoteKey, -// exception)) -// val inProgress = new AmazonUpload.InProgress { -// override def waitForUploadResult: Task[UploadResult] = -// Task.fail(exception) -// } -// new AmazonS3ClientTestFixture { -// (() => fixture.amazonS3TransferManager.upload) -// .when() -// .returns(_ => UIO.succeed(inProgress)) -// private val result = -// invoke(fixture.amazonS3TransferManager)( -// localFile, -// bucket, -// listenerSettings -// ) -// assertResult(expected)(result) -// } -// } -// def invoke(transferManager: AmazonTransferManager)( -// localFile: LocalFile, -// bucket: Bucket, -// listenerSettings: UploadEventListener.Settings -// ) = { -// val program = Uploader -// .upload(transferManager)( -// Uploader.Request(localFile, bucket, listenerSettings)) -// val runtime = new DefaultRuntime {} -// runtime -// .unsafeRunSync( -// program -// .provide(Config.Live)) -// .toEither -// } -// } - -} diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala deleted file mode 100644 index 95cbf45..0000000 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala +++ /dev/null @@ -1,56 +0,0 @@ -package net.kemitix.thorp.storage.aws.hasher - -import com.amazonaws.services.s3.transfer.TransferManagerConfiguration -import net.kemitix.thorp.filesystem.Resource -import org.scalatest.FreeSpec - -class ETagGeneratorTest extends FreeSpec { - - private val bigFile = Resource.select(this, "../big-file") - private val bigFilePath = bigFile.toPath - private val configuration = new TransferManagerConfiguration - private val chunkSize = 1200000 - configuration.setMinimumUploadPartSize(chunkSize) - -// "Create offsets" - { -// "should create offsets" in { -// val offsets = S3ETagGenerator -// .offsets(bigFile.length, chunkSize) -// .foldRight(List[Long]())((l: Long, a: List[Long]) => l :: a) -// assertResult( -// List(0, chunkSize, chunkSize * 2, chunkSize * 3, chunkSize * 4))( -// offsets) -// } -// } - -// "create md5 hash for each chunk" - { -// "should create expected hash for chunks" in { -// val md5Hashes = List( -// "68b7d37e6578297621e06f01800204f1", -// "973475b14a7bda6ad8864a7f9913a947", -// "b9adcfc5b103fe2dd5924a5e5e6817f0", -// "5bd6e10a99fef100fe7bf5eaa0a42384", -// "8a0c1d0778ac8fcf4ca2010eba4711eb" -// ).zipWithIndex -// md5Hashes.foreach { -// case (hash, index) => -// val program = Hasher.hashObjectChunk(bigFilePath, index, chunkSize) -// val result = runtime.unsafeRunSync(program.provide(TestEnv)).toEither -// assertResult(Right(hash))( -// result -// .map(hashes => hashes.get(MD5).get()) -// .map(x => x.hash)) -// } -// } -// } - -// "create etag for whole file" - { -// val expected = "f14327c90ad105244c446c498bfe9a7d-2" -// "should match aws etag for the file" in { -// val program = ETagGenerator.eTag(bigFilePath) -// val result = runtime.unsafeRunSync(program.provide(TestEnv)).toEither -// assertResult(Right(expected))(result) -// } -// } - -}