diff --git a/docs/images/reactor-graph.png b/docs/images/reactor-graph.png
index b0e2940..d9e9193 100644
Binary files a/docs/images/reactor-graph.png and b/docs/images/reactor-graph.png differ
diff --git a/domain/src/main/java/net/kemitix/thorp/domain/StorageEvent.java b/domain/src/main/java/net/kemitix/thorp/domain/StorageEvent.java
index 88620c5..a4c4235 100644
--- a/domain/src/main/java/net/kemitix/thorp/domain/StorageEvent.java
+++ b/domain/src/main/java/net/kemitix/thorp/domain/StorageEvent.java
@@ -1,6 +1,7 @@
package net.kemitix.thorp.domain;
import lombok.AccessLevel;
+import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
@@ -27,6 +28,7 @@ public class StorageEvent {
public static class DoNothingEvent extends StorageEvent {
public final RemoteKey remoteKey;
}
+ @EqualsAndHashCode(callSuper = false)
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public static class CopyEvent extends StorageEvent {
public final RemoteKey sourceKey;
@@ -37,6 +39,7 @@ public class StorageEvent {
public final RemoteKey remoteKey;
public final MD5Hash md5Hash;
}
+ @EqualsAndHashCode(callSuper = false)
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public static class DeleteEvent extends StorageEvent {
public final RemoteKey remoteKey;
diff --git a/filesystem/src/main/java/net/kemitix/thorp/filesystem/MD5HashGenerator.java b/filesystem/src/main/java/net/kemitix/thorp/filesystem/MD5HashGenerator.java
index b67e1e2..6c8c150 100644
--- a/filesystem/src/main/java/net/kemitix/thorp/filesystem/MD5HashGenerator.java
+++ b/filesystem/src/main/java/net/kemitix/thorp/filesystem/MD5HashGenerator.java
@@ -13,7 +13,7 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class MD5HashGenerator implements HashGenerator {
- private static final int maxBufferSize = 8048;
+ private static final int maxBufferSize = 8192;
private static final byte[] defaultBuffer = new byte[maxBufferSize];
public static String hex(byte[] in) throws NoSuchAlgorithmException {
MessageDigest md5 = MessageDigest.getInstance("MD5");
diff --git a/lib/src/main/scala/net/kemitix/thorp/lib/UnversionedMirrorArchive.scala b/lib/src/main/scala/net/kemitix/thorp/lib/UnversionedMirrorArchive.scala
index 9edab94..121511b 100644
--- a/lib/src/main/scala/net/kemitix/thorp/lib/UnversionedMirrorArchive.scala
+++ b/lib/src/main/scala/net/kemitix/thorp/lib/UnversionedMirrorArchive.scala
@@ -2,6 +2,7 @@ package net.kemitix.thorp.lib
import net.kemitix.thorp.config.Configuration
import net.kemitix.thorp.domain.Action.{ToCopy, ToDelete, ToUpload}
+import net.kemitix.thorp.domain.StorageEvent.ActionSummary
import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.Storage
import net.kemitix.thorp.uishell.{UIEvent, UploadEventListener}
@@ -35,7 +36,16 @@ trait UnversionedMirrorArchive extends ThorpArchive {
.copy(bucket, sourceKey, hash, targetKey)
case toDelete: ToDelete =>
val remoteKey = toDelete.remoteKey
- Storage.getInstance().delete(bucket, remoteKey)
+ try {
+ Storage.getInstance().delete(bucket, remoteKey)
+ } catch {
+ case e: Throwable =>
+ StorageEvent.errorEvent(
+ ActionSummary.delete(remoteKey.key()),
+ remoteKey,
+ e
+ );
+ }
case doNothing: Action.DoNothing =>
val remoteKey = doNothing.remoteKey
StorageEvent.doNothingEvent(remoteKey)
diff --git a/storage-aws/pom.xml b/storage-aws/pom.xml
index 49e1b3c..50c2db4 100644
--- a/storage-aws/pom.xml
+++ b/storage-aws/pom.xml
@@ -48,11 +48,10 @@
assertj-core
test
-
-
- org.scala-lang
- scala-library
+ org.mockito
+ mockito-junit-jupiter
+ test
@@ -95,27 +94,5 @@
commons-logging
1.2
-
-
-
- org.scalatest
- scalatest_2.13
- test
-
-
- org.scalamock
- scalamock_2.13
- test
-
-
-
-
-
- net.alchim31.maven
- scala-maven-plugin
-
-
-
-
diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Copier.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Copier.java
index bcf1b6c..b0b1018 100644
--- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Copier.java
+++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Copier.java
@@ -1,5 +1,6 @@
package net.kemitix.thorp.storage.aws;
+import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import net.kemitix.thorp.domain.Bucket;
import net.kemitix.thorp.domain.MD5Hash;
@@ -24,16 +25,19 @@ public interface S3Copier {
return request -> {
RemoteKey sourceKey = RemoteKey.create(request.getSourceKey());
RemoteKey targetKey = RemoteKey.create(request.getDestinationKey());
- return client.copyObject(request)
- .map(success -> StorageEvent.copyEvent(sourceKey, targetKey))
- .orElseGet(() -> errorEvent(sourceKey, targetKey));
+ try {
+ return client.copyObject(request)
+ .map(success -> StorageEvent.copyEvent(sourceKey, targetKey))
+ .orElseGet(() -> StorageEvent.errorEvent(
+ actionSummary(sourceKey, targetKey),
+ targetKey, S3Exception.hashError()));
+ } catch (SdkClientException e) {
+ return StorageEvent.errorEvent(
+ actionSummary(sourceKey, targetKey), targetKey, e);
+ }
};
}
- static StorageEvent.ErrorEvent errorEvent(RemoteKey sourceKey, RemoteKey targetKey) {
- return StorageEvent.errorEvent(actionSummary(sourceKey, targetKey), targetKey, S3Exception.hashError());
- }
-
static StorageEvent.ActionSummary.Copy actionSummary(RemoteKey sourceKey, RemoteKey targetKey) {
return StorageEvent.ActionSummary.copy(
String.format("%s => %s", sourceKey.key(), targetKey.key()));
diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Deleter.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Deleter.java
index e6ff768..b011584 100644
--- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Deleter.java
+++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Deleter.java
@@ -1,5 +1,6 @@
package net.kemitix.thorp.storage.aws;
+import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import net.kemitix.thorp.domain.Bucket;
import net.kemitix.thorp.domain.RemoteKey;
@@ -13,9 +14,15 @@ public interface S3Deleter {
}
static Function deleter(AmazonS3Client client) {
return request -> {
- client.deleteObject(request);
RemoteKey remoteKey = RemoteKey.create(request.getKey());
- return StorageEvent.deleteEvent(remoteKey);
+ try {
+ client.deleteObject(request);
+ return StorageEvent.deleteEvent(remoteKey);
+ } catch (SdkClientException e) {
+ return StorageEvent.errorEvent(
+ StorageEvent.ActionSummary.delete(remoteKey.key()),
+ remoteKey, e);
+ }
};
}
}
diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3ETagGenerator.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3ETagGenerator.java
index 68eff55..9debcfa 100644
--- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3ETagGenerator.java
+++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3ETagGenerator.java
@@ -7,15 +7,18 @@ import net.kemitix.thorp.domain.HashGenerator;
import net.kemitix.thorp.domain.HashType;
import net.kemitix.thorp.domain.Hashes;
import net.kemitix.thorp.domain.MD5Hash;
+import net.kemitix.thorp.filesystem.MD5HashGenerator;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.file.Path;
-import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
+import static net.kemitix.thorp.filesystem.MD5HashGenerator.md5FileChunk;
+
public class S3ETagGenerator implements HashGenerator {
@Deprecated // Use hashFile
public String eTag(Path path) throws IOException, NoSuchAlgorithmException {
@@ -43,7 +46,7 @@ public class S3ETagGenerator implements HashGenerator {
public List offsets(long totalFileSizeBytes, long optimalPartSize) {
return LongStream
- .range(0, totalFileSizeBytes / optimalPartSize)
+ .rangeClosed(0, totalFileSizeBytes / optimalPartSize)
.mapToObj(part -> part * optimalPartSize)
.collect(Collectors.toList());
}
@@ -63,12 +66,11 @@ public class S3ETagGenerator implements HashGenerator {
}
private String eTagHex(Path path, long partSize, long parts) throws IOException, NoSuchAlgorithmException {
- HashGenerator hashGenerator = HashGenerator.generatorFor("MD5");
- MessageDigest md5 = MessageDigest.getInstance("MD5");
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
for (long i = 0; i < parts ; i++ ){
- md5.update(hashGenerator.hashChunk(path, i, partSize).digest());
+ bos.write(md5FileChunk(path, i * partSize, partSize).digest());
}
- return MD5Hash.digestAsString(md5.digest());
+ return MD5HashGenerator.hex(bos.toByteArray());
}
@Override
public HashType hashType() {
diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Lister.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Lister.java
index 3a7b566..4bd8034 100644
--- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Lister.java
+++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Lister.java
@@ -1,5 +1,7 @@
package net.kemitix.thorp.storage.aws;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.S3ObjectSummary;
@@ -71,10 +73,14 @@ public interface S3Lister {
AmazonS3Client client,
ListObjectsV2Request request
) {
- Batch batch = fetchBatch(client, request);
- List more = fetchMore(client, request, batch.more);
- batch.summaries.addAll(more);
- return batch.summaries;
+ try {
+ Batch batch = fetchBatch(client, request);
+ List more = fetchMore(client, request, batch.more);
+ batch.summaries.addAll(more);
+ return batch.summaries;
+ } catch (SdkClientException e) {
+ return Collections.emptyList();
+ }
};
static Optional moreToken(ListObjectsV2Result result) {
diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3TransferManager.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3TransferManager.java
index c66b9fd..760f091 100644
--- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3TransferManager.java
+++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3TransferManager.java
@@ -2,7 +2,6 @@ package net.kemitix.thorp.storage.aws;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.Upload;
import java.util.function.Function;
@@ -17,14 +16,9 @@ public interface S3TransferManager {
}
@Override
public Function uploader() {
- return request -> {
- Upload upload = transferManager.upload(request);
- try {
- return S3Upload.inProgress(upload);
- } catch (S3Exception.UploadError error) {
- return S3Upload.errored(error);
- }
- };
+ return request ->
+ S3Upload.inProgress(
+ transferManager.upload(request));
}
};
}
diff --git a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Uploader.java b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Uploader.java
index 377308e..48c4cc5 100644
--- a/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Uploader.java
+++ b/storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Uploader.java
@@ -1,5 +1,6 @@
package net.kemitix.thorp.storage.aws;
+import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.model.UploadResult;
@@ -29,13 +30,20 @@ public interface S3Uploader {
S3TransferManager transferManager
) {
return request -> {
- UploadResult uploadResult =
- transferManager.uploader()
- .apply(request)
- .waitForUploadResult();
- return StorageEvent.uploadEvent(
- RemoteKey.create(uploadResult.getKey()),
- MD5Hash.create(uploadResult.getETag()));
+ try {
+ UploadResult uploadResult =
+ transferManager.uploader()
+ .apply(request)
+ .waitForUploadResult();
+ return StorageEvent.uploadEvent(
+ RemoteKey.create(uploadResult.getKey()),
+ MD5Hash.create(uploadResult.getETag()));
+ } catch (SdkClientException e) {
+ String key = request.getKey();
+ return StorageEvent.errorEvent(
+ StorageEvent.ActionSummary.upload(key),
+ RemoteKey.create(key), e);
+ }
};
}
}
diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/CopierTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/CopierTest.java
new file mode 100644
index 0000000..952d0e5
--- /dev/null
+++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/CopierTest.java
@@ -0,0 +1,65 @@
+package net.kemitix.thorp.storage.aws;
+
+import com.amazonaws.SdkClientException;
+import com.amazonaws.services.s3.model.CopyObjectRequest;
+import com.amazonaws.services.s3.model.CopyObjectResult;
+import net.kemitix.thorp.domain.Bucket;
+import net.kemitix.thorp.domain.MD5Hash;
+import net.kemitix.thorp.domain.RemoteKey;
+import net.kemitix.thorp.domain.StorageEvent;
+import org.assertj.core.api.WithAssertions;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.util.Optional;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.BDDMockito.given;
+import static org.mockito.Mockito.doThrow;
+
+@ExtendWith(MockitoExtension.class)
+public class CopierTest
+ implements WithAssertions {
+
+ private final Bucket bucket = Bucket.named("aBucket");
+ private final RemoteKey sourceKey = RemoteKey.create("sourceKey");
+ private final MD5Hash hash = MD5Hash.create("aHash");
+ private final RemoteKey targetKey = RemoteKey.create("targetKey");
+ private final CopyObjectRequest request = S3Copier.request(bucket, sourceKey, hash, targetKey);
+
+ private final AmazonS3Client amazonS3Client;
+
+ public CopierTest(@Mock AmazonS3Client amazonS3Client) {
+ this.amazonS3Client = amazonS3Client;
+ }
+
+ @Test
+ @DisplayName("copy produces copy event")
+ public void copy_thenCopyEvent() {
+ //given
+ StorageEvent event = StorageEvent.copyEvent(sourceKey, targetKey);
+ given(amazonS3Client.copyObject(request))
+ .willReturn(Optional.of(new CopyObjectResult()));
+ //when
+ StorageEvent result = S3Copier.copier(amazonS3Client).apply(request);
+ //then
+ assertThat(result).isEqualTo(event);
+ }
+
+ @Test
+ @DisplayName("when error copying then return an error storage event")
+ public void whenCopyErrors_thenErrorEvent() {
+ //given
+ doThrow(SdkClientException.class)
+ .when(amazonS3Client)
+ .copyObject(request);
+ //when
+ StorageEvent result = S3Copier.copier(amazonS3Client).apply(request);
+ //then
+ assertThat(result).isInstanceOf(StorageEvent.ErrorEvent.class);
+ }
+
+}
diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/DeleterTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/DeleterTest.java
new file mode 100644
index 0000000..aa22d99
--- /dev/null
+++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/DeleterTest.java
@@ -0,0 +1,52 @@
+package net.kemitix.thorp.storage.aws;
+
+import com.amazonaws.SdkClientException;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
+import net.kemitix.thorp.domain.Bucket;
+import net.kemitix.thorp.domain.RemoteKey;
+import net.kemitix.thorp.domain.StorageEvent;
+import org.assertj.core.api.WithAssertions;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import static org.mockito.Mockito.doThrow;
+
+@ExtendWith(MockitoExtension.class)
+public class DeleterTest
+ implements WithAssertions {
+
+ private final Bucket bucket = Bucket.named("aBucket");
+ private final RemoteKey remoteKey = RemoteKey.create("aRemoteKey");
+ private final DeleteObjectRequest request = S3Deleter.request(bucket, remoteKey);
+
+ private final AmazonS3Client amazonS3Client;
+
+ public DeleterTest(@Mock AmazonS3Client amazonS3Client) {
+ this.amazonS3Client = amazonS3Client;
+ }
+
+ @Test
+ @DisplayName("when delete then return delete storage event")
+ public void whenDelete_thenDeleteEvent() {
+ //when
+ StorageEvent result = S3Deleter.deleter(amazonS3Client).apply(request);
+ //then
+ assertThat(result).isEqualTo(StorageEvent.deleteEvent(remoteKey));
+ }
+
+ @Test
+ @DisplayName("when error deleting then return an error storage event")
+ public void whenDeleteErrors_thenErrorEvent() {
+ //given
+ doThrow(SdkClientException.class)
+ .when(amazonS3Client)
+ .deleteObject(request);
+ //when
+ StorageEvent result = S3Deleter.deleter(amazonS3Client).apply(request);
+ //then
+ assertThat(result).isInstanceOf(StorageEvent.ErrorEvent.class);
+ }
+}
diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ETagGeneratorTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ETagGeneratorTest.java
new file mode 100644
index 0000000..eb87722
--- /dev/null
+++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ETagGeneratorTest.java
@@ -0,0 +1,54 @@
+package net.kemitix.thorp.storage.aws;
+
+import net.kemitix.thorp.domain.MD5Hash;
+import net.kemitix.thorp.domain.Tuple;
+import net.kemitix.thorp.filesystem.Resource;
+import org.assertj.core.api.WithAssertions;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.List;
+
+public class ETagGeneratorTest
+ implements WithAssertions {
+
+ public static final String BIG_FILE_ETAG = "f14327c90ad105244c446c498bfe9a7d-2";
+ private final Resource bigFile = Resource.select(this, "big-file");
+ private final long chunkSize = 1200000;
+ private final S3ETagGenerator generator = new S3ETagGenerator();
+
+ @Test
+ @DisplayName("creates offsets")
+ public void createsOffsets() {
+ List offsets = generator.offsets(bigFile.length(), chunkSize);
+ assertThat(offsets).containsExactly(
+ 0L, chunkSize, chunkSize * 2, chunkSize * 3, chunkSize * 4);
+ }
+
+ @Test
+ @DisplayName("generate valid hashes")
+ public void generatesValidHashes() throws IOException, NoSuchAlgorithmException {
+ List> md5Hashes = Arrays.asList(
+ Tuple.create(0, "68b7d37e6578297621e06f01800204f1"),
+ Tuple.create(1, "973475b14a7bda6ad8864a7f9913a947"),
+ Tuple.create(2, "b9adcfc5b103fe2dd5924a5e5e6817f0"),
+ Tuple.create(3, "5bd6e10a99fef100fe7bf5eaa0a42384"),
+ Tuple.create(4, "8a0c1d0778ac8fcf4ca2010eba4711eb"));
+ for (Tuple t : md5Hashes) {
+ long offset = t.a * chunkSize;
+ MD5Hash md5Hash =
+ generator.hashChunk(bigFile.toPath(), offset, chunkSize);
+ assertThat(md5Hash.getValue()).isEqualTo(t.b);
+ }
+ }
+
+ @Test
+ @DisplayName("create eTag for whole file")
+ public void createTagForWholeFile() throws IOException, NoSuchAlgorithmException {
+ String result = generator.hashFile(bigFile.toPath());
+ assertThat(result).isEqualTo(BIG_FILE_ETAG);
+ }
+}
diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/HashGeneratorTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/HashGeneratorTest.java
index 7ea82d3..c53d7f6 100644
--- a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/HashGeneratorTest.java
+++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/HashGeneratorTest.java
@@ -42,7 +42,8 @@ public class HashGeneratorTest
//when
Hashes result = HashGenerator.hashObject(path);
//then
- assertThat(result.get(HashType.MD5)).contains(MD5HashData.rootHash());
+ assertThat(result.get(HashType.MD5))
+ .contains(MD5Hash.create("a3a6ac11a0eb577b81b3bb5c95cc8a6e"));
}
@Test
@DisplayName("leaf-file")
@@ -52,7 +53,8 @@ public class HashGeneratorTest
//when
Hashes result = HashGenerator.hashObject(path);
//then
- assertThat(result.get(HashType.MD5)).contains(MD5HashData.leafHash());
+ assertThat(result.get(HashType.MD5))
+ .contains(MD5Hash.create("208386a650bdec61cfcd7bd8dcb6b542"));
}
private Path getResource(String s) {
diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ListerTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ListerTest.java
new file mode 100644
index 0000000..cb2592a
--- /dev/null
+++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ListerTest.java
@@ -0,0 +1,136 @@
+package net.kemitix.thorp.storage.aws;
+
+import com.amazonaws.SdkClientException;
+import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import com.amazonaws.services.s3.model.ListObjectsV2Result;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import net.kemitix.thorp.domain.*;
+import org.assertj.core.api.WithAssertions;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.BDDMockito.given;
+import static org.mockito.Mockito.doThrow;
+
+@ExtendWith(MockitoExtension.class)
+public class ListerTest
+ implements WithAssertions {
+
+ private final Bucket bucket = Bucket.named("aBucket");
+ private final RemoteKey prefix = RemoteKey.create("aRemoteKey");
+ private final ListObjectsV2Request request = S3Lister.request(bucket, prefix);
+
+ private final AmazonS3Client amazonS3Client;
+
+ public ListerTest(@Mock AmazonS3Client amazonS3Client) {
+ this.amazonS3Client = amazonS3Client;
+ }
+
+ @Test
+ @DisplayName("single fetch")
+ public void singleFetch() {
+ //given
+ Date nowDate = new Date();
+ String key = "key";
+ String etag = "etag";
+ Map expectedHashMap =
+ Collections.singletonMap(
+ MD5Hash.create(etag),
+ RemoteKey.create(key)
+ );
+ Map expectedKeyMap =
+ Collections.singletonMap(
+ RemoteKey.create(key),
+ MD5Hash.create(etag)
+ );
+ given(amazonS3Client.listObjects(any()))
+ .willReturn(objectResults(nowDate, key, etag, false));
+ //when
+ RemoteObjects result = S3Lister.lister(amazonS3Client).apply(request);
+ //then
+ assertThat(result.byHash.asMap()).isEqualTo(expectedHashMap);
+ assertThat(result.byKey.asMap()).isEqualTo(expectedKeyMap);
+ }
+
+ @Test
+ @DisplayName("two fetches")
+ public void twoFetches() {
+ //given
+ Date nowDate = new Date();
+ String key1 = "key1";
+ String etag1 = "etag1";
+ String key2 = "key2";
+ String etag2 = "etag2";
+ Map expectedHashMap = new HashMap<>();
+ expectedHashMap.put(
+ MD5Hash.create(etag1),
+ RemoteKey.create(key1));
+ expectedHashMap.put(
+ MD5Hash.create(etag2),
+ RemoteKey.create(key2));
+ Map expectedKeyMap = new HashMap<>();
+ expectedKeyMap.put(
+ RemoteKey.create(key1),
+ MD5Hash.create(etag1));
+ expectedKeyMap.put(
+ RemoteKey.create(key2),
+ MD5Hash.create(etag2));
+ given(amazonS3Client.listObjects(any()))
+ .willReturn(objectResults(nowDate, key1, etag1, true))
+ .willReturn(objectResults(nowDate, key2, etag2, false));
+ //when
+ RemoteObjects result = S3Lister.lister(amazonS3Client).apply(request);
+ //then
+ assertThat(result.byHash.asMap()).isEqualTo(expectedHashMap);
+ assertThat(result.byKey.asMap()).isEqualTo(expectedKeyMap);
+ }
+
+ @Test
+ @DisplayName("when error listing then return an error storage event")
+ public void whenListErrors_thenErrorEvent() {
+ //given
+ doThrow(SdkClientException.class)
+ .when(amazonS3Client)
+ .listObjects(request);
+ //when
+ RemoteObjects result = S3Lister.lister(amazonS3Client).apply(request);
+ //then
+ assertThat(result.byKey.asMap()).isEmpty();
+ assertThat(result.byHash.asMap()).isEmpty();
+ }
+
+ private ListObjectsV2Result objectResults(
+ Date nowDate,
+ String key,
+ String etag,
+ boolean truncated
+ ) {
+ ListObjectsV2Result result = new ListObjectsV2Result();
+ result.getObjectSummaries().add(objectSummary(key, etag, nowDate));
+ result.setNextContinuationToken("next token");
+ result.setTruncated(truncated);
+ return result;
+ }
+
+ private S3ObjectSummary objectSummary(
+ String key,
+ String etag,
+ Date lastModified
+ ) {
+ S3ObjectSummary summary = new S3ObjectSummary();
+ summary.setKey(key);
+ summary.setETag(etag);
+ summary.setLastModified(lastModified);
+ return summary;
+ }
+
+}
diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ObjectsByHashTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ObjectsByHashTest.java
new file mode 100644
index 0000000..0b4f2a2
--- /dev/null
+++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ObjectsByHashTest.java
@@ -0,0 +1,44 @@
+package net.kemitix.thorp.storage.aws;
+
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import lombok.val;
+import net.kemitix.thorp.domain.MD5Hash;
+import net.kemitix.thorp.domain.RemoteKey;
+import org.assertj.core.api.WithAssertions;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class ObjectsByHashTest
+ implements WithAssertions {
+ private S3ObjectSummary s3object(MD5Hash md5Hash,
+ RemoteKey remoteKey) {
+ S3ObjectSummary summary = new S3ObjectSummary();
+ summary.setETag(md5Hash.hash());
+ summary.setKey(remoteKey.key());
+ return summary;
+ }
+
+ @Test
+ @DisplayName("grouping s3 objects together by their hash value")
+ public void groupS3ObjectsByHashValue() {
+ //given
+ MD5Hash hash = MD5Hash.create("hash");
+ RemoteKey key1 = RemoteKey.create("key-1");
+ RemoteKey key2 = RemoteKey.create("key-2");
+ S3ObjectSummary o1 = s3object(hash, key1);
+ S3ObjectSummary o2 = s3object(hash, key2);
+ List os = Arrays.asList(o1, o2);
+ Map expected = Collections.singletonMap(
+ hash, key2);
+ //when
+ Map result = S3Lister.byHash(os);
+ //then
+ assertThat(result).isEqualTo(expected);
+ }
+
+}
diff --git a/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/UploaderTest.java b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/UploaderTest.java
new file mode 100644
index 0000000..6b14937
--- /dev/null
+++ b/storage-aws/src/test/java/net/kemitix/thorp/storage/aws/UploaderTest.java
@@ -0,0 +1,81 @@
+package net.kemitix.thorp.storage.aws;
+
+import com.amazonaws.SdkClientException;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.Upload;
+import com.amazonaws.services.s3.transfer.model.UploadResult;
+import net.kemitix.thorp.domain.HashType;
+import net.kemitix.thorp.domain.*;
+import net.kemitix.thorp.filesystem.Resource;
+import org.assertj.core.api.WithAssertions;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.File;
+
+import static org.mockito.BDDMockito.given;
+import static org.mockito.Mockito.doThrow;
+
+@ExtendWith(MockitoExtension.class)
+public class UploaderTest
+ implements WithAssertions {
+
+ private final File aSource = Resource.select(this, "").toFile();
+ private final File aFile = Resource.select(this, "small-file").toFile();
+ private final MD5Hash aHash = MD5Hash.create("aHash");
+ private final Hashes hashes = Hashes.create(HashType.MD5, aHash);
+ private final RemoteKey remoteKey = RemoteKey.create("aRemoteKey");
+ private final LocalFile localFile =
+ LocalFile.create(aFile, aSource, hashes, remoteKey, aFile.length());
+ private final Bucket bucket = Bucket.named("aBucket");
+ private final PutObjectRequest request =
+ S3Uploader.request(localFile, bucket);
+
+ private final TransferManager transferManager;
+
+ private final S3TransferManager s3TransferManager;
+
+ private final UploadResult uploadResult;
+ private final Upload upload;
+
+ public UploaderTest(@Mock TransferManager transferManager,
+ @Mock Upload upload) {
+ this.transferManager = transferManager;
+ this.upload = upload;
+ s3TransferManager = S3TransferManager.create(transferManager);
+ uploadResult = new UploadResult();
+ uploadResult.setKey(remoteKey.key());
+ uploadResult.setETag(aHash.hash());
+ }
+
+ @Test
+ @DisplayName("when upload then return upload event")
+ public void whenUpload_thenUploadEvent() throws InterruptedException {
+ //given
+ given(transferManager.upload(request))
+ .willReturn(upload);
+ given(upload.waitForUploadResult()).willReturn(uploadResult);
+ //when
+ StorageEvent result = S3Uploader.uploader(s3TransferManager).apply(request);
+ //then
+ assertThat(result).isInstanceOf(StorageEvent.UploadEvent.class);
+ }
+
+ @Test
+ @DisplayName("when error uploading then return an error storage event")
+ public void whenUploadErrors_thenErrorEvent() {
+ //given
+ doThrow(SdkClientException.class)
+ .when(transferManager)
+ .upload(request);
+ //when
+ StorageEvent result = S3Uploader.uploader(s3TransferManager).apply(request);
+ //then
+ assertThat(result).isInstanceOf(StorageEvent.ErrorEvent.class);
+ }
+
+}
diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/AmazonS3ClientTestFixture.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/AmazonS3ClientTestFixture.scala
deleted file mode 100644
index c96c5cc..0000000
--- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/AmazonS3ClientTestFixture.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-package net.kemitix.thorp.storage.aws
-
-import net.kemitix.thorp.storage.Storage
-import org.scalamock.scalatest.MockFactory
-
-trait AmazonS3ClientTestFixture extends MockFactory {
-
- @SuppressWarnings(Array("org.wartremover.warts.PublicInference"))
- private val manager = stub[S3TransferManager]
- @SuppressWarnings(Array("org.wartremover.warts.PublicInference"))
- private val client = stub[AmazonS3Client]
- val fixture: Fixture = Fixture(client, manager)
-
- case class Fixture(amazonS3Client: AmazonS3Client,
- amazonS3TransferManager: S3TransferManager,
- ) {
- lazy val storageService: Storage = Storage.getInstance()
-// new Storage.Service {
-//
-// private val client = amazonS3Client
-// private val transferManager = amazonS3TransferManager
-//
-// override def listObjects(
-// bucket: Bucket,
-// prefix: RemoteKey
-// ): RIO[Storage, RemoteObjects] =
-// UIO {
-// S3Lister.lister(client)(S3Lister.request(bucket, prefix))
-// }
-//
-// override def upload(localFile: LocalFile,
-// bucket: Bucket,
-// listenerSettings: UploadEventListener.Settings,
-// ): UIO[StorageEvent] =
-// UIO(
-// S3Uploader
-// .uploader(transferManager)(S3Uploader.request(localFile, bucket))
-// )
-//
-// override def copy(bucket: Bucket,
-// sourceKey: RemoteKey,
-// hash: MD5Hash,
-// targetKey: RemoteKey): UIO[StorageEvent] =
-// UIO {
-// val request = S3Copier.request(bucket, sourceKey, hash, targetKey)
-// S3Copier.copier(client)(request)
-// }
-//
-// override def delete(bucket: Bucket,
-// remoteKey: RemoteKey): UIO[StorageEvent] =
-// UIO(S3Deleter.deleter(client)(S3Deleter.request(bucket, remoteKey)))
-//
-// override def shutdown: UIO[StorageEvent] = {
-// UIO(transferManager.shutdownNow(true)) *> UIO(client.shutdown())
-// .map(_ => StorageEvent.shutdownEvent())
-// }
-// }
- }
-
-}
diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/CopierTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/CopierTest.scala
deleted file mode 100644
index 7cc02ac..0000000
--- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/CopierTest.scala
+++ /dev/null
@@ -1,88 +0,0 @@
-package net.kemitix.thorp.storage.aws
-
-import org.scalatest.FreeSpec
-
-class CopierTest extends FreeSpec {
-
-// private val runtime = Runtime(Console.Live, PlatformLive.Default)
-//
-// "copier" - {
-// val bucket = Bucket.named("aBucket")
-// val sourceKey = RemoteKey.create("sourceKey")
-// val hash = MD5Hash.create("aHash")
-// val targetKey = RemoteKey.create("targetKey")
-// "when source exists" - {
-// "when source hash matches" - {
-// "copies from source to target" in {
-// val event = StorageEvent.copyEvent(sourceKey, targetKey)
-// val expected = Right(event)
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3Client.copyObject)
-// .when()
-// .returns(_ => Task.succeed(Some(new CopyObjectResult)))
-// private val result =
-// invoke(bucket, sourceKey, hash, targetKey, fixture.amazonS3Client)
-// assertResult(expected)(result)
-// }
-// }
-// }
-// "when source hash does not match" - {
-// "skip the file with an error" in {
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3Client.copyObject)
-// .when()
-// .returns(_ => Task.succeed(None))
-// private val result =
-// invoke(bucket, sourceKey, hash, targetKey, fixture.amazonS3Client)
-// result match {
-// case right: Right[Throwable, StorageEvent] => {
-// val e = right.value.asInstanceOf[ErrorEvent].e
-// e match {
-// case HashError => assert(true)
-// case _ => fail(s"Not a HashError: ${e.getMessage}")
-// }
-// }
-// case e => fail(s"Not an ErrorQueueEvent: $e")
-// }
-// }
-// }
-// }
-// "when client throws an exception" - {
-// "skip the file with an error" in {
-// new AmazonS3ClientTestFixture {
-// private val expectedMessage = "The specified key does not exist"
-// (() => fixture.amazonS3Client.copyObject)
-// .when()
-// .returns(_ => Task.fail(new AmazonS3Exception(expectedMessage)))
-// private val result =
-// invoke(bucket, sourceKey, hash, targetKey, fixture.amazonS3Client)
-// val key = RemoteKey.create("targetKey")
-// result match {
-// case right: Right[Throwable, StorageEvent] => {
-// val e = right.value.asInstanceOf[ErrorEvent].e
-// e match {
-// case CopyError(cause) =>
-// assert(cause.getMessage.startsWith(expectedMessage))
-// case _ => fail(s"Not a CopyError: ${e.getMessage}")
-// }
-// }
-// case e => fail(s"Not an ErrorQueueEvent: ${e}")
-// }
-// }
-// }
-// }
-// }
-// def invoke(
-// bucket: Bucket,
-// sourceKey: RemoteKey,
-// hash: MD5Hash,
-// targetKey: RemoteKey,
-// amazonS3Client: AmazonS3Client
-// ) =
-// runtime.unsafeRunSync {
-// Copier.copy(amazonS3Client)(
-// Copier.Request(bucket, sourceKey, hash, targetKey))
-// }.toEither
-// }
-
-}
diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/DeleterTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/DeleterTest.scala
deleted file mode 100644
index be8a7b3..0000000
--- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/DeleterTest.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-package net.kemitix.thorp.storage.aws
-
-import org.scalatest.FreeSpec
-
-class DeleterTest extends FreeSpec {
-
-// private val runtime = Runtime(Console.Live, PlatformLive.Default)
-//
-// "delete" - {
-// val bucket = Bucket.named("aBucket")
-// val remoteKey = RemoteKey.create("aRemoteKey")
-// "when no errors" in {
-// val expected = Right(StorageEvent.deleteEvent(remoteKey))
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3Client.deleteObject)
-// .when()
-// .returns(_ => UIO.succeed(()))
-// private val result = invoke(fixture.amazonS3Client)(bucket, remoteKey)
-// assertResult(expected)(result)
-// }
-// }
-// "when Amazon Service Exception" in {
-// val exception = new AmazonS3Exception("message")
-// val expected =
-// Right(
-// StorageEvent.errorEvent(ActionSummary.delete(remoteKey.key),
-// remoteKey,
-// exception))
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3Client.deleteObject)
-// .when()
-// .returns(_ => Task.fail(exception))
-// private val result = invoke(fixture.amazonS3Client)(bucket, remoteKey)
-// assertResult(expected)(result)
-// }
-// }
-// "when Amazon SDK Client Exception" in {
-// val exception = new SdkClientException("message")
-// val expected =
-// Right(
-// StorageEvent.errorEvent(ActionSummary.delete(remoteKey.key),
-// remoteKey,
-// exception))
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3Client.deleteObject)
-// .when()
-// .returns(_ => Task.fail(exception))
-// private val result = invoke(fixture.amazonS3Client)(bucket, remoteKey)
-// assertResult(expected)(result)
-// }
-// }
-// def invoke(amazonS3Client: AmazonS3Client.Client)(bucket: Bucket,
-// remoteKey: RemoteKey) =
-// runtime.unsafeRunSync {
-// Deleter.delete(amazonS3Client)(bucket, remoteKey)
-// }.toEither
-//
-// }
-}
diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala
deleted file mode 100644
index f279b1e..0000000
--- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala
+++ /dev/null
@@ -1,119 +0,0 @@
-package net.kemitix.thorp.storage.aws
-
-import org.scalatest.FreeSpec
-
-class ListerTest extends FreeSpec {
-
-// "list" - {
-// val bucket = Bucket.named("aBucket")
-// val prefix = RemoteKey.create("aRemoteKey")
-// "when no errors" - {
-// "when single fetch required" in {
-// val nowDate = new Date
-// val key = "key"
-// val etag = "etag"
-// val expectedHashMap = Map(MD5Hash.create(etag) -> RemoteKey.create(key))
-// val expectedKeyMap = Map(RemoteKey.create(key) -> MD5Hash.create(etag))
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3Client.listObjectsV2)
-// .when()
-// .returns(_ => {
-// UIO.succeed(objectResults(nowDate, key, etag, truncated = false))
-// })
-// private val result = invoke(fixture.amazonS3Client)(bucket, prefix)
-// private val hashMap =
-// result.map(_.byHash).map(m => Map.from(m.asMap.asScala))
-// private val keyMap =
-// result.map(_.byKey).map(m => Map.from(m.asMap.asScala))
-// hashMap should be(Right(expectedHashMap))
-// keyMap should be(Right(expectedKeyMap))
-// }
-// }
-//
-// "when second fetch required" in {
-// val nowDate = new Date
-// val key1 = "key1"
-// val etag1 = "etag1"
-// val key2 = "key2"
-// val etag2 = "etag2"
-// val expectedHashMap = Map(
-// MD5Hash.create(etag1) -> RemoteKey.create(key1),
-// MD5Hash.create(etag2) -> RemoteKey.create(key2)
-// )
-// val expectedKeyMap = Map(
-// RemoteKey.create(key1) -> MD5Hash.create(etag1),
-// RemoteKey.create(key2) -> MD5Hash.create(etag2)
-// )
-// new AmazonS3ClientTestFixture {
-//
-// (() => fixture.amazonS3Client.listObjectsV2)
-// .when()
-// .returns(_ =>
-// UIO(objectResults(nowDate, key1, etag1, truncated = true)))
-// .noMoreThanOnce()
-//
-// (() => fixture.amazonS3Client.listObjectsV2)
-// .when()
-// .returns(_ =>
-// UIO(objectResults(nowDate, key2, etag2, truncated = false)))
-// private val result = invoke(fixture.amazonS3Client)(bucket, prefix)
-// private val hashMap =
-// result.map(_.byHash).map(m => Map.from(m.asMap.asScala))
-// private val keyMap =
-// result.map(_.byKey).map(m => Map.from(m.asMap.asScala))
-// hashMap should be(Right(expectedHashMap))
-// keyMap should be(Right(expectedKeyMap))
-// }
-// }
-//
-// def objectSummary(key: String, etag: String, lastModified: Date) = {
-// val objectSummary = new S3ObjectSummary
-// objectSummary.setKey(key)
-// objectSummary.setETag(etag)
-// objectSummary.setLastModified(lastModified)
-// objectSummary
-// }
-//
-// def objectResults(nowDate: Date,
-// key: String,
-// etag: String,
-// truncated: Boolean) = {
-// val result = new ListObjectsV2Result
-// result.getObjectSummaries.add(objectSummary(key, etag, nowDate))
-// result.setTruncated(truncated)
-// result
-// }
-//
-// }
-// "when Amazon Service Exception" in {
-// val exception = new AmazonS3Exception("message")
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3Client.listObjectsV2)
-// .when()
-// .returns(_ => Task.fail(exception))
-// private val result = invoke(fixture.amazonS3Client)(bucket, prefix)
-// assert(result.isLeft)
-// }
-// }
-// "when Amazon SDK Client Exception" in {
-// val exception = new SdkClientException("message")
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3Client.listObjectsV2)
-// .when()
-// .returns(_ => Task.fail(exception))
-// private val result = invoke(fixture.amazonS3Client)(bucket, prefix)
-// assert(result.isLeft)
-// }
-// }
-// def invoke(amazonS3Client: AmazonS3Client.Client)(bucket: Bucket,
-// prefix: RemoteKey) = {
-// object TestEnv extends Storage.Test
-// val program: RIO[Storage, RemoteObjects] = Lister
-// .listObjects(amazonS3Client)(bucket, prefix)
-// val runtime = new DefaultRuntime {}
-// runtime.unsafeRunSync(program.provide(TestEnv)).toEither
-// }
-//
-// }
-
-}
diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/MD5HashData.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/MD5HashData.scala
deleted file mode 100644
index ffb6047..0000000
--- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/MD5HashData.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package net.kemitix.thorp.storage.aws
-
-import net.kemitix.thorp.domain.MD5Hash
-
-object MD5HashData {
-
- val rootHash = MD5Hash.create("a3a6ac11a0eb577b81b3bb5c95cc8a6e")
-
- val leafHash = MD5Hash.create("208386a650bdec61cfcd7bd8dcb6b542")
-
-}
diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala
deleted file mode 100644
index 08c2285..0000000
--- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-package net.kemitix.thorp.storage.aws
-
-import scala.jdk.CollectionConverters._
-
-import com.amazonaws.services.s3.model.S3ObjectSummary
-import net.kemitix.thorp.domain.{MD5Hash, RemoteKey}
-import org.scalatest.FunSpec
-
-class S3ObjectsByHashSuite extends FunSpec {
-
- describe("grouping s3 object together by their hash values") {
- val hash = MD5Hash.create("hash")
- val key1 = RemoteKey.create("key-1")
- val key2 = RemoteKey.create("key-2")
- val o1 = s3object(hash, key1)
- val o2 = s3object(hash, key2)
- val os = List(o1, o2)
- it("should group by the hash value") {
- val expected: Map[MD5Hash, RemoteKey] = Map(
- hash -> key2
- )
- val result = Map.from(S3Lister.byHash(os.asJava).asScala)
- assertResult(expected)(result)
- }
- }
-
- private def s3object(md5Hash: MD5Hash,
- remoteKey: RemoteKey): S3ObjectSummary = {
- val summary = new S3ObjectSummary()
- summary.setETag(md5Hash.hash())
- summary.setKey(remoteKey.key)
- summary
- }
-
-}
diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala
deleted file mode 100644
index a6f00a0..0000000
--- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala
+++ /dev/null
@@ -1,109 +0,0 @@
-package net.kemitix.thorp.storage.aws
-
-import org.scalamock.scalatest.MockFactory
-import org.scalatest.FreeSpec
-
-class UploaderTest extends FreeSpec with MockFactory {
-
-// val uiChannel: UChannel[Any, UIEvent] = zioMessage => ()
-//
-// "upload" - {
-// val aSource: File = Resource(this, "").toFile
-// val aFile: File = Resource(this, "small-file").toFile
-// val aHash = MD5Hash.create("aHash")
-// val hashes = Hashes.create(MD5, aHash)
-// val remoteKey = RemoteKey.create("aRemoteKey")
-// val localFile =
-// LocalFile.create(aFile, aSource, hashes, remoteKey, aFile.length)
-// val bucket = Bucket.named("aBucket")
-// val uploadResult = new UploadResult
-// uploadResult.setKey(remoteKey.key)
-// uploadResult.setETag(aHash.hash())
-// val listenerSettings =
-// UploadEventListener.Settings(uiChannel, localFile, 0, 0, batchMode = true)
-// "when no error" in {
-// val expected =
-// Right(StorageEvent.uploadEvent(remoteKey, aHash))
-// val inProgress = new AmazonUpload.InProgress {
-// override def waitForUploadResult: Task[UploadResult] =
-// Task(uploadResult)
-// }
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3TransferManager.uploader)
-// .when()
-// .returns(_ => UIO.succeed(inProgress))
-// private val result =
-// invoke(fixture.amazonS3TransferManager)(
-// localFile,
-// bucket,
-// listenerSettings
-// )
-// assertResult(expected)(result)
-// }
-// }
-// "when Amazon Service Exception" in {
-// val exception = new AmazonS3Exception("message")
-// val expected =
-// Right(
-// StorageEvent.errorEvent(ActionSummary.upload(remoteKey.key),
-// remoteKey,
-// exception))
-// val inProgress = new AmazonUpload.InProgress {
-// override def waitForUploadResult: Task[UploadResult] =
-// Task.fail(exception)
-// }
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3TransferManager.upload)
-// .when()
-// .returns(_ => UIO.succeed(inProgress))
-// private val result =
-// invoke(fixture.amazonS3TransferManager)(
-// localFile,
-// bucket,
-// listenerSettings
-// )
-// assertResult(expected)(result)
-// }
-// }
-// "when Amazon SDK Client Exception" in {
-// val exception = new SdkClientException("message")
-// val expected =
-// Right(
-// StorageEvent.errorEvent(ActionSummary.upload(remoteKey.key),
-// remoteKey,
-// exception))
-// val inProgress = new AmazonUpload.InProgress {
-// override def waitForUploadResult: Task[UploadResult] =
-// Task.fail(exception)
-// }
-// new AmazonS3ClientTestFixture {
-// (() => fixture.amazonS3TransferManager.upload)
-// .when()
-// .returns(_ => UIO.succeed(inProgress))
-// private val result =
-// invoke(fixture.amazonS3TransferManager)(
-// localFile,
-// bucket,
-// listenerSettings
-// )
-// assertResult(expected)(result)
-// }
-// }
-// def invoke(transferManager: AmazonTransferManager)(
-// localFile: LocalFile,
-// bucket: Bucket,
-// listenerSettings: UploadEventListener.Settings
-// ) = {
-// val program = Uploader
-// .upload(transferManager)(
-// Uploader.Request(localFile, bucket, listenerSettings))
-// val runtime = new DefaultRuntime {}
-// runtime
-// .unsafeRunSync(
-// program
-// .provide(Config.Live))
-// .toEither
-// }
-// }
-
-}
diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala
deleted file mode 100644
index 95cbf45..0000000
--- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-package net.kemitix.thorp.storage.aws.hasher
-
-import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
-import net.kemitix.thorp.filesystem.Resource
-import org.scalatest.FreeSpec
-
-class ETagGeneratorTest extends FreeSpec {
-
- private val bigFile = Resource.select(this, "../big-file")
- private val bigFilePath = bigFile.toPath
- private val configuration = new TransferManagerConfiguration
- private val chunkSize = 1200000
- configuration.setMinimumUploadPartSize(chunkSize)
-
-// "Create offsets" - {
-// "should create offsets" in {
-// val offsets = S3ETagGenerator
-// .offsets(bigFile.length, chunkSize)
-// .foldRight(List[Long]())((l: Long, a: List[Long]) => l :: a)
-// assertResult(
-// List(0, chunkSize, chunkSize * 2, chunkSize * 3, chunkSize * 4))(
-// offsets)
-// }
-// }
-
-// "create md5 hash for each chunk" - {
-// "should create expected hash for chunks" in {
-// val md5Hashes = List(
-// "68b7d37e6578297621e06f01800204f1",
-// "973475b14a7bda6ad8864a7f9913a947",
-// "b9adcfc5b103fe2dd5924a5e5e6817f0",
-// "5bd6e10a99fef100fe7bf5eaa0a42384",
-// "8a0c1d0778ac8fcf4ca2010eba4711eb"
-// ).zipWithIndex
-// md5Hashes.foreach {
-// case (hash, index) =>
-// val program = Hasher.hashObjectChunk(bigFilePath, index, chunkSize)
-// val result = runtime.unsafeRunSync(program.provide(TestEnv)).toEither
-// assertResult(Right(hash))(
-// result
-// .map(hashes => hashes.get(MD5).get())
-// .map(x => x.hash))
-// }
-// }
-// }
-
-// "create etag for whole file" - {
-// val expected = "f14327c90ad105244c446c498bfe9a7d-2"
-// "should match aws etag for the file" in {
-// val program = ETagGenerator.eTag(bigFilePath)
-// val result = runtime.unsafeRunSync(program.provide(TestEnv)).toEither
-// assertResult(Right(expected))(result)
-// }
-// }
-
-}