kemitix/thorp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Convert storage-aws tests to Java (#484)

Browse source 
* storage-aws.ListerTest: convert to Java

* storage-aws: convert to Java
This commit is contained in:
Paul Campbell 2020-06-26 08:01:29 +01:00 committed by GitHub
parent 5661c9e7da
commit 78b1bcf6ac
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 510 additions and 602 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
docs/images/reactor-graph.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 211 KiB

After

Width:  |  Height:  |  Size: 193 KiB

3
domain/src/main/java/net/kemitix/thorp/domain/StorageEvent.java
View file

@ -1,6 +1,7 @@
package net.kemitix.thorp.domain;
import lombok.AccessLevel;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
@ -27,6 +28,7 @@ public class StorageEvent {
public static class DoNothingEvent extends StorageEvent {
public final RemoteKey remoteKey;
}
@EqualsAndHashCode(callSuper = false)
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public static class CopyEvent extends StorageEvent {
public final RemoteKey sourceKey;
@ -37,6 +39,7 @@ public class StorageEvent {
public final RemoteKey remoteKey;
public final MD5Hash md5Hash;
}
@EqualsAndHashCode(callSuper = false)
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public static class DeleteEvent extends StorageEvent {
public final RemoteKey remoteKey;

2
filesystem/src/main/java/net/kemitix/thorp/filesystem/MD5HashGenerator.java
View file

@ -13,7 +13,7 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class MD5HashGenerator implements HashGenerator {
private static final int maxBufferSize = 8048;
private static final int maxBufferSize = 8192;
private static final byte[] defaultBuffer = new byte[maxBufferSize];
public static String hex(byte[] in) throws NoSuchAlgorithmException {
MessageDigest md5 = MessageDigest.getInstance("MD5");

12
lib/src/main/scala/net/kemitix/thorp/lib/UnversionedMirrorArchive.scala
View file

@ -2,6 +2,7 @@ package net.kemitix.thorp.lib
import net.kemitix.thorp.config.Configuration
import net.kemitix.thorp.domain.Action.{ToCopy, ToDelete, ToUpload}
import net.kemitix.thorp.domain.StorageEvent.ActionSummary
import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.Storage
import net.kemitix.thorp.uishell.{UIEvent, UploadEventListener}
@ -35,7 +36,16 @@ trait UnversionedMirrorArchive extends ThorpArchive {
.copy(bucket, sourceKey, hash, targetKey)
case toDelete: ToDelete =>
val remoteKey = toDelete.remoteKey
Storage.getInstance().delete(bucket, remoteKey)
try {
Storage.getInstance().delete(bucket, remoteKey)
} catch {
case e: Throwable =>
StorageEvent.errorEvent(
ActionSummary.delete(remoteKey.key()),
remoteKey,
e
);
}
case doNothing: Action.DoNothing =>
val remoteKey = doNothing.remoteKey
StorageEvent.doNothingEvent(remoteKey)

29
storage-aws/pom.xml
View file

@ -48,11 +48,10 @@
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<!-- scala -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<!-- aws-sdk -->
@ -95,27 +94,5 @@
<groupId>commons-logging</groupId>
<version>1.2</version>
</dependency>
<!-- scala - testing -->
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.13</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock_2.13</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

18
storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Copier.java
View file

@ -1,5 +1,6 @@
package net.kemitix.thorp.storage.aws;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import net.kemitix.thorp.domain.Bucket;
import net.kemitix.thorp.domain.MD5Hash;
@ -24,16 +25,19 @@ public interface S3Copier {
return request -> {
RemoteKey sourceKey = RemoteKey.create(request.getSourceKey());
RemoteKey targetKey = RemoteKey.create(request.getDestinationKey());
return client.copyObject(request)
.map(success -> StorageEvent.copyEvent(sourceKey, targetKey))
.orElseGet(() -> errorEvent(sourceKey, targetKey));
try {
return client.copyObject(request)
.map(success -> StorageEvent.copyEvent(sourceKey, targetKey))
.orElseGet(() -> StorageEvent.errorEvent(
actionSummary(sourceKey, targetKey),
targetKey, S3Exception.hashError()));
} catch (SdkClientException e) {
return StorageEvent.errorEvent(
actionSummary(sourceKey, targetKey), targetKey, e);
}
};
}
static StorageEvent.ErrorEvent errorEvent(RemoteKey sourceKey, RemoteKey targetKey) {
return StorageEvent.errorEvent(actionSummary(sourceKey, targetKey), targetKey, S3Exception.hashError());
}
static StorageEvent.ActionSummary.Copy actionSummary(RemoteKey sourceKey, RemoteKey targetKey) {
return StorageEvent.ActionSummary.copy(
String.format("%s => %s", sourceKey.key(), targetKey.key()));

11
storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Deleter.java
View file

@ -1,5 +1,6 @@
package net.kemitix.thorp.storage.aws;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import net.kemitix.thorp.domain.Bucket;
import net.kemitix.thorp.domain.RemoteKey;
@ -13,9 +14,15 @@ public interface S3Deleter {
}
static Function<DeleteObjectRequest, StorageEvent> deleter(AmazonS3Client client) {
return request -> {
client.deleteObject(request);
RemoteKey remoteKey = RemoteKey.create(request.getKey());
return StorageEvent.deleteEvent(remoteKey);
try {
client.deleteObject(request);
return StorageEvent.deleteEvent(remoteKey);
} catch (SdkClientException e) {
return StorageEvent.errorEvent(
StorageEvent.ActionSummary.delete(remoteKey.key()),
remoteKey, e);
}
};
}
}

14
storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3ETagGenerator.java
View file

@ -7,15 +7,18 @@ import net.kemitix.thorp.domain.HashGenerator;
import net.kemitix.thorp.domain.HashType;
import net.kemitix.thorp.domain.Hashes;
import net.kemitix.thorp.domain.MD5Hash;
import net.kemitix.thorp.filesystem.MD5HashGenerator;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
import static net.kemitix.thorp.filesystem.MD5HashGenerator.md5FileChunk;
public class S3ETagGenerator implements HashGenerator {
@Deprecated // Use hashFile
public String eTag(Path path) throws IOException, NoSuchAlgorithmException {
@ -43,7 +46,7 @@ public class S3ETagGenerator implements HashGenerator {
public List<Long> offsets(long totalFileSizeBytes, long optimalPartSize) {
return LongStream
.range(0, totalFileSizeBytes / optimalPartSize)
.rangeClosed(0, totalFileSizeBytes / optimalPartSize)
.mapToObj(part -> part * optimalPartSize)
.collect(Collectors.toList());
}
@ -63,12 +66,11 @@ public class S3ETagGenerator implements HashGenerator {
}
private String eTagHex(Path path, long partSize, long parts) throws IOException, NoSuchAlgorithmException {
HashGenerator hashGenerator = HashGenerator.generatorFor("MD5");
MessageDigest md5 = MessageDigest.getInstance("MD5");
ByteArrayOutputStream bos = new ByteArrayOutputStream();
for (long i = 0; i < parts ; i++ ){
md5.update(hashGenerator.hashChunk(path, i, partSize).digest());
bos.write(md5FileChunk(path, i * partSize, partSize).digest());
}
return MD5Hash.digestAsString(md5.digest());
return MD5HashGenerator.hex(bos.toByteArray());
}
@Override
public HashType hashType() {

14
storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Lister.java
View file

@ -1,5 +1,7 @@
package net.kemitix.thorp.storage.aws;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.S3ObjectSummary;
@ -71,10 +73,14 @@ public interface S3Lister {
AmazonS3Client client,
ListObjectsV2Request request
) {
Batch batch = fetchBatch(client, request);
List<S3ObjectSummary> more = fetchMore(client, request, batch.more);
batch.summaries.addAll(more);
return batch.summaries;
try {
Batch batch = fetchBatch(client, request);
List<S3ObjectSummary> more = fetchMore(client, request, batch.more);
batch.summaries.addAll(more);
return batch.summaries;
} catch (SdkClientException e) {
return Collections.emptyList();
}
};
static Optional<String> moreToken(ListObjectsV2Result result) {

12
storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3TransferManager.java
View file

@ -2,7 +2,6 @@ package net.kemitix.thorp.storage.aws;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import java.util.function.Function;
@ -17,14 +16,9 @@ public interface S3TransferManager {
}
@Override
public Function<PutObjectRequest, S3Upload> uploader() {
return request -> {
Upload upload = transferManager.upload(request);
try {
return S3Upload.inProgress(upload);
} catch (S3Exception.UploadError error) {
return S3Upload.errored(error);
}
};
return request ->
S3Upload.inProgress(
transferManager.upload(request));
}
};
}

22
storage-aws/src/main/java/net/kemitix/thorp/storage/aws/S3Uploader.java
View file

@ -1,5 +1,6 @@
package net.kemitix.thorp.storage.aws;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.model.UploadResult;
@ -29,13 +30,20 @@ public interface S3Uploader {
S3TransferManager transferManager
) {
return request -> {
UploadResult uploadResult =
transferManager.uploader()
.apply(request)
.waitForUploadResult();
return StorageEvent.uploadEvent(
RemoteKey.create(uploadResult.getKey()),
MD5Hash.create(uploadResult.getETag()));
try {
UploadResult uploadResult =
transferManager.uploader()
.apply(request)
.waitForUploadResult();
return StorageEvent.uploadEvent(
RemoteKey.create(uploadResult.getKey()),
MD5Hash.create(uploadResult.getETag()));
} catch (SdkClientException e) {
String key = request.getKey();
return StorageEvent.errorEvent(
StorageEvent.ActionSummary.upload(key),
RemoteKey.create(key), e);
}
};
}
}

65
storage-aws/src/test/java/net/kemitix/thorp/storage/aws/CopierTest.java Normal file
View file

@ -0,0 +1,65 @@
package net.kemitix.thorp.storage.aws;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.CopyObjectResult;
import net.kemitix.thorp.domain.Bucket;
import net.kemitix.thorp.domain.MD5Hash;
import net.kemitix.thorp.domain.RemoteKey;
import net.kemitix.thorp.domain.StorageEvent;
import org.assertj.core.api.WithAssertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Optional;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.doThrow;
@ExtendWith(MockitoExtension.class)
public class CopierTest
implements WithAssertions {
private final Bucket bucket = Bucket.named("aBucket");
private final RemoteKey sourceKey = RemoteKey.create("sourceKey");
private final MD5Hash hash = MD5Hash.create("aHash");
private final RemoteKey targetKey = RemoteKey.create("targetKey");
private final CopyObjectRequest request = S3Copier.request(bucket, sourceKey, hash, targetKey);
private final AmazonS3Client amazonS3Client;
public CopierTest(@Mock AmazonS3Client amazonS3Client) {
this.amazonS3Client = amazonS3Client;
}
@Test
@DisplayName("copy produces copy event")
public void copy_thenCopyEvent() {
//given
StorageEvent event = StorageEvent.copyEvent(sourceKey, targetKey);
given(amazonS3Client.copyObject(request))
.willReturn(Optional.of(new CopyObjectResult()));
//when
StorageEvent result = S3Copier.copier(amazonS3Client).apply(request);
//then
assertThat(result).isEqualTo(event);
}
@Test
@DisplayName("when error copying then return an error storage event")
public void whenCopyErrors_thenErrorEvent() {
//given
doThrow(SdkClientException.class)
.when(amazonS3Client)
.copyObject(request);
//when
StorageEvent result = S3Copier.copier(amazonS3Client).apply(request);
//then
assertThat(result).isInstanceOf(StorageEvent.ErrorEvent.class);
}
}

52
storage-aws/src/test/java/net/kemitix/thorp/storage/aws/DeleterTest.java Normal file
View file

@ -0,0 +1,52 @@
package net.kemitix.thorp.storage.aws;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import net.kemitix.thorp.domain.Bucket;
import net.kemitix.thorp.domain.RemoteKey;
import net.kemitix.thorp.domain.StorageEvent;
import org.assertj.core.api.WithAssertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.mockito.Mockito.doThrow;
@ExtendWith(MockitoExtension.class)
public class DeleterTest
implements WithAssertions {
private final Bucket bucket = Bucket.named("aBucket");
private final RemoteKey remoteKey = RemoteKey.create("aRemoteKey");
private final DeleteObjectRequest request = S3Deleter.request(bucket, remoteKey);
private final AmazonS3Client amazonS3Client;
public DeleterTest(@Mock AmazonS3Client amazonS3Client) {
this.amazonS3Client = amazonS3Client;
}
@Test
@DisplayName("when delete then return delete storage event")
public void whenDelete_thenDeleteEvent() {
//when
StorageEvent result = S3Deleter.deleter(amazonS3Client).apply(request);
//then
assertThat(result).isEqualTo(StorageEvent.deleteEvent(remoteKey));
}
@Test
@DisplayName("when error deleting then return an error storage event")
public void whenDeleteErrors_thenErrorEvent() {
//given
doThrow(SdkClientException.class)
.when(amazonS3Client)
.deleteObject(request);
//when
StorageEvent result = S3Deleter.deleter(amazonS3Client).apply(request);
//then
assertThat(result).isInstanceOf(StorageEvent.ErrorEvent.class);
}
}

54
storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ETagGeneratorTest.java Normal file
View file

@ -0,0 +1,54 @@
package net.kemitix.thorp.storage.aws;
import net.kemitix.thorp.domain.MD5Hash;
import net.kemitix.thorp.domain.Tuple;
import net.kemitix.thorp.filesystem.Resource;
import org.assertj.core.api.WithAssertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.List;
public class ETagGeneratorTest
implements WithAssertions {
public static final String BIG_FILE_ETAG = "f14327c90ad105244c446c498bfe9a7d-2";
private final Resource bigFile = Resource.select(this, "big-file");
private final long chunkSize = 1200000;
private final S3ETagGenerator generator = new S3ETagGenerator();
@Test
@DisplayName("creates offsets")
public void createsOffsets() {
List<Long> offsets = generator.offsets(bigFile.length(), chunkSize);
assertThat(offsets).containsExactly(
0L, chunkSize, chunkSize * 2, chunkSize * 3, chunkSize * 4);
}
@Test
@DisplayName("generate valid hashes")
public void generatesValidHashes() throws IOException, NoSuchAlgorithmException {
List<Tuple<Integer, String>> md5Hashes = Arrays.asList(
Tuple.create(0, "68b7d37e6578297621e06f01800204f1"),
Tuple.create(1, "973475b14a7bda6ad8864a7f9913a947"),
Tuple.create(2, "b9adcfc5b103fe2dd5924a5e5e6817f0"),
Tuple.create(3, "5bd6e10a99fef100fe7bf5eaa0a42384"),
Tuple.create(4, "8a0c1d0778ac8fcf4ca2010eba4711eb"));
for (Tuple<Integer, String> t : md5Hashes) {
long offset = t.a * chunkSize;
MD5Hash md5Hash =
generator.hashChunk(bigFile.toPath(), offset, chunkSize);
assertThat(md5Hash.getValue()).isEqualTo(t.b);
}
}
@Test
@DisplayName("create eTag for whole file")
public void createTagForWholeFile() throws IOException, NoSuchAlgorithmException {
String result = generator.hashFile(bigFile.toPath());
assertThat(result).isEqualTo(BIG_FILE_ETAG);
}
}

6
storage-aws/src/test/java/net/kemitix/thorp/storage/aws/HashGeneratorTest.java
View file

@ -42,7 +42,8 @@ public class HashGeneratorTest
//when
Hashes result = HashGenerator.hashObject(path);
//then
assertThat(result.get(HashType.MD5)).contains(MD5HashData.rootHash());
assertThat(result.get(HashType.MD5))
.contains(MD5Hash.create("a3a6ac11a0eb577b81b3bb5c95cc8a6e"));
}
@Test
@DisplayName("leaf-file")
@ -52,7 +53,8 @@ public class HashGeneratorTest
//when
Hashes result = HashGenerator.hashObject(path);
//then
assertThat(result.get(HashType.MD5)).contains(MD5HashData.leafHash());
assertThat(result.get(HashType.MD5))
.contains(MD5Hash.create("208386a650bdec61cfcd7bd8dcb6b542"));
}
private Path getResource(String s) {

136
storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ListerTest.java Normal file
View file

@ -0,0 +1,136 @@
package net.kemitix.thorp.storage.aws;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import net.kemitix.thorp.domain.*;
import org.assertj.core.api.WithAssertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.doThrow;
@ExtendWith(MockitoExtension.class)
public class ListerTest
implements WithAssertions {
private final Bucket bucket = Bucket.named("aBucket");
private final RemoteKey prefix = RemoteKey.create("aRemoteKey");
private final ListObjectsV2Request request = S3Lister.request(bucket, prefix);
private final AmazonS3Client amazonS3Client;
public ListerTest(@Mock AmazonS3Client amazonS3Client) {
this.amazonS3Client = amazonS3Client;
}
@Test
@DisplayName("single fetch")
public void singleFetch() {
//given
Date nowDate = new Date();
String key = "key";
String etag = "etag";
Map<MD5Hash, RemoteKey> expectedHashMap =
Collections.singletonMap(
MD5Hash.create(etag),
RemoteKey.create(key)
);
Map<RemoteKey, MD5Hash> expectedKeyMap =
Collections.singletonMap(
RemoteKey.create(key),
MD5Hash.create(etag)
);
given(amazonS3Client.listObjects(any()))
.willReturn(objectResults(nowDate, key, etag, false));
//when
RemoteObjects result = S3Lister.lister(amazonS3Client).apply(request);
//then
assertThat(result.byHash.asMap()).isEqualTo(expectedHashMap);
assertThat(result.byKey.asMap()).isEqualTo(expectedKeyMap);
}
@Test
@DisplayName("two fetches")
public void twoFetches() {
//given
Date nowDate = new Date();
String key1 = "key1";
String etag1 = "etag1";
String key2 = "key2";
String etag2 = "etag2";
Map<MD5Hash, RemoteKey> expectedHashMap = new HashMap<>();
expectedHashMap.put(
MD5Hash.create(etag1),
RemoteKey.create(key1));
expectedHashMap.put(
MD5Hash.create(etag2),
RemoteKey.create(key2));
Map<RemoteKey, MD5Hash> expectedKeyMap = new HashMap<>();
expectedKeyMap.put(
RemoteKey.create(key1),
MD5Hash.create(etag1));
expectedKeyMap.put(
RemoteKey.create(key2),
MD5Hash.create(etag2));
given(amazonS3Client.listObjects(any()))
.willReturn(objectResults(nowDate, key1, etag1, true))
.willReturn(objectResults(nowDate, key2, etag2, false));
//when
RemoteObjects result = S3Lister.lister(amazonS3Client).apply(request);
//then
assertThat(result.byHash.asMap()).isEqualTo(expectedHashMap);
assertThat(result.byKey.asMap()).isEqualTo(expectedKeyMap);
}
@Test
@DisplayName("when error listing then return an error storage event")
public void whenListErrors_thenErrorEvent() {
//given
doThrow(SdkClientException.class)
.when(amazonS3Client)
.listObjects(request);
//when
RemoteObjects result = S3Lister.lister(amazonS3Client).apply(request);
//then
assertThat(result.byKey.asMap()).isEmpty();
assertThat(result.byHash.asMap()).isEmpty();
}
private ListObjectsV2Result objectResults(
Date nowDate,
String key,
String etag,
boolean truncated
) {
ListObjectsV2Result result = new ListObjectsV2Result();
result.getObjectSummaries().add(objectSummary(key, etag, nowDate));
result.setNextContinuationToken("next token");
result.setTruncated(truncated);
return result;
}
private S3ObjectSummary objectSummary(
String key,
String etag,
Date lastModified
) {
S3ObjectSummary summary = new S3ObjectSummary();
summary.setKey(key);
summary.setETag(etag);
summary.setLastModified(lastModified);
return summary;
}
}

44
storage-aws/src/test/java/net/kemitix/thorp/storage/aws/ObjectsByHashTest.java Normal file
View file

@ -0,0 +1,44 @@
package net.kemitix.thorp.storage.aws;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import lombok.val;
import net.kemitix.thorp.domain.MD5Hash;
import net.kemitix.thorp.domain.RemoteKey;
import org.assertj.core.api.WithAssertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class ObjectsByHashTest
implements WithAssertions {
private S3ObjectSummary s3object(MD5Hash md5Hash,
RemoteKey remoteKey) {
S3ObjectSummary summary = new S3ObjectSummary();
summary.setETag(md5Hash.hash());
summary.setKey(remoteKey.key());
return summary;
}
@Test
@DisplayName("grouping s3 objects together by their hash value")
public void groupS3ObjectsByHashValue() {
//given
MD5Hash hash = MD5Hash.create("hash");
RemoteKey key1 = RemoteKey.create("key-1");
RemoteKey key2 = RemoteKey.create("key-2");
S3ObjectSummary o1 = s3object(hash, key1);
S3ObjectSummary o2 = s3object(hash, key2);
List<S3ObjectSummary> os = Arrays.asList(o1, o2);
Map<MD5Hash, RemoteKey> expected = Collections.singletonMap(
hash, key2);
//when
Map<MD5Hash, RemoteKey> result = S3Lister.byHash(os);
//then
assertThat(result).isEqualTo(expected);
}
}

81
storage-aws/src/test/java/net/kemitix/thorp/storage/aws/UploaderTest.java Normal file
View file

@ -0,0 +1,81 @@
package net.kemitix.thorp.storage.aws;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.services.s3.transfer.model.UploadResult;
import net.kemitix.thorp.domain.HashType;
import net.kemitix.thorp.domain.*;
import net.kemitix.thorp.filesystem.Resource;
import org.assertj.core.api.WithAssertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.File;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.doThrow;
@ExtendWith(MockitoExtension.class)
public class UploaderTest
implements WithAssertions {
private final File aSource = Resource.select(this, "").toFile();
private final File aFile = Resource.select(this, "small-file").toFile();
private final MD5Hash aHash = MD5Hash.create("aHash");
private final Hashes hashes = Hashes.create(HashType.MD5, aHash);
private final RemoteKey remoteKey = RemoteKey.create("aRemoteKey");
private final LocalFile localFile =
LocalFile.create(aFile, aSource, hashes, remoteKey, aFile.length());
private final Bucket bucket = Bucket.named("aBucket");
private final PutObjectRequest request =
S3Uploader.request(localFile, bucket);
private final TransferManager transferManager;
private final S3TransferManager s3TransferManager;
private final UploadResult uploadResult;
private final Upload upload;
public UploaderTest(@Mock TransferManager transferManager,
@Mock Upload upload) {
this.transferManager = transferManager;
this.upload = upload;
s3TransferManager = S3TransferManager.create(transferManager);
uploadResult = new UploadResult();
uploadResult.setKey(remoteKey.key());
uploadResult.setETag(aHash.hash());
}
@Test
@DisplayName("when upload then return upload event")
public void whenUpload_thenUploadEvent() throws InterruptedException {
//given
given(transferManager.upload(request))
.willReturn(upload);
given(upload.waitForUploadResult()).willReturn(uploadResult);
//when
StorageEvent result = S3Uploader.uploader(s3TransferManager).apply(request);
//then
assertThat(result).isInstanceOf(StorageEvent.UploadEvent.class);
}
@Test
@DisplayName("when error uploading then return an error storage event")
public void whenUploadErrors_thenErrorEvent() {
//given
doThrow(SdkClientException.class)
.when(transferManager)
.upload(request);
//when
StorageEvent result = S3Uploader.uploader(s3TransferManager).apply(request);
//then
assertThat(result).isInstanceOf(StorageEvent.ErrorEvent.class);
}
}

60
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/AmazonS3ClientTestFixture.scala
View file

@ -1,60 +0,0 @@
package net.kemitix.thorp.storage.aws
import net.kemitix.thorp.storage.Storage
import org.scalamock.scalatest.MockFactory
trait AmazonS3ClientTestFixture extends MockFactory {
@SuppressWarnings(Array("org.wartremover.warts.PublicInference"))
private val manager = stub[S3TransferManager]
@SuppressWarnings(Array("org.wartremover.warts.PublicInference"))
private val client = stub[AmazonS3Client]
val fixture: Fixture = Fixture(client, manager)
case class Fixture(amazonS3Client: AmazonS3Client,
amazonS3TransferManager: S3TransferManager,
) {
lazy val storageService: Storage = Storage.getInstance()
// new Storage.Service {
//
// private val client = amazonS3Client
// private val transferManager = amazonS3TransferManager
//
// override def listObjects(
// bucket: Bucket,
// prefix: RemoteKey
// ): RIO[Storage, RemoteObjects] =
// UIO {
// S3Lister.lister(client)(S3Lister.request(bucket, prefix))
// }
//
// override def upload(localFile: LocalFile,
// bucket: Bucket,
// listenerSettings: UploadEventListener.Settings,
// ): UIO[StorageEvent] =
// UIO(
// S3Uploader
// .uploader(transferManager)(S3Uploader.request(localFile, bucket))
// )
//
// override def copy(bucket: Bucket,
// sourceKey: RemoteKey,
// hash: MD5Hash,
// targetKey: RemoteKey): UIO[StorageEvent] =
// UIO {
// val request = S3Copier.request(bucket, sourceKey, hash, targetKey)
// S3Copier.copier(client)(request)
// }
//
// override def delete(bucket: Bucket,
// remoteKey: RemoteKey): UIO[StorageEvent] =
// UIO(S3Deleter.deleter(client)(S3Deleter.request(bucket, remoteKey)))
//
// override def shutdown: UIO[StorageEvent] = {
// UIO(transferManager.shutdownNow(true)) *> UIO(client.shutdown())
// .map(_ => StorageEvent.shutdownEvent())
// }
// }
}
}

88
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/CopierTest.scala
View file

@ -1,88 +0,0 @@
package net.kemitix.thorp.storage.aws
import org.scalatest.FreeSpec
class CopierTest extends FreeSpec {
// private val runtime = Runtime(Console.Live, PlatformLive.Default)
//
// "copier" - {
// val bucket = Bucket.named("aBucket")
// val sourceKey = RemoteKey.create("sourceKey")
// val hash = MD5Hash.create("aHash")
// val targetKey = RemoteKey.create("targetKey")
// "when source exists" - {
// "when source hash matches" - {
// "copies from source to target" in {
// val event = StorageEvent.copyEvent(sourceKey, targetKey)
// val expected = Right(event)
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3Client.copyObject)
// .when()
// .returns(_ => Task.succeed(Some(new CopyObjectResult)))
// private val result =
// invoke(bucket, sourceKey, hash, targetKey, fixture.amazonS3Client)
// assertResult(expected)(result)
// }
// }
// }
// "when source hash does not match" - {
// "skip the file with an error" in {
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3Client.copyObject)
// .when()
// .returns(_ => Task.succeed(None))
// private val result =
// invoke(bucket, sourceKey, hash, targetKey, fixture.amazonS3Client)
// result match {
// case right: Right[Throwable, StorageEvent] => {
// val e = right.value.asInstanceOf[ErrorEvent].e
// e match {
// case HashError => assert(true)
// case _ => fail(s"Not a HashError: ${e.getMessage}")
// }
// }
// case e => fail(s"Not an ErrorQueueEvent: $e")
// }
// }
// }
// }
// "when client throws an exception" - {
// "skip the file with an error" in {
// new AmazonS3ClientTestFixture {
// private val expectedMessage = "The specified key does not exist"
// (() => fixture.amazonS3Client.copyObject)
// .when()
// .returns(_ => Task.fail(new AmazonS3Exception(expectedMessage)))
// private val result =
// invoke(bucket, sourceKey, hash, targetKey, fixture.amazonS3Client)
// val key = RemoteKey.create("targetKey")
// result match {
// case right: Right[Throwable, StorageEvent] => {
// val e = right.value.asInstanceOf[ErrorEvent].e
// e match {
// case CopyError(cause) =>
// assert(cause.getMessage.startsWith(expectedMessage))
// case _ => fail(s"Not a CopyError: ${e.getMessage}")
// }
// }
// case e => fail(s"Not an ErrorQueueEvent: ${e}")
// }
// }
// }
// }
// }
// def invoke(
// bucket: Bucket,
// sourceKey: RemoteKey,
// hash: MD5Hash,
// targetKey: RemoteKey,
// amazonS3Client: AmazonS3Client
// ) =
// runtime.unsafeRunSync {
// Copier.copy(amazonS3Client)(
// Copier.Request(bucket, sourceKey, hash, targetKey))
// }.toEither
// }
}

59
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/DeleterTest.scala
View file

@ -1,59 +0,0 @@
package net.kemitix.thorp.storage.aws
import org.scalatest.FreeSpec
class DeleterTest extends FreeSpec {
// private val runtime = Runtime(Console.Live, PlatformLive.Default)
//
// "delete" - {
// val bucket = Bucket.named("aBucket")
// val remoteKey = RemoteKey.create("aRemoteKey")
// "when no errors" in {
// val expected = Right(StorageEvent.deleteEvent(remoteKey))
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3Client.deleteObject)
// .when()
// .returns(_ => UIO.succeed(()))
// private val result = invoke(fixture.amazonS3Client)(bucket, remoteKey)
// assertResult(expected)(result)
// }
// }
// "when Amazon Service Exception" in {
// val exception = new AmazonS3Exception("message")
// val expected =
// Right(
// StorageEvent.errorEvent(ActionSummary.delete(remoteKey.key),
// remoteKey,
// exception))
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3Client.deleteObject)
// .when()
// .returns(_ => Task.fail(exception))
// private val result = invoke(fixture.amazonS3Client)(bucket, remoteKey)
// assertResult(expected)(result)
// }
// }
// "when Amazon SDK Client Exception" in {
// val exception = new SdkClientException("message")
// val expected =
// Right(
// StorageEvent.errorEvent(ActionSummary.delete(remoteKey.key),
// remoteKey,
// exception))
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3Client.deleteObject)
// .when()
// .returns(_ => Task.fail(exception))
// private val result = invoke(fixture.amazonS3Client)(bucket, remoteKey)
// assertResult(expected)(result)
// }
// }
// def invoke(amazonS3Client: AmazonS3Client.Client)(bucket: Bucket,
// remoteKey: RemoteKey) =
// runtime.unsafeRunSync {
// Deleter.delete(amazonS3Client)(bucket, remoteKey)
// }.toEither
//
// }
}

119
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala
View file

@ -1,119 +0,0 @@
package net.kemitix.thorp.storage.aws
import org.scalatest.FreeSpec
class ListerTest extends FreeSpec {
// "list" - {
// val bucket = Bucket.named("aBucket")
// val prefix = RemoteKey.create("aRemoteKey")
// "when no errors" - {
// "when single fetch required" in {
// val nowDate = new Date
// val key = "key"
// val etag = "etag"
// val expectedHashMap = Map(MD5Hash.create(etag) -> RemoteKey.create(key))
// val expectedKeyMap = Map(RemoteKey.create(key) -> MD5Hash.create(etag))
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3Client.listObjectsV2)
// .when()
// .returns(_ => {
// UIO.succeed(objectResults(nowDate, key, etag, truncated = false))
// })
// private val result = invoke(fixture.amazonS3Client)(bucket, prefix)
// private val hashMap =
// result.map(_.byHash).map(m => Map.from(m.asMap.asScala))
// private val keyMap =
// result.map(_.byKey).map(m => Map.from(m.asMap.asScala))
// hashMap should be(Right(expectedHashMap))
// keyMap should be(Right(expectedKeyMap))
// }
// }
//
// "when second fetch required" in {
// val nowDate = new Date
// val key1 = "key1"
// val etag1 = "etag1"
// val key2 = "key2"
// val etag2 = "etag2"
// val expectedHashMap = Map(
// MD5Hash.create(etag1) -> RemoteKey.create(key1),
// MD5Hash.create(etag2) -> RemoteKey.create(key2)
// )
// val expectedKeyMap = Map(
// RemoteKey.create(key1) -> MD5Hash.create(etag1),
// RemoteKey.create(key2) -> MD5Hash.create(etag2)
// )
// new AmazonS3ClientTestFixture {
//
// (() => fixture.amazonS3Client.listObjectsV2)
// .when()
// .returns(_ =>
// UIO(objectResults(nowDate, key1, etag1, truncated = true)))
// .noMoreThanOnce()
//
// (() => fixture.amazonS3Client.listObjectsV2)
// .when()
// .returns(_ =>
// UIO(objectResults(nowDate, key2, etag2, truncated = false)))
// private val result = invoke(fixture.amazonS3Client)(bucket, prefix)
// private val hashMap =
// result.map(_.byHash).map(m => Map.from(m.asMap.asScala))
// private val keyMap =
// result.map(_.byKey).map(m => Map.from(m.asMap.asScala))
// hashMap should be(Right(expectedHashMap))
// keyMap should be(Right(expectedKeyMap))
// }
// }
//
// def objectSummary(key: String, etag: String, lastModified: Date) = {
// val objectSummary = new S3ObjectSummary
// objectSummary.setKey(key)
// objectSummary.setETag(etag)
// objectSummary.setLastModified(lastModified)
// objectSummary
// }
//
// def objectResults(nowDate: Date,
// key: String,
// etag: String,
// truncated: Boolean) = {
// val result = new ListObjectsV2Result
// result.getObjectSummaries.add(objectSummary(key, etag, nowDate))
// result.setTruncated(truncated)
// result
// }
//
// }
// "when Amazon Service Exception" in {
// val exception = new AmazonS3Exception("message")
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3Client.listObjectsV2)
// .when()
// .returns(_ => Task.fail(exception))
// private val result = invoke(fixture.amazonS3Client)(bucket, prefix)
// assert(result.isLeft)
// }
// }
// "when Amazon SDK Client Exception" in {
// val exception = new SdkClientException("message")
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3Client.listObjectsV2)
// .when()
// .returns(_ => Task.fail(exception))
// private val result = invoke(fixture.amazonS3Client)(bucket, prefix)
// assert(result.isLeft)
// }
// }
// def invoke(amazonS3Client: AmazonS3Client.Client)(bucket: Bucket,
// prefix: RemoteKey) = {
// object TestEnv extends Storage.Test
// val program: RIO[Storage, RemoteObjects] = Lister
// .listObjects(amazonS3Client)(bucket, prefix)
// val runtime = new DefaultRuntime {}
// runtime.unsafeRunSync(program.provide(TestEnv)).toEither
// }
//
// }
}

11
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/MD5HashData.scala
View file

@ -1,11 +0,0 @@
package net.kemitix.thorp.storage.aws
import net.kemitix.thorp.domain.MD5Hash
object MD5HashData {
val rootHash = MD5Hash.create("a3a6ac11a0eb577b81b3bb5c95cc8a6e")
val leafHash = MD5Hash.create("208386a650bdec61cfcd7bd8dcb6b542")
}

35
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala
View file

@ -1,35 +0,0 @@
package net.kemitix.thorp.storage.aws
import scala.jdk.CollectionConverters._
import com.amazonaws.services.s3.model.S3ObjectSummary
import net.kemitix.thorp.domain.{MD5Hash, RemoteKey}
import org.scalatest.FunSpec
class S3ObjectsByHashSuite extends FunSpec {
describe("grouping s3 object together by their hash values") {
val hash = MD5Hash.create("hash")
val key1 = RemoteKey.create("key-1")
val key2 = RemoteKey.create("key-2")
val o1 = s3object(hash, key1)
val o2 = s3object(hash, key2)
val os = List(o1, o2)
it("should group by the hash value") {
val expected: Map[MD5Hash, RemoteKey] = Map(
hash -> key2
)
val result = Map.from(S3Lister.byHash(os.asJava).asScala)
assertResult(expected)(result)
}
}
private def s3object(md5Hash: MD5Hash,
remoteKey: RemoteKey): S3ObjectSummary = {
val summary = new S3ObjectSummary()
summary.setETag(md5Hash.hash())
summary.setKey(remoteKey.key)
summary
}
}

109
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala
View file

@ -1,109 +0,0 @@
package net.kemitix.thorp.storage.aws
import org.scalamock.scalatest.MockFactory
import org.scalatest.FreeSpec
class UploaderTest extends FreeSpec with MockFactory {
// val uiChannel: UChannel[Any, UIEvent] = zioMessage => ()
//
// "upload" - {
// val aSource: File = Resource(this, "").toFile
// val aFile: File = Resource(this, "small-file").toFile
// val aHash = MD5Hash.create("aHash")
// val hashes = Hashes.create(MD5, aHash)
// val remoteKey = RemoteKey.create("aRemoteKey")
// val localFile =
// LocalFile.create(aFile, aSource, hashes, remoteKey, aFile.length)
// val bucket = Bucket.named("aBucket")
// val uploadResult = new UploadResult
// uploadResult.setKey(remoteKey.key)
// uploadResult.setETag(aHash.hash())
// val listenerSettings =
// UploadEventListener.Settings(uiChannel, localFile, 0, 0, batchMode = true)
// "when no error" in {
// val expected =
// Right(StorageEvent.uploadEvent(remoteKey, aHash))
// val inProgress = new AmazonUpload.InProgress {
// override def waitForUploadResult: Task[UploadResult] =
// Task(uploadResult)
// }
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3TransferManager.uploader)
// .when()
// .returns(_ => UIO.succeed(inProgress))
// private val result =
// invoke(fixture.amazonS3TransferManager)(
// localFile,
// bucket,
// listenerSettings
// )
// assertResult(expected)(result)
// }
// }
// "when Amazon Service Exception" in {
// val exception = new AmazonS3Exception("message")
// val expected =
// Right(
// StorageEvent.errorEvent(ActionSummary.upload(remoteKey.key),
// remoteKey,
// exception))
// val inProgress = new AmazonUpload.InProgress {
// override def waitForUploadResult: Task[UploadResult] =
// Task.fail(exception)
// }
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3TransferManager.upload)
// .when()
// .returns(_ => UIO.succeed(inProgress))
// private val result =
// invoke(fixture.amazonS3TransferManager)(
// localFile,
// bucket,
// listenerSettings
// )
// assertResult(expected)(result)
// }
// }
// "when Amazon SDK Client Exception" in {
// val exception = new SdkClientException("message")
// val expected =
// Right(
// StorageEvent.errorEvent(ActionSummary.upload(remoteKey.key),
// remoteKey,
// exception))
// val inProgress = new AmazonUpload.InProgress {
// override def waitForUploadResult: Task[UploadResult] =
// Task.fail(exception)
// }
// new AmazonS3ClientTestFixture {
// (() => fixture.amazonS3TransferManager.upload)
// .when()
// .returns(_ => UIO.succeed(inProgress))
// private val result =
// invoke(fixture.amazonS3TransferManager)(
// localFile,
// bucket,
// listenerSettings
// )
// assertResult(expected)(result)
// }
// }
// def invoke(transferManager: AmazonTransferManager)(
// localFile: LocalFile,
// bucket: Bucket,
// listenerSettings: UploadEventListener.Settings
// ) = {
// val program = Uploader
// .upload(transferManager)(
// Uploader.Request(localFile, bucket, listenerSettings))
// val runtime = new DefaultRuntime {}
// runtime
// .unsafeRunSync(
// program
// .provide(Config.Live))
// .toEither
// }
// }
}

56
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala
View file

@ -1,56 +0,0 @@
package net.kemitix.thorp.storage.aws.hasher
import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
import net.kemitix.thorp.filesystem.Resource
import org.scalatest.FreeSpec
class ETagGeneratorTest extends FreeSpec {
private val bigFile = Resource.select(this, "../big-file")
private val bigFilePath = bigFile.toPath
private val configuration = new TransferManagerConfiguration
private val chunkSize = 1200000
configuration.setMinimumUploadPartSize(chunkSize)
// "Create offsets" - {
// "should create offsets" in {
// val offsets = S3ETagGenerator
// .offsets(bigFile.length, chunkSize)
// .foldRight(List[Long]())((l: Long, a: List[Long]) => l :: a)
// assertResult(
// List(0, chunkSize, chunkSize * 2, chunkSize * 3, chunkSize * 4))(
// offsets)
// }
// }
// "create md5 hash for each chunk" - {
// "should create expected hash for chunks" in {
// val md5Hashes = List(
// "68b7d37e6578297621e06f01800204f1",
// "973475b14a7bda6ad8864a7f9913a947",
// "b9adcfc5b103fe2dd5924a5e5e6817f0",
// "5bd6e10a99fef100fe7bf5eaa0a42384",
// "8a0c1d0778ac8fcf4ca2010eba4711eb"
// ).zipWithIndex
// md5Hashes.foreach {
// case (hash, index) =>
// val program = Hasher.hashObjectChunk(bigFilePath, index, chunkSize)
// val result = runtime.unsafeRunSync(program.provide(TestEnv)).toEither
// assertResult(Right(hash))(
// result
// .map(hashes => hashes.get(MD5).get())
// .map(x => x.hash))
// }
// }
// }
// "create etag for whole file" - {
// val expected = "f14327c90ad105244c446c498bfe9a7d-2"
// "should match aws etag for the file" in {
// val program = ETagGenerator.eTag(bigFilePath)
// val result = runtime.unsafeRunSync(program.provide(TestEnv)).toEither
// assertResult(Right(expected))(result)
// }
// }
}