From 5b397ce1810541c7c22119b2c45be3168b804e1e Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 10 May 2019 22:35:01 +0100 Subject: [PATCH] [s3client] upload returns S3's md5hash of the uploaded file --- src/main/scala/net/kemitix/s3thorp/Sync.scala | 2 +- .../scala/net/kemitix/s3thorp/awssdk/S3Client.scala | 2 +- .../net/kemitix/s3thorp/awssdk/ThropS3Client.scala | 10 ++++------ .../scala/net/kemitix/s3thorp/DummyS3Client.scala | 2 +- .../scala/net/kemitix/s3thorp/S3UploaderSuite.scala | 8 +++++--- .../net/kemitix/s3thorp/awssdk/S3ClientSuite.scala | 13 +++++++------ 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/main/scala/net/kemitix/s3thorp/Sync.scala b/src/main/scala/net/kemitix/s3thorp/Sync.scala index 919c554..a95421c 100644 --- a/src/main/scala/net/kemitix/s3thorp/Sync.scala +++ b/src/main/scala/net/kemitix/s3thorp/Sync.scala @@ -17,7 +17,7 @@ class Sync(s3Client: S3Client) override def objectHead(bucket: Bucket, remoteKey: RemoteKey)= s3Client.objectHead(bucket, remoteKey) - override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = + override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey) = s3Client.upload(localFile, bucket, remoteKey) def run(c: Config): IO[Unit] = for { diff --git a/src/main/scala/net/kemitix/s3thorp/awssdk/S3Client.scala b/src/main/scala/net/kemitix/s3thorp/awssdk/S3Client.scala index 5c788cb..e5ea8d8 100644 --- a/src/main/scala/net/kemitix/s3thorp/awssdk/S3Client.scala +++ b/src/main/scala/net/kemitix/s3thorp/awssdk/S3Client.scala @@ -8,7 +8,7 @@ trait S3Client { def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] - def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] + def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Either[Throwable, MD5Hash]] } diff --git a/src/main/scala/net/kemitix/s3thorp/awssdk/ThropS3Client.scala b/src/main/scala/net/kemitix/s3thorp/awssdk/ThropS3Client.scala index 6287f3e..934e2ca 100644 --- a/src/main/scala/net/kemitix/s3thorp/awssdk/ThropS3Client.scala +++ b/src/main/scala/net/kemitix/s3thorp/awssdk/ThropS3Client.scala @@ -2,13 +2,13 @@ package net.kemitix.s3thorp.awssdk import cats.effect.IO import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient -import net.kemitix.s3thorp.Sync.{Bucket, LastModified, LocalFile, MD5Hash, RemoteKey} +import net.kemitix.s3thorp.Sync._ import software.amazon.awssdk.core.async.AsyncRequestBody import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, PutObjectRequest} private class ThropS3Client(s3Client: S3CatsIOClient) extends S3Client { - def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] = { + def objectHead(bucket: Bucket, remoteKey: RemoteKey) = { val request = HeadObjectRequest.builder().bucket(bucket).key(remoteKey).build() s3Client.headObject(request).attempt.map { case Right(response) => Some((response.eTag(), response.lastModified())) @@ -16,12 +16,10 @@ private class ThropS3Client(s3Client: S3CatsIOClient) extends S3Client { } } - def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = { + def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Either[Throwable, MD5Hash]] = { val request = PutObjectRequest.builder().bucket(bucket).key(remoteKey).build() val body = AsyncRequestBody.fromFile(localFile) - for { - _ <- s3Client.putObject(request, body) - } yield () + s3Client.putObject(request, body).map{response => Right(response.eTag())} } } diff --git a/src/test/scala/net/kemitix/s3thorp/DummyS3Client.scala b/src/test/scala/net/kemitix/s3thorp/DummyS3Client.scala index 78f86be..a83bdbd 100644 --- a/src/test/scala/net/kemitix/s3thorp/DummyS3Client.scala +++ b/src/test/scala/net/kemitix/s3thorp/DummyS3Client.scala @@ -8,5 +8,5 @@ trait DummyS3Client extends S3Client { override def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] = ??? - override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = ??? + override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Either[Throwable, MD5Hash]] = ??? } diff --git a/src/test/scala/net/kemitix/s3thorp/S3UploaderSuite.scala b/src/test/scala/net/kemitix/s3thorp/S3UploaderSuite.scala index 976a9f4..5c1afd1 100644 --- a/src/test/scala/net/kemitix/s3thorp/S3UploaderSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/S3UploaderSuite.scala @@ -3,14 +3,16 @@ package net.kemitix.s3thorp import java.io.File import cats.effect.IO -import net.kemitix.s3thorp.Sync.{Bucket, LastModified, LocalFile, MD5Hash, RemoteKey} +import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey} import org.scalatest.FunSpec class S3UploaderSuite extends FunSpec { new S3Uploader { - override def objectHead(bucket: String, key: String): IO[Option[(MD5Hash, LastModified)]] = ??? - override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = IO.unit + val md5Hash = "the-md5hash" + override def objectHead(bucket: String, key: String) = ??? + override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey) = + IO(Right(md5Hash)) describe("upload") { val config: Config = Config("bucket", "prefix", new File("/path/to/files")) diff --git a/src/test/scala/net/kemitix/s3thorp/awssdk/S3ClientSuite.scala b/src/test/scala/net/kemitix/s3thorp/awssdk/S3ClientSuite.scala index 8479fbb..e4d63c0 100644 --- a/src/test/scala/net/kemitix/s3thorp/awssdk/S3ClientSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/awssdk/S3ClientSuite.scala @@ -4,11 +4,10 @@ import java.io.File import java.time.Instant import cats.effect.IO -import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey} import org.scalatest.FunSpec -import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, HeadObjectResponse, NoSuchKeyException, PutObjectRequest, PutObjectResponse} +import software.amazon.awssdk.services.s3.model._ class S3ClientSuite extends FunSpec { @@ -44,19 +43,21 @@ class S3ClientSuite extends FunSpec { } - describe("upload") { + def invoke(s3Client: ThropS3Client, localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey) = + s3Client.upload(localFile, bucket, remoteKey).unsafeRunSync describe("when uploading a file") { + val md5Hash = "the-md5hash" val s3Client = new ThropS3Client( new S3CatsIOClient with JavaClientWrapper { override def putObject(putObjectRequest: PutObjectRequest, requestBody: RB) = - IO(PutObjectResponse.builder().build()) + IO(PutObjectResponse.builder().eTag(md5Hash).build()) }) val localFile: LocalFile = new File("/some/file") val bucket: Bucket = "a-bucket" val remoteKey: RemoteKey = "prefix/file" - it("should not throw any exceptions") { - s3Client.upload(localFile, bucket, remoteKey) + it("should return hash of uploaded file") { + assertResult(Right(md5Hash))(invoke(s3Client, localFile, bucket, remoteKey)) } } }