kemitix/thorp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[s3client] upload returns S3's md5hash of the uploaded file

Browse source
This commit is contained in:
Paul Campbell 2019-05-10 22:35:01 +01:00
parent 41e38f5cee
commit 5b397ce181
6 changed files with 19 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/scala/net/kemitix/s3thorp/Sync.scala
View file

@ -17,7 +17,7 @@ class Sync(s3Client: S3Client)
override def objectHead(bucket: Bucket, remoteKey: RemoteKey)= override def objectHead(bucket: Bucket, remoteKey: RemoteKey)=
s3Client.objectHead(bucket, remoteKey) s3Client.objectHead(bucket, remoteKey)
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey) =
s3Client.upload(localFile, bucket, remoteKey) s3Client.upload(localFile, bucket, remoteKey)
def run(c: Config): IO[Unit] = for { def run(c: Config): IO[Unit] = for {

2
src/main/scala/net/kemitix/s3thorp/awssdk/S3Client.scala
View file

@ -8,7 +8,7 @@ trait S3Client {
def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]]
def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Either[Throwable, MD5Hash]]
} }

10
src/main/scala/net/kemitix/s3thorp/awssdk/ThropS3Client.scala
View file

@ -2,13 +2,13 @@ package net.kemitix.s3thorp.awssdk
import cats.effect.IO import cats.effect.IO
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
import net.kemitix.s3thorp.Sync.{Bucket, LastModified, LocalFile, MD5Hash, RemoteKey} import net.kemitix.s3thorp.Sync._
import software.amazon.awssdk.core.async.AsyncRequestBody import software.amazon.awssdk.core.async.AsyncRequestBody
import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, PutObjectRequest} import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, PutObjectRequest}
private class ThropS3Client(s3Client: S3CatsIOClient) extends S3Client { private class ThropS3Client(s3Client: S3CatsIOClient) extends S3Client {
def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] = { def objectHead(bucket: Bucket, remoteKey: RemoteKey) = {
val request = HeadObjectRequest.builder().bucket(bucket).key(remoteKey).build() val request = HeadObjectRequest.builder().bucket(bucket).key(remoteKey).build()
s3Client.headObject(request).attempt.map { s3Client.headObject(request).attempt.map {
case Right(response) => Some((response.eTag(), response.lastModified())) case Right(response) => Some((response.eTag(), response.lastModified()))
@ -16,12 +16,10 @@ private class ThropS3Client(s3Client: S3CatsIOClient) extends S3Client {
} }
} }
def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = { def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Either[Throwable, MD5Hash]] = {
val request = PutObjectRequest.builder().bucket(bucket).key(remoteKey).build() val request = PutObjectRequest.builder().bucket(bucket).key(remoteKey).build()
val body = AsyncRequestBody.fromFile(localFile) val body = AsyncRequestBody.fromFile(localFile)
for { s3Client.putObject(request, body).map{response => Right(response.eTag())}
_ <- s3Client.putObject(request, body)
} yield ()
} }
} }

2
src/test/scala/net/kemitix/s3thorp/DummyS3Client.scala
View file

@ -8,5 +8,5 @@ trait DummyS3Client extends S3Client {
override def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] = ??? override def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] = ???
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = ??? override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Either[Throwable, MD5Hash]] = ???
} }

8
src/test/scala/net/kemitix/s3thorp/S3UploaderSuite.scala
View file

@ -3,14 +3,16 @@ package net.kemitix.s3thorp
import java.io.File import java.io.File
import cats.effect.IO import cats.effect.IO
import net.kemitix.s3thorp.Sync.{Bucket, LastModified, LocalFile, MD5Hash, RemoteKey} import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey}
import org.scalatest.FunSpec import org.scalatest.FunSpec
class S3UploaderSuite extends FunSpec { class S3UploaderSuite extends FunSpec {
new S3Uploader { new S3Uploader {
override def objectHead(bucket: String, key: String): IO[Option[(MD5Hash, LastModified)]] = ??? val md5Hash = "the-md5hash"
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = IO.unit override def objectHead(bucket: String, key: String) = ???
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey) =
IO(Right(md5Hash))
describe("upload") { describe("upload") {
val config: Config = Config("bucket", "prefix", new File("/path/to/files")) val config: Config = Config("bucket", "prefix", new File("/path/to/files"))

13
src/test/scala/net/kemitix/s3thorp/awssdk/S3ClientSuite.scala
View file

@ -4,11 +4,10 @@ import java.io.File
import java.time.Instant import java.time.Instant
import cats.effect.IO import cats.effect.IO
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey} import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey}
import org.scalatest.FunSpec import org.scalatest.FunSpec
import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, HeadObjectResponse, NoSuchKeyException, PutObjectRequest, PutObjectResponse} import software.amazon.awssdk.services.s3.model._
class S3ClientSuite extends FunSpec { class S3ClientSuite extends FunSpec {
@ -44,19 +43,21 @@ class S3ClientSuite extends FunSpec {
} }
describe("upload") { describe("upload") {
def invoke(s3Client: ThropS3Client, localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey) =
s3Client.upload(localFile, bucket, remoteKey).unsafeRunSync
describe("when uploading a file") { describe("when uploading a file") {
val md5Hash = "the-md5hash"
val s3Client = new ThropS3Client( val s3Client = new ThropS3Client(
new S3CatsIOClient with JavaClientWrapper { new S3CatsIOClient with JavaClientWrapper {
override def putObject(putObjectRequest: PutObjectRequest, requestBody: RB) = override def putObject(putObjectRequest: PutObjectRequest, requestBody: RB) =
IO(PutObjectResponse.builder().build()) IO(PutObjectResponse.builder().eTag(md5Hash).build())
}) })
val localFile: LocalFile = new File("/some/file") val localFile: LocalFile = new File("/some/file")
val bucket: Bucket = "a-bucket" val bucket: Bucket = "a-bucket"
val remoteKey: RemoteKey = "prefix/file" val remoteKey: RemoteKey = "prefix/file"
it("should not throw any exceptions") { it("should return hash of uploaded file") {
s3Client.upload(localFile, bucket, remoteKey) assertResult(Right(md5Hash))(invoke(s3Client, localFile, bucket, remoteKey))
} }
} }
} }