[s3client] upload returns S3's md5hash of the uploaded file
This commit is contained in:
parent
41e38f5cee
commit
5b397ce181
6 changed files with 19 additions and 18 deletions
|
@ -17,7 +17,7 @@ class Sync(s3Client: S3Client)
|
||||||
override def objectHead(bucket: Bucket, remoteKey: RemoteKey)=
|
override def objectHead(bucket: Bucket, remoteKey: RemoteKey)=
|
||||||
s3Client.objectHead(bucket, remoteKey)
|
s3Client.objectHead(bucket, remoteKey)
|
||||||
|
|
||||||
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] =
|
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey) =
|
||||||
s3Client.upload(localFile, bucket, remoteKey)
|
s3Client.upload(localFile, bucket, remoteKey)
|
||||||
|
|
||||||
def run(c: Config): IO[Unit] = for {
|
def run(c: Config): IO[Unit] = for {
|
||||||
|
|
|
@ -8,7 +8,7 @@ trait S3Client {
|
||||||
|
|
||||||
def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]]
|
def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]]
|
||||||
|
|
||||||
def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit]
|
def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Either[Throwable, MD5Hash]]
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,13 +2,13 @@ package net.kemitix.s3thorp.awssdk
|
||||||
|
|
||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||||
import net.kemitix.s3thorp.Sync.{Bucket, LastModified, LocalFile, MD5Hash, RemoteKey}
|
import net.kemitix.s3thorp.Sync._
|
||||||
import software.amazon.awssdk.core.async.AsyncRequestBody
|
import software.amazon.awssdk.core.async.AsyncRequestBody
|
||||||
import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, PutObjectRequest}
|
import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, PutObjectRequest}
|
||||||
|
|
||||||
private class ThropS3Client(s3Client: S3CatsIOClient) extends S3Client {
|
private class ThropS3Client(s3Client: S3CatsIOClient) extends S3Client {
|
||||||
|
|
||||||
def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] = {
|
def objectHead(bucket: Bucket, remoteKey: RemoteKey) = {
|
||||||
val request = HeadObjectRequest.builder().bucket(bucket).key(remoteKey).build()
|
val request = HeadObjectRequest.builder().bucket(bucket).key(remoteKey).build()
|
||||||
s3Client.headObject(request).attempt.map {
|
s3Client.headObject(request).attempt.map {
|
||||||
case Right(response) => Some((response.eTag(), response.lastModified()))
|
case Right(response) => Some((response.eTag(), response.lastModified()))
|
||||||
|
@ -16,12 +16,10 @@ private class ThropS3Client(s3Client: S3CatsIOClient) extends S3Client {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = {
|
def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Either[Throwable, MD5Hash]] = {
|
||||||
val request = PutObjectRequest.builder().bucket(bucket).key(remoteKey).build()
|
val request = PutObjectRequest.builder().bucket(bucket).key(remoteKey).build()
|
||||||
val body = AsyncRequestBody.fromFile(localFile)
|
val body = AsyncRequestBody.fromFile(localFile)
|
||||||
for {
|
s3Client.putObject(request, body).map{response => Right(response.eTag())}
|
||||||
_ <- s3Client.putObject(request, body)
|
|
||||||
} yield ()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,5 +8,5 @@ trait DummyS3Client extends S3Client {
|
||||||
|
|
||||||
override def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] = ???
|
override def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] = ???
|
||||||
|
|
||||||
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = ???
|
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Either[Throwable, MD5Hash]] = ???
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,14 +3,16 @@ package net.kemitix.s3thorp
|
||||||
import java.io.File
|
import java.io.File
|
||||||
|
|
||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
import net.kemitix.s3thorp.Sync.{Bucket, LastModified, LocalFile, MD5Hash, RemoteKey}
|
import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey}
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
|
||||||
class S3UploaderSuite extends FunSpec {
|
class S3UploaderSuite extends FunSpec {
|
||||||
|
|
||||||
new S3Uploader {
|
new S3Uploader {
|
||||||
override def objectHead(bucket: String, key: String): IO[Option[(MD5Hash, LastModified)]] = ???
|
val md5Hash = "the-md5hash"
|
||||||
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = IO.unit
|
override def objectHead(bucket: String, key: String) = ???
|
||||||
|
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey) =
|
||||||
|
IO(Right(md5Hash))
|
||||||
|
|
||||||
describe("upload") {
|
describe("upload") {
|
||||||
val config: Config = Config("bucket", "prefix", new File("/path/to/files"))
|
val config: Config = Config("bucket", "prefix", new File("/path/to/files"))
|
||||||
|
|
|
@ -4,11 +4,10 @@ import java.io.File
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
|
|
||||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||||
import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey}
|
import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey}
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, HeadObjectResponse, NoSuchKeyException, PutObjectRequest, PutObjectResponse}
|
import software.amazon.awssdk.services.s3.model._
|
||||||
|
|
||||||
class S3ClientSuite extends FunSpec {
|
class S3ClientSuite extends FunSpec {
|
||||||
|
|
||||||
|
@ -44,19 +43,21 @@ class S3ClientSuite extends FunSpec {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
describe("upload") {
|
describe("upload") {
|
||||||
|
def invoke(s3Client: ThropS3Client, localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey) =
|
||||||
|
s3Client.upload(localFile, bucket, remoteKey).unsafeRunSync
|
||||||
describe("when uploading a file") {
|
describe("when uploading a file") {
|
||||||
|
val md5Hash = "the-md5hash"
|
||||||
val s3Client = new ThropS3Client(
|
val s3Client = new ThropS3Client(
|
||||||
new S3CatsIOClient with JavaClientWrapper {
|
new S3CatsIOClient with JavaClientWrapper {
|
||||||
override def putObject(putObjectRequest: PutObjectRequest, requestBody: RB) =
|
override def putObject(putObjectRequest: PutObjectRequest, requestBody: RB) =
|
||||||
IO(PutObjectResponse.builder().build())
|
IO(PutObjectResponse.builder().eTag(md5Hash).build())
|
||||||
})
|
})
|
||||||
val localFile: LocalFile = new File("/some/file")
|
val localFile: LocalFile = new File("/some/file")
|
||||||
val bucket: Bucket = "a-bucket"
|
val bucket: Bucket = "a-bucket"
|
||||||
val remoteKey: RemoteKey = "prefix/file"
|
val remoteKey: RemoteKey = "prefix/file"
|
||||||
it("should not throw any exceptions") {
|
it("should return hash of uploaded file") {
|
||||||
s3Client.upload(localFile, bucket, remoteKey)
|
assertResult(Right(md5Hash))(invoke(s3Client, localFile, bucket, remoteKey))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue