[s3client] Add upload method
Add DummyS3Client to help tests
This commit is contained in:
parent
e8a656ae4c
commit
b348f18142
6 changed files with 36 additions and 6 deletions
|
@ -17,6 +17,9 @@ class Sync(s3Client: S3Client)
|
||||||
override def objectHead(bucket: Bucket, remoteKey: RemoteKey)=
|
override def objectHead(bucket: Bucket, remoteKey: RemoteKey)=
|
||||||
s3Client.objectHead(bucket, remoteKey)
|
s3Client.objectHead(bucket, remoteKey)
|
||||||
|
|
||||||
|
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] =
|
||||||
|
s3Client.upload(localFile, bucket, remoteKey)
|
||||||
|
|
||||||
def run(c: Config): IO[Unit] = for {
|
def run(c: Config): IO[Unit] = for {
|
||||||
_ <- putStrLn(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
|
_ <- putStrLn(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
|
||||||
_ <- {
|
_ <- {
|
||||||
|
|
|
@ -3,8 +3,9 @@ package net.kemitix.s3thorp.awssdk
|
||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
|
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
|
||||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||||
import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, NoSuchKeyException}
|
|
||||||
import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey}
|
import net.kemitix.s3thorp.Sync.{Bucket, LocalFile, RemoteKey}
|
||||||
|
import software.amazon.awssdk.core.async.AsyncRequestBody
|
||||||
|
import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, NoSuchKeyException, PutObjectRequest}
|
||||||
import software.amazon.awssdk.services.s3.{S3AsyncClient => JavaS3AsyncClient}
|
import software.amazon.awssdk.services.s3.{S3AsyncClient => JavaS3AsyncClient}
|
||||||
|
|
||||||
private class ReactiveS3Client extends S3Client {
|
private class ReactiveS3Client extends S3Client {
|
||||||
|
@ -24,4 +25,17 @@ private class ReactiveS3Client extends S3Client {
|
||||||
case _: NoSuchKeyException => IO(None)
|
case _: NoSuchKeyException => IO(None)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = {
|
||||||
|
val request = PutObjectRequest.builder()
|
||||||
|
.bucket(bucket)
|
||||||
|
.key(remoteKey)
|
||||||
|
.build()
|
||||||
|
val body = AsyncRequestBody.fromFile(localFile)
|
||||||
|
try {
|
||||||
|
for {
|
||||||
|
_ <- s3Client.putObject(request, body)
|
||||||
|
} yield ()
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,8 @@ trait S3Client {
|
||||||
|
|
||||||
def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]]
|
def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]]
|
||||||
|
|
||||||
|
def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit]
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
object S3Client {
|
object S3Client {
|
||||||
|
|
12
src/test/scala/net/kemitix/s3thorp/DummyS3Client.scala
Normal file
12
src/test/scala/net/kemitix/s3thorp/DummyS3Client.scala
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
package net.kemitix.s3thorp
|
||||||
|
|
||||||
|
import cats.effect.IO
|
||||||
|
import net.kemitix.s3thorp.Sync.{Bucket, LastModified, LocalFile, MD5Hash, RemoteKey}
|
||||||
|
import net.kemitix.s3thorp.awssdk.S3Client
|
||||||
|
|
||||||
|
trait DummyS3Client extends S3Client {
|
||||||
|
|
||||||
|
override def objectHead(bucket: Bucket, remoteKey: RemoteKey): IO[Option[(MD5Hash, LastModified)]] = ???
|
||||||
|
|
||||||
|
override def upload(localFile: LocalFile, bucket: Bucket, remoteKey: RemoteKey): IO[Unit] = ???
|
||||||
|
}
|
|
@ -14,7 +14,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
private val prefix = "prefix"
|
private val prefix = "prefix"
|
||||||
private val config = Config("bucket", prefix, source)
|
private val config = Config("bucket", prefix, source)
|
||||||
|
|
||||||
new S3MetaDataEnricher {
|
new S3MetaDataEnricher with DummyS3Client {
|
||||||
describe("key generator") {
|
describe("key generator") {
|
||||||
val subject = generateKey(config)_
|
val subject = generateKey(config)_
|
||||||
|
|
||||||
|
@ -36,7 +36,6 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
override def objectHead(bucket: String, key: String) = ???
|
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("enrich with metadata") {
|
describe("enrich with metadata") {
|
||||||
|
@ -45,7 +44,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
describe("when remote exists") {
|
describe("when remote exists") {
|
||||||
val hash = "hash"
|
val hash = "hash"
|
||||||
val lastModified = Instant.now()
|
val lastModified = Instant.now()
|
||||||
new S3MetaDataEnricher {
|
new S3MetaDataEnricher with DummyS3Client {
|
||||||
override def objectHead(bucket: String, key: String) = IO(Some((hash, lastModified)))
|
override def objectHead(bucket: String, key: String) = IO(Some((hash, lastModified)))
|
||||||
it("returns metadata") {
|
it("returns metadata") {
|
||||||
val expectedMetadata = S3MetaData(localFile, s"$prefix/$local", hash, lastModified)
|
val expectedMetadata = S3MetaData(localFile, s"$prefix/$local", hash, lastModified)
|
||||||
|
@ -56,7 +55,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("when remote doesn't exist") {
|
describe("when remote doesn't exist") {
|
||||||
new S3MetaDataEnricher {
|
new S3MetaDataEnricher with DummyS3Client {
|
||||||
override def objectHead(bucket: String, key: String) = IO(None)
|
override def objectHead(bucket: String, key: String) = IO(None)
|
||||||
it("returns file to upload") {
|
it("returns file to upload") {
|
||||||
val result = enrichWithS3MetaData(config)(localFile).compile.toList.unsafeRunSync().head
|
val result = enrichWithS3MetaData(config)(localFile).compile.toList.unsafeRunSync().head
|
||||||
|
|
|
@ -11,7 +11,7 @@ class SyncSuite extends FunSpec {
|
||||||
describe("s3client thunk") {
|
describe("s3client thunk") {
|
||||||
val hash = "hash"
|
val hash = "hash"
|
||||||
val lastModified = Instant.now()
|
val lastModified = Instant.now()
|
||||||
val sync = new Sync(new S3Client {
|
val sync = new Sync(new S3Client with DummyS3Client {
|
||||||
override def objectHead(bucket: String, key: String) = IO(Some((hash, lastModified)))
|
override def objectHead(bucket: String, key: String) = IO(Some((hash, lastModified)))
|
||||||
})
|
})
|
||||||
describe("objectHead") {
|
describe("objectHead") {
|
||||||
|
|
Loading…
Reference in a new issue