Display upload progress (#29)
* [S3ClientMultiPartTransferManager] use request object
* [ActionSubmitter] unwrap RemoteKey in log messages
* [ActionSubmitter] rename variable
* [Logging] include log level in info messages
* [LocalFileStream] log when entering directory at level 2
* [UploadProgress{Listener,Logging}: add initial implementations
* [S3Client] def upload not requires an UploadProgressListener as a parameter
* [UploadProgressListener] rename method
* [S3ClientPutObjectUploader] Log upload progress for file <5Mb
Switched to using the AWS SDK V1 for PutObject as the V2 doesn't
support progress callbacks.
* Fix up tests
* Adjust logging levels
This commit is contained in:
GitHub
parent
602c5ef150
commit
574d4c5885
21 changed files with 587 additions and 93 deletions
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp
|
||||
|
||||
import cats.effect.IO
|
||||
import net.kemitix.s3thorp.awssdk.S3Client
|
||||
import net.kemitix.s3thorp.awssdk.{S3Client, UploadProgressListener}
|
||||
|
||||
trait ActionSubmitter
|
||||
extends S3Client
|
||||
|
|
@ -11,14 +11,15 @@ trait ActionSubmitter
|
|||
(implicit c: Config): Stream[IO[S3Action]] = {
|
||||
Stream(
|
||||
action match {
|
||||
case ToUpload(file) =>
|
||||
log4(s" Upload: ${file.relative}")
|
||||
upload(file, c.bucket, 1)
|
||||
case ToUpload(localFile) =>
|
||||
log4(s" Upload: ${localFile.relative}")
|
||||
val progressListener = new UploadProgressListener(localFile)
|
||||
upload(localFile, c.bucket, progressListener, 1)
|
||||
case ToCopy(sourceKey, hash, targetKey) =>
|
||||
log4(s" Copy: $sourceKey => $targetKey")
|
||||
log4(s" Copy: ${sourceKey.key} => ${targetKey.key}")
|
||||
copy(c.bucket, sourceKey, hash, targetKey)
|
||||
case ToDelete(remoteKey) =>
|
||||
log4(s" Delete: $remoteKey")
|
||||
log4(s" Delete: ${remoteKey.key}")
|
||||
delete(c.bucket, remoteKey)
|
||||
case DoNothing(remoteKey) => IO {
|
||||
DoNothingS3Action(remoteKey)}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ trait LocalFileStream
|
|||
|
||||
def findFiles(file: File)
|
||||
(implicit c: Config): Stream[LocalFile] = {
|
||||
log5(s"- Entering: $file")
|
||||
log2(s"- Entering: $file")
|
||||
val files = for {
|
||||
f <- dirPaths(file)
|
||||
.filter { f => f.isDirectory || c.filters.forall { filter => filter isIncluded f.toPath } }
|
||||
|
|
|
|||
|
|
@ -4,18 +4,18 @@ import com.typesafe.scalalogging.LazyLogging
|
|||
|
||||
trait Logging extends LazyLogging {
|
||||
|
||||
def log1(message: String)(implicit config: Config): Unit = if (config.verbose >= 1) logger.info(message)
|
||||
def log1(message: String)(implicit config: Config): Unit = if (config.verbose >= 1) logger.info(s"1:$message")
|
||||
|
||||
def log2(message: String)(implicit config: Config): Unit = if (config.verbose >= 2) logger.info(message)
|
||||
def log2(message: String)(implicit config: Config): Unit = if (config.verbose >= 2) logger.info(s"2:$message")
|
||||
|
||||
def log3(message: String)(implicit config: Config): Unit = if (config.verbose >= 3) logger.info(message)
|
||||
def log3(message: String)(implicit config: Config): Unit = if (config.verbose >= 3) logger.info(s"3:$message")
|
||||
|
||||
def log4(message: String)(implicit config: Config): Unit = if (config.verbose >= 4) logger.info(message)
|
||||
def log4(message: String)(implicit config: Config): Unit = if (config.verbose >= 4) logger.info(s"4:$message")
|
||||
|
||||
def log5(message: String)(implicit config: Config): Unit = if (config.verbose >= 5) logger.info(message)
|
||||
def log5(message: String)(implicit config: Config): Unit = if (config.verbose >= 5) logger.info(s"5:$message")
|
||||
|
||||
def warn(message: String): Unit = logger.warn(message)
|
||||
def warn(message: String): Unit = logger.warn(message)
|
||||
|
||||
def error(message: String): Unit = logger.error(message)
|
||||
def error(message: String): Unit = logger.error(message)
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package net.kemitix.s3thorp
|
|||
|
||||
import cats.effect.IO
|
||||
import cats.implicits._
|
||||
import net.kemitix.s3thorp.awssdk.{S3Client, S3ObjectsData}
|
||||
import net.kemitix.s3thorp.awssdk.{S3Client, S3ObjectsData, UploadProgressListener}
|
||||
|
||||
class Sync(s3Client: S3Client)
|
||||
extends LocalFileStream
|
||||
|
|
@ -39,9 +39,10 @@ class Sync(s3Client: S3Client)
|
|||
|
||||
override def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int)
|
||||
(implicit c: Config): IO[S3Action] =
|
||||
s3Client.upload(localFile, bucket, tryCount)
|
||||
s3Client.upload(localFile, bucket, progressListener, tryCount)
|
||||
|
||||
override def copy(bucket: Bucket,
|
||||
sourceKey: RemoteKey,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
package net.kemitix.s3thorp.awssdk
|
||||
|
||||
import cats.effect.IO
|
||||
import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client, AmazonS3ClientBuilder}
|
||||
import com.amazonaws.services.s3.transfer.{TransferManager, TransferManagerBuilder}
|
||||
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
|
||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||
import net.kemitix.s3thorp._
|
||||
|
|
@ -20,6 +22,7 @@ trait S3Client {
|
|||
|
||||
def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int
|
||||
)(implicit c: Config): IO[S3Action]
|
||||
|
||||
|
|
@ -37,11 +40,15 @@ trait S3Client {
|
|||
|
||||
object S3Client {
|
||||
|
||||
def createClient(s3AsyncClient: S3AsyncClient): S3Client = {
|
||||
new ThorpS3Client(S3CatsIOClient(s3AsyncClient))
|
||||
def createClient(s3AsyncClient: S3AsyncClient,
|
||||
amazonS3Client: AmazonS3,
|
||||
amazonS3TransferManager: TransferManager): S3Client = {
|
||||
new ThorpS3Client(S3CatsIOClient(s3AsyncClient), amazonS3Client, amazonS3TransferManager)
|
||||
}
|
||||
|
||||
val defaultClient: S3Client =
|
||||
createClient(new JavaClientWrapper {}.underlying)
|
||||
createClient(new JavaClientWrapper {}.underlying,
|
||||
AmazonS3ClientBuilder.defaultClient,
|
||||
TransferManagerBuilder.defaultTransferManager)
|
||||
|
||||
}
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
package net.kemitix.s3thorp.awssdk
|
||||
|
||||
import cats.effect.IO
|
||||
import com.amazonaws.services.s3.model.PutObjectResult
|
||||
import net.kemitix.s3thorp.{Bucket, Config, LocalFile, Logging, RemoteKey}
|
||||
import software.amazon.awssdk.services.s3.model.{CopyObjectResponse, DeleteObjectResponse, ListObjectsV2Response, PutObjectResponse}
|
||||
import software.amazon.awssdk.services.s3.model.{CopyObjectResponse, DeleteObjectResponse, ListObjectsV2Response}
|
||||
|
||||
trait S3ClientLogging
|
||||
extends Logging {
|
||||
|
|
@ -26,7 +27,7 @@ trait S3ClientLogging
|
|||
|
||||
def logUploadStart(localFile: LocalFile,
|
||||
bucket: Bucket)
|
||||
(implicit c: Config): PutObjectResponse => IO[PutObjectResponse] = {
|
||||
(implicit c: Config): PutObjectResult => IO[PutObjectResult] = {
|
||||
in => IO {
|
||||
log4(s"Uploading: ${bucket.name}:${localFile.remoteKey.key}")
|
||||
in
|
||||
|
|
@ -35,9 +36,9 @@ trait S3ClientLogging
|
|||
|
||||
def logUploadFinish(localFile: LocalFile,
|
||||
bucket: Bucket)
|
||||
(implicit c: Config): PutObjectResponse => IO[Unit] = {
|
||||
(implicit c: Config): PutObjectResult => IO[Unit] = {
|
||||
in =>IO {
|
||||
log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}")
|
||||
log1(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
package net.kemitix.s3thorp.awssdk
|
||||
|
||||
import cats.effect.IO
|
||||
import com.amazonaws.services.s3.model.PutObjectRequest
|
||||
import com.amazonaws.services.s3.transfer.TransferManager
|
||||
import net.kemitix.s3thorp._
|
||||
|
||||
class S3ClientMultiPartTransferManager(transferManager: TransferManager)
|
||||
class S3ClientMultiPartTransferManager(transferManager: => TransferManager)
|
||||
extends S3ClientUploader
|
||||
with S3ClientMultiPartUploaderLogging {
|
||||
|
||||
|
|
@ -14,11 +16,15 @@ class S3ClientMultiPartTransferManager(transferManager: TransferManager)
|
|||
override
|
||||
def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int)
|
||||
(implicit c: Config): IO[S3Action] = {
|
||||
val putObjectRequest: PutObjectRequest =
|
||||
new PutObjectRequest(bucket.name, localFile.remoteKey.key, localFile.file)
|
||||
.withGeneralProgressListener(progressListener.listener)
|
||||
IO {
|
||||
logMultiPartUploadStart(localFile, tryCount)
|
||||
val result = transferManager.upload(bucket.name, localFile.remoteKey.key, localFile.file)
|
||||
val result = transferManager.upload(putObjectRequest)
|
||||
.waitForUploadResult()
|
||||
logMultiPartUploadFinished(localFile)
|
||||
UploadS3Action(RemoteKey(result.getKey), MD5Hash(result.getETag))
|
||||
|
|
|
|||
|
|
@ -119,6 +119,7 @@ private class S3ClientMultiPartUploader(s3Client: AmazonS3)
|
|||
|
||||
override def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int)
|
||||
(implicit c: Config): IO[S3Action] = {
|
||||
logMultiPartUploadStart(localFile, tryCount)
|
||||
|
|
@ -136,10 +137,10 @@ private class S3ClientMultiPartUploader(s3Client: AmazonS3)
|
|||
.handleErrorWith {
|
||||
case CancellableMultiPartUpload(e, uploadId) =>
|
||||
if (tryCount >= c.maxRetries) IO(logErrorCancelling(e, localFile)) *> cancel(uploadId, localFile) *> IO.pure(ErroredS3Action(localFile.remoteKey, e))
|
||||
else IO(logErrorRetrying(e, localFile, tryCount)) *> upload(localFile, bucket, tryCount + 1)
|
||||
else IO(logErrorRetrying(e, localFile, tryCount)) *> upload(localFile, bucket, progressListener, tryCount + 1)
|
||||
case NonFatal(e) =>
|
||||
if (tryCount >= c.maxRetries) IO(logErrorUnknown(e, localFile)) *> IO.pure(ErroredS3Action(localFile.remoteKey, e))
|
||||
else IO(logErrorRetrying(e, localFile, tryCount)) *> upload(localFile, bucket, tryCount + 1)
|
||||
else IO(logErrorRetrying(e, localFile, tryCount)) *> upload(localFile, bucket, progressListener, tryCount + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ trait S3ClientMultiPartUploaderLogging
|
|||
def logMultiPartUploadStart(localFile: LocalFile,
|
||||
tryCount: Int)
|
||||
(implicit c: Config): Unit =
|
||||
log4(s"$prefix:upload:try $tryCount: ${localFile.remoteKey.key}")
|
||||
log1(s"$prefix:upload:try $tryCount: ${localFile.remoteKey.key}")
|
||||
|
||||
def logMultiPartUploadFinished(localFile: LocalFile)
|
||||
(implicit c: Config): Unit =
|
||||
|
|
@ -56,7 +56,7 @@ trait S3ClientMultiPartUploaderLogging
|
|||
uploadPartResponses: Stream[UploadPartResult],
|
||||
localFile: LocalFile)
|
||||
(implicit c: Config): Unit =
|
||||
log4(s"$prefix:completed:parts ${uploadPartResponses.size}: ${localFile.remoteKey.key}")
|
||||
log1(s"$prefix:completed:parts ${uploadPartResponses.size}: ${localFile.remoteKey.key}")
|
||||
|
||||
def logMultiPartUploadCancelling(localFile: LocalFile)
|
||||
(implicit c: Config): Unit =
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
package net.kemitix.s3thorp.awssdk
|
||||
|
||||
import cats.effect.IO
|
||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||
import net.kemitix.s3thorp.{Bucket, Config, LocalFile, MD5Hash, UploadS3Action}
|
||||
import software.amazon.awssdk.core.async.AsyncRequestBody
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest
|
||||
import com.amazonaws.services.s3.AmazonS3
|
||||
import com.amazonaws.services.s3.model.PutObjectRequest
|
||||
import net.kemitix.s3thorp._
|
||||
|
||||
private class S3ClientPutObjectUploader(s3Client: S3CatsIOClient)
|
||||
private class S3ClientPutObjectUploader(s3Client: => AmazonS3)
|
||||
extends S3ClientUploader
|
||||
with S3ClientLogging
|
||||
with QuoteStripper {
|
||||
|
|
@ -16,17 +15,17 @@ private class S3ClientPutObjectUploader(s3Client: S3CatsIOClient)
|
|||
override
|
||||
def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int)
|
||||
(implicit c: Config): IO[UploadS3Action] = {
|
||||
val request = PutObjectRequest.builder
|
||||
.bucket(bucket.name)
|
||||
.key(localFile.remoteKey.key).build
|
||||
val body = AsyncRequestBody.fromFile(localFile.file)
|
||||
s3Client.putObject(request, body)
|
||||
val request: PutObjectRequest =
|
||||
new PutObjectRequest(bucket.name, localFile.remoteKey.key, localFile.file)
|
||||
.withGeneralProgressListener(progressListener.listener)
|
||||
IO(s3Client.putObject(request))
|
||||
.bracket(
|
||||
logUploadStart(localFile, bucket))(
|
||||
logUploadFinish(localFile, bucket))
|
||||
.map(_.eTag)
|
||||
.map(_.getETag)
|
||||
.map(_ filter stripQuotes)
|
||||
.map(MD5Hash)
|
||||
.map(UploadS3Action(localFile.remoteKey, _))
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ trait S3ClientUploader {
|
|||
|
||||
def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int)
|
||||
(implicit c: Config): IO[S3Action]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,24 +1,26 @@
|
|||
package net.kemitix.s3thorp.awssdk
|
||||
|
||||
import cats.effect.IO
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerBuilder
|
||||
import com.amazonaws.services.s3.{AmazonS3Client, AmazonS3ClientBuilder}
|
||||
import com.amazonaws.services.s3.AmazonS3
|
||||
import com.amazonaws.services.s3.transfer.TransferManager
|
||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||
import net.kemitix.s3thorp._
|
||||
import software.amazon.awssdk.services.s3.model.{Bucket => _, _}
|
||||
import software.amazon.awssdk.services.s3.model.{Bucket => _}
|
||||
|
||||
private class ThorpS3Client(s3Client: S3CatsIOClient)
|
||||
private class ThorpS3Client(ioS3Client: S3CatsIOClient,
|
||||
amazonS3Client: => AmazonS3,
|
||||
amazonS3TransferManager: => TransferManager)
|
||||
extends S3Client
|
||||
with S3ClientLogging
|
||||
with QuoteStripper {
|
||||
|
||||
lazy val amazonS3Client = AmazonS3ClientBuilder.defaultClient
|
||||
lazy val amazonS3TransferManager = TransferManagerBuilder.defaultTransferManager
|
||||
lazy val objectLister = new S3ClientObjectLister(s3Client)
|
||||
lazy val copier = new S3ClientCopier(s3Client)
|
||||
lazy val uploader = new S3ClientPutObjectUploader(s3Client)
|
||||
// lazy val amazonS3Client = AmazonS3ClientBuilder.defaultClient
|
||||
// lazy val amazonS3TransferManager = TransferManagerBuilder.defaultTransferManager
|
||||
lazy val objectLister = new S3ClientObjectLister(ioS3Client)
|
||||
lazy val copier = new S3ClientCopier(ioS3Client)
|
||||
lazy val uploader = new S3ClientPutObjectUploader(amazonS3Client)
|
||||
lazy val multiPartUploader = new S3ClientMultiPartTransferManager(amazonS3TransferManager)
|
||||
lazy val deleter = new S3ClientDeleter(s3Client)
|
||||
lazy val deleter = new S3ClientDeleter(ioS3Client)
|
||||
|
||||
override def listObjects(bucket: Bucket,
|
||||
prefix: RemoteKey)
|
||||
|
|
@ -36,10 +38,12 @@ private class ThorpS3Client(s3Client: S3CatsIOClient)
|
|||
|
||||
override def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int)
|
||||
(implicit c: Config): IO[S3Action] =
|
||||
if (multiPartUploader.accepts(localFile)) multiPartUploader.upload(localFile, bucket, 1)
|
||||
else uploader.upload(localFile, bucket, tryCount)
|
||||
|
||||
if (multiPartUploader.accepts(localFile)) multiPartUploader.upload(localFile, bucket, progressListener, 1)
|
||||
else uploader.upload(localFile, bucket, progressListener, tryCount)
|
||||
|
||||
override def delete(bucket: Bucket,
|
||||
remoteKey: RemoteKey)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
package net.kemitix.s3thorp.awssdk
|
||||
|
||||
import com.amazonaws.event.{ProgressEvent, ProgressListener}
|
||||
import net.kemitix.s3thorp.{Config, LocalFile}
|
||||
|
||||
class UploadProgressListener(localFile: LocalFile)
|
||||
(implicit c: Config)
|
||||
extends UploadProgressLogging {
|
||||
|
||||
def listener: ProgressListener = new ProgressListener {
|
||||
override def progressChanged(progressEvent: ProgressEvent): Unit = {
|
||||
val eventType = progressEvent.getEventType
|
||||
if (eventType.isTransferEvent) logTransfer(localFile, eventType)
|
||||
else logRequestCycle(localFile, eventType, progressEvent.getBytes, progressEvent.getBytesTransferred)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package net.kemitix.s3thorp.awssdk
|
||||
|
||||
import com.amazonaws.event.ProgressEventType
|
||||
import net.kemitix.s3thorp.{Logging, LocalFile, Config}
|
||||
|
||||
trait UploadProgressLogging
|
||||
extends Logging {
|
||||
|
||||
def logTransfer(localFile: LocalFile,
|
||||
eventType: ProgressEventType)
|
||||
(implicit c: Config): Unit =
|
||||
log2(s"Transfer:${eventType.name}: ${localFile.remoteKey.key}")
|
||||
|
||||
def logRequestCycle(localFile: LocalFile,
|
||||
eventType: ProgressEventType,
|
||||
bytes: Long,
|
||||
transferred: Long)
|
||||
(implicit c: Config): Unit =
|
||||
if (eventType equals ProgressEventType.REQUEST_BYTE_TRANSFER_EVENT) print('.')
|
||||
else log3(s"Uploading:${eventType.name}:$transferred/$bytes:${localFile.remoteKey.key}")
|
||||
|
||||
}
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
package net.kemitix.s3thorp
|
||||
|
||||
import cats.effect.IO
|
||||
import net.kemitix.s3thorp.awssdk.{S3ObjectsData, S3Client}
|
||||
import net.kemitix.s3thorp.awssdk.{S3Client, S3ObjectsData, UploadProgressListener}
|
||||
|
||||
trait DummyS3Client extends S3Client {
|
||||
|
||||
override def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int
|
||||
)(implicit c: Config): IO[UploadS3Action] = ???
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,27 @@
|
|||
package net.kemitix.s3thorp
|
||||
|
||||
import java.io.File
|
||||
import java.io.{File, InputStream}
|
||||
import java.net.URL
|
||||
import java.time.Instant
|
||||
import java.util
|
||||
import java.util.Date
|
||||
import java.util.concurrent.CompletableFuture
|
||||
|
||||
import cats.effect.IO
|
||||
import net.kemitix.s3thorp.awssdk.{S3Client, S3ObjectsData}
|
||||
import com.amazonaws.regions.Region
|
||||
import com.amazonaws.services.s3.model._
|
||||
import com.amazonaws.services.s3.model.analytics.AnalyticsConfiguration
|
||||
import com.amazonaws.services.s3.model.inventory.InventoryConfiguration
|
||||
import com.amazonaws.services.s3.model.metrics.MetricsConfiguration
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerBuilder
|
||||
import com.amazonaws.services.s3.waiters.AmazonS3Waiters
|
||||
import com.amazonaws.services.s3.{AmazonS3, S3ClientOptions, S3ResponseMetadata, model}
|
||||
import com.amazonaws.{AmazonWebServiceRequest, HttpMethod}
|
||||
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
|
||||
import software.amazon.awssdk.core.async.AsyncRequestBody
|
||||
import software.amazon.awssdk.services.s3.{S3AsyncClient => JavaS3AsyncClient}
|
||||
import net.kemitix.s3thorp.awssdk.{MyAmazonS3, S3Client, S3ObjectsData, UploadProgressListener}
|
||||
import software.amazon.awssdk.services.s3
|
||||
import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, ListObjectsV2Response, PutObjectRequest, PutObjectResponse}
|
||||
import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, ListObjectsV2Response}
|
||||
import software.amazon.awssdk.services.s3.{S3AsyncClient => JavaS3AsyncClient}
|
||||
|
||||
class SyncSuite
|
||||
extends UnitTest
|
||||
|
|
@ -20,6 +31,11 @@ class SyncSuite
|
|||
private val prefix = RemoteKey("prefix")
|
||||
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
||||
private val lastModified = LastModified(Instant.now)
|
||||
val fileToKey: File => RemoteKey = generateKey(source, prefix)
|
||||
val rootHash = MD5Hash("a3a6ac11a0eb577b81b3bb5c95cc8a6e")
|
||||
val leafHash = MD5Hash("208386a650bdec61cfcd7bd8dcb6b542")
|
||||
val rootFile = aLocalFile("root-file", rootHash, source, fileToKey)
|
||||
val leafFile = aLocalFile("subdir/leaf-file", leafHash, source, fileToKey)
|
||||
|
||||
describe("s3client thunk") {
|
||||
val testBucket = Bucket("bucket")
|
||||
|
|
@ -28,9 +44,11 @@ class SyncSuite
|
|||
describe("upload") {
|
||||
val md5Hash = MD5Hash("the-hash")
|
||||
val testLocalFile = aLocalFile("file", md5Hash, source, generateKey(source, prefix))
|
||||
val progressListener = new UploadProgressListener(testLocalFile)
|
||||
val sync = new Sync(new S3Client with DummyS3Client {
|
||||
override def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int)
|
||||
(implicit c: Config) = IO {
|
||||
assert(bucket == testBucket)
|
||||
|
|
@ -39,11 +57,16 @@ class SyncSuite
|
|||
})
|
||||
it("delegates unmodified to the S3Client") {
|
||||
assertResult(UploadS3Action(RemoteKey(prefix.key + "/file"), md5Hash))(
|
||||
sync.upload(testLocalFile, testBucket, 1).
|
||||
sync.upload(testLocalFile, testBucket, progressListener, 1).
|
||||
unsafeRunSync())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def putObjectRequest(bucket: Bucket, remoteKey: RemoteKey, localFile: LocalFile) = {
|
||||
(bucket.name, remoteKey.key, localFile.file)
|
||||
}
|
||||
|
||||
describe("run") {
|
||||
val testBucket = Bucket("bucket")
|
||||
val source = Resource(this, "upload")
|
||||
|
|
@ -51,8 +74,6 @@ class SyncSuite
|
|||
val config = Config(Bucket("bucket"), RemoteKey("prefix"), source = source)
|
||||
val rootRemoteKey = RemoteKey("prefix/root-file")
|
||||
val leafRemoteKey = RemoteKey("prefix/subdir/leaf-file")
|
||||
val rootHash = MD5Hash("a3a6ac11a0eb577b81b3bb5c95cc8a6e")
|
||||
val leafHash = MD5Hash("208386a650bdec61cfcd7bd8dcb6b542")
|
||||
describe("when all files should be uploaded") {
|
||||
val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(
|
||||
byHash = Map(),
|
||||
|
|
@ -144,16 +165,19 @@ class SyncSuite
|
|||
}
|
||||
}
|
||||
describe("io actions execute") {
|
||||
val recordingS3ClientLegacy = new RecordingS3ClientLegacy
|
||||
val recordingS3Client = new RecordingS3Client
|
||||
val client = S3Client.createClient(recordingS3Client)
|
||||
val transferManager = TransferManagerBuilder.standard
|
||||
.withS3Client(recordingS3Client).build
|
||||
val client = S3Client.createClient(recordingS3ClientLegacy, recordingS3Client, transferManager)
|
||||
val sync = new Sync(client)
|
||||
sync.run(config).unsafeRunSync
|
||||
it("invokes the underlying Java s3client") {
|
||||
val expected = Set(
|
||||
PutObjectRequest.builder().bucket(testBucket.name).key(rootRemoteKey.key).build(),
|
||||
PutObjectRequest.builder().bucket(testBucket.name).key(leafRemoteKey.key).build()
|
||||
putObjectRequest(testBucket, rootRemoteKey, rootFile),
|
||||
putObjectRequest(testBucket, leafRemoteKey, leafFile)
|
||||
)
|
||||
val result = recordingS3Client.puts
|
||||
val result = recordingS3Client.puts map {r => (r.getBucketName, r.getKey, r.getFile)}
|
||||
assertResult(expected)(result)
|
||||
}
|
||||
}
|
||||
|
|
@ -181,6 +205,7 @@ class SyncSuite
|
|||
|
||||
override def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
progressListener: UploadProgressListener,
|
||||
tryCount: Int
|
||||
)(implicit c: Config) = IO {
|
||||
if (bucket == testBucket)
|
||||
|
|
@ -207,7 +232,7 @@ class SyncSuite
|
|||
}
|
||||
}
|
||||
|
||||
class RecordingS3Client extends S3AsyncClient {
|
||||
class RecordingS3ClientLegacy extends S3AsyncClient {
|
||||
var lists: Set[ListObjectsV2Request] = Set()
|
||||
var puts: Set[PutObjectRequest] = Set()
|
||||
override val underlying: s3.S3AsyncClient = new JavaS3AsyncClient {
|
||||
|
|
@ -220,12 +245,19 @@ class SyncSuite
|
|||
CompletableFuture.completedFuture(ListObjectsV2Response.builder().build())
|
||||
}
|
||||
|
||||
override def putObject(putObjectRequest: PutObjectRequest,
|
||||
requestBody: AsyncRequestBody): CompletableFuture[PutObjectResponse] = {
|
||||
puts += putObjectRequest
|
||||
CompletableFuture.completedFuture(PutObjectResponse.builder().eTag("not-null").build())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
class RecordingS3Client extends MyAmazonS3 {
|
||||
var lists: Set[ListObjectsV2Request] = Set()
|
||||
var puts: Set[PutObjectRequest] = Set()
|
||||
|
||||
override def putObject(putObjectRequest: PutObjectRequest): PutObjectResult = {
|
||||
puts += putObjectRequest
|
||||
val result = new PutObjectResult
|
||||
result.setETag("not-null")
|
||||
result
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
375
src/test/scala/net/kemitix/s3thorp/awssdk/MyAmazonS3.scala
Normal file
375
src/test/scala/net/kemitix/s3thorp/awssdk/MyAmazonS3.scala
Normal file
|
|
@ -0,0 +1,375 @@
|
|||
package net.kemitix.s3thorp.awssdk
|
||||
|
||||
import java.io.{File, InputStream}
|
||||
import java.net.URL
|
||||
import java.util
|
||||
import java.util.Date
|
||||
|
||||
import com.amazonaws.{AmazonWebServiceRequest, HttpMethod}
|
||||
import com.amazonaws.regions.Region
|
||||
import com.amazonaws.services.s3.model.analytics.AnalyticsConfiguration
|
||||
import com.amazonaws.services.s3.model.inventory.InventoryConfiguration
|
||||
import com.amazonaws.services.s3.{AmazonS3, S3ClientOptions, S3ResponseMetadata, model}
|
||||
import com.amazonaws.services.s3.model.metrics.MetricsConfiguration
|
||||
import com.amazonaws.services.s3.model._
|
||||
import com.amazonaws.services.s3.waiters.AmazonS3Waiters
|
||||
|
||||
abstract class MyAmazonS3 extends AmazonS3 {
|
||||
override def setEndpoint(endpoint: String): Unit = ???
|
||||
|
||||
override def setRegion(region: Region): Unit = ???
|
||||
|
||||
override def setS3ClientOptions(clientOptions: S3ClientOptions): Unit = ???
|
||||
|
||||
override def changeObjectStorageClass(bucketName: String, key: String, newStorageClass: StorageClass): Unit = ???
|
||||
|
||||
override def setObjectRedirectLocation(bucketName: String, key: String, newRedirectLocation: String): Unit = ???
|
||||
|
||||
override def listObjects(bucketName: String): ObjectListing = ???
|
||||
|
||||
override def listObjects(bucketName: String, prefix: String): ObjectListing = ???
|
||||
|
||||
override def listObjects(listObjectsRequest: ListObjectsRequest): ObjectListing = ???
|
||||
|
||||
override def listObjectsV2(bucketName: String): ListObjectsV2Result = ???
|
||||
|
||||
override def listObjectsV2(bucketName: String, prefix: String): ListObjectsV2Result = ???
|
||||
|
||||
override def listObjectsV2(listObjectsV2Request: ListObjectsV2Request): ListObjectsV2Result = ???
|
||||
|
||||
override def listNextBatchOfObjects(previousObjectListing: ObjectListing): ObjectListing = ???
|
||||
|
||||
override def listNextBatchOfObjects(listNextBatchOfObjectsRequest: ListNextBatchOfObjectsRequest): ObjectListing = ???
|
||||
|
||||
override def listVersions(bucketName: String, prefix: String): VersionListing = ???
|
||||
|
||||
override def listNextBatchOfVersions(previousVersionListing: VersionListing): VersionListing = ???
|
||||
|
||||
override def listNextBatchOfVersions(listNextBatchOfVersionsRequest: ListNextBatchOfVersionsRequest): VersionListing = ???
|
||||
|
||||
override def listVersions(bucketName: String, prefix: String, keyMarker: String, versionIdMarker: String, delimiter: String, maxResults: Integer): VersionListing = ???
|
||||
|
||||
override def listVersions(listVersionsRequest: ListVersionsRequest): VersionListing = ???
|
||||
|
||||
override def getS3AccountOwner: Owner = ???
|
||||
|
||||
override def getS3AccountOwner(getS3AccountOwnerRequest: GetS3AccountOwnerRequest): Owner = ???
|
||||
|
||||
override def doesBucketExist(bucketName: String): Boolean = ???
|
||||
|
||||
override def doesBucketExistV2(bucketName: String): Boolean = ???
|
||||
|
||||
override def headBucket(headBucketRequest: HeadBucketRequest): HeadBucketResult = ???
|
||||
|
||||
override def listBuckets(): util.List[Bucket] = ???
|
||||
|
||||
override def listBuckets(listBucketsRequest: ListBucketsRequest): util.List[Bucket] = ???
|
||||
|
||||
override def getBucketLocation(bucketName: String): String = ???
|
||||
|
||||
override def getBucketLocation(getBucketLocationRequest: GetBucketLocationRequest): String = ???
|
||||
|
||||
override def createBucket(createBucketRequest: CreateBucketRequest): Bucket = ???
|
||||
|
||||
override def createBucket(bucketName: String): Bucket = ???
|
||||
|
||||
override def createBucket(bucketName: String, region: model.Region): Bucket = ???
|
||||
|
||||
override def createBucket(bucketName: String, region: String): Bucket = ???
|
||||
|
||||
override def getObjectAcl(bucketName: String, key: String): AccessControlList = ???
|
||||
|
||||
override def getObjectAcl(bucketName: String, key: String, versionId: String): AccessControlList = ???
|
||||
|
||||
override def getObjectAcl(getObjectAclRequest: GetObjectAclRequest): AccessControlList = ???
|
||||
|
||||
override def setObjectAcl(bucketName: String, key: String, acl: AccessControlList): Unit = ???
|
||||
|
||||
override def setObjectAcl(bucketName: String, key: String, acl: CannedAccessControlList): Unit = ???
|
||||
|
||||
override def setObjectAcl(bucketName: String, key: String, versionId: String, acl: AccessControlList): Unit = ???
|
||||
|
||||
override def setObjectAcl(bucketName: String, key: String, versionId: String, acl: CannedAccessControlList): Unit = ???
|
||||
|
||||
override def setObjectAcl(setObjectAclRequest: SetObjectAclRequest): Unit = ???
|
||||
|
||||
override def getBucketAcl(bucketName: String): AccessControlList = ???
|
||||
|
||||
override def setBucketAcl(setBucketAclRequest: SetBucketAclRequest): Unit = ???
|
||||
|
||||
override def getBucketAcl(getBucketAclRequest: GetBucketAclRequest): AccessControlList = ???
|
||||
|
||||
override def setBucketAcl(bucketName: String, acl: AccessControlList): Unit = ???
|
||||
|
||||
override def setBucketAcl(bucketName: String, acl: CannedAccessControlList): Unit = ???
|
||||
|
||||
override def getObjectMetadata(bucketName: String, key: String): ObjectMetadata = ???
|
||||
|
||||
override def getObjectMetadata(getObjectMetadataRequest: GetObjectMetadataRequest): ObjectMetadata = ???
|
||||
|
||||
override def getObject(bucketName: String, key: String): S3Object = ???
|
||||
|
||||
override def getObject(getObjectRequest: GetObjectRequest): S3Object = ???
|
||||
|
||||
override def getObject(getObjectRequest: GetObjectRequest, destinationFile: File): ObjectMetadata = ???
|
||||
|
||||
override def getObjectAsString(bucketName: String, key: String): String = ???
|
||||
|
||||
override def getObjectTagging(getObjectTaggingRequest: GetObjectTaggingRequest): GetObjectTaggingResult = ???
|
||||
|
||||
override def setObjectTagging(setObjectTaggingRequest: SetObjectTaggingRequest): SetObjectTaggingResult = ???
|
||||
|
||||
override def deleteObjectTagging(deleteObjectTaggingRequest: DeleteObjectTaggingRequest): DeleteObjectTaggingResult = ???
|
||||
|
||||
override def deleteBucket(deleteBucketRequest: DeleteBucketRequest): Unit = ???
|
||||
|
||||
override def deleteBucket(bucketName: String): Unit = ???
|
||||
|
||||
override def putObject(putObjectRequest: PutObjectRequest): PutObjectResult = ???
|
||||
|
||||
override def putObject(bucketName: String, key: String, file: File): PutObjectResult = ???
|
||||
|
||||
override def putObject(bucketName: String, key: String, input: InputStream, metadata: ObjectMetadata): PutObjectResult = ???
|
||||
|
||||
override def putObject(bucketName: String, key: String, content: String): PutObjectResult = ???
|
||||
|
||||
override def copyObject(sourceBucketName: String, sourceKey: String, destinationBucketName: String, destinationKey: String): CopyObjectResult = ???
|
||||
|
||||
override def copyObject(copyObjectRequest: CopyObjectRequest): CopyObjectResult = ???
|
||||
|
||||
override def copyPart(copyPartRequest: CopyPartRequest): CopyPartResult = ???
|
||||
|
||||
override def deleteObject(bucketName: String, key: String): Unit = ???
|
||||
|
||||
override def deleteObject(deleteObjectRequest: DeleteObjectRequest): Unit = ???
|
||||
|
||||
override def deleteObjects(deleteObjectsRequest: DeleteObjectsRequest): DeleteObjectsResult = ???
|
||||
|
||||
override def deleteVersion(bucketName: String, key: String, versionId: String): Unit = ???
|
||||
|
||||
override def deleteVersion(deleteVersionRequest: DeleteVersionRequest): Unit = ???
|
||||
|
||||
override def getBucketLoggingConfiguration(bucketName: String): BucketLoggingConfiguration = ???
|
||||
|
||||
override def getBucketLoggingConfiguration(getBucketLoggingConfigurationRequest: GetBucketLoggingConfigurationRequest): BucketLoggingConfiguration = ???
|
||||
|
||||
override def setBucketLoggingConfiguration(setBucketLoggingConfigurationRequest: SetBucketLoggingConfigurationRequest): Unit = ???
|
||||
|
||||
override def getBucketVersioningConfiguration(bucketName: String): BucketVersioningConfiguration = ???
|
||||
|
||||
override def getBucketVersioningConfiguration(getBucketVersioningConfigurationRequest: GetBucketVersioningConfigurationRequest): BucketVersioningConfiguration = ???
|
||||
|
||||
override def setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest: SetBucketVersioningConfigurationRequest): Unit = ???
|
||||
|
||||
override def getBucketLifecycleConfiguration(bucketName: String): BucketLifecycleConfiguration = ???
|
||||
|
||||
override def getBucketLifecycleConfiguration(getBucketLifecycleConfigurationRequest: GetBucketLifecycleConfigurationRequest): BucketLifecycleConfiguration = ???
|
||||
|
||||
override def setBucketLifecycleConfiguration(bucketName: String, bucketLifecycleConfiguration: BucketLifecycleConfiguration): Unit = ???
|
||||
|
||||
override def setBucketLifecycleConfiguration(setBucketLifecycleConfigurationRequest: SetBucketLifecycleConfigurationRequest): Unit = ???
|
||||
|
||||
override def deleteBucketLifecycleConfiguration(bucketName: String): Unit = ???
|
||||
|
||||
override def deleteBucketLifecycleConfiguration(deleteBucketLifecycleConfigurationRequest: DeleteBucketLifecycleConfigurationRequest): Unit = ???
|
||||
|
||||
override def getBucketCrossOriginConfiguration(bucketName: String): BucketCrossOriginConfiguration = ???
|
||||
|
||||
override def getBucketCrossOriginConfiguration(getBucketCrossOriginConfigurationRequest: GetBucketCrossOriginConfigurationRequest): BucketCrossOriginConfiguration = ???
|
||||
|
||||
override def setBucketCrossOriginConfiguration(bucketName: String, bucketCrossOriginConfiguration: BucketCrossOriginConfiguration): Unit = ???
|
||||
|
||||
override def setBucketCrossOriginConfiguration(setBucketCrossOriginConfigurationRequest: SetBucketCrossOriginConfigurationRequest): Unit = ???
|
||||
|
||||
override def deleteBucketCrossOriginConfiguration(bucketName: String): Unit = ???
|
||||
|
||||
override def deleteBucketCrossOriginConfiguration(deleteBucketCrossOriginConfigurationRequest: DeleteBucketCrossOriginConfigurationRequest): Unit = ???
|
||||
|
||||
override def getBucketTaggingConfiguration(bucketName: String): BucketTaggingConfiguration = ???
|
||||
|
||||
override def getBucketTaggingConfiguration(getBucketTaggingConfigurationRequest: GetBucketTaggingConfigurationRequest): BucketTaggingConfiguration = ???
|
||||
|
||||
override def setBucketTaggingConfiguration(bucketName: String, bucketTaggingConfiguration: BucketTaggingConfiguration): Unit = ???
|
||||
|
||||
override def setBucketTaggingConfiguration(setBucketTaggingConfigurationRequest: SetBucketTaggingConfigurationRequest): Unit = ???
|
||||
|
||||
override def deleteBucketTaggingConfiguration(bucketName: String): Unit = ???
|
||||
|
||||
override def deleteBucketTaggingConfiguration(deleteBucketTaggingConfigurationRequest: DeleteBucketTaggingConfigurationRequest): Unit = ???
|
||||
|
||||
override def getBucketNotificationConfiguration(bucketName: String): BucketNotificationConfiguration = ???
|
||||
|
||||
override def getBucketNotificationConfiguration(getBucketNotificationConfigurationRequest: GetBucketNotificationConfigurationRequest): BucketNotificationConfiguration = ???
|
||||
|
||||
override def setBucketNotificationConfiguration(setBucketNotificationConfigurationRequest: SetBucketNotificationConfigurationRequest): Unit = ???
|
||||
|
||||
override def setBucketNotificationConfiguration(bucketName: String, bucketNotificationConfiguration: BucketNotificationConfiguration): Unit = ???
|
||||
|
||||
override def getBucketWebsiteConfiguration(bucketName: String): BucketWebsiteConfiguration = ???
|
||||
|
||||
override def getBucketWebsiteConfiguration(getBucketWebsiteConfigurationRequest: GetBucketWebsiteConfigurationRequest): BucketWebsiteConfiguration = ???
|
||||
|
||||
override def setBucketWebsiteConfiguration(bucketName: String, configuration: BucketWebsiteConfiguration): Unit = ???
|
||||
|
||||
override def setBucketWebsiteConfiguration(setBucketWebsiteConfigurationRequest: SetBucketWebsiteConfigurationRequest): Unit = ???
|
||||
|
||||
override def deleteBucketWebsiteConfiguration(bucketName: String): Unit = ???
|
||||
|
||||
override def deleteBucketWebsiteConfiguration(deleteBucketWebsiteConfigurationRequest: DeleteBucketWebsiteConfigurationRequest): Unit = ???
|
||||
|
||||
override def getBucketPolicy(bucketName: String): BucketPolicy = ???
|
||||
|
||||
override def getBucketPolicy(getBucketPolicyRequest: GetBucketPolicyRequest): BucketPolicy = ???
|
||||
|
||||
override def setBucketPolicy(bucketName: String, policyText: String): Unit = ???
|
||||
|
||||
override def setBucketPolicy(setBucketPolicyRequest: SetBucketPolicyRequest): Unit = ???
|
||||
|
||||
override def deleteBucketPolicy(bucketName: String): Unit = ???
|
||||
|
||||
override def deleteBucketPolicy(deleteBucketPolicyRequest: DeleteBucketPolicyRequest): Unit = ???
|
||||
|
||||
override def generatePresignedUrl(bucketName: String, key: String, expiration: Date): URL = ???
|
||||
|
||||
override def generatePresignedUrl(bucketName: String, key: String, expiration: Date, method: HttpMethod): URL = ???
|
||||
|
||||
override def generatePresignedUrl(generatePresignedUrlRequest: GeneratePresignedUrlRequest): URL = ???
|
||||
|
||||
override def initiateMultipartUpload(request: InitiateMultipartUploadRequest): InitiateMultipartUploadResult = ???
|
||||
|
||||
override def uploadPart(request: UploadPartRequest): UploadPartResult = ???
|
||||
|
||||
override def listParts(request: ListPartsRequest): PartListing = ???
|
||||
|
||||
override def abortMultipartUpload(request: AbortMultipartUploadRequest): Unit = ???
|
||||
|
||||
override def completeMultipartUpload(request: CompleteMultipartUploadRequest): CompleteMultipartUploadResult = ???
|
||||
|
||||
override def listMultipartUploads(request: ListMultipartUploadsRequest): MultipartUploadListing = ???
|
||||
|
||||
override def getCachedResponseMetadata(request: AmazonWebServiceRequest): S3ResponseMetadata = ???
|
||||
|
||||
override def restoreObject(request: RestoreObjectRequest): Unit = ???
|
||||
|
||||
override def restoreObjectV2(request: RestoreObjectRequest): RestoreObjectResult = ???
|
||||
|
||||
override def restoreObject(bucketName: String, key: String, expirationInDays: Int): Unit = ???
|
||||
|
||||
override def enableRequesterPays(bucketName: String): Unit = ???
|
||||
|
||||
override def disableRequesterPays(bucketName: String): Unit = ???
|
||||
|
||||
override def isRequesterPaysEnabled(bucketName: String): Boolean = ???
|
||||
|
||||
override def setBucketReplicationConfiguration(bucketName: String, configuration: BucketReplicationConfiguration): Unit = ???
|
||||
|
||||
override def setBucketReplicationConfiguration(setBucketReplicationConfigurationRequest: SetBucketReplicationConfigurationRequest): Unit = ???
|
||||
|
||||
override def getBucketReplicationConfiguration(bucketName: String): BucketReplicationConfiguration = ???
|
||||
|
||||
override def getBucketReplicationConfiguration(getBucketReplicationConfigurationRequest: GetBucketReplicationConfigurationRequest): BucketReplicationConfiguration = ???
|
||||
|
||||
override def deleteBucketReplicationConfiguration(bucketName: String): Unit = ???
|
||||
|
||||
override def deleteBucketReplicationConfiguration(request: DeleteBucketReplicationConfigurationRequest): Unit = ???
|
||||
|
||||
override def doesObjectExist(bucketName: String, objectName: String): Boolean = ???
|
||||
|
||||
override def getBucketAccelerateConfiguration(bucketName: String): BucketAccelerateConfiguration = ???
|
||||
|
||||
override def getBucketAccelerateConfiguration(getBucketAccelerateConfigurationRequest: GetBucketAccelerateConfigurationRequest): BucketAccelerateConfiguration = ???
|
||||
|
||||
override def setBucketAccelerateConfiguration(bucketName: String, accelerateConfiguration: BucketAccelerateConfiguration): Unit = ???
|
||||
|
||||
override def setBucketAccelerateConfiguration(setBucketAccelerateConfigurationRequest: SetBucketAccelerateConfigurationRequest): Unit = ???
|
||||
|
||||
override def deleteBucketMetricsConfiguration(bucketName: String, id: String): DeleteBucketMetricsConfigurationResult = ???
|
||||
|
||||
override def deleteBucketMetricsConfiguration(deleteBucketMetricsConfigurationRequest: DeleteBucketMetricsConfigurationRequest): DeleteBucketMetricsConfigurationResult = ???
|
||||
|
||||
override def getBucketMetricsConfiguration(bucketName: String, id: String): GetBucketMetricsConfigurationResult = ???
|
||||
|
||||
override def getBucketMetricsConfiguration(getBucketMetricsConfigurationRequest: GetBucketMetricsConfigurationRequest): GetBucketMetricsConfigurationResult = ???
|
||||
|
||||
override def setBucketMetricsConfiguration(bucketName: String, metricsConfiguration: MetricsConfiguration): SetBucketMetricsConfigurationResult = ???
|
||||
|
||||
override def setBucketMetricsConfiguration(setBucketMetricsConfigurationRequest: SetBucketMetricsConfigurationRequest): SetBucketMetricsConfigurationResult = ???
|
||||
|
||||
override def listBucketMetricsConfigurations(listBucketMetricsConfigurationsRequest: ListBucketMetricsConfigurationsRequest): ListBucketMetricsConfigurationsResult = ???
|
||||
|
||||
override def deleteBucketAnalyticsConfiguration(bucketName: String, id: String): DeleteBucketAnalyticsConfigurationResult = ???
|
||||
|
||||
override def deleteBucketAnalyticsConfiguration(deleteBucketAnalyticsConfigurationRequest: DeleteBucketAnalyticsConfigurationRequest): DeleteBucketAnalyticsConfigurationResult = ???
|
||||
|
||||
override def getBucketAnalyticsConfiguration(bucketName: String, id: String): GetBucketAnalyticsConfigurationResult = ???
|
||||
|
||||
override def getBucketAnalyticsConfiguration(getBucketAnalyticsConfigurationRequest: GetBucketAnalyticsConfigurationRequest): GetBucketAnalyticsConfigurationResult = ???
|
||||
|
||||
override def setBucketAnalyticsConfiguration(bucketName: String, analyticsConfiguration: AnalyticsConfiguration): SetBucketAnalyticsConfigurationResult = ???
|
||||
|
||||
override def setBucketAnalyticsConfiguration(setBucketAnalyticsConfigurationRequest: SetBucketAnalyticsConfigurationRequest): SetBucketAnalyticsConfigurationResult = ???
|
||||
|
||||
override def listBucketAnalyticsConfigurations(listBucketAnalyticsConfigurationsRequest: ListBucketAnalyticsConfigurationsRequest): ListBucketAnalyticsConfigurationsResult = ???
|
||||
|
||||
override def deleteBucketInventoryConfiguration(bucketName: String, id: String): DeleteBucketInventoryConfigurationResult = ???
|
||||
|
||||
override def deleteBucketInventoryConfiguration(deleteBucketInventoryConfigurationRequest: DeleteBucketInventoryConfigurationRequest): DeleteBucketInventoryConfigurationResult = ???
|
||||
|
||||
override def getBucketInventoryConfiguration(bucketName: String, id: String): GetBucketInventoryConfigurationResult = ???
|
||||
|
||||
override def getBucketInventoryConfiguration(getBucketInventoryConfigurationRequest: GetBucketInventoryConfigurationRequest): GetBucketInventoryConfigurationResult = ???
|
||||
|
||||
override def setBucketInventoryConfiguration(bucketName: String, inventoryConfiguration: InventoryConfiguration): SetBucketInventoryConfigurationResult = ???
|
||||
|
||||
override def setBucketInventoryConfiguration(setBucketInventoryConfigurationRequest: SetBucketInventoryConfigurationRequest): SetBucketInventoryConfigurationResult = ???
|
||||
|
||||
override def listBucketInventoryConfigurations(listBucketInventoryConfigurationsRequest: ListBucketInventoryConfigurationsRequest): ListBucketInventoryConfigurationsResult = ???
|
||||
|
||||
override def deleteBucketEncryption(bucketName: String): DeleteBucketEncryptionResult = ???
|
||||
|
||||
override def deleteBucketEncryption(request: DeleteBucketEncryptionRequest): DeleteBucketEncryptionResult = ???
|
||||
|
||||
override def getBucketEncryption(bucketName: String): GetBucketEncryptionResult = ???
|
||||
|
||||
override def getBucketEncryption(request: GetBucketEncryptionRequest): GetBucketEncryptionResult = ???
|
||||
|
||||
override def setBucketEncryption(setBucketEncryptionRequest: SetBucketEncryptionRequest): SetBucketEncryptionResult = ???
|
||||
|
||||
override def setPublicAccessBlock(request: SetPublicAccessBlockRequest): SetPublicAccessBlockResult = ???
|
||||
|
||||
override def getPublicAccessBlock(request: GetPublicAccessBlockRequest): GetPublicAccessBlockResult = ???
|
||||
|
||||
override def deletePublicAccessBlock(request: DeletePublicAccessBlockRequest): DeletePublicAccessBlockResult = ???
|
||||
|
||||
override def getBucketPolicyStatus(request: GetBucketPolicyStatusRequest): GetBucketPolicyStatusResult = ???
|
||||
|
||||
override def selectObjectContent(selectRequest: SelectObjectContentRequest): SelectObjectContentResult = ???
|
||||
|
||||
override def setObjectLegalHold(setObjectLegalHoldRequest: SetObjectLegalHoldRequest): SetObjectLegalHoldResult = ???
|
||||
|
||||
override def getObjectLegalHold(getObjectLegalHoldRequest: GetObjectLegalHoldRequest): GetObjectLegalHoldResult = ???
|
||||
|
||||
override def setObjectLockConfiguration(setObjectLockConfigurationRequest: SetObjectLockConfigurationRequest): SetObjectLockConfigurationResult = ???
|
||||
|
||||
override def getObjectLockConfiguration(getObjectLockConfigurationRequest: GetObjectLockConfigurationRequest): GetObjectLockConfigurationResult = ???
|
||||
|
||||
override def setObjectRetention(setObjectRetentionRequest: SetObjectRetentionRequest): SetObjectRetentionResult = ???
|
||||
|
||||
override def getObjectRetention(getObjectRetentionRequest: GetObjectRetentionRequest): GetObjectRetentionResult = ???
|
||||
|
||||
override def download(presignedUrlDownloadRequest: PresignedUrlDownloadRequest): PresignedUrlDownloadResult = ???
|
||||
|
||||
override def download(presignedUrlDownloadRequest: PresignedUrlDownloadRequest, destinationFile: File): Unit = ???
|
||||
|
||||
override def upload(presignedUrlUploadRequest: PresignedUrlUploadRequest): PresignedUrlUploadResult = ???
|
||||
|
||||
override def shutdown(): Unit = ???
|
||||
|
||||
override def getRegion: model.Region = ???
|
||||
|
||||
override def getRegionName: String = ???
|
||||
|
||||
override def getUrl(bucketName: String, key: String): URL = ???
|
||||
|
||||
override def waiters(): AmazonS3Waiters = ???
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ import java.time.Instant
|
|||
import com.amazonaws.AmazonClientException
|
||||
import com.amazonaws.services.s3.model
|
||||
import com.amazonaws.services.s3.transfer.model.UploadResult
|
||||
import com.amazonaws.services.s3.transfer.{PauseResult, PersistableUpload, Transfer, TransferManager, TransferProgress, Upload}
|
||||
import com.amazonaws.services.s3.transfer.{PauseResult, PersistableUpload, Transfer, TransferManager, TransferManagerBuilder, TransferProgress, Upload}
|
||||
import net.kemitix.s3thorp.{Bucket, Config, KeyGenerator, LastModified, MD5Hash, MD5HashGenerator, RemoteKey, Resource, UnitTest, UploadS3Action}
|
||||
|
||||
class S3ClientMultiPartTransferManagerSuite
|
||||
|
|
@ -43,16 +43,25 @@ class S3ClientMultiPartTransferManagerSuite
|
|||
}
|
||||
}
|
||||
describe("upload") {
|
||||
pending
|
||||
// how much of this test is testing the amazonTransferManager
|
||||
// Should we just test that the correct parameters are passed to initiate, or will this test
|
||||
// just collapse and die if the amazonS3 doesn't respond properly to TransferManager input
|
||||
// dies when putObject is called
|
||||
val returnedKey = RemoteKey("returned-key")
|
||||
val returnedHash = MD5Hash("returned-hash")
|
||||
val bigFile = aLocalFile("small-file", MD5Hash("the-hash"), source, fileToKey)
|
||||
val transferManager = new MyTransferManager(
|
||||
(config.bucket.name, bigFile.remoteKey.key, bigFile.file),
|
||||
returnedKey, returnedHash)
|
||||
val uploader = new S3ClientMultiPartTransferManager(transferManager)
|
||||
val progressListener = new UploadProgressListener(bigFile)
|
||||
val amazonS3 = new MyAmazonS3 {}
|
||||
val amazonS3TransferManager = TransferManagerBuilder.standard().withS3Client(amazonS3).build
|
||||
new MyTransferManager(
|
||||
signature = (config.bucket.name, bigFile.remoteKey.key, bigFile.file),
|
||||
returnedKey = returnedKey,
|
||||
returnedHash = returnedHash)
|
||||
val uploader = new S3ClientMultiPartTransferManager(amazonS3TransferManager)
|
||||
it("should upload") {
|
||||
val expected = UploadS3Action(returnedKey, returnedHash)
|
||||
val result = uploader.upload(bigFile, config.bucket, 1).unsafeRunSync
|
||||
val result = uploader.upload(bigFile, config.bucket, progressListener, 1).unsafeRunSync
|
||||
assertResult(expected)(result)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ class S3ClientMultiPartUploaderSuite
|
|||
|
||||
describe("mulit-part uploader upload") {
|
||||
val theFile = aLocalFile("big-file", MD5Hash(""), source, fileToKey)
|
||||
val progressListener = new UploadProgressListener(theFile)
|
||||
val uploadId = "upload-id"
|
||||
val createUploadResponse = new InitiateMultipartUploadResult()
|
||||
createUploadResponse.setBucketName(config.bucket.name)
|
||||
|
|
@ -167,7 +168,7 @@ class S3ClientMultiPartUploaderSuite
|
|||
describe("upload") {
|
||||
describe("when all okay") {
|
||||
val uploader = new RecordingMultiPartUploader()
|
||||
uploader.upload(theFile, config.bucket, 1).unsafeRunSync
|
||||
uploader.upload(theFile, config.bucket, progressListener, 1).unsafeRunSync
|
||||
it("should initiate the upload") {
|
||||
assert(uploader.initiated.get)
|
||||
}
|
||||
|
|
@ -180,7 +181,7 @@ class S3ClientMultiPartUploaderSuite
|
|||
}
|
||||
describe("when initiate upload fails") {
|
||||
val uploader = new RecordingMultiPartUploader(initOkay = false)
|
||||
uploader.upload(theFile, config.bucket, 1).unsafeRunSync
|
||||
uploader.upload(theFile, config.bucket, progressListener, 1).unsafeRunSync
|
||||
it("should not upload any parts") {
|
||||
assertResult(Set())(uploader.partsUploaded.get)
|
||||
}
|
||||
|
|
@ -190,7 +191,7 @@ class S3ClientMultiPartUploaderSuite
|
|||
}
|
||||
describe("when uploading a part fails once") {
|
||||
val uploader = new RecordingMultiPartUploader(partTriesRequired = 2)
|
||||
uploader.upload(theFile, config.bucket, 1).unsafeRunSync
|
||||
uploader.upload(theFile, config.bucket, progressListener, 1).unsafeRunSync
|
||||
it("should initiate the upload") {
|
||||
assert(uploader.initiated.get)
|
||||
}
|
||||
|
|
@ -203,7 +204,7 @@ class S3ClientMultiPartUploaderSuite
|
|||
}
|
||||
describe("when uploading a part fails too many times") {
|
||||
val uploader = new RecordingMultiPartUploader(partTriesRequired = 4)
|
||||
uploader.upload(theFile, config.bucket, 1).unsafeRunSync
|
||||
uploader.upload(theFile, config.bucket, progressListener, 1).unsafeRunSync
|
||||
it("should initiate the upload") {
|
||||
assert(uploader.initiated.get)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ import java.io.File
|
|||
import java.time.Instant
|
||||
|
||||
import cats.effect.IO
|
||||
import com.amazonaws.services.s3.model
|
||||
import com.amazonaws.services.s3.model.PutObjectResult
|
||||
import com.amazonaws.services.s3.transfer.{TransferManager, TransferManagerBuilder}
|
||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||
import net.kemitix.s3thorp._
|
||||
import software.amazon.awssdk.services.s3.model.{PutObjectRequest, PutObjectResponse}
|
||||
|
|
@ -75,22 +78,31 @@ class S3ClientSuite
|
|||
}
|
||||
|
||||
describe("upload") {
|
||||
def invoke(s3Client: ThorpS3Client, localFile: LocalFile, bucket: Bucket) =
|
||||
s3Client.upload(localFile, bucket, 1).unsafeRunSync
|
||||
def invoke(s3Client: ThorpS3Client, localFile: LocalFile, bucket: Bucket, progressListener: UploadProgressListener) =
|
||||
s3Client.upload(localFile, bucket, progressListener, 1).unsafeRunSync
|
||||
describe("when uploading a file") {
|
||||
val md5Hash = MD5Hash("the-md5hash")
|
||||
val source = Resource(this, "../upload")
|
||||
val md5Hash = new MD5HashGenerator {}.md5File(source.toPath.resolve("root-file").toFile)
|
||||
val amazonS3 = new MyAmazonS3 {
|
||||
override def putObject(putObjectRequest: model.PutObjectRequest): PutObjectResult = {
|
||||
val result = new PutObjectResult
|
||||
result.setETag(md5Hash.hash)
|
||||
result
|
||||
}
|
||||
}
|
||||
val amazonS3TransferManager = TransferManagerBuilder.standard().withS3Client(amazonS3).build
|
||||
val s3Client = new ThorpS3Client(
|
||||
new S3CatsIOClient with JavaClientWrapper {
|
||||
override def putObject(putObjectRequest: PutObjectRequest, requestBody: RB) =
|
||||
IO(PutObjectResponse.builder().eTag(md5Hash.hash).build())
|
||||
})
|
||||
val source = new File("/")
|
||||
// override def putObject(putObjectRequest: PutObjectRequest, requestBody: RB) =
|
||||
// IO(PutObjectResponse.builder().eTag(md5Hash.hash).build())
|
||||
}, amazonS3, amazonS3TransferManager)
|
||||
val prefix = RemoteKey("prefix")
|
||||
val localFile: LocalFile = aLocalFile("/some/file", md5Hash, source, generateKey(source, prefix))
|
||||
val localFile: LocalFile = aLocalFile("root-file", md5Hash, source, generateKey(source, prefix))
|
||||
val bucket: Bucket = Bucket("a-bucket")
|
||||
val remoteKey: RemoteKey = RemoteKey("prefix/some/file")
|
||||
val remoteKey: RemoteKey = RemoteKey("prefix/root-file")
|
||||
val progressListener = new UploadProgressListener(localFile)
|
||||
it("should return hash of uploaded file") {
|
||||
assertResult(UploadS3Action(remoteKey, md5Hash))(invoke(s3Client, localFile, bucket))
|
||||
assertResult(UploadS3Action(remoteKey, md5Hash))(invoke(s3Client, localFile, bucket, progressListener))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ package net.kemitix.s3thorp.awssdk
|
|||
import java.time.Instant
|
||||
|
||||
import cats.effect.IO
|
||||
import com.amazonaws.services.s3.model.{PutObjectRequest, PutObjectResult}
|
||||
import com.amazonaws.services.s3.transfer.{TransferManager, TransferManagerBuilder}
|
||||
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
|
||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||
import net.kemitix.s3thorp._
|
||||
|
|
@ -38,10 +40,12 @@ class ThorpS3ClientSuite extends FunSpec {
|
|||
.contents(List(o1a, o1b, o2).asJava)
|
||||
.build()
|
||||
}
|
||||
val amazonS3 = new MyAmazonS3 {}
|
||||
val amazonS3TransferManager = TransferManagerBuilder.standard().withS3Client(amazonS3).build
|
||||
val s3client = new ThorpS3Client(new MyS3CatsIOClient {
|
||||
override def listObjectsV2(listObjectsV2Request: ListObjectsV2Request): IO[ListObjectsV2Response] =
|
||||
myFakeResponse
|
||||
})
|
||||
}, amazonS3, amazonS3TransferManager)
|
||||
it("should build list of hash lookups, with duplicate objects grouped by hash") {
|
||||
val expected = S3ObjectsData(
|
||||
byHash = Map(
|
||||
|
|
|
|||
Loading…
Reference in a new issue