Simplify logging (#59)
* [cli] Logger add debug method * [cli] Logger: add alternate info method * [cli] replace direct calls to legacy Logger.info * [cli] Extract trait from Logger into core and rename * [cli] Program use debug and new info * [core] MD5HashGenerator uses Logger * [core] Sync uses Logger * [core] ActionSubmitter uses Logger * [core] LocalFileStream uses Logger * [core] SyncLogging uses Logger * [domain] Move Logger into module The allows it to be used in the aws-api module * [aws-lib] S3ClientObjectLister uses Logger * [aws-lib] Uploader uses Logger * [aws-lib] S3ClientDeleter uses Logger * [aws-lib] S3ClientCopier uses Logger * [core] remove unused legacy logging * [aws-lib] remove used logging method * [cli] PrintLogger remove legacy info method
This commit is contained in:
parent
69c09d0091
commit
21b8917395
28 changed files with 157 additions and 134 deletions
|
@ -1,13 +1,13 @@
|
||||||
package net.kemitix.s3thorp.aws.api
|
package net.kemitix.s3thorp.aws.api
|
||||||
|
|
||||||
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action}
|
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action}
|
||||||
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey, S3ObjectsData}
|
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey, S3ObjectsData}
|
||||||
|
|
||||||
trait S3Client[M[_]] {
|
trait S3Client[M[_]] {
|
||||||
|
|
||||||
def listObjects(bucket: Bucket,
|
def listObjects(bucket: Bucket,
|
||||||
prefix: RemoteKey
|
prefix: RemoteKey
|
||||||
)(implicit info: Int => String => M[Unit]): M[S3ObjectsData]
|
)(implicit logger: Logger[M]): M[S3ObjectsData]
|
||||||
|
|
||||||
def upload(localFile: LocalFile,
|
def upload(localFile: LocalFile,
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
|
@ -15,17 +15,16 @@ trait S3Client[M[_]] {
|
||||||
multiPartThreshold: Long,
|
multiPartThreshold: Long,
|
||||||
tryCount: Int,
|
tryCount: Int,
|
||||||
maxRetries: Int)
|
maxRetries: Int)
|
||||||
(implicit info: Int => String => M[Unit],
|
(implicit logger: Logger[M]): M[S3Action]
|
||||||
warn: String => M[Unit]): M[S3Action]
|
|
||||||
|
|
||||||
def copy(bucket: Bucket,
|
def copy(bucket: Bucket,
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
hash: MD5Hash,
|
hash: MD5Hash,
|
||||||
targetKey: RemoteKey
|
targetKey: RemoteKey
|
||||||
)(implicit info: Int => String => M[Unit]): M[CopyS3Action]
|
)(implicit logger: Logger[M]): M[CopyS3Action]
|
||||||
|
|
||||||
def delete(bucket: Bucket,
|
def delete(bucket: Bucket,
|
||||||
remoteKey: RemoteKey
|
remoteKey: RemoteKey
|
||||||
)(implicit info: Int => String => M[Unit]): M[DeleteS3Action]
|
)(implicit logger: Logger[M]): M[DeleteS3Action]
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,7 +6,7 @@ import com.amazonaws.services.s3.AmazonS3
|
||||||
import com.amazonaws.services.s3.model.CopyObjectRequest
|
import com.amazonaws.services.s3.model.CopyObjectRequest
|
||||||
import net.kemitix.s3thorp.aws.api.S3Action.CopyS3Action
|
import net.kemitix.s3thorp.aws.api.S3Action.CopyS3Action
|
||||||
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logCopyFinish, logCopyStart}
|
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logCopyFinish, logCopyStart}
|
||||||
import net.kemitix.s3thorp.domain.{Bucket, MD5Hash, RemoteKey}
|
import net.kemitix.s3thorp.domain.{Bucket, Logger, MD5Hash, RemoteKey}
|
||||||
|
|
||||||
class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) {
|
class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) {
|
||||||
|
|
||||||
|
@ -14,7 +14,7 @@ class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) {
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
hash: MD5Hash,
|
hash: MD5Hash,
|
||||||
targetKey: RemoteKey)
|
targetKey: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[CopyS3Action] =
|
(implicit logger: Logger[M]): M[CopyS3Action] =
|
||||||
for {
|
for {
|
||||||
_ <- logCopyStart[M](bucket, sourceKey, targetKey)
|
_ <- logCopyStart[M](bucket, sourceKey, targetKey)
|
||||||
_ <- copyObject(bucket, sourceKey, hash, targetKey)
|
_ <- copyObject(bucket, sourceKey, hash, targetKey)
|
||||||
|
|
|
@ -6,13 +6,13 @@ import com.amazonaws.services.s3.AmazonS3
|
||||||
import com.amazonaws.services.s3.model.DeleteObjectRequest
|
import com.amazonaws.services.s3.model.DeleteObjectRequest
|
||||||
import net.kemitix.s3thorp.aws.api.S3Action.DeleteS3Action
|
import net.kemitix.s3thorp.aws.api.S3Action.DeleteS3Action
|
||||||
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logDeleteFinish, logDeleteStart}
|
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logDeleteFinish, logDeleteStart}
|
||||||
import net.kemitix.s3thorp.domain.{Bucket, RemoteKey}
|
import net.kemitix.s3thorp.domain.{Bucket, Logger, RemoteKey}
|
||||||
|
|
||||||
class S3ClientDeleter[M[_]: Monad](amazonS3: AmazonS3) {
|
class S3ClientDeleter[M[_]: Monad](amazonS3: AmazonS3) {
|
||||||
|
|
||||||
def delete(bucket: Bucket,
|
def delete(bucket: Bucket,
|
||||||
remoteKey: RemoteKey)
|
remoteKey: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[DeleteS3Action] =
|
(implicit logger: Logger[M]): M[DeleteS3Action] =
|
||||||
for {
|
for {
|
||||||
_ <- logDeleteStart[M](bucket, remoteKey)
|
_ <- logDeleteStart[M](bucket, remoteKey)
|
||||||
_ <- deleteObject(bucket, remoteKey)
|
_ <- deleteObject(bucket, remoteKey)
|
||||||
|
|
|
@ -2,45 +2,40 @@ package net.kemitix.s3thorp.aws.lib
|
||||||
|
|
||||||
import cats.Monad
|
import cats.Monad
|
||||||
import com.amazonaws.services.s3.model.PutObjectResult
|
import com.amazonaws.services.s3.model.PutObjectResult
|
||||||
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, RemoteKey}
|
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, RemoteKey}
|
||||||
|
|
||||||
object S3ClientLogging {
|
object S3ClientLogging {
|
||||||
|
|
||||||
def logListObjectsStart[M[_]: Monad](bucket: Bucket,
|
def logListObjectsStart[M[_]: Monad](bucket: Bucket,
|
||||||
prefix: RemoteKey)
|
prefix: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[Unit] =
|
(implicit logger: Logger[M]): M[Unit] =
|
||||||
info(1)(s"Fetch S3 Summary: ${bucket.name}:${prefix.key}")
|
logger.info(s"Fetch S3 Summary: ${bucket.name}:${prefix.key}")
|
||||||
|
|
||||||
def logListObjectsFinish[M[_]: Monad](bucket: Bucket,
|
def logListObjectsFinish[M[_]: Monad](bucket: Bucket,
|
||||||
prefix: RemoteKey)
|
prefix: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[Unit] =
|
(implicit logger: Logger[M]): M[Unit] =
|
||||||
info(2)(s"Fetched S3 Summary: ${bucket.name}:${prefix.key}")
|
logger.info(s"Fetched S3 Summary: ${bucket.name}:${prefix.key}")
|
||||||
|
|
||||||
def logUploadFinish[M[_]: Monad](localFile: LocalFile,
|
|
||||||
bucket: Bucket)
|
|
||||||
(implicit info: Int => String => M[Unit]): PutObjectResult => M[Unit] =
|
|
||||||
_ => info(2)(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}")
|
|
||||||
|
|
||||||
def logCopyStart[M[_]: Monad](bucket: Bucket,
|
def logCopyStart[M[_]: Monad](bucket: Bucket,
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
targetKey: RemoteKey)
|
targetKey: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[Unit] =
|
(implicit logger: Logger[M]): M[Unit] =
|
||||||
info(1)(s"Copy: ${bucket.name}:${sourceKey.key} => ${targetKey.key}")
|
logger.info(s"Copy: ${bucket.name}:${sourceKey.key} => ${targetKey.key}")
|
||||||
|
|
||||||
def logCopyFinish[M[_]: Monad](bucket: Bucket,
|
def logCopyFinish[M[_]: Monad](bucket: Bucket,
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
targetKey: RemoteKey)
|
targetKey: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[Unit] =
|
(implicit logger: Logger[M]): M[Unit] =
|
||||||
info(2)(s"Copied: ${bucket.name}:${sourceKey.key} => ${targetKey.key}")
|
logger.info(s"Copied: ${bucket.name}:${sourceKey.key} => ${targetKey.key}")
|
||||||
|
|
||||||
def logDeleteStart[M[_]: Monad](bucket: Bucket,
|
def logDeleteStart[M[_]: Monad](bucket: Bucket,
|
||||||
remoteKey: RemoteKey)
|
remoteKey: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[Unit] =
|
(implicit logger: Logger[M]): M[Unit] =
|
||||||
info(1)(s"Delete: ${bucket.name}:${remoteKey.key}")
|
logger.info(s"Delete: ${bucket.name}:${remoteKey.key}")
|
||||||
|
|
||||||
def logDeleteFinish[M[_]: Monad](bucket: Bucket,
|
def logDeleteFinish[M[_]: Monad](bucket: Bucket,
|
||||||
remoteKey: RemoteKey)
|
remoteKey: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[Unit] =
|
(implicit logger: Logger[M]): M[Unit] =
|
||||||
info(2)(s"Deleted: ${bucket.name}:${remoteKey.key}")
|
logger.info(s"Deleted: ${bucket.name}:${remoteKey.key}")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,7 +15,7 @@ class S3ClientObjectLister[M[_]: Monad](amazonS3: AmazonS3) {
|
||||||
|
|
||||||
def listObjects(bucket: Bucket,
|
def listObjects(bucket: Bucket,
|
||||||
prefix: RemoteKey)
|
prefix: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[S3ObjectsData] = {
|
(implicit logger: Logger[M]): M[S3ObjectsData] = {
|
||||||
|
|
||||||
type Token = String
|
type Token = String
|
||||||
type Batch = (Stream[S3ObjectSummary], Option[Token])
|
type Batch = (Stream[S3ObjectSummary], Option[Token])
|
||||||
|
|
|
@ -18,14 +18,14 @@ class ThorpS3Client[M[_]: Monad](amazonS3Client: => AmazonS3,
|
||||||
|
|
||||||
override def listObjects(bucket: Bucket,
|
override def listObjects(bucket: Bucket,
|
||||||
prefix: RemoteKey)
|
prefix: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[S3ObjectsData] =
|
(implicit logger: Logger[M]): M[S3ObjectsData] =
|
||||||
objectLister.listObjects(bucket, prefix)
|
objectLister.listObjects(bucket, prefix)
|
||||||
|
|
||||||
override def copy(bucket: Bucket,
|
override def copy(bucket: Bucket,
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
hash: MD5Hash,
|
hash: MD5Hash,
|
||||||
targetKey: RemoteKey)
|
targetKey: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[CopyS3Action] =
|
(implicit logger: Logger[M]): M[CopyS3Action] =
|
||||||
copier.copy(bucket, sourceKey,hash, targetKey)
|
copier.copy(bucket, sourceKey,hash, targetKey)
|
||||||
|
|
||||||
override def upload(localFile: LocalFile,
|
override def upload(localFile: LocalFile,
|
||||||
|
@ -34,13 +34,12 @@ class ThorpS3Client[M[_]: Monad](amazonS3Client: => AmazonS3,
|
||||||
multiPartThreshold: Long,
|
multiPartThreshold: Long,
|
||||||
tryCount: Int,
|
tryCount: Int,
|
||||||
maxRetries: Int)
|
maxRetries: Int)
|
||||||
(implicit info: Int => String => M[Unit],
|
(implicit logger: Logger[M]): M[S3Action] =
|
||||||
warn: String => M[Unit]): M[S3Action] =
|
|
||||||
uploader.upload(localFile, bucket, progressListener, multiPartThreshold, 1, maxRetries)
|
uploader.upload(localFile, bucket, progressListener, multiPartThreshold, 1, maxRetries)
|
||||||
|
|
||||||
override def delete(bucket: Bucket,
|
override def delete(bucket: Bucket,
|
||||||
remoteKey: RemoteKey)
|
remoteKey: RemoteKey)
|
||||||
(implicit info: Int => String => M[Unit]): M[DeleteS3Action] =
|
(implicit logger: Logger[M]): M[DeleteS3Action] =
|
||||||
deleter.delete(bucket, remoteKey)
|
deleter.delete(bucket, remoteKey)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,7 @@ import net.kemitix.s3thorp.aws.api.S3Action.{ErroredS3Action, UploadS3Action}
|
||||||
import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent}
|
import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent}
|
||||||
import net.kemitix.s3thorp.aws.api.{S3Action, UploadProgressListener}
|
import net.kemitix.s3thorp.aws.api.{S3Action, UploadProgressListener}
|
||||||
import net.kemitix.s3thorp.aws.lib.UploaderLogging.{logMultiPartUploadFinished, logMultiPartUploadStart}
|
import net.kemitix.s3thorp.aws.lib.UploaderLogging.{logMultiPartUploadFinished, logMultiPartUploadStart}
|
||||||
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
|
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey}
|
||||||
|
|
||||||
import scala.util.Try
|
import scala.util.Try
|
||||||
|
|
||||||
|
@ -26,8 +26,7 @@ class Uploader[M[_]: Monad](transferManager: => AmazonTransferManager) {
|
||||||
multiPartThreshold: Long,
|
multiPartThreshold: Long,
|
||||||
tryCount: Int,
|
tryCount: Int,
|
||||||
maxRetries: Int)
|
maxRetries: Int)
|
||||||
(implicit info: Int => String => M[Unit],
|
(implicit logger: Logger[M]): M[S3Action] =
|
||||||
warn: String => M[Unit]): M[S3Action] =
|
|
||||||
for {
|
for {
|
||||||
_ <- logMultiPartUploadStart[M](localFile, tryCount)
|
_ <- logMultiPartUploadStart[M](localFile, tryCount)
|
||||||
upload <- transfer(localFile, bucket, uploadProgressListener)
|
upload <- transfer(localFile, bucket, uploadProgressListener)
|
||||||
|
|
|
@ -3,20 +3,20 @@ package net.kemitix.s3thorp.aws.lib
|
||||||
import cats.Monad
|
import cats.Monad
|
||||||
import net.kemitix.s3thorp.domain.Terminal.clearLine
|
import net.kemitix.s3thorp.domain.Terminal.clearLine
|
||||||
import net.kemitix.s3thorp.domain.SizeTranslation.sizeInEnglish
|
import net.kemitix.s3thorp.domain.SizeTranslation.sizeInEnglish
|
||||||
import net.kemitix.s3thorp.domain.LocalFile
|
import net.kemitix.s3thorp.domain.{LocalFile, Logger}
|
||||||
|
|
||||||
object UploaderLogging {
|
object UploaderLogging {
|
||||||
|
|
||||||
def logMultiPartUploadStart[M[_]: Monad](localFile: LocalFile,
|
def logMultiPartUploadStart[M[_]: Monad](localFile: LocalFile,
|
||||||
tryCount: Int)
|
tryCount: Int)
|
||||||
(implicit info: Int => String => M[Unit]): M[Unit] = {
|
(implicit logger: Logger[M]): M[Unit] = {
|
||||||
val tryMessage = if (tryCount == 1) "" else s"try $tryCount"
|
val tryMessage = if (tryCount == 1) "" else s"try $tryCount"
|
||||||
val size = sizeInEnglish(localFile.file.length)
|
val size = sizeInEnglish(localFile.file.length)
|
||||||
info(1)(s"${clearLine}upload:$tryMessage:$size:${localFile.remoteKey.key}")
|
logger.info(s"${clearLine}upload:$tryMessage:$size:${localFile.remoteKey.key}")
|
||||||
}
|
}
|
||||||
|
|
||||||
def logMultiPartUploadFinished[M[_]: Monad](localFile: LocalFile)
|
def logMultiPartUploadFinished[M[_]: Monad](localFile: LocalFile)
|
||||||
(implicit info: Int => String => M[Unit]): M[Unit] =
|
(implicit logger: Logger[M]): M[Unit] =
|
||||||
info(4)(s"upload:finished: ${localFile.remoteKey.key}")
|
logger.debug(s"upload:finished: ${localFile.remoteKey.key}")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
package net.kemitix.s3thorp.aws.lib
|
||||||
|
|
||||||
|
import cats.Monad
|
||||||
|
import net.kemitix.s3thorp.domain.Logger
|
||||||
|
|
||||||
|
class DummyLogger[M[_]: Monad] extends Logger[M] {
|
||||||
|
|
||||||
|
override def debug(message: => String): M[Unit] = Monad[M].unit
|
||||||
|
|
||||||
|
override def info(message: =>String): M[Unit] = Monad[M].unit
|
||||||
|
|
||||||
|
override def warn(message: String): M[Unit] = Monad[M].unit
|
||||||
|
|
||||||
|
override def error(message: String): M[Unit] = Monad[M].unit
|
||||||
|
|
||||||
|
}
|
|
@ -23,8 +23,7 @@ class S3ClientSuite
|
||||||
|
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
||||||
implicit private val logInfo: Int => String => Id[Unit] = _ => _ => ()
|
implicit private val implLogger: Logger[Id] = new DummyLogger[Id]
|
||||||
implicit private val logWarn: String => Id[Unit] = _ => ()
|
|
||||||
private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _
|
private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _
|
||||||
|
|
||||||
describe("getS3Status") {
|
describe("getS3Status") {
|
||||||
|
|
|
@ -21,7 +21,7 @@ class ThorpS3ClientSuite
|
||||||
val source = Resource(this, "upload")
|
val source = Resource(this, "upload")
|
||||||
val prefix = RemoteKey("prefix")
|
val prefix = RemoteKey("prefix")
|
||||||
implicit val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
implicit val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
||||||
implicit val logInfo: Int => String => Id[Unit] = _ => _ => ()
|
implicit val implLogger: Logger[Id] = new DummyLogger[Id]
|
||||||
|
|
||||||
val lm = LastModified(Instant.now.truncatedTo(ChronoUnit.MILLIS))
|
val lm = LastModified(Instant.now.truncatedTo(ChronoUnit.MILLIS))
|
||||||
|
|
||||||
|
|
|
@ -20,8 +20,7 @@ class UploaderSuite
|
||||||
private val source = Resource(this, ".")
|
private val source = Resource(this, ".")
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
||||||
implicit private val logInfo: Int => String => Id[Unit] = _ => _ => ()
|
implicit private val implLogger: Logger[Id] = new DummyLogger[Id]
|
||||||
implicit private val logWarn: String => Id[Unit] = _ => ()
|
|
||||||
private val fileToKey = generateKey(config.source, config.prefix) _
|
private val fileToKey = generateKey(config.source, config.prefix) _
|
||||||
val lastModified = LastModified(Instant.now())
|
val lastModified = LastModified(Instant.now())
|
||||||
|
|
||||||
|
|
|
@ -1,16 +0,0 @@
|
||||||
package net.kemitix.s3thorp.cli
|
|
||||||
|
|
||||||
import cats.Monad
|
|
||||||
|
|
||||||
|
|
||||||
class Logger[M[_]: Monad](verbosity: Int) {
|
|
||||||
|
|
||||||
def info(level: Int)(message: String): M[Unit] =
|
|
||||||
if (verbosity >= level) Monad[M].pure(println(s"[INFO:$level] $message"))
|
|
||||||
else Monad[M].unit
|
|
||||||
|
|
||||||
def warn(message: String): M[Unit] = Monad[M].pure(println(s"[ WARN] $message"))
|
|
||||||
|
|
||||||
def error(message: String): M[Unit] = Monad[M].pure(println(s"[ ERROR] $message"))
|
|
||||||
|
|
||||||
}
|
|
|
@ -12,14 +12,14 @@ object Main extends IOApp {
|
||||||
Config(source = Paths.get(".").toFile)
|
Config(source = Paths.get(".").toFile)
|
||||||
|
|
||||||
override def run(args: List[String]): IO[ExitCode] = {
|
override def run(args: List[String]): IO[ExitCode] = {
|
||||||
val logger = new Logger[IO](1)
|
val logger = new PrintLogger[IO](1)
|
||||||
ParseArgs(args, defaultConfig)
|
ParseArgs(args, defaultConfig)
|
||||||
.map(Program[IO])
|
.map(Program[IO])
|
||||||
.getOrElse(IO(ExitCode.Error))
|
.getOrElse(IO(ExitCode.Error))
|
||||||
.guaranteeCase {
|
.guaranteeCase {
|
||||||
case Canceled => logger.warn("Interrupted")
|
case Canceled => logger.warn("Interrupted")
|
||||||
case Error(e) => logger.error(e.getMessage)
|
case Error(e) => logger.error(e.getMessage)
|
||||||
case Completed => logger.info(1)("Done")
|
case Completed => logger.info("Done")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
16
cli/src/main/scala/net/kemitix/s3thorp/cli/PrintLogger.scala
Normal file
16
cli/src/main/scala/net/kemitix/s3thorp/cli/PrintLogger.scala
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
package net.kemitix.s3thorp.cli
|
||||||
|
|
||||||
|
import cats.Monad
|
||||||
|
import net.kemitix.s3thorp.domain.Logger
|
||||||
|
|
||||||
|
class PrintLogger[M[_]: Monad](verbosity: Int) extends Logger[M] {
|
||||||
|
|
||||||
|
override def debug(message: => String): M[Unit] = Monad[M].pure(println(s"[ DEBUG] $message"))
|
||||||
|
|
||||||
|
override def info(message: => String): M[Unit] = Monad[M].pure(println(s"[ INFO] $message"))
|
||||||
|
|
||||||
|
override def warn(message: String): M[Unit] = Monad[M].pure(println(s"[ WARN] $message"))
|
||||||
|
|
||||||
|
override def error(message: String): M[Unit] = Monad[M].pure(println(s"[ ERROR] $message"))
|
||||||
|
|
||||||
|
}
|
|
@ -3,27 +3,25 @@ package net.kemitix.s3thorp.cli
|
||||||
import java.io.File
|
import java.io.File
|
||||||
|
|
||||||
import cats.Monad
|
import cats.Monad
|
||||||
import cats.implicits._
|
|
||||||
import cats.effect.ExitCode
|
import cats.effect.ExitCode
|
||||||
|
import cats.implicits._
|
||||||
import net.kemitix.s3thorp.aws.lib.S3ClientBuilder
|
import net.kemitix.s3thorp.aws.lib.S3ClientBuilder
|
||||||
import net.kemitix.s3thorp.core.MD5HashGenerator.md5File
|
import net.kemitix.s3thorp.core.MD5HashGenerator.md5File
|
||||||
import net.kemitix.s3thorp.core.{MD5HashGenerator, Sync}
|
import net.kemitix.s3thorp.core.Sync
|
||||||
import net.kemitix.s3thorp.domain.Config
|
import net.kemitix.s3thorp.domain.{Config, Logger}
|
||||||
|
|
||||||
object Program {
|
object Program {
|
||||||
|
|
||||||
def apply[M[_]: Monad](config: Config): M[ExitCode] = {
|
def apply[M[_]: Monad](config: Config): M[ExitCode] = {
|
||||||
val logger = new Logger[M](config.verbose)
|
val logger = new PrintLogger[M](config.verbose)
|
||||||
val info = (l: Int) => (m: String) => logger.info(l) (m)
|
|
||||||
val warn = (w: String) => logger.warn(w)
|
|
||||||
for {
|
for {
|
||||||
_ <- info(1)("S3Thorp - hashed sync for s3")
|
_ <- logger.info("S3Thorp - hashed sync for s3")
|
||||||
_ <- Sync.run[M](config, S3ClientBuilder.defaultClient, hashGenerator(info), info, warn)
|
_ <- Sync.run[M](config, S3ClientBuilder.defaultClient, hashGenerator(logger), logger)
|
||||||
} yield ExitCode.Success
|
} yield ExitCode.Success
|
||||||
}
|
}
|
||||||
|
|
||||||
private def hashGenerator[M[_]: Monad](info: Int => String => M[Unit]) = {
|
private def hashGenerator[M[_]: Monad](logger: Logger[M]) = {
|
||||||
implicit val logInfo: Int => String => M[Unit] = info
|
implicit val impLogger: Logger[M] = logger
|
||||||
file: File => md5File[M](file)
|
file: File => md5File[M](file)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -5,30 +5,29 @@ import cats.implicits._
|
||||||
import net.kemitix.s3thorp.aws.api.S3Action.DoNothingS3Action
|
import net.kemitix.s3thorp.aws.api.S3Action.DoNothingS3Action
|
||||||
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client, UploadProgressListener}
|
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client, UploadProgressListener}
|
||||||
import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
||||||
import net.kemitix.s3thorp.domain.Config
|
import net.kemitix.s3thorp.domain.{Config, Logger}
|
||||||
|
|
||||||
object ActionSubmitter {
|
object ActionSubmitter {
|
||||||
|
|
||||||
def submitAction[M[_]: Monad](s3Client: S3Client[M], action: Action)
|
def submitAction[M[_]: Monad](s3Client: S3Client[M], action: Action)
|
||||||
(implicit c: Config,
|
(implicit c: Config,
|
||||||
info: Int => String => M[Unit],
|
logger: Logger[M]): Stream[M[S3Action]] = {
|
||||||
warn: String => M[Unit]): Stream[M[S3Action]] = {
|
|
||||||
Stream(
|
Stream(
|
||||||
action match {
|
action match {
|
||||||
case ToUpload(bucket, localFile) =>
|
case ToUpload(bucket, localFile) =>
|
||||||
for {
|
for {
|
||||||
_ <- info(4) (s" Upload: ${localFile.relative}")
|
_ <- logger.info(s" Upload: ${localFile.relative}")
|
||||||
progressListener = new UploadProgressListener(localFile)
|
progressListener = new UploadProgressListener(localFile)
|
||||||
action <- s3Client.upload(localFile, bucket, progressListener, c.multiPartThreshold, 1, c.maxRetries)
|
action <- s3Client.upload(localFile, bucket, progressListener, c.multiPartThreshold, 1, c.maxRetries)
|
||||||
} yield action
|
} yield action
|
||||||
case ToCopy(bucket, sourceKey, hash, targetKey) =>
|
case ToCopy(bucket, sourceKey, hash, targetKey) =>
|
||||||
for {
|
for {
|
||||||
_ <- info(4)(s" Copy: ${sourceKey.key} => ${targetKey.key}")
|
_ <- logger.info(s" Copy: ${sourceKey.key} => ${targetKey.key}")
|
||||||
action <- s3Client.copy(bucket, sourceKey, hash, targetKey)
|
action <- s3Client.copy(bucket, sourceKey, hash, targetKey)
|
||||||
} yield action
|
} yield action
|
||||||
case ToDelete(bucket, remoteKey) =>
|
case ToDelete(bucket, remoteKey) =>
|
||||||
for {
|
for {
|
||||||
_ <- info(4)(s" Delete: ${remoteKey.key}")
|
_ <- logger.info(s" Delete: ${remoteKey.key}")
|
||||||
action <- s3Client.delete(bucket, remoteKey)
|
action <- s3Client.delete(bucket, remoteKey)
|
||||||
} yield action
|
} yield action
|
||||||
case DoNothing(bucket, remoteKey) =>
|
case DoNothing(bucket, remoteKey) =>
|
||||||
|
|
|
@ -6,14 +6,14 @@ import java.nio.file.Path
|
||||||
import cats.Monad
|
import cats.Monad
|
||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
import net.kemitix.s3thorp.core.KeyGenerator.generateKey
|
import net.kemitix.s3thorp.core.KeyGenerator.generateKey
|
||||||
import net.kemitix.s3thorp.domain.{Config, Filter, LocalFile, MD5Hash}
|
import net.kemitix.s3thorp.domain.{Config, Filter, LocalFile, Logger, MD5Hash}
|
||||||
|
|
||||||
object LocalFileStream {
|
object LocalFileStream {
|
||||||
|
|
||||||
def findFiles[M[_]: Monad](file: File,
|
def findFiles[M[_]: Monad](file: File,
|
||||||
md5HashGenerator: File => M[MD5Hash],
|
md5HashGenerator: File => M[MD5Hash])
|
||||||
info: Int => String => M[Unit])
|
(implicit c: Config,
|
||||||
(implicit c: Config): M[Stream[LocalFile]] = {
|
logger: Logger[M]): M[Stream[LocalFile]] = {
|
||||||
|
|
||||||
val filters: Path => Boolean = Filter.isIncluded(c.filters)
|
val filters: Path => Boolean = Filter.isIncluded(c.filters)
|
||||||
|
|
||||||
|
@ -41,10 +41,10 @@ object LocalFileStream {
|
||||||
.flatMap(lfs => acc.map(s => s ++ lfs)))
|
.flatMap(lfs => acc.map(s => s ++ lfs)))
|
||||||
|
|
||||||
for {
|
for {
|
||||||
_ <- info(2)(s"- Entering: $file")
|
_ <- logger.info(s"- Entering: $file")
|
||||||
fs <- dirPaths(file)
|
fs <- dirPaths(file)
|
||||||
lfs <- recurse(fs)
|
lfs <- recurse(fs)
|
||||||
_ <- info(5)(s"- Leaving : $file")
|
_ <- logger.debug(s"- Leaving : $file")
|
||||||
} yield lfs
|
} yield lfs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -5,14 +5,14 @@ import java.security.MessageDigest
|
||||||
|
|
||||||
import cats.Monad
|
import cats.Monad
|
||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
import net.kemitix.s3thorp.domain.MD5Hash
|
import net.kemitix.s3thorp.domain.{Logger, MD5Hash}
|
||||||
|
|
||||||
import scala.collection.immutable.NumericRange
|
import scala.collection.immutable.NumericRange
|
||||||
|
|
||||||
object MD5HashGenerator {
|
object MD5HashGenerator {
|
||||||
|
|
||||||
def md5File[M[_]: Monad](file: File)
|
def md5File[M[_]: Monad](file: File)
|
||||||
(implicit info: Int => String => M[Unit]): M[MD5Hash] = {
|
(implicit logger: Logger[M]): M[MD5Hash] = {
|
||||||
|
|
||||||
val maxBufferSize = 8048
|
val maxBufferSize = 8048
|
||||||
val defaultBuffer = new Array[Byte](maxBufferSize)
|
val defaultBuffer = new Array[Byte](maxBufferSize)
|
||||||
|
@ -54,10 +54,10 @@ object MD5HashGenerator {
|
||||||
} yield md5
|
} yield md5
|
||||||
|
|
||||||
for {
|
for {
|
||||||
_ <- info(5)(s"md5:reading:size ${file.length}:$file")
|
_ <- logger.debug(s"md5:reading:size ${file.length}:$file")
|
||||||
md5 <- readFile
|
md5 <- readFile
|
||||||
hash = MD5Hash(md5)
|
hash = MD5Hash(md5)
|
||||||
_ <- info(4)(s"md5:generated:${hash.hash}:$file")
|
_ <- logger.debug(s"md5:generated:${hash.hash}:$file")
|
||||||
} yield hash
|
} yield hash
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -11,19 +11,17 @@ import net.kemitix.s3thorp.core.ActionSubmitter.submitAction
|
||||||
import net.kemitix.s3thorp.core.LocalFileStream.findFiles
|
import net.kemitix.s3thorp.core.LocalFileStream.findFiles
|
||||||
import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata
|
import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata
|
||||||
import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart}
|
import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart}
|
||||||
import net.kemitix.s3thorp.domain.{Config, LocalFile, MD5Hash, S3MetaData, S3ObjectsData}
|
import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash, S3MetaData, S3ObjectsData}
|
||||||
|
|
||||||
object Sync {
|
object Sync {
|
||||||
|
|
||||||
def run[M[_]: Monad](config: Config,
|
def run[M[_]: Monad](config: Config,
|
||||||
s3Client: S3Client[M],
|
s3Client: S3Client[M],
|
||||||
md5HashGenerator: File => M[MD5Hash],
|
md5HashGenerator: File => M[MD5Hash],
|
||||||
info: Int => String => M[Unit],
|
logger: Logger[M]): M[Unit] = {
|
||||||
warn: String => M[Unit]): M[Unit] = {
|
|
||||||
|
|
||||||
implicit val c: Config = config
|
implicit val c: Config = config
|
||||||
implicit val logInfo: Int => String => M[Unit] = info
|
implicit val implLogger: Logger[M] = logger
|
||||||
implicit val logWarn: String => M[Unit] = warn
|
|
||||||
|
|
||||||
def metaData(s3Data: S3ObjectsData, sFiles: Stream[LocalFile]) =
|
def metaData(s3Data: S3ObjectsData, sFiles: Stream[LocalFile]) =
|
||||||
Monad[M].pure(sFiles.map(file => getMetadata(file, s3Data)))
|
Monad[M].pure(sFiles.map(file => getMetadata(file, s3Data)))
|
||||||
|
@ -36,7 +34,7 @@ object Sync {
|
||||||
|
|
||||||
def copyUploadActions(s3Data: S3ObjectsData): M[Stream[S3Action]] =
|
def copyUploadActions(s3Data: S3ObjectsData): M[Stream[S3Action]] =
|
||||||
(for {
|
(for {
|
||||||
files <- findFiles(c.source, md5HashGenerator, info)
|
files <- findFiles(c.source, md5HashGenerator)
|
||||||
metaData <- metaData(s3Data, files)
|
metaData <- metaData(s3Data, files)
|
||||||
actions <- actions(metaData)
|
actions <- actions(metaData)
|
||||||
s3Actions <- submit(actions)
|
s3Actions <- submit(actions)
|
||||||
|
@ -54,12 +52,12 @@ object Sync {
|
||||||
.sequence
|
.sequence
|
||||||
|
|
||||||
for {
|
for {
|
||||||
_ <- logRunStart(info)
|
_ <- logRunStart[M]
|
||||||
s3data <- s3Client.listObjects(c.bucket, c.prefix)(info)
|
s3data <- s3Client.listObjects(c.bucket, c.prefix)
|
||||||
_ <- logFileScan(info)
|
_ <- logFileScan[M]
|
||||||
copyUploadActions <- copyUploadActions(s3data)
|
copyUploadActions <- copyUploadActions(s3data)
|
||||||
deleteActions <- deleteActions(s3data)
|
deleteActions <- deleteActions(s3data)
|
||||||
_ <- logRunFinished(copyUploadActions ++ deleteActions, info)
|
_ <- logRunFinished[M](copyUploadActions ++ deleteActions)
|
||||||
} yield ()
|
} yield ()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,28 +4,28 @@ import cats.Monad
|
||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
import net.kemitix.s3thorp.aws.api.S3Action
|
import net.kemitix.s3thorp.aws.api.S3Action
|
||||||
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, ErroredS3Action, UploadS3Action}
|
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, ErroredS3Action, UploadS3Action}
|
||||||
import net.kemitix.s3thorp.domain.Config
|
import net.kemitix.s3thorp.domain.{Config, Logger}
|
||||||
|
|
||||||
// Logging for the Sync class
|
// Logging for the Sync class
|
||||||
object SyncLogging {
|
object SyncLogging {
|
||||||
|
|
||||||
def logRunStart[M[_]: Monad](info: Int => String => M[Unit])
|
def logRunStart[M[_]: Monad](implicit c: Config,
|
||||||
(implicit c: Config): M[Unit] =
|
logger: Logger[M]): M[Unit] =
|
||||||
info(1)(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, ")
|
logger.info(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, ")
|
||||||
|
|
||||||
def logFileScan[M[_]: Monad](info: Int => String => M[Unit])
|
def logFileScan[M[_]: Monad](implicit c: Config,
|
||||||
(implicit c: Config): M[Unit] =
|
logger: Logger[M]): M[Unit] =
|
||||||
info(1)(s"Scanning local files: ${c.source}...")
|
logger.info(s"Scanning local files: ${c.source}...")
|
||||||
|
|
||||||
def logRunFinished[M[_]: Monad](actions: Stream[S3Action],
|
def logRunFinished[M[_]: Monad](actions: Stream[S3Action])
|
||||||
info: Int => String => M[Unit])
|
(implicit c: Config,
|
||||||
(implicit c: Config): M[Unit] = {
|
logger: Logger[M]): M[Unit] = {
|
||||||
val counters = actions.foldLeft(Counters())(countActivities)
|
val counters = actions.foldLeft(Counters())(countActivities)
|
||||||
for {
|
for {
|
||||||
_ <- info(1)(s"Uploaded ${counters.uploaded} files")
|
_ <- logger.info(s"Uploaded ${counters.uploaded} files")
|
||||||
_ <- info(1)(s"Copied ${counters.copied} files")
|
_ <- logger.info(s"Copied ${counters.copied} files")
|
||||||
_ <- info(1)(s"Deleted ${counters.deleted} files")
|
_ <- logger.info(s"Deleted ${counters.deleted} files")
|
||||||
_ <- info(1)(s"Errors ${counters.errors}")
|
_ <- logger.info(s"Errors ${counters.errors}")
|
||||||
} yield ()
|
} yield ()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,6 @@ class ActionGeneratorSuite
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
private val bucket = Bucket("bucket")
|
private val bucket = Bucket("bucket")
|
||||||
implicit private val config: Config = Config(bucket, prefix, source = source)
|
implicit private val config: Config = Config(bucket, prefix, source = source)
|
||||||
implicit private val logInfo: Int => String => Unit = l => i => ()
|
|
||||||
private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _
|
private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _
|
||||||
val lastModified = LastModified(Instant.now())
|
val lastModified = LastModified(Instant.now())
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
package net.kemitix.s3thorp.core
|
||||||
|
|
||||||
|
import cats.Monad
|
||||||
|
import net.kemitix.s3thorp.domain.Logger
|
||||||
|
|
||||||
|
class DummyLogger[M[_]: Monad] extends Logger[M] {
|
||||||
|
|
||||||
|
override def debug(message: => String): M[Unit] = Monad[M].unit
|
||||||
|
|
||||||
|
override def info(message: =>String): M[Unit] = Monad[M].unit
|
||||||
|
|
||||||
|
override def warn(message: String): M[Unit] = Monad[M].unit
|
||||||
|
|
||||||
|
override def error(message: String): M[Unit] = Monad[M].unit
|
||||||
|
|
||||||
|
}
|
|
@ -3,20 +3,20 @@ package net.kemitix.s3thorp.core
|
||||||
import java.io.File
|
import java.io.File
|
||||||
|
|
||||||
import cats.Id
|
import cats.Id
|
||||||
import net.kemitix.s3thorp.domain.{Config, LocalFile, MD5Hash}
|
import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash}
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
|
||||||
class LocalFileStreamSuite extends FunSpec {
|
class LocalFileStreamSuite extends FunSpec {
|
||||||
|
|
||||||
val uploadResource = Resource(this, "upload")
|
val uploadResource = Resource(this, "upload")
|
||||||
implicit val config: Config = Config(source = uploadResource)
|
implicit val config: Config = Config(source = uploadResource)
|
||||||
implicit private val logInfo: Int => String => Id[Unit] = l => i => ()
|
implicit private val logger: Logger[Id] = new DummyLogger[Id]
|
||||||
val md5HashGenerator: File => Id[MD5Hash] = file => MD5HashGenerator.md5File[Id](file)
|
val md5HashGenerator: File => Id[MD5Hash] = file => MD5HashGenerator.md5File[Id](file)
|
||||||
|
|
||||||
describe("findFiles") {
|
describe("findFiles") {
|
||||||
it("should find all files") {
|
it("should find all files") {
|
||||||
val result: Set[String] =
|
val result: Set[String] =
|
||||||
LocalFileStream.findFiles[Id](uploadResource, md5HashGenerator, logInfo).toSet
|
LocalFileStream.findFiles[Id](uploadResource, md5HashGenerator).toSet
|
||||||
.map { x: LocalFile => x.relative.toString }
|
.map { x: LocalFile => x.relative.toString }
|
||||||
assertResult(Set("subdir/leaf-file", "root-file"))(result)
|
assertResult(Set("subdir/leaf-file", "root-file"))(result)
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,7 +2,7 @@ package net.kemitix.s3thorp.core
|
||||||
|
|
||||||
import cats.Id
|
import cats.Id
|
||||||
import net.kemitix.s3thorp.core.MD5HashData.rootHash
|
import net.kemitix.s3thorp.core.MD5HashData.rootHash
|
||||||
import net.kemitix.s3thorp.domain.{Bucket, Config, MD5Hash, RemoteKey}
|
import net.kemitix.s3thorp.domain.{Bucket, Config, Logger, MD5Hash, RemoteKey}
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
|
||||||
class MD5HashGeneratorTest extends FunSpec {
|
class MD5HashGeneratorTest extends FunSpec {
|
||||||
|
@ -10,7 +10,7 @@ class MD5HashGeneratorTest extends FunSpec {
|
||||||
private val source = Resource(this, "upload")
|
private val source = Resource(this, "upload")
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
||||||
implicit private val logInfo: Int => String => Id[Unit] = l => i => ()
|
implicit private val logger: Logger[Id] = new DummyLogger[Id]
|
||||||
|
|
||||||
describe("read a small file (smaller than buffer)") {
|
describe("read a small file (smaller than buffer)") {
|
||||||
val file = Resource(this, "upload/root-file")
|
val file = Resource(this, "upload/root-file")
|
||||||
|
|
|
@ -12,7 +12,6 @@ class S3MetaDataEnricherSuite
|
||||||
private val source = Resource(this, "upload")
|
private val source = Resource(this, "upload")
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
|
||||||
implicit private val logInfo: Int => String => Unit = l => i => ()
|
|
||||||
private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _
|
private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _
|
||||||
val lastModified = LastModified(Instant.now())
|
val lastModified = LastModified(Instant.now())
|
||||||
|
|
||||||
|
|
|
@ -17,8 +17,7 @@ class SyncSuite
|
||||||
private val source = Resource(this, "upload")
|
private val source = Resource(this, "upload")
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
val config = Config(Bucket("bucket"), prefix, source = source)
|
val config = Config(Bucket("bucket"), prefix, source = source)
|
||||||
implicit private val logInfo: Int => String => Id[Unit] = _ => _ => ()
|
implicit private val logger: Logger[Id] = new DummyLogger[Id]
|
||||||
implicit private val logWarn: String => Id[Unit] = _ => ()
|
|
||||||
private val lastModified = LastModified(Instant.now)
|
private val lastModified = LastModified(Instant.now)
|
||||||
private val fileToKey: File => RemoteKey = KeyGenerator.generateKey(source, prefix)
|
private val fileToKey: File => RemoteKey = KeyGenerator.generateKey(source, prefix)
|
||||||
private val rootFile = LocalFile.resolve("root-file", rootHash, source, fileToKey)
|
private val rootFile = LocalFile.resolve("root-file", rootHash, source, fileToKey)
|
||||||
|
@ -38,7 +37,7 @@ class SyncSuite
|
||||||
val s3Client = new RecordingClient(testBucket, S3ObjectsData(
|
val s3Client = new RecordingClient(testBucket, S3ObjectsData(
|
||||||
byHash = Map(),
|
byHash = Map(),
|
||||||
byKey = Map()))
|
byKey = Map()))
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
|
Sync.run(config, s3Client, md5HashGenerator, logger)
|
||||||
it("uploads all files") {
|
it("uploads all files") {
|
||||||
val expectedUploads = Map(
|
val expectedUploads = Map(
|
||||||
"subdir/leaf-file" -> leafRemoteKey,
|
"subdir/leaf-file" -> leafRemoteKey,
|
||||||
|
@ -64,7 +63,7 @@ class SyncSuite
|
||||||
RemoteKey("prefix/root-file") -> HashModified(rootHash, lastModified),
|
RemoteKey("prefix/root-file") -> HashModified(rootHash, lastModified),
|
||||||
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
|
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
|
||||||
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
|
Sync.run(config, s3Client, md5HashGenerator, logger)
|
||||||
it("uploads nothing") {
|
it("uploads nothing") {
|
||||||
val expectedUploads = Map()
|
val expectedUploads = Map()
|
||||||
assertResult(expectedUploads)(s3Client.uploadsRecord)
|
assertResult(expectedUploads)(s3Client.uploadsRecord)
|
||||||
|
@ -88,7 +87,7 @@ class SyncSuite
|
||||||
RemoteKey("prefix/root-file-old") -> HashModified(rootHash, lastModified),
|
RemoteKey("prefix/root-file-old") -> HashModified(rootHash, lastModified),
|
||||||
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
|
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
|
||||||
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
|
Sync.run(config, s3Client, md5HashGenerator, logger)
|
||||||
it("uploads nothing") {
|
it("uploads nothing") {
|
||||||
val expectedUploads = Map()
|
val expectedUploads = Map()
|
||||||
assertResult(expectedUploads)(s3Client.uploadsRecord)
|
assertResult(expectedUploads)(s3Client.uploadsRecord)
|
||||||
|
@ -116,7 +115,7 @@ class SyncSuite
|
||||||
byKey = Map(
|
byKey = Map(
|
||||||
deletedKey -> HashModified(deletedHash, lastModified)))
|
deletedKey -> HashModified(deletedHash, lastModified)))
|
||||||
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
|
Sync.run(config, s3Client, md5HashGenerator, logger)
|
||||||
it("deleted key") {
|
it("deleted key") {
|
||||||
val expectedDeletions = Set(deletedKey)
|
val expectedDeletions = Set(deletedKey)
|
||||||
assertResult(expectedDeletions)(s3Client.deletionsRecord)
|
assertResult(expectedDeletions)(s3Client.deletionsRecord)
|
||||||
|
@ -126,7 +125,7 @@ class SyncSuite
|
||||||
val config: Config = Config(Bucket("bucket"), prefix, source = source, filters = List(Exclude("leaf")))
|
val config: Config = Config(Bucket("bucket"), prefix, source = source, filters = List(Exclude("leaf")))
|
||||||
val s3ObjectsData = S3ObjectsData(Map(), Map())
|
val s3ObjectsData = S3ObjectsData(Map(), Map())
|
||||||
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
|
Sync.run(config, s3Client, md5HashGenerator, logger)
|
||||||
it("is not uploaded") {
|
it("is not uploaded") {
|
||||||
val expectedUploads = Map(
|
val expectedUploads = Map(
|
||||||
"root-file" -> rootRemoteKey
|
"root-file" -> rootRemoteKey
|
||||||
|
@ -146,7 +145,7 @@ class SyncSuite
|
||||||
|
|
||||||
override def listObjects(bucket: Bucket,
|
override def listObjects(bucket: Bucket,
|
||||||
prefix: RemoteKey)
|
prefix: RemoteKey)
|
||||||
(implicit info: Int => String => Id[Unit]): S3ObjectsData =
|
(implicit logger: Logger[Id]): S3ObjectsData =
|
||||||
s3ObjectsData
|
s3ObjectsData
|
||||||
|
|
||||||
override def upload(localFile: LocalFile,
|
override def upload(localFile: LocalFile,
|
||||||
|
@ -155,8 +154,7 @@ class SyncSuite
|
||||||
multiPartThreshold: Long,
|
multiPartThreshold: Long,
|
||||||
tryCount: Int,
|
tryCount: Int,
|
||||||
maxRetries: Int)
|
maxRetries: Int)
|
||||||
(implicit info: Int => String => Id[Unit],
|
(implicit logger: Logger[Id]): UploadS3Action = {
|
||||||
warn: String => Id[Unit]): UploadS3Action = {
|
|
||||||
if (bucket == testBucket)
|
if (bucket == testBucket)
|
||||||
uploadsRecord += (localFile.relative.toString -> localFile.remoteKey)
|
uploadsRecord += (localFile.relative.toString -> localFile.remoteKey)
|
||||||
UploadS3Action(localFile.remoteKey, MD5Hash("some hash value"))
|
UploadS3Action(localFile.remoteKey, MD5Hash("some hash value"))
|
||||||
|
@ -166,7 +164,7 @@ class SyncSuite
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
hash: MD5Hash,
|
hash: MD5Hash,
|
||||||
targetKey: RemoteKey
|
targetKey: RemoteKey
|
||||||
)(implicit info: Int => String => Id[Unit]): CopyS3Action = {
|
)(implicit logger: Logger[Id]): CopyS3Action = {
|
||||||
if (bucket == testBucket)
|
if (bucket == testBucket)
|
||||||
copiesRecord += (sourceKey -> targetKey)
|
copiesRecord += (sourceKey -> targetKey)
|
||||||
CopyS3Action(targetKey)
|
CopyS3Action(targetKey)
|
||||||
|
@ -174,7 +172,7 @@ class SyncSuite
|
||||||
|
|
||||||
override def delete(bucket: Bucket,
|
override def delete(bucket: Bucket,
|
||||||
remoteKey: RemoteKey
|
remoteKey: RemoteKey
|
||||||
)(implicit info: Int => String => Id[Unit]): DeleteS3Action = {
|
)(implicit logger: Logger[Id]): DeleteS3Action = {
|
||||||
if (bucket == testBucket)
|
if (bucket == testBucket)
|
||||||
deletionsRecord += remoteKey
|
deletionsRecord += remoteKey
|
||||||
DeleteS3Action(remoteKey)
|
DeleteS3Action(remoteKey)
|
||||||
|
|
|
@ -0,0 +1,10 @@
|
||||||
|
package net.kemitix.s3thorp.domain
|
||||||
|
|
||||||
|
trait Logger[M[_]] {
|
||||||
|
|
||||||
|
def debug(message: => String): M[Unit]
|
||||||
|
def info(message: => String): M[Unit]
|
||||||
|
def warn(message: String): M[Unit]
|
||||||
|
def error(message: String): M[Unit]
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in a new issue