From 21b891739543965402986218900dbc9fe91c2338 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 14 Jun 2019 17:18:53 +0100 Subject: [PATCH] Simplify logging (#59) * [cli] Logger add debug method * [cli] Logger: add alternate info method * [cli] replace direct calls to legacy Logger.info * [cli] Extract trait from Logger into core and rename * [cli] Program use debug and new info * [core] MD5HashGenerator uses Logger * [core] Sync uses Logger * [core] ActionSubmitter uses Logger * [core] LocalFileStream uses Logger * [core] SyncLogging uses Logger * [domain] Move Logger into module The allows it to be used in the aws-api module * [aws-lib] S3ClientObjectLister uses Logger * [aws-lib] Uploader uses Logger * [aws-lib] S3ClientDeleter uses Logger * [aws-lib] S3ClientCopier uses Logger * [core] remove unused legacy logging * [aws-lib] remove used logging method * [cli] PrintLogger remove legacy info method --- .../kemitix/s3thorp/aws/api/S3Client.scala | 11 +++---- .../s3thorp/aws/lib/S3ClientCopier.scala | 4 +-- .../s3thorp/aws/lib/S3ClientDeleter.scala | 4 +-- .../s3thorp/aws/lib/S3ClientLogging.scala | 31 ++++++++----------- .../aws/lib/S3ClientObjectLister.scala | 2 +- .../s3thorp/aws/lib/ThorpS3Client.scala | 9 +++--- .../kemitix/s3thorp/aws/lib/Uploader.scala | 5 ++- .../s3thorp/aws/lib/UploaderLogging.scala | 10 +++--- .../kemitix/s3thorp/aws/lib/DummyLogger.scala | 16 ++++++++++ .../s3thorp/aws/lib/S3ClientSuite.scala | 3 +- .../s3thorp/aws/lib/ThorpS3ClientSuite.scala | 2 +- .../s3thorp/aws/lib/UploaderSuite.scala | 3 +- .../net/kemitix/s3thorp/cli/Logger.scala | 16 ---------- .../scala/net/kemitix/s3thorp/cli/Main.scala | 4 +-- .../net/kemitix/s3thorp/cli/PrintLogger.scala | 16 ++++++++++ .../net/kemitix/s3thorp/cli/Program.scala | 18 +++++------ .../ActionSubmitter.scala | 11 +++---- .../LocalFileStream.scala | 12 +++---- .../MD5HashGenerator.scala | 8 ++--- .../scala/net.kemitix.s3thorp.core/Sync.scala | 18 +++++------ .../SyncLogging.scala | 28 ++++++++--------- .../s3thorp/core/ActionGeneratorSuite.scala | 1 - .../kemitix/s3thorp/core/DummyLogger.scala | 16 ++++++++++ .../s3thorp/core/LocalFileStreamSuite.scala | 6 ++-- .../s3thorp/core/MD5HashGeneratorTest.scala | 4 +-- .../core/S3MetaDataEnricherSuite.scala | 1 - .../net/kemitix/s3thorp/core/SyncSuite.scala | 22 ++++++------- .../net/kemitix/s3thorp/domain/Logger.scala | 10 ++++++ 28 files changed, 157 insertions(+), 134 deletions(-) create mode 100644 aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/DummyLogger.scala delete mode 100644 cli/src/main/scala/net/kemitix/s3thorp/cli/Logger.scala create mode 100644 cli/src/main/scala/net/kemitix/s3thorp/cli/PrintLogger.scala create mode 100644 core/src/test/scala/net/kemitix/s3thorp/core/DummyLogger.scala create mode 100644 domain/src/main/scala/net/kemitix/s3thorp/domain/Logger.scala diff --git a/aws-api/src/main/scala/net/kemitix/s3thorp/aws/api/S3Client.scala b/aws-api/src/main/scala/net/kemitix/s3thorp/aws/api/S3Client.scala index ebe2fb1..7835490 100644 --- a/aws-api/src/main/scala/net/kemitix/s3thorp/aws/api/S3Client.scala +++ b/aws-api/src/main/scala/net/kemitix/s3thorp/aws/api/S3Client.scala @@ -1,13 +1,13 @@ package net.kemitix.s3thorp.aws.api import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action} -import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey, S3ObjectsData} +import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey, S3ObjectsData} trait S3Client[M[_]] { def listObjects(bucket: Bucket, prefix: RemoteKey - )(implicit info: Int => String => M[Unit]): M[S3ObjectsData] + )(implicit logger: Logger[M]): M[S3ObjectsData] def upload(localFile: LocalFile, bucket: Bucket, @@ -15,17 +15,16 @@ trait S3Client[M[_]] { multiPartThreshold: Long, tryCount: Int, maxRetries: Int) - (implicit info: Int => String => M[Unit], - warn: String => M[Unit]): M[S3Action] + (implicit logger: Logger[M]): M[S3Action] def copy(bucket: Bucket, sourceKey: RemoteKey, hash: MD5Hash, targetKey: RemoteKey - )(implicit info: Int => String => M[Unit]): M[CopyS3Action] + )(implicit logger: Logger[M]): M[CopyS3Action] def delete(bucket: Bucket, remoteKey: RemoteKey - )(implicit info: Int => String => M[Unit]): M[DeleteS3Action] + )(implicit logger: Logger[M]): M[DeleteS3Action] } diff --git a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientCopier.scala b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientCopier.scala index 1ee9c32..ae7e638 100644 --- a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientCopier.scala +++ b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientCopier.scala @@ -6,7 +6,7 @@ import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.model.CopyObjectRequest import net.kemitix.s3thorp.aws.api.S3Action.CopyS3Action import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logCopyFinish, logCopyStart} -import net.kemitix.s3thorp.domain.{Bucket, MD5Hash, RemoteKey} +import net.kemitix.s3thorp.domain.{Bucket, Logger, MD5Hash, RemoteKey} class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) { @@ -14,7 +14,7 @@ class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) { sourceKey: RemoteKey, hash: MD5Hash, targetKey: RemoteKey) - (implicit info: Int => String => M[Unit]): M[CopyS3Action] = + (implicit logger: Logger[M]): M[CopyS3Action] = for { _ <- logCopyStart[M](bucket, sourceKey, targetKey) _ <- copyObject(bucket, sourceKey, hash, targetKey) diff --git a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientDeleter.scala b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientDeleter.scala index 170a049..0cae593 100644 --- a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientDeleter.scala +++ b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientDeleter.scala @@ -6,13 +6,13 @@ import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.model.DeleteObjectRequest import net.kemitix.s3thorp.aws.api.S3Action.DeleteS3Action import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logDeleteFinish, logDeleteStart} -import net.kemitix.s3thorp.domain.{Bucket, RemoteKey} +import net.kemitix.s3thorp.domain.{Bucket, Logger, RemoteKey} class S3ClientDeleter[M[_]: Monad](amazonS3: AmazonS3) { def delete(bucket: Bucket, remoteKey: RemoteKey) - (implicit info: Int => String => M[Unit]): M[DeleteS3Action] = + (implicit logger: Logger[M]): M[DeleteS3Action] = for { _ <- logDeleteStart[M](bucket, remoteKey) _ <- deleteObject(bucket, remoteKey) diff --git a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientLogging.scala b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientLogging.scala index 4e9669d..c56cdba 100644 --- a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientLogging.scala +++ b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientLogging.scala @@ -2,45 +2,40 @@ package net.kemitix.s3thorp.aws.lib import cats.Monad import com.amazonaws.services.s3.model.PutObjectResult -import net.kemitix.s3thorp.domain.{Bucket, LocalFile, RemoteKey} +import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, RemoteKey} object S3ClientLogging { def logListObjectsStart[M[_]: Monad](bucket: Bucket, prefix: RemoteKey) - (implicit info: Int => String => M[Unit]): M[Unit] = - info(1)(s"Fetch S3 Summary: ${bucket.name}:${prefix.key}") + (implicit logger: Logger[M]): M[Unit] = + logger.info(s"Fetch S3 Summary: ${bucket.name}:${prefix.key}") def logListObjectsFinish[M[_]: Monad](bucket: Bucket, prefix: RemoteKey) - (implicit info: Int => String => M[Unit]): M[Unit] = - info(2)(s"Fetched S3 Summary: ${bucket.name}:${prefix.key}") - - def logUploadFinish[M[_]: Monad](localFile: LocalFile, - bucket: Bucket) - (implicit info: Int => String => M[Unit]): PutObjectResult => M[Unit] = - _ => info(2)(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}") + (implicit logger: Logger[M]): M[Unit] = + logger.info(s"Fetched S3 Summary: ${bucket.name}:${prefix.key}") def logCopyStart[M[_]: Monad](bucket: Bucket, sourceKey: RemoteKey, targetKey: RemoteKey) - (implicit info: Int => String => M[Unit]): M[Unit] = - info(1)(s"Copy: ${bucket.name}:${sourceKey.key} => ${targetKey.key}") + (implicit logger: Logger[M]): M[Unit] = + logger.info(s"Copy: ${bucket.name}:${sourceKey.key} => ${targetKey.key}") def logCopyFinish[M[_]: Monad](bucket: Bucket, sourceKey: RemoteKey, targetKey: RemoteKey) - (implicit info: Int => String => M[Unit]): M[Unit] = - info(2)(s"Copied: ${bucket.name}:${sourceKey.key} => ${targetKey.key}") + (implicit logger: Logger[M]): M[Unit] = + logger.info(s"Copied: ${bucket.name}:${sourceKey.key} => ${targetKey.key}") def logDeleteStart[M[_]: Monad](bucket: Bucket, remoteKey: RemoteKey) - (implicit info: Int => String => M[Unit]): M[Unit] = - info(1)(s"Delete: ${bucket.name}:${remoteKey.key}") + (implicit logger: Logger[M]): M[Unit] = + logger.info(s"Delete: ${bucket.name}:${remoteKey.key}") def logDeleteFinish[M[_]: Monad](bucket: Bucket, remoteKey: RemoteKey) - (implicit info: Int => String => M[Unit]): M[Unit] = - info(2)(s"Deleted: ${bucket.name}:${remoteKey.key}") + (implicit logger: Logger[M]): M[Unit] = + logger.info(s"Deleted: ${bucket.name}:${remoteKey.key}") } diff --git a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientObjectLister.scala b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientObjectLister.scala index dd5898c..272d90d 100644 --- a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientObjectLister.scala +++ b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientObjectLister.scala @@ -15,7 +15,7 @@ class S3ClientObjectLister[M[_]: Monad](amazonS3: AmazonS3) { def listObjects(bucket: Bucket, prefix: RemoteKey) - (implicit info: Int => String => M[Unit]): M[S3ObjectsData] = { + (implicit logger: Logger[M]): M[S3ObjectsData] = { type Token = String type Batch = (Stream[S3ObjectSummary], Option[Token]) diff --git a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/ThorpS3Client.scala b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/ThorpS3Client.scala index b821c34..cfa5456 100644 --- a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/ThorpS3Client.scala +++ b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/ThorpS3Client.scala @@ -18,14 +18,14 @@ class ThorpS3Client[M[_]: Monad](amazonS3Client: => AmazonS3, override def listObjects(bucket: Bucket, prefix: RemoteKey) - (implicit info: Int => String => M[Unit]): M[S3ObjectsData] = + (implicit logger: Logger[M]): M[S3ObjectsData] = objectLister.listObjects(bucket, prefix) override def copy(bucket: Bucket, sourceKey: RemoteKey, hash: MD5Hash, targetKey: RemoteKey) - (implicit info: Int => String => M[Unit]): M[CopyS3Action] = + (implicit logger: Logger[M]): M[CopyS3Action] = copier.copy(bucket, sourceKey,hash, targetKey) override def upload(localFile: LocalFile, @@ -34,13 +34,12 @@ class ThorpS3Client[M[_]: Monad](amazonS3Client: => AmazonS3, multiPartThreshold: Long, tryCount: Int, maxRetries: Int) - (implicit info: Int => String => M[Unit], - warn: String => M[Unit]): M[S3Action] = + (implicit logger: Logger[M]): M[S3Action] = uploader.upload(localFile, bucket, progressListener, multiPartThreshold, 1, maxRetries) override def delete(bucket: Bucket, remoteKey: RemoteKey) - (implicit info: Int => String => M[Unit]): M[DeleteS3Action] = + (implicit logger: Logger[M]): M[DeleteS3Action] = deleter.delete(bucket, remoteKey) } diff --git a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/Uploader.scala b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/Uploader.scala index 3576bfa..98b928e 100644 --- a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/Uploader.scala +++ b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/Uploader.scala @@ -10,7 +10,7 @@ import net.kemitix.s3thorp.aws.api.S3Action.{ErroredS3Action, UploadS3Action} import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent} import net.kemitix.s3thorp.aws.api.{S3Action, UploadProgressListener} import net.kemitix.s3thorp.aws.lib.UploaderLogging.{logMultiPartUploadFinished, logMultiPartUploadStart} -import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey} +import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey} import scala.util.Try @@ -26,8 +26,7 @@ class Uploader[M[_]: Monad](transferManager: => AmazonTransferManager) { multiPartThreshold: Long, tryCount: Int, maxRetries: Int) - (implicit info: Int => String => M[Unit], - warn: String => M[Unit]): M[S3Action] = + (implicit logger: Logger[M]): M[S3Action] = for { _ <- logMultiPartUploadStart[M](localFile, tryCount) upload <- transfer(localFile, bucket, uploadProgressListener) diff --git a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/UploaderLogging.scala b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/UploaderLogging.scala index 207c111..ab3d8b1 100644 --- a/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/UploaderLogging.scala +++ b/aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/UploaderLogging.scala @@ -3,20 +3,20 @@ package net.kemitix.s3thorp.aws.lib import cats.Monad import net.kemitix.s3thorp.domain.Terminal.clearLine import net.kemitix.s3thorp.domain.SizeTranslation.sizeInEnglish -import net.kemitix.s3thorp.domain.LocalFile +import net.kemitix.s3thorp.domain.{LocalFile, Logger} object UploaderLogging { def logMultiPartUploadStart[M[_]: Monad](localFile: LocalFile, tryCount: Int) - (implicit info: Int => String => M[Unit]): M[Unit] = { + (implicit logger: Logger[M]): M[Unit] = { val tryMessage = if (tryCount == 1) "" else s"try $tryCount" val size = sizeInEnglish(localFile.file.length) - info(1)(s"${clearLine}upload:$tryMessage:$size:${localFile.remoteKey.key}") + logger.info(s"${clearLine}upload:$tryMessage:$size:${localFile.remoteKey.key}") } def logMultiPartUploadFinished[M[_]: Monad](localFile: LocalFile) - (implicit info: Int => String => M[Unit]): M[Unit] = - info(4)(s"upload:finished: ${localFile.remoteKey.key}") + (implicit logger: Logger[M]): M[Unit] = + logger.debug(s"upload:finished: ${localFile.remoteKey.key}") } diff --git a/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/DummyLogger.scala b/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/DummyLogger.scala new file mode 100644 index 0000000..ff4cf04 --- /dev/null +++ b/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/DummyLogger.scala @@ -0,0 +1,16 @@ +package net.kemitix.s3thorp.aws.lib + +import cats.Monad +import net.kemitix.s3thorp.domain.Logger + +class DummyLogger[M[_]: Monad] extends Logger[M] { + + override def debug(message: => String): M[Unit] = Monad[M].unit + + override def info(message: =>String): M[Unit] = Monad[M].unit + + override def warn(message: String): M[Unit] = Monad[M].unit + + override def error(message: String): M[Unit] = Monad[M].unit + +} diff --git a/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/S3ClientSuite.scala b/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/S3ClientSuite.scala index c4a0690..f2c8821 100644 --- a/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/S3ClientSuite.scala +++ b/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/S3ClientSuite.scala @@ -23,8 +23,7 @@ class S3ClientSuite private val prefix = RemoteKey("prefix") implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source) - implicit private val logInfo: Int => String => Id[Unit] = _ => _ => () - implicit private val logWarn: String => Id[Unit] = _ => () + implicit private val implLogger: Logger[Id] = new DummyLogger[Id] private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _ describe("getS3Status") { diff --git a/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/ThorpS3ClientSuite.scala b/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/ThorpS3ClientSuite.scala index 86bc114..03bd7d7 100644 --- a/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/ThorpS3ClientSuite.scala +++ b/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/ThorpS3ClientSuite.scala @@ -21,7 +21,7 @@ class ThorpS3ClientSuite val source = Resource(this, "upload") val prefix = RemoteKey("prefix") implicit val config: Config = Config(Bucket("bucket"), prefix, source = source) - implicit val logInfo: Int => String => Id[Unit] = _ => _ => () + implicit val implLogger: Logger[Id] = new DummyLogger[Id] val lm = LastModified(Instant.now.truncatedTo(ChronoUnit.MILLIS)) diff --git a/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/UploaderSuite.scala b/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/UploaderSuite.scala index 0e681cb..d2d5cde 100644 --- a/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/UploaderSuite.scala +++ b/aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/UploaderSuite.scala @@ -20,8 +20,7 @@ class UploaderSuite private val source = Resource(this, ".") private val prefix = RemoteKey("prefix") implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source) - implicit private val logInfo: Int => String => Id[Unit] = _ => _ => () - implicit private val logWarn: String => Id[Unit] = _ => () + implicit private val implLogger: Logger[Id] = new DummyLogger[Id] private val fileToKey = generateKey(config.source, config.prefix) _ val lastModified = LastModified(Instant.now()) diff --git a/cli/src/main/scala/net/kemitix/s3thorp/cli/Logger.scala b/cli/src/main/scala/net/kemitix/s3thorp/cli/Logger.scala deleted file mode 100644 index 6f8fb89..0000000 --- a/cli/src/main/scala/net/kemitix/s3thorp/cli/Logger.scala +++ /dev/null @@ -1,16 +0,0 @@ -package net.kemitix.s3thorp.cli - -import cats.Monad - - -class Logger[M[_]: Monad](verbosity: Int) { - - def info(level: Int)(message: String): M[Unit] = - if (verbosity >= level) Monad[M].pure(println(s"[INFO:$level] $message")) - else Monad[M].unit - - def warn(message: String): M[Unit] = Monad[M].pure(println(s"[ WARN] $message")) - - def error(message: String): M[Unit] = Monad[M].pure(println(s"[ ERROR] $message")) - -} diff --git a/cli/src/main/scala/net/kemitix/s3thorp/cli/Main.scala b/cli/src/main/scala/net/kemitix/s3thorp/cli/Main.scala index 90b5349..bfa42fd 100644 --- a/cli/src/main/scala/net/kemitix/s3thorp/cli/Main.scala +++ b/cli/src/main/scala/net/kemitix/s3thorp/cli/Main.scala @@ -12,14 +12,14 @@ object Main extends IOApp { Config(source = Paths.get(".").toFile) override def run(args: List[String]): IO[ExitCode] = { - val logger = new Logger[IO](1) + val logger = new PrintLogger[IO](1) ParseArgs(args, defaultConfig) .map(Program[IO]) .getOrElse(IO(ExitCode.Error)) .guaranteeCase { case Canceled => logger.warn("Interrupted") case Error(e) => logger.error(e.getMessage) - case Completed => logger.info(1)("Done") + case Completed => logger.info("Done") } } diff --git a/cli/src/main/scala/net/kemitix/s3thorp/cli/PrintLogger.scala b/cli/src/main/scala/net/kemitix/s3thorp/cli/PrintLogger.scala new file mode 100644 index 0000000..53ad895 --- /dev/null +++ b/cli/src/main/scala/net/kemitix/s3thorp/cli/PrintLogger.scala @@ -0,0 +1,16 @@ +package net.kemitix.s3thorp.cli + +import cats.Monad +import net.kemitix.s3thorp.domain.Logger + +class PrintLogger[M[_]: Monad](verbosity: Int) extends Logger[M] { + + override def debug(message: => String): M[Unit] = Monad[M].pure(println(s"[ DEBUG] $message")) + + override def info(message: => String): M[Unit] = Monad[M].pure(println(s"[ INFO] $message")) + + override def warn(message: String): M[Unit] = Monad[M].pure(println(s"[ WARN] $message")) + + override def error(message: String): M[Unit] = Monad[M].pure(println(s"[ ERROR] $message")) + +} diff --git a/cli/src/main/scala/net/kemitix/s3thorp/cli/Program.scala b/cli/src/main/scala/net/kemitix/s3thorp/cli/Program.scala index c6318a5..0f222da 100644 --- a/cli/src/main/scala/net/kemitix/s3thorp/cli/Program.scala +++ b/cli/src/main/scala/net/kemitix/s3thorp/cli/Program.scala @@ -3,27 +3,25 @@ package net.kemitix.s3thorp.cli import java.io.File import cats.Monad -import cats.implicits._ import cats.effect.ExitCode +import cats.implicits._ import net.kemitix.s3thorp.aws.lib.S3ClientBuilder import net.kemitix.s3thorp.core.MD5HashGenerator.md5File -import net.kemitix.s3thorp.core.{MD5HashGenerator, Sync} -import net.kemitix.s3thorp.domain.Config +import net.kemitix.s3thorp.core.Sync +import net.kemitix.s3thorp.domain.{Config, Logger} object Program { def apply[M[_]: Monad](config: Config): M[ExitCode] = { - val logger = new Logger[M](config.verbose) - val info = (l: Int) => (m: String) => logger.info(l) (m) - val warn = (w: String) => logger.warn(w) + val logger = new PrintLogger[M](config.verbose) for { - _ <- info(1)("S3Thorp - hashed sync for s3") - _ <- Sync.run[M](config, S3ClientBuilder.defaultClient, hashGenerator(info), info, warn) + _ <- logger.info("S3Thorp - hashed sync for s3") + _ <- Sync.run[M](config, S3ClientBuilder.defaultClient, hashGenerator(logger), logger) } yield ExitCode.Success } - private def hashGenerator[M[_]: Monad](info: Int => String => M[Unit]) = { - implicit val logInfo: Int => String => M[Unit] = info + private def hashGenerator[M[_]: Monad](logger: Logger[M]) = { + implicit val impLogger: Logger[M] = logger file: File => md5File[M](file) } diff --git a/core/src/main/scala/net.kemitix.s3thorp.core/ActionSubmitter.scala b/core/src/main/scala/net.kemitix.s3thorp.core/ActionSubmitter.scala index b15ee6b..2e05f83 100644 --- a/core/src/main/scala/net.kemitix.s3thorp.core/ActionSubmitter.scala +++ b/core/src/main/scala/net.kemitix.s3thorp.core/ActionSubmitter.scala @@ -5,30 +5,29 @@ import cats.implicits._ import net.kemitix.s3thorp.aws.api.S3Action.DoNothingS3Action import net.kemitix.s3thorp.aws.api.{S3Action, S3Client, UploadProgressListener} import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload} -import net.kemitix.s3thorp.domain.Config +import net.kemitix.s3thorp.domain.{Config, Logger} object ActionSubmitter { def submitAction[M[_]: Monad](s3Client: S3Client[M], action: Action) (implicit c: Config, - info: Int => String => M[Unit], - warn: String => M[Unit]): Stream[M[S3Action]] = { + logger: Logger[M]): Stream[M[S3Action]] = { Stream( action match { case ToUpload(bucket, localFile) => for { - _ <- info(4) (s" Upload: ${localFile.relative}") + _ <- logger.info(s" Upload: ${localFile.relative}") progressListener = new UploadProgressListener(localFile) action <- s3Client.upload(localFile, bucket, progressListener, c.multiPartThreshold, 1, c.maxRetries) } yield action case ToCopy(bucket, sourceKey, hash, targetKey) => for { - _ <- info(4)(s" Copy: ${sourceKey.key} => ${targetKey.key}") + _ <- logger.info(s" Copy: ${sourceKey.key} => ${targetKey.key}") action <- s3Client.copy(bucket, sourceKey, hash, targetKey) } yield action case ToDelete(bucket, remoteKey) => for { - _ <- info(4)(s" Delete: ${remoteKey.key}") + _ <- logger.info(s" Delete: ${remoteKey.key}") action <- s3Client.delete(bucket, remoteKey) } yield action case DoNothing(bucket, remoteKey) => diff --git a/core/src/main/scala/net.kemitix.s3thorp.core/LocalFileStream.scala b/core/src/main/scala/net.kemitix.s3thorp.core/LocalFileStream.scala index 10f9f5f..8694536 100644 --- a/core/src/main/scala/net.kemitix.s3thorp.core/LocalFileStream.scala +++ b/core/src/main/scala/net.kemitix.s3thorp.core/LocalFileStream.scala @@ -6,14 +6,14 @@ import java.nio.file.Path import cats.Monad import cats.implicits._ import net.kemitix.s3thorp.core.KeyGenerator.generateKey -import net.kemitix.s3thorp.domain.{Config, Filter, LocalFile, MD5Hash} +import net.kemitix.s3thorp.domain.{Config, Filter, LocalFile, Logger, MD5Hash} object LocalFileStream { def findFiles[M[_]: Monad](file: File, - md5HashGenerator: File => M[MD5Hash], - info: Int => String => M[Unit]) - (implicit c: Config): M[Stream[LocalFile]] = { + md5HashGenerator: File => M[MD5Hash]) + (implicit c: Config, + logger: Logger[M]): M[Stream[LocalFile]] = { val filters: Path => Boolean = Filter.isIncluded(c.filters) @@ -41,10 +41,10 @@ object LocalFileStream { .flatMap(lfs => acc.map(s => s ++ lfs))) for { - _ <- info(2)(s"- Entering: $file") + _ <- logger.info(s"- Entering: $file") fs <- dirPaths(file) lfs <- recurse(fs) - _ <- info(5)(s"- Leaving : $file") + _ <- logger.debug(s"- Leaving : $file") } yield lfs } diff --git a/core/src/main/scala/net.kemitix.s3thorp.core/MD5HashGenerator.scala b/core/src/main/scala/net.kemitix.s3thorp.core/MD5HashGenerator.scala index 85f8ee3..95ac20b 100644 --- a/core/src/main/scala/net.kemitix.s3thorp.core/MD5HashGenerator.scala +++ b/core/src/main/scala/net.kemitix.s3thorp.core/MD5HashGenerator.scala @@ -5,14 +5,14 @@ import java.security.MessageDigest import cats.Monad import cats.implicits._ -import net.kemitix.s3thorp.domain.MD5Hash +import net.kemitix.s3thorp.domain.{Logger, MD5Hash} import scala.collection.immutable.NumericRange object MD5HashGenerator { def md5File[M[_]: Monad](file: File) - (implicit info: Int => String => M[Unit]): M[MD5Hash] = { + (implicit logger: Logger[M]): M[MD5Hash] = { val maxBufferSize = 8048 val defaultBuffer = new Array[Byte](maxBufferSize) @@ -54,10 +54,10 @@ object MD5HashGenerator { } yield md5 for { - _ <- info(5)(s"md5:reading:size ${file.length}:$file") + _ <- logger.debug(s"md5:reading:size ${file.length}:$file") md5 <- readFile hash = MD5Hash(md5) - _ <- info(4)(s"md5:generated:${hash.hash}:$file") + _ <- logger.debug(s"md5:generated:${hash.hash}:$file") } yield hash } diff --git a/core/src/main/scala/net.kemitix.s3thorp.core/Sync.scala b/core/src/main/scala/net.kemitix.s3thorp.core/Sync.scala index 0269b47..f509563 100644 --- a/core/src/main/scala/net.kemitix.s3thorp.core/Sync.scala +++ b/core/src/main/scala/net.kemitix.s3thorp.core/Sync.scala @@ -11,19 +11,17 @@ import net.kemitix.s3thorp.core.ActionSubmitter.submitAction import net.kemitix.s3thorp.core.LocalFileStream.findFiles import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart} -import net.kemitix.s3thorp.domain.{Config, LocalFile, MD5Hash, S3MetaData, S3ObjectsData} +import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash, S3MetaData, S3ObjectsData} object Sync { def run[M[_]: Monad](config: Config, s3Client: S3Client[M], md5HashGenerator: File => M[MD5Hash], - info: Int => String => M[Unit], - warn: String => M[Unit]): M[Unit] = { + logger: Logger[M]): M[Unit] = { implicit val c: Config = config - implicit val logInfo: Int => String => M[Unit] = info - implicit val logWarn: String => M[Unit] = warn + implicit val implLogger: Logger[M] = logger def metaData(s3Data: S3ObjectsData, sFiles: Stream[LocalFile]) = Monad[M].pure(sFiles.map(file => getMetadata(file, s3Data))) @@ -36,7 +34,7 @@ object Sync { def copyUploadActions(s3Data: S3ObjectsData): M[Stream[S3Action]] = (for { - files <- findFiles(c.source, md5HashGenerator, info) + files <- findFiles(c.source, md5HashGenerator) metaData <- metaData(s3Data, files) actions <- actions(metaData) s3Actions <- submit(actions) @@ -54,12 +52,12 @@ object Sync { .sequence for { - _ <- logRunStart(info) - s3data <- s3Client.listObjects(c.bucket, c.prefix)(info) - _ <- logFileScan(info) + _ <- logRunStart[M] + s3data <- s3Client.listObjects(c.bucket, c.prefix) + _ <- logFileScan[M] copyUploadActions <- copyUploadActions(s3data) deleteActions <- deleteActions(s3data) - _ <- logRunFinished(copyUploadActions ++ deleteActions, info) + _ <- logRunFinished[M](copyUploadActions ++ deleteActions) } yield () } diff --git a/core/src/main/scala/net.kemitix.s3thorp.core/SyncLogging.scala b/core/src/main/scala/net.kemitix.s3thorp.core/SyncLogging.scala index 806ac16..e121b2b 100644 --- a/core/src/main/scala/net.kemitix.s3thorp.core/SyncLogging.scala +++ b/core/src/main/scala/net.kemitix.s3thorp.core/SyncLogging.scala @@ -4,28 +4,28 @@ import cats.Monad import cats.implicits._ import net.kemitix.s3thorp.aws.api.S3Action import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, ErroredS3Action, UploadS3Action} -import net.kemitix.s3thorp.domain.Config +import net.kemitix.s3thorp.domain.{Config, Logger} // Logging for the Sync class object SyncLogging { - def logRunStart[M[_]: Monad](info: Int => String => M[Unit]) - (implicit c: Config): M[Unit] = - info(1)(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, ") + def logRunStart[M[_]: Monad](implicit c: Config, + logger: Logger[M]): M[Unit] = + logger.info(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, ") - def logFileScan[M[_]: Monad](info: Int => String => M[Unit]) - (implicit c: Config): M[Unit] = - info(1)(s"Scanning local files: ${c.source}...") + def logFileScan[M[_]: Monad](implicit c: Config, + logger: Logger[M]): M[Unit] = + logger.info(s"Scanning local files: ${c.source}...") - def logRunFinished[M[_]: Monad](actions: Stream[S3Action], - info: Int => String => M[Unit]) - (implicit c: Config): M[Unit] = { + def logRunFinished[M[_]: Monad](actions: Stream[S3Action]) + (implicit c: Config, + logger: Logger[M]): M[Unit] = { val counters = actions.foldLeft(Counters())(countActivities) for { - _ <- info(1)(s"Uploaded ${counters.uploaded} files") - _ <- info(1)(s"Copied ${counters.copied} files") - _ <- info(1)(s"Deleted ${counters.deleted} files") - _ <- info(1)(s"Errors ${counters.errors}") + _ <- logger.info(s"Uploaded ${counters.uploaded} files") + _ <- logger.info(s"Copied ${counters.copied} files") + _ <- logger.info(s"Deleted ${counters.deleted} files") + _ <- logger.info(s"Errors ${counters.errors}") } yield () } diff --git a/core/src/test/scala/net/kemitix/s3thorp/core/ActionGeneratorSuite.scala b/core/src/test/scala/net/kemitix/s3thorp/core/ActionGeneratorSuite.scala index 913bd0c..3e6ebc7 100644 --- a/core/src/test/scala/net/kemitix/s3thorp/core/ActionGeneratorSuite.scala +++ b/core/src/test/scala/net/kemitix/s3thorp/core/ActionGeneratorSuite.scala @@ -13,7 +13,6 @@ class ActionGeneratorSuite private val prefix = RemoteKey("prefix") private val bucket = Bucket("bucket") implicit private val config: Config = Config(bucket, prefix, source = source) - implicit private val logInfo: Int => String => Unit = l => i => () private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _ val lastModified = LastModified(Instant.now()) diff --git a/core/src/test/scala/net/kemitix/s3thorp/core/DummyLogger.scala b/core/src/test/scala/net/kemitix/s3thorp/core/DummyLogger.scala new file mode 100644 index 0000000..b7191c5 --- /dev/null +++ b/core/src/test/scala/net/kemitix/s3thorp/core/DummyLogger.scala @@ -0,0 +1,16 @@ +package net.kemitix.s3thorp.core + +import cats.Monad +import net.kemitix.s3thorp.domain.Logger + +class DummyLogger[M[_]: Monad] extends Logger[M] { + + override def debug(message: => String): M[Unit] = Monad[M].unit + + override def info(message: =>String): M[Unit] = Monad[M].unit + + override def warn(message: String): M[Unit] = Monad[M].unit + + override def error(message: String): M[Unit] = Monad[M].unit + +} diff --git a/core/src/test/scala/net/kemitix/s3thorp/core/LocalFileStreamSuite.scala b/core/src/test/scala/net/kemitix/s3thorp/core/LocalFileStreamSuite.scala index 95ec9be..af95145 100644 --- a/core/src/test/scala/net/kemitix/s3thorp/core/LocalFileStreamSuite.scala +++ b/core/src/test/scala/net/kemitix/s3thorp/core/LocalFileStreamSuite.scala @@ -3,20 +3,20 @@ package net.kemitix.s3thorp.core import java.io.File import cats.Id -import net.kemitix.s3thorp.domain.{Config, LocalFile, MD5Hash} +import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash} import org.scalatest.FunSpec class LocalFileStreamSuite extends FunSpec { val uploadResource = Resource(this, "upload") implicit val config: Config = Config(source = uploadResource) - implicit private val logInfo: Int => String => Id[Unit] = l => i => () + implicit private val logger: Logger[Id] = new DummyLogger[Id] val md5HashGenerator: File => Id[MD5Hash] = file => MD5HashGenerator.md5File[Id](file) describe("findFiles") { it("should find all files") { val result: Set[String] = - LocalFileStream.findFiles[Id](uploadResource, md5HashGenerator, logInfo).toSet + LocalFileStream.findFiles[Id](uploadResource, md5HashGenerator).toSet .map { x: LocalFile => x.relative.toString } assertResult(Set("subdir/leaf-file", "root-file"))(result) } diff --git a/core/src/test/scala/net/kemitix/s3thorp/core/MD5HashGeneratorTest.scala b/core/src/test/scala/net/kemitix/s3thorp/core/MD5HashGeneratorTest.scala index 674fa22..a71b57a 100644 --- a/core/src/test/scala/net/kemitix/s3thorp/core/MD5HashGeneratorTest.scala +++ b/core/src/test/scala/net/kemitix/s3thorp/core/MD5HashGeneratorTest.scala @@ -2,7 +2,7 @@ package net.kemitix.s3thorp.core import cats.Id import net.kemitix.s3thorp.core.MD5HashData.rootHash -import net.kemitix.s3thorp.domain.{Bucket, Config, MD5Hash, RemoteKey} +import net.kemitix.s3thorp.domain.{Bucket, Config, Logger, MD5Hash, RemoteKey} import org.scalatest.FunSpec class MD5HashGeneratorTest extends FunSpec { @@ -10,7 +10,7 @@ class MD5HashGeneratorTest extends FunSpec { private val source = Resource(this, "upload") private val prefix = RemoteKey("prefix") implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source) - implicit private val logInfo: Int => String => Id[Unit] = l => i => () + implicit private val logger: Logger[Id] = new DummyLogger[Id] describe("read a small file (smaller than buffer)") { val file = Resource(this, "upload/root-file") diff --git a/core/src/test/scala/net/kemitix/s3thorp/core/S3MetaDataEnricherSuite.scala b/core/src/test/scala/net/kemitix/s3thorp/core/S3MetaDataEnricherSuite.scala index 5b110cd..8ed0610 100644 --- a/core/src/test/scala/net/kemitix/s3thorp/core/S3MetaDataEnricherSuite.scala +++ b/core/src/test/scala/net/kemitix/s3thorp/core/S3MetaDataEnricherSuite.scala @@ -12,7 +12,6 @@ class S3MetaDataEnricherSuite private val source = Resource(this, "upload") private val prefix = RemoteKey("prefix") implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source) - implicit private val logInfo: Int => String => Unit = l => i => () private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _ val lastModified = LastModified(Instant.now()) diff --git a/core/src/test/scala/net/kemitix/s3thorp/core/SyncSuite.scala b/core/src/test/scala/net/kemitix/s3thorp/core/SyncSuite.scala index d310393..2db8659 100644 --- a/core/src/test/scala/net/kemitix/s3thorp/core/SyncSuite.scala +++ b/core/src/test/scala/net/kemitix/s3thorp/core/SyncSuite.scala @@ -17,8 +17,7 @@ class SyncSuite private val source = Resource(this, "upload") private val prefix = RemoteKey("prefix") val config = Config(Bucket("bucket"), prefix, source = source) - implicit private val logInfo: Int => String => Id[Unit] = _ => _ => () - implicit private val logWarn: String => Id[Unit] = _ => () + implicit private val logger: Logger[Id] = new DummyLogger[Id] private val lastModified = LastModified(Instant.now) private val fileToKey: File => RemoteKey = KeyGenerator.generateKey(source, prefix) private val rootFile = LocalFile.resolve("root-file", rootHash, source, fileToKey) @@ -38,7 +37,7 @@ class SyncSuite val s3Client = new RecordingClient(testBucket, S3ObjectsData( byHash = Map(), byKey = Map())) - Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn) + Sync.run(config, s3Client, md5HashGenerator, logger) it("uploads all files") { val expectedUploads = Map( "subdir/leaf-file" -> leafRemoteKey, @@ -64,7 +63,7 @@ class SyncSuite RemoteKey("prefix/root-file") -> HashModified(rootHash, lastModified), RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified))) val s3Client = new RecordingClient(testBucket, s3ObjectsData) - Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn) + Sync.run(config, s3Client, md5HashGenerator, logger) it("uploads nothing") { val expectedUploads = Map() assertResult(expectedUploads)(s3Client.uploadsRecord) @@ -88,7 +87,7 @@ class SyncSuite RemoteKey("prefix/root-file-old") -> HashModified(rootHash, lastModified), RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified))) val s3Client = new RecordingClient(testBucket, s3ObjectsData) - Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn) + Sync.run(config, s3Client, md5HashGenerator, logger) it("uploads nothing") { val expectedUploads = Map() assertResult(expectedUploads)(s3Client.uploadsRecord) @@ -116,7 +115,7 @@ class SyncSuite byKey = Map( deletedKey -> HashModified(deletedHash, lastModified))) val s3Client = new RecordingClient(testBucket, s3ObjectsData) - Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn) + Sync.run(config, s3Client, md5HashGenerator, logger) it("deleted key") { val expectedDeletions = Set(deletedKey) assertResult(expectedDeletions)(s3Client.deletionsRecord) @@ -126,7 +125,7 @@ class SyncSuite val config: Config = Config(Bucket("bucket"), prefix, source = source, filters = List(Exclude("leaf"))) val s3ObjectsData = S3ObjectsData(Map(), Map()) val s3Client = new RecordingClient(testBucket, s3ObjectsData) - Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn) + Sync.run(config, s3Client, md5HashGenerator, logger) it("is not uploaded") { val expectedUploads = Map( "root-file" -> rootRemoteKey @@ -146,7 +145,7 @@ class SyncSuite override def listObjects(bucket: Bucket, prefix: RemoteKey) - (implicit info: Int => String => Id[Unit]): S3ObjectsData = + (implicit logger: Logger[Id]): S3ObjectsData = s3ObjectsData override def upload(localFile: LocalFile, @@ -155,8 +154,7 @@ class SyncSuite multiPartThreshold: Long, tryCount: Int, maxRetries: Int) - (implicit info: Int => String => Id[Unit], - warn: String => Id[Unit]): UploadS3Action = { + (implicit logger: Logger[Id]): UploadS3Action = { if (bucket == testBucket) uploadsRecord += (localFile.relative.toString -> localFile.remoteKey) UploadS3Action(localFile.remoteKey, MD5Hash("some hash value")) @@ -166,7 +164,7 @@ class SyncSuite sourceKey: RemoteKey, hash: MD5Hash, targetKey: RemoteKey - )(implicit info: Int => String => Id[Unit]): CopyS3Action = { + )(implicit logger: Logger[Id]): CopyS3Action = { if (bucket == testBucket) copiesRecord += (sourceKey -> targetKey) CopyS3Action(targetKey) @@ -174,7 +172,7 @@ class SyncSuite override def delete(bucket: Bucket, remoteKey: RemoteKey - )(implicit info: Int => String => Id[Unit]): DeleteS3Action = { + )(implicit logger: Logger[Id]): DeleteS3Action = { if (bucket == testBucket) deletionsRecord += remoteKey DeleteS3Action(remoteKey) diff --git a/domain/src/main/scala/net/kemitix/s3thorp/domain/Logger.scala b/domain/src/main/scala/net/kemitix/s3thorp/domain/Logger.scala new file mode 100644 index 0000000..b2fd355 --- /dev/null +++ b/domain/src/main/scala/net/kemitix/s3thorp/domain/Logger.scala @@ -0,0 +1,10 @@ +package net.kemitix.s3thorp.domain + +trait Logger[M[_]] { + + def debug(message: => String): M[Unit] + def info(message: => String): M[Unit] + def warn(message: String): M[Unit] + def error(message: String): M[Unit] + +}