kemitix/thorp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Simplify logging (#59)

Browse source 
* [cli] Logger add debug method

* [cli] Logger: add alternate info method

* [cli] replace direct calls to legacy Logger.info

* [cli] Extract trait from Logger into core and rename

* [cli] Program use debug and new info

* [core] MD5HashGenerator uses Logger

* [core] Sync uses Logger

* [core] ActionSubmitter uses Logger

* [core] LocalFileStream uses Logger

* [core] SyncLogging uses Logger

* [domain] Move Logger into module

The allows it to be used in the aws-api module

* [aws-lib] S3ClientObjectLister uses Logger

* [aws-lib] Uploader uses Logger

* [aws-lib] S3ClientDeleter uses Logger

* [aws-lib] S3ClientCopier uses Logger

* [core] remove unused legacy logging

* [aws-lib] remove used logging method

* [cli] PrintLogger remove legacy info method
This commit is contained in:
Paul Campbell 2019-06-14 17:18:53 +01:00 committed by GitHub
parent 69c09d0091
commit 21b8917395
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 157 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
aws-api/src/main/scala/net/kemitix/s3thorp/aws/api/S3Client.scala
View file

@ -1,13 +1,13 @@
package net.kemitix.s3thorp.aws.api
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action}
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey, S3ObjectsData}
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey, S3ObjectsData}
trait S3Client[M[_]] {
def listObjects(bucket: Bucket,
prefix: RemoteKey
)(implicit info: Int => String => M[Unit]): M[S3ObjectsData]
)(implicit logger: Logger[M]): M[S3ObjectsData]
def upload(localFile: LocalFile,
bucket: Bucket,
@ -15,17 +15,16 @@ trait S3Client[M[_]] {
multiPartThreshold: Long,
tryCount: Int,
maxRetries: Int)
(implicit info: Int => String => M[Unit],
warn: String => M[Unit]): M[S3Action]
(implicit logger: Logger[M]): M[S3Action]
def copy(bucket: Bucket,
sourceKey: RemoteKey,
hash: MD5Hash,
targetKey: RemoteKey
)(implicit info: Int => String => M[Unit]): M[CopyS3Action]
)(implicit logger: Logger[M]): M[CopyS3Action]
def delete(bucket: Bucket,
remoteKey: RemoteKey
)(implicit info: Int => String => M[Unit]): M[DeleteS3Action]
)(implicit logger: Logger[M]): M[DeleteS3Action]
}

4
aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientCopier.scala
View file

@ -6,7 +6,7 @@ import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.CopyObjectRequest
import net.kemitix.s3thorp.aws.api.S3Action.CopyS3Action
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logCopyFinish, logCopyStart}
import net.kemitix.s3thorp.domain.{Bucket, MD5Hash, RemoteKey}
import net.kemitix.s3thorp.domain.{Bucket, Logger, MD5Hash, RemoteKey}
class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) {
@ -14,7 +14,7 @@ class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) {
sourceKey: RemoteKey,
hash: MD5Hash,
targetKey: RemoteKey)
(implicit info: Int => String => M[Unit]): M[CopyS3Action] =
(implicit logger: Logger[M]): M[CopyS3Action] =
for {
_ <- logCopyStart[M](bucket, sourceKey, targetKey)
_ <- copyObject(bucket, sourceKey, hash, targetKey)

4
aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientDeleter.scala
View file

@ -6,13 +6,13 @@ import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.DeleteObjectRequest
import net.kemitix.s3thorp.aws.api.S3Action.DeleteS3Action
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logDeleteFinish, logDeleteStart}
import net.kemitix.s3thorp.domain.{Bucket, RemoteKey}
import net.kemitix.s3thorp.domain.{Bucket, Logger, RemoteKey}
class S3ClientDeleter[M[_]: Monad](amazonS3: AmazonS3) {
def delete(bucket: Bucket,
remoteKey: RemoteKey)
(implicit info: Int => String => M[Unit]): M[DeleteS3Action] =
(implicit logger: Logger[M]): M[DeleteS3Action] =
for {
_ <- logDeleteStart[M](bucket, remoteKey)
_ <- deleteObject(bucket, remoteKey)

31
aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientLogging.scala
View file

@ -2,45 +2,40 @@ package net.kemitix.s3thorp.aws.lib
import cats.Monad
import com.amazonaws.services.s3.model.PutObjectResult
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, RemoteKey}
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, RemoteKey}
object S3ClientLogging {
def logListObjectsStart[M[_]: Monad](bucket: Bucket,
prefix: RemoteKey)
(implicit info: Int => String => M[Unit]): M[Unit] =
info(1)(s"Fetch S3 Summary: ${bucket.name}:${prefix.key}")
(implicit logger: Logger[M]): M[Unit] =
logger.info(s"Fetch S3 Summary: ${bucket.name}:${prefix.key}")
def logListObjectsFinish[M[_]: Monad](bucket: Bucket,
prefix: RemoteKey)
(implicit info: Int => String => M[Unit]): M[Unit] =
info(2)(s"Fetched S3 Summary: ${bucket.name}:${prefix.key}")
def logUploadFinish[M[_]: Monad](localFile: LocalFile,
bucket: Bucket)
(implicit info: Int => String => M[Unit]): PutObjectResult => M[Unit] =
_ => info(2)(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}")
(implicit logger: Logger[M]): M[Unit] =
logger.info(s"Fetched S3 Summary: ${bucket.name}:${prefix.key}")
def logCopyStart[M[_]: Monad](bucket: Bucket,
sourceKey: RemoteKey,
targetKey: RemoteKey)
(implicit info: Int => String => M[Unit]): M[Unit] =
info(1)(s"Copy: ${bucket.name}:${sourceKey.key} => ${targetKey.key}")
(implicit logger: Logger[M]): M[Unit] =
logger.info(s"Copy: ${bucket.name}:${sourceKey.key} => ${targetKey.key}")
def logCopyFinish[M[_]: Monad](bucket: Bucket,
sourceKey: RemoteKey,
targetKey: RemoteKey)
(implicit info: Int => String => M[Unit]): M[Unit] =
info(2)(s"Copied: ${bucket.name}:${sourceKey.key} => ${targetKey.key}")
(implicit logger: Logger[M]): M[Unit] =
logger.info(s"Copied: ${bucket.name}:${sourceKey.key} => ${targetKey.key}")
def logDeleteStart[M[_]: Monad](bucket: Bucket,
remoteKey: RemoteKey)
(implicit info: Int => String => M[Unit]): M[Unit] =
info(1)(s"Delete: ${bucket.name}:${remoteKey.key}")
(implicit logger: Logger[M]): M[Unit] =
logger.info(s"Delete: ${bucket.name}:${remoteKey.key}")
def logDeleteFinish[M[_]: Monad](bucket: Bucket,
remoteKey: RemoteKey)
(implicit info: Int => String => M[Unit]): M[Unit] =
info(2)(s"Deleted: ${bucket.name}:${remoteKey.key}")
(implicit logger: Logger[M]): M[Unit] =
logger.info(s"Deleted: ${bucket.name}:${remoteKey.key}")
}

2
aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/S3ClientObjectLister.scala
View file

@ -15,7 +15,7 @@ class S3ClientObjectLister[M[_]: Monad](amazonS3: AmazonS3) {
def listObjects(bucket: Bucket,
prefix: RemoteKey)
(implicit info: Int => String => M[Unit]): M[S3ObjectsData] = {
(implicit logger: Logger[M]): M[S3ObjectsData] = {
type Token = String
type Batch = (Stream[S3ObjectSummary], Option[Token])

9
aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/ThorpS3Client.scala
View file

@ -18,14 +18,14 @@ class ThorpS3Client[M[_]: Monad](amazonS3Client: => AmazonS3,
override def listObjects(bucket: Bucket,
prefix: RemoteKey)
(implicit info: Int => String => M[Unit]): M[S3ObjectsData] =
(implicit logger: Logger[M]): M[S3ObjectsData] =
objectLister.listObjects(bucket, prefix)
override def copy(bucket: Bucket,
sourceKey: RemoteKey,
hash: MD5Hash,
targetKey: RemoteKey)
(implicit info: Int => String => M[Unit]): M[CopyS3Action] =
(implicit logger: Logger[M]): M[CopyS3Action] =
copier.copy(bucket, sourceKey,hash, targetKey)
override def upload(localFile: LocalFile,
@ -34,13 +34,12 @@ class ThorpS3Client[M[_]: Monad](amazonS3Client: => AmazonS3,
multiPartThreshold: Long,
tryCount: Int,
maxRetries: Int)
(implicit info: Int => String => M[Unit],
warn: String => M[Unit]): M[S3Action] =
(implicit logger: Logger[M]): M[S3Action] =
uploader.upload(localFile, bucket, progressListener, multiPartThreshold, 1, maxRetries)
override def delete(bucket: Bucket,
remoteKey: RemoteKey)
(implicit info: Int => String => M[Unit]): M[DeleteS3Action] =
(implicit logger: Logger[M]): M[DeleteS3Action] =
deleter.delete(bucket, remoteKey)
}

5
aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/Uploader.scala
View file

@ -10,7 +10,7 @@ import net.kemitix.s3thorp.aws.api.S3Action.{ErroredS3Action, UploadS3Action}
import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent}
import net.kemitix.s3thorp.aws.api.{S3Action, UploadProgressListener}
import net.kemitix.s3thorp.aws.lib.UploaderLogging.{logMultiPartUploadFinished, logMultiPartUploadStart}
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey}
import scala.util.Try
@ -26,8 +26,7 @@ class Uploader[M[_]: Monad](transferManager: => AmazonTransferManager) {
multiPartThreshold: Long,
tryCount: Int,
maxRetries: Int)
(implicit info: Int => String => M[Unit],
warn: String => M[Unit]): M[S3Action] =
(implicit logger: Logger[M]): M[S3Action] =
for {
_ <- logMultiPartUploadStart[M](localFile, tryCount)
upload <- transfer(localFile, bucket, uploadProgressListener)

10
aws-lib/src/main/scala/net/kemitix/s3thorp/aws/lib/UploaderLogging.scala
View file

@ -3,20 +3,20 @@ package net.kemitix.s3thorp.aws.lib
import cats.Monad
import net.kemitix.s3thorp.domain.Terminal.clearLine
import net.kemitix.s3thorp.domain.SizeTranslation.sizeInEnglish
import net.kemitix.s3thorp.domain.LocalFile
import net.kemitix.s3thorp.domain.{LocalFile, Logger}
object UploaderLogging {
def logMultiPartUploadStart[M[_]: Monad](localFile: LocalFile,
tryCount: Int)
(implicit info: Int => String => M[Unit]): M[Unit] = {
(implicit logger: Logger[M]): M[Unit] = {
val tryMessage = if (tryCount == 1) "" else s"try $tryCount"
val size = sizeInEnglish(localFile.file.length)
info(1)(s"${clearLine}upload:$tryMessage:$size:${localFile.remoteKey.key}")
logger.info(s"${clearLine}upload:$tryMessage:$size:${localFile.remoteKey.key}")
}
def logMultiPartUploadFinished[M[_]: Monad](localFile: LocalFile)
(implicit info: Int => String => M[Unit]): M[Unit] =
info(4)(s"upload:finished: ${localFile.remoteKey.key}")
(implicit logger: Logger[M]): M[Unit] =
logger.debug(s"upload:finished: ${localFile.remoteKey.key}")
}

16
aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/DummyLogger.scala Normal file
View file

@ -0,0 +1,16 @@
package net.kemitix.s3thorp.aws.lib
import cats.Monad
import net.kemitix.s3thorp.domain.Logger
class DummyLogger[M[_]: Monad] extends Logger[M] {
override def debug(message: => String): M[Unit] = Monad[M].unit
override def info(message: =>String): M[Unit] = Monad[M].unit
override def warn(message: String): M[Unit] = Monad[M].unit
override def error(message: String): M[Unit] = Monad[M].unit
}

3
aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/S3ClientSuite.scala
View file

@ -23,8 +23,7 @@ class S3ClientSuite
private val prefix = RemoteKey("prefix")
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
implicit private val logInfo: Int => String => Id[Unit] = _ => _ => ()
implicit private val logWarn: String => Id[Unit] = _ => ()
implicit private val implLogger: Logger[Id] = new DummyLogger[Id]
private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _
describe("getS3Status") {

2
aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/ThorpS3ClientSuite.scala
View file

@ -21,7 +21,7 @@ class ThorpS3ClientSuite
val source = Resource(this, "upload")
val prefix = RemoteKey("prefix")
implicit val config: Config = Config(Bucket("bucket"), prefix, source = source)
implicit val logInfo: Int => String => Id[Unit] = _ => _ => ()
implicit val implLogger: Logger[Id] = new DummyLogger[Id]
val lm = LastModified(Instant.now.truncatedTo(ChronoUnit.MILLIS))

3
aws-lib/src/test/scala/net/kemitix/s3thorp/aws/lib/UploaderSuite.scala
View file

@ -20,8 +20,7 @@ class UploaderSuite
private val source = Resource(this, ".")
private val prefix = RemoteKey("prefix")
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
implicit private val logInfo: Int => String => Id[Unit] = _ => _ => ()
implicit private val logWarn: String => Id[Unit] = _ => ()
implicit private val implLogger: Logger[Id] = new DummyLogger[Id]
private val fileToKey = generateKey(config.source, config.prefix) _
val lastModified = LastModified(Instant.now())

16
cli/src/main/scala/net/kemitix/s3thorp/cli/Logger.scala
View file

@ -1,16 +0,0 @@
package net.kemitix.s3thorp.cli
import cats.Monad
class Logger[M[_]: Monad](verbosity: Int) {
def info(level: Int)(message: String): M[Unit] =
if (verbosity >= level) Monad[M].pure(println(s"[INFO:$level] $message"))
else Monad[M].unit
def warn(message: String): M[Unit] = Monad[M].pure(println(s"[ WARN] $message"))
def error(message: String): M[Unit] = Monad[M].pure(println(s"[ ERROR] $message"))
}

4
cli/src/main/scala/net/kemitix/s3thorp/cli/Main.scala
View file

@ -12,14 +12,14 @@ object Main extends IOApp {
Config(source = Paths.get(".").toFile)
override def run(args: List[String]): IO[ExitCode] = {
val logger = new Logger[IO](1)
val logger = new PrintLogger[IO](1)
ParseArgs(args, defaultConfig)
.map(Program[IO])
.getOrElse(IO(ExitCode.Error))
.guaranteeCase {
case Canceled => logger.warn("Interrupted")
case Error(e) => logger.error(e.getMessage)
case Completed => logger.info(1)("Done")
case Completed => logger.info("Done")
}
}

16
cli/src/main/scala/net/kemitix/s3thorp/cli/PrintLogger.scala Normal file
View file

@ -0,0 +1,16 @@
package net.kemitix.s3thorp.cli
import cats.Monad
import net.kemitix.s3thorp.domain.Logger
class PrintLogger[M[_]: Monad](verbosity: Int) extends Logger[M] {
override def debug(message: => String): M[Unit] = Monad[M].pure(println(s"[ DEBUG] $message"))
override def info(message: => String): M[Unit] = Monad[M].pure(println(s"[ INFO] $message"))
override def warn(message: String): M[Unit] = Monad[M].pure(println(s"[ WARN] $message"))
override def error(message: String): M[Unit] = Monad[M].pure(println(s"[ ERROR] $message"))
}

18
cli/src/main/scala/net/kemitix/s3thorp/cli/Program.scala
View file

@ -3,27 +3,25 @@ package net.kemitix.s3thorp.cli
import java.io.File
import cats.Monad
import cats.implicits._
import cats.effect.ExitCode
import cats.implicits._
import net.kemitix.s3thorp.aws.lib.S3ClientBuilder
import net.kemitix.s3thorp.core.MD5HashGenerator.md5File
import net.kemitix.s3thorp.core.{MD5HashGenerator, Sync}
import net.kemitix.s3thorp.domain.Config
import net.kemitix.s3thorp.core.Sync
import net.kemitix.s3thorp.domain.{Config, Logger}
object Program {
def apply[M[_]: Monad](config: Config): M[ExitCode] = {
val logger = new Logger[M](config.verbose)
val info = (l: Int) => (m: String) => logger.info(l) (m)
val warn = (w: String) => logger.warn(w)
val logger = new PrintLogger[M](config.verbose)
for {
_ <- info(1)("S3Thorp - hashed sync for s3")
_ <- Sync.run[M](config, S3ClientBuilder.defaultClient, hashGenerator(info), info, warn)
_ <- logger.info("S3Thorp - hashed sync for s3")
_ <- Sync.run[M](config, S3ClientBuilder.defaultClient, hashGenerator(logger), logger)
} yield ExitCode.Success
}
private def hashGenerator[M[_]: Monad](info: Int => String => M[Unit]) = {
implicit val logInfo: Int => String => M[Unit] = info
private def hashGenerator[M[_]: Monad](logger: Logger[M]) = {
implicit val impLogger: Logger[M] = logger
file: File => md5File[M](file)
}

11
core/src/main/scala/net.kemitix.s3thorp.core/ActionSubmitter.scala
View file

@ -5,30 +5,29 @@ import cats.implicits._
import net.kemitix.s3thorp.aws.api.S3Action.DoNothingS3Action
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client, UploadProgressListener}
import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
import net.kemitix.s3thorp.domain.Config
import net.kemitix.s3thorp.domain.{Config, Logger}
object ActionSubmitter {
def submitAction[M[_]: Monad](s3Client: S3Client[M], action: Action)
(implicit c: Config,
info: Int => String => M[Unit],
warn: String => M[Unit]): Stream[M[S3Action]] = {
logger: Logger[M]): Stream[M[S3Action]] = {
Stream(
action match {
case ToUpload(bucket, localFile) =>
for {
_ <- info(4) (s" Upload: ${localFile.relative}")
_ <- logger.info(s" Upload: ${localFile.relative}")
progressListener = new UploadProgressListener(localFile)
action <- s3Client.upload(localFile, bucket, progressListener, c.multiPartThreshold, 1, c.maxRetries)
} yield action
case ToCopy(bucket, sourceKey, hash, targetKey) =>
for {
_ <- info(4)(s" Copy: ${sourceKey.key} => ${targetKey.key}")
_ <- logger.info(s" Copy: ${sourceKey.key} => ${targetKey.key}")
action <- s3Client.copy(bucket, sourceKey, hash, targetKey)
} yield action
case ToDelete(bucket, remoteKey) =>
for {
_ <- info(4)(s" Delete: ${remoteKey.key}")
_ <- logger.info(s" Delete: ${remoteKey.key}")
action <- s3Client.delete(bucket, remoteKey)
} yield action
case DoNothing(bucket, remoteKey) =>

12
core/src/main/scala/net.kemitix.s3thorp.core/LocalFileStream.scala
View file

@ -6,14 +6,14 @@ import java.nio.file.Path
import cats.Monad
import cats.implicits._
import net.kemitix.s3thorp.core.KeyGenerator.generateKey
import net.kemitix.s3thorp.domain.{Config, Filter, LocalFile, MD5Hash}
import net.kemitix.s3thorp.domain.{Config, Filter, LocalFile, Logger, MD5Hash}
object LocalFileStream {
def findFiles[M[_]: Monad](file: File,
md5HashGenerator: File => M[MD5Hash],
info: Int => String => M[Unit])
(implicit c: Config): M[Stream[LocalFile]] = {
md5HashGenerator: File => M[MD5Hash])
(implicit c: Config,
logger: Logger[M]): M[Stream[LocalFile]] = {
val filters: Path => Boolean = Filter.isIncluded(c.filters)
@ -41,10 +41,10 @@ object LocalFileStream {
.flatMap(lfs => acc.map(s => s ++ lfs)))
for {
_ <- info(2)(s"- Entering: $file")
_ <- logger.info(s"- Entering: $file")
fs <- dirPaths(file)
lfs <- recurse(fs)
_ <- info(5)(s"- Leaving : $file")
_ <- logger.debug(s"- Leaving : $file")
} yield lfs
}

8
core/src/main/scala/net.kemitix.s3thorp.core/MD5HashGenerator.scala
View file

@ -5,14 +5,14 @@ import java.security.MessageDigest
import cats.Monad
import cats.implicits._
import net.kemitix.s3thorp.domain.MD5Hash
import net.kemitix.s3thorp.domain.{Logger, MD5Hash}
import scala.collection.immutable.NumericRange
object MD5HashGenerator {
def md5File[M[_]: Monad](file: File)
(implicit info: Int => String => M[Unit]): M[MD5Hash] = {
(implicit logger: Logger[M]): M[MD5Hash] = {
val maxBufferSize = 8048
val defaultBuffer = new Array[Byte](maxBufferSize)
@ -54,10 +54,10 @@ object MD5HashGenerator {
} yield md5
for {
_ <- info(5)(s"md5:reading:size ${file.length}:$file")
_ <- logger.debug(s"md5:reading:size ${file.length}:$file")
md5 <- readFile
hash = MD5Hash(md5)
_ <- info(4)(s"md5:generated:${hash.hash}:$file")
_ <- logger.debug(s"md5:generated:${hash.hash}:$file")
} yield hash
}

18
core/src/main/scala/net.kemitix.s3thorp.core/Sync.scala
View file

@ -11,19 +11,17 @@ import net.kemitix.s3thorp.core.ActionSubmitter.submitAction
import net.kemitix.s3thorp.core.LocalFileStream.findFiles
import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata
import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart}
import net.kemitix.s3thorp.domain.{Config, LocalFile, MD5Hash, S3MetaData, S3ObjectsData}
import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash, S3MetaData, S3ObjectsData}
object Sync {
def run[M[_]: Monad](config: Config,
s3Client: S3Client[M],
md5HashGenerator: File => M[MD5Hash],
info: Int => String => M[Unit],
warn: String => M[Unit]): M[Unit] = {
logger: Logger[M]): M[Unit] = {
implicit val c: Config = config
implicit val logInfo: Int => String => M[Unit] = info
implicit val logWarn: String => M[Unit] = warn
implicit val implLogger: Logger[M] = logger
def metaData(s3Data: S3ObjectsData, sFiles: Stream[LocalFile]) =
Monad[M].pure(sFiles.map(file => getMetadata(file, s3Data)))
@ -36,7 +34,7 @@ object Sync {
def copyUploadActions(s3Data: S3ObjectsData): M[Stream[S3Action]] =
(for {
files <- findFiles(c.source, md5HashGenerator, info)
files <- findFiles(c.source, md5HashGenerator)
metaData <- metaData(s3Data, files)
actions <- actions(metaData)
s3Actions <- submit(actions)
@ -54,12 +52,12 @@ object Sync {
.sequence
for {
_ <- logRunStart(info)
s3data <- s3Client.listObjects(c.bucket, c.prefix)(info)
_ <- logFileScan(info)
_ <- logRunStart[M]
s3data <- s3Client.listObjects(c.bucket, c.prefix)
_ <- logFileScan[M]
copyUploadActions <- copyUploadActions(s3data)
deleteActions <- deleteActions(s3data)
_ <- logRunFinished(copyUploadActions ++ deleteActions, info)
_ <- logRunFinished[M](copyUploadActions ++ deleteActions)
} yield ()
}

28
core/src/main/scala/net.kemitix.s3thorp.core/SyncLogging.scala
View file

@ -4,28 +4,28 @@ import cats.Monad
import cats.implicits._
import net.kemitix.s3thorp.aws.api.S3Action
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, ErroredS3Action, UploadS3Action}
import net.kemitix.s3thorp.domain.Config
import net.kemitix.s3thorp.domain.{Config, Logger}
// Logging for the Sync class
object SyncLogging {
def logRunStart[M[_]: Monad](info: Int => String => M[Unit])
(implicit c: Config): M[Unit] =
info(1)(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, ")
def logRunStart[M[_]: Monad](implicit c: Config,
logger: Logger[M]): M[Unit] =
logger.info(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, ")
def logFileScan[M[_]: Monad](info: Int => String => M[Unit])
(implicit c: Config): M[Unit] =
info(1)(s"Scanning local files: ${c.source}...")
def logFileScan[M[_]: Monad](implicit c: Config,
logger: Logger[M]): M[Unit] =
logger.info(s"Scanning local files: ${c.source}...")
def logRunFinished[M[_]: Monad](actions: Stream[S3Action],
info: Int => String => M[Unit])
(implicit c: Config): M[Unit] = {
def logRunFinished[M[_]: Monad](actions: Stream[S3Action])
(implicit c: Config,
logger: Logger[M]): M[Unit] = {
val counters = actions.foldLeft(Counters())(countActivities)
for {
_ <- info(1)(s"Uploaded ${counters.uploaded} files")
_ <- info(1)(s"Copied ${counters.copied} files")
_ <- info(1)(s"Deleted ${counters.deleted} files")
_ <- info(1)(s"Errors ${counters.errors}")
_ <- logger.info(s"Uploaded ${counters.uploaded} files")
_ <- logger.info(s"Copied ${counters.copied} files")
_ <- logger.info(s"Deleted ${counters.deleted} files")
_ <- logger.info(s"Errors ${counters.errors}")
} yield ()
}

1
core/src/test/scala/net/kemitix/s3thorp/core/ActionGeneratorSuite.scala
View file

@ -13,7 +13,6 @@ class ActionGeneratorSuite
private val prefix = RemoteKey("prefix")
private val bucket = Bucket("bucket")
implicit private val config: Config = Config(bucket, prefix, source = source)
implicit private val logInfo: Int => String => Unit = l => i => ()
private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _
val lastModified = LastModified(Instant.now())

16
core/src/test/scala/net/kemitix/s3thorp/core/DummyLogger.scala Normal file
View file

@ -0,0 +1,16 @@
package net.kemitix.s3thorp.core
import cats.Monad
import net.kemitix.s3thorp.domain.Logger
class DummyLogger[M[_]: Monad] extends Logger[M] {
override def debug(message: => String): M[Unit] = Monad[M].unit
override def info(message: =>String): M[Unit] = Monad[M].unit
override def warn(message: String): M[Unit] = Monad[M].unit
override def error(message: String): M[Unit] = Monad[M].unit
}

6
core/src/test/scala/net/kemitix/s3thorp/core/LocalFileStreamSuite.scala
View file

@ -3,20 +3,20 @@ package net.kemitix.s3thorp.core
import java.io.File
import cats.Id
import net.kemitix.s3thorp.domain.{Config, LocalFile, MD5Hash}
import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash}
import org.scalatest.FunSpec
class LocalFileStreamSuite extends FunSpec {
val uploadResource = Resource(this, "upload")
implicit val config: Config = Config(source = uploadResource)
implicit private val logInfo: Int => String => Id[Unit] = l => i => ()
implicit private val logger: Logger[Id] = new DummyLogger[Id]
val md5HashGenerator: File => Id[MD5Hash] = file => MD5HashGenerator.md5File[Id](file)
describe("findFiles") {
it("should find all files") {
val result: Set[String] =
LocalFileStream.findFiles[Id](uploadResource, md5HashGenerator, logInfo).toSet
LocalFileStream.findFiles[Id](uploadResource, md5HashGenerator).toSet
.map { x: LocalFile => x.relative.toString }
assertResult(Set("subdir/leaf-file", "root-file"))(result)
}

4
core/src/test/scala/net/kemitix/s3thorp/core/MD5HashGeneratorTest.scala
View file

@ -2,7 +2,7 @@ package net.kemitix.s3thorp.core
import cats.Id
import net.kemitix.s3thorp.core.MD5HashData.rootHash
import net.kemitix.s3thorp.domain.{Bucket, Config, MD5Hash, RemoteKey}
import net.kemitix.s3thorp.domain.{Bucket, Config, Logger, MD5Hash, RemoteKey}
import org.scalatest.FunSpec
class MD5HashGeneratorTest extends FunSpec {
@ -10,7 +10,7 @@ class MD5HashGeneratorTest extends FunSpec {
private val source = Resource(this, "upload")
private val prefix = RemoteKey("prefix")
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
implicit private val logInfo: Int => String => Id[Unit] = l => i => ()
implicit private val logger: Logger[Id] = new DummyLogger[Id]
describe("read a small file (smaller than buffer)") {
val file = Resource(this, "upload/root-file")

1
core/src/test/scala/net/kemitix/s3thorp/core/S3MetaDataEnricherSuite.scala
View file

@ -12,7 +12,6 @@ class S3MetaDataEnricherSuite
private val source = Resource(this, "upload")
private val prefix = RemoteKey("prefix")
implicit private val config: Config = Config(Bucket("bucket"), prefix, source = source)
implicit private val logInfo: Int => String => Unit = l => i => ()
private val fileToKey = KeyGenerator.generateKey(config.source, config.prefix) _
val lastModified = LastModified(Instant.now())

22
core/src/test/scala/net/kemitix/s3thorp/core/SyncSuite.scala
View file

@ -17,8 +17,7 @@ class SyncSuite
private val source = Resource(this, "upload")
private val prefix = RemoteKey("prefix")
val config = Config(Bucket("bucket"), prefix, source = source)
implicit private val logInfo: Int => String => Id[Unit] = _ => _ => ()
implicit private val logWarn: String => Id[Unit] = _ => ()
implicit private val logger: Logger[Id] = new DummyLogger[Id]
private val lastModified = LastModified(Instant.now)
private val fileToKey: File => RemoteKey = KeyGenerator.generateKey(source, prefix)
private val rootFile = LocalFile.resolve("root-file", rootHash, source, fileToKey)
@ -38,7 +37,7 @@ class SyncSuite
val s3Client = new RecordingClient(testBucket, S3ObjectsData(
byHash = Map(),
byKey = Map()))
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
Sync.run(config, s3Client, md5HashGenerator, logger)
it("uploads all files") {
val expectedUploads = Map(
"subdir/leaf-file" -> leafRemoteKey,
@ -64,7 +63,7 @@ class SyncSuite
RemoteKey("prefix/root-file") -> HashModified(rootHash, lastModified),
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
Sync.run(config, s3Client, md5HashGenerator, logger)
it("uploads nothing") {
val expectedUploads = Map()
assertResult(expectedUploads)(s3Client.uploadsRecord)
@ -88,7 +87,7 @@ class SyncSuite
RemoteKey("prefix/root-file-old") -> HashModified(rootHash, lastModified),
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
Sync.run(config, s3Client, md5HashGenerator, logger)
it("uploads nothing") {
val expectedUploads = Map()
assertResult(expectedUploads)(s3Client.uploadsRecord)
@ -116,7 +115,7 @@ class SyncSuite
byKey = Map(
deletedKey -> HashModified(deletedHash, lastModified)))
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
Sync.run(config, s3Client, md5HashGenerator, logger)
it("deleted key") {
val expectedDeletions = Set(deletedKey)
assertResult(expectedDeletions)(s3Client.deletionsRecord)
@ -126,7 +125,7 @@ class SyncSuite
val config: Config = Config(Bucket("bucket"), prefix, source = source, filters = List(Exclude("leaf")))
val s3ObjectsData = S3ObjectsData(Map(), Map())
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
Sync.run(config, s3Client, md5HashGenerator, logInfo, logWarn)
Sync.run(config, s3Client, md5HashGenerator, logger)
it("is not uploaded") {
val expectedUploads = Map(
"root-file" -> rootRemoteKey
@ -146,7 +145,7 @@ class SyncSuite
override def listObjects(bucket: Bucket,
prefix: RemoteKey)
(implicit info: Int => String => Id[Unit]): S3ObjectsData =
(implicit logger: Logger[Id]): S3ObjectsData =
s3ObjectsData
override def upload(localFile: LocalFile,
@ -155,8 +154,7 @@ class SyncSuite
multiPartThreshold: Long,
tryCount: Int,
maxRetries: Int)
(implicit info: Int => String => Id[Unit],
warn: String => Id[Unit]): UploadS3Action = {
(implicit logger: Logger[Id]): UploadS3Action = {
if (bucket == testBucket)
uploadsRecord += (localFile.relative.toString -> localFile.remoteKey)
UploadS3Action(localFile.remoteKey, MD5Hash("some hash value"))
@ -166,7 +164,7 @@ class SyncSuite
sourceKey: RemoteKey,
hash: MD5Hash,
targetKey: RemoteKey
)(implicit info: Int => String => Id[Unit]): CopyS3Action = {
)(implicit logger: Logger[Id]): CopyS3Action = {
if (bucket == testBucket)
copiesRecord += (sourceKey -> targetKey)
CopyS3Action(targetKey)
@ -174,7 +172,7 @@ class SyncSuite
override def delete(bucket: Bucket,
remoteKey: RemoteKey
)(implicit info: Int => String => Id[Unit]): DeleteS3Action = {
)(implicit logger: Logger[Id]): DeleteS3Action = {
if (bucket == testBucket)
deletionsRecord += remoteKey
DeleteS3Action(remoteKey)

10
domain/src/main/scala/net/kemitix/s3thorp/domain/Logger.scala Normal file
View file

@ -0,0 +1,10 @@
package net.kemitix.s3thorp.domain
trait Logger[M[_]] {
def debug(message: => String): M[Unit]
def info(message: => String): M[Unit]
def warn(message: String): M[Unit]
def error(message: String): M[Unit]
}