From 515b8969936ba9ddfaed690960e142b4f339d385 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Thu, 18 Jul 2019 08:57:14 +0100 Subject: [PATCH] Use Lenses (#113) * [sbt] Add monocle for support for lenses * [domain] UploadEventListener make a case class * [domain] Add @Lenses to all case classes * [core] Add @Lenses to case classes * [core] ActionGenerator use lense * [core] ConfigOption use Lenses * [core] ConfigurationBuilder remove unused fields * [core] ConfigurationBuilder refactoring * [core] SyncLogging use lenses * [core] syncLogging refactoring --- build.sbt | 10 ++++++ .../net/kemitix/thorp/cli/ParseArgs.scala | 2 +- .../scala/net/kemitix/thorp/core/Action.scala | 5 +++ .../kemitix/thorp/core/ActionGenerator.scala | 10 +++--- .../net/kemitix/thorp/core/ConfigOption.scala | 26 ++++++++++---- .../kemitix/thorp/core/ConfigOptions.scala | 2 ++ .../thorp/core/ConfigurationBuilder.scala | 35 +++++++++---------- .../net/kemitix/thorp/core/Counters.scala | 3 ++ .../net/kemitix/thorp/core/LocalFiles.scala | 2 ++ .../thorp/core/SourceConfigLoader.scala | 2 +- .../net/kemitix/thorp/core/SyncLogging.scala | 25 ++++++------- .../net/kemitix/thorp/core/SyncPlan.scala | 2 ++ .../thorp/core/UnversionedMirrorArchive.scala | 4 ++- .../net/kemitix/thorp/domain/Bucket.scala | 3 ++ .../net/kemitix/thorp/domain/Config.scala | 3 ++ .../net/kemitix/thorp/domain/Filter.scala | 4 +++ .../kemitix/thorp/domain/HashModified.scala | 3 ++ .../kemitix/thorp/domain/KeyModified.scala | 3 ++ .../kemitix/thorp/domain/LastModified.scala | 3 ++ .../net/kemitix/thorp/domain/LocalFile.scala | 3 ++ .../net/kemitix/thorp/domain/MD5Hash.scala | 3 ++ .../net/kemitix/thorp/domain/RemoteKey.scala | 3 ++ .../kemitix/thorp/domain/RemoteMetaData.scala | 3 ++ .../net/kemitix/thorp/domain/S3MetaData.scala | 3 ++ .../kemitix/thorp/domain/S3ObjectsData.scala | 4 +++ .../net/kemitix/thorp/domain/Sources.scala | 9 +++-- .../thorp/domain/StorageQueueEvent.scala | 8 +++++ .../net/kemitix/thorp/domain/SyncTotals.scala | 3 ++ .../kemitix/thorp/domain/UploadEvent.scala | 5 +++ .../thorp/domain/UploadEventListener.scala | 5 ++- .../storage/aws/StorageServiceSuite.scala | 2 +- .../thorp/storage/aws/UploaderSuite.scala | 2 +- 32 files changed, 148 insertions(+), 52 deletions(-) diff --git a/build.sbt b/build.sbt index 58947f6..c000855 100644 --- a/build.sbt +++ b/build.sbt @@ -26,6 +26,9 @@ val commonSettings = Seq( "-language:postfixOps", "-language:higherKinds", "-Ypartial-unification"), + addCompilerPlugin( + "org.scalameta" % "paradise" % "3.0.0-M11" cross CrossVersion.full + ), test in assembly := {} ) @@ -38,6 +41,12 @@ val testDependencies = Seq( "org.scalamock" %% "scalamock" % "4.3.0" % Test ) ) +val domainDependencies = Seq( + libraryDependencies ++= Seq( + "com.github.julien-truffaut" %% "monocle-core" % "1.6.0", + "com.github.julien-truffaut" %% "monocle-macro" % "1.6.0", + ) +) val commandLineParsing = Seq( libraryDependencies ++= Seq( "com.github.scopt" %% "scopt" % "4.0.0-RC2" @@ -109,6 +118,7 @@ lazy val `storage-api` = (project in file("storage-api")) lazy val domain = (project in file("domain")) .settings(commonSettings) + .settings(domainDependencies) .settings(assemblyJarName in assembly := "domain.jar") .settings(catsEffectsSettings) .settings(testDependencies) diff --git a/cli/src/main/scala/net/kemitix/thorp/cli/ParseArgs.scala b/cli/src/main/scala/net/kemitix/thorp/cli/ParseArgs.scala index 8184fb4..2527fd7 100644 --- a/cli/src/main/scala/net/kemitix/thorp/cli/ParseArgs.scala +++ b/cli/src/main/scala/net/kemitix/thorp/cli/ParseArgs.scala @@ -52,6 +52,6 @@ object ParseArgs { def apply(args: List[String]): Option[ConfigOptions] = OParser .parse(configParser, args, List()) - .map(ConfigOptions) + .map(ConfigOptions(_)) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/Action.scala b/core/src/main/scala/net/kemitix/thorp/core/Action.scala index fa75c66..830947e 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/Action.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/Action.scala @@ -1,5 +1,6 @@ package net.kemitix.thorp.core +import monocle.macros.Lenses import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey} sealed trait Action { @@ -8,18 +9,21 @@ sealed trait Action { } object Action { + @Lenses final case class DoNothing( bucket: Bucket, remoteKey: RemoteKey, size: Long ) extends Action + @Lenses final case class ToUpload( bucket: Bucket, localFile: LocalFile, size: Long ) extends Action + @Lenses final case class ToCopy( bucket: Bucket, sourceKey: RemoteKey, @@ -28,6 +32,7 @@ object Action { size: Long ) extends Action + @Lenses final case class ToDelete( bucket: Bucket, remoteKey: RemoteKey, diff --git a/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala b/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala index bdbe486..ee5cf72 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala @@ -35,15 +35,15 @@ object ActionGenerator { doNothing(c.bucket, localFile.remoteKey) } + private def key = LocalFile.remoteKey ^|-> RemoteKey.key + def isUploadAlreadyQueued( previousActions: Stream[Action] )( localFile: LocalFile - ): Boolean = { - !previousActions.exists { - case ToUpload(_, lf, _) => lf.remoteKey.key equals localFile.remoteKey.key - case _ => false - } + ): Boolean = !previousActions.exists { + case ToUpload(_, lf, _) => key.get(lf) equals key.get(localFile) + case _ => false } private def doNothing( diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala b/core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala index fe4f1f0..4c2b6fc 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala @@ -2,47 +2,57 @@ package net.kemitix.thorp.core import java.nio.file.Path +import monocle.macros.Lenses import net.kemitix.thorp.domain import net.kemitix.thorp.domain.{Config, RemoteKey} +import net.kemitix.thorp.domain.Config._ sealed trait ConfigOption { def update(config: Config): Config } object ConfigOption { + + @Lenses case class Source(path: Path) extends ConfigOption { override def update(config: Config): Config = - config.copy(sources = config.sources ++ path) + sources.modify(_ ++ path)(config) } + @Lenses case class Bucket(name: String) extends ConfigOption { override def update(config: Config): Config = if (config.bucket.name.isEmpty) - config.copy(bucket = domain.Bucket(name)) + bucket.set(domain.Bucket(name))(config) else config } + @Lenses case class Prefix(path: String) extends ConfigOption { override def update(config: Config): Config = if (config.prefix.key.isEmpty) - config.copy(prefix = RemoteKey(path)) + prefix.set(RemoteKey(path))(config) else config } + @Lenses case class Include(pattern: String) extends ConfigOption { override def update(config: Config): Config = - config.copy(filters = domain.Filter.Include(pattern) :: config.filters) + filters.modify(domain.Filter.Include(pattern) :: _)(config) } + @Lenses case class Exclude(pattern: String) extends ConfigOption { override def update(config: Config): Config = - config.copy(filters = domain.Filter.Exclude(pattern) :: config.filters) + filters.modify(domain.Filter.Exclude(pattern) :: _)(config) } + @Lenses case class Debug() extends ConfigOption { - override def update(config: Config): Config = config.copy(debug = true) + override def update(config: Config): Config = + debug.set(true)(config) } case object Version extends ConfigOption { @@ -50,12 +60,14 @@ object ConfigOption { } case object BatchMode extends ConfigOption { - override def update(config: Config): Config = config.copy(batchMode = true) + override def update(config: Config): Config = + batchMode.set(true)(config) } case object IgnoreUserOptions extends ConfigOption { override def update(config: Config): Config = config } + case object IgnoreGlobalOptions extends ConfigOption { override def update(config: Config): Config = config } diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala b/core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala index 907f2f4..2586ede 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala @@ -1,7 +1,9 @@ package net.kemitix.thorp.core import cats.Semigroup +import monocle.macros.Lenses +@Lenses case class ConfigOptions( options: List[ConfigOption] = List() ) extends Semigroup[ConfigOptions] { diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala b/core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala index 3c88da8..9d19da3 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala @@ -1,11 +1,12 @@ package net.kemitix.thorp.core -import java.nio.file.{Path, Paths} +import java.nio.file.Paths import cats.data.NonEmptyChain import cats.effect.IO import net.kemitix.thorp.core.ConfigValidator.validateConfig import net.kemitix.thorp.core.ParseConfigFile.parseFile +import net.kemitix.thorp.core.ConfigOptions.options import net.kemitix.thorp.domain.Config /** @@ -14,20 +15,18 @@ import net.kemitix.thorp.domain.Config */ trait ConfigurationBuilder { - private val sourceConfigFilename = ".thorp.config" - private val userConfigFilename = ".config/thorp.conf" - private val globalConfig = Paths.get("/etc/thorp.conf") - private val userHome = Paths.get(System.getProperty("user.home")) - private val pwd = Paths.get(System.getenv("PWD")) + private val userConfigFilename = ".config/thorp.conf" + private val globalConfig = Paths.get("/etc/thorp.conf") + private val userHome = Paths.get(System.getProperty("user.home")) def buildConfig(priorityOpts: ConfigOptions) - : IO[Either[NonEmptyChain[ConfigValidation], Config]] = { + : IO[Either[NonEmptyChain[ConfigValidation], Config]] = { val sources = ConfigQuery.sources(priorityOpts) for { - sourceOptions <- SourceConfigLoader.loadSourceConfigs(sources) - userOptions <- userOptions(priorityOpts ++ sourceOptions) - globalOptions <- globalOptions(priorityOpts ++ sourceOptions ++ userOptions) - collected = priorityOpts ++ sourceOptions ++ userOptions ++ globalOptions + sourceOpts <- SourceConfigLoader.loadSourceConfigs(sources) + userOpts <- userOptions(priorityOpts ++ sourceOpts) + globalOpts <- globalOptions(priorityOpts ++ sourceOpts ++ userOpts) + collected = priorityOpts ++ sourceOpts ++ userOpts ++ globalOpts config = collateOptions(collected) } yield validateConfig(config).toEither } @@ -36,20 +35,18 @@ trait ConfigurationBuilder { private def userOptions(priorityOpts: ConfigOptions) = if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig - else readFile(userHome, userConfigFilename) + else parseFile(userHome.resolve(userConfigFilename)) private def globalOptions(priorityOpts: ConfigOptions) = if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig else parseFile(globalConfig) - private def readFile( - source: Path, - filename: String - ) = - parseFile(source.resolve(filename)) - private def collateOptions(configOptions: ConfigOptions): Config = - configOptions.options.foldLeft(Config())((c, co) => co.update(c)) + options + .get(configOptions) + .foldLeft(Config()) { (config, configOption) => + configOption.update(config) + } } diff --git a/core/src/main/scala/net/kemitix/thorp/core/Counters.scala b/core/src/main/scala/net/kemitix/thorp/core/Counters.scala index 73b6757..ccfdb58 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/Counters.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/Counters.scala @@ -1,5 +1,8 @@ package net.kemitix.thorp.core +import monocle.macros.Lenses + +@Lenses final case class Counters( uploaded: Int = 0, deleted: Int = 0, diff --git a/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala b/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala index 451e47a..64a31fe 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala @@ -1,7 +1,9 @@ package net.kemitix.thorp.core +import monocle.macros.Lenses import net.kemitix.thorp.domain.LocalFile +@Lenses case class LocalFiles( localFiles: Stream[LocalFile] = Stream(), count: Long = 0, diff --git a/core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala b/core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala index 3add4e1..2b9daa6 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala @@ -12,7 +12,7 @@ trait SourceConfigLoader { sources => { val sourceConfigOptions = - ConfigOptions(sources.paths.map(ConfigOption.Source)) + ConfigOptions(sources.paths.map(ConfigOption.Source(_))) val reduce: List[ConfigOptions] => ConfigOptions = _.foldLeft(sourceConfigOptions) { (acc, co) => acc ++ co } diff --git a/core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala b/core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala index 9b23c2f..d86c651 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala @@ -18,11 +18,14 @@ trait SyncLogging { sources: Sources )(implicit logger: Logger): IO[Unit] = { val sourcesList = sources.paths.mkString(", ") - logger.info(s"Bucket: ${bucket.name}, Prefix: ${prefix.key}, Source: $sourcesList") + logger.info( + List(s"Bucket: ${bucket.name}", + s"Prefix: ${prefix.key}", + s"Source: $sourcesList") + .mkString(", ")) } - def logFileScan(implicit c: Config, - logger: Logger): IO[Unit] = + def logFileScan(implicit c: Config, logger: Logger): IO[Unit] = logger.info(s"Scanning local files: ${c.sources.paths.mkString(", ")}...") def logRunFinished( @@ -50,16 +53,14 @@ trait SyncLogging { private def countActivities: (Counters, StorageQueueEvent) => Counters = (counters: Counters, s3Action: StorageQueueEvent) => { + import Counters._ + val increment: Int => Int = _ + 1 s3Action match { - case _: UploadQueueEvent => - counters.copy(uploaded = counters.uploaded + 1) - case _: CopyQueueEvent => - counters.copy(copied = counters.copied + 1) - case _: DeleteQueueEvent => - counters.copy(deleted = counters.deleted + 1) - case ErrorQueueEvent(_, _) => - counters.copy(errors = counters.errors + 1) - case _ => counters + case _: UploadQueueEvent => uploaded.modify(increment)(counters) + case _: CopyQueueEvent => copied.modify(increment)(counters) + case _: DeleteQueueEvent => deleted.modify(increment)(counters) + case _: ErrorQueueEvent => errors.modify(increment)(counters) + case _ => counters } } diff --git a/core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala b/core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala index 3e7d5da..0e1e6ea 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala @@ -1,7 +1,9 @@ package net.kemitix.thorp.core +import monocle.macros.Lenses import net.kemitix.thorp.domain.SyncTotals +@Lenses case class SyncPlan( actions: Stream[Action] = Stream(), syncTotals: SyncTotals = SyncTotals() diff --git a/core/src/main/scala/net/kemitix/thorp/core/UnversionedMirrorArchive.scala b/core/src/main/scala/net/kemitix/thorp/core/UnversionedMirrorArchive.scala index 6de2e13..08f4f50 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/UnversionedMirrorArchive.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/UnversionedMirrorArchive.scala @@ -1,6 +1,7 @@ package net.kemitix.thorp.core import cats.effect.IO +import monocle.macros.Lenses import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload} import net.kemitix.thorp.domain.StorageQueueEvent.DoNothingQueueEvent import net.kemitix.thorp.domain.{ @@ -13,6 +14,7 @@ import net.kemitix.thorp.domain.{ } import net.kemitix.thorp.storage.api.StorageService +@Lenses case class UnversionedMirrorArchive( storageService: StorageService, batchMode: Boolean, @@ -52,7 +54,7 @@ case class UnversionedMirrorArchive( localFile, bucket, batchMode, - new UploadEventListener(localFile, index, syncTotals, totalBytesSoFar), + UploadEventListener(localFile, index, syncTotals, totalBytesSoFar), 1) } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/Bucket.scala b/domain/src/main/scala/net/kemitix/thorp/domain/Bucket.scala index c152185..a153ed6 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/Bucket.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/Bucket.scala @@ -1,5 +1,8 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + +@Lenses final case class Bucket( name: String ) diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/Config.scala b/domain/src/main/scala/net/kemitix/thorp/domain/Config.scala index f19af05..aa11441 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/Config.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/Config.scala @@ -1,5 +1,8 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + +@Lenses final case class Config( bucket: Bucket = Bucket(""), prefix: RemoteKey = RemoteKey(""), diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala b/domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala index d8b8a8e..858d2d1 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala @@ -3,6 +3,8 @@ package net.kemitix.thorp.domain import java.nio.file.Path import java.util.regex.Pattern +import monocle.macros.Lenses + sealed trait Filter object Filter { @@ -30,6 +32,7 @@ object Filter { } } + @Lenses case class Include( include: String = ".*" ) extends Filter { @@ -40,6 +43,7 @@ object Filter { } + @Lenses case class Exclude( exclude: String ) extends Filter { diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/HashModified.scala b/domain/src/main/scala/net/kemitix/thorp/domain/HashModified.scala index c752573..44530ef 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/HashModified.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/HashModified.scala @@ -1,5 +1,8 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + +@Lenses final case class HashModified( hash: MD5Hash, modified: LastModified diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/KeyModified.scala b/domain/src/main/scala/net/kemitix/thorp/domain/KeyModified.scala index 6627497..0a6e4ec 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/KeyModified.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/KeyModified.scala @@ -1,5 +1,8 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + +@Lenses final case class KeyModified( key: RemoteKey, modified: LastModified diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/LastModified.scala b/domain/src/main/scala/net/kemitix/thorp/domain/LastModified.scala index 0e74f99..f500f30 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/LastModified.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/LastModified.scala @@ -2,6 +2,9 @@ package net.kemitix.thorp.domain import java.time.Instant +import monocle.macros.Lenses + +@Lenses final case class LastModified( when: Instant = Instant.now ) diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala b/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala index 3b57f05..30364dd 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala @@ -3,6 +3,9 @@ package net.kemitix.thorp.domain import java.io.File import java.nio.file.Path +import monocle.macros.Lenses + +@Lenses final case class LocalFile( file: File, source: File, diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala b/domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala index 4210f13..457b0de 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala @@ -4,6 +4,9 @@ import java.util.Base64 import net.kemitix.thorp.domain.QuoteStripper.stripQuotes +import monocle.macros.Lenses + +@Lenses final case class MD5Hash( in: String ) { diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala b/domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala index 130fa4d..a8a824e 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala @@ -3,6 +3,9 @@ package net.kemitix.thorp.domain import java.io.File import java.nio.file.{Path, Paths} +import monocle.macros.Lenses + +@Lenses final case class RemoteKey( key: String ) { diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/RemoteMetaData.scala b/domain/src/main/scala/net/kemitix/thorp/domain/RemoteMetaData.scala index a55b46e..75c8066 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/RemoteMetaData.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/RemoteMetaData.scala @@ -1,5 +1,8 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + +@Lenses final case class RemoteMetaData( remoteKey: RemoteKey, hash: MD5Hash, diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/S3MetaData.scala b/domain/src/main/scala/net/kemitix/thorp/domain/S3MetaData.scala index 93ba71e..f350c02 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/S3MetaData.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/S3MetaData.scala @@ -1,6 +1,9 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + // For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key +@Lenses final case class S3MetaData( localFile: LocalFile, matchByHash: Set[RemoteMetaData], diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/S3ObjectsData.scala b/domain/src/main/scala/net/kemitix/thorp/domain/S3ObjectsData.scala index 72b0aed..be5b66d 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/S3ObjectsData.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/S3ObjectsData.scala @@ -1,8 +1,12 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + /** * A list of objects and their MD5 hash values. */ + +@Lenses final case class S3ObjectsData( byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty, byKey: Map[RemoteKey, HashModified] = Map.empty diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala b/domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala index a6a60f9..2652787 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala @@ -2,6 +2,8 @@ package net.kemitix.thorp.domain import java.nio.file.Path +import monocle.macros.Lenses + /** * The paths to synchronise with target. * @@ -12,12 +14,13 @@ import java.nio.file.Path * * A path should only occur once in paths. */ +@Lenses case class Sources( paths: List[Path] ) { - def ++(path: Path): Sources = this ++ List(path) - def ++(otherPaths: List[Path]): Sources = Sources( - otherPaths.foldLeft(paths) { (acc, path) => + def ++(path: Path): Sources = this ++ List(path) + def ++(otherPaths: List[Path]): Sources = + Sources(otherPaths.foldLeft(paths) { (acc, path) => if (acc.contains(path)) acc else acc ++ List(path) }) diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/StorageQueueEvent.scala b/domain/src/main/scala/net/kemitix/thorp/domain/StorageQueueEvent.scala index 31e7e8d..9ca41fb 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/StorageQueueEvent.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/StorageQueueEvent.scala @@ -1,5 +1,7 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + sealed trait StorageQueueEvent { val order: Int @@ -8,18 +10,21 @@ sealed trait StorageQueueEvent { object StorageQueueEvent { + @Lenses final case class DoNothingQueueEvent( remoteKey: RemoteKey ) extends StorageQueueEvent { override val order: Int = 0 } + @Lenses final case class CopyQueueEvent( remoteKey: RemoteKey ) extends StorageQueueEvent { override val order: Int = 1 } + @Lenses final case class UploadQueueEvent( remoteKey: RemoteKey, md5Hash: MD5Hash @@ -27,12 +32,14 @@ object StorageQueueEvent { override val order: Int = 2 } + @Lenses final case class DeleteQueueEvent( remoteKey: RemoteKey ) extends StorageQueueEvent { override val order: Int = 3 } + @Lenses final case class ErrorQueueEvent( remoteKey: RemoteKey, e: Throwable @@ -40,6 +47,7 @@ object StorageQueueEvent { override val order: Int = 10 } + @Lenses final case class ShutdownQueueEvent() extends StorageQueueEvent { override val order: Int = 99 } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/SyncTotals.scala b/domain/src/main/scala/net/kemitix/thorp/domain/SyncTotals.scala index 7358e2b..2ffb1b9 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/SyncTotals.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/SyncTotals.scala @@ -1,5 +1,8 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + +@Lenses case class SyncTotals( count: Long = 0L, totalSizeBytes: Long = 0L, diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/UploadEvent.scala b/domain/src/main/scala/net/kemitix/thorp/domain/UploadEvent.scala index d416904..811d32f 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/UploadEvent.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/UploadEvent.scala @@ -1,21 +1,26 @@ package net.kemitix.thorp.domain +import monocle.macros.Lenses + sealed trait UploadEvent { def name: String } object UploadEvent { + @Lenses final case class TransferEvent( name: String ) extends UploadEvent + @Lenses final case class RequestEvent( name: String, bytes: Long, transferred: Long ) extends UploadEvent + @Lenses final case class ByteTransferEvent( name: String ) extends UploadEvent diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala b/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala index 392c13a..7f905d0 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala @@ -3,7 +3,10 @@ package net.kemitix.thorp.domain import net.kemitix.thorp.domain.UploadEvent.RequestEvent import net.kemitix.thorp.domain.UploadEventLogger.logRequestCycle -class UploadEventListener( +import monocle.macros.Lenses + +@Lenses +case class UploadEventListener( localFile: LocalFile, index: Int, syncTotals: SyncTotals, diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala index 9cb9427..0f77280 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala @@ -137,7 +137,7 @@ class StorageServiceSuite extends FunSpec with MockFactory { val bucket = Bucket("a-bucket") val remoteKey = RemoteKey("prefix/root-file") val uploadEventListener = - new UploadEventListener(localFile, 1, SyncTotals(), 0L) + UploadEventListener(localFile, 1, SyncTotals(), 0L) val upload = stub[AmazonUpload] (amazonTransferManager upload (_: PutObjectRequest)) diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderSuite.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderSuite.scala index 3d53de6..ad2f4ad 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderSuite.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderSuite.scala @@ -36,7 +36,7 @@ class UploaderSuite extends FunSpec with MockFactory { sourcePath, fileToKey) val uploadEventListener = - new UploadEventListener(bigFile, 1, SyncTotals(), 0L) + UploadEventListener(bigFile, 1, SyncTotals(), 0L) val amazonS3 = mock[AmazonS3] val amazonTransferManager = AmazonTransferManager(