Use Lenses (#113)

* [sbt] Add monocle for support for lenses

* [domain] UploadEventListener make a case class

* [domain] Add @Lenses to all case classes

* [core] Add @Lenses to case classes

* [core] ActionGenerator use lense

* [core] ConfigOption use Lenses

* [core] ConfigurationBuilder remove unused fields

* [core] ConfigurationBuilder refactoring

* [core] SyncLogging use lenses

* [core] syncLogging refactoring

build.sbt

@@ -26,6 +26,9 @@ val commonSettings = Seq(
     "-language:postfixOps",
     "-language:higherKinds",
     "-Ypartial-unification"),
+  addCompilerPlugin(
+    "org.scalameta" % "paradise" % "3.0.0-M11" cross CrossVersion.full
+  ),
   test in assembly := {}
 )
 
@@ -38,6 +41,12 @@ val testDependencies = Seq(
     "org.scalamock" %% "scalamock" % "4.3.0" % Test
   )
 )
+val domainDependencies = Seq(
+  libraryDependencies ++= Seq(
+    "com.github.julien-truffaut"  %%  "monocle-core"    % "1.6.0",
+    "com.github.julien-truffaut"  %%  "monocle-macro"   % "1.6.0",
+  )
+)
 val commandLineParsing = Seq(
   libraryDependencies ++= Seq(
     "com.github.scopt" %% "scopt" % "4.0.0-RC2"
@@ -109,6 +118,7 @@ lazy val `storage-api` = (project in file("storage-api"))
 
 lazy val domain = (project in file("domain"))
   .settings(commonSettings)
+  .settings(domainDependencies)
   .settings(assemblyJarName in assembly := "domain.jar")
   .settings(catsEffectsSettings)
   .settings(testDependencies)

cli/src/main/scala/net/kemitix/thorp/cli/ParseArgs.scala

@@ -52,6 +52,6 @@ object ParseArgs {
   def apply(args: List[String]): Option[ConfigOptions] =
     OParser
       .parse(configParser, args, List())
-      .map(ConfigOptions)
+      .map(ConfigOptions(_))
 
 }

core/src/main/scala/net/kemitix/thorp/core/Action.scala

@@ -1,5 +1,6 @@
 package net.kemitix.thorp.core
 
+import monocle.macros.Lenses
 import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
 
 sealed trait Action {
@@ -8,18 +9,21 @@ sealed trait Action {
 }
 object Action {
 
+  @Lenses
   final case class DoNothing(
       bucket: Bucket,
       remoteKey: RemoteKey,
       size: Long
   ) extends Action
 
+  @Lenses
   final case class ToUpload(
       bucket: Bucket,
       localFile: LocalFile,
       size: Long
   ) extends Action
 
+  @Lenses
   final case class ToCopy(
       bucket: Bucket,
       sourceKey: RemoteKey,
@@ -28,6 +32,7 @@ object Action {
       size: Long
   ) extends Action
 
+  @Lenses
   final case class ToDelete(
       bucket: Bucket,
       remoteKey: RemoteKey,

core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala

@@ -35,15 +35,15 @@ object ActionGenerator {
         doNothing(c.bucket, localFile.remoteKey)
     }
 
+  private def key = LocalFile.remoteKey ^|-> RemoteKey.key
+
   def isUploadAlreadyQueued(
       previousActions: Stream[Action]
   )(
       localFile: LocalFile
-  ): Boolean = {
+  ): Boolean = !previousActions.exists {
-    !previousActions.exists {
+    case ToUpload(_, lf, _) => key.get(lf) equals key.get(localFile)
-      case ToUpload(_, lf, _) => lf.remoteKey.key equals localFile.remoteKey.key
+    case _                  => false
-      case _                  => false
-    }
   }
 
   private def doNothing(

core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala

@@ -2,47 +2,57 @@ package net.kemitix.thorp.core
 
 import java.nio.file.Path
 
+import monocle.macros.Lenses
 import net.kemitix.thorp.domain
 import net.kemitix.thorp.domain.{Config, RemoteKey}
+import net.kemitix.thorp.domain.Config._
 
 sealed trait ConfigOption {
   def update(config: Config): Config
 }
 
 object ConfigOption {
+
+  @Lenses
   case class Source(path: Path) extends ConfigOption {
     override def update(config: Config): Config =
-      config.copy(sources = config.sources ++ path)
+      sources.modify(_ ++ path)(config)
   }
 
+  @Lenses
   case class Bucket(name: String) extends ConfigOption {
     override def update(config: Config): Config =
       if (config.bucket.name.isEmpty)
-        config.copy(bucket = domain.Bucket(name))
+        bucket.set(domain.Bucket(name))(config)
       else
         config
   }
 
+  @Lenses
   case class Prefix(path: String) extends ConfigOption {
     override def update(config: Config): Config =
       if (config.prefix.key.isEmpty)
-        config.copy(prefix = RemoteKey(path))
+        prefix.set(RemoteKey(path))(config)
       else
         config
   }
 
+  @Lenses
   case class Include(pattern: String) extends ConfigOption {
     override def update(config: Config): Config =
-      config.copy(filters = domain.Filter.Include(pattern) :: config.filters)
+      filters.modify(domain.Filter.Include(pattern) :: _)(config)
   }
 
+  @Lenses
   case class Exclude(pattern: String) extends ConfigOption {
     override def update(config: Config): Config =
-      config.copy(filters = domain.Filter.Exclude(pattern) :: config.filters)
+      filters.modify(domain.Filter.Exclude(pattern) :: _)(config)
   }
 
+  @Lenses
   case class Debug() extends ConfigOption {
-    override def update(config: Config): Config = config.copy(debug = true)
+    override def update(config: Config): Config =
+      debug.set(true)(config)
   }
 
   case object Version extends ConfigOption {
@@ -50,12 +60,14 @@ object ConfigOption {
   }
 
   case object BatchMode extends ConfigOption {
-    override def update(config: Config): Config = config.copy(batchMode = true)
+    override def update(config: Config): Config =
+      batchMode.set(true)(config)
   }
 
   case object IgnoreUserOptions extends ConfigOption {
     override def update(config: Config): Config = config
   }
+
   case object IgnoreGlobalOptions extends ConfigOption {
     override def update(config: Config): Config = config
   }

core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala

@@ -1,7 +1,9 @@
 package net.kemitix.thorp.core
 
 import cats.Semigroup
+import monocle.macros.Lenses
 
+@Lenses
 case class ConfigOptions(
     options: List[ConfigOption] = List()
 ) extends Semigroup[ConfigOptions] {

core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala

@@ -1,11 +1,12 @@
 package net.kemitix.thorp.core
 
-import java.nio.file.{Path, Paths}
+import java.nio.file.Paths
 
 import cats.data.NonEmptyChain
 import cats.effect.IO
 import net.kemitix.thorp.core.ConfigValidator.validateConfig
 import net.kemitix.thorp.core.ParseConfigFile.parseFile
+import net.kemitix.thorp.core.ConfigOptions.options
 import net.kemitix.thorp.domain.Config
 
 /**
@@ -14,20 +15,18 @@ import net.kemitix.thorp.domain.Config
   */
 trait ConfigurationBuilder {
 
-  private val sourceConfigFilename = ".thorp.config"
+  private val userConfigFilename = ".config/thorp.conf"
-  private val userConfigFilename   = ".config/thorp.conf"
+  private val globalConfig       = Paths.get("/etc/thorp.conf")
-  private val globalConfig         = Paths.get("/etc/thorp.conf")
+  private val userHome           = Paths.get(System.getProperty("user.home"))
-  private val userHome             = Paths.get(System.getProperty("user.home"))
-  private val pwd                  = Paths.get(System.getenv("PWD"))
 
   def buildConfig(priorityOpts: ConfigOptions)
-      : IO[Either[NonEmptyChain[ConfigValidation], Config]] = {
+    : IO[Either[NonEmptyChain[ConfigValidation], Config]] = {
     val sources = ConfigQuery.sources(priorityOpts)
     for {
-      sourceOptions <- SourceConfigLoader.loadSourceConfigs(sources)
+      sourceOpts <- SourceConfigLoader.loadSourceConfigs(sources)
-      userOptions   <- userOptions(priorityOpts ++ sourceOptions)
+      userOpts   <- userOptions(priorityOpts ++ sourceOpts)
-      globalOptions <- globalOptions(priorityOpts ++ sourceOptions ++ userOptions)
+      globalOpts <- globalOptions(priorityOpts ++ sourceOpts ++ userOpts)
-      collected = priorityOpts ++ sourceOptions ++ userOptions ++ globalOptions
+      collected = priorityOpts ++ sourceOpts ++ userOpts ++ globalOpts
       config    = collateOptions(collected)
     } yield validateConfig(config).toEither
   }
@@ -36,20 +35,18 @@ trait ConfigurationBuilder {
 
   private def userOptions(priorityOpts: ConfigOptions) =
     if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
-    else readFile(userHome, userConfigFilename)
+    else parseFile(userHome.resolve(userConfigFilename))
 
   private def globalOptions(priorityOpts: ConfigOptions) =
     if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig
     else parseFile(globalConfig)
 
-  private def readFile(
-      source: Path,
-      filename: String
-  ) =
-    parseFile(source.resolve(filename))
-
   private def collateOptions(configOptions: ConfigOptions): Config =
-    configOptions.options.foldLeft(Config())((c, co) => co.update(c))
+    options
+      .get(configOptions)
+      .foldLeft(Config()) { (config, configOption) =>
+        configOption.update(config)
+      }
 
 }
 

core/src/main/scala/net/kemitix/thorp/core/Counters.scala

@@ -1,5 +1,8 @@
 package net.kemitix.thorp.core
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class Counters(
     uploaded: Int = 0,
     deleted: Int = 0,

core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala

@@ -1,7 +1,9 @@
 package net.kemitix.thorp.core
 
+import monocle.macros.Lenses
 import net.kemitix.thorp.domain.LocalFile
 
+@Lenses
 case class LocalFiles(
     localFiles: Stream[LocalFile] = Stream(),
     count: Long = 0,

core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala

@@ -12,7 +12,7 @@ trait SourceConfigLoader {
     sources => {
 
       val sourceConfigOptions =
-        ConfigOptions(sources.paths.map(ConfigOption.Source))
+        ConfigOptions(sources.paths.map(ConfigOption.Source(_)))
 
       val reduce: List[ConfigOptions] => ConfigOptions =
         _.foldLeft(sourceConfigOptions) { (acc, co) => acc ++ co }

core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala

@@ -18,11 +18,14 @@ trait SyncLogging {
       sources: Sources
   )(implicit logger: Logger): IO[Unit] = {
     val sourcesList = sources.paths.mkString(", ")
-    logger.info(s"Bucket: ${bucket.name}, Prefix: ${prefix.key}, Source: $sourcesList")
+    logger.info(
+      List(s"Bucket: ${bucket.name}",
+           s"Prefix: ${prefix.key}",
+           s"Source: $sourcesList")
+        .mkString(", "))
   }
 
-  def logFileScan(implicit c: Config,
+  def logFileScan(implicit c: Config, logger: Logger): IO[Unit] =
-                  logger: Logger): IO[Unit] =
     logger.info(s"Scanning local files: ${c.sources.paths.mkString(", ")}...")
 
   def logRunFinished(
@@ -50,16 +53,14 @@ trait SyncLogging {
 
   private def countActivities: (Counters, StorageQueueEvent) => Counters =
     (counters: Counters, s3Action: StorageQueueEvent) => {
+      import Counters._
+      val increment: Int => Int = _ + 1
       s3Action match {
-        case _: UploadQueueEvent =>
+        case _: UploadQueueEvent => uploaded.modify(increment)(counters)
-          counters.copy(uploaded = counters.uploaded + 1)
+        case _: CopyQueueEvent   => copied.modify(increment)(counters)
-        case _: CopyQueueEvent =>
+        case _: DeleteQueueEvent => deleted.modify(increment)(counters)
-          counters.copy(copied = counters.copied + 1)
+        case _: ErrorQueueEvent  => errors.modify(increment)(counters)
-        case _: DeleteQueueEvent =>
+        case _                   => counters
-          counters.copy(deleted = counters.deleted + 1)
-        case ErrorQueueEvent(_, _) =>
-          counters.copy(errors = counters.errors + 1)
-        case _ => counters
       }
     }
 

core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala

@@ -1,7 +1,9 @@
 package net.kemitix.thorp.core
 
+import monocle.macros.Lenses
 import net.kemitix.thorp.domain.SyncTotals
 
+@Lenses
 case class SyncPlan(
     actions: Stream[Action] = Stream(),
     syncTotals: SyncTotals = SyncTotals()

core/src/main/scala/net/kemitix/thorp/core/UnversionedMirrorArchive.scala

@@ -1,6 +1,7 @@
 package net.kemitix.thorp.core
 
 import cats.effect.IO
+import monocle.macros.Lenses
 import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
 import net.kemitix.thorp.domain.StorageQueueEvent.DoNothingQueueEvent
 import net.kemitix.thorp.domain.{
@@ -13,6 +14,7 @@ import net.kemitix.thorp.domain.{
 }
 import net.kemitix.thorp.storage.api.StorageService
 
+@Lenses
 case class UnversionedMirrorArchive(
     storageService: StorageService,
     batchMode: Boolean,
@@ -52,7 +54,7 @@ case class UnversionedMirrorArchive(
       localFile,
       bucket,
       batchMode,
-      new UploadEventListener(localFile, index, syncTotals, totalBytesSoFar),
+      UploadEventListener(localFile, index, syncTotals, totalBytesSoFar),
       1)
 }
 

domain/src/main/scala/net/kemitix/thorp/domain/Bucket.scala

@@ -1,5 +1,8 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class Bucket(
     name: String
 )

domain/src/main/scala/net/kemitix/thorp/domain/Config.scala

@@ -1,5 +1,8 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class Config(
     bucket: Bucket = Bucket(""),
     prefix: RemoteKey = RemoteKey(""),

domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala

@@ -3,6 +3,8 @@ package net.kemitix.thorp.domain
 import java.nio.file.Path
 import java.util.regex.Pattern
 
+import monocle.macros.Lenses
+
 sealed trait Filter
 
 object Filter {
@@ -30,6 +32,7 @@ object Filter {
     }
   }
 
+  @Lenses
   case class Include(
       include: String = ".*"
   ) extends Filter {
@@ -40,6 +43,7 @@ object Filter {
 
   }
 
+  @Lenses
   case class Exclude(
       exclude: String
   ) extends Filter {

domain/src/main/scala/net/kemitix/thorp/domain/HashModified.scala

@@ -1,5 +1,8 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class HashModified(
     hash: MD5Hash,
     modified: LastModified

domain/src/main/scala/net/kemitix/thorp/domain/KeyModified.scala

@@ -1,5 +1,8 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class KeyModified(
     key: RemoteKey,
     modified: LastModified

domain/src/main/scala/net/kemitix/thorp/domain/LastModified.scala

@@ -2,6 +2,9 @@ package net.kemitix.thorp.domain
 
 import java.time.Instant
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class LastModified(
     when: Instant = Instant.now
 )

domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala

@@ -3,6 +3,9 @@ package net.kemitix.thorp.domain
 import java.io.File
 import java.nio.file.Path
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class LocalFile(
     file: File,
     source: File,

domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala

@@ -4,6 +4,9 @@ import java.util.Base64
 
 import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class MD5Hash(
     in: String
 ) {

domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala

@@ -3,6 +3,9 @@ package net.kemitix.thorp.domain
 import java.io.File
 import java.nio.file.{Path, Paths}
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class RemoteKey(
     key: String
 ) {

domain/src/main/scala/net/kemitix/thorp/domain/RemoteMetaData.scala

@@ -1,5 +1,8 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
+@Lenses
 final case class RemoteMetaData(
     remoteKey: RemoteKey,
     hash: MD5Hash,

domain/src/main/scala/net/kemitix/thorp/domain/S3MetaData.scala

@@ -1,6 +1,9 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
 // For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
+@Lenses
 final case class S3MetaData(
     localFile: LocalFile,
     matchByHash: Set[RemoteMetaData],

domain/src/main/scala/net/kemitix/thorp/domain/S3ObjectsData.scala

@@ -1,8 +1,12 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
 /**
   * A list of objects and their MD5 hash values.
   */
+
+@Lenses
 final case class S3ObjectsData(
     byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty,
     byKey: Map[RemoteKey, HashModified] = Map.empty

domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala

@@ -2,6 +2,8 @@ package net.kemitix.thorp.domain
 
 import java.nio.file.Path
 
+import monocle.macros.Lenses
+
 /**
   * The paths to synchronise with target.
   *
@@ -12,12 +14,13 @@ import java.nio.file.Path
   *
   * A path should only occur once in paths.
   */
+@Lenses
 case class Sources(
     paths: List[Path]
 ) {
-  def ++(path: Path): Sources             = this ++ List(path)
+  def ++(path: Path): Sources = this ++ List(path)
-  def ++(otherPaths: List[Path]): Sources = Sources(
+  def ++(otherPaths: List[Path]): Sources =
-    otherPaths.foldLeft(paths) { (acc, path) =>
+    Sources(otherPaths.foldLeft(paths) { (acc, path) =>
       if (acc.contains(path)) acc
       else acc ++ List(path)
     })

domain/src/main/scala/net/kemitix/thorp/domain/StorageQueueEvent.scala

@@ -1,5 +1,7 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
 sealed trait StorageQueueEvent {
 
   val order: Int
@@ -8,18 +10,21 @@ sealed trait StorageQueueEvent {
 
 object StorageQueueEvent {
 
+  @Lenses
   final case class DoNothingQueueEvent(
       remoteKey: RemoteKey
   ) extends StorageQueueEvent {
     override val order: Int = 0
   }
 
+  @Lenses
   final case class CopyQueueEvent(
       remoteKey: RemoteKey
   ) extends StorageQueueEvent {
     override val order: Int = 1
   }
 
+  @Lenses
   final case class UploadQueueEvent(
       remoteKey: RemoteKey,
       md5Hash: MD5Hash
@@ -27,12 +32,14 @@ object StorageQueueEvent {
     override val order: Int = 2
   }
 
+  @Lenses
   final case class DeleteQueueEvent(
       remoteKey: RemoteKey
   ) extends StorageQueueEvent {
     override val order: Int = 3
   }
 
+  @Lenses
   final case class ErrorQueueEvent(
       remoteKey: RemoteKey,
       e: Throwable
@@ -40,6 +47,7 @@ object StorageQueueEvent {
     override val order: Int = 10
   }
 
+  @Lenses
   final case class ShutdownQueueEvent() extends StorageQueueEvent {
     override val order: Int = 99
   }

domain/src/main/scala/net/kemitix/thorp/domain/SyncTotals.scala

@@ -1,5 +1,8 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
+@Lenses
 case class SyncTotals(
     count: Long = 0L,
     totalSizeBytes: Long = 0L,

domain/src/main/scala/net/kemitix/thorp/domain/UploadEvent.scala

@@ -1,21 +1,26 @@
 package net.kemitix.thorp.domain
 
+import monocle.macros.Lenses
+
 sealed trait UploadEvent {
   def name: String
 }
 
 object UploadEvent {
 
+  @Lenses
   final case class TransferEvent(
       name: String
   ) extends UploadEvent
 
+  @Lenses
   final case class RequestEvent(
       name: String,
       bytes: Long,
       transferred: Long
   ) extends UploadEvent
 
+  @Lenses
   final case class ByteTransferEvent(
       name: String
   ) extends UploadEvent

domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala

@@ -3,7 +3,10 @@ package net.kemitix.thorp.domain
 import net.kemitix.thorp.domain.UploadEvent.RequestEvent
 import net.kemitix.thorp.domain.UploadEventLogger.logRequestCycle
 
-class UploadEventListener(
+import monocle.macros.Lenses
+
+@Lenses
+case class UploadEventListener(
     localFile: LocalFile,
     index: Int,
     syncTotals: SyncTotals,

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala

@@ -137,7 +137,7 @@ class StorageServiceSuite extends FunSpec with MockFactory {
       val bucket    = Bucket("a-bucket")
       val remoteKey = RemoteKey("prefix/root-file")
       val uploadEventListener =
-        new UploadEventListener(localFile, 1, SyncTotals(), 0L)
+        UploadEventListener(localFile, 1, SyncTotals(), 0L)
 
       val upload = stub[AmazonUpload]
       (amazonTransferManager upload (_: PutObjectRequest))

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderSuite.scala

@@ -36,7 +36,7 @@ class UploaderSuite extends FunSpec with MockFactory {
                                       sourcePath,
                                       fileToKey)
       val uploadEventListener =
-        new UploadEventListener(bigFile, 1, SyncTotals(), 0L)
+        UploadEventListener(bigFile, 1, SyncTotals(), 0L)
       val amazonS3 = mock[AmazonS3]
       val amazonTransferManager =
         AmazonTransferManager(