Use Lenses (#113)

* [sbt] Add monocle for support for lenses

* [domain] UploadEventListener make a case class

* [domain] Add @Lenses to all case classes

* [core] Add @Lenses to case classes

* [core] ActionGenerator use lense

* [core] ConfigOption use Lenses

* [core] ConfigurationBuilder remove unused fields

* [core] ConfigurationBuilder refactoring

* [core] SyncLogging use lenses

* [core] syncLogging refactoring
This commit is contained in:
Paul Campbell 2019-07-18 08:57:14 +01:00 committed by GitHub
parent 0fbae945a7
commit 515b896993
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 148 additions and 52 deletions

View file

@ -26,6 +26,9 @@ val commonSettings = Seq(
"-language:postfixOps",
"-language:higherKinds",
"-Ypartial-unification"),
addCompilerPlugin(
"org.scalameta" % "paradise" % "3.0.0-M11" cross CrossVersion.full
),
test in assembly := {}
)
@ -38,6 +41,12 @@ val testDependencies = Seq(
"org.scalamock" %% "scalamock" % "4.3.0" % Test
)
)
val domainDependencies = Seq(
libraryDependencies ++= Seq(
"com.github.julien-truffaut" %% "monocle-core" % "1.6.0",
"com.github.julien-truffaut" %% "monocle-macro" % "1.6.0",
)
)
val commandLineParsing = Seq(
libraryDependencies ++= Seq(
"com.github.scopt" %% "scopt" % "4.0.0-RC2"
@ -109,6 +118,7 @@ lazy val `storage-api` = (project in file("storage-api"))
lazy val domain = (project in file("domain"))
.settings(commonSettings)
.settings(domainDependencies)
.settings(assemblyJarName in assembly := "domain.jar")
.settings(catsEffectsSettings)
.settings(testDependencies)

View file

@ -52,6 +52,6 @@ object ParseArgs {
def apply(args: List[String]): Option[ConfigOptions] =
OParser
.parse(configParser, args, List())
.map(ConfigOptions)
.map(ConfigOptions(_))
}

View file

@ -1,5 +1,6 @@
package net.kemitix.thorp.core
import monocle.macros.Lenses
import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
sealed trait Action {
@ -8,18 +9,21 @@ sealed trait Action {
}
object Action {
@Lenses
final case class DoNothing(
bucket: Bucket,
remoteKey: RemoteKey,
size: Long
) extends Action
@Lenses
final case class ToUpload(
bucket: Bucket,
localFile: LocalFile,
size: Long
) extends Action
@Lenses
final case class ToCopy(
bucket: Bucket,
sourceKey: RemoteKey,
@ -28,6 +32,7 @@ object Action {
size: Long
) extends Action
@Lenses
final case class ToDelete(
bucket: Bucket,
remoteKey: RemoteKey,

View file

@ -35,15 +35,15 @@ object ActionGenerator {
doNothing(c.bucket, localFile.remoteKey)
}
private def key = LocalFile.remoteKey ^|-> RemoteKey.key
def isUploadAlreadyQueued(
previousActions: Stream[Action]
)(
localFile: LocalFile
): Boolean = {
!previousActions.exists {
case ToUpload(_, lf, _) => lf.remoteKey.key equals localFile.remoteKey.key
case _ => false
}
): Boolean = !previousActions.exists {
case ToUpload(_, lf, _) => key.get(lf) equals key.get(localFile)
case _ => false
}
private def doNothing(

View file

@ -2,47 +2,57 @@ package net.kemitix.thorp.core
import java.nio.file.Path
import monocle.macros.Lenses
import net.kemitix.thorp.domain
import net.kemitix.thorp.domain.{Config, RemoteKey}
import net.kemitix.thorp.domain.Config._
sealed trait ConfigOption {
def update(config: Config): Config
}
object ConfigOption {
@Lenses
case class Source(path: Path) extends ConfigOption {
override def update(config: Config): Config =
config.copy(sources = config.sources ++ path)
sources.modify(_ ++ path)(config)
}
@Lenses
case class Bucket(name: String) extends ConfigOption {
override def update(config: Config): Config =
if (config.bucket.name.isEmpty)
config.copy(bucket = domain.Bucket(name))
bucket.set(domain.Bucket(name))(config)
else
config
}
@Lenses
case class Prefix(path: String) extends ConfigOption {
override def update(config: Config): Config =
if (config.prefix.key.isEmpty)
config.copy(prefix = RemoteKey(path))
prefix.set(RemoteKey(path))(config)
else
config
}
@Lenses
case class Include(pattern: String) extends ConfigOption {
override def update(config: Config): Config =
config.copy(filters = domain.Filter.Include(pattern) :: config.filters)
filters.modify(domain.Filter.Include(pattern) :: _)(config)
}
@Lenses
case class Exclude(pattern: String) extends ConfigOption {
override def update(config: Config): Config =
config.copy(filters = domain.Filter.Exclude(pattern) :: config.filters)
filters.modify(domain.Filter.Exclude(pattern) :: _)(config)
}
@Lenses
case class Debug() extends ConfigOption {
override def update(config: Config): Config = config.copy(debug = true)
override def update(config: Config): Config =
debug.set(true)(config)
}
case object Version extends ConfigOption {
@ -50,12 +60,14 @@ object ConfigOption {
}
case object BatchMode extends ConfigOption {
override def update(config: Config): Config = config.copy(batchMode = true)
override def update(config: Config): Config =
batchMode.set(true)(config)
}
case object IgnoreUserOptions extends ConfigOption {
override def update(config: Config): Config = config
}
case object IgnoreGlobalOptions extends ConfigOption {
override def update(config: Config): Config = config
}

View file

@ -1,7 +1,9 @@
package net.kemitix.thorp.core
import cats.Semigroup
import monocle.macros.Lenses
@Lenses
case class ConfigOptions(
options: List[ConfigOption] = List()
) extends Semigroup[ConfigOptions] {

View file

@ -1,11 +1,12 @@
package net.kemitix.thorp.core
import java.nio.file.{Path, Paths}
import java.nio.file.Paths
import cats.data.NonEmptyChain
import cats.effect.IO
import net.kemitix.thorp.core.ConfigValidator.validateConfig
import net.kemitix.thorp.core.ParseConfigFile.parseFile
import net.kemitix.thorp.core.ConfigOptions.options
import net.kemitix.thorp.domain.Config
/**
@ -14,20 +15,18 @@ import net.kemitix.thorp.domain.Config
*/
trait ConfigurationBuilder {
private val sourceConfigFilename = ".thorp.config"
private val userConfigFilename = ".config/thorp.conf"
private val globalConfig = Paths.get("/etc/thorp.conf")
private val userHome = Paths.get(System.getProperty("user.home"))
private val pwd = Paths.get(System.getenv("PWD"))
private val userConfigFilename = ".config/thorp.conf"
private val globalConfig = Paths.get("/etc/thorp.conf")
private val userHome = Paths.get(System.getProperty("user.home"))
def buildConfig(priorityOpts: ConfigOptions)
: IO[Either[NonEmptyChain[ConfigValidation], Config]] = {
: IO[Either[NonEmptyChain[ConfigValidation], Config]] = {
val sources = ConfigQuery.sources(priorityOpts)
for {
sourceOptions <- SourceConfigLoader.loadSourceConfigs(sources)
userOptions <- userOptions(priorityOpts ++ sourceOptions)
globalOptions <- globalOptions(priorityOpts ++ sourceOptions ++ userOptions)
collected = priorityOpts ++ sourceOptions ++ userOptions ++ globalOptions
sourceOpts <- SourceConfigLoader.loadSourceConfigs(sources)
userOpts <- userOptions(priorityOpts ++ sourceOpts)
globalOpts <- globalOptions(priorityOpts ++ sourceOpts ++ userOpts)
collected = priorityOpts ++ sourceOpts ++ userOpts ++ globalOpts
config = collateOptions(collected)
} yield validateConfig(config).toEither
}
@ -36,20 +35,18 @@ trait ConfigurationBuilder {
private def userOptions(priorityOpts: ConfigOptions) =
if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
else readFile(userHome, userConfigFilename)
else parseFile(userHome.resolve(userConfigFilename))
private def globalOptions(priorityOpts: ConfigOptions) =
if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig
else parseFile(globalConfig)
private def readFile(
source: Path,
filename: String
) =
parseFile(source.resolve(filename))
private def collateOptions(configOptions: ConfigOptions): Config =
configOptions.options.foldLeft(Config())((c, co) => co.update(c))
options
.get(configOptions)
.foldLeft(Config()) { (config, configOption) =>
configOption.update(config)
}
}

View file

@ -1,5 +1,8 @@
package net.kemitix.thorp.core
import monocle.macros.Lenses
@Lenses
final case class Counters(
uploaded: Int = 0,
deleted: Int = 0,

View file

@ -1,7 +1,9 @@
package net.kemitix.thorp.core
import monocle.macros.Lenses
import net.kemitix.thorp.domain.LocalFile
@Lenses
case class LocalFiles(
localFiles: Stream[LocalFile] = Stream(),
count: Long = 0,

View file

@ -12,7 +12,7 @@ trait SourceConfigLoader {
sources => {
val sourceConfigOptions =
ConfigOptions(sources.paths.map(ConfigOption.Source))
ConfigOptions(sources.paths.map(ConfigOption.Source(_)))
val reduce: List[ConfigOptions] => ConfigOptions =
_.foldLeft(sourceConfigOptions) { (acc, co) => acc ++ co }

View file

@ -18,11 +18,14 @@ trait SyncLogging {
sources: Sources
)(implicit logger: Logger): IO[Unit] = {
val sourcesList = sources.paths.mkString(", ")
logger.info(s"Bucket: ${bucket.name}, Prefix: ${prefix.key}, Source: $sourcesList")
logger.info(
List(s"Bucket: ${bucket.name}",
s"Prefix: ${prefix.key}",
s"Source: $sourcesList")
.mkString(", "))
}
def logFileScan(implicit c: Config,
logger: Logger): IO[Unit] =
def logFileScan(implicit c: Config, logger: Logger): IO[Unit] =
logger.info(s"Scanning local files: ${c.sources.paths.mkString(", ")}...")
def logRunFinished(
@ -50,16 +53,14 @@ trait SyncLogging {
private def countActivities: (Counters, StorageQueueEvent) => Counters =
(counters: Counters, s3Action: StorageQueueEvent) => {
import Counters._
val increment: Int => Int = _ + 1
s3Action match {
case _: UploadQueueEvent =>
counters.copy(uploaded = counters.uploaded + 1)
case _: CopyQueueEvent =>
counters.copy(copied = counters.copied + 1)
case _: DeleteQueueEvent =>
counters.copy(deleted = counters.deleted + 1)
case ErrorQueueEvent(_, _) =>
counters.copy(errors = counters.errors + 1)
case _ => counters
case _: UploadQueueEvent => uploaded.modify(increment)(counters)
case _: CopyQueueEvent => copied.modify(increment)(counters)
case _: DeleteQueueEvent => deleted.modify(increment)(counters)
case _: ErrorQueueEvent => errors.modify(increment)(counters)
case _ => counters
}
}

View file

@ -1,7 +1,9 @@
package net.kemitix.thorp.core
import monocle.macros.Lenses
import net.kemitix.thorp.domain.SyncTotals
@Lenses
case class SyncPlan(
actions: Stream[Action] = Stream(),
syncTotals: SyncTotals = SyncTotals()

View file

@ -1,6 +1,7 @@
package net.kemitix.thorp.core
import cats.effect.IO
import monocle.macros.Lenses
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
import net.kemitix.thorp.domain.StorageQueueEvent.DoNothingQueueEvent
import net.kemitix.thorp.domain.{
@ -13,6 +14,7 @@ import net.kemitix.thorp.domain.{
}
import net.kemitix.thorp.storage.api.StorageService
@Lenses
case class UnversionedMirrorArchive(
storageService: StorageService,
batchMode: Boolean,
@ -52,7 +54,7 @@ case class UnversionedMirrorArchive(
localFile,
bucket,
batchMode,
new UploadEventListener(localFile, index, syncTotals, totalBytesSoFar),
UploadEventListener(localFile, index, syncTotals, totalBytesSoFar),
1)
}

View file

@ -1,5 +1,8 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
@Lenses
final case class Bucket(
name: String
)

View file

@ -1,5 +1,8 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
@Lenses
final case class Config(
bucket: Bucket = Bucket(""),
prefix: RemoteKey = RemoteKey(""),

View file

@ -3,6 +3,8 @@ package net.kemitix.thorp.domain
import java.nio.file.Path
import java.util.regex.Pattern
import monocle.macros.Lenses
sealed trait Filter
object Filter {
@ -30,6 +32,7 @@ object Filter {
}
}
@Lenses
case class Include(
include: String = ".*"
) extends Filter {
@ -40,6 +43,7 @@ object Filter {
}
@Lenses
case class Exclude(
exclude: String
) extends Filter {

View file

@ -1,5 +1,8 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
@Lenses
final case class HashModified(
hash: MD5Hash,
modified: LastModified

View file

@ -1,5 +1,8 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
@Lenses
final case class KeyModified(
key: RemoteKey,
modified: LastModified

View file

@ -2,6 +2,9 @@ package net.kemitix.thorp.domain
import java.time.Instant
import monocle.macros.Lenses
@Lenses
final case class LastModified(
when: Instant = Instant.now
)

View file

@ -3,6 +3,9 @@ package net.kemitix.thorp.domain
import java.io.File
import java.nio.file.Path
import monocle.macros.Lenses
@Lenses
final case class LocalFile(
file: File,
source: File,

View file

@ -4,6 +4,9 @@ import java.util.Base64
import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
import monocle.macros.Lenses
@Lenses
final case class MD5Hash(
in: String
) {

View file

@ -3,6 +3,9 @@ package net.kemitix.thorp.domain
import java.io.File
import java.nio.file.{Path, Paths}
import monocle.macros.Lenses
@Lenses
final case class RemoteKey(
key: String
) {

View file

@ -1,5 +1,8 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
@Lenses
final case class RemoteMetaData(
remoteKey: RemoteKey,
hash: MD5Hash,

View file

@ -1,6 +1,9 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
@Lenses
final case class S3MetaData(
localFile: LocalFile,
matchByHash: Set[RemoteMetaData],

View file

@ -1,8 +1,12 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
/**
* A list of objects and their MD5 hash values.
*/
@Lenses
final case class S3ObjectsData(
byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty,
byKey: Map[RemoteKey, HashModified] = Map.empty

View file

@ -2,6 +2,8 @@ package net.kemitix.thorp.domain
import java.nio.file.Path
import monocle.macros.Lenses
/**
* The paths to synchronise with target.
*
@ -12,12 +14,13 @@ import java.nio.file.Path
*
* A path should only occur once in paths.
*/
@Lenses
case class Sources(
paths: List[Path]
) {
def ++(path: Path): Sources = this ++ List(path)
def ++(otherPaths: List[Path]): Sources = Sources(
otherPaths.foldLeft(paths) { (acc, path) =>
def ++(path: Path): Sources = this ++ List(path)
def ++(otherPaths: List[Path]): Sources =
Sources(otherPaths.foldLeft(paths) { (acc, path) =>
if (acc.contains(path)) acc
else acc ++ List(path)
})

View file

@ -1,5 +1,7 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
sealed trait StorageQueueEvent {
val order: Int
@ -8,18 +10,21 @@ sealed trait StorageQueueEvent {
object StorageQueueEvent {
@Lenses
final case class DoNothingQueueEvent(
remoteKey: RemoteKey
) extends StorageQueueEvent {
override val order: Int = 0
}
@Lenses
final case class CopyQueueEvent(
remoteKey: RemoteKey
) extends StorageQueueEvent {
override val order: Int = 1
}
@Lenses
final case class UploadQueueEvent(
remoteKey: RemoteKey,
md5Hash: MD5Hash
@ -27,12 +32,14 @@ object StorageQueueEvent {
override val order: Int = 2
}
@Lenses
final case class DeleteQueueEvent(
remoteKey: RemoteKey
) extends StorageQueueEvent {
override val order: Int = 3
}
@Lenses
final case class ErrorQueueEvent(
remoteKey: RemoteKey,
e: Throwable
@ -40,6 +47,7 @@ object StorageQueueEvent {
override val order: Int = 10
}
@Lenses
final case class ShutdownQueueEvent() extends StorageQueueEvent {
override val order: Int = 99
}

View file

@ -1,5 +1,8 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
@Lenses
case class SyncTotals(
count: Long = 0L,
totalSizeBytes: Long = 0L,

View file

@ -1,21 +1,26 @@
package net.kemitix.thorp.domain
import monocle.macros.Lenses
sealed trait UploadEvent {
def name: String
}
object UploadEvent {
@Lenses
final case class TransferEvent(
name: String
) extends UploadEvent
@Lenses
final case class RequestEvent(
name: String,
bytes: Long,
transferred: Long
) extends UploadEvent
@Lenses
final case class ByteTransferEvent(
name: String
) extends UploadEvent

View file

@ -3,7 +3,10 @@ package net.kemitix.thorp.domain
import net.kemitix.thorp.domain.UploadEvent.RequestEvent
import net.kemitix.thorp.domain.UploadEventLogger.logRequestCycle
class UploadEventListener(
import monocle.macros.Lenses
@Lenses
case class UploadEventListener(
localFile: LocalFile,
index: Int,
syncTotals: SyncTotals,

View file

@ -137,7 +137,7 @@ class StorageServiceSuite extends FunSpec with MockFactory {
val bucket = Bucket("a-bucket")
val remoteKey = RemoteKey("prefix/root-file")
val uploadEventListener =
new UploadEventListener(localFile, 1, SyncTotals(), 0L)
UploadEventListener(localFile, 1, SyncTotals(), 0L)
val upload = stub[AmazonUpload]
(amazonTransferManager upload (_: PutObjectRequest))

View file

@ -36,7 +36,7 @@ class UploaderSuite extends FunSpec with MockFactory {
sourcePath,
fileToKey)
val uploadEventListener =
new UploadEventListener(bigFile, 1, SyncTotals(), 0L)
UploadEventListener(bigFile, 1, SyncTotals(), 0L)
val amazonS3 = mock[AmazonS3]
val amazonTransferManager =
AmazonTransferManager(