Use Lenses (#113)
* [sbt] Add monocle for support for lenses * [domain] UploadEventListener make a case class * [domain] Add @Lenses to all case classes * [core] Add @Lenses to case classes * [core] ActionGenerator use lense * [core] ConfigOption use Lenses * [core] ConfigurationBuilder remove unused fields * [core] ConfigurationBuilder refactoring * [core] SyncLogging use lenses * [core] syncLogging refactoring
This commit is contained in:
parent
0fbae945a7
commit
515b896993
32 changed files with 148 additions and 52 deletions
10
build.sbt
10
build.sbt
|
@ -26,6 +26,9 @@ val commonSettings = Seq(
|
||||||
"-language:postfixOps",
|
"-language:postfixOps",
|
||||||
"-language:higherKinds",
|
"-language:higherKinds",
|
||||||
"-Ypartial-unification"),
|
"-Ypartial-unification"),
|
||||||
|
addCompilerPlugin(
|
||||||
|
"org.scalameta" % "paradise" % "3.0.0-M11" cross CrossVersion.full
|
||||||
|
),
|
||||||
test in assembly := {}
|
test in assembly := {}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -38,6 +41,12 @@ val testDependencies = Seq(
|
||||||
"org.scalamock" %% "scalamock" % "4.3.0" % Test
|
"org.scalamock" %% "scalamock" % "4.3.0" % Test
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
val domainDependencies = Seq(
|
||||||
|
libraryDependencies ++= Seq(
|
||||||
|
"com.github.julien-truffaut" %% "monocle-core" % "1.6.0",
|
||||||
|
"com.github.julien-truffaut" %% "monocle-macro" % "1.6.0",
|
||||||
|
)
|
||||||
|
)
|
||||||
val commandLineParsing = Seq(
|
val commandLineParsing = Seq(
|
||||||
libraryDependencies ++= Seq(
|
libraryDependencies ++= Seq(
|
||||||
"com.github.scopt" %% "scopt" % "4.0.0-RC2"
|
"com.github.scopt" %% "scopt" % "4.0.0-RC2"
|
||||||
|
@ -109,6 +118,7 @@ lazy val `storage-api` = (project in file("storage-api"))
|
||||||
|
|
||||||
lazy val domain = (project in file("domain"))
|
lazy val domain = (project in file("domain"))
|
||||||
.settings(commonSettings)
|
.settings(commonSettings)
|
||||||
|
.settings(domainDependencies)
|
||||||
.settings(assemblyJarName in assembly := "domain.jar")
|
.settings(assemblyJarName in assembly := "domain.jar")
|
||||||
.settings(catsEffectsSettings)
|
.settings(catsEffectsSettings)
|
||||||
.settings(testDependencies)
|
.settings(testDependencies)
|
||||||
|
|
|
@ -52,6 +52,6 @@ object ParseArgs {
|
||||||
def apply(args: List[String]): Option[ConfigOptions] =
|
def apply(args: List[String]): Option[ConfigOptions] =
|
||||||
OParser
|
OParser
|
||||||
.parse(configParser, args, List())
|
.parse(configParser, args, List())
|
||||||
.map(ConfigOptions)
|
.map(ConfigOptions(_))
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
package net.kemitix.thorp.core
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
|
import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
|
||||||
|
|
||||||
sealed trait Action {
|
sealed trait Action {
|
||||||
|
@ -8,18 +9,21 @@ sealed trait Action {
|
||||||
}
|
}
|
||||||
object Action {
|
object Action {
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class DoNothing(
|
final case class DoNothing(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
remoteKey: RemoteKey,
|
remoteKey: RemoteKey,
|
||||||
size: Long
|
size: Long
|
||||||
) extends Action
|
) extends Action
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class ToUpload(
|
final case class ToUpload(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
size: Long
|
size: Long
|
||||||
) extends Action
|
) extends Action
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class ToCopy(
|
final case class ToCopy(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
|
@ -28,6 +32,7 @@ object Action {
|
||||||
size: Long
|
size: Long
|
||||||
) extends Action
|
) extends Action
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class ToDelete(
|
final case class ToDelete(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
remoteKey: RemoteKey,
|
remoteKey: RemoteKey,
|
||||||
|
|
|
@ -35,15 +35,15 @@ object ActionGenerator {
|
||||||
doNothing(c.bucket, localFile.remoteKey)
|
doNothing(c.bucket, localFile.remoteKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private def key = LocalFile.remoteKey ^|-> RemoteKey.key
|
||||||
|
|
||||||
def isUploadAlreadyQueued(
|
def isUploadAlreadyQueued(
|
||||||
previousActions: Stream[Action]
|
previousActions: Stream[Action]
|
||||||
)(
|
)(
|
||||||
localFile: LocalFile
|
localFile: LocalFile
|
||||||
): Boolean = {
|
): Boolean = !previousActions.exists {
|
||||||
!previousActions.exists {
|
case ToUpload(_, lf, _) => key.get(lf) equals key.get(localFile)
|
||||||
case ToUpload(_, lf, _) => lf.remoteKey.key equals localFile.remoteKey.key
|
case _ => false
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private def doNothing(
|
private def doNothing(
|
||||||
|
|
|
@ -2,47 +2,57 @@ package net.kemitix.thorp.core
|
||||||
|
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
import net.kemitix.thorp.domain
|
import net.kemitix.thorp.domain
|
||||||
import net.kemitix.thorp.domain.{Config, RemoteKey}
|
import net.kemitix.thorp.domain.{Config, RemoteKey}
|
||||||
|
import net.kemitix.thorp.domain.Config._
|
||||||
|
|
||||||
sealed trait ConfigOption {
|
sealed trait ConfigOption {
|
||||||
def update(config: Config): Config
|
def update(config: Config): Config
|
||||||
}
|
}
|
||||||
|
|
||||||
object ConfigOption {
|
object ConfigOption {
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class Source(path: Path) extends ConfigOption {
|
case class Source(path: Path) extends ConfigOption {
|
||||||
override def update(config: Config): Config =
|
override def update(config: Config): Config =
|
||||||
config.copy(sources = config.sources ++ path)
|
sources.modify(_ ++ path)(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class Bucket(name: String) extends ConfigOption {
|
case class Bucket(name: String) extends ConfigOption {
|
||||||
override def update(config: Config): Config =
|
override def update(config: Config): Config =
|
||||||
if (config.bucket.name.isEmpty)
|
if (config.bucket.name.isEmpty)
|
||||||
config.copy(bucket = domain.Bucket(name))
|
bucket.set(domain.Bucket(name))(config)
|
||||||
else
|
else
|
||||||
config
|
config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class Prefix(path: String) extends ConfigOption {
|
case class Prefix(path: String) extends ConfigOption {
|
||||||
override def update(config: Config): Config =
|
override def update(config: Config): Config =
|
||||||
if (config.prefix.key.isEmpty)
|
if (config.prefix.key.isEmpty)
|
||||||
config.copy(prefix = RemoteKey(path))
|
prefix.set(RemoteKey(path))(config)
|
||||||
else
|
else
|
||||||
config
|
config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class Include(pattern: String) extends ConfigOption {
|
case class Include(pattern: String) extends ConfigOption {
|
||||||
override def update(config: Config): Config =
|
override def update(config: Config): Config =
|
||||||
config.copy(filters = domain.Filter.Include(pattern) :: config.filters)
|
filters.modify(domain.Filter.Include(pattern) :: _)(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class Exclude(pattern: String) extends ConfigOption {
|
case class Exclude(pattern: String) extends ConfigOption {
|
||||||
override def update(config: Config): Config =
|
override def update(config: Config): Config =
|
||||||
config.copy(filters = domain.Filter.Exclude(pattern) :: config.filters)
|
filters.modify(domain.Filter.Exclude(pattern) :: _)(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class Debug() extends ConfigOption {
|
case class Debug() extends ConfigOption {
|
||||||
override def update(config: Config): Config = config.copy(debug = true)
|
override def update(config: Config): Config =
|
||||||
|
debug.set(true)(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
case object Version extends ConfigOption {
|
case object Version extends ConfigOption {
|
||||||
|
@ -50,12 +60,14 @@ object ConfigOption {
|
||||||
}
|
}
|
||||||
|
|
||||||
case object BatchMode extends ConfigOption {
|
case object BatchMode extends ConfigOption {
|
||||||
override def update(config: Config): Config = config.copy(batchMode = true)
|
override def update(config: Config): Config =
|
||||||
|
batchMode.set(true)(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
case object IgnoreUserOptions extends ConfigOption {
|
case object IgnoreUserOptions extends ConfigOption {
|
||||||
override def update(config: Config): Config = config
|
override def update(config: Config): Config = config
|
||||||
}
|
}
|
||||||
|
|
||||||
case object IgnoreGlobalOptions extends ConfigOption {
|
case object IgnoreGlobalOptions extends ConfigOption {
|
||||||
override def update(config: Config): Config = config
|
override def update(config: Config): Config = config
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
package net.kemitix.thorp.core
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
import cats.Semigroup
|
import cats.Semigroup
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class ConfigOptions(
|
case class ConfigOptions(
|
||||||
options: List[ConfigOption] = List()
|
options: List[ConfigOption] = List()
|
||||||
) extends Semigroup[ConfigOptions] {
|
) extends Semigroup[ConfigOptions] {
|
||||||
|
|
|
@ -1,11 +1,12 @@
|
||||||
package net.kemitix.thorp.core
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
import java.nio.file.{Path, Paths}
|
import java.nio.file.Paths
|
||||||
|
|
||||||
import cats.data.NonEmptyChain
|
import cats.data.NonEmptyChain
|
||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
import net.kemitix.thorp.core.ConfigValidator.validateConfig
|
import net.kemitix.thorp.core.ConfigValidator.validateConfig
|
||||||
import net.kemitix.thorp.core.ParseConfigFile.parseFile
|
import net.kemitix.thorp.core.ParseConfigFile.parseFile
|
||||||
|
import net.kemitix.thorp.core.ConfigOptions.options
|
||||||
import net.kemitix.thorp.domain.Config
|
import net.kemitix.thorp.domain.Config
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -14,20 +15,18 @@ import net.kemitix.thorp.domain.Config
|
||||||
*/
|
*/
|
||||||
trait ConfigurationBuilder {
|
trait ConfigurationBuilder {
|
||||||
|
|
||||||
private val sourceConfigFilename = ".thorp.config"
|
private val userConfigFilename = ".config/thorp.conf"
|
||||||
private val userConfigFilename = ".config/thorp.conf"
|
private val globalConfig = Paths.get("/etc/thorp.conf")
|
||||||
private val globalConfig = Paths.get("/etc/thorp.conf")
|
private val userHome = Paths.get(System.getProperty("user.home"))
|
||||||
private val userHome = Paths.get(System.getProperty("user.home"))
|
|
||||||
private val pwd = Paths.get(System.getenv("PWD"))
|
|
||||||
|
|
||||||
def buildConfig(priorityOpts: ConfigOptions)
|
def buildConfig(priorityOpts: ConfigOptions)
|
||||||
: IO[Either[NonEmptyChain[ConfigValidation], Config]] = {
|
: IO[Either[NonEmptyChain[ConfigValidation], Config]] = {
|
||||||
val sources = ConfigQuery.sources(priorityOpts)
|
val sources = ConfigQuery.sources(priorityOpts)
|
||||||
for {
|
for {
|
||||||
sourceOptions <- SourceConfigLoader.loadSourceConfigs(sources)
|
sourceOpts <- SourceConfigLoader.loadSourceConfigs(sources)
|
||||||
userOptions <- userOptions(priorityOpts ++ sourceOptions)
|
userOpts <- userOptions(priorityOpts ++ sourceOpts)
|
||||||
globalOptions <- globalOptions(priorityOpts ++ sourceOptions ++ userOptions)
|
globalOpts <- globalOptions(priorityOpts ++ sourceOpts ++ userOpts)
|
||||||
collected = priorityOpts ++ sourceOptions ++ userOptions ++ globalOptions
|
collected = priorityOpts ++ sourceOpts ++ userOpts ++ globalOpts
|
||||||
config = collateOptions(collected)
|
config = collateOptions(collected)
|
||||||
} yield validateConfig(config).toEither
|
} yield validateConfig(config).toEither
|
||||||
}
|
}
|
||||||
|
@ -36,20 +35,18 @@ trait ConfigurationBuilder {
|
||||||
|
|
||||||
private def userOptions(priorityOpts: ConfigOptions) =
|
private def userOptions(priorityOpts: ConfigOptions) =
|
||||||
if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
|
if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
|
||||||
else readFile(userHome, userConfigFilename)
|
else parseFile(userHome.resolve(userConfigFilename))
|
||||||
|
|
||||||
private def globalOptions(priorityOpts: ConfigOptions) =
|
private def globalOptions(priorityOpts: ConfigOptions) =
|
||||||
if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig
|
if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig
|
||||||
else parseFile(globalConfig)
|
else parseFile(globalConfig)
|
||||||
|
|
||||||
private def readFile(
|
|
||||||
source: Path,
|
|
||||||
filename: String
|
|
||||||
) =
|
|
||||||
parseFile(source.resolve(filename))
|
|
||||||
|
|
||||||
private def collateOptions(configOptions: ConfigOptions): Config =
|
private def collateOptions(configOptions: ConfigOptions): Config =
|
||||||
configOptions.options.foldLeft(Config())((c, co) => co.update(c))
|
options
|
||||||
|
.get(configOptions)
|
||||||
|
.foldLeft(Config()) { (config, configOption) =>
|
||||||
|
configOption.update(config)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
package net.kemitix.thorp.core
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class Counters(
|
final case class Counters(
|
||||||
uploaded: Int = 0,
|
uploaded: Int = 0,
|
||||||
deleted: Int = 0,
|
deleted: Int = 0,
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
package net.kemitix.thorp.core
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
import net.kemitix.thorp.domain.LocalFile
|
import net.kemitix.thorp.domain.LocalFile
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class LocalFiles(
|
case class LocalFiles(
|
||||||
localFiles: Stream[LocalFile] = Stream(),
|
localFiles: Stream[LocalFile] = Stream(),
|
||||||
count: Long = 0,
|
count: Long = 0,
|
||||||
|
|
|
@ -12,7 +12,7 @@ trait SourceConfigLoader {
|
||||||
sources => {
|
sources => {
|
||||||
|
|
||||||
val sourceConfigOptions =
|
val sourceConfigOptions =
|
||||||
ConfigOptions(sources.paths.map(ConfigOption.Source))
|
ConfigOptions(sources.paths.map(ConfigOption.Source(_)))
|
||||||
|
|
||||||
val reduce: List[ConfigOptions] => ConfigOptions =
|
val reduce: List[ConfigOptions] => ConfigOptions =
|
||||||
_.foldLeft(sourceConfigOptions) { (acc, co) => acc ++ co }
|
_.foldLeft(sourceConfigOptions) { (acc, co) => acc ++ co }
|
||||||
|
|
|
@ -18,11 +18,14 @@ trait SyncLogging {
|
||||||
sources: Sources
|
sources: Sources
|
||||||
)(implicit logger: Logger): IO[Unit] = {
|
)(implicit logger: Logger): IO[Unit] = {
|
||||||
val sourcesList = sources.paths.mkString(", ")
|
val sourcesList = sources.paths.mkString(", ")
|
||||||
logger.info(s"Bucket: ${bucket.name}, Prefix: ${prefix.key}, Source: $sourcesList")
|
logger.info(
|
||||||
|
List(s"Bucket: ${bucket.name}",
|
||||||
|
s"Prefix: ${prefix.key}",
|
||||||
|
s"Source: $sourcesList")
|
||||||
|
.mkString(", "))
|
||||||
}
|
}
|
||||||
|
|
||||||
def logFileScan(implicit c: Config,
|
def logFileScan(implicit c: Config, logger: Logger): IO[Unit] =
|
||||||
logger: Logger): IO[Unit] =
|
|
||||||
logger.info(s"Scanning local files: ${c.sources.paths.mkString(", ")}...")
|
logger.info(s"Scanning local files: ${c.sources.paths.mkString(", ")}...")
|
||||||
|
|
||||||
def logRunFinished(
|
def logRunFinished(
|
||||||
|
@ -50,16 +53,14 @@ trait SyncLogging {
|
||||||
|
|
||||||
private def countActivities: (Counters, StorageQueueEvent) => Counters =
|
private def countActivities: (Counters, StorageQueueEvent) => Counters =
|
||||||
(counters: Counters, s3Action: StorageQueueEvent) => {
|
(counters: Counters, s3Action: StorageQueueEvent) => {
|
||||||
|
import Counters._
|
||||||
|
val increment: Int => Int = _ + 1
|
||||||
s3Action match {
|
s3Action match {
|
||||||
case _: UploadQueueEvent =>
|
case _: UploadQueueEvent => uploaded.modify(increment)(counters)
|
||||||
counters.copy(uploaded = counters.uploaded + 1)
|
case _: CopyQueueEvent => copied.modify(increment)(counters)
|
||||||
case _: CopyQueueEvent =>
|
case _: DeleteQueueEvent => deleted.modify(increment)(counters)
|
||||||
counters.copy(copied = counters.copied + 1)
|
case _: ErrorQueueEvent => errors.modify(increment)(counters)
|
||||||
case _: DeleteQueueEvent =>
|
case _ => counters
|
||||||
counters.copy(deleted = counters.deleted + 1)
|
|
||||||
case ErrorQueueEvent(_, _) =>
|
|
||||||
counters.copy(errors = counters.errors + 1)
|
|
||||||
case _ => counters
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
package net.kemitix.thorp.core
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
import net.kemitix.thorp.domain.SyncTotals
|
import net.kemitix.thorp.domain.SyncTotals
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class SyncPlan(
|
case class SyncPlan(
|
||||||
actions: Stream[Action] = Stream(),
|
actions: Stream[Action] = Stream(),
|
||||||
syncTotals: SyncTotals = SyncTotals()
|
syncTotals: SyncTotals = SyncTotals()
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package net.kemitix.thorp.core
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
|
import monocle.macros.Lenses
|
||||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
||||||
import net.kemitix.thorp.domain.StorageQueueEvent.DoNothingQueueEvent
|
import net.kemitix.thorp.domain.StorageQueueEvent.DoNothingQueueEvent
|
||||||
import net.kemitix.thorp.domain.{
|
import net.kemitix.thorp.domain.{
|
||||||
|
@ -13,6 +14,7 @@ import net.kemitix.thorp.domain.{
|
||||||
}
|
}
|
||||||
import net.kemitix.thorp.storage.api.StorageService
|
import net.kemitix.thorp.storage.api.StorageService
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class UnversionedMirrorArchive(
|
case class UnversionedMirrorArchive(
|
||||||
storageService: StorageService,
|
storageService: StorageService,
|
||||||
batchMode: Boolean,
|
batchMode: Boolean,
|
||||||
|
@ -52,7 +54,7 @@ case class UnversionedMirrorArchive(
|
||||||
localFile,
|
localFile,
|
||||||
bucket,
|
bucket,
|
||||||
batchMode,
|
batchMode,
|
||||||
new UploadEventListener(localFile, index, syncTotals, totalBytesSoFar),
|
UploadEventListener(localFile, index, syncTotals, totalBytesSoFar),
|
||||||
1)
|
1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class Bucket(
|
final case class Bucket(
|
||||||
name: String
|
name: String
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class Config(
|
final case class Config(
|
||||||
bucket: Bucket = Bucket(""),
|
bucket: Bucket = Bucket(""),
|
||||||
prefix: RemoteKey = RemoteKey(""),
|
prefix: RemoteKey = RemoteKey(""),
|
||||||
|
|
|
@ -3,6 +3,8 @@ package net.kemitix.thorp.domain
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
sealed trait Filter
|
sealed trait Filter
|
||||||
|
|
||||||
object Filter {
|
object Filter {
|
||||||
|
@ -30,6 +32,7 @@ object Filter {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class Include(
|
case class Include(
|
||||||
include: String = ".*"
|
include: String = ".*"
|
||||||
) extends Filter {
|
) extends Filter {
|
||||||
|
@ -40,6 +43,7 @@ object Filter {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class Exclude(
|
case class Exclude(
|
||||||
exclude: String
|
exclude: String
|
||||||
) extends Filter {
|
) extends Filter {
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class HashModified(
|
final case class HashModified(
|
||||||
hash: MD5Hash,
|
hash: MD5Hash,
|
||||||
modified: LastModified
|
modified: LastModified
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class KeyModified(
|
final case class KeyModified(
|
||||||
key: RemoteKey,
|
key: RemoteKey,
|
||||||
modified: LastModified
|
modified: LastModified
|
||||||
|
|
|
@ -2,6 +2,9 @@ package net.kemitix.thorp.domain
|
||||||
|
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class LastModified(
|
final case class LastModified(
|
||||||
when: Instant = Instant.now
|
when: Instant = Instant.now
|
||||||
)
|
)
|
||||||
|
|
|
@ -3,6 +3,9 @@ package net.kemitix.thorp.domain
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class LocalFile(
|
final case class LocalFile(
|
||||||
file: File,
|
file: File,
|
||||||
source: File,
|
source: File,
|
||||||
|
|
|
@ -4,6 +4,9 @@ import java.util.Base64
|
||||||
|
|
||||||
import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
|
import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class MD5Hash(
|
final case class MD5Hash(
|
||||||
in: String
|
in: String
|
||||||
) {
|
) {
|
||||||
|
|
|
@ -3,6 +3,9 @@ package net.kemitix.thorp.domain
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.nio.file.{Path, Paths}
|
import java.nio.file.{Path, Paths}
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class RemoteKey(
|
final case class RemoteKey(
|
||||||
key: String
|
key: String
|
||||||
) {
|
) {
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class RemoteMetaData(
|
final case class RemoteMetaData(
|
||||||
remoteKey: RemoteKey,
|
remoteKey: RemoteKey,
|
||||||
hash: MD5Hash,
|
hash: MD5Hash,
|
||||||
|
|
|
@ -1,6 +1,9 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
|
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
|
||||||
|
@Lenses
|
||||||
final case class S3MetaData(
|
final case class S3MetaData(
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
matchByHash: Set[RemoteMetaData],
|
matchByHash: Set[RemoteMetaData],
|
||||||
|
|
|
@ -1,8 +1,12 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A list of objects and their MD5 hash values.
|
* A list of objects and their MD5 hash values.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class S3ObjectsData(
|
final case class S3ObjectsData(
|
||||||
byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty,
|
byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty,
|
||||||
byKey: Map[RemoteKey, HashModified] = Map.empty
|
byKey: Map[RemoteKey, HashModified] = Map.empty
|
||||||
|
|
|
@ -2,6 +2,8 @@ package net.kemitix.thorp.domain
|
||||||
|
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The paths to synchronise with target.
|
* The paths to synchronise with target.
|
||||||
*
|
*
|
||||||
|
@ -12,12 +14,13 @@ import java.nio.file.Path
|
||||||
*
|
*
|
||||||
* A path should only occur once in paths.
|
* A path should only occur once in paths.
|
||||||
*/
|
*/
|
||||||
|
@Lenses
|
||||||
case class Sources(
|
case class Sources(
|
||||||
paths: List[Path]
|
paths: List[Path]
|
||||||
) {
|
) {
|
||||||
def ++(path: Path): Sources = this ++ List(path)
|
def ++(path: Path): Sources = this ++ List(path)
|
||||||
def ++(otherPaths: List[Path]): Sources = Sources(
|
def ++(otherPaths: List[Path]): Sources =
|
||||||
otherPaths.foldLeft(paths) { (acc, path) =>
|
Sources(otherPaths.foldLeft(paths) { (acc, path) =>
|
||||||
if (acc.contains(path)) acc
|
if (acc.contains(path)) acc
|
||||||
else acc ++ List(path)
|
else acc ++ List(path)
|
||||||
})
|
})
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
sealed trait StorageQueueEvent {
|
sealed trait StorageQueueEvent {
|
||||||
|
|
||||||
val order: Int
|
val order: Int
|
||||||
|
@ -8,18 +10,21 @@ sealed trait StorageQueueEvent {
|
||||||
|
|
||||||
object StorageQueueEvent {
|
object StorageQueueEvent {
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class DoNothingQueueEvent(
|
final case class DoNothingQueueEvent(
|
||||||
remoteKey: RemoteKey
|
remoteKey: RemoteKey
|
||||||
) extends StorageQueueEvent {
|
) extends StorageQueueEvent {
|
||||||
override val order: Int = 0
|
override val order: Int = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class CopyQueueEvent(
|
final case class CopyQueueEvent(
|
||||||
remoteKey: RemoteKey
|
remoteKey: RemoteKey
|
||||||
) extends StorageQueueEvent {
|
) extends StorageQueueEvent {
|
||||||
override val order: Int = 1
|
override val order: Int = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class UploadQueueEvent(
|
final case class UploadQueueEvent(
|
||||||
remoteKey: RemoteKey,
|
remoteKey: RemoteKey,
|
||||||
md5Hash: MD5Hash
|
md5Hash: MD5Hash
|
||||||
|
@ -27,12 +32,14 @@ object StorageQueueEvent {
|
||||||
override val order: Int = 2
|
override val order: Int = 2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class DeleteQueueEvent(
|
final case class DeleteQueueEvent(
|
||||||
remoteKey: RemoteKey
|
remoteKey: RemoteKey
|
||||||
) extends StorageQueueEvent {
|
) extends StorageQueueEvent {
|
||||||
override val order: Int = 3
|
override val order: Int = 3
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class ErrorQueueEvent(
|
final case class ErrorQueueEvent(
|
||||||
remoteKey: RemoteKey,
|
remoteKey: RemoteKey,
|
||||||
e: Throwable
|
e: Throwable
|
||||||
|
@ -40,6 +47,7 @@ object StorageQueueEvent {
|
||||||
override val order: Int = 10
|
override val order: Int = 10
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class ShutdownQueueEvent() extends StorageQueueEvent {
|
final case class ShutdownQueueEvent() extends StorageQueueEvent {
|
||||||
override val order: Int = 99
|
override val order: Int = 99
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
case class SyncTotals(
|
case class SyncTotals(
|
||||||
count: Long = 0L,
|
count: Long = 0L,
|
||||||
totalSizeBytes: Long = 0L,
|
totalSizeBytes: Long = 0L,
|
||||||
|
|
|
@ -1,21 +1,26 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
sealed trait UploadEvent {
|
sealed trait UploadEvent {
|
||||||
def name: String
|
def name: String
|
||||||
}
|
}
|
||||||
|
|
||||||
object UploadEvent {
|
object UploadEvent {
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class TransferEvent(
|
final case class TransferEvent(
|
||||||
name: String
|
name: String
|
||||||
) extends UploadEvent
|
) extends UploadEvent
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class RequestEvent(
|
final case class RequestEvent(
|
||||||
name: String,
|
name: String,
|
||||||
bytes: Long,
|
bytes: Long,
|
||||||
transferred: Long
|
transferred: Long
|
||||||
) extends UploadEvent
|
) extends UploadEvent
|
||||||
|
|
||||||
|
@Lenses
|
||||||
final case class ByteTransferEvent(
|
final case class ByteTransferEvent(
|
||||||
name: String
|
name: String
|
||||||
) extends UploadEvent
|
) extends UploadEvent
|
||||||
|
|
|
@ -3,7 +3,10 @@ package net.kemitix.thorp.domain
|
||||||
import net.kemitix.thorp.domain.UploadEvent.RequestEvent
|
import net.kemitix.thorp.domain.UploadEvent.RequestEvent
|
||||||
import net.kemitix.thorp.domain.UploadEventLogger.logRequestCycle
|
import net.kemitix.thorp.domain.UploadEventLogger.logRequestCycle
|
||||||
|
|
||||||
class UploadEventListener(
|
import monocle.macros.Lenses
|
||||||
|
|
||||||
|
@Lenses
|
||||||
|
case class UploadEventListener(
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
index: Int,
|
index: Int,
|
||||||
syncTotals: SyncTotals,
|
syncTotals: SyncTotals,
|
||||||
|
|
|
@ -137,7 +137,7 @@ class StorageServiceSuite extends FunSpec with MockFactory {
|
||||||
val bucket = Bucket("a-bucket")
|
val bucket = Bucket("a-bucket")
|
||||||
val remoteKey = RemoteKey("prefix/root-file")
|
val remoteKey = RemoteKey("prefix/root-file")
|
||||||
val uploadEventListener =
|
val uploadEventListener =
|
||||||
new UploadEventListener(localFile, 1, SyncTotals(), 0L)
|
UploadEventListener(localFile, 1, SyncTotals(), 0L)
|
||||||
|
|
||||||
val upload = stub[AmazonUpload]
|
val upload = stub[AmazonUpload]
|
||||||
(amazonTransferManager upload (_: PutObjectRequest))
|
(amazonTransferManager upload (_: PutObjectRequest))
|
||||||
|
|
|
@ -36,7 +36,7 @@ class UploaderSuite extends FunSpec with MockFactory {
|
||||||
sourcePath,
|
sourcePath,
|
||||||
fileToKey)
|
fileToKey)
|
||||||
val uploadEventListener =
|
val uploadEventListener =
|
||||||
new UploadEventListener(bigFile, 1, SyncTotals(), 0L)
|
UploadEventListener(bigFile, 1, SyncTotals(), 0L)
|
||||||
val amazonS3 = mock[AmazonS3]
|
val amazonS3 = mock[AmazonS3]
|
||||||
val amazonTransferManager =
|
val amazonTransferManager =
|
||||||
AmazonTransferManager(
|
AmazonTransferManager(
|
||||||
|
|
Loading…
Reference in a new issue