From ccefd286f91db59f16b67badc1aec549e43e6d6f Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sun, 28 Jul 2019 21:47:01 +0100 Subject: [PATCH] Convert Config to full ZIO effect module (#134) * [config] new module * [config] stub module * [domain] Rename domain.Config as domain.LegacyConfig * [config] Move LegacyConfig to config module * [config] Move config parsing and validation into module * [config] Complete migration to module for Config * [config] Config You should not name methods after their defining object * [config] Rename LegacyConfig as Configuration Also remove redundant uses * [core] LocalFileStream Refactoring * [changelog] update --- CHANGELOG.org | 1 + build.sbt | 11 +- .../scala/net/kemitix/thorp/cli/Main.scala | 3 +- .../scala/net/kemitix/thorp/cli/Program.scala | 20 +-- .../net/kemitix/thorp/config}/CliArgs.scala | 3 +- .../net/kemitix/thorp/config/Config.scala | 50 ++++++ .../kemitix/thorp/config}/ConfigOption.scala | 28 ++-- .../kemitix/thorp/config}/ConfigOptions.scala | 2 +- .../kemitix/thorp/config}/ConfigQuery.scala | 6 +- .../thorp/config}/ConfigValidation.scala | 2 +- .../config}/ConfigValidationException.scala | 2 +- .../thorp/config}/ConfigValidator.scala | 8 +- .../kemitix/thorp/config/Configuration.scala | 29 ++++ .../thorp/config}/ConfigurationBuilder.scala | 29 ++-- .../thorp/config}/ParseConfigFile.scala | 2 +- .../thorp/config}/ParseConfigLines.scala | 11 +- .../net/kemitix/thorp/config}/Resource.scala | 2 +- .../thorp/config}/SourceConfigLoader.scala | 2 +- .../net/kemitix/thorp/config/package.scala | 29 ++++ .../kemitix/thorp/config}/CliArgsTest.scala | 5 +- .../net/kemitix/thorp/console/package.scala | 2 +- .../kemitix/thorp/core/ActionGenerator.scala | 60 ++++--- .../net/kemitix/thorp/core/CoreTypes.scala | 3 +- .../kemitix/thorp/core/LocalFileStream.scala | 100 ++++++------ .../net/kemitix/thorp/core/LocalFiles.scala | 7 + .../net/kemitix/thorp/core/PlanBuilder.scala | 148 ++++++++---------- .../thorp/core/S3MetaDataEnricher.scala | 2 +- .../net/kemitix/thorp/core/SyncLogging.scala | 19 ++- .../thorp/core/ActionGeneratorSuite.scala | 82 +++++++--- .../kemitix/thorp/core/ConfigOptionTest.scala | 6 + .../kemitix/thorp/core/ConfigQueryTest.scala | 31 ++-- .../thorp/core/ConfigurationBuilderTest.scala | 5 + .../thorp/core/KeyGeneratorSuite.scala | 8 +- .../thorp/core/LocalFileStreamSuite.scala | 41 ++++- .../thorp/core/MD5HashGeneratorTest.scala | 8 +- .../thorp/core/ParseConfigFileTest.scala | 6 + .../thorp/core/ParseConfigLinesTest.scala | 1 + .../kemitix/thorp/core/PlanBuilderTest.scala | 24 +-- .../thorp/core/S3MetaDataEnricherSuite.scala | 6 +- .../net/kemitix/thorp/domain/Config.scala | 25 --- .../thorp/storage/aws/ETagGeneratorTest.scala | 2 +- .../storage/aws/StorageServiceSuite.scala | 11 +- .../thorp/storage/aws/UploaderTest.scala | 2 +- 43 files changed, 517 insertions(+), 327 deletions(-) rename {cli/src/main/scala/net/kemitix/thorp/cli => config/src/main/scala/net/kemitix/thorp/config}/CliArgs.scala (95%) create mode 100644 config/src/main/scala/net/kemitix/thorp/config/Config.scala rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/ConfigOption.scala (58%) rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/ConfigOptions.scala (95%) rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/ConfigQuery.scala (88%) rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/ConfigValidation.scala (95%) rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/ConfigValidationException.scala (75%) rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/ConfigValidator.scala (89%) create mode 100644 config/src/main/scala/net/kemitix/thorp/config/Configuration.scala rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/ConfigurationBuilder.scala (72%) rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/ParseConfigFile.scala (96%) rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/ParseConfigLines.scala (89%) rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/Resource.scala (89%) rename {core/src/main/scala/net/kemitix/thorp/core => config/src/main/scala/net/kemitix/thorp/config}/SourceConfigLoader.scala (95%) create mode 100644 config/src/main/scala/net/kemitix/thorp/config/package.scala rename {cli/src/test/scala/net/kemitix/thorp/cli => config/src/test/scala/net/kemitix/thorp/config}/CliArgsTest.scala (93%) delete mode 100644 domain/src/main/scala/net/kemitix/thorp/domain/Config.scala diff --git a/CHANGELOG.org b/CHANGELOG.org index c3556ae..34d57c9 100644 --- a/CHANGELOG.org +++ b/CHANGELOG.org @@ -20,6 +20,7 @@ The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog] - [internal] Replace Monocle with local SimpleLens implementation (#121) - [internal] Don't use String as key in Map for hashes (#124) - [internal] Convert Storage to full ZIO effect module (#133) + - [internal] Convert Config to full ZIO effect module (#134) ** Dependencies diff --git a/build.sbt b/build.sbt index bc493c7..8d94dba 100644 --- a/build.sbt +++ b/build.sbt @@ -58,6 +58,7 @@ val zioDependencies = Seq( ) // cli -> thorp-lib -> storage-aws -> core -> storage-api -> console -> domain +// core -> config -> domain lazy val thorp = (project in file(".")) .settings(commonSettings) @@ -67,7 +68,6 @@ lazy val cli = (project in file("cli")) .settings(commonSettings) .settings(mainClass in assembly := Some("net.kemitix.thorp.cli.Main")) .settings(applicationSettings) - .settings(commandLineParsing) .settings(testDependencies) .enablePlugins(BuildInfoPlugin) .settings( @@ -101,6 +101,7 @@ lazy val core = (project in file("core")) .settings(testDependencies) .dependsOn(`storage-api`) .dependsOn(domain % "compile->compile;test->test") + .dependsOn(config) lazy val `storage-api` = (project in file("storage-api")) .settings(commonSettings) @@ -114,6 +115,14 @@ lazy val console = (project in file("console")) .settings(assemblyJarName in assembly := "console.jar") .dependsOn(domain) +lazy val config = (project in file("config")) + .settings(commonSettings) + .settings(zioDependencies) + .settings(testDependencies) + .settings(commandLineParsing) + .settings(assemblyJarName in assembly := "config.jar") + .dependsOn(domain) + lazy val domain = (project in file("domain")) .settings(commonSettings) .settings(assemblyJarName in assembly := "domain.jar") diff --git a/cli/src/main/scala/net/kemitix/thorp/cli/Main.scala b/cli/src/main/scala/net/kemitix/thorp/cli/Main.scala index 1a36fbd..d87fa9b 100644 --- a/cli/src/main/scala/net/kemitix/thorp/cli/Main.scala +++ b/cli/src/main/scala/net/kemitix/thorp/cli/Main.scala @@ -1,12 +1,13 @@ package net.kemitix.thorp.cli +import net.kemitix.thorp.config.Config import net.kemitix.thorp.console.Console import net.kemitix.thorp.storage.aws.S3Storage import zio.{App, ZIO} object Main extends App { - object LiveThorpApp extends S3Storage.Live with Console.Live + object LiveThorpApp extends S3Storage.Live with Console.Live with Config.Live override def run(args: List[String]): ZIO[Environment, Nothing, Int] = Program diff --git a/cli/src/main/scala/net/kemitix/thorp/cli/Program.scala b/cli/src/main/scala/net/kemitix/thorp/cli/Program.scala index cb05e74..0c67d08 100644 --- a/cli/src/main/scala/net/kemitix/thorp/cli/Program.scala +++ b/cli/src/main/scala/net/kemitix/thorp/cli/Program.scala @@ -1,11 +1,12 @@ package net.kemitix.thorp.cli +import net.kemitix.thorp.config._ import net.kemitix.thorp.console._ import net.kemitix.thorp.core.CoreTypes.CoreProgram import net.kemitix.thorp.core._ import net.kemitix.thorp.domain.StorageQueueEvent import net.kemitix.thorp.storage.aws.S3HashService.defaultHashService -import zio.{UIO, ZIO} +import zio.ZIO trait Program { @@ -13,28 +14,27 @@ trait Program { def run(args: List[String]): CoreProgram[Unit] = { for { - cli <- CliArgs.parse(args) - _ <- ZIO.when(showVersion(cli))(putStrLn(version)) - _ <- ZIO.when(!showVersion(cli))(execute(cli).catchAll(handleErrors)) + cli <- CliArgs.parse(args) + config <- ConfigurationBuilder.buildConfig(cli) + _ <- setConfiguration(config) + _ <- ZIO.when(showVersion(cli))(putStrLn(version)) + _ <- ZIO.when(!showVersion(cli))(execute.catchAll(handleErrors)) } yield () } private def showVersion: ConfigOptions => Boolean = cli => ConfigQuery.showVersion(cli) - private def execute(cliOptions: ConfigOptions) = { + private def execute = { for { - plan <- PlanBuilder.createPlan(defaultHashService, cliOptions) - batchMode <- isBatchMode(cliOptions) + plan <- PlanBuilder.createPlan(defaultHashService) + batchMode <- isBatchMode archive <- UnversionedMirrorArchive.default(batchMode, plan.syncTotals) events <- applyPlan(archive, plan) _ <- SyncLogging.logRunFinished(events) } yield () } - private def isBatchMode(cliOptions: ConfigOptions) = - UIO(ConfigQuery.batchMode(cliOptions)) - private def handleErrors(throwable: Throwable) = for { _ <- putStrLn("There were errors:") diff --git a/cli/src/main/scala/net/kemitix/thorp/cli/CliArgs.scala b/config/src/main/scala/net/kemitix/thorp/config/CliArgs.scala similarity index 95% rename from cli/src/main/scala/net/kemitix/thorp/cli/CliArgs.scala rename to config/src/main/scala/net/kemitix/thorp/config/CliArgs.scala index 26901db..0bd8028 100644 --- a/cli/src/main/scala/net/kemitix/thorp/cli/CliArgs.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/CliArgs.scala @@ -1,8 +1,7 @@ -package net.kemitix.thorp.cli +package net.kemitix.thorp.config import java.nio.file.Paths -import net.kemitix.thorp.core.{ConfigOption, ConfigOptions} import scopt.OParser import zio.Task diff --git a/config/src/main/scala/net/kemitix/thorp/config/Config.scala b/config/src/main/scala/net/kemitix/thorp/config/Config.scala new file mode 100644 index 0000000..19f7722 --- /dev/null +++ b/config/src/main/scala/net/kemitix/thorp/config/Config.scala @@ -0,0 +1,50 @@ +package net.kemitix.thorp.config + +import java.util.concurrent.atomic.AtomicReference + +import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, Sources} +import zio.{UIO, ZIO} + +trait Config { + val config: Config.Service +} + +object Config { + + trait Service { + def setConfiguration(config: Configuration): ZIO[Config, Nothing, Unit] + def isBatchMode: ZIO[Config, Nothing, Boolean] + def bucket: ZIO[Config, Nothing, Bucket] + def prefix: ZIO[Config, Nothing, RemoteKey] + def sources: ZIO[Config, Nothing, Sources] + def filters: ZIO[Config, Nothing, List[Filter]] + } + + trait Live extends Config { + + val config: Service = new Service { + private val configRef = new AtomicReference(Configuration()) + override def setConfiguration( + config: Configuration): ZIO[Config, Nothing, Unit] = + UIO(configRef.set(config)) + + override def bucket: ZIO[Config, Nothing, Bucket] = + UIO(configRef.get).map(_.bucket) + + override def sources: ZIO[Config, Nothing, Sources] = + UIO(configRef.get).map(_.sources) + + override def prefix: ZIO[Config, Nothing, RemoteKey] = + UIO(configRef.get).map(_.prefix) + + override def isBatchMode: ZIO[Config, Nothing, Boolean] = + UIO(configRef.get).map(_.batchMode) + + override def filters: ZIO[Config, Nothing, List[Filter]] = + UIO(configRef.get).map(_.filters) + } + } + + object Live extends Live + +} diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala b/config/src/main/scala/net/kemitix/thorp/config/ConfigOption.scala similarity index 58% rename from core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala rename to config/src/main/scala/net/kemitix/thorp/config/ConfigOption.scala index c34174b..04e9b34 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ConfigOption.scala @@ -1,24 +1,24 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import java.nio.file.Path +import net.kemitix.thorp.config.Configuration._ import net.kemitix.thorp.domain -import net.kemitix.thorp.domain.{Config, RemoteKey} -import net.kemitix.thorp.domain.Config._ +import net.kemitix.thorp.domain.RemoteKey sealed trait ConfigOption { - def update(config: Config): Config + def update(config: Configuration): Configuration } object ConfigOption { case class Source(path: Path) extends ConfigOption { - override def update(config: Config): Config = + override def update(config: Configuration): Configuration = sources.modify(_ ++ path)(config) } case class Bucket(name: String) extends ConfigOption { - override def update(config: Config): Config = + override def update(config: Configuration): Configuration = if (config.bucket.name.isEmpty) bucket.set(domain.Bucket(name))(config) else @@ -26,7 +26,7 @@ object ConfigOption { } case class Prefix(path: String) extends ConfigOption { - override def update(config: Config): Config = + override def update(config: Configuration): Configuration = if (config.prefix.key.isEmpty) prefix.set(RemoteKey(path))(config) else @@ -34,35 +34,35 @@ object ConfigOption { } case class Include(pattern: String) extends ConfigOption { - override def update(config: Config): Config = + override def update(config: Configuration): Configuration = filters.modify(domain.Filter.Include(pattern) :: _)(config) } case class Exclude(pattern: String) extends ConfigOption { - override def update(config: Config): Config = + override def update(config: Configuration): Configuration = filters.modify(domain.Filter.Exclude(pattern) :: _)(config) } case class Debug() extends ConfigOption { - override def update(config: Config): Config = + override def update(config: Configuration): Configuration = debug.set(true)(config) } case object Version extends ConfigOption { - override def update(config: Config): Config = config + override def update(config: Configuration): Configuration = config } case object BatchMode extends ConfigOption { - override def update(config: Config): Config = + override def update(config: Configuration): Configuration = batchMode.set(true)(config) } case object IgnoreUserOptions extends ConfigOption { - override def update(config: Config): Config = config + override def update(config: Configuration): Configuration = config } case object IgnoreGlobalOptions extends ConfigOption { - override def update(config: Config): Config = config + override def update(config: Configuration): Configuration = config } } diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala b/config/src/main/scala/net/kemitix/thorp/config/ConfigOptions.scala similarity index 95% rename from core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala rename to config/src/main/scala/net/kemitix/thorp/config/ConfigOptions.scala index 9f0662b..858bcf1 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ConfigOptions.scala @@ -1,4 +1,4 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import net.kemitix.thorp.domain.SimpleLens diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigQuery.scala b/config/src/main/scala/net/kemitix/thorp/config/ConfigQuery.scala similarity index 88% rename from core/src/main/scala/net/kemitix/thorp/core/ConfigQuery.scala rename to config/src/main/scala/net/kemitix/thorp/config/ConfigQuery.scala index 55abfa6..7d38c73 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigQuery.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ConfigQuery.scala @@ -1,4 +1,4 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import java.nio.file.Paths @@ -21,11 +21,11 @@ trait ConfigQuery { def sources(configOptions: ConfigOptions): Sources = { val paths = configOptions.options.flatMap { case ConfigOption.Source(sourcePath) => Some(sourcePath) - case _ => None + case _ => None } Sources(paths match { case List() => List(Paths.get(System.getenv("PWD"))) - case _ => paths + case _ => paths }) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigValidation.scala b/config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala similarity index 95% rename from core/src/main/scala/net/kemitix/thorp/core/ConfigValidation.scala rename to config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala index 10a3c73..23ab79c 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigValidation.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala @@ -1,4 +1,4 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import java.nio.file.Path diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigValidationException.scala b/config/src/main/scala/net/kemitix/thorp/config/ConfigValidationException.scala similarity index 75% rename from core/src/main/scala/net/kemitix/thorp/core/ConfigValidationException.scala rename to config/src/main/scala/net/kemitix/thorp/config/ConfigValidationException.scala index 37e4e20..b78e35f 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigValidationException.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ConfigValidationException.scala @@ -1,4 +1,4 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config final case class ConfigValidationException( errors: List[ConfigValidation] diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigValidator.scala b/config/src/main/scala/net/kemitix/thorp/config/ConfigValidator.scala similarity index 89% rename from core/src/main/scala/net/kemitix/thorp/core/ConfigValidator.scala rename to config/src/main/scala/net/kemitix/thorp/config/ConfigValidator.scala index 0fdb2ce..605c82e 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigValidator.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ConfigValidator.scala @@ -1,15 +1,15 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import java.nio.file.Path -import net.kemitix.thorp.domain.{Bucket, Config, Sources} +import net.kemitix.thorp.domain.{Bucket, Sources} import zio.IO sealed trait ConfigValidator { def validateConfig( - config: Config - ): IO[List[ConfigValidation], Config] = IO.fromEither { + config: Configuration + ): IO[List[ConfigValidation], Configuration] = IO.fromEither { for { _ <- validateSources(config.sources) _ <- validateBucket(config.bucket) diff --git a/config/src/main/scala/net/kemitix/thorp/config/Configuration.scala b/config/src/main/scala/net/kemitix/thorp/config/Configuration.scala new file mode 100644 index 0000000..118cdb8 --- /dev/null +++ b/config/src/main/scala/net/kemitix/thorp/config/Configuration.scala @@ -0,0 +1,29 @@ +package net.kemitix.thorp.config + +import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, SimpleLens, Sources} + +private[config] final case class Configuration( + bucket: Bucket = Bucket(""), + prefix: RemoteKey = RemoteKey(""), + filters: List[Filter] = List(), + debug: Boolean = false, + batchMode: Boolean = false, + sources: Sources = Sources(List()) +) + +private[config] object Configuration { + val sources: SimpleLens[Configuration, Sources] = + SimpleLens[Configuration, Sources](_.sources, b => a => b.copy(sources = a)) + val bucket: SimpleLens[Configuration, Bucket] = + SimpleLens[Configuration, Bucket](_.bucket, b => a => b.copy(bucket = a)) + val prefix: SimpleLens[Configuration, RemoteKey] = + SimpleLens[Configuration, RemoteKey](_.prefix, b => a => b.copy(prefix = a)) + val filters: SimpleLens[Configuration, List[Filter]] = + SimpleLens[Configuration, List[Filter]](_.filters, + b => a => b.copy(filters = a)) + val debug: SimpleLens[Configuration, Boolean] = + SimpleLens[Configuration, Boolean](_.debug, b => a => b.copy(debug = a)) + val batchMode: SimpleLens[Configuration, Boolean] = + SimpleLens[Configuration, Boolean](_.batchMode, + b => a => b.copy(batchMode = a)) +} diff --git a/core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala b/config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala similarity index 72% rename from core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala rename to config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala index df50081..2fe5563 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala @@ -1,12 +1,8 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import java.nio.file.Paths -import net.kemitix.thorp.core.ConfigOptions.options -import net.kemitix.thorp.core.ConfigValidator.validateConfig -import net.kemitix.thorp.core.ParseConfigFile.parseFile -import net.kemitix.thorp.domain.Config -import zio.IO +import zio.{IO, TaskR} /** * Builds a configuration from settings in a file within the @@ -18,12 +14,13 @@ trait ConfigurationBuilder { private val globalConfig = Paths.get("/etc/thorp.conf") private val userHome = Paths.get(System.getProperty("user.home")) - def buildConfig( - priorityOpts: ConfigOptions): IO[List[ConfigValidation], Config] = - for { + def buildConfig(priorityOpts: ConfigOptions) + : IO[ConfigValidationException, Configuration] = + (for { config <- getConfigOptions(priorityOpts).map(collateOptions) - valid <- validateConfig(config) - } yield valid + valid <- ConfigValidator.validateConfig(config) + } yield valid) + .catchAll(errors => TaskR.fail(ConfigValidationException(errors))) private def getConfigOptions( priorityOpts: ConfigOptions): IO[List[ConfigValidation], ConfigOptions] = @@ -39,17 +36,17 @@ trait ConfigurationBuilder { private def userOptions( priorityOpts: ConfigOptions): IO[List[ConfigValidation], ConfigOptions] = if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig - else parseFile(userHome.resolve(userConfigFilename)) + else ParseConfigFile.parseFile(userHome.resolve(userConfigFilename)) private def globalOptions( priorityOpts: ConfigOptions): IO[List[ConfigValidation], ConfigOptions] = if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig - else parseFile(globalConfig) + else ParseConfigFile.parseFile(globalConfig) - private def collateOptions(configOptions: ConfigOptions): Config = - options + private def collateOptions(configOptions: ConfigOptions): Configuration = + ConfigOptions.options .get(configOptions) - .foldLeft(Config()) { (config, configOption) => + .foldLeft(Configuration()) { (config, configOption) => configOption.update(config) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/ParseConfigFile.scala b/config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala similarity index 96% rename from core/src/main/scala/net/kemitix/thorp/core/ParseConfigFile.scala rename to config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala index 2d4bd07..eae3b28 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ParseConfigFile.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala @@ -1,4 +1,4 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import java.nio.file.{Files, Path} diff --git a/core/src/main/scala/net/kemitix/thorp/core/ParseConfigLines.scala b/config/src/main/scala/net/kemitix/thorp/config/ParseConfigLines.scala similarity index 89% rename from core/src/main/scala/net/kemitix/thorp/core/ParseConfigLines.scala rename to config/src/main/scala/net/kemitix/thorp/config/ParseConfigLines.scala index 80c0c1c..c7a1c84 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ParseConfigLines.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ParseConfigLines.scala @@ -1,9 +1,16 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import java.nio.file.Paths import java.util.regex.Pattern -import net.kemitix.thorp.core.ConfigOption._ +import net.kemitix.thorp.config.ConfigOption.{ + Bucket, + Debug, + Exclude, + Include, + Prefix, + Source +} trait ParseConfigLines { diff --git a/core/src/main/scala/net/kemitix/thorp/core/Resource.scala b/config/src/main/scala/net/kemitix/thorp/config/Resource.scala similarity index 89% rename from core/src/main/scala/net/kemitix/thorp/core/Resource.scala rename to config/src/main/scala/net/kemitix/thorp/config/Resource.scala index 77f75b3..b1b9c7c 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/Resource.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/Resource.scala @@ -1,4 +1,4 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import java.io.{File, FileNotFoundException} diff --git a/core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala b/config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala similarity index 95% rename from core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala rename to config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala index 2669802..9dd401b 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala @@ -1,4 +1,4 @@ -package net.kemitix.thorp.core +package net.kemitix.thorp.config import net.kemitix.thorp.domain.Sources import zio.IO diff --git a/config/src/main/scala/net/kemitix/thorp/config/package.scala b/config/src/main/scala/net/kemitix/thorp/config/package.scala new file mode 100644 index 0000000..ca38a9c --- /dev/null +++ b/config/src/main/scala/net/kemitix/thorp/config/package.scala @@ -0,0 +1,29 @@ +package net.kemitix.thorp + +import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, Sources} +import zio.ZIO + +package object config { + + final val configService: ZIO[Config, Nothing, Config.Service] = + ZIO.access(_.config) + + final def setConfiguration( + config: Configuration): ZIO[Config, Nothing, Unit] = + ZIO.accessM(_.config setConfiguration config) + + final def isBatchMode: ZIO[Config, Nothing, Boolean] = + ZIO.accessM(_.config isBatchMode) + + final def getBucket: ZIO[Config, Nothing, Bucket] = + ZIO.accessM(_.config bucket) + + final def getPrefix: ZIO[Config, Nothing, RemoteKey] = + ZIO.accessM(_.config prefix) + + final def getSources: ZIO[Config, Nothing, Sources] = + ZIO.accessM(_.config sources) + + final def getFilters: ZIO[Config, Nothing, List[Filter]] = + ZIO.accessM(_.config filters) +} diff --git a/cli/src/test/scala/net/kemitix/thorp/cli/CliArgsTest.scala b/config/src/test/scala/net/kemitix/thorp/config/CliArgsTest.scala similarity index 93% rename from cli/src/test/scala/net/kemitix/thorp/cli/CliArgsTest.scala rename to config/src/test/scala/net/kemitix/thorp/config/CliArgsTest.scala index 1596b6c..a45aacd 100644 --- a/cli/src/test/scala/net/kemitix/thorp/cli/CliArgsTest.scala +++ b/config/src/test/scala/net/kemitix/thorp/config/CliArgsTest.scala @@ -1,9 +1,8 @@ -package net.kemitix.thorp.cli +package net.kemitix.thorp.config import java.nio.file.Paths -import net.kemitix.thorp.core.ConfigOption.Debug -import net.kemitix.thorp.core.{ConfigOptions, ConfigQuery, Resource} +import net.kemitix.thorp.config.ConfigOption.Debug import org.scalatest.FunSpec import zio.DefaultRuntime diff --git a/console/src/main/scala/net/kemitix/thorp/console/package.scala b/console/src/main/scala/net/kemitix/thorp/console/package.scala index b96d052..722201a 100644 --- a/console/src/main/scala/net/kemitix/thorp/console/package.scala +++ b/console/src/main/scala/net/kemitix/thorp/console/package.scala @@ -10,7 +10,7 @@ package object console { final def putStrLn(line: String): ZIO[Console, Nothing, Unit] = ZIO.accessM(_.console putStrLn line) - final def putStrLn(line: ConsoleOut): ZIO[Console, Nothing, Unit] = + final def putMessageLn(line: ConsoleOut): ZIO[Console, Nothing, Unit] = ZIO.accessM(_.console putStrLn line) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala b/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala index ee5cf72..c885139 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala @@ -1,39 +1,47 @@ package net.kemitix.thorp.core +import net.kemitix.thorp.config._ import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload} import net.kemitix.thorp.domain._ +import zio.ZIO object ActionGenerator { def createActions( s3MetaData: S3MetaData, previousActions: Stream[Action] - )(implicit c: Config): Stream[Action] = - s3MetaData match { - // #1 local exists, remote exists, remote matches - do nothing - case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _))) - if localFile.matches(hash) => - doNothing(c.bucket, key) - // #2 local exists, remote is missing, other matches - copy - case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty => - copyFile(c.bucket, localFile, matchByHash) - // #3 local exists, remote is missing, other no matches - upload - case S3MetaData(localFile, matchByHash, None) - if matchByHash.isEmpty && - isUploadAlreadyQueued(previousActions)(localFile) => - uploadFile(c.bucket, localFile) - // #4 local exists, remote exists, remote no match, other matches - copy - case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _))) - if !localFile.matches(hash) && - matchByHash.nonEmpty => - copyFile(c.bucket, localFile, matchByHash) - // #5 local exists, remote exists, remote no match, other no matches - upload - case S3MetaData(localFile, matchByHash, Some(_)) if matchByHash.isEmpty => - uploadFile(c.bucket, localFile) - // fallback - case S3MetaData(localFile, _, _) => - doNothing(c.bucket, localFile.remoteKey) - } + ): ZIO[Config, Nothing, Stream[Action]] = + for { + bucket <- getBucket + } yield + s3MetaData match { + // #1 local exists, remote exists, remote matches - do nothing + case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _))) + if localFile.matches(hash) => + doNothing(bucket, key) + // #2 local exists, remote is missing, other matches - copy + case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty => + copyFile(bucket, localFile, matchByHash) + // #3 local exists, remote is missing, other no matches - upload + case S3MetaData(localFile, matchByHash, None) + if matchByHash.isEmpty && + isUploadAlreadyQueued(previousActions)(localFile) => + uploadFile(bucket, localFile) + // #4 local exists, remote exists, remote no match, other matches - copy + case S3MetaData(localFile, + matchByHash, + Some(RemoteMetaData(_, hash, _))) + if !localFile.matches(hash) && + matchByHash.nonEmpty => + copyFile(bucket, localFile, matchByHash) + // #5 local exists, remote exists, remote no match, other no matches - upload + case S3MetaData(localFile, matchByHash, Some(_)) + if matchByHash.isEmpty => + uploadFile(bucket, localFile) + // fallback + case S3MetaData(localFile, _, _) => + doNothing(bucket, localFile.remoteKey) + } private def key = LocalFile.remoteKey ^|-> RemoteKey.key diff --git a/core/src/main/scala/net/kemitix/thorp/core/CoreTypes.scala b/core/src/main/scala/net/kemitix/thorp/core/CoreTypes.scala index b60da04..3c82fe9 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/CoreTypes.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/CoreTypes.scala @@ -1,12 +1,13 @@ package net.kemitix.thorp.core +import net.kemitix.thorp.config.Config import net.kemitix.thorp.console.Console import net.kemitix.thorp.storage.api.Storage import zio.ZIO object CoreTypes { - type CoreEnv = Storage with Console + type CoreEnv = Storage with Console with Config type CoreProgram[A] = ZIO[CoreEnv, Throwable, A] } diff --git a/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala b/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala index 42ceca3..7a54941 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala @@ -2,42 +2,30 @@ package net.kemitix.thorp.core import java.nio.file.Path +import net.kemitix.thorp.config._ import net.kemitix.thorp.core.KeyGenerator.generateKey -import net.kemitix.thorp.domain import net.kemitix.thorp.domain._ import net.kemitix.thorp.storage.api.HashService -import zio.Task +import zio.{Task, TaskR, ZIO} object LocalFileStream { - def findFiles( - source: Path, - hashService: HashService - )( - implicit c: Config - ): Task[LocalFiles] = { + def findFiles(hashService: HashService)( + source: Path + ): TaskR[Config, LocalFiles] = { - val isIncluded: Path => Boolean = Filter.isIncluded(c.filters) + def recurseIntoSubDirectories(path: Path): TaskR[Config, LocalFiles] = + path.toFile match { + case f if f.isDirectory => loop(path) + case _ => pathToLocalFile(hashService)(path) + } - val pathToLocalFile: Path => Task[LocalFiles] = path => - localFile(hashService, c)(path) + def recurse(paths: Stream[Path]): TaskR[Config, LocalFiles] = + for { + recursed <- ZIO.foreach(paths)(path => recurseIntoSubDirectories(path)) + } yield LocalFiles.reduce(recursed.toStream) - def loop(path: Path): Task[LocalFiles] = { - - def dirPaths(path: Path): Task[Stream[Path]] = - listFiles(path) - .map(_.filter(isIncluded)) - - def recurseIntoSubDirectories(path: Path): Task[LocalFiles] = - path.toFile match { - case f if f.isDirectory => loop(path) - case _ => pathToLocalFile(path) - } - - def recurse(paths: Stream[Path]): Task[LocalFiles] = - Task.foldLeft(paths)(LocalFiles())((acc, path) => { - recurseIntoSubDirectories(path).map(localFiles => acc ++ localFiles) - }) + def loop(path: Path): TaskR[Config, LocalFiles] = { for { paths <- dirPaths(path) @@ -48,30 +36,50 @@ object LocalFileStream { loop(source) } - def localFile( - hashService: HashService, - c: Config - ): Path => Task[LocalFiles] = - path => { - val file = path.toFile - val source = c.sources.forPath(path) - for { - hash <- hashService.hashLocalObject(path) - } yield - LocalFiles(localFiles = Stream( - domain.LocalFile(file, - source.toFile, - hash, - generateKey(c.sources, c.prefix)(path))), - count = 1, - totalSizeBytes = file.length) - } + private def dirPaths(path: Path) = + for { + paths <- listFiles(path) + filtered <- includedDirPaths(paths) + } yield filtered - private def listFiles(path: Path): Task[Stream[Path]] = + private def includedDirPaths(paths: Stream[Path]) = + for { + flaggedPaths <- TaskR.foreach(paths)(path => + isIncluded(path).map((path, _))) + } yield + flaggedPaths.toStream + .filter({ case (_, included) => included }) + .map({ case (path, _) => path }) + + private def localFile(hashService: HashService)(path: Path) = { + val file = path.toFile + for { + sources <- getSources + prefix <- getPrefix + hash <- hashService.hashLocalObject(path) + localFile = LocalFile(file, + sources.forPath(path).toFile, + hash, + generateKey(sources, prefix)(path)) + } yield + LocalFiles(localFiles = Stream(localFile), + count = 1, + totalSizeBytes = file.length) + } + + private def listFiles(path: Path) = for { files <- Task(path.toFile.listFiles) _ <- Task.when(files == null)( Task.fail(new IllegalArgumentException(s"Directory not found $path"))) } yield Stream(files: _*).map(_.toPath) + private def isIncluded(path: Path) = + for { + filters <- getFilters + } yield Filter.isIncluded(filters)(path) + + private def pathToLocalFile(hashService: HashService)(path: Path) = + localFile(hashService)(path) + } diff --git a/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala b/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala index 451e47a..c70f0f7 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala @@ -15,3 +15,10 @@ case class LocalFiles( ) } + +object LocalFiles { + + def reduce: Stream[LocalFiles] => LocalFiles = + list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf) + +} diff --git a/core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala b/core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala index 91d8271..43ec6cf 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala @@ -1,121 +1,111 @@ package net.kemitix.thorp.core +import net.kemitix.thorp.config._ import net.kemitix.thorp.console._ import net.kemitix.thorp.core.Action._ import net.kemitix.thorp.domain._ import net.kemitix.thorp.storage._ import net.kemitix.thorp.storage.api.{HashService, Storage} -import zio.{Task, TaskR} +import zio.{TaskR, ZIO} trait PlanBuilder { - def createPlan( - hashService: HashService, - configOptions: ConfigOptions - ): TaskR[Storage with Console, SyncPlan] = - ConfigurationBuilder - .buildConfig(configOptions) - .catchAll(errors => TaskR.fail(ConfigValidationException(errors))) - .flatMap(config => useValidConfig(hashService)(config)) - - private def useValidConfig( - hashService: HashService - )(implicit c: Config) = { + def createPlan(hashService: HashService) + : TaskR[Storage with Console with Config, SyncPlan] = for { - _ <- SyncLogging.logRunStart(c.bucket, c.prefix, c.sources) + _ <- SyncLogging.logRunStart actions <- buildPlan(hashService) } yield actions - } - private def buildPlan( - hashService: HashService - )(implicit c: Config) = + private def buildPlan(hashService: HashService) = for { metadata <- gatherMetadata(hashService) - } yield assemblePlan(c)(metadata) + plan <- assemblePlan(metadata) + } yield plan - private def assemblePlan( - implicit c: Config): ((S3ObjectsData, LocalFiles)) => SyncPlan = { - case (remoteData, localData) => - SyncPlan( - actions = createActions(c)(remoteData)(localData) - .filter(doesSomething) - .sortBy(SequencePlan.order), - syncTotals = SyncTotals(count = localData.count, - totalSizeBytes = localData.totalSizeBytes) - ) - } + private def assemblePlan(metadata: (S3ObjectsData, LocalFiles)) = + metadata match { + case (remoteData, localData) => + createActions(remoteData, localData) + .map(_.filter(doesSomething).sortBy(SequencePlan.order)) + .map( + SyncPlan(_, SyncTotals(localData.count, localData.totalSizeBytes))) + } - private def createActions - : Config => S3ObjectsData => LocalFiles => Stream[Action] = - c => - remoteData => - localData => - actionsForLocalFiles(c)(remoteData)(localData) ++ - actionsForRemoteKeys(c)(remoteData) + private def createActions( + remoteData: S3ObjectsData, + localData: LocalFiles + ) = + for { + fileActions <- actionsForLocalFiles(remoteData, localData) + remoteActions <- actionsForRemoteKeys(remoteData) + } yield fileActions ++ remoteActions private def doesSomething: Action => Boolean = { case _: DoNothing => false case _ => true } - private def actionsForLocalFiles - : Config => S3ObjectsData => LocalFiles => Stream[Action] = - c => - remoteData => - localData => - localData.localFiles.foldLeft(Stream.empty[Action])((acc, lf) => - createActionFromLocalFile(c)(lf)(remoteData)(acc) ++ acc) + private def actionsForLocalFiles( + remoteData: S3ObjectsData, + localData: LocalFiles + ) = + ZIO.foldLeft(localData.localFiles)(Stream.empty[Action])( + (acc, localFile) => + createActionFromLocalFile(remoteData, acc, localFile) + .map(actions => actions ++ acc)) - private def createActionFromLocalFile - : Config => LocalFile => S3ObjectsData => Stream[Action] => Stream[Action] = - c => - lf => - remoteData => - previousActions => - ActionGenerator.createActions( - S3MetaDataEnricher.getMetadata(lf, remoteData)(c), - previousActions)(c) + private def createActionFromLocalFile( + remoteData: S3ObjectsData, + previousActions: Stream[Action], + localFile: LocalFile + ) = + ActionGenerator.createActions( + S3MetaDataEnricher.getMetadata(localFile, remoteData), + previousActions) - private def actionsForRemoteKeys: Config => S3ObjectsData => Stream[Action] = - c => - remoteData => - remoteData.byKey.keys.foldLeft(Stream.empty[Action])((acc, rk) => - createActionFromRemoteKey(c)(rk) #:: acc) + private def actionsForRemoteKeys(remoteData: S3ObjectsData) = + ZIO.foldLeft(remoteData.byKey.keys)(Stream.empty[Action]) { + (acc, remoteKey) => + createActionFromRemoteKey(remoteKey).map(action => action #:: acc) + } - private def createActionFromRemoteKey: Config => RemoteKey => Action = - c => - rk => - if (rk.isMissingLocally(c.sources, c.prefix)) - Action.ToDelete(c.bucket, rk, 0L) - else DoNothing(c.bucket, rk, 0L) + private def createActionFromRemoteKey(remoteKey: RemoteKey) = + for { + bucket <- getBucket + prefix <- getPrefix + sources <- getSources + needsDeleted = remoteKey.isMissingLocally(sources, prefix) + } yield + if (needsDeleted) ToDelete(bucket, remoteKey, 0L) + else DoNothing(bucket, remoteKey, 0L) - private def gatherMetadata( - hashService: HashService - )(implicit c: Config) = + private def gatherMetadata(hashService: HashService) = for { remoteData <- fetchRemoteData localData <- findLocalFiles(hashService) } yield (remoteData, localData) - private def fetchRemoteData(implicit c: Config) = - listObjects(c.bucket, c.prefix) + private def fetchRemoteData = + for { + bucket <- getBucket + prefix <- getPrefix + objects <- listObjects(bucket, prefix) + } yield objects - private def findLocalFiles( - hashService: HashService - )(implicit config: Config) = + private def findLocalFiles(hashService: HashService) = for { _ <- SyncLogging.logFileScan localFiles <- findFiles(hashService) } yield localFiles - private def findFiles( - hashService: HashService - )(implicit c: Config) = { - Task - .foreach(c.sources.paths)(LocalFileStream.findFiles(_, hashService)) - .map(_.foldLeft(LocalFiles())((acc, localFile) => acc ++ localFile)) - } + private def findFiles(hashService: HashService) = + for { + sources <- getSources + paths = sources.paths + found <- ZIO.foreach(paths)(path => + LocalFileStream.findFiles(hashService)(path)) + } yield LocalFiles.reduce(found.toStream) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/S3MetaDataEnricher.scala b/core/src/main/scala/net/kemitix/thorp/core/S3MetaDataEnricher.scala index 8e5be81..6a41611 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/S3MetaDataEnricher.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/S3MetaDataEnricher.scala @@ -7,7 +7,7 @@ object S3MetaDataEnricher { def getMetadata( localFile: LocalFile, s3ObjectsData: S3ObjectsData - )(implicit c: Config): S3MetaData = { + ): S3MetaData = { val (keyMatches, hashMatches) = getS3Status(localFile, s3ObjectsData) S3MetaData( localFile, diff --git a/core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala b/core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala index bbe0034..fc5f9f8 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala @@ -1,5 +1,6 @@ package net.kemitix.thorp.core +import net.kemitix.thorp.config._ import net.kemitix.thorp.console._ import net.kemitix.thorp.domain.StorageQueueEvent.{ CopyQueueEvent, @@ -13,17 +14,19 @@ import zio.ZIO trait SyncLogging { - def logRunStart( - bucket: Bucket, - prefix: RemoteKey, - sources: Sources - ): ZIO[Console, Nothing, Unit] = + def logRunStart: ZIO[Console with Config, Nothing, Unit] = for { - _ <- putStrLn(ConsoleOut.ValidConfig(bucket, prefix, sources)) + bucket <- getBucket + prefix <- getPrefix + sources <- getSources + _ <- putMessageLn(ConsoleOut.ValidConfig(bucket, prefix, sources)) } yield () - def logFileScan(implicit c: Config): ZIO[Console, Nothing, Unit] = - putStrLn(s"Scanning local files: ${c.sources.paths.mkString(", ")}...") + def logFileScan: ZIO[Config with Console, Nothing, Unit] = + for { + sources <- getSources + _ <- putStrLn(s"Scanning local files: ${sources.paths.mkString(", ")}...") + } yield () def logRunFinished( actions: Stream[StorageQueueEvent] diff --git a/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala index 146c280..3aa1e84 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala @@ -2,29 +2,35 @@ package net.kemitix.thorp.core import java.time.Instant +import net.kemitix.thorp.config._ import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload} import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain._ import org.scalatest.FunSpec +import zio.DefaultRuntime class ActionGeneratorSuite extends FunSpec { val lastModified = LastModified(Instant.now()) private val source = Resource(this, "upload") private val sourcePath = source.toPath + private val sources = Sources(List(sourcePath)) private val prefix = RemoteKey("prefix") private val bucket = Bucket("bucket") - implicit private val config: Config = - Config(bucket, prefix, sources = Sources(List(sourcePath))) + private val configOptions = ConfigOptions( + List( + ConfigOption.Bucket("bucket"), + ConfigOption.Prefix("prefix"), + ConfigOption.Source(sourcePath), + ConfigOption.IgnoreUserOptions, + ConfigOption.IgnoreGlobalOptions + )) private val fileToKey = - KeyGenerator.generateKey(config.sources, config.prefix) _ + KeyGenerator.generateKey(sources, prefix) _ describe("create actions") { val previousActions = Stream.empty[Action] - def invoke(input: S3MetaData) = - ActionGenerator.createActions(input, previousActions).toList - describe("#1 local exists, remote exists, remote matches - do nothing") { val theHash = MD5Hash("the-hash") val theFile = LocalFile.resolve("the-file", @@ -40,8 +46,9 @@ class ActionGeneratorSuite extends FunSpec { ) it("do nothing") { val expected = - List(DoNothing(bucket, theFile.remoteKey, theFile.file.length)) - val result = invoke(input) + Right( + Stream(DoNothing(bucket, theFile.remoteKey, theFile.file.length))) + val result = invoke(input, previousActions) assertResult(expected)(result) } } @@ -60,13 +67,14 @@ class ActionGeneratorSuite extends FunSpec { matchByHash = Set(otherRemoteMetadata), // other matches matchByKey = None) // remote is missing it("copy from other key") { - val expected = List( - ToCopy(bucket, - otherRemoteKey, - theHash, - theRemoteKey, - theFile.file.length)) // copy - val result = invoke(input) + val expected = Right( + Stream( + ToCopy(bucket, + otherRemoteKey, + theHash, + theRemoteKey, + theFile.file.length))) // copy + val result = invoke(input, previousActions) assertResult(expected)(result) } } @@ -80,8 +88,9 @@ class ActionGeneratorSuite extends FunSpec { matchByHash = Set.empty, // other no matches matchByKey = None) // remote is missing it("upload") { - val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload - val result = invoke(input) + val expected = Right( + Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload + val result = invoke(input, previousActions) assertResult(expected)(result) } } @@ -105,13 +114,14 @@ class ActionGeneratorSuite extends FunSpec { matchByHash = Set(otherRemoteMetadata), // other matches matchByKey = Some(oldRemoteMetadata)) // remote exists it("copy from other key") { - val expected = List( - ToCopy(bucket, - otherRemoteKey, - theHash, - theRemoteKey, - theFile.file.length)) // copy - val result = invoke(input) + val expected = Right( + Stream( + ToCopy(bucket, + otherRemoteKey, + theHash, + theRemoteKey, + theFile.file.length))) // copy + val result = invoke(input, previousActions) assertResult(expected)(result) } } @@ -132,8 +142,9 @@ class ActionGeneratorSuite extends FunSpec { matchByKey = Some(theRemoteMetadata) // remote exists ) it("upload") { - val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload - val result = invoke(input) + val expected = Right( + Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload + val result = invoke(input, previousActions) assertResult(expected)(result) } } @@ -147,4 +158,23 @@ class ActionGeneratorSuite extends FunSpec { private def md5HashMap(theHash: MD5Hash): Map[HashType, MD5Hash] = { Map(MD5 -> theHash) } + + private def invoke( + input: S3MetaData, + previousActions: Stream[Action] + ) = { + type TestEnv = Config + val testEnv: TestEnv = new Config.Live {} + + def testProgram = + for { + config <- ConfigurationBuilder.buildConfig(configOptions) + _ <- setConfiguration(config) + actions <- ActionGenerator.createActions(input, previousActions) + } yield actions + + new DefaultRuntime {}.unsafeRunSync { + testProgram.provide(testEnv) + }.toEither + } } diff --git a/core/src/test/scala/net/kemitix/thorp/core/ConfigOptionTest.scala b/core/src/test/scala/net/kemitix/thorp/core/ConfigOptionTest.scala index 73f5476..fd1f788 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/ConfigOptionTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/ConfigOptionTest.scala @@ -1,5 +1,11 @@ package net.kemitix.thorp.core +import net.kemitix.thorp.config.{ + ConfigOption, + ConfigOptions, + ConfigQuery, + ConfigurationBuilder +} import net.kemitix.thorp.domain.Sources import org.scalatest.FunSpec import zio.DefaultRuntime diff --git a/core/src/test/scala/net/kemitix/thorp/core/ConfigQueryTest.scala b/core/src/test/scala/net/kemitix/thorp/core/ConfigQueryTest.scala index 48ed4fb..8050300 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/ConfigQueryTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/ConfigQueryTest.scala @@ -2,6 +2,7 @@ package net.kemitix.thorp.core import java.nio.file.Paths +import net.kemitix.thorp.config.{ConfigOption, ConfigOptions, ConfigQuery} import net.kemitix.thorp.domain.Sources import org.scalatest.FreeSpec @@ -10,8 +11,8 @@ class ConfigQueryTest extends FreeSpec { "show version" - { "when is set" - { "should be true" in { - val result = ConfigQuery.showVersion(ConfigOptions(List( - ConfigOption.Version))) + val result = + ConfigQuery.showVersion(ConfigOptions(List(ConfigOption.Version))) assertResult(true)(result) } } @@ -25,8 +26,8 @@ class ConfigQueryTest extends FreeSpec { "batch mode" - { "when is set" - { "should be true" in { - val result = ConfigQuery.batchMode(ConfigOptions(List( - ConfigOption.BatchMode))) + val result = + ConfigQuery.batchMode(ConfigOptions(List(ConfigOption.BatchMode))) assertResult(true)(result) } } @@ -40,8 +41,8 @@ class ConfigQueryTest extends FreeSpec { "ignore user options" - { "when is set" - { "should be true" in { - val result = ConfigQuery.ignoreUserOptions(ConfigOptions(List( - ConfigOption.IgnoreUserOptions))) + val result = ConfigQuery.ignoreUserOptions( + ConfigOptions(List(ConfigOption.IgnoreUserOptions))) assertResult(true)(result) } } @@ -55,8 +56,8 @@ class ConfigQueryTest extends FreeSpec { "ignore global options" - { "when is set" - { "should be true" in { - val result = ConfigQuery.ignoreGlobalOptions(ConfigOptions(List( - ConfigOption.IgnoreGlobalOptions))) + val result = ConfigQuery.ignoreGlobalOptions( + ConfigOptions(List(ConfigOption.IgnoreGlobalOptions))) assertResult(true)(result) } } @@ -72,26 +73,26 @@ class ConfigQueryTest extends FreeSpec { val pathB = Paths.get("b-path") "when not set" - { "should have current dir" - { - val pwd = Paths.get(System.getenv("PWD")) + val pwd = Paths.get(System.getenv("PWD")) val expected = Sources(List(pwd)) - val result = ConfigQuery.sources(ConfigOptions(List())) + val result = ConfigQuery.sources(ConfigOptions(List())) assertResult(expected)(result) } } "when is set once" - { "should have one source" in { val expected = Sources(List(pathA)) - val result = ConfigQuery.sources(ConfigOptions(List( - ConfigOption.Source(pathA)))) + val result = + ConfigQuery.sources(ConfigOptions(List(ConfigOption.Source(pathA)))) assertResult(expected)(result) } } "when is set twice" - { "should have two sources" in { val expected = Sources(List(pathA, pathB)) - val result = ConfigQuery.sources(ConfigOptions(List( - ConfigOption.Source(pathA), - ConfigOption.Source(pathB)))) + val result = ConfigQuery.sources( + ConfigOptions( + List(ConfigOption.Source(pathA), ConfigOption.Source(pathB)))) assertResult(expected)(result) } } diff --git a/core/src/test/scala/net/kemitix/thorp/core/ConfigurationBuilderTest.scala b/core/src/test/scala/net/kemitix/thorp/core/ConfigurationBuilderTest.scala index df75c5f..9b85961 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/ConfigurationBuilderTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/ConfigurationBuilderTest.scala @@ -2,6 +2,11 @@ package net.kemitix.thorp.core import java.nio.file.{Path, Paths} +import net.kemitix.thorp.config.{ + ConfigOption, + ConfigOptions, + ConfigurationBuilder +} import net.kemitix.thorp.domain.Filter.{Exclude, Include} import net.kemitix.thorp.domain._ import org.scalatest.FunSpec diff --git a/core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala index 3d3f471..f541c20 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala @@ -2,7 +2,8 @@ package net.kemitix.thorp.core import java.io.File -import net.kemitix.thorp.domain.{Bucket, Config, RemoteKey, Sources} +import net.kemitix.thorp.config.Resource +import net.kemitix.thorp.domain.{RemoteKey, Sources} import org.scalatest.FunSpec class KeyGeneratorSuite extends FunSpec { @@ -10,10 +11,9 @@ class KeyGeneratorSuite extends FunSpec { private val source: File = Resource(this, "upload") private val sourcePath = source.toPath private val prefix = RemoteKey("prefix") - implicit private val config: Config = - Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath))) + private val sources = Sources(List(sourcePath)) private val fileToKey = - KeyGenerator.generateKey(config.sources, config.prefix) _ + KeyGenerator.generateKey(sources, prefix) _ describe("key generator") { diff --git a/core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala index 077f3aa..2047296 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala @@ -2,11 +2,13 @@ package net.kemitix.thorp.core import java.nio.file.Paths +import net.kemitix.thorp.config._ +import net.kemitix.thorp.console._ import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain._ -import net.kemitix.thorp.storage.api.HashService +import net.kemitix.thorp.storage.api.{HashService, Storage} import org.scalatest.FunSpec -import zio.DefaultRuntime +import zio.{DefaultRuntime, Task, UIO} class LocalFileStreamSuite extends FunSpec { @@ -21,8 +23,13 @@ class LocalFileStreamSuite extends FunSpec { private def file(filename: String) = sourcePath.resolve(Paths.get(filename)) - implicit private val config: Config = Config( - sources = Sources(List(sourcePath))) + private val configOptions = ConfigOptions( + List( + ConfigOption.IgnoreGlobalOptions, + ConfigOption.IgnoreUserOptions, + ConfigOption.Source(sourcePath), + ConfigOption.Bucket("aBucket") + )) describe("findFiles") { it("should find all files") { @@ -47,9 +54,29 @@ class LocalFileStreamSuite extends FunSpec { } private def invoke() = { - val runtime = new DefaultRuntime {} - runtime.unsafeRunSync { - LocalFileStream.findFiles(sourcePath, hashService) + type TestEnv = Storage with Console with Config + val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live { + override def listResult: Task[S3ObjectsData] = + Task.die(new NotImplementedError) + override def uploadResult: UIO[StorageQueueEvent] = + Task.die(new NotImplementedError) + override def copyResult: UIO[StorageQueueEvent] = + Task.die(new NotImplementedError) + override def deleteResult: UIO[StorageQueueEvent] = + Task.die(new NotImplementedError) + override def shutdownResult: UIO[StorageQueueEvent] = + Task.die(new NotImplementedError) + } + + def testProgram = + for { + config <- ConfigurationBuilder.buildConfig(configOptions) + _ <- setConfiguration(config) + files <- LocalFileStream.findFiles(hashService)(sourcePath) + } yield files + + new DefaultRuntime {}.unsafeRunSync { + testProgram.provide(testEnv) }.toEither } diff --git a/core/src/test/scala/net/kemitix/thorp/core/MD5HashGeneratorTest.scala b/core/src/test/scala/net/kemitix/thorp/core/MD5HashGeneratorTest.scala index e6a091e..e414aa0 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/MD5HashGeneratorTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/MD5HashGeneratorTest.scala @@ -2,8 +2,8 @@ package net.kemitix.thorp.core import java.nio.file.Path +import net.kemitix.thorp.config.Resource import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root} -import net.kemitix.thorp.domain._ import org.scalatest.FunSpec import zio.DefaultRuntime @@ -11,11 +11,7 @@ class MD5HashGeneratorTest extends FunSpec { private val runtime = new DefaultRuntime {} - private val source = Resource(this, "upload") - private val sourcePath = source.toPath - private val prefix = RemoteKey("prefix") - implicit private val config: Config = - Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath))) + private val source = Resource(this, "upload") describe("md5File()") { describe("read a small file (smaller than buffer)") { diff --git a/core/src/test/scala/net/kemitix/thorp/core/ParseConfigFileTest.scala b/core/src/test/scala/net/kemitix/thorp/core/ParseConfigFileTest.scala index e3cb0df..3aeee04 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/ParseConfigFileTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/ParseConfigFileTest.scala @@ -2,6 +2,12 @@ package net.kemitix.thorp.core import java.nio.file.{Path, Paths} +import net.kemitix.thorp.config.{ + ConfigOption, + ConfigOptions, + ParseConfigFile, + Resource +} import org.scalatest.FunSpec import zio.DefaultRuntime diff --git a/core/src/test/scala/net/kemitix/thorp/core/ParseConfigLinesTest.scala b/core/src/test/scala/net/kemitix/thorp/core/ParseConfigLinesTest.scala index 270cd8f..9f0853f 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/ParseConfigLinesTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/ParseConfigLinesTest.scala @@ -2,6 +2,7 @@ package net.kemitix.thorp.core import java.nio.file.Paths +import net.kemitix.thorp.config.{ConfigOption, ConfigOptions, ParseConfigLines} import org.scalatest.FunSpec class ParseConfigLinesTest extends FunSpec { diff --git a/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala b/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala index 8e19aee..711159f 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala @@ -3,6 +3,7 @@ package net.kemitix.thorp.core import java.io.File import java.nio.file.Path +import net.kemitix.thorp.config._ import net.kemitix.thorp.console._ import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload} import net.kemitix.thorp.domain.HashType.MD5 @@ -24,7 +25,9 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { val options: Path => ConfigOptions = source => configOptions(ConfigOption.Source(source), - ConfigOption.Bucket("a-bucket")) + ConfigOption.Bucket("a-bucket"), + ConfigOption.IgnoreUserOptions, + ConfigOption.IgnoreGlobalOptions) "a file" - { val filename = "aFile" val remoteKey = RemoteKey(filename) @@ -325,9 +328,9 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { hashService: HashService, configOptions: ConfigOptions, result: Task[S3ObjectsData] - ): Either[Any, List[(String, String, String, String, String)]] = { - type TestEnv = Storage.Test with Console.Test - val testEnv: TestEnv = new Storage.Test with Console.Test { + ) = { + type TestEnv = Storage with Console with Config + val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live { override def listResult: Task[S3ObjectsData] = result override def uploadResult: UIO[StorageQueueEvent] = Task.die(new NotImplementedError) @@ -339,12 +342,15 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { Task.die(new NotImplementedError) } + def testProgram = + for { + config <- ConfigurationBuilder.buildConfig(configOptions) + _ <- setConfiguration(config) + plan <- PlanBuilder.createPlan(hashService) + } yield plan + new DefaultRuntime {} - .unsafeRunSync { - PlanBuilder - .createPlan(hashService, configOptions) - .provide(testEnv) - } + .unsafeRunSync(testProgram.provide(testEnv)) .toEither .map(convertResult) } diff --git a/core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala index 3dc1a74..05eb4a5 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala @@ -2,6 +2,7 @@ package net.kemitix.thorp.core import java.time.Instant +import net.kemitix.thorp.config.Resource import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status} import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain._ @@ -11,11 +12,10 @@ class S3MetaDataEnricherSuite extends FunSpec { val lastModified = LastModified(Instant.now()) private val source = Resource(this, "upload") private val sourcePath = source.toPath + private val sources = Sources(List(sourcePath)) private val prefix = RemoteKey("prefix") - implicit private val config: Config = - Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath))) private val fileToKey = - KeyGenerator.generateKey(config.sources, config.prefix) _ + KeyGenerator.generateKey(sources, prefix) _ def getMatchesByKey( status: (Option[HashModified], Set[(MD5Hash, KeyModified)])) diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/Config.scala b/domain/src/main/scala/net/kemitix/thorp/domain/Config.scala deleted file mode 100644 index 5fff0a6..0000000 --- a/domain/src/main/scala/net/kemitix/thorp/domain/Config.scala +++ /dev/null @@ -1,25 +0,0 @@ -package net.kemitix.thorp.domain - -final case class Config( - bucket: Bucket = Bucket(""), - prefix: RemoteKey = RemoteKey(""), - filters: List[Filter] = List(), - debug: Boolean = false, - batchMode: Boolean = false, - sources: Sources = Sources(List()) -) - -object Config { - val sources: SimpleLens[Config, Sources] = - SimpleLens[Config, Sources](_.sources, b => a => b.copy(sources = a)) - val bucket: SimpleLens[Config, Bucket] = - SimpleLens[Config, Bucket](_.bucket, b => a => b.copy(bucket = a)) - val prefix: SimpleLens[Config, RemoteKey] = - SimpleLens[Config, RemoteKey](_.prefix, b => a => b.copy(prefix = a)) - val filters: SimpleLens[Config, List[Filter]] = - SimpleLens[Config, List[Filter]](_.filters, b => a => b.copy(filters = a)) - val debug: SimpleLens[Config, Boolean] = - SimpleLens[Config, Boolean](_.debug, b => a => b.copy(debug = a)) - val batchMode: SimpleLens[Config, Boolean] = - SimpleLens[Config, Boolean](_.batchMode, b => a => b.copy(batchMode = a)) -} diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ETagGeneratorTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ETagGeneratorTest.scala index eb901bc..53ba8eb 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ETagGeneratorTest.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ETagGeneratorTest.scala @@ -3,7 +3,7 @@ package net.kemitix.thorp.storage.aws import java.nio.file.Path import com.amazonaws.services.s3.transfer.TransferManagerConfiguration -import net.kemitix.thorp.core.Resource +import net.kemitix.thorp.config.Resource import org.scalatest.FunSpec import zio.DefaultRuntime diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala index 00b2576..5cec8f3 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala @@ -2,7 +2,8 @@ package net.kemitix.thorp.storage.aws import java.time.Instant -import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher} +import net.kemitix.thorp.config.Resource +import net.kemitix.thorp.core.{KeyGenerator, S3MetaDataEnricher} import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain._ import org.scalamock.scalatest.MockFactory @@ -12,12 +13,10 @@ class StorageServiceSuite extends FunSpec with MockFactory { private val source = Resource(this, "upload") private val sourcePath = source.toPath - - private val prefix = RemoteKey("prefix") - implicit private val config: Config = - Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath))) + private val sources = Sources(List(sourcePath)) + private val prefix = RemoteKey("prefix") private val fileToKey = - KeyGenerator.generateKey(config.sources, config.prefix) _ + KeyGenerator.generateKey(sources, prefix) _ describe("getS3Status") { diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala index ccc5e0f..aa68d03 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala @@ -5,8 +5,8 @@ import java.io.File import com.amazonaws.SdkClientException import com.amazonaws.services.s3.model.AmazonS3Exception import com.amazonaws.services.s3.transfer.model.UploadResult +import net.kemitix.thorp.config.Resource import net.kemitix.thorp.console._ -import net.kemitix.thorp.core.Resource import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain.StorageQueueEvent.{ Action,