Convert Config to full ZIO effect module (#134)
* [config] new module * [config] stub module * [domain] Rename domain.Config as domain.LegacyConfig * [config] Move LegacyConfig to config module * [config] Move config parsing and validation into module * [config] Complete migration to module for Config * [config] Config You should not name methods after their defining object * [config] Rename LegacyConfig as Configuration Also remove redundant uses * [core] LocalFileStream Refactoring * [changelog] update
This commit is contained in:
parent
96a83e6c3e
commit
ccefd286f9
43 changed files with 517 additions and 327 deletions
|
@ -20,6 +20,7 @@ The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]
|
|||
- [internal] Replace Monocle with local SimpleLens implementation (#121)
|
||||
- [internal] Don't use String as key in Map for hashes (#124)
|
||||
- [internal] Convert Storage to full ZIO effect module (#133)
|
||||
- [internal] Convert Config to full ZIO effect module (#134)
|
||||
|
||||
** Dependencies
|
||||
|
||||
|
|
11
build.sbt
11
build.sbt
|
@ -58,6 +58,7 @@ val zioDependencies = Seq(
|
|||
)
|
||||
|
||||
// cli -> thorp-lib -> storage-aws -> core -> storage-api -> console -> domain
|
||||
// core -> config -> domain
|
||||
|
||||
lazy val thorp = (project in file("."))
|
||||
.settings(commonSettings)
|
||||
|
@ -67,7 +68,6 @@ lazy val cli = (project in file("cli"))
|
|||
.settings(commonSettings)
|
||||
.settings(mainClass in assembly := Some("net.kemitix.thorp.cli.Main"))
|
||||
.settings(applicationSettings)
|
||||
.settings(commandLineParsing)
|
||||
.settings(testDependencies)
|
||||
.enablePlugins(BuildInfoPlugin)
|
||||
.settings(
|
||||
|
@ -101,6 +101,7 @@ lazy val core = (project in file("core"))
|
|||
.settings(testDependencies)
|
||||
.dependsOn(`storage-api`)
|
||||
.dependsOn(domain % "compile->compile;test->test")
|
||||
.dependsOn(config)
|
||||
|
||||
lazy val `storage-api` = (project in file("storage-api"))
|
||||
.settings(commonSettings)
|
||||
|
@ -114,6 +115,14 @@ lazy val console = (project in file("console"))
|
|||
.settings(assemblyJarName in assembly := "console.jar")
|
||||
.dependsOn(domain)
|
||||
|
||||
lazy val config = (project in file("config"))
|
||||
.settings(commonSettings)
|
||||
.settings(zioDependencies)
|
||||
.settings(testDependencies)
|
||||
.settings(commandLineParsing)
|
||||
.settings(assemblyJarName in assembly := "config.jar")
|
||||
.dependsOn(domain)
|
||||
|
||||
lazy val domain = (project in file("domain"))
|
||||
.settings(commonSettings)
|
||||
.settings(assemblyJarName in assembly := "domain.jar")
|
||||
|
|
|
@ -1,12 +1,13 @@
|
|||
package net.kemitix.thorp.cli
|
||||
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.console.Console
|
||||
import net.kemitix.thorp.storage.aws.S3Storage
|
||||
import zio.{App, ZIO}
|
||||
|
||||
object Main extends App {
|
||||
|
||||
object LiveThorpApp extends S3Storage.Live with Console.Live
|
||||
object LiveThorpApp extends S3Storage.Live with Console.Live with Config.Live
|
||||
|
||||
override def run(args: List[String]): ZIO[Environment, Nothing, Int] =
|
||||
Program
|
||||
|
|
|
@ -1,11 +1,12 @@
|
|||
package net.kemitix.thorp.cli
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.core.CoreTypes.CoreProgram
|
||||
import net.kemitix.thorp.core._
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent
|
||||
import net.kemitix.thorp.storage.aws.S3HashService.defaultHashService
|
||||
import zio.{UIO, ZIO}
|
||||
import zio.ZIO
|
||||
|
||||
trait Program {
|
||||
|
||||
|
@ -14,27 +15,26 @@ trait Program {
|
|||
def run(args: List[String]): CoreProgram[Unit] = {
|
||||
for {
|
||||
cli <- CliArgs.parse(args)
|
||||
config <- ConfigurationBuilder.buildConfig(cli)
|
||||
_ <- setConfiguration(config)
|
||||
_ <- ZIO.when(showVersion(cli))(putStrLn(version))
|
||||
_ <- ZIO.when(!showVersion(cli))(execute(cli).catchAll(handleErrors))
|
||||
_ <- ZIO.when(!showVersion(cli))(execute.catchAll(handleErrors))
|
||||
} yield ()
|
||||
}
|
||||
|
||||
private def showVersion: ConfigOptions => Boolean =
|
||||
cli => ConfigQuery.showVersion(cli)
|
||||
|
||||
private def execute(cliOptions: ConfigOptions) = {
|
||||
private def execute = {
|
||||
for {
|
||||
plan <- PlanBuilder.createPlan(defaultHashService, cliOptions)
|
||||
batchMode <- isBatchMode(cliOptions)
|
||||
plan <- PlanBuilder.createPlan(defaultHashService)
|
||||
batchMode <- isBatchMode
|
||||
archive <- UnversionedMirrorArchive.default(batchMode, plan.syncTotals)
|
||||
events <- applyPlan(archive, plan)
|
||||
_ <- SyncLogging.logRunFinished(events)
|
||||
} yield ()
|
||||
}
|
||||
|
||||
private def isBatchMode(cliOptions: ConfigOptions) =
|
||||
UIO(ConfigQuery.batchMode(cliOptions))
|
||||
|
||||
private def handleErrors(throwable: Throwable) =
|
||||
for {
|
||||
_ <- putStrLn("There were errors:")
|
||||
|
|
|
@ -1,8 +1,7 @@
|
|||
package net.kemitix.thorp.cli
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.thorp.core.{ConfigOption, ConfigOptions}
|
||||
import scopt.OParser
|
||||
import zio.Task
|
||||
|
50
config/src/main/scala/net/kemitix/thorp/config/Config.scala
Normal file
50
config/src/main/scala/net/kemitix/thorp/config/Config.scala
Normal file
|
@ -0,0 +1,50 @@
|
|||
package net.kemitix.thorp.config
|
||||
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, Sources}
|
||||
import zio.{UIO, ZIO}
|
||||
|
||||
trait Config {
|
||||
val config: Config.Service
|
||||
}
|
||||
|
||||
object Config {
|
||||
|
||||
trait Service {
|
||||
def setConfiguration(config: Configuration): ZIO[Config, Nothing, Unit]
|
||||
def isBatchMode: ZIO[Config, Nothing, Boolean]
|
||||
def bucket: ZIO[Config, Nothing, Bucket]
|
||||
def prefix: ZIO[Config, Nothing, RemoteKey]
|
||||
def sources: ZIO[Config, Nothing, Sources]
|
||||
def filters: ZIO[Config, Nothing, List[Filter]]
|
||||
}
|
||||
|
||||
trait Live extends Config {
|
||||
|
||||
val config: Service = new Service {
|
||||
private val configRef = new AtomicReference(Configuration())
|
||||
override def setConfiguration(
|
||||
config: Configuration): ZIO[Config, Nothing, Unit] =
|
||||
UIO(configRef.set(config))
|
||||
|
||||
override def bucket: ZIO[Config, Nothing, Bucket] =
|
||||
UIO(configRef.get).map(_.bucket)
|
||||
|
||||
override def sources: ZIO[Config, Nothing, Sources] =
|
||||
UIO(configRef.get).map(_.sources)
|
||||
|
||||
override def prefix: ZIO[Config, Nothing, RemoteKey] =
|
||||
UIO(configRef.get).map(_.prefix)
|
||||
|
||||
override def isBatchMode: ZIO[Config, Nothing, Boolean] =
|
||||
UIO(configRef.get).map(_.batchMode)
|
||||
|
||||
override def filters: ZIO[Config, Nothing, List[Filter]] =
|
||||
UIO(configRef.get).map(_.filters)
|
||||
}
|
||||
}
|
||||
|
||||
object Live extends Live
|
||||
|
||||
}
|
|
@ -1,24 +1,24 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import net.kemitix.thorp.config.Configuration._
|
||||
import net.kemitix.thorp.domain
|
||||
import net.kemitix.thorp.domain.{Config, RemoteKey}
|
||||
import net.kemitix.thorp.domain.Config._
|
||||
import net.kemitix.thorp.domain.RemoteKey
|
||||
|
||||
sealed trait ConfigOption {
|
||||
def update(config: Config): Config
|
||||
def update(config: Configuration): Configuration
|
||||
}
|
||||
|
||||
object ConfigOption {
|
||||
|
||||
case class Source(path: Path) extends ConfigOption {
|
||||
override def update(config: Config): Config =
|
||||
override def update(config: Configuration): Configuration =
|
||||
sources.modify(_ ++ path)(config)
|
||||
}
|
||||
|
||||
case class Bucket(name: String) extends ConfigOption {
|
||||
override def update(config: Config): Config =
|
||||
override def update(config: Configuration): Configuration =
|
||||
if (config.bucket.name.isEmpty)
|
||||
bucket.set(domain.Bucket(name))(config)
|
||||
else
|
||||
|
@ -26,7 +26,7 @@ object ConfigOption {
|
|||
}
|
||||
|
||||
case class Prefix(path: String) extends ConfigOption {
|
||||
override def update(config: Config): Config =
|
||||
override def update(config: Configuration): Configuration =
|
||||
if (config.prefix.key.isEmpty)
|
||||
prefix.set(RemoteKey(path))(config)
|
||||
else
|
||||
|
@ -34,35 +34,35 @@ object ConfigOption {
|
|||
}
|
||||
|
||||
case class Include(pattern: String) extends ConfigOption {
|
||||
override def update(config: Config): Config =
|
||||
override def update(config: Configuration): Configuration =
|
||||
filters.modify(domain.Filter.Include(pattern) :: _)(config)
|
||||
}
|
||||
|
||||
case class Exclude(pattern: String) extends ConfigOption {
|
||||
override def update(config: Config): Config =
|
||||
override def update(config: Configuration): Configuration =
|
||||
filters.modify(domain.Filter.Exclude(pattern) :: _)(config)
|
||||
}
|
||||
|
||||
case class Debug() extends ConfigOption {
|
||||
override def update(config: Config): Config =
|
||||
override def update(config: Configuration): Configuration =
|
||||
debug.set(true)(config)
|
||||
}
|
||||
|
||||
case object Version extends ConfigOption {
|
||||
override def update(config: Config): Config = config
|
||||
override def update(config: Configuration): Configuration = config
|
||||
}
|
||||
|
||||
case object BatchMode extends ConfigOption {
|
||||
override def update(config: Config): Config =
|
||||
override def update(config: Configuration): Configuration =
|
||||
batchMode.set(true)(config)
|
||||
}
|
||||
|
||||
case object IgnoreUserOptions extends ConfigOption {
|
||||
override def update(config: Config): Config = config
|
||||
override def update(config: Configuration): Configuration = config
|
||||
}
|
||||
|
||||
case object IgnoreGlobalOptions extends ConfigOption {
|
||||
override def update(config: Config): Config = config
|
||||
override def update(config: Configuration): Configuration = config
|
||||
}
|
||||
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import net.kemitix.thorp.domain.SimpleLens
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.nio.file.Path
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
final case class ConfigValidationException(
|
||||
errors: List[ConfigValidation]
|
|
@ -1,15 +1,15 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
import net.kemitix.thorp.domain.{Bucket, Config, Sources}
|
||||
import net.kemitix.thorp.domain.{Bucket, Sources}
|
||||
import zio.IO
|
||||
|
||||
sealed trait ConfigValidator {
|
||||
|
||||
def validateConfig(
|
||||
config: Config
|
||||
): IO[List[ConfigValidation], Config] = IO.fromEither {
|
||||
config: Configuration
|
||||
): IO[List[ConfigValidation], Configuration] = IO.fromEither {
|
||||
for {
|
||||
_ <- validateSources(config.sources)
|
||||
_ <- validateBucket(config.bucket)
|
|
@ -0,0 +1,29 @@
|
|||
package net.kemitix.thorp.config
|
||||
|
||||
import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, SimpleLens, Sources}
|
||||
|
||||
private[config] final case class Configuration(
|
||||
bucket: Bucket = Bucket(""),
|
||||
prefix: RemoteKey = RemoteKey(""),
|
||||
filters: List[Filter] = List(),
|
||||
debug: Boolean = false,
|
||||
batchMode: Boolean = false,
|
||||
sources: Sources = Sources(List())
|
||||
)
|
||||
|
||||
private[config] object Configuration {
|
||||
val sources: SimpleLens[Configuration, Sources] =
|
||||
SimpleLens[Configuration, Sources](_.sources, b => a => b.copy(sources = a))
|
||||
val bucket: SimpleLens[Configuration, Bucket] =
|
||||
SimpleLens[Configuration, Bucket](_.bucket, b => a => b.copy(bucket = a))
|
||||
val prefix: SimpleLens[Configuration, RemoteKey] =
|
||||
SimpleLens[Configuration, RemoteKey](_.prefix, b => a => b.copy(prefix = a))
|
||||
val filters: SimpleLens[Configuration, List[Filter]] =
|
||||
SimpleLens[Configuration, List[Filter]](_.filters,
|
||||
b => a => b.copy(filters = a))
|
||||
val debug: SimpleLens[Configuration, Boolean] =
|
||||
SimpleLens[Configuration, Boolean](_.debug, b => a => b.copy(debug = a))
|
||||
val batchMode: SimpleLens[Configuration, Boolean] =
|
||||
SimpleLens[Configuration, Boolean](_.batchMode,
|
||||
b => a => b.copy(batchMode = a))
|
||||
}
|
|
@ -1,12 +1,8 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.thorp.core.ConfigOptions.options
|
||||
import net.kemitix.thorp.core.ConfigValidator.validateConfig
|
||||
import net.kemitix.thorp.core.ParseConfigFile.parseFile
|
||||
import net.kemitix.thorp.domain.Config
|
||||
import zio.IO
|
||||
import zio.{IO, TaskR}
|
||||
|
||||
/**
|
||||
* Builds a configuration from settings in a file within the
|
||||
|
@ -18,12 +14,13 @@ trait ConfigurationBuilder {
|
|||
private val globalConfig = Paths.get("/etc/thorp.conf")
|
||||
private val userHome = Paths.get(System.getProperty("user.home"))
|
||||
|
||||
def buildConfig(
|
||||
priorityOpts: ConfigOptions): IO[List[ConfigValidation], Config] =
|
||||
for {
|
||||
def buildConfig(priorityOpts: ConfigOptions)
|
||||
: IO[ConfigValidationException, Configuration] =
|
||||
(for {
|
||||
config <- getConfigOptions(priorityOpts).map(collateOptions)
|
||||
valid <- validateConfig(config)
|
||||
} yield valid
|
||||
valid <- ConfigValidator.validateConfig(config)
|
||||
} yield valid)
|
||||
.catchAll(errors => TaskR.fail(ConfigValidationException(errors)))
|
||||
|
||||
private def getConfigOptions(
|
||||
priorityOpts: ConfigOptions): IO[List[ConfigValidation], ConfigOptions] =
|
||||
|
@ -39,17 +36,17 @@ trait ConfigurationBuilder {
|
|||
private def userOptions(
|
||||
priorityOpts: ConfigOptions): IO[List[ConfigValidation], ConfigOptions] =
|
||||
if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
|
||||
else parseFile(userHome.resolve(userConfigFilename))
|
||||
else ParseConfigFile.parseFile(userHome.resolve(userConfigFilename))
|
||||
|
||||
private def globalOptions(
|
||||
priorityOpts: ConfigOptions): IO[List[ConfigValidation], ConfigOptions] =
|
||||
if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig
|
||||
else parseFile(globalConfig)
|
||||
else ParseConfigFile.parseFile(globalConfig)
|
||||
|
||||
private def collateOptions(configOptions: ConfigOptions): Config =
|
||||
options
|
||||
private def collateOptions(configOptions: ConfigOptions): Configuration =
|
||||
ConfigOptions.options
|
||||
.get(configOptions)
|
||||
.foldLeft(Config()) { (config, configOption) =>
|
||||
.foldLeft(Configuration()) { (config, configOption) =>
|
||||
configOption.update(config)
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.nio.file.{Files, Path}
|
||||
|
|
@ -1,9 +1,16 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.nio.file.Paths
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import net.kemitix.thorp.core.ConfigOption._
|
||||
import net.kemitix.thorp.config.ConfigOption.{
|
||||
Bucket,
|
||||
Debug,
|
||||
Exclude,
|
||||
Include,
|
||||
Prefix,
|
||||
Source
|
||||
}
|
||||
|
||||
trait ParseConfigLines {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.io.{File, FileNotFoundException}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.thorp.core
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import net.kemitix.thorp.domain.Sources
|
||||
import zio.IO
|
29
config/src/main/scala/net/kemitix/thorp/config/package.scala
Normal file
29
config/src/main/scala/net/kemitix/thorp/config/package.scala
Normal file
|
@ -0,0 +1,29 @@
|
|||
package net.kemitix.thorp
|
||||
|
||||
import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, Sources}
|
||||
import zio.ZIO
|
||||
|
||||
package object config {
|
||||
|
||||
final val configService: ZIO[Config, Nothing, Config.Service] =
|
||||
ZIO.access(_.config)
|
||||
|
||||
final def setConfiguration(
|
||||
config: Configuration): ZIO[Config, Nothing, Unit] =
|
||||
ZIO.accessM(_.config setConfiguration config)
|
||||
|
||||
final def isBatchMode: ZIO[Config, Nothing, Boolean] =
|
||||
ZIO.accessM(_.config isBatchMode)
|
||||
|
||||
final def getBucket: ZIO[Config, Nothing, Bucket] =
|
||||
ZIO.accessM(_.config bucket)
|
||||
|
||||
final def getPrefix: ZIO[Config, Nothing, RemoteKey] =
|
||||
ZIO.accessM(_.config prefix)
|
||||
|
||||
final def getSources: ZIO[Config, Nothing, Sources] =
|
||||
ZIO.accessM(_.config sources)
|
||||
|
||||
final def getFilters: ZIO[Config, Nothing, List[Filter]] =
|
||||
ZIO.accessM(_.config filters)
|
||||
}
|
|
@ -1,9 +1,8 @@
|
|||
package net.kemitix.thorp.cli
|
||||
package net.kemitix.thorp.config
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.thorp.core.ConfigOption.Debug
|
||||
import net.kemitix.thorp.core.{ConfigOptions, ConfigQuery, Resource}
|
||||
import net.kemitix.thorp.config.ConfigOption.Debug
|
||||
import org.scalatest.FunSpec
|
||||
import zio.DefaultRuntime
|
||||
|
|
@ -10,7 +10,7 @@ package object console {
|
|||
final def putStrLn(line: String): ZIO[Console, Nothing, Unit] =
|
||||
ZIO.accessM(_.console putStrLn line)
|
||||
|
||||
final def putStrLn(line: ConsoleOut): ZIO[Console, Nothing, Unit] =
|
||||
final def putMessageLn(line: ConsoleOut): ZIO[Console, Nothing, Unit] =
|
||||
ZIO.accessM(_.console putStrLn line)
|
||||
|
||||
}
|
||||
|
|
|
@ -1,38 +1,46 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||
import net.kemitix.thorp.domain._
|
||||
import zio.ZIO
|
||||
|
||||
object ActionGenerator {
|
||||
|
||||
def createActions(
|
||||
s3MetaData: S3MetaData,
|
||||
previousActions: Stream[Action]
|
||||
)(implicit c: Config): Stream[Action] =
|
||||
): ZIO[Config, Nothing, Stream[Action]] =
|
||||
for {
|
||||
bucket <- getBucket
|
||||
} yield
|
||||
s3MetaData match {
|
||||
// #1 local exists, remote exists, remote matches - do nothing
|
||||
case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
|
||||
if localFile.matches(hash) =>
|
||||
doNothing(c.bucket, key)
|
||||
doNothing(bucket, key)
|
||||
// #2 local exists, remote is missing, other matches - copy
|
||||
case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
|
||||
copyFile(c.bucket, localFile, matchByHash)
|
||||
copyFile(bucket, localFile, matchByHash)
|
||||
// #3 local exists, remote is missing, other no matches - upload
|
||||
case S3MetaData(localFile, matchByHash, None)
|
||||
if matchByHash.isEmpty &&
|
||||
isUploadAlreadyQueued(previousActions)(localFile) =>
|
||||
uploadFile(c.bucket, localFile)
|
||||
uploadFile(bucket, localFile)
|
||||
// #4 local exists, remote exists, remote no match, other matches - copy
|
||||
case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _)))
|
||||
case S3MetaData(localFile,
|
||||
matchByHash,
|
||||
Some(RemoteMetaData(_, hash, _)))
|
||||
if !localFile.matches(hash) &&
|
||||
matchByHash.nonEmpty =>
|
||||
copyFile(c.bucket, localFile, matchByHash)
|
||||
copyFile(bucket, localFile, matchByHash)
|
||||
// #5 local exists, remote exists, remote no match, other no matches - upload
|
||||
case S3MetaData(localFile, matchByHash, Some(_)) if matchByHash.isEmpty =>
|
||||
uploadFile(c.bucket, localFile)
|
||||
case S3MetaData(localFile, matchByHash, Some(_))
|
||||
if matchByHash.isEmpty =>
|
||||
uploadFile(bucket, localFile)
|
||||
// fallback
|
||||
case S3MetaData(localFile, _, _) =>
|
||||
doNothing(c.bucket, localFile.remoteKey)
|
||||
doNothing(bucket, localFile.remoteKey)
|
||||
}
|
||||
|
||||
private def key = LocalFile.remoteKey ^|-> RemoteKey.key
|
||||
|
|
|
@ -1,12 +1,13 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.console.Console
|
||||
import net.kemitix.thorp.storage.api.Storage
|
||||
import zio.ZIO
|
||||
|
||||
object CoreTypes {
|
||||
|
||||
type CoreEnv = Storage with Console
|
||||
type CoreEnv = Storage with Console with Config
|
||||
type CoreProgram[A] = ZIO[CoreEnv, Throwable, A]
|
||||
|
||||
}
|
||||
|
|
|
@ -2,42 +2,30 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.nio.file.Path
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.core.KeyGenerator.generateKey
|
||||
import net.kemitix.thorp.domain
|
||||
import net.kemitix.thorp.domain._
|
||||
import net.kemitix.thorp.storage.api.HashService
|
||||
import zio.Task
|
||||
import zio.{Task, TaskR, ZIO}
|
||||
|
||||
object LocalFileStream {
|
||||
|
||||
def findFiles(
|
||||
source: Path,
|
||||
hashService: HashService
|
||||
)(
|
||||
implicit c: Config
|
||||
): Task[LocalFiles] = {
|
||||
def findFiles(hashService: HashService)(
|
||||
source: Path
|
||||
): TaskR[Config, LocalFiles] = {
|
||||
|
||||
val isIncluded: Path => Boolean = Filter.isIncluded(c.filters)
|
||||
|
||||
val pathToLocalFile: Path => Task[LocalFiles] = path =>
|
||||
localFile(hashService, c)(path)
|
||||
|
||||
def loop(path: Path): Task[LocalFiles] = {
|
||||
|
||||
def dirPaths(path: Path): Task[Stream[Path]] =
|
||||
listFiles(path)
|
||||
.map(_.filter(isIncluded))
|
||||
|
||||
def recurseIntoSubDirectories(path: Path): Task[LocalFiles] =
|
||||
def recurseIntoSubDirectories(path: Path): TaskR[Config, LocalFiles] =
|
||||
path.toFile match {
|
||||
case f if f.isDirectory => loop(path)
|
||||
case _ => pathToLocalFile(path)
|
||||
case _ => pathToLocalFile(hashService)(path)
|
||||
}
|
||||
|
||||
def recurse(paths: Stream[Path]): Task[LocalFiles] =
|
||||
Task.foldLeft(paths)(LocalFiles())((acc, path) => {
|
||||
recurseIntoSubDirectories(path).map(localFiles => acc ++ localFiles)
|
||||
})
|
||||
def recurse(paths: Stream[Path]): TaskR[Config, LocalFiles] =
|
||||
for {
|
||||
recursed <- ZIO.foreach(paths)(path => recurseIntoSubDirectories(path))
|
||||
} yield LocalFiles.reduce(recursed.toStream)
|
||||
|
||||
def loop(path: Path): TaskR[Config, LocalFiles] = {
|
||||
|
||||
for {
|
||||
paths <- dirPaths(path)
|
||||
|
@ -48,30 +36,50 @@ object LocalFileStream {
|
|||
loop(source)
|
||||
}
|
||||
|
||||
def localFile(
|
||||
hashService: HashService,
|
||||
c: Config
|
||||
): Path => Task[LocalFiles] =
|
||||
path => {
|
||||
val file = path.toFile
|
||||
val source = c.sources.forPath(path)
|
||||
private def dirPaths(path: Path) =
|
||||
for {
|
||||
hash <- hashService.hashLocalObject(path)
|
||||
paths <- listFiles(path)
|
||||
filtered <- includedDirPaths(paths)
|
||||
} yield filtered
|
||||
|
||||
private def includedDirPaths(paths: Stream[Path]) =
|
||||
for {
|
||||
flaggedPaths <- TaskR.foreach(paths)(path =>
|
||||
isIncluded(path).map((path, _)))
|
||||
} yield
|
||||
LocalFiles(localFiles = Stream(
|
||||
domain.LocalFile(file,
|
||||
source.toFile,
|
||||
flaggedPaths.toStream
|
||||
.filter({ case (_, included) => included })
|
||||
.map({ case (path, _) => path })
|
||||
|
||||
private def localFile(hashService: HashService)(path: Path) = {
|
||||
val file = path.toFile
|
||||
for {
|
||||
sources <- getSources
|
||||
prefix <- getPrefix
|
||||
hash <- hashService.hashLocalObject(path)
|
||||
localFile = LocalFile(file,
|
||||
sources.forPath(path).toFile,
|
||||
hash,
|
||||
generateKey(c.sources, c.prefix)(path))),
|
||||
generateKey(sources, prefix)(path))
|
||||
} yield
|
||||
LocalFiles(localFiles = Stream(localFile),
|
||||
count = 1,
|
||||
totalSizeBytes = file.length)
|
||||
}
|
||||
|
||||
private def listFiles(path: Path): Task[Stream[Path]] =
|
||||
private def listFiles(path: Path) =
|
||||
for {
|
||||
files <- Task(path.toFile.listFiles)
|
||||
_ <- Task.when(files == null)(
|
||||
Task.fail(new IllegalArgumentException(s"Directory not found $path")))
|
||||
} yield Stream(files: _*).map(_.toPath)
|
||||
|
||||
private def isIncluded(path: Path) =
|
||||
for {
|
||||
filters <- getFilters
|
||||
} yield Filter.isIncluded(filters)(path)
|
||||
|
||||
private def pathToLocalFile(hashService: HashService)(path: Path) =
|
||||
localFile(hashService)(path)
|
||||
|
||||
}
|
||||
|
|
|
@ -15,3 +15,10 @@ case class LocalFiles(
|
|||
)
|
||||
|
||||
}
|
||||
|
||||
object LocalFiles {
|
||||
|
||||
def reduce: Stream[LocalFiles] => LocalFiles =
|
||||
list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf)
|
||||
|
||||
}
|
||||
|
|
|
@ -1,121 +1,111 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.core.Action._
|
||||
import net.kemitix.thorp.domain._
|
||||
import net.kemitix.thorp.storage._
|
||||
import net.kemitix.thorp.storage.api.{HashService, Storage}
|
||||
import zio.{Task, TaskR}
|
||||
import zio.{TaskR, ZIO}
|
||||
|
||||
trait PlanBuilder {
|
||||
|
||||
def createPlan(
|
||||
hashService: HashService,
|
||||
configOptions: ConfigOptions
|
||||
): TaskR[Storage with Console, SyncPlan] =
|
||||
ConfigurationBuilder
|
||||
.buildConfig(configOptions)
|
||||
.catchAll(errors => TaskR.fail(ConfigValidationException(errors)))
|
||||
.flatMap(config => useValidConfig(hashService)(config))
|
||||
|
||||
private def useValidConfig(
|
||||
hashService: HashService
|
||||
)(implicit c: Config) = {
|
||||
def createPlan(hashService: HashService)
|
||||
: TaskR[Storage with Console with Config, SyncPlan] =
|
||||
for {
|
||||
_ <- SyncLogging.logRunStart(c.bucket, c.prefix, c.sources)
|
||||
_ <- SyncLogging.logRunStart
|
||||
actions <- buildPlan(hashService)
|
||||
} yield actions
|
||||
}
|
||||
|
||||
private def buildPlan(
|
||||
hashService: HashService
|
||||
)(implicit c: Config) =
|
||||
private def buildPlan(hashService: HashService) =
|
||||
for {
|
||||
metadata <- gatherMetadata(hashService)
|
||||
} yield assemblePlan(c)(metadata)
|
||||
plan <- assemblePlan(metadata)
|
||||
} yield plan
|
||||
|
||||
private def assemblePlan(
|
||||
implicit c: Config): ((S3ObjectsData, LocalFiles)) => SyncPlan = {
|
||||
private def assemblePlan(metadata: (S3ObjectsData, LocalFiles)) =
|
||||
metadata match {
|
||||
case (remoteData, localData) =>
|
||||
SyncPlan(
|
||||
actions = createActions(c)(remoteData)(localData)
|
||||
.filter(doesSomething)
|
||||
.sortBy(SequencePlan.order),
|
||||
syncTotals = SyncTotals(count = localData.count,
|
||||
totalSizeBytes = localData.totalSizeBytes)
|
||||
)
|
||||
createActions(remoteData, localData)
|
||||
.map(_.filter(doesSomething).sortBy(SequencePlan.order))
|
||||
.map(
|
||||
SyncPlan(_, SyncTotals(localData.count, localData.totalSizeBytes)))
|
||||
}
|
||||
|
||||
private def createActions
|
||||
: Config => S3ObjectsData => LocalFiles => Stream[Action] =
|
||||
c =>
|
||||
remoteData =>
|
||||
localData =>
|
||||
actionsForLocalFiles(c)(remoteData)(localData) ++
|
||||
actionsForRemoteKeys(c)(remoteData)
|
||||
private def createActions(
|
||||
remoteData: S3ObjectsData,
|
||||
localData: LocalFiles
|
||||
) =
|
||||
for {
|
||||
fileActions <- actionsForLocalFiles(remoteData, localData)
|
||||
remoteActions <- actionsForRemoteKeys(remoteData)
|
||||
} yield fileActions ++ remoteActions
|
||||
|
||||
private def doesSomething: Action => Boolean = {
|
||||
case _: DoNothing => false
|
||||
case _ => true
|
||||
}
|
||||
|
||||
private def actionsForLocalFiles
|
||||
: Config => S3ObjectsData => LocalFiles => Stream[Action] =
|
||||
c =>
|
||||
remoteData =>
|
||||
localData =>
|
||||
localData.localFiles.foldLeft(Stream.empty[Action])((acc, lf) =>
|
||||
createActionFromLocalFile(c)(lf)(remoteData)(acc) ++ acc)
|
||||
private def actionsForLocalFiles(
|
||||
remoteData: S3ObjectsData,
|
||||
localData: LocalFiles
|
||||
) =
|
||||
ZIO.foldLeft(localData.localFiles)(Stream.empty[Action])(
|
||||
(acc, localFile) =>
|
||||
createActionFromLocalFile(remoteData, acc, localFile)
|
||||
.map(actions => actions ++ acc))
|
||||
|
||||
private def createActionFromLocalFile
|
||||
: Config => LocalFile => S3ObjectsData => Stream[Action] => Stream[Action] =
|
||||
c =>
|
||||
lf =>
|
||||
remoteData =>
|
||||
previousActions =>
|
||||
private def createActionFromLocalFile(
|
||||
remoteData: S3ObjectsData,
|
||||
previousActions: Stream[Action],
|
||||
localFile: LocalFile
|
||||
) =
|
||||
ActionGenerator.createActions(
|
||||
S3MetaDataEnricher.getMetadata(lf, remoteData)(c),
|
||||
previousActions)(c)
|
||||
S3MetaDataEnricher.getMetadata(localFile, remoteData),
|
||||
previousActions)
|
||||
|
||||
private def actionsForRemoteKeys: Config => S3ObjectsData => Stream[Action] =
|
||||
c =>
|
||||
remoteData =>
|
||||
remoteData.byKey.keys.foldLeft(Stream.empty[Action])((acc, rk) =>
|
||||
createActionFromRemoteKey(c)(rk) #:: acc)
|
||||
private def actionsForRemoteKeys(remoteData: S3ObjectsData) =
|
||||
ZIO.foldLeft(remoteData.byKey.keys)(Stream.empty[Action]) {
|
||||
(acc, remoteKey) =>
|
||||
createActionFromRemoteKey(remoteKey).map(action => action #:: acc)
|
||||
}
|
||||
|
||||
private def createActionFromRemoteKey: Config => RemoteKey => Action =
|
||||
c =>
|
||||
rk =>
|
||||
if (rk.isMissingLocally(c.sources, c.prefix))
|
||||
Action.ToDelete(c.bucket, rk, 0L)
|
||||
else DoNothing(c.bucket, rk, 0L)
|
||||
private def createActionFromRemoteKey(remoteKey: RemoteKey) =
|
||||
for {
|
||||
bucket <- getBucket
|
||||
prefix <- getPrefix
|
||||
sources <- getSources
|
||||
needsDeleted = remoteKey.isMissingLocally(sources, prefix)
|
||||
} yield
|
||||
if (needsDeleted) ToDelete(bucket, remoteKey, 0L)
|
||||
else DoNothing(bucket, remoteKey, 0L)
|
||||
|
||||
private def gatherMetadata(
|
||||
hashService: HashService
|
||||
)(implicit c: Config) =
|
||||
private def gatherMetadata(hashService: HashService) =
|
||||
for {
|
||||
remoteData <- fetchRemoteData
|
||||
localData <- findLocalFiles(hashService)
|
||||
} yield (remoteData, localData)
|
||||
|
||||
private def fetchRemoteData(implicit c: Config) =
|
||||
listObjects(c.bucket, c.prefix)
|
||||
private def fetchRemoteData =
|
||||
for {
|
||||
bucket <- getBucket
|
||||
prefix <- getPrefix
|
||||
objects <- listObjects(bucket, prefix)
|
||||
} yield objects
|
||||
|
||||
private def findLocalFiles(
|
||||
hashService: HashService
|
||||
)(implicit config: Config) =
|
||||
private def findLocalFiles(hashService: HashService) =
|
||||
for {
|
||||
_ <- SyncLogging.logFileScan
|
||||
localFiles <- findFiles(hashService)
|
||||
} yield localFiles
|
||||
|
||||
private def findFiles(
|
||||
hashService: HashService
|
||||
)(implicit c: Config) = {
|
||||
Task
|
||||
.foreach(c.sources.paths)(LocalFileStream.findFiles(_, hashService))
|
||||
.map(_.foldLeft(LocalFiles())((acc, localFile) => acc ++ localFile))
|
||||
}
|
||||
private def findFiles(hashService: HashService) =
|
||||
for {
|
||||
sources <- getSources
|
||||
paths = sources.paths
|
||||
found <- ZIO.foreach(paths)(path =>
|
||||
LocalFileStream.findFiles(hashService)(path))
|
||||
} yield LocalFiles.reduce(found.toStream)
|
||||
|
||||
}
|
||||
|
||||
|
|
|
@ -7,7 +7,7 @@ object S3MetaDataEnricher {
|
|||
def getMetadata(
|
||||
localFile: LocalFile,
|
||||
s3ObjectsData: S3ObjectsData
|
||||
)(implicit c: Config): S3MetaData = {
|
||||
): S3MetaData = {
|
||||
val (keyMatches, hashMatches) = getS3Status(localFile, s3ObjectsData)
|
||||
S3MetaData(
|
||||
localFile,
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent.{
|
||||
CopyQueueEvent,
|
||||
|
@ -13,17 +14,19 @@ import zio.ZIO
|
|||
|
||||
trait SyncLogging {
|
||||
|
||||
def logRunStart(
|
||||
bucket: Bucket,
|
||||
prefix: RemoteKey,
|
||||
sources: Sources
|
||||
): ZIO[Console, Nothing, Unit] =
|
||||
def logRunStart: ZIO[Console with Config, Nothing, Unit] =
|
||||
for {
|
||||
_ <- putStrLn(ConsoleOut.ValidConfig(bucket, prefix, sources))
|
||||
bucket <- getBucket
|
||||
prefix <- getPrefix
|
||||
sources <- getSources
|
||||
_ <- putMessageLn(ConsoleOut.ValidConfig(bucket, prefix, sources))
|
||||
} yield ()
|
||||
|
||||
def logFileScan(implicit c: Config): ZIO[Console, Nothing, Unit] =
|
||||
putStrLn(s"Scanning local files: ${c.sources.paths.mkString(", ")}...")
|
||||
def logFileScan: ZIO[Config with Console, Nothing, Unit] =
|
||||
for {
|
||||
sources <- getSources
|
||||
_ <- putStrLn(s"Scanning local files: ${sources.paths.mkString(", ")}...")
|
||||
} yield ()
|
||||
|
||||
def logRunFinished(
|
||||
actions: Stream[StorageQueueEvent]
|
||||
|
|
|
@ -2,29 +2,35 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.time.Instant
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalatest.FunSpec
|
||||
import zio.DefaultRuntime
|
||||
|
||||
class ActionGeneratorSuite extends FunSpec {
|
||||
val lastModified = LastModified(Instant.now())
|
||||
private val source = Resource(this, "upload")
|
||||
private val sourcePath = source.toPath
|
||||
private val sources = Sources(List(sourcePath))
|
||||
private val prefix = RemoteKey("prefix")
|
||||
private val bucket = Bucket("bucket")
|
||||
implicit private val config: Config =
|
||||
Config(bucket, prefix, sources = Sources(List(sourcePath)))
|
||||
private val configOptions = ConfigOptions(
|
||||
List(
|
||||
ConfigOption.Bucket("bucket"),
|
||||
ConfigOption.Prefix("prefix"),
|
||||
ConfigOption.Source(sourcePath),
|
||||
ConfigOption.IgnoreUserOptions,
|
||||
ConfigOption.IgnoreGlobalOptions
|
||||
))
|
||||
private val fileToKey =
|
||||
KeyGenerator.generateKey(config.sources, config.prefix) _
|
||||
KeyGenerator.generateKey(sources, prefix) _
|
||||
|
||||
describe("create actions") {
|
||||
|
||||
val previousActions = Stream.empty[Action]
|
||||
|
||||
def invoke(input: S3MetaData) =
|
||||
ActionGenerator.createActions(input, previousActions).toList
|
||||
|
||||
describe("#1 local exists, remote exists, remote matches - do nothing") {
|
||||
val theHash = MD5Hash("the-hash")
|
||||
val theFile = LocalFile.resolve("the-file",
|
||||
|
@ -40,8 +46,9 @@ class ActionGeneratorSuite extends FunSpec {
|
|||
)
|
||||
it("do nothing") {
|
||||
val expected =
|
||||
List(DoNothing(bucket, theFile.remoteKey, theFile.file.length))
|
||||
val result = invoke(input)
|
||||
Right(
|
||||
Stream(DoNothing(bucket, theFile.remoteKey, theFile.file.length)))
|
||||
val result = invoke(input, previousActions)
|
||||
assertResult(expected)(result)
|
||||
}
|
||||
}
|
||||
|
@ -60,13 +67,14 @@ class ActionGeneratorSuite extends FunSpec {
|
|||
matchByHash = Set(otherRemoteMetadata), // other matches
|
||||
matchByKey = None) // remote is missing
|
||||
it("copy from other key") {
|
||||
val expected = List(
|
||||
val expected = Right(
|
||||
Stream(
|
||||
ToCopy(bucket,
|
||||
otherRemoteKey,
|
||||
theHash,
|
||||
theRemoteKey,
|
||||
theFile.file.length)) // copy
|
||||
val result = invoke(input)
|
||||
theFile.file.length))) // copy
|
||||
val result = invoke(input, previousActions)
|
||||
assertResult(expected)(result)
|
||||
}
|
||||
}
|
||||
|
@ -80,8 +88,9 @@ class ActionGeneratorSuite extends FunSpec {
|
|||
matchByHash = Set.empty, // other no matches
|
||||
matchByKey = None) // remote is missing
|
||||
it("upload") {
|
||||
val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload
|
||||
val result = invoke(input)
|
||||
val expected = Right(
|
||||
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
|
||||
val result = invoke(input, previousActions)
|
||||
assertResult(expected)(result)
|
||||
}
|
||||
}
|
||||
|
@ -105,13 +114,14 @@ class ActionGeneratorSuite extends FunSpec {
|
|||
matchByHash = Set(otherRemoteMetadata), // other matches
|
||||
matchByKey = Some(oldRemoteMetadata)) // remote exists
|
||||
it("copy from other key") {
|
||||
val expected = List(
|
||||
val expected = Right(
|
||||
Stream(
|
||||
ToCopy(bucket,
|
||||
otherRemoteKey,
|
||||
theHash,
|
||||
theRemoteKey,
|
||||
theFile.file.length)) // copy
|
||||
val result = invoke(input)
|
||||
theFile.file.length))) // copy
|
||||
val result = invoke(input, previousActions)
|
||||
assertResult(expected)(result)
|
||||
}
|
||||
}
|
||||
|
@ -132,8 +142,9 @@ class ActionGeneratorSuite extends FunSpec {
|
|||
matchByKey = Some(theRemoteMetadata) // remote exists
|
||||
)
|
||||
it("upload") {
|
||||
val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload
|
||||
val result = invoke(input)
|
||||
val expected = Right(
|
||||
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
|
||||
val result = invoke(input, previousActions)
|
||||
assertResult(expected)(result)
|
||||
}
|
||||
}
|
||||
|
@ -147,4 +158,23 @@ class ActionGeneratorSuite extends FunSpec {
|
|||
private def md5HashMap(theHash: MD5Hash): Map[HashType, MD5Hash] = {
|
||||
Map(MD5 -> theHash)
|
||||
}
|
||||
|
||||
private def invoke(
|
||||
input: S3MetaData,
|
||||
previousActions: Stream[Action]
|
||||
) = {
|
||||
type TestEnv = Config
|
||||
val testEnv: TestEnv = new Config.Live {}
|
||||
|
||||
def testProgram =
|
||||
for {
|
||||
config <- ConfigurationBuilder.buildConfig(configOptions)
|
||||
_ <- setConfiguration(config)
|
||||
actions <- ActionGenerator.createActions(input, previousActions)
|
||||
} yield actions
|
||||
|
||||
new DefaultRuntime {}.unsafeRunSync {
|
||||
testProgram.provide(testEnv)
|
||||
}.toEither
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,5 +1,11 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config.{
|
||||
ConfigOption,
|
||||
ConfigOptions,
|
||||
ConfigQuery,
|
||||
ConfigurationBuilder
|
||||
}
|
||||
import net.kemitix.thorp.domain.Sources
|
||||
import org.scalatest.FunSpec
|
||||
import zio.DefaultRuntime
|
||||
|
|
|
@ -2,6 +2,7 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.thorp.config.{ConfigOption, ConfigOptions, ConfigQuery}
|
||||
import net.kemitix.thorp.domain.Sources
|
||||
import org.scalatest.FreeSpec
|
||||
|
||||
|
@ -10,8 +11,8 @@ class ConfigQueryTest extends FreeSpec {
|
|||
"show version" - {
|
||||
"when is set" - {
|
||||
"should be true" in {
|
||||
val result = ConfigQuery.showVersion(ConfigOptions(List(
|
||||
ConfigOption.Version)))
|
||||
val result =
|
||||
ConfigQuery.showVersion(ConfigOptions(List(ConfigOption.Version)))
|
||||
assertResult(true)(result)
|
||||
}
|
||||
}
|
||||
|
@ -25,8 +26,8 @@ class ConfigQueryTest extends FreeSpec {
|
|||
"batch mode" - {
|
||||
"when is set" - {
|
||||
"should be true" in {
|
||||
val result = ConfigQuery.batchMode(ConfigOptions(List(
|
||||
ConfigOption.BatchMode)))
|
||||
val result =
|
||||
ConfigQuery.batchMode(ConfigOptions(List(ConfigOption.BatchMode)))
|
||||
assertResult(true)(result)
|
||||
}
|
||||
}
|
||||
|
@ -40,8 +41,8 @@ class ConfigQueryTest extends FreeSpec {
|
|||
"ignore user options" - {
|
||||
"when is set" - {
|
||||
"should be true" in {
|
||||
val result = ConfigQuery.ignoreUserOptions(ConfigOptions(List(
|
||||
ConfigOption.IgnoreUserOptions)))
|
||||
val result = ConfigQuery.ignoreUserOptions(
|
||||
ConfigOptions(List(ConfigOption.IgnoreUserOptions)))
|
||||
assertResult(true)(result)
|
||||
}
|
||||
}
|
||||
|
@ -55,8 +56,8 @@ class ConfigQueryTest extends FreeSpec {
|
|||
"ignore global options" - {
|
||||
"when is set" - {
|
||||
"should be true" in {
|
||||
val result = ConfigQuery.ignoreGlobalOptions(ConfigOptions(List(
|
||||
ConfigOption.IgnoreGlobalOptions)))
|
||||
val result = ConfigQuery.ignoreGlobalOptions(
|
||||
ConfigOptions(List(ConfigOption.IgnoreGlobalOptions)))
|
||||
assertResult(true)(result)
|
||||
}
|
||||
}
|
||||
|
@ -81,17 +82,17 @@ class ConfigQueryTest extends FreeSpec {
|
|||
"when is set once" - {
|
||||
"should have one source" in {
|
||||
val expected = Sources(List(pathA))
|
||||
val result = ConfigQuery.sources(ConfigOptions(List(
|
||||
ConfigOption.Source(pathA))))
|
||||
val result =
|
||||
ConfigQuery.sources(ConfigOptions(List(ConfigOption.Source(pathA))))
|
||||
assertResult(expected)(result)
|
||||
}
|
||||
}
|
||||
"when is set twice" - {
|
||||
"should have two sources" in {
|
||||
val expected = Sources(List(pathA, pathB))
|
||||
val result = ConfigQuery.sources(ConfigOptions(List(
|
||||
ConfigOption.Source(pathA),
|
||||
ConfigOption.Source(pathB))))
|
||||
val result = ConfigQuery.sources(
|
||||
ConfigOptions(
|
||||
List(ConfigOption.Source(pathA), ConfigOption.Source(pathB))))
|
||||
assertResult(expected)(result)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,6 +2,11 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.nio.file.{Path, Paths}
|
||||
|
||||
import net.kemitix.thorp.config.{
|
||||
ConfigOption,
|
||||
ConfigOptions,
|
||||
ConfigurationBuilder
|
||||
}
|
||||
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalatest.FunSpec
|
||||
|
|
|
@ -2,7 +2,8 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.io.File
|
||||
|
||||
import net.kemitix.thorp.domain.{Bucket, Config, RemoteKey, Sources}
|
||||
import net.kemitix.thorp.config.Resource
|
||||
import net.kemitix.thorp.domain.{RemoteKey, Sources}
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class KeyGeneratorSuite extends FunSpec {
|
||||
|
@ -10,10 +11,9 @@ class KeyGeneratorSuite extends FunSpec {
|
|||
private val source: File = Resource(this, "upload")
|
||||
private val sourcePath = source.toPath
|
||||
private val prefix = RemoteKey("prefix")
|
||||
implicit private val config: Config =
|
||||
Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
|
||||
private val sources = Sources(List(sourcePath))
|
||||
private val fileToKey =
|
||||
KeyGenerator.generateKey(config.sources, config.prefix) _
|
||||
KeyGenerator.generateKey(sources, prefix) _
|
||||
|
||||
describe("key generator") {
|
||||
|
||||
|
|
|
@ -2,11 +2,13 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
import net.kemitix.thorp.domain._
|
||||
import net.kemitix.thorp.storage.api.HashService
|
||||
import net.kemitix.thorp.storage.api.{HashService, Storage}
|
||||
import org.scalatest.FunSpec
|
||||
import zio.DefaultRuntime
|
||||
import zio.{DefaultRuntime, Task, UIO}
|
||||
|
||||
class LocalFileStreamSuite extends FunSpec {
|
||||
|
||||
|
@ -21,8 +23,13 @@ class LocalFileStreamSuite extends FunSpec {
|
|||
private def file(filename: String) =
|
||||
sourcePath.resolve(Paths.get(filename))
|
||||
|
||||
implicit private val config: Config = Config(
|
||||
sources = Sources(List(sourcePath)))
|
||||
private val configOptions = ConfigOptions(
|
||||
List(
|
||||
ConfigOption.IgnoreGlobalOptions,
|
||||
ConfigOption.IgnoreUserOptions,
|
||||
ConfigOption.Source(sourcePath),
|
||||
ConfigOption.Bucket("aBucket")
|
||||
))
|
||||
|
||||
describe("findFiles") {
|
||||
it("should find all files") {
|
||||
|
@ -47,9 +54,29 @@ class LocalFileStreamSuite extends FunSpec {
|
|||
}
|
||||
|
||||
private def invoke() = {
|
||||
val runtime = new DefaultRuntime {}
|
||||
runtime.unsafeRunSync {
|
||||
LocalFileStream.findFiles(sourcePath, hashService)
|
||||
type TestEnv = Storage with Console with Config
|
||||
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live {
|
||||
override def listResult: Task[S3ObjectsData] =
|
||||
Task.die(new NotImplementedError)
|
||||
override def uploadResult: UIO[StorageQueueEvent] =
|
||||
Task.die(new NotImplementedError)
|
||||
override def copyResult: UIO[StorageQueueEvent] =
|
||||
Task.die(new NotImplementedError)
|
||||
override def deleteResult: UIO[StorageQueueEvent] =
|
||||
Task.die(new NotImplementedError)
|
||||
override def shutdownResult: UIO[StorageQueueEvent] =
|
||||
Task.die(new NotImplementedError)
|
||||
}
|
||||
|
||||
def testProgram =
|
||||
for {
|
||||
config <- ConfigurationBuilder.buildConfig(configOptions)
|
||||
_ <- setConfiguration(config)
|
||||
files <- LocalFileStream.findFiles(hashService)(sourcePath)
|
||||
} yield files
|
||||
|
||||
new DefaultRuntime {}.unsafeRunSync {
|
||||
testProgram.provide(testEnv)
|
||||
}.toEither
|
||||
}
|
||||
|
||||
|
|
|
@ -2,8 +2,8 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.nio.file.Path
|
||||
|
||||
import net.kemitix.thorp.config.Resource
|
||||
import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root}
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalatest.FunSpec
|
||||
import zio.DefaultRuntime
|
||||
|
||||
|
@ -12,10 +12,6 @@ class MD5HashGeneratorTest extends FunSpec {
|
|||
private val runtime = new DefaultRuntime {}
|
||||
|
||||
private val source = Resource(this, "upload")
|
||||
private val sourcePath = source.toPath
|
||||
private val prefix = RemoteKey("prefix")
|
||||
implicit private val config: Config =
|
||||
Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
|
||||
|
||||
describe("md5File()") {
|
||||
describe("read a small file (smaller than buffer)") {
|
||||
|
|
|
@ -2,6 +2,12 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.nio.file.{Path, Paths}
|
||||
|
||||
import net.kemitix.thorp.config.{
|
||||
ConfigOption,
|
||||
ConfigOptions,
|
||||
ParseConfigFile,
|
||||
Resource
|
||||
}
|
||||
import org.scalatest.FunSpec
|
||||
import zio.DefaultRuntime
|
||||
|
||||
|
|
|
@ -2,6 +2,7 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.thorp.config.{ConfigOption, ConfigOptions, ParseConfigLines}
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class ParseConfigLinesTest extends FunSpec {
|
||||
|
|
|
@ -3,6 +3,7 @@ package net.kemitix.thorp.core
|
|||
import java.io.File
|
||||
import java.nio.file.Path
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
|
@ -24,7 +25,9 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
|||
val options: Path => ConfigOptions =
|
||||
source =>
|
||||
configOptions(ConfigOption.Source(source),
|
||||
ConfigOption.Bucket("a-bucket"))
|
||||
ConfigOption.Bucket("a-bucket"),
|
||||
ConfigOption.IgnoreUserOptions,
|
||||
ConfigOption.IgnoreGlobalOptions)
|
||||
"a file" - {
|
||||
val filename = "aFile"
|
||||
val remoteKey = RemoteKey(filename)
|
||||
|
@ -325,9 +328,9 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
|||
hashService: HashService,
|
||||
configOptions: ConfigOptions,
|
||||
result: Task[S3ObjectsData]
|
||||
): Either[Any, List[(String, String, String, String, String)]] = {
|
||||
type TestEnv = Storage.Test with Console.Test
|
||||
val testEnv: TestEnv = new Storage.Test with Console.Test {
|
||||
) = {
|
||||
type TestEnv = Storage with Console with Config
|
||||
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live {
|
||||
override def listResult: Task[S3ObjectsData] = result
|
||||
override def uploadResult: UIO[StorageQueueEvent] =
|
||||
Task.die(new NotImplementedError)
|
||||
|
@ -339,12 +342,15 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
|||
Task.die(new NotImplementedError)
|
||||
}
|
||||
|
||||
def testProgram =
|
||||
for {
|
||||
config <- ConfigurationBuilder.buildConfig(configOptions)
|
||||
_ <- setConfiguration(config)
|
||||
plan <- PlanBuilder.createPlan(hashService)
|
||||
} yield plan
|
||||
|
||||
new DefaultRuntime {}
|
||||
.unsafeRunSync {
|
||||
PlanBuilder
|
||||
.createPlan(hashService, configOptions)
|
||||
.provide(testEnv)
|
||||
}
|
||||
.unsafeRunSync(testProgram.provide(testEnv))
|
||||
.toEither
|
||||
.map(convertResult)
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.time.Instant
|
||||
|
||||
import net.kemitix.thorp.config.Resource
|
||||
import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
import net.kemitix.thorp.domain._
|
||||
|
@ -11,11 +12,10 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
|||
val lastModified = LastModified(Instant.now())
|
||||
private val source = Resource(this, "upload")
|
||||
private val sourcePath = source.toPath
|
||||
private val sources = Sources(List(sourcePath))
|
||||
private val prefix = RemoteKey("prefix")
|
||||
implicit private val config: Config =
|
||||
Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
|
||||
private val fileToKey =
|
||||
KeyGenerator.generateKey(config.sources, config.prefix) _
|
||||
KeyGenerator.generateKey(sources, prefix) _
|
||||
|
||||
def getMatchesByKey(
|
||||
status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
|
||||
|
|
|
@ -1,25 +0,0 @@
|
|||
package net.kemitix.thorp.domain
|
||||
|
||||
final case class Config(
|
||||
bucket: Bucket = Bucket(""),
|
||||
prefix: RemoteKey = RemoteKey(""),
|
||||
filters: List[Filter] = List(),
|
||||
debug: Boolean = false,
|
||||
batchMode: Boolean = false,
|
||||
sources: Sources = Sources(List())
|
||||
)
|
||||
|
||||
object Config {
|
||||
val sources: SimpleLens[Config, Sources] =
|
||||
SimpleLens[Config, Sources](_.sources, b => a => b.copy(sources = a))
|
||||
val bucket: SimpleLens[Config, Bucket] =
|
||||
SimpleLens[Config, Bucket](_.bucket, b => a => b.copy(bucket = a))
|
||||
val prefix: SimpleLens[Config, RemoteKey] =
|
||||
SimpleLens[Config, RemoteKey](_.prefix, b => a => b.copy(prefix = a))
|
||||
val filters: SimpleLens[Config, List[Filter]] =
|
||||
SimpleLens[Config, List[Filter]](_.filters, b => a => b.copy(filters = a))
|
||||
val debug: SimpleLens[Config, Boolean] =
|
||||
SimpleLens[Config, Boolean](_.debug, b => a => b.copy(debug = a))
|
||||
val batchMode: SimpleLens[Config, Boolean] =
|
||||
SimpleLens[Config, Boolean](_.batchMode, b => a => b.copy(batchMode = a))
|
||||
}
|
|
@ -3,7 +3,7 @@ package net.kemitix.thorp.storage.aws
|
|||
import java.nio.file.Path
|
||||
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
|
||||
import net.kemitix.thorp.core.Resource
|
||||
import net.kemitix.thorp.config.Resource
|
||||
import org.scalatest.FunSpec
|
||||
import zio.DefaultRuntime
|
||||
|
||||
|
|
|
@ -2,7 +2,8 @@ package net.kemitix.thorp.storage.aws
|
|||
|
||||
import java.time.Instant
|
||||
|
||||
import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
|
||||
import net.kemitix.thorp.config.Resource
|
||||
import net.kemitix.thorp.core.{KeyGenerator, S3MetaDataEnricher}
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalamock.scalatest.MockFactory
|
||||
|
@ -12,12 +13,10 @@ class StorageServiceSuite extends FunSpec with MockFactory {
|
|||
|
||||
private val source = Resource(this, "upload")
|
||||
private val sourcePath = source.toPath
|
||||
|
||||
private val sources = Sources(List(sourcePath))
|
||||
private val prefix = RemoteKey("prefix")
|
||||
implicit private val config: Config =
|
||||
Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
|
||||
private val fileToKey =
|
||||
KeyGenerator.generateKey(config.sources, config.prefix) _
|
||||
KeyGenerator.generateKey(sources, prefix) _
|
||||
|
||||
describe("getS3Status") {
|
||||
|
||||
|
|
|
@ -5,8 +5,8 @@ import java.io.File
|
|||
import com.amazonaws.SdkClientException
|
||||
import com.amazonaws.services.s3.model.AmazonS3Exception
|
||||
import com.amazonaws.services.s3.transfer.model.UploadResult
|
||||
import net.kemitix.thorp.config.Resource
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.core.Resource
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent.{
|
||||
Action,
|
||||
|
|
Loading…
Reference in a new issue