kemitix/thorp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Convert Config to full ZIO effect module (#134)

Browse source 
* [config] new module

* [config] stub module

* [domain] Rename domain.Config as domain.LegacyConfig

* [config] Move LegacyConfig to config module

* [config] Move config parsing and validation into module

* [config] Complete migration to module for Config

* [config] Config You should not name methods after their defining object

* [config] Rename LegacyConfig as Configuration

Also remove redundant uses

* [core] LocalFileStream Refactoring

* [changelog] update
This commit is contained in:
Paul Campbell 2019-07-28 21:47:01 +01:00 committed by GitHub
parent 96a83e6c3e
commit ccefd286f9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
43 changed files with 517 additions and 327 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.org
View file

@ -20,6 +20,7 @@ The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]
- [internal] Replace Monocle with local SimpleLens implementation (#121)
- [internal] Don't use String as key in Map for hashes (#124)
- [internal] Convert Storage to full ZIO effect module (#133)
- [internal] Convert Config to full ZIO effect module (#134)
** Dependencies

11
build.sbt
View file

@ -58,6 +58,7 @@ val zioDependencies = Seq(
)
// cli -> thorp-lib -> storage-aws -> core -> storage-api -> console -> domain
// core -> config -> domain
lazy val thorp = (project in file("."))
.settings(commonSettings)
@ -67,7 +68,6 @@ lazy val cli = (project in file("cli"))
.settings(commonSettings)
.settings(mainClass in assembly := Some("net.kemitix.thorp.cli.Main"))
.settings(applicationSettings)
.settings(commandLineParsing)
.settings(testDependencies)
.enablePlugins(BuildInfoPlugin)
.settings(
@ -101,6 +101,7 @@ lazy val core = (project in file("core"))
.settings(testDependencies)
.dependsOn(`storage-api`)
.dependsOn(domain % "compile->compile;test->test")
.dependsOn(config)
lazy val `storage-api` = (project in file("storage-api"))
.settings(commonSettings)
@ -114,6 +115,14 @@ lazy val console = (project in file("console"))
.settings(assemblyJarName in assembly := "console.jar")
.dependsOn(domain)
lazy val config = (project in file("config"))
.settings(commonSettings)
.settings(zioDependencies)
.settings(testDependencies)
.settings(commandLineParsing)
.settings(assemblyJarName in assembly := "config.jar")
.dependsOn(domain)
lazy val domain = (project in file("domain"))
.settings(commonSettings)
.settings(assemblyJarName in assembly := "domain.jar")

3
cli/src/main/scala/net/kemitix/thorp/cli/Main.scala
View file

@ -1,12 +1,13 @@
package net.kemitix.thorp.cli
import net.kemitix.thorp.config.Config
import net.kemitix.thorp.console.Console
import net.kemitix.thorp.storage.aws.S3Storage
import zio.{App, ZIO}
object Main extends App {
object LiveThorpApp extends S3Storage.Live with Console.Live
object LiveThorpApp extends S3Storage.Live with Console.Live with Config.Live
override def run(args: List[String]): ZIO[Environment, Nothing, Int] =
Program

20
cli/src/main/scala/net/kemitix/thorp/cli/Program.scala
View file

@ -1,11 +1,12 @@
package net.kemitix.thorp.cli
import net.kemitix.thorp.config._
import net.kemitix.thorp.console._
import net.kemitix.thorp.core.CoreTypes.CoreProgram
import net.kemitix.thorp.core._
import net.kemitix.thorp.domain.StorageQueueEvent
import net.kemitix.thorp.storage.aws.S3HashService.defaultHashService
import zio.{UIO, ZIO}
import zio.ZIO
trait Program {
@ -13,28 +14,27 @@ trait Program {
def run(args: List[String]): CoreProgram[Unit] = {
for {
cli <- CliArgs.parse(args)
_ <- ZIO.when(showVersion(cli))(putStrLn(version))
_ <- ZIO.when(!showVersion(cli))(execute(cli).catchAll(handleErrors))
cli <- CliArgs.parse(args)
config <- ConfigurationBuilder.buildConfig(cli)
_ <- setConfiguration(config)
_ <- ZIO.when(showVersion(cli))(putStrLn(version))
_ <- ZIO.when(!showVersion(cli))(execute.catchAll(handleErrors))
} yield ()
}
private def showVersion: ConfigOptions => Boolean =
cli => ConfigQuery.showVersion(cli)
private def execute(cliOptions: ConfigOptions) = {
private def execute = {
for {
plan <- PlanBuilder.createPlan(defaultHashService, cliOptions)
batchMode <- isBatchMode(cliOptions)
plan <- PlanBuilder.createPlan(defaultHashService)
batchMode <- isBatchMode
archive <- UnversionedMirrorArchive.default(batchMode, plan.syncTotals)
events <- applyPlan(archive, plan)
_ <- SyncLogging.logRunFinished(events)
} yield ()
}
private def isBatchMode(cliOptions: ConfigOptions) =
UIO(ConfigQuery.batchMode(cliOptions))
private def handleErrors(throwable: Throwable) =
for {
_ <- putStrLn("There were errors:")

3
cli/src/main/scala/net/kemitix/thorp/cli/CliArgs.scala → config/src/main/scala/net/kemitix/thorp/config/CliArgs.scala
View file

@ -1,8 +1,7 @@
package net.kemitix.thorp.cli
package net.kemitix.thorp.config
import java.nio.file.Paths
import net.kemitix.thorp.core.{ConfigOption, ConfigOptions}
import scopt.OParser
import zio.Task

50
config/src/main/scala/net/kemitix/thorp/config/Config.scala Normal file
View file

@ -0,0 +1,50 @@
package net.kemitix.thorp.config
import java.util.concurrent.atomic.AtomicReference
import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, Sources}
import zio.{UIO, ZIO}
trait Config {
val config: Config.Service
}
object Config {
trait Service {
def setConfiguration(config: Configuration): ZIO[Config, Nothing, Unit]
def isBatchMode: ZIO[Config, Nothing, Boolean]
def bucket: ZIO[Config, Nothing, Bucket]
def prefix: ZIO[Config, Nothing, RemoteKey]
def sources: ZIO[Config, Nothing, Sources]
def filters: ZIO[Config, Nothing, List[Filter]]
}
trait Live extends Config {
val config: Service = new Service {
private val configRef = new AtomicReference(Configuration())
override def setConfiguration(
config: Configuration): ZIO[Config, Nothing, Unit] =
UIO(configRef.set(config))
override def bucket: ZIO[Config, Nothing, Bucket] =
UIO(configRef.get).map(_.bucket)
override def sources: ZIO[Config, Nothing, Sources] =
UIO(configRef.get).map(_.sources)
override def prefix: ZIO[Config, Nothing, RemoteKey] =
UIO(configRef.get).map(_.prefix)
override def isBatchMode: ZIO[Config, Nothing, Boolean] =
UIO(configRef.get).map(_.batchMode)
override def filters: ZIO[Config, Nothing, List[Filter]] =
UIO(configRef.get).map(_.filters)
}
}
object Live extends Live
}

28
core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala → config/src/main/scala/net/kemitix/thorp/config/ConfigOption.scala
View file

@ -1,24 +1,24 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import java.nio.file.Path
import net.kemitix.thorp.config.Configuration._
import net.kemitix.thorp.domain
import net.kemitix.thorp.domain.{Config, RemoteKey}
import net.kemitix.thorp.domain.Config._
import net.kemitix.thorp.domain.RemoteKey
sealed trait ConfigOption {
def update(config: Config): Config
def update(config: Configuration): Configuration
}
object ConfigOption {
case class Source(path: Path) extends ConfigOption {
override def update(config: Config): Config =
override def update(config: Configuration): Configuration =
sources.modify(_ ++ path)(config)
}
case class Bucket(name: String) extends ConfigOption {
override def update(config: Config): Config =
override def update(config: Configuration): Configuration =
if (config.bucket.name.isEmpty)
bucket.set(domain.Bucket(name))(config)
else
@ -26,7 +26,7 @@ object ConfigOption {
}
case class Prefix(path: String) extends ConfigOption {
override def update(config: Config): Config =
override def update(config: Configuration): Configuration =
if (config.prefix.key.isEmpty)
prefix.set(RemoteKey(path))(config)
else
@ -34,35 +34,35 @@ object ConfigOption {
}
case class Include(pattern: String) extends ConfigOption {
override def update(config: Config): Config =
override def update(config: Configuration): Configuration =
filters.modify(domain.Filter.Include(pattern) :: _)(config)
}
case class Exclude(pattern: String) extends ConfigOption {
override def update(config: Config): Config =
override def update(config: Configuration): Configuration =
filters.modify(domain.Filter.Exclude(pattern) :: _)(config)
}
case class Debug() extends ConfigOption {
override def update(config: Config): Config =
override def update(config: Configuration): Configuration =
debug.set(true)(config)
}
case object Version extends ConfigOption {
override def update(config: Config): Config = config
override def update(config: Configuration): Configuration = config
}
case object BatchMode extends ConfigOption {
override def update(config: Config): Config =
override def update(config: Configuration): Configuration =
batchMode.set(true)(config)
}
case object IgnoreUserOptions extends ConfigOption {
override def update(config: Config): Config = config
override def update(config: Configuration): Configuration = config
}
case object IgnoreGlobalOptions extends ConfigOption {
override def update(config: Config): Config = config
override def update(config: Configuration): Configuration = config
}
}

2
core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala → config/src/main/scala/net/kemitix/thorp/config/ConfigOptions.scala
View file

@ -1,4 +1,4 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import net.kemitix.thorp.domain.SimpleLens

6
core/src/main/scala/net/kemitix/thorp/core/ConfigQuery.scala → config/src/main/scala/net/kemitix/thorp/config/ConfigQuery.scala
View file

@ -1,4 +1,4 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import java.nio.file.Paths
@ -21,11 +21,11 @@ trait ConfigQuery {
def sources(configOptions: ConfigOptions): Sources = {
val paths = configOptions.options.flatMap {
case ConfigOption.Source(sourcePath) => Some(sourcePath)
case _ => None
case _ => None
}
Sources(paths match {
case List() => List(Paths.get(System.getenv("PWD")))
case _ => paths
case _ => paths
})
}

2
core/src/main/scala/net/kemitix/thorp/core/ConfigValidation.scala → config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala
View file

@ -1,4 +1,4 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import java.nio.file.Path

2
core/src/main/scala/net/kemitix/thorp/core/ConfigValidationException.scala → config/src/main/scala/net/kemitix/thorp/config/ConfigValidationException.scala
View file

@ -1,4 +1,4 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
final case class ConfigValidationException(
errors: List[ConfigValidation]

8
core/src/main/scala/net/kemitix/thorp/core/ConfigValidator.scala → config/src/main/scala/net/kemitix/thorp/config/ConfigValidator.scala
View file

@ -1,15 +1,15 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import java.nio.file.Path
import net.kemitix.thorp.domain.{Bucket, Config, Sources}
import net.kemitix.thorp.domain.{Bucket, Sources}
import zio.IO
sealed trait ConfigValidator {
def validateConfig(
config: Config
): IO[List[ConfigValidation], Config] = IO.fromEither {
config: Configuration
): IO[List[ConfigValidation], Configuration] = IO.fromEither {
for {
_ <- validateSources(config.sources)
_ <- validateBucket(config.bucket)

29
config/src/main/scala/net/kemitix/thorp/config/Configuration.scala Normal file
View file

@ -0,0 +1,29 @@
package net.kemitix.thorp.config
import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, SimpleLens, Sources}
private[config] final case class Configuration(
bucket: Bucket = Bucket(""),
prefix: RemoteKey = RemoteKey(""),
filters: List[Filter] = List(),
debug: Boolean = false,
batchMode: Boolean = false,
sources: Sources = Sources(List())
)
private[config] object Configuration {
val sources: SimpleLens[Configuration, Sources] =
SimpleLens[Configuration, Sources](_.sources, b => a => b.copy(sources = a))
val bucket: SimpleLens[Configuration, Bucket] =
SimpleLens[Configuration, Bucket](_.bucket, b => a => b.copy(bucket = a))
val prefix: SimpleLens[Configuration, RemoteKey] =
SimpleLens[Configuration, RemoteKey](_.prefix, b => a => b.copy(prefix = a))
val filters: SimpleLens[Configuration, List[Filter]] =
SimpleLens[Configuration, List[Filter]](_.filters,
b => a => b.copy(filters = a))
val debug: SimpleLens[Configuration, Boolean] =
SimpleLens[Configuration, Boolean](_.debug, b => a => b.copy(debug = a))
val batchMode: SimpleLens[Configuration, Boolean] =
SimpleLens[Configuration, Boolean](_.batchMode,
b => a => b.copy(batchMode = a))
}

29
core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala → config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala
View file

@ -1,12 +1,8 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import java.nio.file.Paths
import net.kemitix.thorp.core.ConfigOptions.options
import net.kemitix.thorp.core.ConfigValidator.validateConfig
import net.kemitix.thorp.core.ParseConfigFile.parseFile
import net.kemitix.thorp.domain.Config
import zio.IO
import zio.{IO, TaskR}
/**
* Builds a configuration from settings in a file within the
@ -18,12 +14,13 @@ trait ConfigurationBuilder {
private val globalConfig = Paths.get("/etc/thorp.conf")
private val userHome = Paths.get(System.getProperty("user.home"))
def buildConfig(
priorityOpts: ConfigOptions): IO[List[ConfigValidation], Config] =
for {
def buildConfig(priorityOpts: ConfigOptions)
: IO[ConfigValidationException, Configuration] =
(for {
config <- getConfigOptions(priorityOpts).map(collateOptions)
valid <- validateConfig(config)
} yield valid
valid <- ConfigValidator.validateConfig(config)
} yield valid)
.catchAll(errors => TaskR.fail(ConfigValidationException(errors)))
private def getConfigOptions(
priorityOpts: ConfigOptions): IO[List[ConfigValidation], ConfigOptions] =
@ -39,17 +36,17 @@ trait ConfigurationBuilder {
private def userOptions(
priorityOpts: ConfigOptions): IO[List[ConfigValidation], ConfigOptions] =
if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
else parseFile(userHome.resolve(userConfigFilename))
else ParseConfigFile.parseFile(userHome.resolve(userConfigFilename))
private def globalOptions(
priorityOpts: ConfigOptions): IO[List[ConfigValidation], ConfigOptions] =
if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig
else parseFile(globalConfig)
else ParseConfigFile.parseFile(globalConfig)
private def collateOptions(configOptions: ConfigOptions): Config =
options
private def collateOptions(configOptions: ConfigOptions): Configuration =
ConfigOptions.options
.get(configOptions)
.foldLeft(Config()) { (config, configOption) =>
.foldLeft(Configuration()) { (config, configOption) =>
configOption.update(config)
}

2
core/src/main/scala/net/kemitix/thorp/core/ParseConfigFile.scala → config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala
View file

@ -1,4 +1,4 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import java.nio.file.{Files, Path}

11
core/src/main/scala/net/kemitix/thorp/core/ParseConfigLines.scala → config/src/main/scala/net/kemitix/thorp/config/ParseConfigLines.scala
View file

@ -1,9 +1,16 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import java.nio.file.Paths
import java.util.regex.Pattern
import net.kemitix.thorp.core.ConfigOption._
import net.kemitix.thorp.config.ConfigOption.{
Bucket,
Debug,
Exclude,
Include,
Prefix,
Source
}
trait ParseConfigLines {

2
core/src/main/scala/net/kemitix/thorp/core/Resource.scala → config/src/main/scala/net/kemitix/thorp/config/Resource.scala
View file

@ -1,4 +1,4 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import java.io.{File, FileNotFoundException}

2
core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala → config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala
View file

@ -1,4 +1,4 @@
package net.kemitix.thorp.core
package net.kemitix.thorp.config
import net.kemitix.thorp.domain.Sources
import zio.IO

29
config/src/main/scala/net/kemitix/thorp/config/package.scala Normal file
View file

@ -0,0 +1,29 @@
package net.kemitix.thorp
import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, Sources}
import zio.ZIO
package object config {
final val configService: ZIO[Config, Nothing, Config.Service] =
ZIO.access(_.config)
final def setConfiguration(
config: Configuration): ZIO[Config, Nothing, Unit] =
ZIO.accessM(_.config setConfiguration config)
final def isBatchMode: ZIO[Config, Nothing, Boolean] =
ZIO.accessM(_.config isBatchMode)
final def getBucket: ZIO[Config, Nothing, Bucket] =
ZIO.accessM(_.config bucket)
final def getPrefix: ZIO[Config, Nothing, RemoteKey] =
ZIO.accessM(_.config prefix)
final def getSources: ZIO[Config, Nothing, Sources] =
ZIO.accessM(_.config sources)
final def getFilters: ZIO[Config, Nothing, List[Filter]] =
ZIO.accessM(_.config filters)
}

5
cli/src/test/scala/net/kemitix/thorp/cli/CliArgsTest.scala → config/src/test/scala/net/kemitix/thorp/config/CliArgsTest.scala
View file

@ -1,9 +1,8 @@
package net.kemitix.thorp.cli
package net.kemitix.thorp.config
import java.nio.file.Paths
import net.kemitix.thorp.core.ConfigOption.Debug
import net.kemitix.thorp.core.{ConfigOptions, ConfigQuery, Resource}
import net.kemitix.thorp.config.ConfigOption.Debug
import org.scalatest.FunSpec
import zio.DefaultRuntime

2
console/src/main/scala/net/kemitix/thorp/console/package.scala
View file

@ -10,7 +10,7 @@ package object console {
final def putStrLn(line: String): ZIO[Console, Nothing, Unit] =
ZIO.accessM(_.console putStrLn line)
final def putStrLn(line: ConsoleOut): ZIO[Console, Nothing, Unit] =
final def putMessageLn(line: ConsoleOut): ZIO[Console, Nothing, Unit] =
ZIO.accessM(_.console putStrLn line)
}

60
core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala
View file

@ -1,39 +1,47 @@
package net.kemitix.thorp.core
import net.kemitix.thorp.config._
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
import net.kemitix.thorp.domain._
import zio.ZIO
object ActionGenerator {
def createActions(
s3MetaData: S3MetaData,
previousActions: Stream[Action]
)(implicit c: Config): Stream[Action] =
s3MetaData match {
// #1 local exists, remote exists, remote matches - do nothing
case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
if localFile.matches(hash) =>
doNothing(c.bucket, key)
// #2 local exists, remote is missing, other matches - copy
case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
copyFile(c.bucket, localFile, matchByHash)
// #3 local exists, remote is missing, other no matches - upload
case S3MetaData(localFile, matchByHash, None)
if matchByHash.isEmpty &&
isUploadAlreadyQueued(previousActions)(localFile) =>
uploadFile(c.bucket, localFile)
// #4 local exists, remote exists, remote no match, other matches - copy
case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _)))
if !localFile.matches(hash) &&
matchByHash.nonEmpty =>
copyFile(c.bucket, localFile, matchByHash)
// #5 local exists, remote exists, remote no match, other no matches - upload
case S3MetaData(localFile, matchByHash, Some(_)) if matchByHash.isEmpty =>
uploadFile(c.bucket, localFile)
// fallback
case S3MetaData(localFile, _, _) =>
doNothing(c.bucket, localFile.remoteKey)
}
): ZIO[Config, Nothing, Stream[Action]] =
for {
bucket <- getBucket
} yield
s3MetaData match {
// #1 local exists, remote exists, remote matches - do nothing
case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
if localFile.matches(hash) =>
doNothing(bucket, key)
// #2 local exists, remote is missing, other matches - copy
case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
copyFile(bucket, localFile, matchByHash)
// #3 local exists, remote is missing, other no matches - upload
case S3MetaData(localFile, matchByHash, None)
if matchByHash.isEmpty &&
isUploadAlreadyQueued(previousActions)(localFile) =>
uploadFile(bucket, localFile)
// #4 local exists, remote exists, remote no match, other matches - copy
case S3MetaData(localFile,
matchByHash,
Some(RemoteMetaData(_, hash, _)))
if !localFile.matches(hash) &&
matchByHash.nonEmpty =>
copyFile(bucket, localFile, matchByHash)
// #5 local exists, remote exists, remote no match, other no matches - upload
case S3MetaData(localFile, matchByHash, Some(_))
if matchByHash.isEmpty =>
uploadFile(bucket, localFile)
// fallback
case S3MetaData(localFile, _, _) =>
doNothing(bucket, localFile.remoteKey)
}
private def key = LocalFile.remoteKey ^|-> RemoteKey.key

3
core/src/main/scala/net/kemitix/thorp/core/CoreTypes.scala
View file

@ -1,12 +1,13 @@
package net.kemitix.thorp.core
import net.kemitix.thorp.config.Config
import net.kemitix.thorp.console.Console
import net.kemitix.thorp.storage.api.Storage
import zio.ZIO
object CoreTypes {
type CoreEnv = Storage with Console
type CoreEnv = Storage with Console with Config
type CoreProgram[A] = ZIO[CoreEnv, Throwable, A]
}

100
core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala
View file

@ -2,42 +2,30 @@ package net.kemitix.thorp.core
import java.nio.file.Path
import net.kemitix.thorp.config._
import net.kemitix.thorp.core.KeyGenerator.generateKey
import net.kemitix.thorp.domain
import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.api.HashService
import zio.Task
import zio.{Task, TaskR, ZIO}
object LocalFileStream {
def findFiles(
source: Path,
hashService: HashService
)(
implicit c: Config
): Task[LocalFiles] = {
def findFiles(hashService: HashService)(
source: Path
): TaskR[Config, LocalFiles] = {
val isIncluded: Path => Boolean = Filter.isIncluded(c.filters)
def recurseIntoSubDirectories(path: Path): TaskR[Config, LocalFiles] =
path.toFile match {
case f if f.isDirectory => loop(path)
case _ => pathToLocalFile(hashService)(path)
}
val pathToLocalFile: Path => Task[LocalFiles] = path =>
localFile(hashService, c)(path)
def recurse(paths: Stream[Path]): TaskR[Config, LocalFiles] =
for {
recursed <- ZIO.foreach(paths)(path => recurseIntoSubDirectories(path))
} yield LocalFiles.reduce(recursed.toStream)
def loop(path: Path): Task[LocalFiles] = {
def dirPaths(path: Path): Task[Stream[Path]] =
listFiles(path)
.map(_.filter(isIncluded))
def recurseIntoSubDirectories(path: Path): Task[LocalFiles] =
path.toFile match {
case f if f.isDirectory => loop(path)
case _ => pathToLocalFile(path)
}
def recurse(paths: Stream[Path]): Task[LocalFiles] =
Task.foldLeft(paths)(LocalFiles())((acc, path) => {
recurseIntoSubDirectories(path).map(localFiles => acc ++ localFiles)
})
def loop(path: Path): TaskR[Config, LocalFiles] = {
for {
paths <- dirPaths(path)
@ -48,30 +36,50 @@ object LocalFileStream {
loop(source)
}
def localFile(
hashService: HashService,
c: Config
): Path => Task[LocalFiles] =
path => {
val file = path.toFile
val source = c.sources.forPath(path)
for {
hash <- hashService.hashLocalObject(path)
} yield
LocalFiles(localFiles = Stream(
domain.LocalFile(file,
source.toFile,
hash,
generateKey(c.sources, c.prefix)(path))),
count = 1,
totalSizeBytes = file.length)
}
private def dirPaths(path: Path) =
for {
paths <- listFiles(path)
filtered <- includedDirPaths(paths)
} yield filtered
private def listFiles(path: Path): Task[Stream[Path]] =
private def includedDirPaths(paths: Stream[Path]) =
for {
flaggedPaths <- TaskR.foreach(paths)(path =>
isIncluded(path).map((path, _)))
} yield
flaggedPaths.toStream
.filter({ case (_, included) => included })
.map({ case (path, _) => path })
private def localFile(hashService: HashService)(path: Path) = {
val file = path.toFile
for {
sources <- getSources
prefix <- getPrefix
hash <- hashService.hashLocalObject(path)
localFile = LocalFile(file,
sources.forPath(path).toFile,
hash,
generateKey(sources, prefix)(path))
} yield
LocalFiles(localFiles = Stream(localFile),
count = 1,
totalSizeBytes = file.length)
}
private def listFiles(path: Path) =
for {
files <- Task(path.toFile.listFiles)
_ <- Task.when(files == null)(
Task.fail(new IllegalArgumentException(s"Directory not found $path")))
} yield Stream(files: _*).map(_.toPath)
private def isIncluded(path: Path) =
for {
filters <- getFilters
} yield Filter.isIncluded(filters)(path)
private def pathToLocalFile(hashService: HashService)(path: Path) =
localFile(hashService)(path)
}

7
core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala
View file

@ -15,3 +15,10 @@ case class LocalFiles(
)
}
object LocalFiles {
def reduce: Stream[LocalFiles] => LocalFiles =
list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf)
}

148
core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala
View file

@ -1,121 +1,111 @@
package net.kemitix.thorp.core
import net.kemitix.thorp.config._
import net.kemitix.thorp.console._
import net.kemitix.thorp.core.Action._
import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage._
import net.kemitix.thorp.storage.api.{HashService, Storage}
import zio.{Task, TaskR}
import zio.{TaskR, ZIO}
trait PlanBuilder {
def createPlan(
hashService: HashService,
configOptions: ConfigOptions
): TaskR[Storage with Console, SyncPlan] =
ConfigurationBuilder
.buildConfig(configOptions)
.catchAll(errors => TaskR.fail(ConfigValidationException(errors)))
.flatMap(config => useValidConfig(hashService)(config))
private def useValidConfig(
hashService: HashService
)(implicit c: Config) = {
def createPlan(hashService: HashService)
: TaskR[Storage with Console with Config, SyncPlan] =
for {
_ <- SyncLogging.logRunStart(c.bucket, c.prefix, c.sources)
_ <- SyncLogging.logRunStart
actions <- buildPlan(hashService)
} yield actions
}
private def buildPlan(
hashService: HashService
)(implicit c: Config) =
private def buildPlan(hashService: HashService) =
for {
metadata <- gatherMetadata(hashService)
} yield assemblePlan(c)(metadata)
plan <- assemblePlan(metadata)
} yield plan
private def assemblePlan(
implicit c: Config): ((S3ObjectsData, LocalFiles)) => SyncPlan = {
case (remoteData, localData) =>
SyncPlan(
actions = createActions(c)(remoteData)(localData)
.filter(doesSomething)
.sortBy(SequencePlan.order),
syncTotals = SyncTotals(count = localData.count,
totalSizeBytes = localData.totalSizeBytes)
)
}
private def assemblePlan(metadata: (S3ObjectsData, LocalFiles)) =
metadata match {
case (remoteData, localData) =>
createActions(remoteData, localData)
.map(_.filter(doesSomething).sortBy(SequencePlan.order))
.map(
SyncPlan(_, SyncTotals(localData.count, localData.totalSizeBytes)))
}
private def createActions
: Config => S3ObjectsData => LocalFiles => Stream[Action] =
c =>
remoteData =>
localData =>
actionsForLocalFiles(c)(remoteData)(localData) ++
actionsForRemoteKeys(c)(remoteData)
private def createActions(
remoteData: S3ObjectsData,
localData: LocalFiles
) =
for {
fileActions <- actionsForLocalFiles(remoteData, localData)
remoteActions <- actionsForRemoteKeys(remoteData)
} yield fileActions ++ remoteActions
private def doesSomething: Action => Boolean = {
case _: DoNothing => false
case _ => true
}
private def actionsForLocalFiles
: Config => S3ObjectsData => LocalFiles => Stream[Action] =
c =>
remoteData =>
localData =>
localData.localFiles.foldLeft(Stream.empty[Action])((acc, lf) =>
createActionFromLocalFile(c)(lf)(remoteData)(acc) ++ acc)
private def actionsForLocalFiles(
remoteData: S3ObjectsData,
localData: LocalFiles
) =
ZIO.foldLeft(localData.localFiles)(Stream.empty[Action])(
(acc, localFile) =>
createActionFromLocalFile(remoteData, acc, localFile)
.map(actions => actions ++ acc))
private def createActionFromLocalFile
: Config => LocalFile => S3ObjectsData => Stream[Action] => Stream[Action] =
c =>
lf =>
remoteData =>
previousActions =>
ActionGenerator.createActions(
S3MetaDataEnricher.getMetadata(lf, remoteData)(c),
previousActions)(c)
private def createActionFromLocalFile(
remoteData: S3ObjectsData,
previousActions: Stream[Action],
localFile: LocalFile
) =
ActionGenerator.createActions(
S3MetaDataEnricher.getMetadata(localFile, remoteData),
previousActions)
private def actionsForRemoteKeys: Config => S3ObjectsData => Stream[Action] =
c =>
remoteData =>
remoteData.byKey.keys.foldLeft(Stream.empty[Action])((acc, rk) =>
createActionFromRemoteKey(c)(rk) #:: acc)
private def actionsForRemoteKeys(remoteData: S3ObjectsData) =
ZIO.foldLeft(remoteData.byKey.keys)(Stream.empty[Action]) {
(acc, remoteKey) =>
createActionFromRemoteKey(remoteKey).map(action => action #:: acc)
}
private def createActionFromRemoteKey: Config => RemoteKey => Action =
c =>
rk =>
if (rk.isMissingLocally(c.sources, c.prefix))
Action.ToDelete(c.bucket, rk, 0L)
else DoNothing(c.bucket, rk, 0L)
private def createActionFromRemoteKey(remoteKey: RemoteKey) =
for {
bucket <- getBucket
prefix <- getPrefix
sources <- getSources
needsDeleted = remoteKey.isMissingLocally(sources, prefix)
} yield
if (needsDeleted) ToDelete(bucket, remoteKey, 0L)
else DoNothing(bucket, remoteKey, 0L)
private def gatherMetadata(
hashService: HashService
)(implicit c: Config) =
private def gatherMetadata(hashService: HashService) =
for {
remoteData <- fetchRemoteData
localData <- findLocalFiles(hashService)
} yield (remoteData, localData)
private def fetchRemoteData(implicit c: Config) =
listObjects(c.bucket, c.prefix)
private def fetchRemoteData =
for {
bucket <- getBucket
prefix <- getPrefix
objects <- listObjects(bucket, prefix)
} yield objects
private def findLocalFiles(
hashService: HashService
)(implicit config: Config) =
private def findLocalFiles(hashService: HashService) =
for {
_ <- SyncLogging.logFileScan
localFiles <- findFiles(hashService)
} yield localFiles
private def findFiles(
hashService: HashService
)(implicit c: Config) = {
Task
.foreach(c.sources.paths)(LocalFileStream.findFiles(_, hashService))
.map(_.foldLeft(LocalFiles())((acc, localFile) => acc ++ localFile))
}
private def findFiles(hashService: HashService) =
for {
sources <- getSources
paths = sources.paths
found <- ZIO.foreach(paths)(path =>
LocalFileStream.findFiles(hashService)(path))
} yield LocalFiles.reduce(found.toStream)
}

2
core/src/main/scala/net/kemitix/thorp/core/S3MetaDataEnricher.scala
View file

@ -7,7 +7,7 @@ object S3MetaDataEnricher {
def getMetadata(
localFile: LocalFile,
s3ObjectsData: S3ObjectsData
)(implicit c: Config): S3MetaData = {
): S3MetaData = {
val (keyMatches, hashMatches) = getS3Status(localFile, s3ObjectsData)
S3MetaData(
localFile,

19
core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala
View file

@ -1,5 +1,6 @@
package net.kemitix.thorp.core
import net.kemitix.thorp.config._
import net.kemitix.thorp.console._
import net.kemitix.thorp.domain.StorageQueueEvent.{
CopyQueueEvent,
@ -13,17 +14,19 @@ import zio.ZIO
trait SyncLogging {
def logRunStart(
bucket: Bucket,
prefix: RemoteKey,
sources: Sources
): ZIO[Console, Nothing, Unit] =
def logRunStart: ZIO[Console with Config, Nothing, Unit] =
for {
_ <- putStrLn(ConsoleOut.ValidConfig(bucket, prefix, sources))
bucket <- getBucket
prefix <- getPrefix
sources <- getSources
_ <- putMessageLn(ConsoleOut.ValidConfig(bucket, prefix, sources))
} yield ()
def logFileScan(implicit c: Config): ZIO[Console, Nothing, Unit] =
putStrLn(s"Scanning local files: ${c.sources.paths.mkString(", ")}...")
def logFileScan: ZIO[Config with Console, Nothing, Unit] =
for {
sources <- getSources
_ <- putStrLn(s"Scanning local files: ${sources.paths.mkString(", ")}...")
} yield ()
def logRunFinished(
actions: Stream[StorageQueueEvent]

82
core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala
View file

@ -2,29 +2,35 @@ package net.kemitix.thorp.core
import java.time.Instant
import net.kemitix.thorp.config._
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
import org.scalatest.FunSpec
import zio.DefaultRuntime
class ActionGeneratorSuite extends FunSpec {
val lastModified = LastModified(Instant.now())
private val source = Resource(this, "upload")
private val sourcePath = source.toPath
private val sources = Sources(List(sourcePath))
private val prefix = RemoteKey("prefix")
private val bucket = Bucket("bucket")
implicit private val config: Config =
Config(bucket, prefix, sources = Sources(List(sourcePath)))
private val configOptions = ConfigOptions(
List(
ConfigOption.Bucket("bucket"),
ConfigOption.Prefix("prefix"),
ConfigOption.Source(sourcePath),
ConfigOption.IgnoreUserOptions,
ConfigOption.IgnoreGlobalOptions
))
private val fileToKey =
KeyGenerator.generateKey(config.sources, config.prefix) _
KeyGenerator.generateKey(sources, prefix) _
describe("create actions") {
val previousActions = Stream.empty[Action]
def invoke(input: S3MetaData) =
ActionGenerator.createActions(input, previousActions).toList
describe("#1 local exists, remote exists, remote matches - do nothing") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
@ -40,8 +46,9 @@ class ActionGeneratorSuite extends FunSpec {
)
it("do nothing") {
val expected =
List(DoNothing(bucket, theFile.remoteKey, theFile.file.length))
val result = invoke(input)
Right(
Stream(DoNothing(bucket, theFile.remoteKey, theFile.file.length)))
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
}
@ -60,13 +67,14 @@ class ActionGeneratorSuite extends FunSpec {
matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = None) // remote is missing
it("copy from other key") {
val expected = List(
ToCopy(bucket,
otherRemoteKey,
theHash,
theRemoteKey,
theFile.file.length)) // copy
val result = invoke(input)
val expected = Right(
Stream(
ToCopy(bucket,
otherRemoteKey,
theHash,
theRemoteKey,
theFile.file.length))) // copy
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
}
@ -80,8 +88,9 @@ class ActionGeneratorSuite extends FunSpec {
matchByHash = Set.empty, // other no matches
matchByKey = None) // remote is missing
it("upload") {
val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload
val result = invoke(input)
val expected = Right(
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
}
@ -105,13 +114,14 @@ class ActionGeneratorSuite extends FunSpec {
matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = Some(oldRemoteMetadata)) // remote exists
it("copy from other key") {
val expected = List(
ToCopy(bucket,
otherRemoteKey,
theHash,
theRemoteKey,
theFile.file.length)) // copy
val result = invoke(input)
val expected = Right(
Stream(
ToCopy(bucket,
otherRemoteKey,
theHash,
theRemoteKey,
theFile.file.length))) // copy
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
}
@ -132,8 +142,9 @@ class ActionGeneratorSuite extends FunSpec {
matchByKey = Some(theRemoteMetadata) // remote exists
)
it("upload") {
val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload
val result = invoke(input)
val expected = Right(
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
}
@ -147,4 +158,23 @@ class ActionGeneratorSuite extends FunSpec {
private def md5HashMap(theHash: MD5Hash): Map[HashType, MD5Hash] = {
Map(MD5 -> theHash)
}
private def invoke(
input: S3MetaData,
previousActions: Stream[Action]
) = {
type TestEnv = Config
val testEnv: TestEnv = new Config.Live {}
def testProgram =
for {
config <- ConfigurationBuilder.buildConfig(configOptions)
_ <- setConfiguration(config)
actions <- ActionGenerator.createActions(input, previousActions)
} yield actions
new DefaultRuntime {}.unsafeRunSync {
testProgram.provide(testEnv)
}.toEither
}
}

6
core/src/test/scala/net/kemitix/thorp/core/ConfigOptionTest.scala
View file

@ -1,5 +1,11 @@
package net.kemitix.thorp.core
import net.kemitix.thorp.config.{
ConfigOption,
ConfigOptions,
ConfigQuery,
ConfigurationBuilder
}
import net.kemitix.thorp.domain.Sources
import org.scalatest.FunSpec
import zio.DefaultRuntime

31
core/src/test/scala/net/kemitix/thorp/core/ConfigQueryTest.scala
View file

@ -2,6 +2,7 @@ package net.kemitix.thorp.core
import java.nio.file.Paths
import net.kemitix.thorp.config.{ConfigOption, ConfigOptions, ConfigQuery}
import net.kemitix.thorp.domain.Sources
import org.scalatest.FreeSpec
@ -10,8 +11,8 @@ class ConfigQueryTest extends FreeSpec {
"show version" - {
"when is set" - {
"should be true" in {
val result = ConfigQuery.showVersion(ConfigOptions(List(
ConfigOption.Version)))
val result =
ConfigQuery.showVersion(ConfigOptions(List(ConfigOption.Version)))
assertResult(true)(result)
}
}
@ -25,8 +26,8 @@ class ConfigQueryTest extends FreeSpec {
"batch mode" - {
"when is set" - {
"should be true" in {
val result = ConfigQuery.batchMode(ConfigOptions(List(
ConfigOption.BatchMode)))
val result =
ConfigQuery.batchMode(ConfigOptions(List(ConfigOption.BatchMode)))
assertResult(true)(result)
}
}
@ -40,8 +41,8 @@ class ConfigQueryTest extends FreeSpec {
"ignore user options" - {
"when is set" - {
"should be true" in {
val result = ConfigQuery.ignoreUserOptions(ConfigOptions(List(
ConfigOption.IgnoreUserOptions)))
val result = ConfigQuery.ignoreUserOptions(
ConfigOptions(List(ConfigOption.IgnoreUserOptions)))
assertResult(true)(result)
}
}
@ -55,8 +56,8 @@ class ConfigQueryTest extends FreeSpec {
"ignore global options" - {
"when is set" - {
"should be true" in {
val result = ConfigQuery.ignoreGlobalOptions(ConfigOptions(List(
ConfigOption.IgnoreGlobalOptions)))
val result = ConfigQuery.ignoreGlobalOptions(
ConfigOptions(List(ConfigOption.IgnoreGlobalOptions)))
assertResult(true)(result)
}
}
@ -72,26 +73,26 @@ class ConfigQueryTest extends FreeSpec {
val pathB = Paths.get("b-path")
"when not set" - {
"should have current dir" - {
val pwd = Paths.get(System.getenv("PWD"))
val pwd = Paths.get(System.getenv("PWD"))
val expected = Sources(List(pwd))
val result = ConfigQuery.sources(ConfigOptions(List()))
val result = ConfigQuery.sources(ConfigOptions(List()))
assertResult(expected)(result)
}
}
"when is set once" - {
"should have one source" in {
val expected = Sources(List(pathA))
val result = ConfigQuery.sources(ConfigOptions(List(
ConfigOption.Source(pathA))))
val result =
ConfigQuery.sources(ConfigOptions(List(ConfigOption.Source(pathA))))
assertResult(expected)(result)
}
}
"when is set twice" - {
"should have two sources" in {
val expected = Sources(List(pathA, pathB))
val result = ConfigQuery.sources(ConfigOptions(List(
ConfigOption.Source(pathA),
ConfigOption.Source(pathB))))
val result = ConfigQuery.sources(
ConfigOptions(
List(ConfigOption.Source(pathA), ConfigOption.Source(pathB))))
assertResult(expected)(result)
}
}

5
core/src/test/scala/net/kemitix/thorp/core/ConfigurationBuilderTest.scala
View file

@ -2,6 +2,11 @@ package net.kemitix.thorp.core
import java.nio.file.{Path, Paths}
import net.kemitix.thorp.config.{
ConfigOption,
ConfigOptions,
ConfigurationBuilder
}
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
import net.kemitix.thorp.domain._
import org.scalatest.FunSpec

8
core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala
View file

@ -2,7 +2,8 @@ package net.kemitix.thorp.core
import java.io.File
import net.kemitix.thorp.domain.{Bucket, Config, RemoteKey, Sources}
import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.domain.{RemoteKey, Sources}
import org.scalatest.FunSpec
class KeyGeneratorSuite extends FunSpec {
@ -10,10 +11,9 @@ class KeyGeneratorSuite extends FunSpec {
private val source: File = Resource(this, "upload")
private val sourcePath = source.toPath
private val prefix = RemoteKey("prefix")
implicit private val config: Config =
Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
private val sources = Sources(List(sourcePath))
private val fileToKey =
KeyGenerator.generateKey(config.sources, config.prefix) _
KeyGenerator.generateKey(sources, prefix) _
describe("key generator") {

41
core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala
View file

@ -2,11 +2,13 @@ package net.kemitix.thorp.core
import java.nio.file.Paths
import net.kemitix.thorp.config._
import net.kemitix.thorp.console._
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.api.HashService
import net.kemitix.thorp.storage.api.{HashService, Storage}
import org.scalatest.FunSpec
import zio.DefaultRuntime
import zio.{DefaultRuntime, Task, UIO}
class LocalFileStreamSuite extends FunSpec {
@ -21,8 +23,13 @@ class LocalFileStreamSuite extends FunSpec {
private def file(filename: String) =
sourcePath.resolve(Paths.get(filename))
implicit private val config: Config = Config(
sources = Sources(List(sourcePath)))
private val configOptions = ConfigOptions(
List(
ConfigOption.IgnoreGlobalOptions,
ConfigOption.IgnoreUserOptions,
ConfigOption.Source(sourcePath),
ConfigOption.Bucket("aBucket")
))
describe("findFiles") {
it("should find all files") {
@ -47,9 +54,29 @@ class LocalFileStreamSuite extends FunSpec {
}
private def invoke() = {
val runtime = new DefaultRuntime {}
runtime.unsafeRunSync {
LocalFileStream.findFiles(sourcePath, hashService)
type TestEnv = Storage with Console with Config
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live {
override def listResult: Task[S3ObjectsData] =
Task.die(new NotImplementedError)
override def uploadResult: UIO[StorageQueueEvent] =
Task.die(new NotImplementedError)
override def copyResult: UIO[StorageQueueEvent] =
Task.die(new NotImplementedError)
override def deleteResult: UIO[StorageQueueEvent] =
Task.die(new NotImplementedError)
override def shutdownResult: UIO[StorageQueueEvent] =
Task.die(new NotImplementedError)
}
def testProgram =
for {
config <- ConfigurationBuilder.buildConfig(configOptions)
_ <- setConfiguration(config)
files <- LocalFileStream.findFiles(hashService)(sourcePath)
} yield files
new DefaultRuntime {}.unsafeRunSync {
testProgram.provide(testEnv)
}.toEither
}

8
core/src/test/scala/net/kemitix/thorp/core/MD5HashGeneratorTest.scala
View file

@ -2,8 +2,8 @@ package net.kemitix.thorp.core
import java.nio.file.Path
import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root}
import net.kemitix.thorp.domain._
import org.scalatest.FunSpec
import zio.DefaultRuntime
@ -11,11 +11,7 @@ class MD5HashGeneratorTest extends FunSpec {
private val runtime = new DefaultRuntime {}
private val source = Resource(this, "upload")
private val sourcePath = source.toPath
private val prefix = RemoteKey("prefix")
implicit private val config: Config =
Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
private val source = Resource(this, "upload")
describe("md5File()") {
describe("read a small file (smaller than buffer)") {

6
core/src/test/scala/net/kemitix/thorp/core/ParseConfigFileTest.scala
View file

@ -2,6 +2,12 @@ package net.kemitix.thorp.core
import java.nio.file.{Path, Paths}
import net.kemitix.thorp.config.{
ConfigOption,
ConfigOptions,
ParseConfigFile,
Resource
}
import org.scalatest.FunSpec
import zio.DefaultRuntime

1
core/src/test/scala/net/kemitix/thorp/core/ParseConfigLinesTest.scala
View file

@ -2,6 +2,7 @@ package net.kemitix.thorp.core
import java.nio.file.Paths
import net.kemitix.thorp.config.{ConfigOption, ConfigOptions, ParseConfigLines}
import org.scalatest.FunSpec
class ParseConfigLinesTest extends FunSpec {

24
core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala
View file

@ -3,6 +3,7 @@ package net.kemitix.thorp.core
import java.io.File
import java.nio.file.Path
import net.kemitix.thorp.config._
import net.kemitix.thorp.console._
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5
@ -24,7 +25,9 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
val options: Path => ConfigOptions =
source =>
configOptions(ConfigOption.Source(source),
ConfigOption.Bucket("a-bucket"))
ConfigOption.Bucket("a-bucket"),
ConfigOption.IgnoreUserOptions,
ConfigOption.IgnoreGlobalOptions)
"a file" - {
val filename = "aFile"
val remoteKey = RemoteKey(filename)
@ -325,9 +328,9 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
hashService: HashService,
configOptions: ConfigOptions,
result: Task[S3ObjectsData]
): Either[Any, List[(String, String, String, String, String)]] = {
type TestEnv = Storage.Test with Console.Test
val testEnv: TestEnv = new Storage.Test with Console.Test {
) = {
type TestEnv = Storage with Console with Config
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live {
override def listResult: Task[S3ObjectsData] = result
override def uploadResult: UIO[StorageQueueEvent] =
Task.die(new NotImplementedError)
@ -339,12 +342,15 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
Task.die(new NotImplementedError)
}
def testProgram =
for {
config <- ConfigurationBuilder.buildConfig(configOptions)
_ <- setConfiguration(config)
plan <- PlanBuilder.createPlan(hashService)
} yield plan
new DefaultRuntime {}
.unsafeRunSync {
PlanBuilder
.createPlan(hashService, configOptions)
.provide(testEnv)
}
.unsafeRunSync(testProgram.provide(testEnv))
.toEither
.map(convertResult)
}

6
core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala
View file

@ -2,6 +2,7 @@ package net.kemitix.thorp.core
import java.time.Instant
import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
@ -11,11 +12,10 @@ class S3MetaDataEnricherSuite extends FunSpec {
val lastModified = LastModified(Instant.now())
private val source = Resource(this, "upload")
private val sourcePath = source.toPath
private val sources = Sources(List(sourcePath))
private val prefix = RemoteKey("prefix")
implicit private val config: Config =
Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
private val fileToKey =
KeyGenerator.generateKey(config.sources, config.prefix) _
KeyGenerator.generateKey(sources, prefix) _
def getMatchesByKey(
status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))

25
domain/src/main/scala/net/kemitix/thorp/domain/Config.scala
View file

@ -1,25 +0,0 @@
package net.kemitix.thorp.domain
final case class Config(
bucket: Bucket = Bucket(""),
prefix: RemoteKey = RemoteKey(""),
filters: List[Filter] = List(),
debug: Boolean = false,
batchMode: Boolean = false,
sources: Sources = Sources(List())
)
object Config {
val sources: SimpleLens[Config, Sources] =
SimpleLens[Config, Sources](_.sources, b => a => b.copy(sources = a))
val bucket: SimpleLens[Config, Bucket] =
SimpleLens[Config, Bucket](_.bucket, b => a => b.copy(bucket = a))
val prefix: SimpleLens[Config, RemoteKey] =
SimpleLens[Config, RemoteKey](_.prefix, b => a => b.copy(prefix = a))
val filters: SimpleLens[Config, List[Filter]] =
SimpleLens[Config, List[Filter]](_.filters, b => a => b.copy(filters = a))
val debug: SimpleLens[Config, Boolean] =
SimpleLens[Config, Boolean](_.debug, b => a => b.copy(debug = a))
val batchMode: SimpleLens[Config, Boolean] =
SimpleLens[Config, Boolean](_.batchMode, b => a => b.copy(batchMode = a))
}

2
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ETagGeneratorTest.scala
View file

@ -3,7 +3,7 @@ package net.kemitix.thorp.storage.aws
import java.nio.file.Path
import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
import net.kemitix.thorp.core.Resource
import net.kemitix.thorp.config.Resource
import org.scalatest.FunSpec
import zio.DefaultRuntime

11
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala
View file

@ -2,7 +2,8 @@ package net.kemitix.thorp.storage.aws
import java.time.Instant
import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.core.{KeyGenerator, S3MetaDataEnricher}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
import org.scalamock.scalatest.MockFactory
@ -12,12 +13,10 @@ class StorageServiceSuite extends FunSpec with MockFactory {
private val source = Resource(this, "upload")
private val sourcePath = source.toPath
private val prefix = RemoteKey("prefix")
implicit private val config: Config =
Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
private val sources = Sources(List(sourcePath))
private val prefix = RemoteKey("prefix")
private val fileToKey =
KeyGenerator.generateKey(config.sources, config.prefix) _
KeyGenerator.generateKey(sources, prefix) _
describe("getS3Status") {

2
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala
View file

@ -5,8 +5,8 @@ import java.io.File
import com.amazonaws.SdkClientException
import com.amazonaws.services.s3.model.AmazonS3Exception
import com.amazonaws.services.s3.transfer.model.UploadResult
import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.console._
import net.kemitix.thorp.core.Resource
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.StorageQueueEvent.{
Action,