case classes shouldn't be OO objects (#147)

* [core] Extract Filters from domain.Filter

* [core] extract LocalFileValidator

* [domain] LocalFile remove unused isDirectory

* [domain] LocalFile move/rename relative as relativeToSource on companion

* [domain] LocalFile move and rename matches as matchesHash on companion

* [domain] LocalFile move md5base64 to companion

* [domain] Logger remove

* [domain] MD5Hash move hash to companion

* [domain] MD5Hash move digest to companion

* [domain] MD5Hash move hash64 to companion

* [domain] RemoteKey move class methods to companion

* [domain] Sources move forPath to companion

Led to being able to cleanup LocalFileStream.localFile and adding
LocalFiles.one

* [domain] UploadEventLogger rename method as apply

build.sbt

@@ -135,3 +135,4 @@ lazy val domain = (project in file("domain"))
   .settings(commonSettings)
   .settings(assemblyJarName in assembly := "domain.jar")
   .settings(testDependencies)
+  .settings(zioDependencies)

config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala

@@ -1,6 +1,6 @@
 package net.kemitix.thorp.config
 
-import java.nio.file.Path
+import java.io.File
 
 sealed trait ConfigValidation {
 
@@ -22,10 +22,10 @@ object ConfigValidation {
   }
 
   case class ErrorReadingFile(
-      path: Path,
+      file: File,
       message: String
   ) extends ConfigValidation {
-    override def errorMessage: String = s"Error reading file '$path': $message"
+    override def errorMessage: String = s"Error reading file '$file': $message"
   }
 
 }

config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala

@@ -1,6 +1,6 @@
 package net.kemitix.thorp.config
 
-import java.nio.file.Paths
+import java.io.File
 
 import net.kemitix.thorp.filesystem.FileSystem
 import zio.ZIO
@@ -11,9 +11,9 @@ import zio.ZIO
   */
 trait ConfigurationBuilder {
 
-  private val userConfigFilename = ".config/thorp.conf"
+  private val userConfigFile = ".config/thorp.conf"
-  private val globalConfig       = Paths.get("/etc/thorp.conf")
+  private val globalConfig   = new File("/etc/thorp.conf")
-  private val userHome           = Paths.get(System.getProperty("user.home"))
+  private val userHome       = new File(System.getProperty("user.home"))
 
   def buildConfig(priorityOpts: ConfigOptions)
     : ZIO[FileSystem, ConfigValidationException, Configuration] =
@@ -33,7 +33,7 @@ trait ConfigurationBuilder {
 
   private def userOptions(priorityOpts: ConfigOptions) =
     if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
-    else ParseConfigFile.parseFile(userHome.resolve(userConfigFilename))
+    else ParseConfigFile.parseFile(new File(userHome, userConfigFile))
 
   private def globalOptions(priorityOpts: ConfigOptions) =
     if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig

config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala

@@ -1,6 +1,6 @@
 package net.kemitix.thorp.config
 
-import java.nio.file.Path
+import java.io.File
 
 import net.kemitix.thorp.filesystem.FileSystem
 import zio.{IO, TaskR, ZIO}
@@ -8,19 +8,14 @@ import zio.{IO, TaskR, ZIO}
 trait ParseConfigFile {
 
   def parseFile(
-      filename: Path): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
+      file: File): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
-    (readFile(filename) >>= ParseConfigLines.parseLines)
+    (FileSystem.exists(file) >>= readLines(file) >>= ParseConfigLines.parseLines)
-      .catchAll(
+      .catchAll(h =>
-        h =>
+        IO.fail(List(ConfigValidation.ErrorReadingFile(file, h.getMessage))))
-          IO.fail(
-            List(ConfigValidation.ErrorReadingFile(filename, h.getMessage))))
 
-  private def readFile(filename: Path) =
+  private def readLines(file: File)(
-    FileSystem.exists(filename.toFile) >>= readLines(filename)
-
-  private def readLines(filename: Path)(
       exists: Boolean): TaskR[FileSystem, Seq[String]] =
-    if (exists) FileSystem.lines(filename.toFile)
+    if (exists) FileSystem.lines(file)
     else ZIO.succeed(Seq.empty)
 
 }

config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala

@@ -1,5 +1,7 @@
 package net.kemitix.thorp.config
 
+import java.io.File
+
 import net.kemitix.thorp.domain.Sources
 import net.kemitix.thorp.filesystem.FileSystem
 import zio.ZIO
@@ -12,7 +14,7 @@ trait SourceConfigLoader {
       sources: Sources): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
     ZIO
       .foreach(sources.paths) { path =>
-        ParseConfigFile.parseFile(path.resolve(thorpConfigFileName))
+        ParseConfigFile.parseFile(new File(path.toFile, thorpConfigFileName))
       }
       .map(_.foldLeft(ConfigOptions(sources.paths.map(ConfigOption.Source))) {
         (acc, co) =>

config/src/test/scala/net/kemitix/thorp/config/ParseConfigFileTest.scala

@@ -1,47 +1,58 @@
 package net.kemitix.thorp.config
 
-import java.nio.file.{Path, Paths}
+import java.io.File
+import java.nio.file.Paths
 
+import net.kemitix.thorp.domain.TemporaryFolder
 import net.kemitix.thorp.filesystem.FileSystem
 import org.scalatest.FunSpec
 import zio.DefaultRuntime
 
-class ParseConfigFileTest extends FunSpec {
+class ParseConfigFileTest extends FunSpec with TemporaryFolder {
 
   private val empty = Right(ConfigOptions())
 
   describe("parse a missing file") {
-    val filename = Paths.get("/path/to/missing/file")
+    val file = new File("/path/to/missing/file")
     it("should return no options") {
-      assertResult(empty)(invoke(filename))
+      assertResult(empty)(invoke(file))
     }
   }
   describe("parse an empty file") {
-    val filename = Resource(this, "empty-file").toPath
     it("should return no options") {
-      assertResult(empty)(invoke(filename))
+      withDirectory(dir => {
+        val file = writeFile(dir, "empty-file")
+        assertResult(empty)(invoke(file))
+      })
     }
   }
   describe("parse a file with no valid entries") {
-    val filename = Resource(this, "invalid-config").toPath
     it("should return no options") {
-      assertResult(empty)(invoke(filename))
+      withDirectory(dir => {
+        val file = writeFile(dir, "invalid-config", "no valid = config items")
+        assertResult(empty)(invoke(file))
+      })
     }
   }
   describe("parse a file with properties") {
-    val filename = Resource(this, "simple-config").toPath
+    it("should return some options") {
       val expected = Right(
         ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source")),
                            ConfigOption.Bucket("bucket-name"))))
-    it("should return some options") {
+      withDirectory(dir => {
-      assertResult(expected)(invoke(filename))
+        val file = writeFile(dir,
+                             "simple-config",
+                             "source = /path/to/source",
+                             "bucket = bucket-name")
+        assertResult(expected)(invoke(file))
+      })
     }
   }
 
-  private def invoke(filename: Path) = {
+  private def invoke(file: File) = {
     new DefaultRuntime {}.unsafeRunSync {
       ParseConfigFile
-        .parseFile(filename)
+        .parseFile(file)
         .provide(FileSystem.Live)
     }.toEither
   }

core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala

@@ -21,7 +21,7 @@ object ActionGenerator {
     s3MetaData match {
       // #1 local exists, remote exists, remote matches - do nothing
       case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
-          if localFile.matches(hash) =>
+          if LocalFile.matchesHash(localFile)(hash) =>
         doNothing(bucket, key)
       // #2 local exists, remote is missing, other matches - copy
       case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
@@ -33,7 +33,7 @@ object ActionGenerator {
         uploadFile(bucket, localFile)
       // #4 local exists, remote exists, remote no match, other matches - copy
       case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _)))
-          if !localFile.matches(hash) &&
+          if !LocalFile.matchesHash(localFile)(hash) &&
             matchByHash.nonEmpty =>
         copyFile(bucket, localFile, matchByHash)
       // #5 local exists, remote exists, remote no match, other no matches - upload

core/src/main/scala/net/kemitix/thorp/core/Filters.scala

@@ -0,0 +1,41 @@
+package net.kemitix.thorp.core
+
+import java.nio.file.Path
+
+import net.kemitix.thorp.domain.Filter
+import net.kemitix.thorp.domain.Filter.{Exclude, Include}
+
+object Filters {
+
+  def isIncluded(p: Path)(filters: List[Filter]): Boolean = {
+    sealed trait State
+    case class Unknown()   extends State
+    case class Accepted()  extends State
+    case class Discarded() extends State
+    val excluded = isExcludedByFilter(p)(_)
+    val included = isIncludedByFilter(p)(_)
+    filters.foldRight(Unknown(): State)((filter, state) =>
+      (filter, state) match {
+        case (_, Accepted())                => Accepted()
+        case (_, Discarded())               => Discarded()
+        case (x: Exclude, _) if excluded(x) => Discarded()
+        case (i: Include, _) if included(i) => Accepted()
+        case _                              => Unknown()
+    }) match {
+      case Accepted()  => true
+      case Discarded() => false
+      case Unknown() =>
+        filters.forall {
+          case _: Include => false
+          case _          => true
+        }
+    }
+  }
+
+  def isIncludedByFilter(path: Path)(filter: Filter): Boolean =
+    filter.predicate.test(path.toString)
+
+  def isExcludedByFilter(path: Path)(filter: Filter): Boolean =
+    filter.predicate.test(path.toString)
+
+}

core/src/main/scala/net/kemitix/thorp/core/KeyGenerator.scala

@@ -3,21 +3,17 @@ package net.kemitix.thorp.core
 import java.nio.file.Path
 
 import net.kemitix.thorp.domain.{RemoteKey, Sources}
+import zio.Task
 
 object KeyGenerator {
 
   def generateKey(
       sources: Sources,
       prefix: RemoteKey
-  )(path: Path): RemoteKey = {
+  )(path: Path): Task[RemoteKey] =
-    val source       = sources.forPath(path)
+    Sources
-    val relativePath = source.relativize(path.toAbsolutePath).toString
+      .forPath(path)(sources)
-    RemoteKey(
+      .map(p => p.relativize(path.toAbsolutePath).toString)
-      List(
+      .map(RemoteKey.resolve(_)(prefix))
-        prefix.key,
-        relativePath
-      ).filter(_.nonEmpty)
-        .mkString("/"))
-  }
 
 }

core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala

@@ -4,9 +4,8 @@ import java.io.File
 import java.nio.file.Path
 
 import net.kemitix.thorp.config.Config
-import net.kemitix.thorp.core.KeyGenerator.generateKey
 import net.kemitix.thorp.core.hasher.Hasher
-import net.kemitix.thorp.domain._
+import net.kemitix.thorp.domain.Sources
 import net.kemitix.thorp.filesystem.FileSystem
 import zio.{Task, TaskR, ZIO}
 
@@ -48,21 +47,18 @@ object LocalFileStream {
         .filter({ case (_, included) => included })
         .map({ case (path, _) => path })
 
-  private def localFile(path: Path) = {
+  private def localFile(path: Path) =
-    val file = path.toFile
     for {
       sources <- Config.sources
       prefix  <- Config.prefix
+      source  <- Sources.forPath(path)(sources)
       hash    <- Hasher.hashObject(path)
-      localFile = LocalFile(file,
+      localFile <- LocalFileValidator.validate(path,
-                            sources.forPath(path).toFile,
+                                               source.toFile,
                                                hash,
-                            generateKey(sources, prefix)(path))
+                                               sources,
-    } yield
+                                               prefix)
-      LocalFiles(localFiles = Stream(localFile),
+    } yield LocalFiles.one(localFile)
-                 count = 1,
-                 totalSizeBytes = file.length)
-  }
 
   private def listFiles(path: Path) =
     for {
@@ -78,6 +74,6 @@ object LocalFileStream {
   private def isIncluded(path: Path) =
     for {
       filters <- Config.filters
-    } yield Filter.isIncluded(path)(filters)
+    } yield Filters.isIncluded(path)(filters)
 
 }

core/src/main/scala/net/kemitix/thorp/core/LocalFileValidator.scala

@@ -0,0 +1,66 @@
+package net.kemitix.thorp.core
+
+import java.io.File
+import java.nio.file.Path
+
+import net.kemitix.thorp.domain.{
+  HashType,
+  LocalFile,
+  MD5Hash,
+  RemoteKey,
+  Sources
+}
+import zio.{IO, ZIO}
+
+object LocalFileValidator {
+
+  def validate(
+      path: Path,
+      source: File,
+      hash: Map[HashType, MD5Hash],
+      sources: Sources,
+      prefix: RemoteKey
+  ): IO[Violation, LocalFile] =
+    for {
+      vFile     <- validateFile(path.toFile)
+      remoteKey <- validateRemoteKey(sources, prefix, path)
+    } yield LocalFile(vFile, source, hash, remoteKey)
+
+  private def validateFile(file: File) =
+    if (file.isDirectory)
+      ZIO.fail(Violation.IsNotAFile(file))
+    else
+      ZIO.succeed(file)
+
+  private def validateRemoteKey(sources: Sources,
+                                prefix: RemoteKey,
+                                path: Path) =
+    KeyGenerator
+      .generateKey(sources, prefix)(path)
+      .mapError(e => Violation.InvalidRemoteKey(path, e))
+
+  sealed trait Violation extends Throwable {
+    def getMessage: String
+  }
+  object Violation {
+    case class IsNotAFile(file: File) extends Violation {
+      override def getMessage: String = s"Local File must be a file: ${file}"
+    }
+    case class InvalidRemoteKey(path: Path, e: Throwable) extends Violation {
+      override def getMessage: String =
+        s"Remote Key for '${path}' is invalid: ${e.getMessage}"
+    }
+  }
+
+  def resolve(
+      path: String,
+      md5Hashes: Map[HashType, MD5Hash],
+      source: Path,
+      sources: Sources,
+      prefix: RemoteKey
+  ): IO[Violation, LocalFile] = {
+    val resolvedPath = source.resolve(path)
+    validate(resolvedPath, source.toFile, md5Hashes, sources, prefix)
+  }
+
+}

core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala

@@ -13,12 +13,11 @@ case class LocalFiles(
       count = count + append.count,
       totalSizeBytes = totalSizeBytes + append.totalSizeBytes
     )
-
 }
 
 object LocalFiles {
-
   def reduce: Stream[LocalFiles] => LocalFiles =
     list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf)
-
+  def one(localFile: LocalFile): LocalFiles =
+    LocalFiles(Stream(localFile), 1, localFile.file.length)
 }

core/src/main/scala/net/kemitix/thorp/core/Remote.scala

@@ -18,7 +18,7 @@ object Remote {
       remoteKey: RemoteKey
   ): TaskR[FileSystem, Boolean] = {
     def existsInSource(source: Path) =
-      remoteKey.asFile(source, prefix) match {
+      RemoteKey.asFile(source, prefix)(remoteKey) match {
         case Some(file) => FileSystem.exists(file)
         case None       => ZIO.succeed(false)
       }

core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala

@@ -2,13 +2,7 @@ package net.kemitix.thorp.core
 
 import java.time.Instant
 
-import net.kemitix.thorp.config.{
+import net.kemitix.thorp.config._
-  Config,
-  ConfigOption,
-  ConfigOptions,
-  ConfigurationBuilder,
-  Resource
-}
 import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
 import net.kemitix.thorp.domain.HashType.MD5
 import net.kemitix.thorp.domain._
@@ -31,8 +25,6 @@ class ActionGeneratorSuite extends FunSpec {
       ConfigOption.IgnoreUserOptions,
       ConfigOption.IgnoreGlobalOptions
     ))
-  private val fileToKey =
-    KeyGenerator.generateKey(sources, prefix) _
 
   describe("create actions") {
 
@@ -40,40 +32,54 @@ class ActionGeneratorSuite extends FunSpec {
 
     describe("#1 local exists, remote exists, remote matches - do nothing") {
       val theHash = MD5Hash("the-hash")
-      val theFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                      fileToKey)
+                                              sources,
-      val theRemoteMetadata =
+                                              prefix)
-        RemoteMetaData(theFile.remoteKey, theHash, lastModified)
+        theRemoteMetadata = RemoteMetaData(theFile.remoteKey,
-      val input =
+                                           theHash,
-        S3MetaData(theFile, // local exists
+                                           lastModified)
+        input = S3MetaData(
+          theFile, // local exists
           matchByHash = Set(theRemoteMetadata), // remote matches
           matchByKey = Some(theRemoteMetadata) // remote exists
         )
+      } yield (theFile, input)
       it("do nothing") {
+        env.map({
+          case (theFile, input) => {
             val expected =
-          Right(
+              Right(Stream(
-            Stream(DoNothing(bucket, theFile.remoteKey, theFile.file.length)))
+                DoNothing(bucket, theFile.remoteKey, theFile.file.length + 1)))
             val result = invoke(input, previousActions)
             assertResult(expected)(result)
           }
+        })
+      }
     }
     describe("#2 local exists, remote is missing, other matches - copy") {
       val theHash = MD5Hash("the-hash")
-      val theFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                      fileToKey)
+                                              sources,
-      val theRemoteKey   = theFile.remoteKey
+                                              prefix)
-      val otherRemoteKey = prefix.resolve("other-key")
+        theRemoteKey   = theFile.remoteKey
-      val otherRemoteMetadata =
+        otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
-        RemoteMetaData(otherRemoteKey, theHash, lastModified)
+        otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
-      val input =
+                                             theHash,
-        S3MetaData(theFile, // local exists
+                                             lastModified)
+        input = S3MetaData(
+          theFile, // local exists
           matchByHash = Set(otherRemoteMetadata), // other matches
           matchByKey = None) // remote is missing
+      } yield (theFile, theRemoteKey, input, otherRemoteKey)
       it("copy from other key") {
+        env.map({
+          case (theFile, theRemoteKey, input, otherRemoteKey) => {
             val expected = Right(
               Stream(
                 ToCopy(bucket,
@@ -84,43 +90,58 @@ class ActionGeneratorSuite extends FunSpec {
             val result = invoke(input, previousActions)
             assertResult(expected)(result)
           }
+        })
       }
       describe("#3 local exists, remote is missing, other no matches - upload") {
         val theHash = MD5Hash("the-hash")
-      val theFile = LocalFile.resolve("the-file",
+        val env = for {
+          theFile <- LocalFileValidator.resolve("the-file",
                                                 md5HashMap(theHash),
                                                 sourcePath,
-                                      fileToKey)
+                                                sources,
-      val input = S3MetaData(theFile, // local exists
+                                                prefix)
+          input = S3MetaData(theFile, // local exists
                              matchByHash = Set.empty, // other no matches
                              matchByKey = None) // remote is missing
+        } yield (theFile, input)
         it("upload") {
-        val expected = Right(
+          env.map({
-          Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
+            case (theFile, input) => {
+              val expected = Right(Stream(
+                ToUpload(bucket, theFile, theFile.file.length))) // upload
               val result = invoke(input, previousActions)
               assertResult(expected)(result)
             }
+          })
+        }
+      }
     }
     describe(
       "#4 local exists, remote exists, remote no match, other matches - copy") {
       val theHash = MD5Hash("the-hash")
-      val theFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                      fileToKey)
+                                              sources,
-      val theRemoteKey   = theFile.remoteKey
+                                              prefix)
-      val oldHash        = MD5Hash("old-hash")
+        theRemoteKey   = theFile.remoteKey
-      val otherRemoteKey = prefix.resolve("other-key")
+        oldHash        = MD5Hash("old-hash")
-      val otherRemoteMetadata =
+        otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
-        RemoteMetaData(otherRemoteKey, theHash, lastModified)
+        otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
-      val oldRemoteMetadata = RemoteMetaData(theRemoteKey,
+                                             theHash,
+                                             lastModified)
+        oldRemoteMetadata = RemoteMetaData(theRemoteKey,
                                            hash = oldHash, // remote no match
                                            lastModified = lastModified)
-      val input =
+        input = S3MetaData(
-        S3MetaData(theFile, // local exists
+          theFile, // local exists
           matchByHash = Set(otherRemoteMetadata), // other matches
           matchByKey = Some(oldRemoteMetadata)) // remote exists
+      } yield (theFile, theRemoteKey, input, otherRemoteKey)
       it("copy from other key") {
+        env.map({
+          case (theFile, theRemoteKey, input, otherRemoteKey) => {
             val expected = Right(
               Stream(
                 ToCopy(bucket,
@@ -131,29 +152,37 @@ class ActionGeneratorSuite extends FunSpec {
             val result = invoke(input, previousActions)
             assertResult(expected)(result)
           }
+        })
+      }
     }
     describe(
       "#5 local exists, remote exists, remote no match, other no matches - upload") {
       val theHash = MD5Hash("the-hash")
-      val theFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                      fileToKey)
+                                              sources,
-      val theRemoteKey = theFile.remoteKey
+                                              prefix)
-      val oldHash      = MD5Hash("old-hash")
+        theRemoteKey      = theFile.remoteKey
-      val theRemoteMetadata =
+        oldHash           = MD5Hash("old-hash")
-        RemoteMetaData(theRemoteKey, oldHash, lastModified)
+        theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
-      val input =
+        input = S3MetaData(
-        S3MetaData(theFile, // local exists
+          theFile, // local exists
           matchByHash = Set.empty, // remote no match, other no match
           matchByKey = Some(theRemoteMetadata) // remote exists
         )
+      } yield (theFile, input)
       it("upload") {
+        env.map({
+          case (theFile, input) => {
             val expected = Right(
               Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
             val result = invoke(input, previousActions)
             assertResult(expected)(result)
           }
+        })
+      }
     }
     describe("#6 local missing, remote exists - delete") {
       it("TODO") {

core/src/test/scala/net/kemitix/thorp/core/FiltersSuite.scala

@@ -1,7 +1,8 @@
-package net.kemitix.thorp.domain
+package net.kemitix.thorp.core
 
 import java.nio.file.Paths
 
+import net.kemitix.thorp.domain.Filter
 import net.kemitix.thorp.domain.Filter.{Exclude, Include}
 import org.scalatest.FunSpec
 
@@ -21,31 +22,32 @@ class FiltersSuite extends FunSpec {
     describe("default filter") {
       val include = Include()
       it("should include files") {
-        paths.foreach(path => assertResult(true)(include.isIncluded(path)))
+        paths.foreach(path =>
+          assertResult(true)(Filters.isIncludedByFilter(path)(include)))
       }
     }
     describe("directory exact match include '/upload/subdir/'") {
       val include = Include("/upload/subdir/")
       it("include matching directory") {
         val matching = Paths.get("/upload/subdir/leaf-file")
-        assertResult(true)(include.isIncluded(matching))
+        assertResult(true)(Filters.isIncludedByFilter(matching)(include))
       }
       it("exclude non-matching files") {
         val nonMatching = Paths.get("/upload/other-file")
-        assertResult(false)(include.isIncluded(nonMatching))
+        assertResult(false)(Filters.isIncludedByFilter(nonMatching)(include))
       }
     }
     describe("file partial match 'root'") {
       val include = Include("root")
       it("include matching file '/upload/root-file") {
         val matching = Paths.get("/upload/root-file")
-        assertResult(true)(include.isIncluded(matching))
+        assertResult(true)(Filters.isIncludedByFilter(matching)(include))
       }
       it("exclude non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
         val nonMatching1 = Paths.get("/test-file-for-hash.txt")
         val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
-        assertResult(false)(include.isIncluded(nonMatching1))
+        assertResult(false)(Filters.isIncludedByFilter(nonMatching1)(include))
-        assertResult(false)(include.isIncluded(nonMatching2))
+        assertResult(false)(Filters.isIncludedByFilter(nonMatching2)(include))
       }
     }
   }
@@ -63,30 +65,30 @@ class FiltersSuite extends FunSpec {
       val exclude = Exclude("/upload/subdir/")
       it("exclude matching directory") {
         val matching = Paths.get("/upload/subdir/leaf-file")
-        assertResult(true)(exclude.isExcluded(matching))
+        assertResult(true)(Filters.isExcludedByFilter(matching)(exclude))
       }
       it("include non-matching files") {
         val nonMatching = Paths.get("/upload/other-file")
-        assertResult(false)(exclude.isExcluded(nonMatching))
+        assertResult(false)(Filters.isExcludedByFilter(nonMatching)(exclude))
       }
     }
     describe("file partial match 'root'") {
       val exclude = Exclude("root")
       it("exclude matching file '/upload/root-file") {
         val matching = Paths.get("/upload/root-file")
-        assertResult(true)(exclude.isExcluded(matching))
+        assertResult(true)(Filters.isExcludedByFilter(matching)(exclude))
       }
       it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
         val nonMatching1 = Paths.get("/test-file-for-hash.txt")
         val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
-        assertResult(false)(exclude.isExcluded(nonMatching1))
+        assertResult(false)(Filters.isExcludedByFilter(nonMatching1)(exclude))
-        assertResult(false)(exclude.isExcluded(nonMatching2))
+        assertResult(false)(Filters.isExcludedByFilter(nonMatching2)(exclude))
       }
     }
   }
   describe("isIncluded") {
     def invoke(filters: List[Filter]) = {
-      paths.filter(path => Filter.isIncluded(path)(filters))
+      paths.filter(path => Filters.isIncluded(path)(filters))
     }
 
     describe("when there are no filters") {

core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala

@@ -1,10 +1,12 @@
 package net.kemitix.thorp.core
 
 import java.io.File
+import java.nio.file.Path
 
 import net.kemitix.thorp.config.Resource
 import net.kemitix.thorp.domain.{RemoteKey, Sources}
 import org.scalatest.FunSpec
+import zio.DefaultRuntime
 
 class KeyGeneratorSuite extends FunSpec {
 
@@ -12,26 +14,32 @@ class KeyGeneratorSuite extends FunSpec {
   private val sourcePath   = source.toPath
   private val prefix       = RemoteKey("prefix")
   private val sources      = Sources(List(sourcePath))
-  private val fileToKey =
-    KeyGenerator.generateKey(sources, prefix) _
 
   describe("key generator") {
 
     describe("when file is within source") {
       it("has a valid key") {
         val subdir   = "subdir"
-        assertResult(RemoteKey(s"${prefix.key}/$subdir"))(
+        val expected = Right(RemoteKey(s"${prefix.key}/$subdir"))
-          fileToKey(sourcePath.resolve(subdir)))
+        val result   = invoke(sourcePath.resolve(subdir))
+        assertResult(expected)(result)
       }
     }
 
     describe("when file is deeper within source") {
       it("has a valid key") {
         val subdir   = "subdir/deeper/still"
-        assertResult(RemoteKey(s"${prefix.key}/$subdir"))(
+        val expected = Right(RemoteKey(s"${prefix.key}/$subdir"))
-          fileToKey(sourcePath.resolve(subdir)))
+        val result   = invoke(sourcePath.resolve(subdir))
+        assertResult(expected)(result)
       }
     }
+
+    def invoke(path: Path) = {
+      new DefaultRuntime {}.unsafeRunSync {
+        KeyGenerator.generateKey(sources, prefix)(path)
+      }.toEither
+    }
   }
 
 }

core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala

@@ -32,7 +32,7 @@ class LocalFileStreamSuite extends FunSpec {
       val result =
         invoke()
           .map(_.localFiles)
-          .map(_.map(_.relative.toString))
+          .map(_.map(LocalFile.relativeToSource(_).toString))
           .map(_.toSet)
       assertResult(expected)(result)
     }

core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala

@@ -333,13 +333,17 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
                        md5Hash: MD5Hash,
                        source: Path,
                        file: File): (String, String, String, String, String) =
-    ("upload", remoteKey.key, md5Hash.hash, source.toString, file.toString)
+    ("upload",
+     remoteKey.key,
+     MD5Hash.hash(md5Hash),
+     source.toString,
+     file.toString)
 
   private def toCopy(
       sourceKey: RemoteKey,
       md5Hash: MD5Hash,
       targetKey: RemoteKey): (String, String, String, String, String) =
-    ("copy", sourceKey.key, md5Hash.hash, targetKey.key, "")
+    ("copy", sourceKey.key, MD5Hash.hash(md5Hash), targetKey.key, "")
 
   private def toDelete(
       remoteKey: RemoteKey): (String, String, String, String, String) =
@@ -386,12 +390,12 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
       case ToUpload(_, lf, _) =>
         ("upload",
          lf.remoteKey.key,
-         lf.hashes(MD5).hash,
+         MD5Hash.hash(lf.hashes(MD5)),
          lf.source.toString,
          lf.file.toString)
       case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")
       case ToCopy(_, sourceKey, hash, targetKey, _) =>
-        ("copy", sourceKey.key, hash.hash, targetKey.key, "")
+        ("copy", sourceKey.key, MD5Hash.hash(hash), targetKey.key, "")
       case DoNothing(_, remoteKey, _) =>
         ("do-nothing", remoteKey.key, "", "", "")
       case x => ("other", x.toString, "", "", "")

core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala

@@ -14,8 +14,6 @@ class S3MetaDataEnricherSuite extends FunSpec {
   private val sourcePath = source.toPath
   private val sources    = Sources(List(sourcePath))
   private val prefix     = RemoteKey("prefix")
-  private val fileToKey =
-    KeyGenerator.generateKey(sources, prefix) _
 
   def getMatchesByKey(
       status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
@@ -36,139 +34,181 @@ class S3MetaDataEnricherSuite extends FunSpec {
     describe(
       "#1a local exists, remote exists, remote matches, other matches - do nothing") {
       val theHash: MD5Hash = MD5Hash("the-file-hash")
-      val theFile: LocalFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                                 fileToKey)
+                                              sources,
-      val theRemoteKey: RemoteKey = theFile.remoteKey
+                                              prefix)
-      val s3: S3ObjectsData = S3ObjectsData(
+        theRemoteKey = theFile.remoteKey
+        s3 = S3ObjectsData(
           byHash = Map(theHash     -> Set(KeyModified(theRemoteKey, lastModified))),
           byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
         )
-      val theRemoteMetadata =
+        theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
-        RemoteMetaData(theRemoteKey, theHash, lastModified)
+      } yield (theFile, theRemoteMetadata, s3)
       it("generates valid metadata") {
+        env.map({
+          case (theFile, theRemoteMetadata, s3) => {
             val expected = S3MetaData(theFile,
                                       matchByHash = Set(theRemoteMetadata),
                                       matchByKey = Some(theRemoteMetadata))
             val result = getMetadata(theFile, s3)
             assertResult(expected)(result)
           }
+        })
+      }
     }
     describe(
       "#1b local exists, remote exists, remote matches, other no matches - do nothing") {
       val theHash: MD5Hash = MD5Hash("the-file-hash")
-      val theFile: LocalFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                                 fileToKey)
+                                              sources,
-      val theRemoteKey: RemoteKey = prefix.resolve("the-file")
+                                              prefix)
-      val s3: S3ObjectsData = S3ObjectsData(
+        theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix)
+        s3: S3ObjectsData = S3ObjectsData(
           byHash = Map(theHash     -> Set(KeyModified(theRemoteKey, lastModified))),
           byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
         )
-      val theRemoteMetadata =
+        theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
-        RemoteMetaData(theRemoteKey, theHash, lastModified)
+      } yield (theFile, theRemoteMetadata, s3)
       it("generates valid metadata") {
+        env.map({
+          case (theFile, theRemoteMetadata, s3) => {
             val expected = S3MetaData(theFile,
                                       matchByHash = Set(theRemoteMetadata),
                                       matchByKey = Some(theRemoteMetadata))
             val result = getMetadata(theFile, s3)
             assertResult(expected)(result)
           }
+        })
+      }
     }
     describe(
       "#2 local exists, remote is missing, remote no match, other matches - copy") {
       val theHash = MD5Hash("the-hash")
-      val theFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                      fileToKey)
+                                              sources,
-      val otherRemoteKey = RemoteKey("other-key")
+                                              prefix)
-      val s3: S3ObjectsData = S3ObjectsData(
+        otherRemoteKey = RemoteKey("other-key")
-        byHash = Map(theHash       -> Set(KeyModified(otherRemoteKey, lastModified))),
+        s3: S3ObjectsData = S3ObjectsData(
+          byHash =
+            Map(theHash              -> Set(KeyModified(otherRemoteKey, lastModified))),
           byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
         )
-      val otherRemoteMetadata =
+        otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
-        RemoteMetaData(otherRemoteKey, theHash, lastModified)
+                                             theHash,
+                                             lastModified)
+      } yield (theFile, otherRemoteMetadata, s3)
       it("generates valid metadata") {
+        env.map({
+          case (theFile, otherRemoteMetadata, s3) => {
             val expected = S3MetaData(theFile,
                                       matchByHash = Set(otherRemoteMetadata),
                                       matchByKey = None)
             val result = getMetadata(theFile, s3)
             assertResult(expected)(result)
           }
+        })
+      }
     }
     describe(
       "#3 local exists, remote is missing, remote no match, other no matches - upload") {
       val theHash = MD5Hash("the-hash")
-      val theFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                      fileToKey)
+                                              sources,
-      val s3: S3ObjectsData = S3ObjectsData()
+                                              prefix)
+        s3: S3ObjectsData = S3ObjectsData()
+      } yield (theFile, s3)
       it("generates valid metadata") {
+        env.map({
+          case (theFile, s3) => {
             val expected =
               S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None)
             val result = getMetadata(theFile, s3)
             assertResult(expected)(result)
           }
+        })
+      }
     }
     describe(
       "#4 local exists, remote exists, remote no match, other matches - copy") {
       val theHash = MD5Hash("the-hash")
-      val theFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                      fileToKey)
+                                              sources,
-      val theRemoteKey   = theFile.remoteKey
+                                              prefix)
-      val oldHash        = MD5Hash("old-hash")
+        theRemoteKey   = theFile.remoteKey
-      val otherRemoteKey = prefix.resolve("other-key")
+        oldHash        = MD5Hash("old-hash")
-      val s3: S3ObjectsData = S3ObjectsData(
+        otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
-        byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
+        s3: S3ObjectsData = S3ObjectsData(
+          byHash =
+            Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
                 theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
           byKey = Map(
             theRemoteKey   -> HashModified(oldHash, lastModified),
             otherRemoteKey -> HashModified(theHash, lastModified)
           )
         )
-      val theRemoteMetadata =
+        theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
-        RemoteMetaData(theRemoteKey, oldHash, lastModified)
+        otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
-      val otherRemoteMetadata =
+                                             theHash,
-        RemoteMetaData(otherRemoteKey, theHash, lastModified)
+                                             lastModified)
+      } yield (theFile, theRemoteMetadata, otherRemoteMetadata, s3)
       it("generates valid metadata") {
+        env.map({
+          case (theFile, theRemoteMetadata, otherRemoteMetadata, s3) => {
             val expected = S3MetaData(theFile,
                                       matchByHash = Set(otherRemoteMetadata),
                                       matchByKey = Some(theRemoteMetadata))
             val result = getMetadata(theFile, s3)
             assertResult(expected)(result)
           }
+        })
+      }
     }
     describe(
       "#5 local exists, remote exists, remote no match, other no matches - upload") {
       val theHash = MD5Hash("the-hash")
-      val theFile = LocalFile.resolve("the-file",
+      val env = for {
+        theFile <- LocalFileValidator.resolve("the-file",
                                               md5HashMap(theHash),
                                               sourcePath,
-                                      fileToKey)
+                                              sources,
-      val theRemoteKey = theFile.remoteKey
+                                              prefix)
-      val oldHash      = MD5Hash("old-hash")
+        theRemoteKey = theFile.remoteKey
-      val s3: S3ObjectsData = S3ObjectsData(
+        oldHash      = MD5Hash("old-hash")
+        s3: S3ObjectsData = S3ObjectsData(
           byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
                        theHash -> Set.empty),
           byKey = Map(
             theRemoteKey -> HashModified(oldHash, lastModified)
           )
         )
-      val theRemoteMetadata =
+        theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
-        RemoteMetaData(theRemoteKey, oldHash, lastModified)
+      } yield (theFile, theRemoteMetadata, s3)
       it("generates valid metadata") {
+        env.map({
+          case (theFile, theRemoteMetadata, s3) => {
             val expected = S3MetaData(theFile,
                                       matchByHash = Set.empty,
                                       matchByKey = Some(theRemoteMetadata))
             val result = getMetadata(theFile, s3)
             assertResult(expected)(result)
           }
+        })
+      }
     }
   }
 
@@ -178,20 +218,26 @@ class S3MetaDataEnricherSuite extends FunSpec {
 
   describe("getS3Status") {
     val hash = MD5Hash("hash")
-    val localFile =
+    val env = for {
-      LocalFile.resolve("the-file", md5HashMap(hash), sourcePath, fileToKey)
+      localFile <- LocalFileValidator.resolve("the-file",
-    val key = localFile.remoteKey
-    val keyOtherKey = LocalFile.resolve("other-key-same-hash",
                                               md5HashMap(hash),
                                               sourcePath,
-                                        fileToKey)
+                                              sources,
-    val diffHash = MD5Hash("diff")
+                                              prefix)
-    val keyDiffHash = LocalFile.resolve("other-key-diff-hash",
+      key = localFile.remoteKey
+      keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash",
+                                                md5HashMap(hash),
+                                                sourcePath,
+                                                sources,
+                                                prefix)
+      diffHash = MD5Hash("diff")
+      keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash",
                                                 md5HashMap(diffHash),
                                                 sourcePath,
-                                        fileToKey)
+                                                sources,
-    val lastModified = LastModified(Instant.now)
+                                                prefix)
-    val s3ObjectsData: S3ObjectsData = S3ObjectsData(
+      lastModified = LastModified(Instant.now)
+      s3ObjectsData = S3ObjectsData(
         byHash = Map(
           hash -> Set(KeyModified(key, lastModified),
                       KeyModified(keyOtherKey.remoteKey, lastModified)),
@@ -203,46 +249,72 @@ class S3MetaDataEnricherSuite extends FunSpec {
           keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
         )
       )
+    } yield (s3ObjectsData, localFile, keyDiffHash, diffHash)
 
-    def invoke(localFile: LocalFile) = {
+    def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) = {
       getS3Status(localFile, s3ObjectsData)
     }
 
     describe("when remote key exists") {
       it("should return a result for matching key") {
-        val result = getMatchesByKey(invoke(localFile))
+        env.map({
+          case (s3ObjectsData, localFile: LocalFile, _, _) =>
+            val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
             assert(result.contains(HashModified(hash, lastModified)))
+        })
       }
     }
 
     describe("when remote key does not exist and no others matches hash") {
-      val localFile = LocalFile.resolve("missing-file",
+      val env2 = for {
+        localFile <- LocalFileValidator.resolve("missing-remote",
                                                 md5HashMap(MD5Hash("unique")),
                                                 sourcePath,
-                                        fileToKey)
+                                                sources,
+                                                prefix)
+      } yield (localFile)
       it("should return no matches by key") {
-        val result = getMatchesByKey(invoke(localFile))
+        env.map({
+          case (s3ObjectsData, _, _, _) => {
+            env2.map({
+              case (localFile) => {
+                val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
                 assert(result.isEmpty)
               }
+            })
+          }
+        })
+      }
       it("should return no matches by hash") {
-        val result = getMatchesByHash(invoke(localFile))
+        env.map({
+          case (s3ObjectsData, _, _, _) => {
+            env2.map({
+              case (localFile) => {
+                val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
                 assert(result.isEmpty)
               }
+            })
+          }
+        })
+      }
     }
 
     describe("when remote key exists and no others match hash") {
+      env.map({
+        case (s3ObjectsData, _, keyDiffHash, diffHash) => {
           it("should return match by key") {
-        val result = getMatchesByKey(invoke(keyDiffHash))
+            val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData))
             assert(result.contains(HashModified(diffHash, lastModified)))
           }
           it("should return only itself in match by hash") {
-        val result = getMatchesByHash(invoke(keyDiffHash))
+            val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData))
             assert(
-          result.equals(
+              result.equals(Set(
-            Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
+                (diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
           }
         }
-
+      })
+    }
   }
 
 }

core/src/test/scala/net/kemitix/thorp/core/hasher/MD5HashGeneratorTest.scala

@@ -3,6 +3,7 @@ package net.kemitix.thorp.core.hasher
 import java.nio.file.Path
 
 import net.kemitix.thorp.config.Resource
+import net.kemitix.thorp.domain.MD5Hash
 import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root}
 import net.kemitix.thorp.filesystem.FileSystem
 import org.scalatest.FunSpec
@@ -42,14 +43,14 @@ class MD5HashGeneratorTest extends FunSpec {
       val path = Resource(this, "../big-file").toPath
       it("should generate the correct hash for first chunk of the file") {
         val part1    = BigFile.Part1
-        val expected = Right(part1.hash.hash)
+        val expected = Right(MD5Hash.hash(part1.hash))
-        val result   = invoke(path, part1.offset, part1.size).map(_.hash)
+        val result   = invoke(path, part1.offset, part1.size).map(MD5Hash.hash)
         assertResult(expected)(result)
       }
       it("should generate the correct hash for second chunk of the file") {
         val part2    = BigFile.Part2
-        val expected = Right(part2.hash.hash)
+        val expected = Right(MD5Hash.hash(part2.hash))
-        val result   = invoke(path, part2.offset, part2.size).map(_.hash)
+        val result   = invoke(path, part2.offset, part2.size).map(MD5Hash.hash)
         assertResult(expected)(result)
       }
     }

domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala

@@ -1,53 +1,18 @@
 package net.kemitix.thorp.domain
 
-import java.nio.file.Path
+import java.util.function.Predicate
 import java.util.regex.Pattern
 
-sealed trait Filter
+sealed trait Filter {
+  def predicate: Predicate[String]
+}
 
 object Filter {
-
+  case class Include(include: String = ".*") extends Filter {
-  def isIncluded(p: Path)(filters: List[Filter]): Boolean = {
+    lazy val predicate: Predicate[String] = Pattern.compile(include).asPredicate
-    sealed trait State
-    case class Unknown()   extends State
-    case class Accepted()  extends State
-    case class Discarded() extends State
-    filters.foldRight(Unknown(): State)((filter, state) =>
-      (filter, state) match {
-        case (_, Accepted())                    => Accepted()
-        case (_, Discarded())                   => Discarded()
-        case (x: Exclude, _) if x.isExcluded(p) => Discarded()
-        case (i: Include, _) if i.isIncluded(p) => Accepted()
-        case _                                  => Unknown()
-    }) match {
-      case Accepted()  => true
-      case Discarded() => false
-      case Unknown() =>
-        filters.forall {
-          case _: Include => false
-          case _          => true
   }
+  case class Exclude(exclude: String) extends Filter {
+    lazy val predicate: Predicate[String] =
+      Pattern.compile(exclude).asPredicate()
   }
-  }
-
-  case class Include(
-      include: String = ".*"
-  ) extends Filter {
-
-    private lazy val predicate = Pattern.compile(include).asPredicate
-
-    def isIncluded(path: Path): Boolean = predicate.test(path.toString)
-
-  }
-
-  case class Exclude(
-      exclude: String
-  ) extends Filter {
-
-    private lazy val predicate = Pattern.compile(exclude).asPredicate()
-
-    def isExcluded(path: Path): Boolean = predicate.test(path.toString)
-
-  }
-
 }

domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala

@@ -5,42 +5,22 @@ import java.nio.file.Path
 
 import net.kemitix.thorp.domain.HashType.MD5
 
-final case class LocalFile(
+final case class LocalFile private (
     file: File,
     source: File,
     hashes: Map[HashType, MD5Hash],
     remoteKey: RemoteKey
-) {
+)
-
-  require(!file.isDirectory, s"LocalFile must not be a directory: $file")
-
-  def isDirectory: Boolean = file.isDirectory
-
-  // the path of the file within the source
-  def relative: Path = source.toPath.relativize(file.toPath)
-
-  def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _)
-
-  def md5base64: Option[String] = hashes.get(MD5).map(_.hash64)
-
-}
 
 object LocalFile {
-
-  def resolve(
-      path: String,
-      md5Hashes: Map[HashType, MD5Hash],
-      source: Path,
-      pathToKey: Path => RemoteKey
-  ): LocalFile = {
-    val resolvedPath = source.resolve(path)
-    LocalFile(resolvedPath.toFile,
-              source.toFile,
-              md5Hashes,
-              pathToKey(resolvedPath))
-  }
-
   val remoteKey: SimpleLens[LocalFile, RemoteKey] =
     SimpleLens[LocalFile, RemoteKey](_.remoteKey,
                                      b => a => b.copy(remoteKey = a))
+  // the path of the file within the source
+  def relativeToSource(localFile: LocalFile): Path =
+    localFile.source.toPath.relativize(localFile.file.toPath)
+  def matchesHash(localFile: LocalFile)(other: MD5Hash): Boolean =
+    localFile.hashes.values.exists(other equals _)
+  def md5base64(localFile: LocalFile): Option[String] =
+    localFile.hashes.get(MD5).map(MD5Hash.hash64)
 }

domain/src/main/scala/net/kemitix/thorp/domain/Logger.scala

@@ -1,15 +0,0 @@
-package net.kemitix.thorp.domain
-
-trait Logger {
-
-  // returns an instance of Logger with debug set as indicated
-  // where the current Logger already matches this state, then
-  // it returns itself, unmodified
-  def withDebug(debug: Boolean): Logger
-
-  def debug(message: => String): Unit
-  def info(message: => String): Unit
-  def warn(message: String): Unit
-  def error(message: String): Unit
-
-}

domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala

@@ -4,20 +4,14 @@ import java.util.Base64
 
 import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
 
-final case class MD5Hash(
+final case class MD5Hash(in: String)
-    in: String
-) {
-
-  lazy val hash: String = in filter stripQuotes
-
-  lazy val digest: Array[Byte] = HexEncoder.decode(hash)
-
-  lazy val hash64: String = Base64.getEncoder.encodeToString(digest)
-}
 
 object MD5Hash {
-  def fromDigest(digest: Array[Byte]): MD5Hash = {
+  def fromDigest(digest: Array[Byte]): MD5Hash =
-    val hexDigest = (digest map ("%02x" format _)).mkString
+    MD5Hash((digest map ("%02x" format _)).mkString)
-    MD5Hash(hexDigest)
+  def hash(md5Hash: MD5Hash): String = md5Hash.in.filter(stripQuotes)
-  }
+  def digest(md5Hash: MD5Hash): Array[Byte] =
+    HexEncoder.decode(MD5Hash.hash(md5Hash))
+  def hash64(md5Hash: MD5Hash): String =
+    Base64.getEncoder.encodeToString(MD5Hash.digest(md5Hash))
 }

domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala

@@ -3,31 +3,21 @@ package net.kemitix.thorp.domain
 import java.io.File
 import java.nio.file.{Path, Paths}
 
-final case class RemoteKey(
+final case class RemoteKey(key: String)
-    key: String
-) {
-
-  def asFile(
-      source: Path,
-      prefix: RemoteKey
-  ): Option[File] =
-    if (key.length == 0) None
-    else Some(source.resolve(relativeTo(prefix)).toFile)
-
-  private def relativeTo(prefix: RemoteKey) = {
-    prefix match {
-      case RemoteKey("") => Paths.get(key)
-      case _             => Paths.get(prefix.key).relativize(Paths.get(key))
-    }
-  }
-
-  def resolve(path: String): RemoteKey =
-    RemoteKey(List(key, path).filterNot(_.isEmpty).mkString("/"))
-
-}
 
 object RemoteKey {
   val key: SimpleLens[RemoteKey, String] =
     SimpleLens[RemoteKey, String](_.key, b => a => b.copy(key = a))
-
+  def asFile(source: Path, prefix: RemoteKey)(
+      remoteKey: RemoteKey): Option[File] =
+    if (remoteKey.key.length == 0) None
+    else Some(source.resolve(RemoteKey.relativeTo(prefix)(remoteKey)).toFile)
+  def relativeTo(prefix: RemoteKey)(remoteKey: RemoteKey): Path = {
+    prefix match {
+      case RemoteKey("") => Paths.get(remoteKey.key)
+      case _             => Paths.get(prefix.key).relativize(Paths.get(remoteKey.key))
+    }
+  }
+  def resolve(path: String)(remoteKey: RemoteKey): RemoteKey =
+    RemoteKey(List(remoteKey.key, path).filterNot(_.isEmpty).mkString("/"))
 }

domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala

@@ -2,6 +2,8 @@ package net.kemitix.thorp.domain
 
 import java.nio.file.Path
 
+import zio.{Task, ZIO}
+
 /**
   * The paths to synchronise with target.
   *
@@ -18,18 +20,10 @@ case class Sources(
   def +(path: Path)(implicit m: Monoid[Sources]): Sources = this ++ List(path)
   def ++(otherPaths: List[Path])(implicit m: Monoid[Sources]): Sources =
     m.op(this, Sources(otherPaths))
-
-  /**
-    * Returns the source path for the given path.
-    */
-  def forPath(path: Path): Path =
-    paths.find(source => path.startsWith(source)).get
 }
 
 object Sources {
-
   final val emptySources = Sources(List.empty)
-
   implicit def sourcesAppendMonoid: Monoid[Sources] = new Monoid[Sources] {
     override def zero: Sources = emptySources
     override def op(t1: Sources, t2: Sources): Sources =
@@ -38,4 +32,12 @@ object Sources {
         else acc ++ List(path)
       })
   }
+
+  /**
+    * Returns the source path for the given path.
+    */
+  def forPath(path: Path)(sources: Sources): Task[Path] =
+    ZIO
+      .fromOption(sources.paths.find(s => path.startsWith(s)))
+      .mapError(_ => new Exception("Path is not within any known source"))
 }

domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala

@@ -1,10 +1,7 @@
 package net.kemitix.thorp.domain
 
 import net.kemitix.thorp.domain.UploadEvent.RequestEvent
-import net.kemitix.thorp.domain.UploadEventLogger.{
+import net.kemitix.thorp.domain.UploadEventLogger.RequestCycle
-  RequestCycle,
-  logRequestCycle
-}
 
 object UploadEventListener {
 
@@ -21,7 +18,7 @@ object UploadEventListener {
       uploadEvent match {
         case e: RequestEvent =>
           bytesTransferred += e.transferred
-          logRequestCycle(
+          UploadEventLogger(
             RequestCycle(settings.localFile,
                          bytesTransferred,
                          settings.index,

domain/src/main/scala/net/kemitix/thorp/domain/UploadEventLogger.scala

@@ -14,7 +14,7 @@ object UploadEventLogger {
       totalBytesSoFar: Long
   )
 
-  def logRequestCycle(requestCycle: RequestCycle): Unit = {
+  def apply(requestCycle: RequestCycle): Unit = {
     val remoteKey    = requestCycle.localFile.remoteKey.key
     val fileLength   = requestCycle.localFile.file.length
     val statusHeight = 7

domain/src/test/scala/net/kemitix/thorp/domain/MD5HashTest.scala

@@ -7,11 +7,11 @@ class MD5HashTest extends FunSpec {
   describe("recover base64 hash") {
     it("should recover base 64 #1") {
       val rootHash = MD5HashData.Root.hash
-      assertResult(MD5HashData.Root.base64)(rootHash.hash64)
+      assertResult(MD5HashData.Root.base64)(MD5Hash.hash64(rootHash))
     }
     it("should recover base 64 #2") {
       val leafHash = MD5HashData.Leaf.hash
-      assertResult(MD5HashData.Leaf.base64)(leafHash.hash64)
+      assertResult(MD5HashData.Leaf.base64)(MD5Hash.hash64(leafHash))
     }
   }
 }

domain/src/test/scala/net/kemitix/thorp/domain/RemoteKeyTest.scala

@@ -15,21 +15,21 @@ class RemoteKeyTest extends FreeSpec {
         val key      = emptyKey
         val path     = "path"
         val expected = RemoteKey("path")
-        val result   = key.resolve(path)
+        val result   = RemoteKey.resolve(path)(key)
         assertResult(expected)(result)
       }
       "when path is empty" in {
         val key      = RemoteKey("key")
         val path     = ""
         val expected = RemoteKey("key")
-        val result   = key.resolve(path)
+        val result   = RemoteKey.resolve(path)(key)
         assertResult(expected)(result)
       }
       "when key and path are empty" in {
         val key      = emptyKey
         val path     = ""
         val expected = emptyKey
-        val result   = key.resolve(path)
+        val result   = RemoteKey.resolve(path)(key)
         assertResult(expected)(result)
       }
     }
@@ -39,7 +39,7 @@ class RemoteKeyTest extends FreeSpec {
         val source   = Paths.get("source")
         val prefix   = RemoteKey("prefix")
         val expected = Some(new File("source/key"))
-        val result   = key.asFile(source, prefix)
+        val result   = RemoteKey.asFile(source, prefix)(key)
         assertResult(expected)(result)
       }
       "when prefix is empty" in {
@@ -47,7 +47,7 @@ class RemoteKeyTest extends FreeSpec {
         val source   = Paths.get("source")
         val prefix   = emptyKey
         val expected = Some(new File("source/key"))
-        val result   = key.asFile(source, prefix)
+        val result   = RemoteKey.asFile(source, prefix)(key)
         assertResult(expected)(result)
       }
       "when key is empty" in {
@@ -55,7 +55,7 @@ class RemoteKeyTest extends FreeSpec {
         val source   = Paths.get("source")
         val prefix   = RemoteKey("prefix")
         val expected = None
-        val result   = key.asFile(source, prefix)
+        val result   = RemoteKey.asFile(source, prefix)(key)
         assertResult(expected)(result)
       }
       "when key and prefix are empty" in {
@@ -63,7 +63,7 @@ class RemoteKeyTest extends FreeSpec {
         val source   = Paths.get("source")
         val prefix   = emptyKey
         val expected = None
-        val result   = key.asFile(source, prefix)
+        val result   = RemoteKey.asFile(source, prefix)(key)
         assertResult(expected)(result)
       }
     }

domain/src/test/scala/net/kemitix/thorp/domain/TemporaryFolder.scala

@@ -38,11 +38,13 @@ trait TemporaryFolder {
     path.resolve(name).toFile
   }
 
-  def writeFile(directory: Path, name: String, contents: String*): Unit = {
+  def writeFile(directory: Path, name: String, contents: String*): File = {
     directory.toFile.mkdirs
-    val pw = new PrintWriter(directory.resolve(name).toFile, "UTF-8")
+    val file   = directory.resolve(name).toFile
-    contents.foreach(pw.println)
+    val writer = new PrintWriter(file, "UTF-8")
-    pw.close()
+    contents.foreach(writer.println)
+    writer.close()
+    file
   }
 
 }

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Copier.scala

@@ -41,7 +41,7 @@ trait Copier {
       copyRequest.sourceKey.key,
       copyRequest.bucket.name,
       copyRequest.targetKey.key
-    ).withMatchingETagConstraint(copyRequest.hash.hash)
+    ).withMatchingETagConstraint(MD5Hash.hash(copyRequest.hash))
 
   private def foldFailure(
       sourceKey: RemoteKey,

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Uploader.scala

@@ -68,7 +68,7 @@ trait Uploader {
 
   private def metadata: LocalFile => ObjectMetadata = localFile => {
     val metadata = new ObjectMetadata()
-    localFile.md5base64.foreach(metadata.setContentMD5)
+    LocalFile.md5base64(localFile).foreach(metadata.setContentMD5)
     metadata
   }
 

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala

@@ -7,6 +7,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
 import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils
 import net.kemitix.thorp.core.hasher.Hasher
 import net.kemitix.thorp.domain.HashType.MD5
+import net.kemitix.thorp.domain.MD5Hash
 import net.kemitix.thorp.filesystem.FileSystem
 import zio.{TaskR, ZIO}
 
@@ -56,7 +57,7 @@ private trait ETagGenerator {
     Hasher
       .hashObjectChunk(path, chunkNumber, chunkSize)
       .map(_(MD5))
-      .map(_.digest)
+      .map(MD5Hash.digest)
 
   def offsets(
       totalFileSizeBytes: Long,

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala

@@ -32,7 +32,7 @@ class S3ObjectsByHashSuite extends FunSpec {
                        remoteKey: RemoteKey,
                        lastModified: LastModified): S3ObjectSummary = {
     val summary = new S3ObjectSummary()
-    summary.setETag(md5Hash.hash)
+    summary.setETag(MD5Hash.hash(md5Hash))
     summary.setKey(remoteKey.key)
     summary.setLastModified(Date.from(lastModified.when))
     summary

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala

@@ -3,7 +3,7 @@ package net.kemitix.thorp.storage.aws
 import java.time.Instant
 
 import net.kemitix.thorp.config.Resource
-import net.kemitix.thorp.core.{KeyGenerator, S3MetaDataEnricher}
+import net.kemitix.thorp.core.{LocalFileValidator, S3MetaDataEnricher}
 import net.kemitix.thorp.domain.HashType.MD5
 import net.kemitix.thorp.domain._
 import org.scalamock.scalatest.MockFactory
@@ -15,26 +15,30 @@ class StorageServiceSuite extends FunSpec with MockFactory {
   private val sourcePath = source.toPath
   private val sources    = Sources(List(sourcePath))
   private val prefix     = RemoteKey("prefix")
-  private val fileToKey =
-    KeyGenerator.generateKey(sources, prefix) _
 
   describe("getS3Status") {
 
     val hash = MD5Hash("hash")
-    val localFile =
+    val env = for {
-      LocalFile.resolve("the-file", Map(MD5 -> hash), sourcePath, fileToKey)
+      localFile <- LocalFileValidator.resolve("the-file",
-    val key = localFile.remoteKey
-    val keyOtherKey = LocalFile.resolve("other-key-same-hash",
                                               Map(MD5 -> hash),
                                               sourcePath,
-                                        fileToKey)
+                                              sources,
-    val diffHash = MD5Hash("diff")
+                                              prefix)
-    val keyDiffHash = LocalFile.resolve("other-key-diff-hash",
+      key = localFile.remoteKey
+      keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash",
+                                                Map(MD5 -> hash),
+                                                sourcePath,
+                                                sources,
+                                                prefix)
+      diffHash = MD5Hash("diff")
+      keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash",
                                                 Map(MD5 -> diffHash),
                                                 sourcePath,
-                                        fileToKey)
+                                                sources,
-    val lastModified = LastModified(Instant.now)
+                                                prefix)
-    val s3ObjectsData: S3ObjectsData = S3ObjectsData(
+      lastModified = LastModified(Instant.now)
+      s3ObjectsData = S3ObjectsData(
         byHash = Map(
           hash -> Set(KeyModified(key, lastModified),
                       KeyModified(keyOtherKey.remoteKey, lastModified)),
@@ -46,8 +50,16 @@ class StorageServiceSuite extends FunSpec with MockFactory {
           keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
         )
       )
+    } yield
+      (s3ObjectsData,
+       localFile: LocalFile,
+       lastModified,
+       keyOtherKey,
+       keyDiffHash,
+       diffHash,
+       key)
 
-    def invoke(localFile: LocalFile) =
+    def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) =
       S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData)
 
     def getMatchesByKey(
@@ -67,47 +79,94 @@ class StorageServiceSuite extends FunSpec with MockFactory {
     describe(
       "when remote key exists, unmodified and other key matches the hash") {
       it("should return the match by key") {
-        val result = getMatchesByKey(invoke(localFile))
+        env.map({
+          case (s3ObjectsData, localFile, lastModified, _, _, _, _) => {
+            val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
             assert(result.contains(HashModified(hash, lastModified)))
           }
+        })
+      }
       it("should return both matches for the hash") {
-        val result = getMatchesByHash(invoke(localFile))
+        env.map({
+          case (s3ObjectsData,
+                localFile,
+                lastModified,
+                keyOtherKey,
+                _,
+                _,
+                key) => {
+            val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
             assertResult(
               Set((hash, KeyModified(key, lastModified)),
                   (hash, KeyModified(keyOtherKey.remoteKey, lastModified)))
             )(result)
           }
+        })
+      }
     }
 
     describe("when remote key does not exist and no others matches hash") {
-      val localFile = LocalFile.resolve("missing-file",
+      val env2 = LocalFileValidator
+        .resolve("missing-file",
                  Map(MD5 -> MD5Hash("unique")),
                  sourcePath,
-                                        fileToKey)
+                 sources,
+                 prefix)
       it("should return no matches by key") {
-        val result = getMatchesByKey(invoke(localFile))
+        env2.map(localFile => {
+          env.map({
+            case (s3ObjectsData, _, _, _, _, _, _) => {
+              val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
               assert(result.isEmpty)
             }
+          })
+        })
+      }
       it("should return no matches by hash") {
-        val result = getMatchesByHash(invoke(localFile))
+        env2.map(localFile => {
+          env.map({
+            case (s3ObjectsData, _, _, _, _, _, _) => {
+              val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
               assert(result.isEmpty)
             }
+          })
+        })
+      }
     }
 
     describe("when remote key exists and no others match hash") {
-      val localFile = keyDiffHash
       it("should return the match by key") {
-        val result = getMatchesByKey(invoke(localFile))
+        env.map({
+            case (s3ObjectsData,
+                  _,
+                  lastModified,
+                  _,
+                  keyDiffHash,
+                  diffHash,
+                  _) => {
+              val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData))
               assert(result.contains(HashModified(diffHash, lastModified)))
             }
+          })
+      }
       it("should return one match by hash") {
-        val result = getMatchesByHash(invoke(localFile))
+        env.map({
+            case (s3ObjectsData,
+                  _,
+                  lastModified,
+                  _,
+                  keyDiffHash,
+                  diffHash,
+                  _) => {
+              val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData))
               assertResult(
-          Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))
+                Set(
+                  (diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))
               )(result)
             }
+          })
+      }
     }
-
   }
 
 }

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala

@@ -29,7 +29,7 @@ class UploaderTest extends FreeSpec with MockFactory {
     val bucket        = Bucket("aBucket")
     val uploadResult  = new UploadResult
     uploadResult.setKey(remoteKey.key)
-    uploadResult.setETag(aHash.hash)
+    uploadResult.setETag(MD5Hash.hash(aHash))
     val inProgress = new AmazonUpload.InProgress {
       override def waitForUploadResult: UploadResult = uploadResult
     }

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala

@@ -6,6 +6,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
 import net.kemitix.thorp.config.Resource
 import net.kemitix.thorp.core.hasher.Hasher
 import net.kemitix.thorp.domain.HashType.MD5
+import net.kemitix.thorp.domain.MD5Hash
 import net.kemitix.thorp.filesystem.FileSystem
 import org.scalatest.FunSpec
 import zio.DefaultRuntime
@@ -45,7 +46,7 @@ class ETagGeneratorTest extends FunSpec {
           assertResult(Right(hash))(
             invoke(bigFilePath, index, chunkSize)
               .map(_(MD5))
-              .map(_.hash))
+              .map(MD5Hash.hash))
       }
     }
     def invoke(path: Path, index: Long, size: Long) =