kemitix/thorp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

case classes shouldn't be OO objects (#147)

Browse source 
* [core] Extract Filters from domain.Filter

* [core] extract LocalFileValidator

* [domain] LocalFile remove unused isDirectory

* [domain] LocalFile move/rename relative as relativeToSource on companion

* [domain] LocalFile move and rename matches as matchesHash on companion

* [domain] LocalFile move md5base64 to companion

* [domain] Logger remove

* [domain] MD5Hash move hash to companion

* [domain] MD5Hash move digest to companion

* [domain] MD5Hash move hash64 to companion

* [domain] RemoteKey move class methods to companion

* [domain] Sources move forPath to companion

Led to being able to cleanup LocalFileStream.localFile and adding
LocalFiles.one

* [domain] UploadEventLogger rename method as apply
This commit is contained in:
Paul Campbell 2019-08-05 08:38:44 +01:00 committed by GitHub
parent 96ecedbe61
commit 8fad680a96
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 750 additions and 551 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
build.sbt
View file

@ -135,3 +135,4 @@ lazy val domain = (project in file("domain"))
.settings(commonSettings)
.settings(assemblyJarName in assembly := "domain.jar")
.settings(testDependencies)
.settings(zioDependencies)

6
config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala
View file

@ -1,6 +1,6 @@
package net.kemitix.thorp.config
import java.nio.file.Path
import java.io.File
sealed trait ConfigValidation {
@ -22,10 +22,10 @@ object ConfigValidation {
}
case class ErrorReadingFile(
path: Path,
file: File,
message: String
) extends ConfigValidation {
override def errorMessage: String = s"Error reading file '$path': $message"
override def errorMessage: String = s"Error reading file '$file': $message"
}
}

10
config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala
View file

@ -1,6 +1,6 @@
package net.kemitix.thorp.config
import java.nio.file.Paths
import java.io.File
import net.kemitix.thorp.filesystem.FileSystem
import zio.ZIO
@ -11,9 +11,9 @@ import zio.ZIO
*/
trait ConfigurationBuilder {
private val userConfigFilename = ".config/thorp.conf"
private val globalConfig = Paths.get("/etc/thorp.conf")
private val userHome = Paths.get(System.getProperty("user.home"))
private val userConfigFile = ".config/thorp.conf"
private val globalConfig = new File("/etc/thorp.conf")
private val userHome = new File(System.getProperty("user.home"))
def buildConfig(priorityOpts: ConfigOptions)
: ZIO[FileSystem, ConfigValidationException, Configuration] =
@ -33,7 +33,7 @@ trait ConfigurationBuilder {
private def userOptions(priorityOpts: ConfigOptions) =
if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
else ParseConfigFile.parseFile(userHome.resolve(userConfigFilename))
else ParseConfigFile.parseFile(new File(userHome, userConfigFile))
private def globalOptions(priorityOpts: ConfigOptions) =
if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig

19
config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala
View file

@ -1,6 +1,6 @@
package net.kemitix.thorp.config
import java.nio.file.Path
import java.io.File
import net.kemitix.thorp.filesystem.FileSystem
import zio.{IO, TaskR, ZIO}
@ -8,19 +8,14 @@ import zio.{IO, TaskR, ZIO}
trait ParseConfigFile {
def parseFile(
filename: Path): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
(readFile(filename) >>= ParseConfigLines.parseLines)
.catchAll(
h =>
IO.fail(
List(ConfigValidation.ErrorReadingFile(filename, h.getMessage))))
file: File): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
(FileSystem.exists(file) >>= readLines(file) >>= ParseConfigLines.parseLines)
.catchAll(h =>
IO.fail(List(ConfigValidation.ErrorReadingFile(file, h.getMessage))))
private def readFile(filename: Path) =
FileSystem.exists(filename.toFile) >>= readLines(filename)
private def readLines(filename: Path)(
private def readLines(file: File)(
exists: Boolean): TaskR[FileSystem, Seq[String]] =
if (exists) FileSystem.lines(filename.toFile)
if (exists) FileSystem.lines(file)
else ZIO.succeed(Seq.empty)
}

4
config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala
View file

@ -1,5 +1,7 @@
package net.kemitix.thorp.config
import java.io.File
import net.kemitix.thorp.domain.Sources
import net.kemitix.thorp.filesystem.FileSystem
import zio.ZIO
@ -12,7 +14,7 @@ trait SourceConfigLoader {
sources: Sources): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
ZIO
.foreach(sources.paths) { path =>
ParseConfigFile.parseFile(path.resolve(thorpConfigFileName))
ParseConfigFile.parseFile(new File(path.toFile, thorpConfigFileName))
}
.map(_.foldLeft(ConfigOptions(sources.paths.map(ConfigOption.Source))) {
(acc, co) =>

41
config/src/test/scala/net/kemitix/thorp/config/ParseConfigFileTest.scala
View file

@ -1,47 +1,58 @@
package net.kemitix.thorp.config
import java.nio.file.{Path, Paths}
import java.io.File
import java.nio.file.Paths
import net.kemitix.thorp.domain.TemporaryFolder
import net.kemitix.thorp.filesystem.FileSystem
import org.scalatest.FunSpec
import zio.DefaultRuntime
class ParseConfigFileTest extends FunSpec {
class ParseConfigFileTest extends FunSpec with TemporaryFolder {
private val empty = Right(ConfigOptions())
describe("parse a missing file") {
val filename = Paths.get("/path/to/missing/file")
val file = new File("/path/to/missing/file")
it("should return no options") {
assertResult(empty)(invoke(filename))
assertResult(empty)(invoke(file))
}
}
describe("parse an empty file") {
val filename = Resource(this, "empty-file").toPath
it("should return no options") {
assertResult(empty)(invoke(filename))
withDirectory(dir => {
val file = writeFile(dir, "empty-file")
assertResult(empty)(invoke(file))
})
}
}
describe("parse a file with no valid entries") {
val filename = Resource(this, "invalid-config").toPath
it("should return no options") {
assertResult(empty)(invoke(filename))
withDirectory(dir => {
val file = writeFile(dir, "invalid-config", "no valid = config items")
assertResult(empty)(invoke(file))
})
}
}
describe("parse a file with properties") {
val filename = Resource(this, "simple-config").toPath
val expected = Right(
ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source")),
ConfigOption.Bucket("bucket-name"))))
it("should return some options") {
assertResult(expected)(invoke(filename))
val expected = Right(
ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source")),
ConfigOption.Bucket("bucket-name"))))
withDirectory(dir => {
val file = writeFile(dir,
"simple-config",
"source = /path/to/source",
"bucket = bucket-name")
assertResult(expected)(invoke(file))
})
}
}
private def invoke(filename: Path) = {
private def invoke(file: File) = {
new DefaultRuntime {}.unsafeRunSync {
ParseConfigFile
.parseFile(filename)
.parseFile(file)
.provide(FileSystem.Live)
}.toEither
}

4
core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala
View file

@ -21,7 +21,7 @@ object ActionGenerator {
s3MetaData match {
// #1 local exists, remote exists, remote matches - do nothing
case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
if localFile.matches(hash) =>
if LocalFile.matchesHash(localFile)(hash) =>
doNothing(bucket, key)
// #2 local exists, remote is missing, other matches - copy
case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
@ -33,7 +33,7 @@ object ActionGenerator {
uploadFile(bucket, localFile)
// #4 local exists, remote exists, remote no match, other matches - copy
case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _)))
if !localFile.matches(hash) &&
if !LocalFile.matchesHash(localFile)(hash) &&
matchByHash.nonEmpty =>
copyFile(bucket, localFile, matchByHash)
// #5 local exists, remote exists, remote no match, other no matches - upload

41
core/src/main/scala/net/kemitix/thorp/core/Filters.scala Normal file
View file

@ -0,0 +1,41 @@
package net.kemitix.thorp.core
import java.nio.file.Path
import net.kemitix.thorp.domain.Filter
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
object Filters {
def isIncluded(p: Path)(filters: List[Filter]): Boolean = {
sealed trait State
case class Unknown() extends State
case class Accepted() extends State
case class Discarded() extends State
val excluded = isExcludedByFilter(p)(_)
val included = isIncludedByFilter(p)(_)
filters.foldRight(Unknown(): State)((filter, state) =>
(filter, state) match {
case (_, Accepted()) => Accepted()
case (_, Discarded()) => Discarded()
case (x: Exclude, _) if excluded(x) => Discarded()
case (i: Include, _) if included(i) => Accepted()
case _ => Unknown()
}) match {
case Accepted() => true
case Discarded() => false
case Unknown() =>
filters.forall {
case _: Include => false
case _ => true
}
}
}
def isIncludedByFilter(path: Path)(filter: Filter): Boolean =
filter.predicate.test(path.toString)
def isExcludedByFilter(path: Path)(filter: Filter): Boolean =
filter.predicate.test(path.toString)
}

16
core/src/main/scala/net/kemitix/thorp/core/KeyGenerator.scala
View file

@ -3,21 +3,17 @@ package net.kemitix.thorp.core
import java.nio.file.Path
import net.kemitix.thorp.domain.{RemoteKey, Sources}
import zio.Task
object KeyGenerator {
def generateKey(
sources: Sources,
prefix: RemoteKey
)(path: Path): RemoteKey = {
val source = sources.forPath(path)
val relativePath = source.relativize(path.toAbsolutePath).toString
RemoteKey(
List(
prefix.key,
relativePath
).filter(_.nonEmpty)
.mkString("/"))
}
)(path: Path): Task[RemoteKey] =
Sources
.forPath(path)(sources)
.map(p => p.relativize(path.toAbsolutePath).toString)
.map(RemoteKey.resolve(_)(prefix))
}

24
core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala
View file

@ -4,9 +4,8 @@ import java.io.File
import java.nio.file.Path
import net.kemitix.thorp.config.Config
import net.kemitix.thorp.core.KeyGenerator.generateKey
import net.kemitix.thorp.core.hasher.Hasher
import net.kemitix.thorp.domain._
import net.kemitix.thorp.domain.Sources
import net.kemitix.thorp.filesystem.FileSystem
import zio.{Task, TaskR, ZIO}
@ -48,21 +47,18 @@ object LocalFileStream {
.filter({ case (_, included) => included })
.map({ case (path, _) => path })
private def localFile(path: Path) = {
val file = path.toFile
private def localFile(path: Path) =
for {
sources <- Config.sources
prefix <- Config.prefix
source <- Sources.forPath(path)(sources)
hash <- Hasher.hashObject(path)
localFile = LocalFile(file,
sources.forPath(path).toFile,
hash,
generateKey(sources, prefix)(path))
} yield
LocalFiles(localFiles = Stream(localFile),
count = 1,
totalSizeBytes = file.length)
}
localFile <- LocalFileValidator.validate(path,
source.toFile,
hash,
sources,
prefix)
} yield LocalFiles.one(localFile)
private def listFiles(path: Path) =
for {
@ -78,6 +74,6 @@ object LocalFileStream {
private def isIncluded(path: Path) =
for {
filters <- Config.filters
} yield Filter.isIncluded(path)(filters)
} yield Filters.isIncluded(path)(filters)
}

66
core/src/main/scala/net/kemitix/thorp/core/LocalFileValidator.scala Normal file
View file

@ -0,0 +1,66 @@
package net.kemitix.thorp.core
import java.io.File
import java.nio.file.Path
import net.kemitix.thorp.domain.{
HashType,
LocalFile,
MD5Hash,
RemoteKey,
Sources
}
import zio.{IO, ZIO}
object LocalFileValidator {
def validate(
path: Path,
source: File,
hash: Map[HashType, MD5Hash],
sources: Sources,
prefix: RemoteKey
): IO[Violation, LocalFile] =
for {
vFile <- validateFile(path.toFile)
remoteKey <- validateRemoteKey(sources, prefix, path)
} yield LocalFile(vFile, source, hash, remoteKey)
private def validateFile(file: File) =
if (file.isDirectory)
ZIO.fail(Violation.IsNotAFile(file))
else
ZIO.succeed(file)
private def validateRemoteKey(sources: Sources,
prefix: RemoteKey,
path: Path) =
KeyGenerator
.generateKey(sources, prefix)(path)
.mapError(e => Violation.InvalidRemoteKey(path, e))
sealed trait Violation extends Throwable {
def getMessage: String
}
object Violation {
case class IsNotAFile(file: File) extends Violation {
override def getMessage: String = s"Local File must be a file: ${file}"
}
case class InvalidRemoteKey(path: Path, e: Throwable) extends Violation {
override def getMessage: String =
s"Remote Key for '${path}' is invalid: ${e.getMessage}"
}
}
def resolve(
path: String,
md5Hashes: Map[HashType, MD5Hash],
source: Path,
sources: Sources,
prefix: RemoteKey
): IO[Violation, LocalFile] = {
val resolvedPath = source.resolve(path)
validate(resolvedPath, source.toFile, md5Hashes, sources, prefix)
}
}

5
core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala
View file

@ -13,12 +13,11 @@ case class LocalFiles(
count = count + append.count,
totalSizeBytes = totalSizeBytes + append.totalSizeBytes
)
}
object LocalFiles {
def reduce: Stream[LocalFiles] => LocalFiles =
list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf)
def one(localFile: LocalFile): LocalFiles =
LocalFiles(Stream(localFile), 1, localFile.file.length)
}

2
core/src/main/scala/net/kemitix/thorp/core/Remote.scala
View file

@ -18,7 +18,7 @@ object Remote {
remoteKey: RemoteKey
): TaskR[FileSystem, Boolean] = {
def existsInSource(source: Path) =
remoteKey.asFile(source, prefix) match {
RemoteKey.asFile(source, prefix)(remoteKey) match {
case Some(file) => FileSystem.exists(file)
case None => ZIO.succeed(false)
}

227
core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala
View file

@ -2,13 +2,7 @@ package net.kemitix.thorp.core
import java.time.Instant
import net.kemitix.thorp.config.{
Config,
ConfigOption,
ConfigOptions,
ConfigurationBuilder,
Resource
}
import net.kemitix.thorp.config._
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
@ -31,8 +25,6 @@ class ActionGeneratorSuite extends FunSpec {
ConfigOption.IgnoreUserOptions,
ConfigOption.IgnoreGlobalOptions
))
private val fileToKey =
KeyGenerator.generateKey(sources, prefix) _
describe("create actions") {
@ -40,119 +32,156 @@ class ActionGeneratorSuite extends FunSpec {
describe("#1 local exists, remote exists, remote matches - do nothing") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val theRemoteMetadata =
RemoteMetaData(theFile.remoteKey, theHash, lastModified)
val input =
S3MetaData(theFile, // local exists
matchByHash = Set(theRemoteMetadata), // remote matches
matchByKey = Some(theRemoteMetadata) // remote exists
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
theRemoteMetadata = RemoteMetaData(theFile.remoteKey,
theHash,
lastModified)
input = S3MetaData(
theFile, // local exists
matchByHash = Set(theRemoteMetadata), // remote matches
matchByKey = Some(theRemoteMetadata) // remote exists
)
} yield (theFile, input)
it("do nothing") {
val expected =
Right(
Stream(DoNothing(bucket, theFile.remoteKey, theFile.file.length)))
val result = invoke(input, previousActions)
assertResult(expected)(result)
env.map({
case (theFile, input) => {
val expected =
Right(Stream(
DoNothing(bucket, theFile.remoteKey, theFile.file.length + 1)))
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
}
}
describe("#2 local exists, remote is missing, other matches - copy") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val theRemoteKey = theFile.remoteKey
val otherRemoteKey = prefix.resolve("other-key")
val otherRemoteMetadata =
RemoteMetaData(otherRemoteKey, theHash, lastModified)
val input =
S3MetaData(theFile, // local exists
matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = None) // remote is missing
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
theRemoteKey = theFile.remoteKey
otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
theHash,
lastModified)
input = S3MetaData(
theFile, // local exists
matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = None) // remote is missing
} yield (theFile, theRemoteKey, input, otherRemoteKey)
it("copy from other key") {
val expected = Right(
Stream(
ToCopy(bucket,
otherRemoteKey,
theHash,
theRemoteKey,
theFile.file.length))) // copy
val result = invoke(input, previousActions)
assertResult(expected)(result)
env.map({
case (theFile, theRemoteKey, input, otherRemoteKey) => {
val expected = Right(
Stream(
ToCopy(bucket,
otherRemoteKey,
theHash,
theRemoteKey,
theFile.file.length))) // copy
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
}
}
describe("#3 local exists, remote is missing, other no matches - upload") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val input = S3MetaData(theFile, // local exists
describe("#3 local exists, remote is missing, other no matches - upload") {
val theHash = MD5Hash("the-hash")
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
input = S3MetaData(theFile, // local exists
matchByHash = Set.empty, // other no matches
matchByKey = None) // remote is missing
it("upload") {
val expected = Right(
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
val result = invoke(input, previousActions)
assertResult(expected)(result)
} yield (theFile, input)
it("upload") {
env.map({
case (theFile, input) => {
val expected = Right(Stream(
ToUpload(bucket, theFile, theFile.file.length))) // upload
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
}
}
}
describe(
"#4 local exists, remote exists, remote no match, other matches - copy") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val theRemoteKey = theFile.remoteKey
val oldHash = MD5Hash("old-hash")
val otherRemoteKey = prefix.resolve("other-key")
val otherRemoteMetadata =
RemoteMetaData(otherRemoteKey, theHash, lastModified)
val oldRemoteMetadata = RemoteMetaData(theRemoteKey,
hash = oldHash, // remote no match
lastModified = lastModified)
val input =
S3MetaData(theFile, // local exists
matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = Some(oldRemoteMetadata)) // remote exists
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
theRemoteKey = theFile.remoteKey
oldHash = MD5Hash("old-hash")
otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
theHash,
lastModified)
oldRemoteMetadata = RemoteMetaData(theRemoteKey,
hash = oldHash, // remote no match
lastModified = lastModified)
input = S3MetaData(
theFile, // local exists
matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = Some(oldRemoteMetadata)) // remote exists
} yield (theFile, theRemoteKey, input, otherRemoteKey)
it("copy from other key") {
val expected = Right(
Stream(
ToCopy(bucket,
otherRemoteKey,
theHash,
theRemoteKey,
theFile.file.length))) // copy
val result = invoke(input, previousActions)
assertResult(expected)(result)
env.map({
case (theFile, theRemoteKey, input, otherRemoteKey) => {
val expected = Right(
Stream(
ToCopy(bucket,
otherRemoteKey,
theHash,
theRemoteKey,
theFile.file.length))) // copy
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
}
}
describe(
"#5 local exists, remote exists, remote no match, other no matches - upload") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val theRemoteKey = theFile.remoteKey
val oldHash = MD5Hash("old-hash")
val theRemoteMetadata =
RemoteMetaData(theRemoteKey, oldHash, lastModified)
val input =
S3MetaData(theFile, // local exists
matchByHash = Set.empty, // remote no match, other no match
matchByKey = Some(theRemoteMetadata) // remote exists
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
theRemoteKey = theFile.remoteKey
oldHash = MD5Hash("old-hash")
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
input = S3MetaData(
theFile, // local exists
matchByHash = Set.empty, // remote no match, other no match
matchByKey = Some(theRemoteMetadata) // remote exists
)
} yield (theFile, input)
it("upload") {
val expected = Right(
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
val result = invoke(input, previousActions)
assertResult(expected)(result)
env.map({
case (theFile, input) => {
val expected = Right(
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
}
}
describe("#6 local missing, remote exists - delete") {

28
domain/src/test/scala/net/kemitix/thorp/domain/FiltersSuite.scala → core/src/test/scala/net/kemitix/thorp/core/FiltersSuite.scala
View file

@ -1,7 +1,8 @@
package net.kemitix.thorp.domain
package net.kemitix.thorp.core
import java.nio.file.Paths
import net.kemitix.thorp.domain.Filter
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
import org.scalatest.FunSpec
@ -21,31 +22,32 @@ class FiltersSuite extends FunSpec {
describe("default filter") {
val include = Include()
it("should include files") {
paths.foreach(path => assertResult(true)(include.isIncluded(path)))
paths.foreach(path =>
assertResult(true)(Filters.isIncludedByFilter(path)(include)))
}
}
describe("directory exact match include '/upload/subdir/'") {
val include = Include("/upload/subdir/")
it("include matching directory") {
val matching = Paths.get("/upload/subdir/leaf-file")
assertResult(true)(include.isIncluded(matching))
assertResult(true)(Filters.isIncludedByFilter(matching)(include))
}
it("exclude non-matching files") {
val nonMatching = Paths.get("/upload/other-file")
assertResult(false)(include.isIncluded(nonMatching))
assertResult(false)(Filters.isIncludedByFilter(nonMatching)(include))
}
}
describe("file partial match 'root'") {
val include = Include("root")
it("include matching file '/upload/root-file") {
val matching = Paths.get("/upload/root-file")
assertResult(true)(include.isIncluded(matching))
assertResult(true)(Filters.isIncludedByFilter(matching)(include))
}
it("exclude non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
val nonMatching1 = Paths.get("/test-file-for-hash.txt")
val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
assertResult(false)(include.isIncluded(nonMatching1))
assertResult(false)(include.isIncluded(nonMatching2))
assertResult(false)(Filters.isIncludedByFilter(nonMatching1)(include))
assertResult(false)(Filters.isIncludedByFilter(nonMatching2)(include))
}
}
}
@ -63,30 +65,30 @@ class FiltersSuite extends FunSpec {
val exclude = Exclude("/upload/subdir/")
it("exclude matching directory") {
val matching = Paths.get("/upload/subdir/leaf-file")
assertResult(true)(exclude.isExcluded(matching))
assertResult(true)(Filters.isExcludedByFilter(matching)(exclude))
}
it("include non-matching files") {
val nonMatching = Paths.get("/upload/other-file")
assertResult(false)(exclude.isExcluded(nonMatching))
assertResult(false)(Filters.isExcludedByFilter(nonMatching)(exclude))
}
}
describe("file partial match 'root'") {
val exclude = Exclude("root")
it("exclude matching file '/upload/root-file") {
val matching = Paths.get("/upload/root-file")
assertResult(true)(exclude.isExcluded(matching))
assertResult(true)(Filters.isExcludedByFilter(matching)(exclude))
}
it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
val nonMatching1 = Paths.get("/test-file-for-hash.txt")
val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
assertResult(false)(exclude.isExcluded(nonMatching1))
assertResult(false)(exclude.isExcluded(nonMatching2))
assertResult(false)(Filters.isExcludedByFilter(nonMatching1)(exclude))
assertResult(false)(Filters.isExcludedByFilter(nonMatching2)(exclude))
}
}
}
describe("isIncluded") {
def invoke(filters: List[Filter]) = {
paths.filter(path => Filter.isIncluded(path)(filters))
paths.filter(path => Filters.isIncluded(path)(filters))
}
describe("when there are no filters") {

24
core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala
View file

@ -1,10 +1,12 @@
package net.kemitix.thorp.core
import java.io.File
import java.nio.file.Path
import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.domain.{RemoteKey, Sources}
import org.scalatest.FunSpec
import zio.DefaultRuntime
class KeyGeneratorSuite extends FunSpec {
@ -12,26 +14,32 @@ class KeyGeneratorSuite extends FunSpec {
private val sourcePath = source.toPath
private val prefix = RemoteKey("prefix")
private val sources = Sources(List(sourcePath))
private val fileToKey =
KeyGenerator.generateKey(sources, prefix) _
describe("key generator") {
describe("when file is within source") {
it("has a valid key") {
val subdir = "subdir"
assertResult(RemoteKey(s"${prefix.key}/$subdir"))(
fileToKey(sourcePath.resolve(subdir)))
val subdir = "subdir"
val expected = Right(RemoteKey(s"${prefix.key}/$subdir"))
val result = invoke(sourcePath.resolve(subdir))
assertResult(expected)(result)
}
}
describe("when file is deeper within source") {
it("has a valid key") {
val subdir = "subdir/deeper/still"
assertResult(RemoteKey(s"${prefix.key}/$subdir"))(
fileToKey(sourcePath.resolve(subdir)))
val subdir = "subdir/deeper/still"
val expected = Right(RemoteKey(s"${prefix.key}/$subdir"))
val result = invoke(sourcePath.resolve(subdir))
assertResult(expected)(result)
}
}
def invoke(path: Path) = {
new DefaultRuntime {}.unsafeRunSync {
KeyGenerator.generateKey(sources, prefix)(path)
}.toEither
}
}
}

2
core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala
View file

@ -32,7 +32,7 @@ class LocalFileStreamSuite extends FunSpec {
val result =
invoke()
.map(_.localFiles)
.map(_.map(_.relative.toString))
.map(_.map(LocalFile.relativeToSource(_).toString))
.map(_.toSet)
assertResult(expected)(result)
}

12
core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala
View file

@ -333,13 +333,17 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
md5Hash: MD5Hash,
source: Path,
file: File): (String, String, String, String, String) =
("upload", remoteKey.key, md5Hash.hash, source.toString, file.toString)
("upload",
remoteKey.key,
MD5Hash.hash(md5Hash),
source.toString,
file.toString)
private def toCopy(
sourceKey: RemoteKey,
md5Hash: MD5Hash,
targetKey: RemoteKey): (String, String, String, String, String) =
("copy", sourceKey.key, md5Hash.hash, targetKey.key, "")
("copy", sourceKey.key, MD5Hash.hash(md5Hash), targetKey.key, "")
private def toDelete(
remoteKey: RemoteKey): (String, String, String, String, String) =
@ -386,12 +390,12 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
case ToUpload(_, lf, _) =>
("upload",
lf.remoteKey.key,
lf.hashes(MD5).hash,
MD5Hash.hash(lf.hashes(MD5)),
lf.source.toString,
lf.file.toString)
case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")
case ToCopy(_, sourceKey, hash, targetKey, _) =>
("copy", sourceKey.key, hash.hash, targetKey.key, "")
("copy", sourceKey.key, MD5Hash.hash(hash), targetKey.key, "")
case DoNothing(_, remoteKey, _) =>
("do-nothing", remoteKey.key, "", "", "")
case x => ("other", x.toString, "", "", "")

366
core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala
View file

@ -14,8 +14,6 @@ class S3MetaDataEnricherSuite extends FunSpec {
private val sourcePath = source.toPath
private val sources = Sources(List(sourcePath))
private val prefix = RemoteKey("prefix")
private val fileToKey =
KeyGenerator.generateKey(sources, prefix) _
def getMatchesByKey(
status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
@ -36,138 +34,180 @@ class S3MetaDataEnricherSuite extends FunSpec {
describe(
"#1a local exists, remote exists, remote matches, other matches - do nothing") {
val theHash: MD5Hash = MD5Hash("the-file-hash")
val theFile: LocalFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val theRemoteKey: RemoteKey = theFile.remoteKey
val s3: S3ObjectsData = S3ObjectsData(
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
)
val theRemoteMetadata =
RemoteMetaData(theRemoteKey, theHash, lastModified)
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
theRemoteKey = theFile.remoteKey
s3 = S3ObjectsData(
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
)
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
} yield (theFile, theRemoteMetadata, s3)
it("generates valid metadata") {
val expected = S3MetaData(theFile,
matchByHash = Set(theRemoteMetadata),
matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
env.map({
case (theFile, theRemoteMetadata, s3) => {
val expected = S3MetaData(theFile,
matchByHash = Set(theRemoteMetadata),
matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
}
}
describe(
"#1b local exists, remote exists, remote matches, other no matches - do nothing") {
val theHash: MD5Hash = MD5Hash("the-file-hash")
val theFile: LocalFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val theRemoteKey: RemoteKey = prefix.resolve("the-file")
val s3: S3ObjectsData = S3ObjectsData(
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
)
val theRemoteMetadata =
RemoteMetaData(theRemoteKey, theHash, lastModified)
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix)
s3: S3ObjectsData = S3ObjectsData(
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
)
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
} yield (theFile, theRemoteMetadata, s3)
it("generates valid metadata") {
val expected = S3MetaData(theFile,
matchByHash = Set(theRemoteMetadata),
matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
env.map({
case (theFile, theRemoteMetadata, s3) => {
val expected = S3MetaData(theFile,
matchByHash = Set(theRemoteMetadata),
matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
}
}
describe(
"#2 local exists, remote is missing, remote no match, other matches - copy") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val otherRemoteKey = RemoteKey("other-key")
val s3: S3ObjectsData = S3ObjectsData(
byHash = Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
)
val otherRemoteMetadata =
RemoteMetaData(otherRemoteKey, theHash, lastModified)
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
otherRemoteKey = RemoteKey("other-key")
s3: S3ObjectsData = S3ObjectsData(
byHash =
Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
)
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
theHash,
lastModified)
} yield (theFile, otherRemoteMetadata, s3)
it("generates valid metadata") {
val expected = S3MetaData(theFile,
matchByHash = Set(otherRemoteMetadata),
matchByKey = None)
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
env.map({
case (theFile, otherRemoteMetadata, s3) => {
val expected = S3MetaData(theFile,
matchByHash = Set(otherRemoteMetadata),
matchByKey = None)
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
}
}
describe(
"#3 local exists, remote is missing, remote no match, other no matches - upload") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val s3: S3ObjectsData = S3ObjectsData()
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
s3: S3ObjectsData = S3ObjectsData()
} yield (theFile, s3)
it("generates valid metadata") {
val expected =
S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None)
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
env.map({
case (theFile, s3) => {
val expected =
S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None)
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
}
}
describe(
"#4 local exists, remote exists, remote no match, other matches - copy") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val theRemoteKey = theFile.remoteKey
val oldHash = MD5Hash("old-hash")
val otherRemoteKey = prefix.resolve("other-key")
val s3: S3ObjectsData = S3ObjectsData(
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
byKey = Map(
theRemoteKey -> HashModified(oldHash, lastModified),
otherRemoteKey -> HashModified(theHash, lastModified)
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
theRemoteKey = theFile.remoteKey
oldHash = MD5Hash("old-hash")
otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
s3: S3ObjectsData = S3ObjectsData(
byHash =
Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
byKey = Map(
theRemoteKey -> HashModified(oldHash, lastModified),
otherRemoteKey -> HashModified(theHash, lastModified)
)
)
)
val theRemoteMetadata =
RemoteMetaData(theRemoteKey, oldHash, lastModified)
val otherRemoteMetadata =
RemoteMetaData(otherRemoteKey, theHash, lastModified)
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
theHash,
lastModified)
} yield (theFile, theRemoteMetadata, otherRemoteMetadata, s3)
it("generates valid metadata") {
val expected = S3MetaData(theFile,
matchByHash = Set(otherRemoteMetadata),
matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
env.map({
case (theFile, theRemoteMetadata, otherRemoteMetadata, s3) => {
val expected = S3MetaData(theFile,
matchByHash = Set(otherRemoteMetadata),
matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
}
}
describe(
"#5 local exists, remote exists, remote no match, other no matches - upload") {
val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file",
md5HashMap(theHash),
sourcePath,
fileToKey)
val theRemoteKey = theFile.remoteKey
val oldHash = MD5Hash("old-hash")
val s3: S3ObjectsData = S3ObjectsData(
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
theHash -> Set.empty),
byKey = Map(
theRemoteKey -> HashModified(oldHash, lastModified)
val env = for {
theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash),
sourcePath,
sources,
prefix)
theRemoteKey = theFile.remoteKey
oldHash = MD5Hash("old-hash")
s3: S3ObjectsData = S3ObjectsData(
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
theHash -> Set.empty),
byKey = Map(
theRemoteKey -> HashModified(oldHash, lastModified)
)
)
)
val theRemoteMetadata =
RemoteMetaData(theRemoteKey, oldHash, lastModified)
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
} yield (theFile, theRemoteMetadata, s3)
it("generates valid metadata") {
val expected = S3MetaData(theFile,
matchByHash = Set.empty,
matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
env.map({
case (theFile, theRemoteMetadata, s3) => {
val expected = S3MetaData(theFile,
matchByHash = Set.empty,
matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
}
}
}
@ -178,71 +218,103 @@ class S3MetaDataEnricherSuite extends FunSpec {
describe("getS3Status") {
val hash = MD5Hash("hash")
val localFile =
LocalFile.resolve("the-file", md5HashMap(hash), sourcePath, fileToKey)
val key = localFile.remoteKey
val keyOtherKey = LocalFile.resolve("other-key-same-hash",
md5HashMap(hash),
sourcePath,
fileToKey)
val diffHash = MD5Hash("diff")
val keyDiffHash = LocalFile.resolve("other-key-diff-hash",
md5HashMap(diffHash),
sourcePath,
fileToKey)
val lastModified = LastModified(Instant.now)
val s3ObjectsData: S3ObjectsData = S3ObjectsData(
byHash = Map(
hash -> Set(KeyModified(key, lastModified),
KeyModified(keyOtherKey.remoteKey, lastModified)),
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
),
byKey = Map(
key -> HashModified(hash, lastModified),
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
val env = for {
localFile <- LocalFileValidator.resolve("the-file",
md5HashMap(hash),
sourcePath,
sources,
prefix)
key = localFile.remoteKey
keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash",
md5HashMap(hash),
sourcePath,
sources,
prefix)
diffHash = MD5Hash("diff")
keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash",
md5HashMap(diffHash),
sourcePath,
sources,
prefix)
lastModified = LastModified(Instant.now)
s3ObjectsData = S3ObjectsData(
byHash = Map(
hash -> Set(KeyModified(key, lastModified),
KeyModified(keyOtherKey.remoteKey, lastModified)),
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
),
byKey = Map(
key -> HashModified(hash, lastModified),
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
)
)
)
} yield (s3ObjectsData, localFile, keyDiffHash, diffHash)
def invoke(localFile: LocalFile) = {
def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) = {
getS3Status(localFile, s3ObjectsData)
}
describe("when remote key exists") {
it("should return a result for matching key") {
val result = getMatchesByKey(invoke(localFile))
assert(result.contains(HashModified(hash, lastModified)))
env.map({
case (s3ObjectsData, localFile: LocalFile, _, _) =>
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
assert(result.contains(HashModified(hash, lastModified)))
})
}
}
describe("when remote key does not exist and no others matches hash") {
val localFile = LocalFile.resolve("missing-file",
md5HashMap(MD5Hash("unique")),
sourcePath,
fileToKey)
val env2 = for {
localFile <- LocalFileValidator.resolve("missing-remote",
md5HashMap(MD5Hash("unique")),
sourcePath,
sources,
prefix)
} yield (localFile)
it("should return no matches by key") {
val result = getMatchesByKey(invoke(localFile))
assert(result.isEmpty)
env.map({
case (s3ObjectsData, _, _, _) => {
env2.map({
case (localFile) => {
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
assert(result.isEmpty)
}
})
}
})
}
it("should return no matches by hash") {
val result = getMatchesByHash(invoke(localFile))
assert(result.isEmpty)
env.map({
case (s3ObjectsData, _, _, _) => {
env2.map({
case (localFile) => {
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
assert(result.isEmpty)
}
})
}
})
}
}
describe("when remote key exists and no others match hash") {
it("should return match by key") {
val result = getMatchesByKey(invoke(keyDiffHash))
assert(result.contains(HashModified(diffHash, lastModified)))
}
it("should return only itself in match by hash") {
val result = getMatchesByHash(invoke(keyDiffHash))
assert(
result.equals(
Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
}
env.map({
case (s3ObjectsData, _, keyDiffHash, diffHash) => {
it("should return match by key") {
val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData))
assert(result.contains(HashModified(diffHash, lastModified)))
}
it("should return only itself in match by hash") {
val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData))
assert(
result.equals(Set(
(diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
}
}
})
}
}
}

9
core/src/test/scala/net/kemitix/thorp/core/hasher/MD5HashGeneratorTest.scala
View file

@ -3,6 +3,7 @@ package net.kemitix.thorp.core.hasher
import java.nio.file.Path
import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root}
import net.kemitix.thorp.filesystem.FileSystem
import org.scalatest.FunSpec
@ -42,14 +43,14 @@ class MD5HashGeneratorTest extends FunSpec {
val path = Resource(this, "../big-file").toPath
it("should generate the correct hash for first chunk of the file") {
val part1 = BigFile.Part1
val expected = Right(part1.hash.hash)
val result = invoke(path, part1.offset, part1.size).map(_.hash)
val expected = Right(MD5Hash.hash(part1.hash))
val result = invoke(path, part1.offset, part1.size).map(MD5Hash.hash)
assertResult(expected)(result)
}
it("should generate the correct hash for second chunk of the file") {
val part2 = BigFile.Part2
val expected = Right(part2.hash.hash)
val result = invoke(path, part2.offset, part2.size).map(_.hash)
val expected = Right(MD5Hash.hash(part2.hash))
val result = invoke(path, part2.offset, part2.size).map(MD5Hash.hash)
assertResult(expected)(result)
}
}

53
domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala
View file

@ -1,53 +1,18 @@
package net.kemitix.thorp.domain
import java.nio.file.Path
import java.util.function.Predicate
import java.util.regex.Pattern
sealed trait Filter
sealed trait Filter {
def predicate: Predicate[String]
}
object Filter {
def isIncluded(p: Path)(filters: List[Filter]): Boolean = {
sealed trait State
case class Unknown() extends State
case class Accepted() extends State
case class Discarded() extends State
filters.foldRight(Unknown(): State)((filter, state) =>
(filter, state) match {
case (_, Accepted()) => Accepted()
case (_, Discarded()) => Discarded()
case (x: Exclude, _) if x.isExcluded(p) => Discarded()
case (i: Include, _) if i.isIncluded(p) => Accepted()
case _ => Unknown()
}) match {
case Accepted() => true
case Discarded() => false
case Unknown() =>
filters.forall {
case _: Include => false
case _ => true
}
}
case class Include(include: String = ".*") extends Filter {
lazy val predicate: Predicate[String] = Pattern.compile(include).asPredicate
}
case class Include(
include: String = ".*"
) extends Filter {
private lazy val predicate = Pattern.compile(include).asPredicate
def isIncluded(path: Path): Boolean = predicate.test(path.toString)
case class Exclude(exclude: String) extends Filter {
lazy val predicate: Predicate[String] =
Pattern.compile(exclude).asPredicate()
}
case class Exclude(
exclude: String
) extends Filter {
private lazy val predicate = Pattern.compile(exclude).asPredicate()
def isExcluded(path: Path): Boolean = predicate.test(path.toString)
}
}

38
domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala
View file

@ -5,42 +5,22 @@ import java.nio.file.Path
import net.kemitix.thorp.domain.HashType.MD5
final case class LocalFile(
final case class LocalFile private (
file: File,
source: File,
hashes: Map[HashType, MD5Hash],
remoteKey: RemoteKey
) {
require(!file.isDirectory, s"LocalFile must not be a directory: $file")
def isDirectory: Boolean = file.isDirectory
// the path of the file within the source
def relative: Path = source.toPath.relativize(file.toPath)
def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _)
def md5base64: Option[String] = hashes.get(MD5).map(_.hash64)
}
)
object LocalFile {
def resolve(
path: String,
md5Hashes: Map[HashType, MD5Hash],
source: Path,
pathToKey: Path => RemoteKey
): LocalFile = {
val resolvedPath = source.resolve(path)
LocalFile(resolvedPath.toFile,
source.toFile,
md5Hashes,
pathToKey(resolvedPath))
}
val remoteKey: SimpleLens[LocalFile, RemoteKey] =
SimpleLens[LocalFile, RemoteKey](_.remoteKey,
b => a => b.copy(remoteKey = a))
// the path of the file within the source
def relativeToSource(localFile: LocalFile): Path =
localFile.source.toPath.relativize(localFile.file.toPath)
def matchesHash(localFile: LocalFile)(other: MD5Hash): Boolean =
localFile.hashes.values.exists(other equals _)
def md5base64(localFile: LocalFile): Option[String] =
localFile.hashes.get(MD5).map(MD5Hash.hash64)
}

15
domain/src/main/scala/net/kemitix/thorp/domain/Logger.scala
View file

@ -1,15 +0,0 @@
package net.kemitix.thorp.domain
trait Logger {
// returns an instance of Logger with debug set as indicated
// where the current Logger already matches this state, then
// it returns itself, unmodified
def withDebug(debug: Boolean): Logger
def debug(message: => String): Unit
def info(message: => String): Unit
def warn(message: String): Unit
def error(message: String): Unit
}

22
domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala
View file

@ -4,20 +4,14 @@ import java.util.Base64
import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
final case class MD5Hash(
in: String
) {
lazy val hash: String = in filter stripQuotes
lazy val digest: Array[Byte] = HexEncoder.decode(hash)
lazy val hash64: String = Base64.getEncoder.encodeToString(digest)
}
final case class MD5Hash(in: String)
object MD5Hash {
def fromDigest(digest: Array[Byte]): MD5Hash = {
val hexDigest = (digest map ("%02x" format _)).mkString
MD5Hash(hexDigest)
}
def fromDigest(digest: Array[Byte]): MD5Hash =
MD5Hash((digest map ("%02x" format _)).mkString)
def hash(md5Hash: MD5Hash): String = md5Hash.in.filter(stripQuotes)
def digest(md5Hash: MD5Hash): Array[Byte] =
HexEncoder.decode(MD5Hash.hash(md5Hash))
def hash64(md5Hash: MD5Hash): String =
Base64.getEncoder.encodeToString(MD5Hash.digest(md5Hash))
}

36
domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala
View file

@ -3,31 +3,21 @@ package net.kemitix.thorp.domain
import java.io.File
import java.nio.file.{Path, Paths}
final case class RemoteKey(
key: String
) {
def asFile(
source: Path,
prefix: RemoteKey
): Option[File] =
if (key.length == 0) None
else Some(source.resolve(relativeTo(prefix)).toFile)
private def relativeTo(prefix: RemoteKey) = {
prefix match {
case RemoteKey("") => Paths.get(key)
case _ => Paths.get(prefix.key).relativize(Paths.get(key))
}
}
def resolve(path: String): RemoteKey =
RemoteKey(List(key, path).filterNot(_.isEmpty).mkString("/"))
}
final case class RemoteKey(key: String)
object RemoteKey {
val key: SimpleLens[RemoteKey, String] =
SimpleLens[RemoteKey, String](_.key, b => a => b.copy(key = a))
def asFile(source: Path, prefix: RemoteKey)(
remoteKey: RemoteKey): Option[File] =
if (remoteKey.key.length == 0) None
else Some(source.resolve(RemoteKey.relativeTo(prefix)(remoteKey)).toFile)
def relativeTo(prefix: RemoteKey)(remoteKey: RemoteKey): Path = {
prefix match {
case RemoteKey("") => Paths.get(remoteKey.key)
case _ => Paths.get(prefix.key).relativize(Paths.get(remoteKey.key))
}
}
def resolve(path: String)(remoteKey: RemoteKey): RemoteKey =
RemoteKey(List(remoteKey.key, path).filterNot(_.isEmpty).mkString("/"))
}

18
domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala
View file

@ -2,6 +2,8 @@ package net.kemitix.thorp.domain
import java.nio.file.Path
import zio.{Task, ZIO}
/**
* The paths to synchronise with target.
*
@ -18,18 +20,10 @@ case class Sources(
def +(path: Path)(implicit m: Monoid[Sources]): Sources = this ++ List(path)
def ++(otherPaths: List[Path])(implicit m: Monoid[Sources]): Sources =
m.op(this, Sources(otherPaths))
/**
* Returns the source path for the given path.
*/
def forPath(path: Path): Path =
paths.find(source => path.startsWith(source)).get
}
object Sources {
final val emptySources = Sources(List.empty)
implicit def sourcesAppendMonoid: Monoid[Sources] = new Monoid[Sources] {
override def zero: Sources = emptySources
override def op(t1: Sources, t2: Sources): Sources =
@ -38,4 +32,12 @@ object Sources {
else acc ++ List(path)
})
}
/**
* Returns the source path for the given path.
*/
def forPath(path: Path)(sources: Sources): Task[Path] =
ZIO
.fromOption(sources.paths.find(s => path.startsWith(s)))
.mapError(_ => new Exception("Path is not within any known source"))
}

7
domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala
View file

@ -1,10 +1,7 @@
package net.kemitix.thorp.domain
import net.kemitix.thorp.domain.UploadEvent.RequestEvent
import net.kemitix.thorp.domain.UploadEventLogger.{
RequestCycle,
logRequestCycle
}
import net.kemitix.thorp.domain.UploadEventLogger.RequestCycle
object UploadEventListener {
@ -21,7 +18,7 @@ object UploadEventListener {
uploadEvent match {
case e: RequestEvent =>
bytesTransferred += e.transferred
logRequestCycle(
UploadEventLogger(
RequestCycle(settings.localFile,
bytesTransferred,
settings.index,

2
domain/src/main/scala/net/kemitix/thorp/domain/UploadEventLogger.scala
View file

@ -14,7 +14,7 @@ object UploadEventLogger {
totalBytesSoFar: Long
)
def logRequestCycle(requestCycle: RequestCycle): Unit = {
def apply(requestCycle: RequestCycle): Unit = {
val remoteKey = requestCycle.localFile.remoteKey.key
val fileLength = requestCycle.localFile.file.length
val statusHeight = 7

4
domain/src/test/scala/net/kemitix/thorp/domain/MD5HashTest.scala
View file

@ -7,11 +7,11 @@ class MD5HashTest extends FunSpec {
describe("recover base64 hash") {
it("should recover base 64 #1") {
val rootHash = MD5HashData.Root.hash
assertResult(MD5HashData.Root.base64)(rootHash.hash64)
assertResult(MD5HashData.Root.base64)(MD5Hash.hash64(rootHash))
}
it("should recover base 64 #2") {
val leafHash = MD5HashData.Leaf.hash
assertResult(MD5HashData.Leaf.base64)(leafHash.hash64)
assertResult(MD5HashData.Leaf.base64)(MD5Hash.hash64(leafHash))
}
}
}

14
domain/src/test/scala/net/kemitix/thorp/domain/RemoteKeyTest.scala
View file

@ -15,21 +15,21 @@ class RemoteKeyTest extends FreeSpec {
val key = emptyKey
val path = "path"
val expected = RemoteKey("path")
val result = key.resolve(path)
val result = RemoteKey.resolve(path)(key)
assertResult(expected)(result)
}
"when path is empty" in {
val key = RemoteKey("key")
val path = ""
val expected = RemoteKey("key")
val result = key.resolve(path)
val result = RemoteKey.resolve(path)(key)
assertResult(expected)(result)
}
"when key and path are empty" in {
val key = emptyKey
val path = ""
val expected = emptyKey
val result = key.resolve(path)
val result = RemoteKey.resolve(path)(key)
assertResult(expected)(result)
}
}
@ -39,7 +39,7 @@ class RemoteKeyTest extends FreeSpec {
val source = Paths.get("source")
val prefix = RemoteKey("prefix")
val expected = Some(new File("source/key"))
val result = key.asFile(source, prefix)
val result = RemoteKey.asFile(source, prefix)(key)
assertResult(expected)(result)
}
"when prefix is empty" in {
@ -47,7 +47,7 @@ class RemoteKeyTest extends FreeSpec {
val source = Paths.get("source")
val prefix = emptyKey
val expected = Some(new File("source/key"))
val result = key.asFile(source, prefix)
val result = RemoteKey.asFile(source, prefix)(key)
assertResult(expected)(result)
}
"when key is empty" in {
@ -55,7 +55,7 @@ class RemoteKeyTest extends FreeSpec {
val source = Paths.get("source")
val prefix = RemoteKey("prefix")
val expected = None
val result = key.asFile(source, prefix)
val result = RemoteKey.asFile(source, prefix)(key)
assertResult(expected)(result)
}
"when key and prefix are empty" in {
@ -63,7 +63,7 @@ class RemoteKeyTest extends FreeSpec {
val source = Paths.get("source")
val prefix = emptyKey
val expected = None
val result = key.asFile(source, prefix)
val result = RemoteKey.asFile(source, prefix)(key)
assertResult(expected)(result)
}
}

10
domain/src/test/scala/net/kemitix/thorp/domain/TemporaryFolder.scala
View file

@ -38,11 +38,13 @@ trait TemporaryFolder {
path.resolve(name).toFile
}
def writeFile(directory: Path, name: String, contents: String*): Unit = {
def writeFile(directory: Path, name: String, contents: String*): File = {
directory.toFile.mkdirs
val pw = new PrintWriter(directory.resolve(name).toFile, "UTF-8")
contents.foreach(pw.println)
pw.close()
val file = directory.resolve(name).toFile
val writer = new PrintWriter(file, "UTF-8")
contents.foreach(writer.println)
writer.close()
file
}
}

2
storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Copier.scala
View file

@ -41,7 +41,7 @@ trait Copier {
copyRequest.sourceKey.key,
copyRequest.bucket.name,
copyRequest.targetKey.key
).withMatchingETagConstraint(copyRequest.hash.hash)
).withMatchingETagConstraint(MD5Hash.hash(copyRequest.hash))
private def foldFailure(
sourceKey: RemoteKey,

2
storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Uploader.scala
View file

@ -68,7 +68,7 @@ trait Uploader {
private def metadata: LocalFile => ObjectMetadata = localFile => {
val metadata = new ObjectMetadata()
localFile.md5base64.foreach(metadata.setContentMD5)
LocalFile.md5base64(localFile).foreach(metadata.setContentMD5)
metadata
}

3
storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala
View file

@ -7,6 +7,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils
import net.kemitix.thorp.core.hasher.Hasher
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.filesystem.FileSystem
import zio.{TaskR, ZIO}
@ -56,7 +57,7 @@ private trait ETagGenerator {
Hasher
.hashObjectChunk(path, chunkNumber, chunkSize)
.map(_(MD5))
.map(_.digest)
.map(MD5Hash.digest)
def offsets(
totalFileSizeBytes: Long,

2
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala
View file

@ -32,7 +32,7 @@ class S3ObjectsByHashSuite extends FunSpec {
remoteKey: RemoteKey,
lastModified: LastModified): S3ObjectSummary = {
val summary = new S3ObjectSummary()
summary.setETag(md5Hash.hash)
summary.setETag(MD5Hash.hash(md5Hash))
summary.setKey(remoteKey.key)
summary.setLastModified(Date.from(lastModified.when))
summary

161
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala
View file

@ -3,7 +3,7 @@ package net.kemitix.thorp.storage.aws
import java.time.Instant
import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.core.{KeyGenerator, S3MetaDataEnricher}
import net.kemitix.thorp.core.{LocalFileValidator, S3MetaDataEnricher}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
import org.scalamock.scalatest.MockFactory
@ -15,39 +15,51 @@ class StorageServiceSuite extends FunSpec with MockFactory {
private val sourcePath = source.toPath
private val sources = Sources(List(sourcePath))
private val prefix = RemoteKey("prefix")
private val fileToKey =
KeyGenerator.generateKey(sources, prefix) _
describe("getS3Status") {
val hash = MD5Hash("hash")
val localFile =
LocalFile.resolve("the-file", Map(MD5 -> hash), sourcePath, fileToKey)
val key = localFile.remoteKey
val keyOtherKey = LocalFile.resolve("other-key-same-hash",
Map(MD5 -> hash),
sourcePath,
fileToKey)
val diffHash = MD5Hash("diff")
val keyDiffHash = LocalFile.resolve("other-key-diff-hash",
Map(MD5 -> diffHash),
sourcePath,
fileToKey)
val lastModified = LastModified(Instant.now)
val s3ObjectsData: S3ObjectsData = S3ObjectsData(
byHash = Map(
hash -> Set(KeyModified(key, lastModified),
KeyModified(keyOtherKey.remoteKey, lastModified)),
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
),
byKey = Map(
key -> HashModified(hash, lastModified),
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
val env = for {
localFile <- LocalFileValidator.resolve("the-file",
Map(MD5 -> hash),
sourcePath,
sources,
prefix)
key = localFile.remoteKey
keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash",
Map(MD5 -> hash),
sourcePath,
sources,
prefix)
diffHash = MD5Hash("diff")
keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash",
Map(MD5 -> diffHash),
sourcePath,
sources,
prefix)
lastModified = LastModified(Instant.now)
s3ObjectsData = S3ObjectsData(
byHash = Map(
hash -> Set(KeyModified(key, lastModified),
KeyModified(keyOtherKey.remoteKey, lastModified)),
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
),
byKey = Map(
key -> HashModified(hash, lastModified),
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
)
)
)
} yield
(s3ObjectsData,
localFile: LocalFile,
lastModified,
keyOtherKey,
keyDiffHash,
diffHash,
key)
def invoke(localFile: LocalFile) =
def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) =
S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData)
def getMatchesByKey(
@ -67,47 +79,94 @@ class StorageServiceSuite extends FunSpec with MockFactory {
describe(
"when remote key exists, unmodified and other key matches the hash") {
it("should return the match by key") {
val result = getMatchesByKey(invoke(localFile))
assert(result.contains(HashModified(hash, lastModified)))
env.map({
case (s3ObjectsData, localFile, lastModified, _, _, _, _) => {
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
assert(result.contains(HashModified(hash, lastModified)))
}
})
}
it("should return both matches for the hash") {
val result = getMatchesByHash(invoke(localFile))
assertResult(
Set((hash, KeyModified(key, lastModified)),
(hash, KeyModified(keyOtherKey.remoteKey, lastModified)))
)(result)
env.map({
case (s3ObjectsData,
localFile,
lastModified,
keyOtherKey,
_,
_,
key) => {
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
assertResult(
Set((hash, KeyModified(key, lastModified)),
(hash, KeyModified(keyOtherKey.remoteKey, lastModified)))
)(result)
}
})
}
}
describe("when remote key does not exist and no others matches hash") {
val localFile = LocalFile.resolve("missing-file",
Map(MD5 -> MD5Hash("unique")),
sourcePath,
fileToKey)
val env2 = LocalFileValidator
.resolve("missing-file",
Map(MD5 -> MD5Hash("unique")),
sourcePath,
sources,
prefix)
it("should return no matches by key") {
val result = getMatchesByKey(invoke(localFile))
assert(result.isEmpty)
env2.map(localFile => {
env.map({
case (s3ObjectsData, _, _, _, _, _, _) => {
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
assert(result.isEmpty)
}
})
})
}
it("should return no matches by hash") {
val result = getMatchesByHash(invoke(localFile))
assert(result.isEmpty)
env2.map(localFile => {
env.map({
case (s3ObjectsData, _, _, _, _, _, _) => {
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
assert(result.isEmpty)
}
})
})
}
}
describe("when remote key exists and no others match hash") {
val localFile = keyDiffHash
it("should return the match by key") {
val result = getMatchesByKey(invoke(localFile))
assert(result.contains(HashModified(diffHash, lastModified)))
env.map({
case (s3ObjectsData,
_,
lastModified,
_,
keyDiffHash,
diffHash,
_) => {
val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData))
assert(result.contains(HashModified(diffHash, lastModified)))
}
})
}
it("should return one match by hash") {
val result = getMatchesByHash(invoke(localFile))
assertResult(
Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))
)(result)
env.map({
case (s3ObjectsData,
_,
lastModified,
_,
keyDiffHash,
diffHash,
_) => {
val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData))
assertResult(
Set(
(diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))
)(result)
}
})
}
}
}
}

2
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala
View file

@ -29,7 +29,7 @@ class UploaderTest extends FreeSpec with MockFactory {
val bucket = Bucket("aBucket")
val uploadResult = new UploadResult
uploadResult.setKey(remoteKey.key)
uploadResult.setETag(aHash.hash)
uploadResult.setETag(MD5Hash.hash(aHash))
val inProgress = new AmazonUpload.InProgress {
override def waitForUploadResult: UploadResult = uploadResult
}

3
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala
View file

@ -6,6 +6,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.core.hasher.Hasher
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.filesystem.FileSystem
import org.scalatest.FunSpec
import zio.DefaultRuntime
@ -45,7 +46,7 @@ class ETagGeneratorTest extends FunSpec {
assertResult(Right(hash))(
invoke(bigFilePath, index, chunkSize)
.map(_(MD5))
.map(_.hash))
.map(MD5Hash.hash))
}
}
def invoke(path: Path, index: Long, size: Long) =