case classes shouldn't be OO objects (#147)

* [core] Extract Filters from domain.Filter

* [core] extract LocalFileValidator

* [domain] LocalFile remove unused isDirectory

* [domain] LocalFile move/rename relative as relativeToSource on companion

* [domain] LocalFile move and rename matches as matchesHash on companion

* [domain] LocalFile move md5base64 to companion

* [domain] Logger remove

* [domain] MD5Hash move hash to companion

* [domain] MD5Hash move digest to companion

* [domain] MD5Hash move hash64 to companion

* [domain] RemoteKey move class methods to companion

* [domain] Sources move forPath to companion

Led to being able to cleanup LocalFileStream.localFile and adding
LocalFiles.one

* [domain] UploadEventLogger rename method as apply
This commit is contained in:
Paul Campbell 2019-08-05 08:38:44 +01:00 committed by GitHub
parent 96ecedbe61
commit 8fad680a96
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 750 additions and 551 deletions

View file

@ -135,3 +135,4 @@ lazy val domain = (project in file("domain"))
.settings(commonSettings) .settings(commonSettings)
.settings(assemblyJarName in assembly := "domain.jar") .settings(assemblyJarName in assembly := "domain.jar")
.settings(testDependencies) .settings(testDependencies)
.settings(zioDependencies)

View file

@ -1,6 +1,6 @@
package net.kemitix.thorp.config package net.kemitix.thorp.config
import java.nio.file.Path import java.io.File
sealed trait ConfigValidation { sealed trait ConfigValidation {
@ -22,10 +22,10 @@ object ConfigValidation {
} }
case class ErrorReadingFile( case class ErrorReadingFile(
path: Path, file: File,
message: String message: String
) extends ConfigValidation { ) extends ConfigValidation {
override def errorMessage: String = s"Error reading file '$path': $message" override def errorMessage: String = s"Error reading file '$file': $message"
} }
} }

View file

@ -1,6 +1,6 @@
package net.kemitix.thorp.config package net.kemitix.thorp.config
import java.nio.file.Paths import java.io.File
import net.kemitix.thorp.filesystem.FileSystem import net.kemitix.thorp.filesystem.FileSystem
import zio.ZIO import zio.ZIO
@ -11,9 +11,9 @@ import zio.ZIO
*/ */
trait ConfigurationBuilder { trait ConfigurationBuilder {
private val userConfigFilename = ".config/thorp.conf" private val userConfigFile = ".config/thorp.conf"
private val globalConfig = Paths.get("/etc/thorp.conf") private val globalConfig = new File("/etc/thorp.conf")
private val userHome = Paths.get(System.getProperty("user.home")) private val userHome = new File(System.getProperty("user.home"))
def buildConfig(priorityOpts: ConfigOptions) def buildConfig(priorityOpts: ConfigOptions)
: ZIO[FileSystem, ConfigValidationException, Configuration] = : ZIO[FileSystem, ConfigValidationException, Configuration] =
@ -33,7 +33,7 @@ trait ConfigurationBuilder {
private def userOptions(priorityOpts: ConfigOptions) = private def userOptions(priorityOpts: ConfigOptions) =
if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
else ParseConfigFile.parseFile(userHome.resolve(userConfigFilename)) else ParseConfigFile.parseFile(new File(userHome, userConfigFile))
private def globalOptions(priorityOpts: ConfigOptions) = private def globalOptions(priorityOpts: ConfigOptions) =
if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig

View file

@ -1,6 +1,6 @@
package net.kemitix.thorp.config package net.kemitix.thorp.config
import java.nio.file.Path import java.io.File
import net.kemitix.thorp.filesystem.FileSystem import net.kemitix.thorp.filesystem.FileSystem
import zio.{IO, TaskR, ZIO} import zio.{IO, TaskR, ZIO}
@ -8,19 +8,14 @@ import zio.{IO, TaskR, ZIO}
trait ParseConfigFile { trait ParseConfigFile {
def parseFile( def parseFile(
filename: Path): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] = file: File): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
(readFile(filename) >>= ParseConfigLines.parseLines) (FileSystem.exists(file) >>= readLines(file) >>= ParseConfigLines.parseLines)
.catchAll( .catchAll(h =>
h => IO.fail(List(ConfigValidation.ErrorReadingFile(file, h.getMessage))))
IO.fail(
List(ConfigValidation.ErrorReadingFile(filename, h.getMessage))))
private def readFile(filename: Path) = private def readLines(file: File)(
FileSystem.exists(filename.toFile) >>= readLines(filename)
private def readLines(filename: Path)(
exists: Boolean): TaskR[FileSystem, Seq[String]] = exists: Boolean): TaskR[FileSystem, Seq[String]] =
if (exists) FileSystem.lines(filename.toFile) if (exists) FileSystem.lines(file)
else ZIO.succeed(Seq.empty) else ZIO.succeed(Seq.empty)
} }

View file

@ -1,5 +1,7 @@
package net.kemitix.thorp.config package net.kemitix.thorp.config
import java.io.File
import net.kemitix.thorp.domain.Sources import net.kemitix.thorp.domain.Sources
import net.kemitix.thorp.filesystem.FileSystem import net.kemitix.thorp.filesystem.FileSystem
import zio.ZIO import zio.ZIO
@ -12,7 +14,7 @@ trait SourceConfigLoader {
sources: Sources): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] = sources: Sources): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
ZIO ZIO
.foreach(sources.paths) { path => .foreach(sources.paths) { path =>
ParseConfigFile.parseFile(path.resolve(thorpConfigFileName)) ParseConfigFile.parseFile(new File(path.toFile, thorpConfigFileName))
} }
.map(_.foldLeft(ConfigOptions(sources.paths.map(ConfigOption.Source))) { .map(_.foldLeft(ConfigOptions(sources.paths.map(ConfigOption.Source))) {
(acc, co) => (acc, co) =>

View file

@ -1,47 +1,58 @@
package net.kemitix.thorp.config package net.kemitix.thorp.config
import java.nio.file.{Path, Paths} import java.io.File
import java.nio.file.Paths
import net.kemitix.thorp.domain.TemporaryFolder
import net.kemitix.thorp.filesystem.FileSystem import net.kemitix.thorp.filesystem.FileSystem
import org.scalatest.FunSpec import org.scalatest.FunSpec
import zio.DefaultRuntime import zio.DefaultRuntime
class ParseConfigFileTest extends FunSpec { class ParseConfigFileTest extends FunSpec with TemporaryFolder {
private val empty = Right(ConfigOptions()) private val empty = Right(ConfigOptions())
describe("parse a missing file") { describe("parse a missing file") {
val filename = Paths.get("/path/to/missing/file") val file = new File("/path/to/missing/file")
it("should return no options") { it("should return no options") {
assertResult(empty)(invoke(filename)) assertResult(empty)(invoke(file))
} }
} }
describe("parse an empty file") { describe("parse an empty file") {
val filename = Resource(this, "empty-file").toPath
it("should return no options") { it("should return no options") {
assertResult(empty)(invoke(filename)) withDirectory(dir => {
val file = writeFile(dir, "empty-file")
assertResult(empty)(invoke(file))
})
} }
} }
describe("parse a file with no valid entries") { describe("parse a file with no valid entries") {
val filename = Resource(this, "invalid-config").toPath
it("should return no options") { it("should return no options") {
assertResult(empty)(invoke(filename)) withDirectory(dir => {
val file = writeFile(dir, "invalid-config", "no valid = config items")
assertResult(empty)(invoke(file))
})
} }
} }
describe("parse a file with properties") { describe("parse a file with properties") {
val filename = Resource(this, "simple-config").toPath
val expected = Right(
ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source")),
ConfigOption.Bucket("bucket-name"))))
it("should return some options") { it("should return some options") {
assertResult(expected)(invoke(filename)) val expected = Right(
ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source")),
ConfigOption.Bucket("bucket-name"))))
withDirectory(dir => {
val file = writeFile(dir,
"simple-config",
"source = /path/to/source",
"bucket = bucket-name")
assertResult(expected)(invoke(file))
})
} }
} }
private def invoke(filename: Path) = { private def invoke(file: File) = {
new DefaultRuntime {}.unsafeRunSync { new DefaultRuntime {}.unsafeRunSync {
ParseConfigFile ParseConfigFile
.parseFile(filename) .parseFile(file)
.provide(FileSystem.Live) .provide(FileSystem.Live)
}.toEither }.toEither
} }

View file

@ -21,7 +21,7 @@ object ActionGenerator {
s3MetaData match { s3MetaData match {
// #1 local exists, remote exists, remote matches - do nothing // #1 local exists, remote exists, remote matches - do nothing
case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _))) case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
if localFile.matches(hash) => if LocalFile.matchesHash(localFile)(hash) =>
doNothing(bucket, key) doNothing(bucket, key)
// #2 local exists, remote is missing, other matches - copy // #2 local exists, remote is missing, other matches - copy
case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty => case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
@ -33,7 +33,7 @@ object ActionGenerator {
uploadFile(bucket, localFile) uploadFile(bucket, localFile)
// #4 local exists, remote exists, remote no match, other matches - copy // #4 local exists, remote exists, remote no match, other matches - copy
case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _))) case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _)))
if !localFile.matches(hash) && if !LocalFile.matchesHash(localFile)(hash) &&
matchByHash.nonEmpty => matchByHash.nonEmpty =>
copyFile(bucket, localFile, matchByHash) copyFile(bucket, localFile, matchByHash)
// #5 local exists, remote exists, remote no match, other no matches - upload // #5 local exists, remote exists, remote no match, other no matches - upload

View file

@ -0,0 +1,41 @@
package net.kemitix.thorp.core
import java.nio.file.Path
import net.kemitix.thorp.domain.Filter
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
object Filters {
def isIncluded(p: Path)(filters: List[Filter]): Boolean = {
sealed trait State
case class Unknown() extends State
case class Accepted() extends State
case class Discarded() extends State
val excluded = isExcludedByFilter(p)(_)
val included = isIncludedByFilter(p)(_)
filters.foldRight(Unknown(): State)((filter, state) =>
(filter, state) match {
case (_, Accepted()) => Accepted()
case (_, Discarded()) => Discarded()
case (x: Exclude, _) if excluded(x) => Discarded()
case (i: Include, _) if included(i) => Accepted()
case _ => Unknown()
}) match {
case Accepted() => true
case Discarded() => false
case Unknown() =>
filters.forall {
case _: Include => false
case _ => true
}
}
}
def isIncludedByFilter(path: Path)(filter: Filter): Boolean =
filter.predicate.test(path.toString)
def isExcludedByFilter(path: Path)(filter: Filter): Boolean =
filter.predicate.test(path.toString)
}

View file

@ -3,21 +3,17 @@ package net.kemitix.thorp.core
import java.nio.file.Path import java.nio.file.Path
import net.kemitix.thorp.domain.{RemoteKey, Sources} import net.kemitix.thorp.domain.{RemoteKey, Sources}
import zio.Task
object KeyGenerator { object KeyGenerator {
def generateKey( def generateKey(
sources: Sources, sources: Sources,
prefix: RemoteKey prefix: RemoteKey
)(path: Path): RemoteKey = { )(path: Path): Task[RemoteKey] =
val source = sources.forPath(path) Sources
val relativePath = source.relativize(path.toAbsolutePath).toString .forPath(path)(sources)
RemoteKey( .map(p => p.relativize(path.toAbsolutePath).toString)
List( .map(RemoteKey.resolve(_)(prefix))
prefix.key,
relativePath
).filter(_.nonEmpty)
.mkString("/"))
}
} }

View file

@ -4,9 +4,8 @@ import java.io.File
import java.nio.file.Path import java.nio.file.Path
import net.kemitix.thorp.config.Config import net.kemitix.thorp.config.Config
import net.kemitix.thorp.core.KeyGenerator.generateKey
import net.kemitix.thorp.core.hasher.Hasher import net.kemitix.thorp.core.hasher.Hasher
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain.Sources
import net.kemitix.thorp.filesystem.FileSystem import net.kemitix.thorp.filesystem.FileSystem
import zio.{Task, TaskR, ZIO} import zio.{Task, TaskR, ZIO}
@ -48,21 +47,18 @@ object LocalFileStream {
.filter({ case (_, included) => included }) .filter({ case (_, included) => included })
.map({ case (path, _) => path }) .map({ case (path, _) => path })
private def localFile(path: Path) = { private def localFile(path: Path) =
val file = path.toFile
for { for {
sources <- Config.sources sources <- Config.sources
prefix <- Config.prefix prefix <- Config.prefix
source <- Sources.forPath(path)(sources)
hash <- Hasher.hashObject(path) hash <- Hasher.hashObject(path)
localFile = LocalFile(file, localFile <- LocalFileValidator.validate(path,
sources.forPath(path).toFile, source.toFile,
hash, hash,
generateKey(sources, prefix)(path)) sources,
} yield prefix)
LocalFiles(localFiles = Stream(localFile), } yield LocalFiles.one(localFile)
count = 1,
totalSizeBytes = file.length)
}
private def listFiles(path: Path) = private def listFiles(path: Path) =
for { for {
@ -78,6 +74,6 @@ object LocalFileStream {
private def isIncluded(path: Path) = private def isIncluded(path: Path) =
for { for {
filters <- Config.filters filters <- Config.filters
} yield Filter.isIncluded(path)(filters) } yield Filters.isIncluded(path)(filters)
} }

View file

@ -0,0 +1,66 @@
package net.kemitix.thorp.core
import java.io.File
import java.nio.file.Path
import net.kemitix.thorp.domain.{
HashType,
LocalFile,
MD5Hash,
RemoteKey,
Sources
}
import zio.{IO, ZIO}
object LocalFileValidator {
def validate(
path: Path,
source: File,
hash: Map[HashType, MD5Hash],
sources: Sources,
prefix: RemoteKey
): IO[Violation, LocalFile] =
for {
vFile <- validateFile(path.toFile)
remoteKey <- validateRemoteKey(sources, prefix, path)
} yield LocalFile(vFile, source, hash, remoteKey)
private def validateFile(file: File) =
if (file.isDirectory)
ZIO.fail(Violation.IsNotAFile(file))
else
ZIO.succeed(file)
private def validateRemoteKey(sources: Sources,
prefix: RemoteKey,
path: Path) =
KeyGenerator
.generateKey(sources, prefix)(path)
.mapError(e => Violation.InvalidRemoteKey(path, e))
sealed trait Violation extends Throwable {
def getMessage: String
}
object Violation {
case class IsNotAFile(file: File) extends Violation {
override def getMessage: String = s"Local File must be a file: ${file}"
}
case class InvalidRemoteKey(path: Path, e: Throwable) extends Violation {
override def getMessage: String =
s"Remote Key for '${path}' is invalid: ${e.getMessage}"
}
}
def resolve(
path: String,
md5Hashes: Map[HashType, MD5Hash],
source: Path,
sources: Sources,
prefix: RemoteKey
): IO[Violation, LocalFile] = {
val resolvedPath = source.resolve(path)
validate(resolvedPath, source.toFile, md5Hashes, sources, prefix)
}
}

View file

@ -13,12 +13,11 @@ case class LocalFiles(
count = count + append.count, count = count + append.count,
totalSizeBytes = totalSizeBytes + append.totalSizeBytes totalSizeBytes = totalSizeBytes + append.totalSizeBytes
) )
} }
object LocalFiles { object LocalFiles {
def reduce: Stream[LocalFiles] => LocalFiles = def reduce: Stream[LocalFiles] => LocalFiles =
list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf) list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf)
def one(localFile: LocalFile): LocalFiles =
LocalFiles(Stream(localFile), 1, localFile.file.length)
} }

View file

@ -18,7 +18,7 @@ object Remote {
remoteKey: RemoteKey remoteKey: RemoteKey
): TaskR[FileSystem, Boolean] = { ): TaskR[FileSystem, Boolean] = {
def existsInSource(source: Path) = def existsInSource(source: Path) =
remoteKey.asFile(source, prefix) match { RemoteKey.asFile(source, prefix)(remoteKey) match {
case Some(file) => FileSystem.exists(file) case Some(file) => FileSystem.exists(file)
case None => ZIO.succeed(false) case None => ZIO.succeed(false)
} }

View file

@ -2,13 +2,7 @@ package net.kemitix.thorp.core
import java.time.Instant import java.time.Instant
import net.kemitix.thorp.config.{ import net.kemitix.thorp.config._
Config,
ConfigOption,
ConfigOptions,
ConfigurationBuilder,
Resource
}
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload} import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain._
@ -31,8 +25,6 @@ class ActionGeneratorSuite extends FunSpec {
ConfigOption.IgnoreUserOptions, ConfigOption.IgnoreUserOptions,
ConfigOption.IgnoreGlobalOptions ConfigOption.IgnoreGlobalOptions
)) ))
private val fileToKey =
KeyGenerator.generateKey(sources, prefix) _
describe("create actions") { describe("create actions") {
@ -40,119 +32,156 @@ class ActionGeneratorSuite extends FunSpec {
describe("#1 local exists, remote exists, remote matches - do nothing") { describe("#1 local exists, remote exists, remote matches - do nothing") {
val theHash = MD5Hash("the-hash") val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val theRemoteMetadata = sources,
RemoteMetaData(theFile.remoteKey, theHash, lastModified) prefix)
val input = theRemoteMetadata = RemoteMetaData(theFile.remoteKey,
S3MetaData(theFile, // local exists theHash,
matchByHash = Set(theRemoteMetadata), // remote matches lastModified)
matchByKey = Some(theRemoteMetadata) // remote exists input = S3MetaData(
theFile, // local exists
matchByHash = Set(theRemoteMetadata), // remote matches
matchByKey = Some(theRemoteMetadata) // remote exists
) )
} yield (theFile, input)
it("do nothing") { it("do nothing") {
val expected = env.map({
Right( case (theFile, input) => {
Stream(DoNothing(bucket, theFile.remoteKey, theFile.file.length))) val expected =
val result = invoke(input, previousActions) Right(Stream(
assertResult(expected)(result) DoNothing(bucket, theFile.remoteKey, theFile.file.length + 1)))
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
} }
} }
describe("#2 local exists, remote is missing, other matches - copy") { describe("#2 local exists, remote is missing, other matches - copy") {
val theHash = MD5Hash("the-hash") val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val theRemoteKey = theFile.remoteKey sources,
val otherRemoteKey = prefix.resolve("other-key") prefix)
val otherRemoteMetadata = theRemoteKey = theFile.remoteKey
RemoteMetaData(otherRemoteKey, theHash, lastModified) otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
val input = otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
S3MetaData(theFile, // local exists theHash,
matchByHash = Set(otherRemoteMetadata), // other matches lastModified)
matchByKey = None) // remote is missing input = S3MetaData(
theFile, // local exists
matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = None) // remote is missing
} yield (theFile, theRemoteKey, input, otherRemoteKey)
it("copy from other key") { it("copy from other key") {
val expected = Right( env.map({
Stream( case (theFile, theRemoteKey, input, otherRemoteKey) => {
ToCopy(bucket, val expected = Right(
otherRemoteKey, Stream(
theHash, ToCopy(bucket,
theRemoteKey, otherRemoteKey,
theFile.file.length))) // copy theHash,
val result = invoke(input, previousActions) theRemoteKey,
assertResult(expected)(result) theFile.file.length))) // copy
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
} }
} describe("#3 local exists, remote is missing, other no matches - upload") {
describe("#3 local exists, remote is missing, other no matches - upload") { val theHash = MD5Hash("the-hash")
val theHash = MD5Hash("the-hash") val env = for {
val theFile = LocalFile.resolve("the-file", theFile <- LocalFileValidator.resolve("the-file",
md5HashMap(theHash), md5HashMap(theHash),
sourcePath, sourcePath,
fileToKey) sources,
val input = S3MetaData(theFile, // local exists prefix)
input = S3MetaData(theFile, // local exists
matchByHash = Set.empty, // other no matches matchByHash = Set.empty, // other no matches
matchByKey = None) // remote is missing matchByKey = None) // remote is missing
it("upload") { } yield (theFile, input)
val expected = Right( it("upload") {
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload env.map({
val result = invoke(input, previousActions) case (theFile, input) => {
assertResult(expected)(result) val expected = Right(Stream(
ToUpload(bucket, theFile, theFile.file.length))) // upload
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
}
} }
} }
describe( describe(
"#4 local exists, remote exists, remote no match, other matches - copy") { "#4 local exists, remote exists, remote no match, other matches - copy") {
val theHash = MD5Hash("the-hash") val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val theRemoteKey = theFile.remoteKey sources,
val oldHash = MD5Hash("old-hash") prefix)
val otherRemoteKey = prefix.resolve("other-key") theRemoteKey = theFile.remoteKey
val otherRemoteMetadata = oldHash = MD5Hash("old-hash")
RemoteMetaData(otherRemoteKey, theHash, lastModified) otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
val oldRemoteMetadata = RemoteMetaData(theRemoteKey, otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
hash = oldHash, // remote no match theHash,
lastModified = lastModified) lastModified)
val input = oldRemoteMetadata = RemoteMetaData(theRemoteKey,
S3MetaData(theFile, // local exists hash = oldHash, // remote no match
matchByHash = Set(otherRemoteMetadata), // other matches lastModified = lastModified)
matchByKey = Some(oldRemoteMetadata)) // remote exists input = S3MetaData(
theFile, // local exists
matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = Some(oldRemoteMetadata)) // remote exists
} yield (theFile, theRemoteKey, input, otherRemoteKey)
it("copy from other key") { it("copy from other key") {
val expected = Right( env.map({
Stream( case (theFile, theRemoteKey, input, otherRemoteKey) => {
ToCopy(bucket, val expected = Right(
otherRemoteKey, Stream(
theHash, ToCopy(bucket,
theRemoteKey, otherRemoteKey,
theFile.file.length))) // copy theHash,
val result = invoke(input, previousActions) theRemoteKey,
assertResult(expected)(result) theFile.file.length))) // copy
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
} }
} }
describe( describe(
"#5 local exists, remote exists, remote no match, other no matches - upload") { "#5 local exists, remote exists, remote no match, other no matches - upload") {
val theHash = MD5Hash("the-hash") val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val theRemoteKey = theFile.remoteKey sources,
val oldHash = MD5Hash("old-hash") prefix)
val theRemoteMetadata = theRemoteKey = theFile.remoteKey
RemoteMetaData(theRemoteKey, oldHash, lastModified) oldHash = MD5Hash("old-hash")
val input = theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
S3MetaData(theFile, // local exists input = S3MetaData(
matchByHash = Set.empty, // remote no match, other no match theFile, // local exists
matchByKey = Some(theRemoteMetadata) // remote exists matchByHash = Set.empty, // remote no match, other no match
matchByKey = Some(theRemoteMetadata) // remote exists
) )
} yield (theFile, input)
it("upload") { it("upload") {
val expected = Right( env.map({
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload case (theFile, input) => {
val result = invoke(input, previousActions) val expected = Right(
assertResult(expected)(result) Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
val result = invoke(input, previousActions)
assertResult(expected)(result)
}
})
} }
} }
describe("#6 local missing, remote exists - delete") { describe("#6 local missing, remote exists - delete") {

View file

@ -1,7 +1,8 @@
package net.kemitix.thorp.domain package net.kemitix.thorp.core
import java.nio.file.Paths import java.nio.file.Paths
import net.kemitix.thorp.domain.Filter
import net.kemitix.thorp.domain.Filter.{Exclude, Include} import net.kemitix.thorp.domain.Filter.{Exclude, Include}
import org.scalatest.FunSpec import org.scalatest.FunSpec
@ -21,31 +22,32 @@ class FiltersSuite extends FunSpec {
describe("default filter") { describe("default filter") {
val include = Include() val include = Include()
it("should include files") { it("should include files") {
paths.foreach(path => assertResult(true)(include.isIncluded(path))) paths.foreach(path =>
assertResult(true)(Filters.isIncludedByFilter(path)(include)))
} }
} }
describe("directory exact match include '/upload/subdir/'") { describe("directory exact match include '/upload/subdir/'") {
val include = Include("/upload/subdir/") val include = Include("/upload/subdir/")
it("include matching directory") { it("include matching directory") {
val matching = Paths.get("/upload/subdir/leaf-file") val matching = Paths.get("/upload/subdir/leaf-file")
assertResult(true)(include.isIncluded(matching)) assertResult(true)(Filters.isIncludedByFilter(matching)(include))
} }
it("exclude non-matching files") { it("exclude non-matching files") {
val nonMatching = Paths.get("/upload/other-file") val nonMatching = Paths.get("/upload/other-file")
assertResult(false)(include.isIncluded(nonMatching)) assertResult(false)(Filters.isIncludedByFilter(nonMatching)(include))
} }
} }
describe("file partial match 'root'") { describe("file partial match 'root'") {
val include = Include("root") val include = Include("root")
it("include matching file '/upload/root-file") { it("include matching file '/upload/root-file") {
val matching = Paths.get("/upload/root-file") val matching = Paths.get("/upload/root-file")
assertResult(true)(include.isIncluded(matching)) assertResult(true)(Filters.isIncludedByFilter(matching)(include))
} }
it("exclude non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") { it("exclude non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
val nonMatching1 = Paths.get("/test-file-for-hash.txt") val nonMatching1 = Paths.get("/test-file-for-hash.txt")
val nonMatching2 = Paths.get("/upload/subdir/leaf-file") val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
assertResult(false)(include.isIncluded(nonMatching1)) assertResult(false)(Filters.isIncludedByFilter(nonMatching1)(include))
assertResult(false)(include.isIncluded(nonMatching2)) assertResult(false)(Filters.isIncludedByFilter(nonMatching2)(include))
} }
} }
} }
@ -63,30 +65,30 @@ class FiltersSuite extends FunSpec {
val exclude = Exclude("/upload/subdir/") val exclude = Exclude("/upload/subdir/")
it("exclude matching directory") { it("exclude matching directory") {
val matching = Paths.get("/upload/subdir/leaf-file") val matching = Paths.get("/upload/subdir/leaf-file")
assertResult(true)(exclude.isExcluded(matching)) assertResult(true)(Filters.isExcludedByFilter(matching)(exclude))
} }
it("include non-matching files") { it("include non-matching files") {
val nonMatching = Paths.get("/upload/other-file") val nonMatching = Paths.get("/upload/other-file")
assertResult(false)(exclude.isExcluded(nonMatching)) assertResult(false)(Filters.isExcludedByFilter(nonMatching)(exclude))
} }
} }
describe("file partial match 'root'") { describe("file partial match 'root'") {
val exclude = Exclude("root") val exclude = Exclude("root")
it("exclude matching file '/upload/root-file") { it("exclude matching file '/upload/root-file") {
val matching = Paths.get("/upload/root-file") val matching = Paths.get("/upload/root-file")
assertResult(true)(exclude.isExcluded(matching)) assertResult(true)(Filters.isExcludedByFilter(matching)(exclude))
} }
it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") { it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
val nonMatching1 = Paths.get("/test-file-for-hash.txt") val nonMatching1 = Paths.get("/test-file-for-hash.txt")
val nonMatching2 = Paths.get("/upload/subdir/leaf-file") val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
assertResult(false)(exclude.isExcluded(nonMatching1)) assertResult(false)(Filters.isExcludedByFilter(nonMatching1)(exclude))
assertResult(false)(exclude.isExcluded(nonMatching2)) assertResult(false)(Filters.isExcludedByFilter(nonMatching2)(exclude))
} }
} }
} }
describe("isIncluded") { describe("isIncluded") {
def invoke(filters: List[Filter]) = { def invoke(filters: List[Filter]) = {
paths.filter(path => Filter.isIncluded(path)(filters)) paths.filter(path => Filters.isIncluded(path)(filters))
} }
describe("when there are no filters") { describe("when there are no filters") {

View file

@ -1,10 +1,12 @@
package net.kemitix.thorp.core package net.kemitix.thorp.core
import java.io.File import java.io.File
import java.nio.file.Path
import net.kemitix.thorp.config.Resource import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.domain.{RemoteKey, Sources} import net.kemitix.thorp.domain.{RemoteKey, Sources}
import org.scalatest.FunSpec import org.scalatest.FunSpec
import zio.DefaultRuntime
class KeyGeneratorSuite extends FunSpec { class KeyGeneratorSuite extends FunSpec {
@ -12,26 +14,32 @@ class KeyGeneratorSuite extends FunSpec {
private val sourcePath = source.toPath private val sourcePath = source.toPath
private val prefix = RemoteKey("prefix") private val prefix = RemoteKey("prefix")
private val sources = Sources(List(sourcePath)) private val sources = Sources(List(sourcePath))
private val fileToKey =
KeyGenerator.generateKey(sources, prefix) _
describe("key generator") { describe("key generator") {
describe("when file is within source") { describe("when file is within source") {
it("has a valid key") { it("has a valid key") {
val subdir = "subdir" val subdir = "subdir"
assertResult(RemoteKey(s"${prefix.key}/$subdir"))( val expected = Right(RemoteKey(s"${prefix.key}/$subdir"))
fileToKey(sourcePath.resolve(subdir))) val result = invoke(sourcePath.resolve(subdir))
assertResult(expected)(result)
} }
} }
describe("when file is deeper within source") { describe("when file is deeper within source") {
it("has a valid key") { it("has a valid key") {
val subdir = "subdir/deeper/still" val subdir = "subdir/deeper/still"
assertResult(RemoteKey(s"${prefix.key}/$subdir"))( val expected = Right(RemoteKey(s"${prefix.key}/$subdir"))
fileToKey(sourcePath.resolve(subdir))) val result = invoke(sourcePath.resolve(subdir))
assertResult(expected)(result)
} }
} }
def invoke(path: Path) = {
new DefaultRuntime {}.unsafeRunSync {
KeyGenerator.generateKey(sources, prefix)(path)
}.toEither
}
} }
} }

View file

@ -32,7 +32,7 @@ class LocalFileStreamSuite extends FunSpec {
val result = val result =
invoke() invoke()
.map(_.localFiles) .map(_.localFiles)
.map(_.map(_.relative.toString)) .map(_.map(LocalFile.relativeToSource(_).toString))
.map(_.toSet) .map(_.toSet)
assertResult(expected)(result) assertResult(expected)(result)
} }

View file

@ -333,13 +333,17 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
md5Hash: MD5Hash, md5Hash: MD5Hash,
source: Path, source: Path,
file: File): (String, String, String, String, String) = file: File): (String, String, String, String, String) =
("upload", remoteKey.key, md5Hash.hash, source.toString, file.toString) ("upload",
remoteKey.key,
MD5Hash.hash(md5Hash),
source.toString,
file.toString)
private def toCopy( private def toCopy(
sourceKey: RemoteKey, sourceKey: RemoteKey,
md5Hash: MD5Hash, md5Hash: MD5Hash,
targetKey: RemoteKey): (String, String, String, String, String) = targetKey: RemoteKey): (String, String, String, String, String) =
("copy", sourceKey.key, md5Hash.hash, targetKey.key, "") ("copy", sourceKey.key, MD5Hash.hash(md5Hash), targetKey.key, "")
private def toDelete( private def toDelete(
remoteKey: RemoteKey): (String, String, String, String, String) = remoteKey: RemoteKey): (String, String, String, String, String) =
@ -386,12 +390,12 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
case ToUpload(_, lf, _) => case ToUpload(_, lf, _) =>
("upload", ("upload",
lf.remoteKey.key, lf.remoteKey.key,
lf.hashes(MD5).hash, MD5Hash.hash(lf.hashes(MD5)),
lf.source.toString, lf.source.toString,
lf.file.toString) lf.file.toString)
case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "") case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")
case ToCopy(_, sourceKey, hash, targetKey, _) => case ToCopy(_, sourceKey, hash, targetKey, _) =>
("copy", sourceKey.key, hash.hash, targetKey.key, "") ("copy", sourceKey.key, MD5Hash.hash(hash), targetKey.key, "")
case DoNothing(_, remoteKey, _) => case DoNothing(_, remoteKey, _) =>
("do-nothing", remoteKey.key, "", "", "") ("do-nothing", remoteKey.key, "", "", "")
case x => ("other", x.toString, "", "", "") case x => ("other", x.toString, "", "", "")

View file

@ -14,8 +14,6 @@ class S3MetaDataEnricherSuite extends FunSpec {
private val sourcePath = source.toPath private val sourcePath = source.toPath
private val sources = Sources(List(sourcePath)) private val sources = Sources(List(sourcePath))
private val prefix = RemoteKey("prefix") private val prefix = RemoteKey("prefix")
private val fileToKey =
KeyGenerator.generateKey(sources, prefix) _
def getMatchesByKey( def getMatchesByKey(
status: (Option[HashModified], Set[(MD5Hash, KeyModified)])) status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
@ -36,138 +34,180 @@ class S3MetaDataEnricherSuite extends FunSpec {
describe( describe(
"#1a local exists, remote exists, remote matches, other matches - do nothing") { "#1a local exists, remote exists, remote matches, other matches - do nothing") {
val theHash: MD5Hash = MD5Hash("the-file-hash") val theHash: MD5Hash = MD5Hash("the-file-hash")
val theFile: LocalFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val theRemoteKey: RemoteKey = theFile.remoteKey sources,
val s3: S3ObjectsData = S3ObjectsData( prefix)
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))), theRemoteKey = theFile.remoteKey
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified)) s3 = S3ObjectsData(
) byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
val theRemoteMetadata = byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
RemoteMetaData(theRemoteKey, theHash, lastModified) )
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
} yield (theFile, theRemoteMetadata, s3)
it("generates valid metadata") { it("generates valid metadata") {
val expected = S3MetaData(theFile, env.map({
matchByHash = Set(theRemoteMetadata), case (theFile, theRemoteMetadata, s3) => {
matchByKey = Some(theRemoteMetadata)) val expected = S3MetaData(theFile,
val result = getMetadata(theFile, s3) matchByHash = Set(theRemoteMetadata),
assertResult(expected)(result) matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
} }
} }
describe( describe(
"#1b local exists, remote exists, remote matches, other no matches - do nothing") { "#1b local exists, remote exists, remote matches, other no matches - do nothing") {
val theHash: MD5Hash = MD5Hash("the-file-hash") val theHash: MD5Hash = MD5Hash("the-file-hash")
val theFile: LocalFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val theRemoteKey: RemoteKey = prefix.resolve("the-file") sources,
val s3: S3ObjectsData = S3ObjectsData( prefix)
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))), theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix)
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified)) s3: S3ObjectsData = S3ObjectsData(
) byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
val theRemoteMetadata = byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
RemoteMetaData(theRemoteKey, theHash, lastModified) )
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
} yield (theFile, theRemoteMetadata, s3)
it("generates valid metadata") { it("generates valid metadata") {
val expected = S3MetaData(theFile, env.map({
matchByHash = Set(theRemoteMetadata), case (theFile, theRemoteMetadata, s3) => {
matchByKey = Some(theRemoteMetadata)) val expected = S3MetaData(theFile,
val result = getMetadata(theFile, s3) matchByHash = Set(theRemoteMetadata),
assertResult(expected)(result) matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
} }
} }
describe( describe(
"#2 local exists, remote is missing, remote no match, other matches - copy") { "#2 local exists, remote is missing, remote no match, other matches - copy") {
val theHash = MD5Hash("the-hash") val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val otherRemoteKey = RemoteKey("other-key") sources,
val s3: S3ObjectsData = S3ObjectsData( prefix)
byHash = Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))), otherRemoteKey = RemoteKey("other-key")
byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified)) s3: S3ObjectsData = S3ObjectsData(
) byHash =
val otherRemoteMetadata = Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
RemoteMetaData(otherRemoteKey, theHash, lastModified) byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
)
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
theHash,
lastModified)
} yield (theFile, otherRemoteMetadata, s3)
it("generates valid metadata") { it("generates valid metadata") {
val expected = S3MetaData(theFile, env.map({
matchByHash = Set(otherRemoteMetadata), case (theFile, otherRemoteMetadata, s3) => {
matchByKey = None) val expected = S3MetaData(theFile,
val result = getMetadata(theFile, s3) matchByHash = Set(otherRemoteMetadata),
assertResult(expected)(result) matchByKey = None)
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
} }
} }
describe( describe(
"#3 local exists, remote is missing, remote no match, other no matches - upload") { "#3 local exists, remote is missing, remote no match, other no matches - upload") {
val theHash = MD5Hash("the-hash") val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val s3: S3ObjectsData = S3ObjectsData() sources,
prefix)
s3: S3ObjectsData = S3ObjectsData()
} yield (theFile, s3)
it("generates valid metadata") { it("generates valid metadata") {
val expected = env.map({
S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None) case (theFile, s3) => {
val result = getMetadata(theFile, s3) val expected =
assertResult(expected)(result) S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None)
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
} }
} }
describe( describe(
"#4 local exists, remote exists, remote no match, other matches - copy") { "#4 local exists, remote exists, remote no match, other matches - copy") {
val theHash = MD5Hash("the-hash") val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val theRemoteKey = theFile.remoteKey sources,
val oldHash = MD5Hash("old-hash") prefix)
val otherRemoteKey = prefix.resolve("other-key") theRemoteKey = theFile.remoteKey
val s3: S3ObjectsData = S3ObjectsData( oldHash = MD5Hash("old-hash")
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)), otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
theHash -> Set(KeyModified(otherRemoteKey, lastModified))), s3: S3ObjectsData = S3ObjectsData(
byKey = Map( byHash =
theRemoteKey -> HashModified(oldHash, lastModified), Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
otherRemoteKey -> HashModified(theHash, lastModified) theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
byKey = Map(
theRemoteKey -> HashModified(oldHash, lastModified),
otherRemoteKey -> HashModified(theHash, lastModified)
)
) )
) theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
val theRemoteMetadata = otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
RemoteMetaData(theRemoteKey, oldHash, lastModified) theHash,
val otherRemoteMetadata = lastModified)
RemoteMetaData(otherRemoteKey, theHash, lastModified) } yield (theFile, theRemoteMetadata, otherRemoteMetadata, s3)
it("generates valid metadata") { it("generates valid metadata") {
val expected = S3MetaData(theFile, env.map({
matchByHash = Set(otherRemoteMetadata), case (theFile, theRemoteMetadata, otherRemoteMetadata, s3) => {
matchByKey = Some(theRemoteMetadata)) val expected = S3MetaData(theFile,
val result = getMetadata(theFile, s3) matchByHash = Set(otherRemoteMetadata),
assertResult(expected)(result) matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
} }
} }
describe( describe(
"#5 local exists, remote exists, remote no match, other no matches - upload") { "#5 local exists, remote exists, remote no match, other no matches - upload") {
val theHash = MD5Hash("the-hash") val theHash = MD5Hash("the-hash")
val theFile = LocalFile.resolve("the-file", val env = for {
md5HashMap(theHash), theFile <- LocalFileValidator.resolve("the-file",
sourcePath, md5HashMap(theHash),
fileToKey) sourcePath,
val theRemoteKey = theFile.remoteKey sources,
val oldHash = MD5Hash("old-hash") prefix)
val s3: S3ObjectsData = S3ObjectsData( theRemoteKey = theFile.remoteKey
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)), oldHash = MD5Hash("old-hash")
theHash -> Set.empty), s3: S3ObjectsData = S3ObjectsData(
byKey = Map( byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
theRemoteKey -> HashModified(oldHash, lastModified) theHash -> Set.empty),
byKey = Map(
theRemoteKey -> HashModified(oldHash, lastModified)
)
) )
) theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
val theRemoteMetadata = } yield (theFile, theRemoteMetadata, s3)
RemoteMetaData(theRemoteKey, oldHash, lastModified)
it("generates valid metadata") { it("generates valid metadata") {
val expected = S3MetaData(theFile, env.map({
matchByHash = Set.empty, case (theFile, theRemoteMetadata, s3) => {
matchByKey = Some(theRemoteMetadata)) val expected = S3MetaData(theFile,
val result = getMetadata(theFile, s3) matchByHash = Set.empty,
assertResult(expected)(result) matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3)
assertResult(expected)(result)
}
})
} }
} }
} }
@ -178,71 +218,103 @@ class S3MetaDataEnricherSuite extends FunSpec {
describe("getS3Status") { describe("getS3Status") {
val hash = MD5Hash("hash") val hash = MD5Hash("hash")
val localFile = val env = for {
LocalFile.resolve("the-file", md5HashMap(hash), sourcePath, fileToKey) localFile <- LocalFileValidator.resolve("the-file",
val key = localFile.remoteKey md5HashMap(hash),
val keyOtherKey = LocalFile.resolve("other-key-same-hash", sourcePath,
md5HashMap(hash), sources,
sourcePath, prefix)
fileToKey) key = localFile.remoteKey
val diffHash = MD5Hash("diff") keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash",
val keyDiffHash = LocalFile.resolve("other-key-diff-hash", md5HashMap(hash),
md5HashMap(diffHash), sourcePath,
sourcePath, sources,
fileToKey) prefix)
val lastModified = LastModified(Instant.now) diffHash = MD5Hash("diff")
val s3ObjectsData: S3ObjectsData = S3ObjectsData( keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash",
byHash = Map( md5HashMap(diffHash),
hash -> Set(KeyModified(key, lastModified), sourcePath,
KeyModified(keyOtherKey.remoteKey, lastModified)), sources,
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified)) prefix)
), lastModified = LastModified(Instant.now)
byKey = Map( s3ObjectsData = S3ObjectsData(
key -> HashModified(hash, lastModified), byHash = Map(
keyOtherKey.remoteKey -> HashModified(hash, lastModified), hash -> Set(KeyModified(key, lastModified),
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified) KeyModified(keyOtherKey.remoteKey, lastModified)),
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
),
byKey = Map(
key -> HashModified(hash, lastModified),
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
)
) )
) } yield (s3ObjectsData, localFile, keyDiffHash, diffHash)
def invoke(localFile: LocalFile) = { def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) = {
getS3Status(localFile, s3ObjectsData) getS3Status(localFile, s3ObjectsData)
} }
describe("when remote key exists") { describe("when remote key exists") {
it("should return a result for matching key") { it("should return a result for matching key") {
val result = getMatchesByKey(invoke(localFile)) env.map({
assert(result.contains(HashModified(hash, lastModified))) case (s3ObjectsData, localFile: LocalFile, _, _) =>
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
assert(result.contains(HashModified(hash, lastModified)))
})
} }
} }
describe("when remote key does not exist and no others matches hash") { describe("when remote key does not exist and no others matches hash") {
val localFile = LocalFile.resolve("missing-file", val env2 = for {
md5HashMap(MD5Hash("unique")), localFile <- LocalFileValidator.resolve("missing-remote",
sourcePath, md5HashMap(MD5Hash("unique")),
fileToKey) sourcePath,
sources,
prefix)
} yield (localFile)
it("should return no matches by key") { it("should return no matches by key") {
val result = getMatchesByKey(invoke(localFile)) env.map({
assert(result.isEmpty) case (s3ObjectsData, _, _, _) => {
env2.map({
case (localFile) => {
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
assert(result.isEmpty)
}
})
}
})
} }
it("should return no matches by hash") { it("should return no matches by hash") {
val result = getMatchesByHash(invoke(localFile)) env.map({
assert(result.isEmpty) case (s3ObjectsData, _, _, _) => {
env2.map({
case (localFile) => {
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
assert(result.isEmpty)
}
})
}
})
} }
} }
describe("when remote key exists and no others match hash") { describe("when remote key exists and no others match hash") {
it("should return match by key") { env.map({
val result = getMatchesByKey(invoke(keyDiffHash)) case (s3ObjectsData, _, keyDiffHash, diffHash) => {
assert(result.contains(HashModified(diffHash, lastModified))) it("should return match by key") {
} val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData))
it("should return only itself in match by hash") { assert(result.contains(HashModified(diffHash, lastModified)))
val result = getMatchesByHash(invoke(keyDiffHash)) }
assert( it("should return only itself in match by hash") {
result.equals( val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData))
Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified))))) assert(
} result.equals(Set(
(diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
}
}
})
} }
} }
} }

View file

@ -3,6 +3,7 @@ package net.kemitix.thorp.core.hasher
import java.nio.file.Path import java.nio.file.Path
import net.kemitix.thorp.config.Resource import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root} import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root}
import net.kemitix.thorp.filesystem.FileSystem import net.kemitix.thorp.filesystem.FileSystem
import org.scalatest.FunSpec import org.scalatest.FunSpec
@ -42,14 +43,14 @@ class MD5HashGeneratorTest extends FunSpec {
val path = Resource(this, "../big-file").toPath val path = Resource(this, "../big-file").toPath
it("should generate the correct hash for first chunk of the file") { it("should generate the correct hash for first chunk of the file") {
val part1 = BigFile.Part1 val part1 = BigFile.Part1
val expected = Right(part1.hash.hash) val expected = Right(MD5Hash.hash(part1.hash))
val result = invoke(path, part1.offset, part1.size).map(_.hash) val result = invoke(path, part1.offset, part1.size).map(MD5Hash.hash)
assertResult(expected)(result) assertResult(expected)(result)
} }
it("should generate the correct hash for second chunk of the file") { it("should generate the correct hash for second chunk of the file") {
val part2 = BigFile.Part2 val part2 = BigFile.Part2
val expected = Right(part2.hash.hash) val expected = Right(MD5Hash.hash(part2.hash))
val result = invoke(path, part2.offset, part2.size).map(_.hash) val result = invoke(path, part2.offset, part2.size).map(MD5Hash.hash)
assertResult(expected)(result) assertResult(expected)(result)
} }
} }

View file

@ -1,53 +1,18 @@
package net.kemitix.thorp.domain package net.kemitix.thorp.domain
import java.nio.file.Path import java.util.function.Predicate
import java.util.regex.Pattern import java.util.regex.Pattern
sealed trait Filter sealed trait Filter {
def predicate: Predicate[String]
}
object Filter { object Filter {
case class Include(include: String = ".*") extends Filter {
def isIncluded(p: Path)(filters: List[Filter]): Boolean = { lazy val predicate: Predicate[String] = Pattern.compile(include).asPredicate
sealed trait State
case class Unknown() extends State
case class Accepted() extends State
case class Discarded() extends State
filters.foldRight(Unknown(): State)((filter, state) =>
(filter, state) match {
case (_, Accepted()) => Accepted()
case (_, Discarded()) => Discarded()
case (x: Exclude, _) if x.isExcluded(p) => Discarded()
case (i: Include, _) if i.isIncluded(p) => Accepted()
case _ => Unknown()
}) match {
case Accepted() => true
case Discarded() => false
case Unknown() =>
filters.forall {
case _: Include => false
case _ => true
}
}
} }
case class Exclude(exclude: String) extends Filter {
case class Include( lazy val predicate: Predicate[String] =
include: String = ".*" Pattern.compile(exclude).asPredicate()
) extends Filter {
private lazy val predicate = Pattern.compile(include).asPredicate
def isIncluded(path: Path): Boolean = predicate.test(path.toString)
} }
case class Exclude(
exclude: String
) extends Filter {
private lazy val predicate = Pattern.compile(exclude).asPredicate()
def isExcluded(path: Path): Boolean = predicate.test(path.toString)
}
} }

View file

@ -5,42 +5,22 @@ import java.nio.file.Path
import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain.HashType.MD5
final case class LocalFile( final case class LocalFile private (
file: File, file: File,
source: File, source: File,
hashes: Map[HashType, MD5Hash], hashes: Map[HashType, MD5Hash],
remoteKey: RemoteKey remoteKey: RemoteKey
) { )
require(!file.isDirectory, s"LocalFile must not be a directory: $file")
def isDirectory: Boolean = file.isDirectory
// the path of the file within the source
def relative: Path = source.toPath.relativize(file.toPath)
def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _)
def md5base64: Option[String] = hashes.get(MD5).map(_.hash64)
}
object LocalFile { object LocalFile {
def resolve(
path: String,
md5Hashes: Map[HashType, MD5Hash],
source: Path,
pathToKey: Path => RemoteKey
): LocalFile = {
val resolvedPath = source.resolve(path)
LocalFile(resolvedPath.toFile,
source.toFile,
md5Hashes,
pathToKey(resolvedPath))
}
val remoteKey: SimpleLens[LocalFile, RemoteKey] = val remoteKey: SimpleLens[LocalFile, RemoteKey] =
SimpleLens[LocalFile, RemoteKey](_.remoteKey, SimpleLens[LocalFile, RemoteKey](_.remoteKey,
b => a => b.copy(remoteKey = a)) b => a => b.copy(remoteKey = a))
// the path of the file within the source
def relativeToSource(localFile: LocalFile): Path =
localFile.source.toPath.relativize(localFile.file.toPath)
def matchesHash(localFile: LocalFile)(other: MD5Hash): Boolean =
localFile.hashes.values.exists(other equals _)
def md5base64(localFile: LocalFile): Option[String] =
localFile.hashes.get(MD5).map(MD5Hash.hash64)
} }

View file

@ -1,15 +0,0 @@
package net.kemitix.thorp.domain
trait Logger {
// returns an instance of Logger with debug set as indicated
// where the current Logger already matches this state, then
// it returns itself, unmodified
def withDebug(debug: Boolean): Logger
def debug(message: => String): Unit
def info(message: => String): Unit
def warn(message: String): Unit
def error(message: String): Unit
}

View file

@ -4,20 +4,14 @@ import java.util.Base64
import net.kemitix.thorp.domain.QuoteStripper.stripQuotes import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
final case class MD5Hash( final case class MD5Hash(in: String)
in: String
) {
lazy val hash: String = in filter stripQuotes
lazy val digest: Array[Byte] = HexEncoder.decode(hash)
lazy val hash64: String = Base64.getEncoder.encodeToString(digest)
}
object MD5Hash { object MD5Hash {
def fromDigest(digest: Array[Byte]): MD5Hash = { def fromDigest(digest: Array[Byte]): MD5Hash =
val hexDigest = (digest map ("%02x" format _)).mkString MD5Hash((digest map ("%02x" format _)).mkString)
MD5Hash(hexDigest) def hash(md5Hash: MD5Hash): String = md5Hash.in.filter(stripQuotes)
} def digest(md5Hash: MD5Hash): Array[Byte] =
HexEncoder.decode(MD5Hash.hash(md5Hash))
def hash64(md5Hash: MD5Hash): String =
Base64.getEncoder.encodeToString(MD5Hash.digest(md5Hash))
} }

View file

@ -3,31 +3,21 @@ package net.kemitix.thorp.domain
import java.io.File import java.io.File
import java.nio.file.{Path, Paths} import java.nio.file.{Path, Paths}
final case class RemoteKey( final case class RemoteKey(key: String)
key: String
) {
def asFile(
source: Path,
prefix: RemoteKey
): Option[File] =
if (key.length == 0) None
else Some(source.resolve(relativeTo(prefix)).toFile)
private def relativeTo(prefix: RemoteKey) = {
prefix match {
case RemoteKey("") => Paths.get(key)
case _ => Paths.get(prefix.key).relativize(Paths.get(key))
}
}
def resolve(path: String): RemoteKey =
RemoteKey(List(key, path).filterNot(_.isEmpty).mkString("/"))
}
object RemoteKey { object RemoteKey {
val key: SimpleLens[RemoteKey, String] = val key: SimpleLens[RemoteKey, String] =
SimpleLens[RemoteKey, String](_.key, b => a => b.copy(key = a)) SimpleLens[RemoteKey, String](_.key, b => a => b.copy(key = a))
def asFile(source: Path, prefix: RemoteKey)(
remoteKey: RemoteKey): Option[File] =
if (remoteKey.key.length == 0) None
else Some(source.resolve(RemoteKey.relativeTo(prefix)(remoteKey)).toFile)
def relativeTo(prefix: RemoteKey)(remoteKey: RemoteKey): Path = {
prefix match {
case RemoteKey("") => Paths.get(remoteKey.key)
case _ => Paths.get(prefix.key).relativize(Paths.get(remoteKey.key))
}
}
def resolve(path: String)(remoteKey: RemoteKey): RemoteKey =
RemoteKey(List(remoteKey.key, path).filterNot(_.isEmpty).mkString("/"))
} }

View file

@ -2,6 +2,8 @@ package net.kemitix.thorp.domain
import java.nio.file.Path import java.nio.file.Path
import zio.{Task, ZIO}
/** /**
* The paths to synchronise with target. * The paths to synchronise with target.
* *
@ -18,18 +20,10 @@ case class Sources(
def +(path: Path)(implicit m: Monoid[Sources]): Sources = this ++ List(path) def +(path: Path)(implicit m: Monoid[Sources]): Sources = this ++ List(path)
def ++(otherPaths: List[Path])(implicit m: Monoid[Sources]): Sources = def ++(otherPaths: List[Path])(implicit m: Monoid[Sources]): Sources =
m.op(this, Sources(otherPaths)) m.op(this, Sources(otherPaths))
/**
* Returns the source path for the given path.
*/
def forPath(path: Path): Path =
paths.find(source => path.startsWith(source)).get
} }
object Sources { object Sources {
final val emptySources = Sources(List.empty) final val emptySources = Sources(List.empty)
implicit def sourcesAppendMonoid: Monoid[Sources] = new Monoid[Sources] { implicit def sourcesAppendMonoid: Monoid[Sources] = new Monoid[Sources] {
override def zero: Sources = emptySources override def zero: Sources = emptySources
override def op(t1: Sources, t2: Sources): Sources = override def op(t1: Sources, t2: Sources): Sources =
@ -38,4 +32,12 @@ object Sources {
else acc ++ List(path) else acc ++ List(path)
}) })
} }
/**
* Returns the source path for the given path.
*/
def forPath(path: Path)(sources: Sources): Task[Path] =
ZIO
.fromOption(sources.paths.find(s => path.startsWith(s)))
.mapError(_ => new Exception("Path is not within any known source"))
} }

View file

@ -1,10 +1,7 @@
package net.kemitix.thorp.domain package net.kemitix.thorp.domain
import net.kemitix.thorp.domain.UploadEvent.RequestEvent import net.kemitix.thorp.domain.UploadEvent.RequestEvent
import net.kemitix.thorp.domain.UploadEventLogger.{ import net.kemitix.thorp.domain.UploadEventLogger.RequestCycle
RequestCycle,
logRequestCycle
}
object UploadEventListener { object UploadEventListener {
@ -21,7 +18,7 @@ object UploadEventListener {
uploadEvent match { uploadEvent match {
case e: RequestEvent => case e: RequestEvent =>
bytesTransferred += e.transferred bytesTransferred += e.transferred
logRequestCycle( UploadEventLogger(
RequestCycle(settings.localFile, RequestCycle(settings.localFile,
bytesTransferred, bytesTransferred,
settings.index, settings.index,

View file

@ -14,7 +14,7 @@ object UploadEventLogger {
totalBytesSoFar: Long totalBytesSoFar: Long
) )
def logRequestCycle(requestCycle: RequestCycle): Unit = { def apply(requestCycle: RequestCycle): Unit = {
val remoteKey = requestCycle.localFile.remoteKey.key val remoteKey = requestCycle.localFile.remoteKey.key
val fileLength = requestCycle.localFile.file.length val fileLength = requestCycle.localFile.file.length
val statusHeight = 7 val statusHeight = 7

View file

@ -7,11 +7,11 @@ class MD5HashTest extends FunSpec {
describe("recover base64 hash") { describe("recover base64 hash") {
it("should recover base 64 #1") { it("should recover base 64 #1") {
val rootHash = MD5HashData.Root.hash val rootHash = MD5HashData.Root.hash
assertResult(MD5HashData.Root.base64)(rootHash.hash64) assertResult(MD5HashData.Root.base64)(MD5Hash.hash64(rootHash))
} }
it("should recover base 64 #2") { it("should recover base 64 #2") {
val leafHash = MD5HashData.Leaf.hash val leafHash = MD5HashData.Leaf.hash
assertResult(MD5HashData.Leaf.base64)(leafHash.hash64) assertResult(MD5HashData.Leaf.base64)(MD5Hash.hash64(leafHash))
} }
} }
} }

View file

@ -15,21 +15,21 @@ class RemoteKeyTest extends FreeSpec {
val key = emptyKey val key = emptyKey
val path = "path" val path = "path"
val expected = RemoteKey("path") val expected = RemoteKey("path")
val result = key.resolve(path) val result = RemoteKey.resolve(path)(key)
assertResult(expected)(result) assertResult(expected)(result)
} }
"when path is empty" in { "when path is empty" in {
val key = RemoteKey("key") val key = RemoteKey("key")
val path = "" val path = ""
val expected = RemoteKey("key") val expected = RemoteKey("key")
val result = key.resolve(path) val result = RemoteKey.resolve(path)(key)
assertResult(expected)(result) assertResult(expected)(result)
} }
"when key and path are empty" in { "when key and path are empty" in {
val key = emptyKey val key = emptyKey
val path = "" val path = ""
val expected = emptyKey val expected = emptyKey
val result = key.resolve(path) val result = RemoteKey.resolve(path)(key)
assertResult(expected)(result) assertResult(expected)(result)
} }
} }
@ -39,7 +39,7 @@ class RemoteKeyTest extends FreeSpec {
val source = Paths.get("source") val source = Paths.get("source")
val prefix = RemoteKey("prefix") val prefix = RemoteKey("prefix")
val expected = Some(new File("source/key")) val expected = Some(new File("source/key"))
val result = key.asFile(source, prefix) val result = RemoteKey.asFile(source, prefix)(key)
assertResult(expected)(result) assertResult(expected)(result)
} }
"when prefix is empty" in { "when prefix is empty" in {
@ -47,7 +47,7 @@ class RemoteKeyTest extends FreeSpec {
val source = Paths.get("source") val source = Paths.get("source")
val prefix = emptyKey val prefix = emptyKey
val expected = Some(new File("source/key")) val expected = Some(new File("source/key"))
val result = key.asFile(source, prefix) val result = RemoteKey.asFile(source, prefix)(key)
assertResult(expected)(result) assertResult(expected)(result)
} }
"when key is empty" in { "when key is empty" in {
@ -55,7 +55,7 @@ class RemoteKeyTest extends FreeSpec {
val source = Paths.get("source") val source = Paths.get("source")
val prefix = RemoteKey("prefix") val prefix = RemoteKey("prefix")
val expected = None val expected = None
val result = key.asFile(source, prefix) val result = RemoteKey.asFile(source, prefix)(key)
assertResult(expected)(result) assertResult(expected)(result)
} }
"when key and prefix are empty" in { "when key and prefix are empty" in {
@ -63,7 +63,7 @@ class RemoteKeyTest extends FreeSpec {
val source = Paths.get("source") val source = Paths.get("source")
val prefix = emptyKey val prefix = emptyKey
val expected = None val expected = None
val result = key.asFile(source, prefix) val result = RemoteKey.asFile(source, prefix)(key)
assertResult(expected)(result) assertResult(expected)(result)
} }
} }

View file

@ -38,11 +38,13 @@ trait TemporaryFolder {
path.resolve(name).toFile path.resolve(name).toFile
} }
def writeFile(directory: Path, name: String, contents: String*): Unit = { def writeFile(directory: Path, name: String, contents: String*): File = {
directory.toFile.mkdirs directory.toFile.mkdirs
val pw = new PrintWriter(directory.resolve(name).toFile, "UTF-8") val file = directory.resolve(name).toFile
contents.foreach(pw.println) val writer = new PrintWriter(file, "UTF-8")
pw.close() contents.foreach(writer.println)
writer.close()
file
} }
} }

View file

@ -41,7 +41,7 @@ trait Copier {
copyRequest.sourceKey.key, copyRequest.sourceKey.key,
copyRequest.bucket.name, copyRequest.bucket.name,
copyRequest.targetKey.key copyRequest.targetKey.key
).withMatchingETagConstraint(copyRequest.hash.hash) ).withMatchingETagConstraint(MD5Hash.hash(copyRequest.hash))
private def foldFailure( private def foldFailure(
sourceKey: RemoteKey, sourceKey: RemoteKey,

View file

@ -68,7 +68,7 @@ trait Uploader {
private def metadata: LocalFile => ObjectMetadata = localFile => { private def metadata: LocalFile => ObjectMetadata = localFile => {
val metadata = new ObjectMetadata() val metadata = new ObjectMetadata()
localFile.md5base64.foreach(metadata.setContentMD5) LocalFile.md5base64(localFile).foreach(metadata.setContentMD5)
metadata metadata
} }

View file

@ -7,6 +7,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils
import net.kemitix.thorp.core.hasher.Hasher import net.kemitix.thorp.core.hasher.Hasher
import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.filesystem.FileSystem import net.kemitix.thorp.filesystem.FileSystem
import zio.{TaskR, ZIO} import zio.{TaskR, ZIO}
@ -56,7 +57,7 @@ private trait ETagGenerator {
Hasher Hasher
.hashObjectChunk(path, chunkNumber, chunkSize) .hashObjectChunk(path, chunkNumber, chunkSize)
.map(_(MD5)) .map(_(MD5))
.map(_.digest) .map(MD5Hash.digest)
def offsets( def offsets(
totalFileSizeBytes: Long, totalFileSizeBytes: Long,

View file

@ -32,7 +32,7 @@ class S3ObjectsByHashSuite extends FunSpec {
remoteKey: RemoteKey, remoteKey: RemoteKey,
lastModified: LastModified): S3ObjectSummary = { lastModified: LastModified): S3ObjectSummary = {
val summary = new S3ObjectSummary() val summary = new S3ObjectSummary()
summary.setETag(md5Hash.hash) summary.setETag(MD5Hash.hash(md5Hash))
summary.setKey(remoteKey.key) summary.setKey(remoteKey.key)
summary.setLastModified(Date.from(lastModified.when)) summary.setLastModified(Date.from(lastModified.when))
summary summary

View file

@ -3,7 +3,7 @@ package net.kemitix.thorp.storage.aws
import java.time.Instant import java.time.Instant
import net.kemitix.thorp.config.Resource import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.core.{KeyGenerator, S3MetaDataEnricher} import net.kemitix.thorp.core.{LocalFileValidator, S3MetaDataEnricher}
import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain._
import org.scalamock.scalatest.MockFactory import org.scalamock.scalatest.MockFactory
@ -15,39 +15,51 @@ class StorageServiceSuite extends FunSpec with MockFactory {
private val sourcePath = source.toPath private val sourcePath = source.toPath
private val sources = Sources(List(sourcePath)) private val sources = Sources(List(sourcePath))
private val prefix = RemoteKey("prefix") private val prefix = RemoteKey("prefix")
private val fileToKey =
KeyGenerator.generateKey(sources, prefix) _
describe("getS3Status") { describe("getS3Status") {
val hash = MD5Hash("hash") val hash = MD5Hash("hash")
val localFile = val env = for {
LocalFile.resolve("the-file", Map(MD5 -> hash), sourcePath, fileToKey) localFile <- LocalFileValidator.resolve("the-file",
val key = localFile.remoteKey Map(MD5 -> hash),
val keyOtherKey = LocalFile.resolve("other-key-same-hash", sourcePath,
Map(MD5 -> hash), sources,
sourcePath, prefix)
fileToKey) key = localFile.remoteKey
val diffHash = MD5Hash("diff") keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash",
val keyDiffHash = LocalFile.resolve("other-key-diff-hash", Map(MD5 -> hash),
Map(MD5 -> diffHash), sourcePath,
sourcePath, sources,
fileToKey) prefix)
val lastModified = LastModified(Instant.now) diffHash = MD5Hash("diff")
val s3ObjectsData: S3ObjectsData = S3ObjectsData( keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash",
byHash = Map( Map(MD5 -> diffHash),
hash -> Set(KeyModified(key, lastModified), sourcePath,
KeyModified(keyOtherKey.remoteKey, lastModified)), sources,
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified)) prefix)
), lastModified = LastModified(Instant.now)
byKey = Map( s3ObjectsData = S3ObjectsData(
key -> HashModified(hash, lastModified), byHash = Map(
keyOtherKey.remoteKey -> HashModified(hash, lastModified), hash -> Set(KeyModified(key, lastModified),
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified) KeyModified(keyOtherKey.remoteKey, lastModified)),
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
),
byKey = Map(
key -> HashModified(hash, lastModified),
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
)
) )
) } yield
(s3ObjectsData,
localFile: LocalFile,
lastModified,
keyOtherKey,
keyDiffHash,
diffHash,
key)
def invoke(localFile: LocalFile) = def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) =
S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData) S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData)
def getMatchesByKey( def getMatchesByKey(
@ -67,47 +79,94 @@ class StorageServiceSuite extends FunSpec with MockFactory {
describe( describe(
"when remote key exists, unmodified and other key matches the hash") { "when remote key exists, unmodified and other key matches the hash") {
it("should return the match by key") { it("should return the match by key") {
val result = getMatchesByKey(invoke(localFile)) env.map({
assert(result.contains(HashModified(hash, lastModified))) case (s3ObjectsData, localFile, lastModified, _, _, _, _) => {
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
assert(result.contains(HashModified(hash, lastModified)))
}
})
} }
it("should return both matches for the hash") { it("should return both matches for the hash") {
val result = getMatchesByHash(invoke(localFile)) env.map({
assertResult( case (s3ObjectsData,
Set((hash, KeyModified(key, lastModified)), localFile,
(hash, KeyModified(keyOtherKey.remoteKey, lastModified))) lastModified,
)(result) keyOtherKey,
_,
_,
key) => {
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
assertResult(
Set((hash, KeyModified(key, lastModified)),
(hash, KeyModified(keyOtherKey.remoteKey, lastModified)))
)(result)
}
})
} }
} }
describe("when remote key does not exist and no others matches hash") { describe("when remote key does not exist and no others matches hash") {
val localFile = LocalFile.resolve("missing-file", val env2 = LocalFileValidator
Map(MD5 -> MD5Hash("unique")), .resolve("missing-file",
sourcePath, Map(MD5 -> MD5Hash("unique")),
fileToKey) sourcePath,
sources,
prefix)
it("should return no matches by key") { it("should return no matches by key") {
val result = getMatchesByKey(invoke(localFile)) env2.map(localFile => {
assert(result.isEmpty) env.map({
case (s3ObjectsData, _, _, _, _, _, _) => {
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
assert(result.isEmpty)
}
})
})
} }
it("should return no matches by hash") { it("should return no matches by hash") {
val result = getMatchesByHash(invoke(localFile)) env2.map(localFile => {
assert(result.isEmpty) env.map({
case (s3ObjectsData, _, _, _, _, _, _) => {
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
assert(result.isEmpty)
}
})
})
} }
} }
describe("when remote key exists and no others match hash") { describe("when remote key exists and no others match hash") {
val localFile = keyDiffHash
it("should return the match by key") { it("should return the match by key") {
val result = getMatchesByKey(invoke(localFile)) env.map({
assert(result.contains(HashModified(diffHash, lastModified))) case (s3ObjectsData,
_,
lastModified,
_,
keyDiffHash,
diffHash,
_) => {
val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData))
assert(result.contains(HashModified(diffHash, lastModified)))
}
})
} }
it("should return one match by hash") { it("should return one match by hash") {
val result = getMatchesByHash(invoke(localFile)) env.map({
assertResult( case (s3ObjectsData,
Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified))) _,
)(result) lastModified,
_,
keyDiffHash,
diffHash,
_) => {
val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData))
assertResult(
Set(
(diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))
)(result)
}
})
} }
} }
} }
} }

View file

@ -29,7 +29,7 @@ class UploaderTest extends FreeSpec with MockFactory {
val bucket = Bucket("aBucket") val bucket = Bucket("aBucket")
val uploadResult = new UploadResult val uploadResult = new UploadResult
uploadResult.setKey(remoteKey.key) uploadResult.setKey(remoteKey.key)
uploadResult.setETag(aHash.hash) uploadResult.setETag(MD5Hash.hash(aHash))
val inProgress = new AmazonUpload.InProgress { val inProgress = new AmazonUpload.InProgress {
override def waitForUploadResult: UploadResult = uploadResult override def waitForUploadResult: UploadResult = uploadResult
} }

View file

@ -6,6 +6,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
import net.kemitix.thorp.config.Resource import net.kemitix.thorp.config.Resource
import net.kemitix.thorp.core.hasher.Hasher import net.kemitix.thorp.core.hasher.Hasher
import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.filesystem.FileSystem import net.kemitix.thorp.filesystem.FileSystem
import org.scalatest.FunSpec import org.scalatest.FunSpec
import zio.DefaultRuntime import zio.DefaultRuntime
@ -45,7 +46,7 @@ class ETagGeneratorTest extends FunSpec {
assertResult(Right(hash))( assertResult(Right(hash))(
invoke(bigFilePath, index, chunkSize) invoke(bigFilePath, index, chunkSize)
.map(_(MD5)) .map(_(MD5))
.map(_.hash)) .map(MD5Hash.hash))
} }
} }
def invoke(path: Path, index: Long, size: Long) = def invoke(path: Path, index: Long, size: Long) =