case classes shouldn't be OO objects (#147)
* [core] Extract Filters from domain.Filter * [core] extract LocalFileValidator * [domain] LocalFile remove unused isDirectory * [domain] LocalFile move/rename relative as relativeToSource on companion * [domain] LocalFile move and rename matches as matchesHash on companion * [domain] LocalFile move md5base64 to companion * [domain] Logger remove * [domain] MD5Hash move hash to companion * [domain] MD5Hash move digest to companion * [domain] MD5Hash move hash64 to companion * [domain] RemoteKey move class methods to companion * [domain] Sources move forPath to companion Led to being able to cleanup LocalFileStream.localFile and adding LocalFiles.one * [domain] UploadEventLogger rename method as apply
This commit is contained in:
parent
96ecedbe61
commit
8fad680a96
38 changed files with 750 additions and 551 deletions
|
@ -135,3 +135,4 @@ lazy val domain = (project in file("domain"))
|
||||||
.settings(commonSettings)
|
.settings(commonSettings)
|
||||||
.settings(assemblyJarName in assembly := "domain.jar")
|
.settings(assemblyJarName in assembly := "domain.jar")
|
||||||
.settings(testDependencies)
|
.settings(testDependencies)
|
||||||
|
.settings(zioDependencies)
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
package net.kemitix.thorp.config
|
package net.kemitix.thorp.config
|
||||||
|
|
||||||
import java.nio.file.Path
|
import java.io.File
|
||||||
|
|
||||||
sealed trait ConfigValidation {
|
sealed trait ConfigValidation {
|
||||||
|
|
||||||
|
@ -22,10 +22,10 @@ object ConfigValidation {
|
||||||
}
|
}
|
||||||
|
|
||||||
case class ErrorReadingFile(
|
case class ErrorReadingFile(
|
||||||
path: Path,
|
file: File,
|
||||||
message: String
|
message: String
|
||||||
) extends ConfigValidation {
|
) extends ConfigValidation {
|
||||||
override def errorMessage: String = s"Error reading file '$path': $message"
|
override def errorMessage: String = s"Error reading file '$file': $message"
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
package net.kemitix.thorp.config
|
package net.kemitix.thorp.config
|
||||||
|
|
||||||
import java.nio.file.Paths
|
import java.io.File
|
||||||
|
|
||||||
import net.kemitix.thorp.filesystem.FileSystem
|
import net.kemitix.thorp.filesystem.FileSystem
|
||||||
import zio.ZIO
|
import zio.ZIO
|
||||||
|
@ -11,9 +11,9 @@ import zio.ZIO
|
||||||
*/
|
*/
|
||||||
trait ConfigurationBuilder {
|
trait ConfigurationBuilder {
|
||||||
|
|
||||||
private val userConfigFilename = ".config/thorp.conf"
|
private val userConfigFile = ".config/thorp.conf"
|
||||||
private val globalConfig = Paths.get("/etc/thorp.conf")
|
private val globalConfig = new File("/etc/thorp.conf")
|
||||||
private val userHome = Paths.get(System.getProperty("user.home"))
|
private val userHome = new File(System.getProperty("user.home"))
|
||||||
|
|
||||||
def buildConfig(priorityOpts: ConfigOptions)
|
def buildConfig(priorityOpts: ConfigOptions)
|
||||||
: ZIO[FileSystem, ConfigValidationException, Configuration] =
|
: ZIO[FileSystem, ConfigValidationException, Configuration] =
|
||||||
|
@ -33,7 +33,7 @@ trait ConfigurationBuilder {
|
||||||
|
|
||||||
private def userOptions(priorityOpts: ConfigOptions) =
|
private def userOptions(priorityOpts: ConfigOptions) =
|
||||||
if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
|
if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
|
||||||
else ParseConfigFile.parseFile(userHome.resolve(userConfigFilename))
|
else ParseConfigFile.parseFile(new File(userHome, userConfigFile))
|
||||||
|
|
||||||
private def globalOptions(priorityOpts: ConfigOptions) =
|
private def globalOptions(priorityOpts: ConfigOptions) =
|
||||||
if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig
|
if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
package net.kemitix.thorp.config
|
package net.kemitix.thorp.config
|
||||||
|
|
||||||
import java.nio.file.Path
|
import java.io.File
|
||||||
|
|
||||||
import net.kemitix.thorp.filesystem.FileSystem
|
import net.kemitix.thorp.filesystem.FileSystem
|
||||||
import zio.{IO, TaskR, ZIO}
|
import zio.{IO, TaskR, ZIO}
|
||||||
|
@ -8,19 +8,14 @@ import zio.{IO, TaskR, ZIO}
|
||||||
trait ParseConfigFile {
|
trait ParseConfigFile {
|
||||||
|
|
||||||
def parseFile(
|
def parseFile(
|
||||||
filename: Path): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
|
file: File): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
|
||||||
(readFile(filename) >>= ParseConfigLines.parseLines)
|
(FileSystem.exists(file) >>= readLines(file) >>= ParseConfigLines.parseLines)
|
||||||
.catchAll(
|
.catchAll(h =>
|
||||||
h =>
|
IO.fail(List(ConfigValidation.ErrorReadingFile(file, h.getMessage))))
|
||||||
IO.fail(
|
|
||||||
List(ConfigValidation.ErrorReadingFile(filename, h.getMessage))))
|
|
||||||
|
|
||||||
private def readFile(filename: Path) =
|
private def readLines(file: File)(
|
||||||
FileSystem.exists(filename.toFile) >>= readLines(filename)
|
|
||||||
|
|
||||||
private def readLines(filename: Path)(
|
|
||||||
exists: Boolean): TaskR[FileSystem, Seq[String]] =
|
exists: Boolean): TaskR[FileSystem, Seq[String]] =
|
||||||
if (exists) FileSystem.lines(filename.toFile)
|
if (exists) FileSystem.lines(file)
|
||||||
else ZIO.succeed(Seq.empty)
|
else ZIO.succeed(Seq.empty)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
package net.kemitix.thorp.config
|
package net.kemitix.thorp.config
|
||||||
|
|
||||||
|
import java.io.File
|
||||||
|
|
||||||
import net.kemitix.thorp.domain.Sources
|
import net.kemitix.thorp.domain.Sources
|
||||||
import net.kemitix.thorp.filesystem.FileSystem
|
import net.kemitix.thorp.filesystem.FileSystem
|
||||||
import zio.ZIO
|
import zio.ZIO
|
||||||
|
@ -12,7 +14,7 @@ trait SourceConfigLoader {
|
||||||
sources: Sources): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
|
sources: Sources): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] =
|
||||||
ZIO
|
ZIO
|
||||||
.foreach(sources.paths) { path =>
|
.foreach(sources.paths) { path =>
|
||||||
ParseConfigFile.parseFile(path.resolve(thorpConfigFileName))
|
ParseConfigFile.parseFile(new File(path.toFile, thorpConfigFileName))
|
||||||
}
|
}
|
||||||
.map(_.foldLeft(ConfigOptions(sources.paths.map(ConfigOption.Source))) {
|
.map(_.foldLeft(ConfigOptions(sources.paths.map(ConfigOption.Source))) {
|
||||||
(acc, co) =>
|
(acc, co) =>
|
||||||
|
|
|
@ -1,47 +1,58 @@
|
||||||
package net.kemitix.thorp.config
|
package net.kemitix.thorp.config
|
||||||
|
|
||||||
import java.nio.file.{Path, Paths}
|
import java.io.File
|
||||||
|
import java.nio.file.Paths
|
||||||
|
|
||||||
|
import net.kemitix.thorp.domain.TemporaryFolder
|
||||||
import net.kemitix.thorp.filesystem.FileSystem
|
import net.kemitix.thorp.filesystem.FileSystem
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
import zio.DefaultRuntime
|
import zio.DefaultRuntime
|
||||||
|
|
||||||
class ParseConfigFileTest extends FunSpec {
|
class ParseConfigFileTest extends FunSpec with TemporaryFolder {
|
||||||
|
|
||||||
private val empty = Right(ConfigOptions())
|
private val empty = Right(ConfigOptions())
|
||||||
|
|
||||||
describe("parse a missing file") {
|
describe("parse a missing file") {
|
||||||
val filename = Paths.get("/path/to/missing/file")
|
val file = new File("/path/to/missing/file")
|
||||||
it("should return no options") {
|
it("should return no options") {
|
||||||
assertResult(empty)(invoke(filename))
|
assertResult(empty)(invoke(file))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("parse an empty file") {
|
describe("parse an empty file") {
|
||||||
val filename = Resource(this, "empty-file").toPath
|
|
||||||
it("should return no options") {
|
it("should return no options") {
|
||||||
assertResult(empty)(invoke(filename))
|
withDirectory(dir => {
|
||||||
|
val file = writeFile(dir, "empty-file")
|
||||||
|
assertResult(empty)(invoke(file))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("parse a file with no valid entries") {
|
describe("parse a file with no valid entries") {
|
||||||
val filename = Resource(this, "invalid-config").toPath
|
|
||||||
it("should return no options") {
|
it("should return no options") {
|
||||||
assertResult(empty)(invoke(filename))
|
withDirectory(dir => {
|
||||||
|
val file = writeFile(dir, "invalid-config", "no valid = config items")
|
||||||
|
assertResult(empty)(invoke(file))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("parse a file with properties") {
|
describe("parse a file with properties") {
|
||||||
val filename = Resource(this, "simple-config").toPath
|
|
||||||
val expected = Right(
|
|
||||||
ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source")),
|
|
||||||
ConfigOption.Bucket("bucket-name"))))
|
|
||||||
it("should return some options") {
|
it("should return some options") {
|
||||||
assertResult(expected)(invoke(filename))
|
val expected = Right(
|
||||||
|
ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source")),
|
||||||
|
ConfigOption.Bucket("bucket-name"))))
|
||||||
|
withDirectory(dir => {
|
||||||
|
val file = writeFile(dir,
|
||||||
|
"simple-config",
|
||||||
|
"source = /path/to/source",
|
||||||
|
"bucket = bucket-name")
|
||||||
|
assertResult(expected)(invoke(file))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def invoke(filename: Path) = {
|
private def invoke(file: File) = {
|
||||||
new DefaultRuntime {}.unsafeRunSync {
|
new DefaultRuntime {}.unsafeRunSync {
|
||||||
ParseConfigFile
|
ParseConfigFile
|
||||||
.parseFile(filename)
|
.parseFile(file)
|
||||||
.provide(FileSystem.Live)
|
.provide(FileSystem.Live)
|
||||||
}.toEither
|
}.toEither
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,7 +21,7 @@ object ActionGenerator {
|
||||||
s3MetaData match {
|
s3MetaData match {
|
||||||
// #1 local exists, remote exists, remote matches - do nothing
|
// #1 local exists, remote exists, remote matches - do nothing
|
||||||
case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
|
case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
|
||||||
if localFile.matches(hash) =>
|
if LocalFile.matchesHash(localFile)(hash) =>
|
||||||
doNothing(bucket, key)
|
doNothing(bucket, key)
|
||||||
// #2 local exists, remote is missing, other matches - copy
|
// #2 local exists, remote is missing, other matches - copy
|
||||||
case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
|
case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
|
||||||
|
@ -33,7 +33,7 @@ object ActionGenerator {
|
||||||
uploadFile(bucket, localFile)
|
uploadFile(bucket, localFile)
|
||||||
// #4 local exists, remote exists, remote no match, other matches - copy
|
// #4 local exists, remote exists, remote no match, other matches - copy
|
||||||
case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _)))
|
case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _)))
|
||||||
if !localFile.matches(hash) &&
|
if !LocalFile.matchesHash(localFile)(hash) &&
|
||||||
matchByHash.nonEmpty =>
|
matchByHash.nonEmpty =>
|
||||||
copyFile(bucket, localFile, matchByHash)
|
copyFile(bucket, localFile, matchByHash)
|
||||||
// #5 local exists, remote exists, remote no match, other no matches - upload
|
// #5 local exists, remote exists, remote no match, other no matches - upload
|
||||||
|
|
41
core/src/main/scala/net/kemitix/thorp/core/Filters.scala
Normal file
41
core/src/main/scala/net/kemitix/thorp/core/Filters.scala
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
import net.kemitix.thorp.domain.Filter
|
||||||
|
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
|
||||||
|
|
||||||
|
object Filters {
|
||||||
|
|
||||||
|
def isIncluded(p: Path)(filters: List[Filter]): Boolean = {
|
||||||
|
sealed trait State
|
||||||
|
case class Unknown() extends State
|
||||||
|
case class Accepted() extends State
|
||||||
|
case class Discarded() extends State
|
||||||
|
val excluded = isExcludedByFilter(p)(_)
|
||||||
|
val included = isIncludedByFilter(p)(_)
|
||||||
|
filters.foldRight(Unknown(): State)((filter, state) =>
|
||||||
|
(filter, state) match {
|
||||||
|
case (_, Accepted()) => Accepted()
|
||||||
|
case (_, Discarded()) => Discarded()
|
||||||
|
case (x: Exclude, _) if excluded(x) => Discarded()
|
||||||
|
case (i: Include, _) if included(i) => Accepted()
|
||||||
|
case _ => Unknown()
|
||||||
|
}) match {
|
||||||
|
case Accepted() => true
|
||||||
|
case Discarded() => false
|
||||||
|
case Unknown() =>
|
||||||
|
filters.forall {
|
||||||
|
case _: Include => false
|
||||||
|
case _ => true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def isIncludedByFilter(path: Path)(filter: Filter): Boolean =
|
||||||
|
filter.predicate.test(path.toString)
|
||||||
|
|
||||||
|
def isExcludedByFilter(path: Path)(filter: Filter): Boolean =
|
||||||
|
filter.predicate.test(path.toString)
|
||||||
|
|
||||||
|
}
|
|
@ -3,21 +3,17 @@ package net.kemitix.thorp.core
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.domain.{RemoteKey, Sources}
|
import net.kemitix.thorp.domain.{RemoteKey, Sources}
|
||||||
|
import zio.Task
|
||||||
|
|
||||||
object KeyGenerator {
|
object KeyGenerator {
|
||||||
|
|
||||||
def generateKey(
|
def generateKey(
|
||||||
sources: Sources,
|
sources: Sources,
|
||||||
prefix: RemoteKey
|
prefix: RemoteKey
|
||||||
)(path: Path): RemoteKey = {
|
)(path: Path): Task[RemoteKey] =
|
||||||
val source = sources.forPath(path)
|
Sources
|
||||||
val relativePath = source.relativize(path.toAbsolutePath).toString
|
.forPath(path)(sources)
|
||||||
RemoteKey(
|
.map(p => p.relativize(path.toAbsolutePath).toString)
|
||||||
List(
|
.map(RemoteKey.resolve(_)(prefix))
|
||||||
prefix.key,
|
|
||||||
relativePath
|
|
||||||
).filter(_.nonEmpty)
|
|
||||||
.mkString("/"))
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,9 +4,8 @@ import java.io.File
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.config.Config
|
import net.kemitix.thorp.config.Config
|
||||||
import net.kemitix.thorp.core.KeyGenerator.generateKey
|
|
||||||
import net.kemitix.thorp.core.hasher.Hasher
|
import net.kemitix.thorp.core.hasher.Hasher
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain.Sources
|
||||||
import net.kemitix.thorp.filesystem.FileSystem
|
import net.kemitix.thorp.filesystem.FileSystem
|
||||||
import zio.{Task, TaskR, ZIO}
|
import zio.{Task, TaskR, ZIO}
|
||||||
|
|
||||||
|
@ -48,21 +47,18 @@ object LocalFileStream {
|
||||||
.filter({ case (_, included) => included })
|
.filter({ case (_, included) => included })
|
||||||
.map({ case (path, _) => path })
|
.map({ case (path, _) => path })
|
||||||
|
|
||||||
private def localFile(path: Path) = {
|
private def localFile(path: Path) =
|
||||||
val file = path.toFile
|
|
||||||
for {
|
for {
|
||||||
sources <- Config.sources
|
sources <- Config.sources
|
||||||
prefix <- Config.prefix
|
prefix <- Config.prefix
|
||||||
|
source <- Sources.forPath(path)(sources)
|
||||||
hash <- Hasher.hashObject(path)
|
hash <- Hasher.hashObject(path)
|
||||||
localFile = LocalFile(file,
|
localFile <- LocalFileValidator.validate(path,
|
||||||
sources.forPath(path).toFile,
|
source.toFile,
|
||||||
hash,
|
hash,
|
||||||
generateKey(sources, prefix)(path))
|
sources,
|
||||||
} yield
|
prefix)
|
||||||
LocalFiles(localFiles = Stream(localFile),
|
} yield LocalFiles.one(localFile)
|
||||||
count = 1,
|
|
||||||
totalSizeBytes = file.length)
|
|
||||||
}
|
|
||||||
|
|
||||||
private def listFiles(path: Path) =
|
private def listFiles(path: Path) =
|
||||||
for {
|
for {
|
||||||
|
@ -78,6 +74,6 @@ object LocalFileStream {
|
||||||
private def isIncluded(path: Path) =
|
private def isIncluded(path: Path) =
|
||||||
for {
|
for {
|
||||||
filters <- Config.filters
|
filters <- Config.filters
|
||||||
} yield Filter.isIncluded(path)(filters)
|
} yield Filters.isIncluded(path)(filters)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,66 @@
|
||||||
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
|
import java.io.File
|
||||||
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
import net.kemitix.thorp.domain.{
|
||||||
|
HashType,
|
||||||
|
LocalFile,
|
||||||
|
MD5Hash,
|
||||||
|
RemoteKey,
|
||||||
|
Sources
|
||||||
|
}
|
||||||
|
import zio.{IO, ZIO}
|
||||||
|
|
||||||
|
object LocalFileValidator {
|
||||||
|
|
||||||
|
def validate(
|
||||||
|
path: Path,
|
||||||
|
source: File,
|
||||||
|
hash: Map[HashType, MD5Hash],
|
||||||
|
sources: Sources,
|
||||||
|
prefix: RemoteKey
|
||||||
|
): IO[Violation, LocalFile] =
|
||||||
|
for {
|
||||||
|
vFile <- validateFile(path.toFile)
|
||||||
|
remoteKey <- validateRemoteKey(sources, prefix, path)
|
||||||
|
} yield LocalFile(vFile, source, hash, remoteKey)
|
||||||
|
|
||||||
|
private def validateFile(file: File) =
|
||||||
|
if (file.isDirectory)
|
||||||
|
ZIO.fail(Violation.IsNotAFile(file))
|
||||||
|
else
|
||||||
|
ZIO.succeed(file)
|
||||||
|
|
||||||
|
private def validateRemoteKey(sources: Sources,
|
||||||
|
prefix: RemoteKey,
|
||||||
|
path: Path) =
|
||||||
|
KeyGenerator
|
||||||
|
.generateKey(sources, prefix)(path)
|
||||||
|
.mapError(e => Violation.InvalidRemoteKey(path, e))
|
||||||
|
|
||||||
|
sealed trait Violation extends Throwable {
|
||||||
|
def getMessage: String
|
||||||
|
}
|
||||||
|
object Violation {
|
||||||
|
case class IsNotAFile(file: File) extends Violation {
|
||||||
|
override def getMessage: String = s"Local File must be a file: ${file}"
|
||||||
|
}
|
||||||
|
case class InvalidRemoteKey(path: Path, e: Throwable) extends Violation {
|
||||||
|
override def getMessage: String =
|
||||||
|
s"Remote Key for '${path}' is invalid: ${e.getMessage}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def resolve(
|
||||||
|
path: String,
|
||||||
|
md5Hashes: Map[HashType, MD5Hash],
|
||||||
|
source: Path,
|
||||||
|
sources: Sources,
|
||||||
|
prefix: RemoteKey
|
||||||
|
): IO[Violation, LocalFile] = {
|
||||||
|
val resolvedPath = source.resolve(path)
|
||||||
|
validate(resolvedPath, source.toFile, md5Hashes, sources, prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -13,12 +13,11 @@ case class LocalFiles(
|
||||||
count = count + append.count,
|
count = count + append.count,
|
||||||
totalSizeBytes = totalSizeBytes + append.totalSizeBytes
|
totalSizeBytes = totalSizeBytes + append.totalSizeBytes
|
||||||
)
|
)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
object LocalFiles {
|
object LocalFiles {
|
||||||
|
|
||||||
def reduce: Stream[LocalFiles] => LocalFiles =
|
def reduce: Stream[LocalFiles] => LocalFiles =
|
||||||
list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf)
|
list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf)
|
||||||
|
def one(localFile: LocalFile): LocalFiles =
|
||||||
|
LocalFiles(Stream(localFile), 1, localFile.file.length)
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,7 @@ object Remote {
|
||||||
remoteKey: RemoteKey
|
remoteKey: RemoteKey
|
||||||
): TaskR[FileSystem, Boolean] = {
|
): TaskR[FileSystem, Boolean] = {
|
||||||
def existsInSource(source: Path) =
|
def existsInSource(source: Path) =
|
||||||
remoteKey.asFile(source, prefix) match {
|
RemoteKey.asFile(source, prefix)(remoteKey) match {
|
||||||
case Some(file) => FileSystem.exists(file)
|
case Some(file) => FileSystem.exists(file)
|
||||||
case None => ZIO.succeed(false)
|
case None => ZIO.succeed(false)
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,13 +2,7 @@ package net.kemitix.thorp.core
|
||||||
|
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
import net.kemitix.thorp.config.{
|
import net.kemitix.thorp.config._
|
||||||
Config,
|
|
||||||
ConfigOption,
|
|
||||||
ConfigOptions,
|
|
||||||
ConfigurationBuilder,
|
|
||||||
Resource
|
|
||||||
}
|
|
||||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||||
import net.kemitix.thorp.domain.HashType.MD5
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain._
|
||||||
|
@ -31,8 +25,6 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
ConfigOption.IgnoreUserOptions,
|
ConfigOption.IgnoreUserOptions,
|
||||||
ConfigOption.IgnoreGlobalOptions
|
ConfigOption.IgnoreGlobalOptions
|
||||||
))
|
))
|
||||||
private val fileToKey =
|
|
||||||
KeyGenerator.generateKey(sources, prefix) _
|
|
||||||
|
|
||||||
describe("create actions") {
|
describe("create actions") {
|
||||||
|
|
||||||
|
@ -40,119 +32,156 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
|
|
||||||
describe("#1 local exists, remote exists, remote matches - do nothing") {
|
describe("#1 local exists, remote exists, remote matches - do nothing") {
|
||||||
val theHash = MD5Hash("the-hash")
|
val theHash = MD5Hash("the-hash")
|
||||||
val theFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val theRemoteMetadata =
|
sources,
|
||||||
RemoteMetaData(theFile.remoteKey, theHash, lastModified)
|
prefix)
|
||||||
val input =
|
theRemoteMetadata = RemoteMetaData(theFile.remoteKey,
|
||||||
S3MetaData(theFile, // local exists
|
theHash,
|
||||||
matchByHash = Set(theRemoteMetadata), // remote matches
|
lastModified)
|
||||||
matchByKey = Some(theRemoteMetadata) // remote exists
|
input = S3MetaData(
|
||||||
|
theFile, // local exists
|
||||||
|
matchByHash = Set(theRemoteMetadata), // remote matches
|
||||||
|
matchByKey = Some(theRemoteMetadata) // remote exists
|
||||||
)
|
)
|
||||||
|
} yield (theFile, input)
|
||||||
it("do nothing") {
|
it("do nothing") {
|
||||||
val expected =
|
env.map({
|
||||||
Right(
|
case (theFile, input) => {
|
||||||
Stream(DoNothing(bucket, theFile.remoteKey, theFile.file.length)))
|
val expected =
|
||||||
val result = invoke(input, previousActions)
|
Right(Stream(
|
||||||
assertResult(expected)(result)
|
DoNothing(bucket, theFile.remoteKey, theFile.file.length + 1)))
|
||||||
|
val result = invoke(input, previousActions)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("#2 local exists, remote is missing, other matches - copy") {
|
describe("#2 local exists, remote is missing, other matches - copy") {
|
||||||
val theHash = MD5Hash("the-hash")
|
val theHash = MD5Hash("the-hash")
|
||||||
val theFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val theRemoteKey = theFile.remoteKey
|
sources,
|
||||||
val otherRemoteKey = prefix.resolve("other-key")
|
prefix)
|
||||||
val otherRemoteMetadata =
|
theRemoteKey = theFile.remoteKey
|
||||||
RemoteMetaData(otherRemoteKey, theHash, lastModified)
|
otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
|
||||||
val input =
|
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
||||||
S3MetaData(theFile, // local exists
|
theHash,
|
||||||
matchByHash = Set(otherRemoteMetadata), // other matches
|
lastModified)
|
||||||
matchByKey = None) // remote is missing
|
input = S3MetaData(
|
||||||
|
theFile, // local exists
|
||||||
|
matchByHash = Set(otherRemoteMetadata), // other matches
|
||||||
|
matchByKey = None) // remote is missing
|
||||||
|
} yield (theFile, theRemoteKey, input, otherRemoteKey)
|
||||||
it("copy from other key") {
|
it("copy from other key") {
|
||||||
val expected = Right(
|
env.map({
|
||||||
Stream(
|
case (theFile, theRemoteKey, input, otherRemoteKey) => {
|
||||||
ToCopy(bucket,
|
val expected = Right(
|
||||||
otherRemoteKey,
|
Stream(
|
||||||
theHash,
|
ToCopy(bucket,
|
||||||
theRemoteKey,
|
otherRemoteKey,
|
||||||
theFile.file.length))) // copy
|
theHash,
|
||||||
val result = invoke(input, previousActions)
|
theRemoteKey,
|
||||||
assertResult(expected)(result)
|
theFile.file.length))) // copy
|
||||||
|
val result = invoke(input, previousActions)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
describe("#3 local exists, remote is missing, other no matches - upload") {
|
||||||
describe("#3 local exists, remote is missing, other no matches - upload") {
|
val theHash = MD5Hash("the-hash")
|
||||||
val theHash = MD5Hash("the-hash")
|
val env = for {
|
||||||
val theFile = LocalFile.resolve("the-file",
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
md5HashMap(theHash),
|
md5HashMap(theHash),
|
||||||
sourcePath,
|
sourcePath,
|
||||||
fileToKey)
|
sources,
|
||||||
val input = S3MetaData(theFile, // local exists
|
prefix)
|
||||||
|
input = S3MetaData(theFile, // local exists
|
||||||
matchByHash = Set.empty, // other no matches
|
matchByHash = Set.empty, // other no matches
|
||||||
matchByKey = None) // remote is missing
|
matchByKey = None) // remote is missing
|
||||||
it("upload") {
|
} yield (theFile, input)
|
||||||
val expected = Right(
|
it("upload") {
|
||||||
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
|
env.map({
|
||||||
val result = invoke(input, previousActions)
|
case (theFile, input) => {
|
||||||
assertResult(expected)(result)
|
val expected = Right(Stream(
|
||||||
|
ToUpload(bucket, theFile, theFile.file.length))) // upload
|
||||||
|
val result = invoke(input, previousActions)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe(
|
describe(
|
||||||
"#4 local exists, remote exists, remote no match, other matches - copy") {
|
"#4 local exists, remote exists, remote no match, other matches - copy") {
|
||||||
val theHash = MD5Hash("the-hash")
|
val theHash = MD5Hash("the-hash")
|
||||||
val theFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val theRemoteKey = theFile.remoteKey
|
sources,
|
||||||
val oldHash = MD5Hash("old-hash")
|
prefix)
|
||||||
val otherRemoteKey = prefix.resolve("other-key")
|
theRemoteKey = theFile.remoteKey
|
||||||
val otherRemoteMetadata =
|
oldHash = MD5Hash("old-hash")
|
||||||
RemoteMetaData(otherRemoteKey, theHash, lastModified)
|
otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
|
||||||
val oldRemoteMetadata = RemoteMetaData(theRemoteKey,
|
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
||||||
hash = oldHash, // remote no match
|
theHash,
|
||||||
lastModified = lastModified)
|
lastModified)
|
||||||
val input =
|
oldRemoteMetadata = RemoteMetaData(theRemoteKey,
|
||||||
S3MetaData(theFile, // local exists
|
hash = oldHash, // remote no match
|
||||||
matchByHash = Set(otherRemoteMetadata), // other matches
|
lastModified = lastModified)
|
||||||
matchByKey = Some(oldRemoteMetadata)) // remote exists
|
input = S3MetaData(
|
||||||
|
theFile, // local exists
|
||||||
|
matchByHash = Set(otherRemoteMetadata), // other matches
|
||||||
|
matchByKey = Some(oldRemoteMetadata)) // remote exists
|
||||||
|
} yield (theFile, theRemoteKey, input, otherRemoteKey)
|
||||||
it("copy from other key") {
|
it("copy from other key") {
|
||||||
val expected = Right(
|
env.map({
|
||||||
Stream(
|
case (theFile, theRemoteKey, input, otherRemoteKey) => {
|
||||||
ToCopy(bucket,
|
val expected = Right(
|
||||||
otherRemoteKey,
|
Stream(
|
||||||
theHash,
|
ToCopy(bucket,
|
||||||
theRemoteKey,
|
otherRemoteKey,
|
||||||
theFile.file.length))) // copy
|
theHash,
|
||||||
val result = invoke(input, previousActions)
|
theRemoteKey,
|
||||||
assertResult(expected)(result)
|
theFile.file.length))) // copy
|
||||||
|
val result = invoke(input, previousActions)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe(
|
describe(
|
||||||
"#5 local exists, remote exists, remote no match, other no matches - upload") {
|
"#5 local exists, remote exists, remote no match, other no matches - upload") {
|
||||||
val theHash = MD5Hash("the-hash")
|
val theHash = MD5Hash("the-hash")
|
||||||
val theFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val theRemoteKey = theFile.remoteKey
|
sources,
|
||||||
val oldHash = MD5Hash("old-hash")
|
prefix)
|
||||||
val theRemoteMetadata =
|
theRemoteKey = theFile.remoteKey
|
||||||
RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
oldHash = MD5Hash("old-hash")
|
||||||
val input =
|
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
||||||
S3MetaData(theFile, // local exists
|
input = S3MetaData(
|
||||||
matchByHash = Set.empty, // remote no match, other no match
|
theFile, // local exists
|
||||||
matchByKey = Some(theRemoteMetadata) // remote exists
|
matchByHash = Set.empty, // remote no match, other no match
|
||||||
|
matchByKey = Some(theRemoteMetadata) // remote exists
|
||||||
)
|
)
|
||||||
|
} yield (theFile, input)
|
||||||
it("upload") {
|
it("upload") {
|
||||||
val expected = Right(
|
env.map({
|
||||||
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
|
case (theFile, input) => {
|
||||||
val result = invoke(input, previousActions)
|
val expected = Right(
|
||||||
assertResult(expected)(result)
|
Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload
|
||||||
|
val result = invoke(input, previousActions)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("#6 local missing, remote exists - delete") {
|
describe("#6 local missing, remote exists - delete") {
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
import java.nio.file.Paths
|
import java.nio.file.Paths
|
||||||
|
|
||||||
|
import net.kemitix.thorp.domain.Filter
|
||||||
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
|
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
|
||||||
|
@ -21,31 +22,32 @@ class FiltersSuite extends FunSpec {
|
||||||
describe("default filter") {
|
describe("default filter") {
|
||||||
val include = Include()
|
val include = Include()
|
||||||
it("should include files") {
|
it("should include files") {
|
||||||
paths.foreach(path => assertResult(true)(include.isIncluded(path)))
|
paths.foreach(path =>
|
||||||
|
assertResult(true)(Filters.isIncludedByFilter(path)(include)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("directory exact match include '/upload/subdir/'") {
|
describe("directory exact match include '/upload/subdir/'") {
|
||||||
val include = Include("/upload/subdir/")
|
val include = Include("/upload/subdir/")
|
||||||
it("include matching directory") {
|
it("include matching directory") {
|
||||||
val matching = Paths.get("/upload/subdir/leaf-file")
|
val matching = Paths.get("/upload/subdir/leaf-file")
|
||||||
assertResult(true)(include.isIncluded(matching))
|
assertResult(true)(Filters.isIncludedByFilter(matching)(include))
|
||||||
}
|
}
|
||||||
it("exclude non-matching files") {
|
it("exclude non-matching files") {
|
||||||
val nonMatching = Paths.get("/upload/other-file")
|
val nonMatching = Paths.get("/upload/other-file")
|
||||||
assertResult(false)(include.isIncluded(nonMatching))
|
assertResult(false)(Filters.isIncludedByFilter(nonMatching)(include))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("file partial match 'root'") {
|
describe("file partial match 'root'") {
|
||||||
val include = Include("root")
|
val include = Include("root")
|
||||||
it("include matching file '/upload/root-file") {
|
it("include matching file '/upload/root-file") {
|
||||||
val matching = Paths.get("/upload/root-file")
|
val matching = Paths.get("/upload/root-file")
|
||||||
assertResult(true)(include.isIncluded(matching))
|
assertResult(true)(Filters.isIncludedByFilter(matching)(include))
|
||||||
}
|
}
|
||||||
it("exclude non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
|
it("exclude non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
|
||||||
val nonMatching1 = Paths.get("/test-file-for-hash.txt")
|
val nonMatching1 = Paths.get("/test-file-for-hash.txt")
|
||||||
val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
|
val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
|
||||||
assertResult(false)(include.isIncluded(nonMatching1))
|
assertResult(false)(Filters.isIncludedByFilter(nonMatching1)(include))
|
||||||
assertResult(false)(include.isIncluded(nonMatching2))
|
assertResult(false)(Filters.isIncludedByFilter(nonMatching2)(include))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -63,30 +65,30 @@ class FiltersSuite extends FunSpec {
|
||||||
val exclude = Exclude("/upload/subdir/")
|
val exclude = Exclude("/upload/subdir/")
|
||||||
it("exclude matching directory") {
|
it("exclude matching directory") {
|
||||||
val matching = Paths.get("/upload/subdir/leaf-file")
|
val matching = Paths.get("/upload/subdir/leaf-file")
|
||||||
assertResult(true)(exclude.isExcluded(matching))
|
assertResult(true)(Filters.isExcludedByFilter(matching)(exclude))
|
||||||
}
|
}
|
||||||
it("include non-matching files") {
|
it("include non-matching files") {
|
||||||
val nonMatching = Paths.get("/upload/other-file")
|
val nonMatching = Paths.get("/upload/other-file")
|
||||||
assertResult(false)(exclude.isExcluded(nonMatching))
|
assertResult(false)(Filters.isExcludedByFilter(nonMatching)(exclude))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("file partial match 'root'") {
|
describe("file partial match 'root'") {
|
||||||
val exclude = Exclude("root")
|
val exclude = Exclude("root")
|
||||||
it("exclude matching file '/upload/root-file") {
|
it("exclude matching file '/upload/root-file") {
|
||||||
val matching = Paths.get("/upload/root-file")
|
val matching = Paths.get("/upload/root-file")
|
||||||
assertResult(true)(exclude.isExcluded(matching))
|
assertResult(true)(Filters.isExcludedByFilter(matching)(exclude))
|
||||||
}
|
}
|
||||||
it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
|
it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
|
||||||
val nonMatching1 = Paths.get("/test-file-for-hash.txt")
|
val nonMatching1 = Paths.get("/test-file-for-hash.txt")
|
||||||
val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
|
val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
|
||||||
assertResult(false)(exclude.isExcluded(nonMatching1))
|
assertResult(false)(Filters.isExcludedByFilter(nonMatching1)(exclude))
|
||||||
assertResult(false)(exclude.isExcluded(nonMatching2))
|
assertResult(false)(Filters.isExcludedByFilter(nonMatching2)(exclude))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("isIncluded") {
|
describe("isIncluded") {
|
||||||
def invoke(filters: List[Filter]) = {
|
def invoke(filters: List[Filter]) = {
|
||||||
paths.filter(path => Filter.isIncluded(path)(filters))
|
paths.filter(path => Filters.isIncluded(path)(filters))
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("when there are no filters") {
|
describe("when there are no filters") {
|
|
@ -1,10 +1,12 @@
|
||||||
package net.kemitix.thorp.core
|
package net.kemitix.thorp.core
|
||||||
|
|
||||||
import java.io.File
|
import java.io.File
|
||||||
|
import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.config.Resource
|
import net.kemitix.thorp.config.Resource
|
||||||
import net.kemitix.thorp.domain.{RemoteKey, Sources}
|
import net.kemitix.thorp.domain.{RemoteKey, Sources}
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
import zio.DefaultRuntime
|
||||||
|
|
||||||
class KeyGeneratorSuite extends FunSpec {
|
class KeyGeneratorSuite extends FunSpec {
|
||||||
|
|
||||||
|
@ -12,26 +14,32 @@ class KeyGeneratorSuite extends FunSpec {
|
||||||
private val sourcePath = source.toPath
|
private val sourcePath = source.toPath
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
private val sources = Sources(List(sourcePath))
|
private val sources = Sources(List(sourcePath))
|
||||||
private val fileToKey =
|
|
||||||
KeyGenerator.generateKey(sources, prefix) _
|
|
||||||
|
|
||||||
describe("key generator") {
|
describe("key generator") {
|
||||||
|
|
||||||
describe("when file is within source") {
|
describe("when file is within source") {
|
||||||
it("has a valid key") {
|
it("has a valid key") {
|
||||||
val subdir = "subdir"
|
val subdir = "subdir"
|
||||||
assertResult(RemoteKey(s"${prefix.key}/$subdir"))(
|
val expected = Right(RemoteKey(s"${prefix.key}/$subdir"))
|
||||||
fileToKey(sourcePath.resolve(subdir)))
|
val result = invoke(sourcePath.resolve(subdir))
|
||||||
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("when file is deeper within source") {
|
describe("when file is deeper within source") {
|
||||||
it("has a valid key") {
|
it("has a valid key") {
|
||||||
val subdir = "subdir/deeper/still"
|
val subdir = "subdir/deeper/still"
|
||||||
assertResult(RemoteKey(s"${prefix.key}/$subdir"))(
|
val expected = Right(RemoteKey(s"${prefix.key}/$subdir"))
|
||||||
fileToKey(sourcePath.resolve(subdir)))
|
val result = invoke(sourcePath.resolve(subdir))
|
||||||
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def invoke(path: Path) = {
|
||||||
|
new DefaultRuntime {}.unsafeRunSync {
|
||||||
|
KeyGenerator.generateKey(sources, prefix)(path)
|
||||||
|
}.toEither
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,7 +32,7 @@ class LocalFileStreamSuite extends FunSpec {
|
||||||
val result =
|
val result =
|
||||||
invoke()
|
invoke()
|
||||||
.map(_.localFiles)
|
.map(_.localFiles)
|
||||||
.map(_.map(_.relative.toString))
|
.map(_.map(LocalFile.relativeToSource(_).toString))
|
||||||
.map(_.toSet)
|
.map(_.toSet)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
|
|
|
@ -333,13 +333,17 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
md5Hash: MD5Hash,
|
md5Hash: MD5Hash,
|
||||||
source: Path,
|
source: Path,
|
||||||
file: File): (String, String, String, String, String) =
|
file: File): (String, String, String, String, String) =
|
||||||
("upload", remoteKey.key, md5Hash.hash, source.toString, file.toString)
|
("upload",
|
||||||
|
remoteKey.key,
|
||||||
|
MD5Hash.hash(md5Hash),
|
||||||
|
source.toString,
|
||||||
|
file.toString)
|
||||||
|
|
||||||
private def toCopy(
|
private def toCopy(
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
md5Hash: MD5Hash,
|
md5Hash: MD5Hash,
|
||||||
targetKey: RemoteKey): (String, String, String, String, String) =
|
targetKey: RemoteKey): (String, String, String, String, String) =
|
||||||
("copy", sourceKey.key, md5Hash.hash, targetKey.key, "")
|
("copy", sourceKey.key, MD5Hash.hash(md5Hash), targetKey.key, "")
|
||||||
|
|
||||||
private def toDelete(
|
private def toDelete(
|
||||||
remoteKey: RemoteKey): (String, String, String, String, String) =
|
remoteKey: RemoteKey): (String, String, String, String, String) =
|
||||||
|
@ -386,12 +390,12 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
case ToUpload(_, lf, _) =>
|
case ToUpload(_, lf, _) =>
|
||||||
("upload",
|
("upload",
|
||||||
lf.remoteKey.key,
|
lf.remoteKey.key,
|
||||||
lf.hashes(MD5).hash,
|
MD5Hash.hash(lf.hashes(MD5)),
|
||||||
lf.source.toString,
|
lf.source.toString,
|
||||||
lf.file.toString)
|
lf.file.toString)
|
||||||
case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")
|
case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")
|
||||||
case ToCopy(_, sourceKey, hash, targetKey, _) =>
|
case ToCopy(_, sourceKey, hash, targetKey, _) =>
|
||||||
("copy", sourceKey.key, hash.hash, targetKey.key, "")
|
("copy", sourceKey.key, MD5Hash.hash(hash), targetKey.key, "")
|
||||||
case DoNothing(_, remoteKey, _) =>
|
case DoNothing(_, remoteKey, _) =>
|
||||||
("do-nothing", remoteKey.key, "", "", "")
|
("do-nothing", remoteKey.key, "", "", "")
|
||||||
case x => ("other", x.toString, "", "", "")
|
case x => ("other", x.toString, "", "", "")
|
||||||
|
|
|
@ -14,8 +14,6 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
private val sourcePath = source.toPath
|
private val sourcePath = source.toPath
|
||||||
private val sources = Sources(List(sourcePath))
|
private val sources = Sources(List(sourcePath))
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
private val fileToKey =
|
|
||||||
KeyGenerator.generateKey(sources, prefix) _
|
|
||||||
|
|
||||||
def getMatchesByKey(
|
def getMatchesByKey(
|
||||||
status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
|
status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
|
||||||
|
@ -36,138 +34,180 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
describe(
|
describe(
|
||||||
"#1a local exists, remote exists, remote matches, other matches - do nothing") {
|
"#1a local exists, remote exists, remote matches, other matches - do nothing") {
|
||||||
val theHash: MD5Hash = MD5Hash("the-file-hash")
|
val theHash: MD5Hash = MD5Hash("the-file-hash")
|
||||||
val theFile: LocalFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val theRemoteKey: RemoteKey = theFile.remoteKey
|
sources,
|
||||||
val s3: S3ObjectsData = S3ObjectsData(
|
prefix)
|
||||||
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
|
theRemoteKey = theFile.remoteKey
|
||||||
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
|
s3 = S3ObjectsData(
|
||||||
)
|
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
|
||||||
val theRemoteMetadata =
|
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
|
||||||
RemoteMetaData(theRemoteKey, theHash, lastModified)
|
)
|
||||||
|
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
|
||||||
|
} yield (theFile, theRemoteMetadata, s3)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
val expected = S3MetaData(theFile,
|
env.map({
|
||||||
matchByHash = Set(theRemoteMetadata),
|
case (theFile, theRemoteMetadata, s3) => {
|
||||||
matchByKey = Some(theRemoteMetadata))
|
val expected = S3MetaData(theFile,
|
||||||
val result = getMetadata(theFile, s3)
|
matchByHash = Set(theRemoteMetadata),
|
||||||
assertResult(expected)(result)
|
matchByKey = Some(theRemoteMetadata))
|
||||||
|
val result = getMetadata(theFile, s3)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe(
|
describe(
|
||||||
"#1b local exists, remote exists, remote matches, other no matches - do nothing") {
|
"#1b local exists, remote exists, remote matches, other no matches - do nothing") {
|
||||||
val theHash: MD5Hash = MD5Hash("the-file-hash")
|
val theHash: MD5Hash = MD5Hash("the-file-hash")
|
||||||
val theFile: LocalFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val theRemoteKey: RemoteKey = prefix.resolve("the-file")
|
sources,
|
||||||
val s3: S3ObjectsData = S3ObjectsData(
|
prefix)
|
||||||
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
|
theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix)
|
||||||
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
|
s3: S3ObjectsData = S3ObjectsData(
|
||||||
)
|
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
|
||||||
val theRemoteMetadata =
|
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
|
||||||
RemoteMetaData(theRemoteKey, theHash, lastModified)
|
)
|
||||||
|
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
|
||||||
|
} yield (theFile, theRemoteMetadata, s3)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
val expected = S3MetaData(theFile,
|
env.map({
|
||||||
matchByHash = Set(theRemoteMetadata),
|
case (theFile, theRemoteMetadata, s3) => {
|
||||||
matchByKey = Some(theRemoteMetadata))
|
val expected = S3MetaData(theFile,
|
||||||
val result = getMetadata(theFile, s3)
|
matchByHash = Set(theRemoteMetadata),
|
||||||
assertResult(expected)(result)
|
matchByKey = Some(theRemoteMetadata))
|
||||||
|
val result = getMetadata(theFile, s3)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe(
|
describe(
|
||||||
"#2 local exists, remote is missing, remote no match, other matches - copy") {
|
"#2 local exists, remote is missing, remote no match, other matches - copy") {
|
||||||
val theHash = MD5Hash("the-hash")
|
val theHash = MD5Hash("the-hash")
|
||||||
val theFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val otherRemoteKey = RemoteKey("other-key")
|
sources,
|
||||||
val s3: S3ObjectsData = S3ObjectsData(
|
prefix)
|
||||||
byHash = Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
|
otherRemoteKey = RemoteKey("other-key")
|
||||||
byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
|
s3: S3ObjectsData = S3ObjectsData(
|
||||||
)
|
byHash =
|
||||||
val otherRemoteMetadata =
|
Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
|
||||||
RemoteMetaData(otherRemoteKey, theHash, lastModified)
|
byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
|
||||||
|
)
|
||||||
|
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
||||||
|
theHash,
|
||||||
|
lastModified)
|
||||||
|
} yield (theFile, otherRemoteMetadata, s3)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
val expected = S3MetaData(theFile,
|
env.map({
|
||||||
matchByHash = Set(otherRemoteMetadata),
|
case (theFile, otherRemoteMetadata, s3) => {
|
||||||
matchByKey = None)
|
val expected = S3MetaData(theFile,
|
||||||
val result = getMetadata(theFile, s3)
|
matchByHash = Set(otherRemoteMetadata),
|
||||||
assertResult(expected)(result)
|
matchByKey = None)
|
||||||
|
val result = getMetadata(theFile, s3)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe(
|
describe(
|
||||||
"#3 local exists, remote is missing, remote no match, other no matches - upload") {
|
"#3 local exists, remote is missing, remote no match, other no matches - upload") {
|
||||||
val theHash = MD5Hash("the-hash")
|
val theHash = MD5Hash("the-hash")
|
||||||
val theFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val s3: S3ObjectsData = S3ObjectsData()
|
sources,
|
||||||
|
prefix)
|
||||||
|
s3: S3ObjectsData = S3ObjectsData()
|
||||||
|
} yield (theFile, s3)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
val expected =
|
env.map({
|
||||||
S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None)
|
case (theFile, s3) => {
|
||||||
val result = getMetadata(theFile, s3)
|
val expected =
|
||||||
assertResult(expected)(result)
|
S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None)
|
||||||
|
val result = getMetadata(theFile, s3)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe(
|
describe(
|
||||||
"#4 local exists, remote exists, remote no match, other matches - copy") {
|
"#4 local exists, remote exists, remote no match, other matches - copy") {
|
||||||
val theHash = MD5Hash("the-hash")
|
val theHash = MD5Hash("the-hash")
|
||||||
val theFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val theRemoteKey = theFile.remoteKey
|
sources,
|
||||||
val oldHash = MD5Hash("old-hash")
|
prefix)
|
||||||
val otherRemoteKey = prefix.resolve("other-key")
|
theRemoteKey = theFile.remoteKey
|
||||||
val s3: S3ObjectsData = S3ObjectsData(
|
oldHash = MD5Hash("old-hash")
|
||||||
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
|
otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
|
||||||
theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
|
s3: S3ObjectsData = S3ObjectsData(
|
||||||
byKey = Map(
|
byHash =
|
||||||
theRemoteKey -> HashModified(oldHash, lastModified),
|
Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
|
||||||
otherRemoteKey -> HashModified(theHash, lastModified)
|
theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
|
||||||
|
byKey = Map(
|
||||||
|
theRemoteKey -> HashModified(oldHash, lastModified),
|
||||||
|
otherRemoteKey -> HashModified(theHash, lastModified)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
||||||
val theRemoteMetadata =
|
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
||||||
RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
theHash,
|
||||||
val otherRemoteMetadata =
|
lastModified)
|
||||||
RemoteMetaData(otherRemoteKey, theHash, lastModified)
|
} yield (theFile, theRemoteMetadata, otherRemoteMetadata, s3)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
val expected = S3MetaData(theFile,
|
env.map({
|
||||||
matchByHash = Set(otherRemoteMetadata),
|
case (theFile, theRemoteMetadata, otherRemoteMetadata, s3) => {
|
||||||
matchByKey = Some(theRemoteMetadata))
|
val expected = S3MetaData(theFile,
|
||||||
val result = getMetadata(theFile, s3)
|
matchByHash = Set(otherRemoteMetadata),
|
||||||
assertResult(expected)(result)
|
matchByKey = Some(theRemoteMetadata))
|
||||||
|
val result = getMetadata(theFile, s3)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe(
|
describe(
|
||||||
"#5 local exists, remote exists, remote no match, other no matches - upload") {
|
"#5 local exists, remote exists, remote no match, other no matches - upload") {
|
||||||
val theHash = MD5Hash("the-hash")
|
val theHash = MD5Hash("the-hash")
|
||||||
val theFile = LocalFile.resolve("the-file",
|
val env = for {
|
||||||
md5HashMap(theHash),
|
theFile <- LocalFileValidator.resolve("the-file",
|
||||||
sourcePath,
|
md5HashMap(theHash),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
val theRemoteKey = theFile.remoteKey
|
sources,
|
||||||
val oldHash = MD5Hash("old-hash")
|
prefix)
|
||||||
val s3: S3ObjectsData = S3ObjectsData(
|
theRemoteKey = theFile.remoteKey
|
||||||
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
|
oldHash = MD5Hash("old-hash")
|
||||||
theHash -> Set.empty),
|
s3: S3ObjectsData = S3ObjectsData(
|
||||||
byKey = Map(
|
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
|
||||||
theRemoteKey -> HashModified(oldHash, lastModified)
|
theHash -> Set.empty),
|
||||||
|
byKey = Map(
|
||||||
|
theRemoteKey -> HashModified(oldHash, lastModified)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
||||||
val theRemoteMetadata =
|
} yield (theFile, theRemoteMetadata, s3)
|
||||||
RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
val expected = S3MetaData(theFile,
|
env.map({
|
||||||
matchByHash = Set.empty,
|
case (theFile, theRemoteMetadata, s3) => {
|
||||||
matchByKey = Some(theRemoteMetadata))
|
val expected = S3MetaData(theFile,
|
||||||
val result = getMetadata(theFile, s3)
|
matchByHash = Set.empty,
|
||||||
assertResult(expected)(result)
|
matchByKey = Some(theRemoteMetadata))
|
||||||
|
val result = getMetadata(theFile, s3)
|
||||||
|
assertResult(expected)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -178,71 +218,103 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
|
|
||||||
describe("getS3Status") {
|
describe("getS3Status") {
|
||||||
val hash = MD5Hash("hash")
|
val hash = MD5Hash("hash")
|
||||||
val localFile =
|
val env = for {
|
||||||
LocalFile.resolve("the-file", md5HashMap(hash), sourcePath, fileToKey)
|
localFile <- LocalFileValidator.resolve("the-file",
|
||||||
val key = localFile.remoteKey
|
md5HashMap(hash),
|
||||||
val keyOtherKey = LocalFile.resolve("other-key-same-hash",
|
sourcePath,
|
||||||
md5HashMap(hash),
|
sources,
|
||||||
sourcePath,
|
prefix)
|
||||||
fileToKey)
|
key = localFile.remoteKey
|
||||||
val diffHash = MD5Hash("diff")
|
keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash",
|
||||||
val keyDiffHash = LocalFile.resolve("other-key-diff-hash",
|
md5HashMap(hash),
|
||||||
md5HashMap(diffHash),
|
sourcePath,
|
||||||
sourcePath,
|
sources,
|
||||||
fileToKey)
|
prefix)
|
||||||
val lastModified = LastModified(Instant.now)
|
diffHash = MD5Hash("diff")
|
||||||
val s3ObjectsData: S3ObjectsData = S3ObjectsData(
|
keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash",
|
||||||
byHash = Map(
|
md5HashMap(diffHash),
|
||||||
hash -> Set(KeyModified(key, lastModified),
|
sourcePath,
|
||||||
KeyModified(keyOtherKey.remoteKey, lastModified)),
|
sources,
|
||||||
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
|
prefix)
|
||||||
),
|
lastModified = LastModified(Instant.now)
|
||||||
byKey = Map(
|
s3ObjectsData = S3ObjectsData(
|
||||||
key -> HashModified(hash, lastModified),
|
byHash = Map(
|
||||||
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
|
hash -> Set(KeyModified(key, lastModified),
|
||||||
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
|
KeyModified(keyOtherKey.remoteKey, lastModified)),
|
||||||
|
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
|
||||||
|
),
|
||||||
|
byKey = Map(
|
||||||
|
key -> HashModified(hash, lastModified),
|
||||||
|
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
|
||||||
|
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
} yield (s3ObjectsData, localFile, keyDiffHash, diffHash)
|
||||||
|
|
||||||
def invoke(localFile: LocalFile) = {
|
def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) = {
|
||||||
getS3Status(localFile, s3ObjectsData)
|
getS3Status(localFile, s3ObjectsData)
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("when remote key exists") {
|
describe("when remote key exists") {
|
||||||
it("should return a result for matching key") {
|
it("should return a result for matching key") {
|
||||||
val result = getMatchesByKey(invoke(localFile))
|
env.map({
|
||||||
assert(result.contains(HashModified(hash, lastModified)))
|
case (s3ObjectsData, localFile: LocalFile, _, _) =>
|
||||||
|
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
|
||||||
|
assert(result.contains(HashModified(hash, lastModified)))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("when remote key does not exist and no others matches hash") {
|
describe("when remote key does not exist and no others matches hash") {
|
||||||
val localFile = LocalFile.resolve("missing-file",
|
val env2 = for {
|
||||||
md5HashMap(MD5Hash("unique")),
|
localFile <- LocalFileValidator.resolve("missing-remote",
|
||||||
sourcePath,
|
md5HashMap(MD5Hash("unique")),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
|
sources,
|
||||||
|
prefix)
|
||||||
|
} yield (localFile)
|
||||||
it("should return no matches by key") {
|
it("should return no matches by key") {
|
||||||
val result = getMatchesByKey(invoke(localFile))
|
env.map({
|
||||||
assert(result.isEmpty)
|
case (s3ObjectsData, _, _, _) => {
|
||||||
|
env2.map({
|
||||||
|
case (localFile) => {
|
||||||
|
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
|
||||||
|
assert(result.isEmpty)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
it("should return no matches by hash") {
|
it("should return no matches by hash") {
|
||||||
val result = getMatchesByHash(invoke(localFile))
|
env.map({
|
||||||
assert(result.isEmpty)
|
case (s3ObjectsData, _, _, _) => {
|
||||||
|
env2.map({
|
||||||
|
case (localFile) => {
|
||||||
|
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
|
||||||
|
assert(result.isEmpty)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("when remote key exists and no others match hash") {
|
describe("when remote key exists and no others match hash") {
|
||||||
it("should return match by key") {
|
env.map({
|
||||||
val result = getMatchesByKey(invoke(keyDiffHash))
|
case (s3ObjectsData, _, keyDiffHash, diffHash) => {
|
||||||
assert(result.contains(HashModified(diffHash, lastModified)))
|
it("should return match by key") {
|
||||||
}
|
val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData))
|
||||||
it("should return only itself in match by hash") {
|
assert(result.contains(HashModified(diffHash, lastModified)))
|
||||||
val result = getMatchesByHash(invoke(keyDiffHash))
|
}
|
||||||
assert(
|
it("should return only itself in match by hash") {
|
||||||
result.equals(
|
val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData))
|
||||||
Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
|
assert(
|
||||||
}
|
result.equals(Set(
|
||||||
|
(diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ package net.kemitix.thorp.core.hasher
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.config.Resource
|
import net.kemitix.thorp.config.Resource
|
||||||
|
import net.kemitix.thorp.domain.MD5Hash
|
||||||
import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root}
|
import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root}
|
||||||
import net.kemitix.thorp.filesystem.FileSystem
|
import net.kemitix.thorp.filesystem.FileSystem
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
@ -42,14 +43,14 @@ class MD5HashGeneratorTest extends FunSpec {
|
||||||
val path = Resource(this, "../big-file").toPath
|
val path = Resource(this, "../big-file").toPath
|
||||||
it("should generate the correct hash for first chunk of the file") {
|
it("should generate the correct hash for first chunk of the file") {
|
||||||
val part1 = BigFile.Part1
|
val part1 = BigFile.Part1
|
||||||
val expected = Right(part1.hash.hash)
|
val expected = Right(MD5Hash.hash(part1.hash))
|
||||||
val result = invoke(path, part1.offset, part1.size).map(_.hash)
|
val result = invoke(path, part1.offset, part1.size).map(MD5Hash.hash)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
it("should generate the correct hash for second chunk of the file") {
|
it("should generate the correct hash for second chunk of the file") {
|
||||||
val part2 = BigFile.Part2
|
val part2 = BigFile.Part2
|
||||||
val expected = Right(part2.hash.hash)
|
val expected = Right(MD5Hash.hash(part2.hash))
|
||||||
val result = invoke(path, part2.offset, part2.size).map(_.hash)
|
val result = invoke(path, part2.offset, part2.size).map(MD5Hash.hash)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,53 +1,18 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
import java.nio.file.Path
|
import java.util.function.Predicate
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
sealed trait Filter
|
sealed trait Filter {
|
||||||
|
def predicate: Predicate[String]
|
||||||
|
}
|
||||||
|
|
||||||
object Filter {
|
object Filter {
|
||||||
|
case class Include(include: String = ".*") extends Filter {
|
||||||
def isIncluded(p: Path)(filters: List[Filter]): Boolean = {
|
lazy val predicate: Predicate[String] = Pattern.compile(include).asPredicate
|
||||||
sealed trait State
|
|
||||||
case class Unknown() extends State
|
|
||||||
case class Accepted() extends State
|
|
||||||
case class Discarded() extends State
|
|
||||||
filters.foldRight(Unknown(): State)((filter, state) =>
|
|
||||||
(filter, state) match {
|
|
||||||
case (_, Accepted()) => Accepted()
|
|
||||||
case (_, Discarded()) => Discarded()
|
|
||||||
case (x: Exclude, _) if x.isExcluded(p) => Discarded()
|
|
||||||
case (i: Include, _) if i.isIncluded(p) => Accepted()
|
|
||||||
case _ => Unknown()
|
|
||||||
}) match {
|
|
||||||
case Accepted() => true
|
|
||||||
case Discarded() => false
|
|
||||||
case Unknown() =>
|
|
||||||
filters.forall {
|
|
||||||
case _: Include => false
|
|
||||||
case _ => true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
case class Exclude(exclude: String) extends Filter {
|
||||||
case class Include(
|
lazy val predicate: Predicate[String] =
|
||||||
include: String = ".*"
|
Pattern.compile(exclude).asPredicate()
|
||||||
) extends Filter {
|
|
||||||
|
|
||||||
private lazy val predicate = Pattern.compile(include).asPredicate
|
|
||||||
|
|
||||||
def isIncluded(path: Path): Boolean = predicate.test(path.toString)
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
case class Exclude(
|
|
||||||
exclude: String
|
|
||||||
) extends Filter {
|
|
||||||
|
|
||||||
private lazy val predicate = Pattern.compile(exclude).asPredicate()
|
|
||||||
|
|
||||||
def isExcluded(path: Path): Boolean = predicate.test(path.toString)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,42 +5,22 @@ import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.domain.HashType.MD5
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
|
|
||||||
final case class LocalFile(
|
final case class LocalFile private (
|
||||||
file: File,
|
file: File,
|
||||||
source: File,
|
source: File,
|
||||||
hashes: Map[HashType, MD5Hash],
|
hashes: Map[HashType, MD5Hash],
|
||||||
remoteKey: RemoteKey
|
remoteKey: RemoteKey
|
||||||
) {
|
)
|
||||||
|
|
||||||
require(!file.isDirectory, s"LocalFile must not be a directory: $file")
|
|
||||||
|
|
||||||
def isDirectory: Boolean = file.isDirectory
|
|
||||||
|
|
||||||
// the path of the file within the source
|
|
||||||
def relative: Path = source.toPath.relativize(file.toPath)
|
|
||||||
|
|
||||||
def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _)
|
|
||||||
|
|
||||||
def md5base64: Option[String] = hashes.get(MD5).map(_.hash64)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
object LocalFile {
|
object LocalFile {
|
||||||
|
|
||||||
def resolve(
|
|
||||||
path: String,
|
|
||||||
md5Hashes: Map[HashType, MD5Hash],
|
|
||||||
source: Path,
|
|
||||||
pathToKey: Path => RemoteKey
|
|
||||||
): LocalFile = {
|
|
||||||
val resolvedPath = source.resolve(path)
|
|
||||||
LocalFile(resolvedPath.toFile,
|
|
||||||
source.toFile,
|
|
||||||
md5Hashes,
|
|
||||||
pathToKey(resolvedPath))
|
|
||||||
}
|
|
||||||
|
|
||||||
val remoteKey: SimpleLens[LocalFile, RemoteKey] =
|
val remoteKey: SimpleLens[LocalFile, RemoteKey] =
|
||||||
SimpleLens[LocalFile, RemoteKey](_.remoteKey,
|
SimpleLens[LocalFile, RemoteKey](_.remoteKey,
|
||||||
b => a => b.copy(remoteKey = a))
|
b => a => b.copy(remoteKey = a))
|
||||||
|
// the path of the file within the source
|
||||||
|
def relativeToSource(localFile: LocalFile): Path =
|
||||||
|
localFile.source.toPath.relativize(localFile.file.toPath)
|
||||||
|
def matchesHash(localFile: LocalFile)(other: MD5Hash): Boolean =
|
||||||
|
localFile.hashes.values.exists(other equals _)
|
||||||
|
def md5base64(localFile: LocalFile): Option[String] =
|
||||||
|
localFile.hashes.get(MD5).map(MD5Hash.hash64)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,15 +0,0 @@
|
||||||
package net.kemitix.thorp.domain
|
|
||||||
|
|
||||||
trait Logger {
|
|
||||||
|
|
||||||
// returns an instance of Logger with debug set as indicated
|
|
||||||
// where the current Logger already matches this state, then
|
|
||||||
// it returns itself, unmodified
|
|
||||||
def withDebug(debug: Boolean): Logger
|
|
||||||
|
|
||||||
def debug(message: => String): Unit
|
|
||||||
def info(message: => String): Unit
|
|
||||||
def warn(message: String): Unit
|
|
||||||
def error(message: String): Unit
|
|
||||||
|
|
||||||
}
|
|
|
@ -4,20 +4,14 @@ import java.util.Base64
|
||||||
|
|
||||||
import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
|
import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
|
||||||
|
|
||||||
final case class MD5Hash(
|
final case class MD5Hash(in: String)
|
||||||
in: String
|
|
||||||
) {
|
|
||||||
|
|
||||||
lazy val hash: String = in filter stripQuotes
|
|
||||||
|
|
||||||
lazy val digest: Array[Byte] = HexEncoder.decode(hash)
|
|
||||||
|
|
||||||
lazy val hash64: String = Base64.getEncoder.encodeToString(digest)
|
|
||||||
}
|
|
||||||
|
|
||||||
object MD5Hash {
|
object MD5Hash {
|
||||||
def fromDigest(digest: Array[Byte]): MD5Hash = {
|
def fromDigest(digest: Array[Byte]): MD5Hash =
|
||||||
val hexDigest = (digest map ("%02x" format _)).mkString
|
MD5Hash((digest map ("%02x" format _)).mkString)
|
||||||
MD5Hash(hexDigest)
|
def hash(md5Hash: MD5Hash): String = md5Hash.in.filter(stripQuotes)
|
||||||
}
|
def digest(md5Hash: MD5Hash): Array[Byte] =
|
||||||
|
HexEncoder.decode(MD5Hash.hash(md5Hash))
|
||||||
|
def hash64(md5Hash: MD5Hash): String =
|
||||||
|
Base64.getEncoder.encodeToString(MD5Hash.digest(md5Hash))
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,31 +3,21 @@ package net.kemitix.thorp.domain
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.nio.file.{Path, Paths}
|
import java.nio.file.{Path, Paths}
|
||||||
|
|
||||||
final case class RemoteKey(
|
final case class RemoteKey(key: String)
|
||||||
key: String
|
|
||||||
) {
|
|
||||||
|
|
||||||
def asFile(
|
|
||||||
source: Path,
|
|
||||||
prefix: RemoteKey
|
|
||||||
): Option[File] =
|
|
||||||
if (key.length == 0) None
|
|
||||||
else Some(source.resolve(relativeTo(prefix)).toFile)
|
|
||||||
|
|
||||||
private def relativeTo(prefix: RemoteKey) = {
|
|
||||||
prefix match {
|
|
||||||
case RemoteKey("") => Paths.get(key)
|
|
||||||
case _ => Paths.get(prefix.key).relativize(Paths.get(key))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def resolve(path: String): RemoteKey =
|
|
||||||
RemoteKey(List(key, path).filterNot(_.isEmpty).mkString("/"))
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
object RemoteKey {
|
object RemoteKey {
|
||||||
val key: SimpleLens[RemoteKey, String] =
|
val key: SimpleLens[RemoteKey, String] =
|
||||||
SimpleLens[RemoteKey, String](_.key, b => a => b.copy(key = a))
|
SimpleLens[RemoteKey, String](_.key, b => a => b.copy(key = a))
|
||||||
|
def asFile(source: Path, prefix: RemoteKey)(
|
||||||
|
remoteKey: RemoteKey): Option[File] =
|
||||||
|
if (remoteKey.key.length == 0) None
|
||||||
|
else Some(source.resolve(RemoteKey.relativeTo(prefix)(remoteKey)).toFile)
|
||||||
|
def relativeTo(prefix: RemoteKey)(remoteKey: RemoteKey): Path = {
|
||||||
|
prefix match {
|
||||||
|
case RemoteKey("") => Paths.get(remoteKey.key)
|
||||||
|
case _ => Paths.get(prefix.key).relativize(Paths.get(remoteKey.key))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def resolve(path: String)(remoteKey: RemoteKey): RemoteKey =
|
||||||
|
RemoteKey(List(remoteKey.key, path).filterNot(_.isEmpty).mkString("/"))
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,8 @@ package net.kemitix.thorp.domain
|
||||||
|
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
import zio.{Task, ZIO}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The paths to synchronise with target.
|
* The paths to synchronise with target.
|
||||||
*
|
*
|
||||||
|
@ -18,18 +20,10 @@ case class Sources(
|
||||||
def +(path: Path)(implicit m: Monoid[Sources]): Sources = this ++ List(path)
|
def +(path: Path)(implicit m: Monoid[Sources]): Sources = this ++ List(path)
|
||||||
def ++(otherPaths: List[Path])(implicit m: Monoid[Sources]): Sources =
|
def ++(otherPaths: List[Path])(implicit m: Monoid[Sources]): Sources =
|
||||||
m.op(this, Sources(otherPaths))
|
m.op(this, Sources(otherPaths))
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns the source path for the given path.
|
|
||||||
*/
|
|
||||||
def forPath(path: Path): Path =
|
|
||||||
paths.find(source => path.startsWith(source)).get
|
|
||||||
}
|
}
|
||||||
|
|
||||||
object Sources {
|
object Sources {
|
||||||
|
|
||||||
final val emptySources = Sources(List.empty)
|
final val emptySources = Sources(List.empty)
|
||||||
|
|
||||||
implicit def sourcesAppendMonoid: Monoid[Sources] = new Monoid[Sources] {
|
implicit def sourcesAppendMonoid: Monoid[Sources] = new Monoid[Sources] {
|
||||||
override def zero: Sources = emptySources
|
override def zero: Sources = emptySources
|
||||||
override def op(t1: Sources, t2: Sources): Sources =
|
override def op(t1: Sources, t2: Sources): Sources =
|
||||||
|
@ -38,4 +32,12 @@ object Sources {
|
||||||
else acc ++ List(path)
|
else acc ++ List(path)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the source path for the given path.
|
||||||
|
*/
|
||||||
|
def forPath(path: Path)(sources: Sources): Task[Path] =
|
||||||
|
ZIO
|
||||||
|
.fromOption(sources.paths.find(s => path.startsWith(s)))
|
||||||
|
.mapError(_ => new Exception("Path is not within any known source"))
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,10 +1,7 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
import net.kemitix.thorp.domain.UploadEvent.RequestEvent
|
import net.kemitix.thorp.domain.UploadEvent.RequestEvent
|
||||||
import net.kemitix.thorp.domain.UploadEventLogger.{
|
import net.kemitix.thorp.domain.UploadEventLogger.RequestCycle
|
||||||
RequestCycle,
|
|
||||||
logRequestCycle
|
|
||||||
}
|
|
||||||
|
|
||||||
object UploadEventListener {
|
object UploadEventListener {
|
||||||
|
|
||||||
|
@ -21,7 +18,7 @@ object UploadEventListener {
|
||||||
uploadEvent match {
|
uploadEvent match {
|
||||||
case e: RequestEvent =>
|
case e: RequestEvent =>
|
||||||
bytesTransferred += e.transferred
|
bytesTransferred += e.transferred
|
||||||
logRequestCycle(
|
UploadEventLogger(
|
||||||
RequestCycle(settings.localFile,
|
RequestCycle(settings.localFile,
|
||||||
bytesTransferred,
|
bytesTransferred,
|
||||||
settings.index,
|
settings.index,
|
||||||
|
|
|
@ -14,7 +14,7 @@ object UploadEventLogger {
|
||||||
totalBytesSoFar: Long
|
totalBytesSoFar: Long
|
||||||
)
|
)
|
||||||
|
|
||||||
def logRequestCycle(requestCycle: RequestCycle): Unit = {
|
def apply(requestCycle: RequestCycle): Unit = {
|
||||||
val remoteKey = requestCycle.localFile.remoteKey.key
|
val remoteKey = requestCycle.localFile.remoteKey.key
|
||||||
val fileLength = requestCycle.localFile.file.length
|
val fileLength = requestCycle.localFile.file.length
|
||||||
val statusHeight = 7
|
val statusHeight = 7
|
||||||
|
|
|
@ -7,11 +7,11 @@ class MD5HashTest extends FunSpec {
|
||||||
describe("recover base64 hash") {
|
describe("recover base64 hash") {
|
||||||
it("should recover base 64 #1") {
|
it("should recover base 64 #1") {
|
||||||
val rootHash = MD5HashData.Root.hash
|
val rootHash = MD5HashData.Root.hash
|
||||||
assertResult(MD5HashData.Root.base64)(rootHash.hash64)
|
assertResult(MD5HashData.Root.base64)(MD5Hash.hash64(rootHash))
|
||||||
}
|
}
|
||||||
it("should recover base 64 #2") {
|
it("should recover base 64 #2") {
|
||||||
val leafHash = MD5HashData.Leaf.hash
|
val leafHash = MD5HashData.Leaf.hash
|
||||||
assertResult(MD5HashData.Leaf.base64)(leafHash.hash64)
|
assertResult(MD5HashData.Leaf.base64)(MD5Hash.hash64(leafHash))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,21 +15,21 @@ class RemoteKeyTest extends FreeSpec {
|
||||||
val key = emptyKey
|
val key = emptyKey
|
||||||
val path = "path"
|
val path = "path"
|
||||||
val expected = RemoteKey("path")
|
val expected = RemoteKey("path")
|
||||||
val result = key.resolve(path)
|
val result = RemoteKey.resolve(path)(key)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
"when path is empty" in {
|
"when path is empty" in {
|
||||||
val key = RemoteKey("key")
|
val key = RemoteKey("key")
|
||||||
val path = ""
|
val path = ""
|
||||||
val expected = RemoteKey("key")
|
val expected = RemoteKey("key")
|
||||||
val result = key.resolve(path)
|
val result = RemoteKey.resolve(path)(key)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
"when key and path are empty" in {
|
"when key and path are empty" in {
|
||||||
val key = emptyKey
|
val key = emptyKey
|
||||||
val path = ""
|
val path = ""
|
||||||
val expected = emptyKey
|
val expected = emptyKey
|
||||||
val result = key.resolve(path)
|
val result = RemoteKey.resolve(path)(key)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -39,7 +39,7 @@ class RemoteKeyTest extends FreeSpec {
|
||||||
val source = Paths.get("source")
|
val source = Paths.get("source")
|
||||||
val prefix = RemoteKey("prefix")
|
val prefix = RemoteKey("prefix")
|
||||||
val expected = Some(new File("source/key"))
|
val expected = Some(new File("source/key"))
|
||||||
val result = key.asFile(source, prefix)
|
val result = RemoteKey.asFile(source, prefix)(key)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
"when prefix is empty" in {
|
"when prefix is empty" in {
|
||||||
|
@ -47,7 +47,7 @@ class RemoteKeyTest extends FreeSpec {
|
||||||
val source = Paths.get("source")
|
val source = Paths.get("source")
|
||||||
val prefix = emptyKey
|
val prefix = emptyKey
|
||||||
val expected = Some(new File("source/key"))
|
val expected = Some(new File("source/key"))
|
||||||
val result = key.asFile(source, prefix)
|
val result = RemoteKey.asFile(source, prefix)(key)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
"when key is empty" in {
|
"when key is empty" in {
|
||||||
|
@ -55,7 +55,7 @@ class RemoteKeyTest extends FreeSpec {
|
||||||
val source = Paths.get("source")
|
val source = Paths.get("source")
|
||||||
val prefix = RemoteKey("prefix")
|
val prefix = RemoteKey("prefix")
|
||||||
val expected = None
|
val expected = None
|
||||||
val result = key.asFile(source, prefix)
|
val result = RemoteKey.asFile(source, prefix)(key)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
"when key and prefix are empty" in {
|
"when key and prefix are empty" in {
|
||||||
|
@ -63,7 +63,7 @@ class RemoteKeyTest extends FreeSpec {
|
||||||
val source = Paths.get("source")
|
val source = Paths.get("source")
|
||||||
val prefix = emptyKey
|
val prefix = emptyKey
|
||||||
val expected = None
|
val expected = None
|
||||||
val result = key.asFile(source, prefix)
|
val result = RemoteKey.asFile(source, prefix)(key)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,11 +38,13 @@ trait TemporaryFolder {
|
||||||
path.resolve(name).toFile
|
path.resolve(name).toFile
|
||||||
}
|
}
|
||||||
|
|
||||||
def writeFile(directory: Path, name: String, contents: String*): Unit = {
|
def writeFile(directory: Path, name: String, contents: String*): File = {
|
||||||
directory.toFile.mkdirs
|
directory.toFile.mkdirs
|
||||||
val pw = new PrintWriter(directory.resolve(name).toFile, "UTF-8")
|
val file = directory.resolve(name).toFile
|
||||||
contents.foreach(pw.println)
|
val writer = new PrintWriter(file, "UTF-8")
|
||||||
pw.close()
|
contents.foreach(writer.println)
|
||||||
|
writer.close()
|
||||||
|
file
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,7 +41,7 @@ trait Copier {
|
||||||
copyRequest.sourceKey.key,
|
copyRequest.sourceKey.key,
|
||||||
copyRequest.bucket.name,
|
copyRequest.bucket.name,
|
||||||
copyRequest.targetKey.key
|
copyRequest.targetKey.key
|
||||||
).withMatchingETagConstraint(copyRequest.hash.hash)
|
).withMatchingETagConstraint(MD5Hash.hash(copyRequest.hash))
|
||||||
|
|
||||||
private def foldFailure(
|
private def foldFailure(
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
|
|
|
@ -68,7 +68,7 @@ trait Uploader {
|
||||||
|
|
||||||
private def metadata: LocalFile => ObjectMetadata = localFile => {
|
private def metadata: LocalFile => ObjectMetadata = localFile => {
|
||||||
val metadata = new ObjectMetadata()
|
val metadata = new ObjectMetadata()
|
||||||
localFile.md5base64.foreach(metadata.setContentMD5)
|
LocalFile.md5base64(localFile).foreach(metadata.setContentMD5)
|
||||||
metadata
|
metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
|
||||||
import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils
|
import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils
|
||||||
import net.kemitix.thorp.core.hasher.Hasher
|
import net.kemitix.thorp.core.hasher.Hasher
|
||||||
import net.kemitix.thorp.domain.HashType.MD5
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
|
import net.kemitix.thorp.domain.MD5Hash
|
||||||
import net.kemitix.thorp.filesystem.FileSystem
|
import net.kemitix.thorp.filesystem.FileSystem
|
||||||
import zio.{TaskR, ZIO}
|
import zio.{TaskR, ZIO}
|
||||||
|
|
||||||
|
@ -56,7 +57,7 @@ private trait ETagGenerator {
|
||||||
Hasher
|
Hasher
|
||||||
.hashObjectChunk(path, chunkNumber, chunkSize)
|
.hashObjectChunk(path, chunkNumber, chunkSize)
|
||||||
.map(_(MD5))
|
.map(_(MD5))
|
||||||
.map(_.digest)
|
.map(MD5Hash.digest)
|
||||||
|
|
||||||
def offsets(
|
def offsets(
|
||||||
totalFileSizeBytes: Long,
|
totalFileSizeBytes: Long,
|
||||||
|
|
|
@ -32,7 +32,7 @@ class S3ObjectsByHashSuite extends FunSpec {
|
||||||
remoteKey: RemoteKey,
|
remoteKey: RemoteKey,
|
||||||
lastModified: LastModified): S3ObjectSummary = {
|
lastModified: LastModified): S3ObjectSummary = {
|
||||||
val summary = new S3ObjectSummary()
|
val summary = new S3ObjectSummary()
|
||||||
summary.setETag(md5Hash.hash)
|
summary.setETag(MD5Hash.hash(md5Hash))
|
||||||
summary.setKey(remoteKey.key)
|
summary.setKey(remoteKey.key)
|
||||||
summary.setLastModified(Date.from(lastModified.when))
|
summary.setLastModified(Date.from(lastModified.when))
|
||||||
summary
|
summary
|
||||||
|
|
|
@ -3,7 +3,7 @@ package net.kemitix.thorp.storage.aws
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
import net.kemitix.thorp.config.Resource
|
import net.kemitix.thorp.config.Resource
|
||||||
import net.kemitix.thorp.core.{KeyGenerator, S3MetaDataEnricher}
|
import net.kemitix.thorp.core.{LocalFileValidator, S3MetaDataEnricher}
|
||||||
import net.kemitix.thorp.domain.HashType.MD5
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain._
|
||||||
import org.scalamock.scalatest.MockFactory
|
import org.scalamock.scalatest.MockFactory
|
||||||
|
@ -15,39 +15,51 @@ class StorageServiceSuite extends FunSpec with MockFactory {
|
||||||
private val sourcePath = source.toPath
|
private val sourcePath = source.toPath
|
||||||
private val sources = Sources(List(sourcePath))
|
private val sources = Sources(List(sourcePath))
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
private val fileToKey =
|
|
||||||
KeyGenerator.generateKey(sources, prefix) _
|
|
||||||
|
|
||||||
describe("getS3Status") {
|
describe("getS3Status") {
|
||||||
|
|
||||||
val hash = MD5Hash("hash")
|
val hash = MD5Hash("hash")
|
||||||
val localFile =
|
val env = for {
|
||||||
LocalFile.resolve("the-file", Map(MD5 -> hash), sourcePath, fileToKey)
|
localFile <- LocalFileValidator.resolve("the-file",
|
||||||
val key = localFile.remoteKey
|
Map(MD5 -> hash),
|
||||||
val keyOtherKey = LocalFile.resolve("other-key-same-hash",
|
sourcePath,
|
||||||
Map(MD5 -> hash),
|
sources,
|
||||||
sourcePath,
|
prefix)
|
||||||
fileToKey)
|
key = localFile.remoteKey
|
||||||
val diffHash = MD5Hash("diff")
|
keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash",
|
||||||
val keyDiffHash = LocalFile.resolve("other-key-diff-hash",
|
Map(MD5 -> hash),
|
||||||
Map(MD5 -> diffHash),
|
sourcePath,
|
||||||
sourcePath,
|
sources,
|
||||||
fileToKey)
|
prefix)
|
||||||
val lastModified = LastModified(Instant.now)
|
diffHash = MD5Hash("diff")
|
||||||
val s3ObjectsData: S3ObjectsData = S3ObjectsData(
|
keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash",
|
||||||
byHash = Map(
|
Map(MD5 -> diffHash),
|
||||||
hash -> Set(KeyModified(key, lastModified),
|
sourcePath,
|
||||||
KeyModified(keyOtherKey.remoteKey, lastModified)),
|
sources,
|
||||||
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
|
prefix)
|
||||||
),
|
lastModified = LastModified(Instant.now)
|
||||||
byKey = Map(
|
s3ObjectsData = S3ObjectsData(
|
||||||
key -> HashModified(hash, lastModified),
|
byHash = Map(
|
||||||
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
|
hash -> Set(KeyModified(key, lastModified),
|
||||||
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
|
KeyModified(keyOtherKey.remoteKey, lastModified)),
|
||||||
|
diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
|
||||||
|
),
|
||||||
|
byKey = Map(
|
||||||
|
key -> HashModified(hash, lastModified),
|
||||||
|
keyOtherKey.remoteKey -> HashModified(hash, lastModified),
|
||||||
|
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
} yield
|
||||||
|
(s3ObjectsData,
|
||||||
|
localFile: LocalFile,
|
||||||
|
lastModified,
|
||||||
|
keyOtherKey,
|
||||||
|
keyDiffHash,
|
||||||
|
diffHash,
|
||||||
|
key)
|
||||||
|
|
||||||
def invoke(localFile: LocalFile) =
|
def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) =
|
||||||
S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData)
|
S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData)
|
||||||
|
|
||||||
def getMatchesByKey(
|
def getMatchesByKey(
|
||||||
|
@ -67,47 +79,94 @@ class StorageServiceSuite extends FunSpec with MockFactory {
|
||||||
describe(
|
describe(
|
||||||
"when remote key exists, unmodified and other key matches the hash") {
|
"when remote key exists, unmodified and other key matches the hash") {
|
||||||
it("should return the match by key") {
|
it("should return the match by key") {
|
||||||
val result = getMatchesByKey(invoke(localFile))
|
env.map({
|
||||||
assert(result.contains(HashModified(hash, lastModified)))
|
case (s3ObjectsData, localFile, lastModified, _, _, _, _) => {
|
||||||
|
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
|
||||||
|
assert(result.contains(HashModified(hash, lastModified)))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
it("should return both matches for the hash") {
|
it("should return both matches for the hash") {
|
||||||
val result = getMatchesByHash(invoke(localFile))
|
env.map({
|
||||||
assertResult(
|
case (s3ObjectsData,
|
||||||
Set((hash, KeyModified(key, lastModified)),
|
localFile,
|
||||||
(hash, KeyModified(keyOtherKey.remoteKey, lastModified)))
|
lastModified,
|
||||||
)(result)
|
keyOtherKey,
|
||||||
|
_,
|
||||||
|
_,
|
||||||
|
key) => {
|
||||||
|
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
|
||||||
|
assertResult(
|
||||||
|
Set((hash, KeyModified(key, lastModified)),
|
||||||
|
(hash, KeyModified(keyOtherKey.remoteKey, lastModified)))
|
||||||
|
)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("when remote key does not exist and no others matches hash") {
|
describe("when remote key does not exist and no others matches hash") {
|
||||||
val localFile = LocalFile.resolve("missing-file",
|
val env2 = LocalFileValidator
|
||||||
Map(MD5 -> MD5Hash("unique")),
|
.resolve("missing-file",
|
||||||
sourcePath,
|
Map(MD5 -> MD5Hash("unique")),
|
||||||
fileToKey)
|
sourcePath,
|
||||||
|
sources,
|
||||||
|
prefix)
|
||||||
it("should return no matches by key") {
|
it("should return no matches by key") {
|
||||||
val result = getMatchesByKey(invoke(localFile))
|
env2.map(localFile => {
|
||||||
assert(result.isEmpty)
|
env.map({
|
||||||
|
case (s3ObjectsData, _, _, _, _, _, _) => {
|
||||||
|
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
|
||||||
|
assert(result.isEmpty)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
}
|
}
|
||||||
it("should return no matches by hash") {
|
it("should return no matches by hash") {
|
||||||
val result = getMatchesByHash(invoke(localFile))
|
env2.map(localFile => {
|
||||||
assert(result.isEmpty)
|
env.map({
|
||||||
|
case (s3ObjectsData, _, _, _, _, _, _) => {
|
||||||
|
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
|
||||||
|
assert(result.isEmpty)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("when remote key exists and no others match hash") {
|
describe("when remote key exists and no others match hash") {
|
||||||
val localFile = keyDiffHash
|
|
||||||
it("should return the match by key") {
|
it("should return the match by key") {
|
||||||
val result = getMatchesByKey(invoke(localFile))
|
env.map({
|
||||||
assert(result.contains(HashModified(diffHash, lastModified)))
|
case (s3ObjectsData,
|
||||||
|
_,
|
||||||
|
lastModified,
|
||||||
|
_,
|
||||||
|
keyDiffHash,
|
||||||
|
diffHash,
|
||||||
|
_) => {
|
||||||
|
val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData))
|
||||||
|
assert(result.contains(HashModified(diffHash, lastModified)))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
it("should return one match by hash") {
|
it("should return one match by hash") {
|
||||||
val result = getMatchesByHash(invoke(localFile))
|
env.map({
|
||||||
assertResult(
|
case (s3ObjectsData,
|
||||||
Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))
|
_,
|
||||||
)(result)
|
lastModified,
|
||||||
|
_,
|
||||||
|
keyDiffHash,
|
||||||
|
diffHash,
|
||||||
|
_) => {
|
||||||
|
val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData))
|
||||||
|
assertResult(
|
||||||
|
Set(
|
||||||
|
(diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))
|
||||||
|
)(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,7 +29,7 @@ class UploaderTest extends FreeSpec with MockFactory {
|
||||||
val bucket = Bucket("aBucket")
|
val bucket = Bucket("aBucket")
|
||||||
val uploadResult = new UploadResult
|
val uploadResult = new UploadResult
|
||||||
uploadResult.setKey(remoteKey.key)
|
uploadResult.setKey(remoteKey.key)
|
||||||
uploadResult.setETag(aHash.hash)
|
uploadResult.setETag(MD5Hash.hash(aHash))
|
||||||
val inProgress = new AmazonUpload.InProgress {
|
val inProgress = new AmazonUpload.InProgress {
|
||||||
override def waitForUploadResult: UploadResult = uploadResult
|
override def waitForUploadResult: UploadResult = uploadResult
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
|
||||||
import net.kemitix.thorp.config.Resource
|
import net.kemitix.thorp.config.Resource
|
||||||
import net.kemitix.thorp.core.hasher.Hasher
|
import net.kemitix.thorp.core.hasher.Hasher
|
||||||
import net.kemitix.thorp.domain.HashType.MD5
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
|
import net.kemitix.thorp.domain.MD5Hash
|
||||||
import net.kemitix.thorp.filesystem.FileSystem
|
import net.kemitix.thorp.filesystem.FileSystem
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
import zio.DefaultRuntime
|
import zio.DefaultRuntime
|
||||||
|
@ -45,7 +46,7 @@ class ETagGeneratorTest extends FunSpec {
|
||||||
assertResult(Right(hash))(
|
assertResult(Right(hash))(
|
||||||
invoke(bigFilePath, index, chunkSize)
|
invoke(bigFilePath, index, chunkSize)
|
||||||
.map(_(MD5))
|
.map(_(MD5))
|
||||||
.map(_.hash))
|
.map(MD5Hash.hash))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
def invoke(path: Path, index: Long, size: Long) =
|
def invoke(path: Path, index: Long, size: Long) =
|
||||||
|
|
Loading…
Reference in a new issue