From 8fad680a961bf532d30aff9e28c989f10d919694 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Mon, 5 Aug 2019 08:38:44 +0100 Subject: [PATCH] case classes shouldn't be OO objects (#147) * [core] Extract Filters from domain.Filter * [core] extract LocalFileValidator * [domain] LocalFile remove unused isDirectory * [domain] LocalFile move/rename relative as relativeToSource on companion * [domain] LocalFile move and rename matches as matchesHash on companion * [domain] LocalFile move md5base64 to companion * [domain] Logger remove * [domain] MD5Hash move hash to companion * [domain] MD5Hash move digest to companion * [domain] MD5Hash move hash64 to companion * [domain] RemoteKey move class methods to companion * [domain] Sources move forPath to companion Led to being able to cleanup LocalFileStream.localFile and adding LocalFiles.one * [domain] UploadEventLogger rename method as apply --- build.sbt | 1 + .../thorp/config/ConfigValidation.scala | 6 +- .../thorp/config/ConfigurationBuilder.scala | 10 +- .../thorp/config/ParseConfigFile.scala | 19 +- .../thorp/config/SourceConfigLoader.scala | 4 +- .../thorp/config/ParseConfigFileTest.scala | 41 +- .../kemitix/thorp/core/ActionGenerator.scala | 4 +- .../net/kemitix/thorp/core/Filters.scala | 41 ++ .../net/kemitix/thorp/core/KeyGenerator.scala | 16 +- .../kemitix/thorp/core/LocalFileStream.scala | 24 +- .../thorp/core/LocalFileValidator.scala | 66 ++++ .../net/kemitix/thorp/core/LocalFiles.scala | 5 +- .../scala/net/kemitix/thorp/core/Remote.scala | 2 +- .../thorp/core/ActionGeneratorSuite.scala | 227 ++++++----- .../kemitix/thorp/core}/FiltersSuite.scala | 28 +- .../thorp/core/KeyGeneratorSuite.scala | 24 +- .../thorp/core/LocalFileStreamSuite.scala | 2 +- .../kemitix/thorp/core/PlanBuilderTest.scala | 12 +- .../thorp/core/S3MetaDataEnricherSuite.scala | 366 +++++++++++------- .../core/hasher/MD5HashGeneratorTest.scala | 9 +- .../net/kemitix/thorp/domain/Filter.scala | 53 +-- .../net/kemitix/thorp/domain/LocalFile.scala | 38 +- .../net/kemitix/thorp/domain/Logger.scala | 15 - .../net/kemitix/thorp/domain/MD5Hash.scala | 22 +- .../net/kemitix/thorp/domain/RemoteKey.scala | 36 +- .../net/kemitix/thorp/domain/Sources.scala | 18 +- .../thorp/domain/UploadEventListener.scala | 7 +- .../thorp/domain/UploadEventLogger.scala | 2 +- .../kemitix/thorp/domain/MD5HashTest.scala | 4 +- .../kemitix/thorp/domain/RemoteKeyTest.scala | 14 +- .../thorp/domain/TemporaryFolder.scala | 10 +- .../kemitix/thorp/storage/aws/Copier.scala | 2 +- .../kemitix/thorp/storage/aws/Uploader.scala | 2 +- .../storage/aws/hasher/ETagGenerator.scala | 3 +- .../storage/aws/S3ObjectsByHashSuite.scala | 2 +- .../storage/aws/StorageServiceSuite.scala | 161 +++++--- .../thorp/storage/aws/UploaderTest.scala | 2 +- .../aws/hasher/ETagGeneratorTest.scala | 3 +- 38 files changed, 750 insertions(+), 551 deletions(-) create mode 100644 core/src/main/scala/net/kemitix/thorp/core/Filters.scala create mode 100644 core/src/main/scala/net/kemitix/thorp/core/LocalFileValidator.scala rename {domain/src/test/scala/net/kemitix/thorp/domain => core/src/test/scala/net/kemitix/thorp/core}/FiltersSuite.scala (79%) delete mode 100644 domain/src/main/scala/net/kemitix/thorp/domain/Logger.scala diff --git a/build.sbt b/build.sbt index f7eddbd..6f46ceb 100644 --- a/build.sbt +++ b/build.sbt @@ -135,3 +135,4 @@ lazy val domain = (project in file("domain")) .settings(commonSettings) .settings(assemblyJarName in assembly := "domain.jar") .settings(testDependencies) + .settings(zioDependencies) diff --git a/config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala b/config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala index 23ab79c..1d0d769 100644 --- a/config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ConfigValidation.scala @@ -1,6 +1,6 @@ package net.kemitix.thorp.config -import java.nio.file.Path +import java.io.File sealed trait ConfigValidation { @@ -22,10 +22,10 @@ object ConfigValidation { } case class ErrorReadingFile( - path: Path, + file: File, message: String ) extends ConfigValidation { - override def errorMessage: String = s"Error reading file '$path': $message" + override def errorMessage: String = s"Error reading file '$file': $message" } } diff --git a/config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala b/config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala index 3b4fd07..b046269 100644 --- a/config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ConfigurationBuilder.scala @@ -1,6 +1,6 @@ package net.kemitix.thorp.config -import java.nio.file.Paths +import java.io.File import net.kemitix.thorp.filesystem.FileSystem import zio.ZIO @@ -11,9 +11,9 @@ import zio.ZIO */ trait ConfigurationBuilder { - private val userConfigFilename = ".config/thorp.conf" - private val globalConfig = Paths.get("/etc/thorp.conf") - private val userHome = Paths.get(System.getProperty("user.home")) + private val userConfigFile = ".config/thorp.conf" + private val globalConfig = new File("/etc/thorp.conf") + private val userHome = new File(System.getProperty("user.home")) def buildConfig(priorityOpts: ConfigOptions) : ZIO[FileSystem, ConfigValidationException, Configuration] = @@ -33,7 +33,7 @@ trait ConfigurationBuilder { private def userOptions(priorityOpts: ConfigOptions) = if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig - else ParseConfigFile.parseFile(userHome.resolve(userConfigFilename)) + else ParseConfigFile.parseFile(new File(userHome, userConfigFile)) private def globalOptions(priorityOpts: ConfigOptions) = if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig diff --git a/config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala b/config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala index 142ee52..2f68094 100644 --- a/config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/ParseConfigFile.scala @@ -1,6 +1,6 @@ package net.kemitix.thorp.config -import java.nio.file.Path +import java.io.File import net.kemitix.thorp.filesystem.FileSystem import zio.{IO, TaskR, ZIO} @@ -8,19 +8,14 @@ import zio.{IO, TaskR, ZIO} trait ParseConfigFile { def parseFile( - filename: Path): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] = - (readFile(filename) >>= ParseConfigLines.parseLines) - .catchAll( - h => - IO.fail( - List(ConfigValidation.ErrorReadingFile(filename, h.getMessage)))) + file: File): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] = + (FileSystem.exists(file) >>= readLines(file) >>= ParseConfigLines.parseLines) + .catchAll(h => + IO.fail(List(ConfigValidation.ErrorReadingFile(file, h.getMessage)))) - private def readFile(filename: Path) = - FileSystem.exists(filename.toFile) >>= readLines(filename) - - private def readLines(filename: Path)( + private def readLines(file: File)( exists: Boolean): TaskR[FileSystem, Seq[String]] = - if (exists) FileSystem.lines(filename.toFile) + if (exists) FileSystem.lines(file) else ZIO.succeed(Seq.empty) } diff --git a/config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala b/config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala index ae941b1..cc5daa8 100644 --- a/config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala +++ b/config/src/main/scala/net/kemitix/thorp/config/SourceConfigLoader.scala @@ -1,5 +1,7 @@ package net.kemitix.thorp.config +import java.io.File + import net.kemitix.thorp.domain.Sources import net.kemitix.thorp.filesystem.FileSystem import zio.ZIO @@ -12,7 +14,7 @@ trait SourceConfigLoader { sources: Sources): ZIO[FileSystem, Seq[ConfigValidation], ConfigOptions] = ZIO .foreach(sources.paths) { path => - ParseConfigFile.parseFile(path.resolve(thorpConfigFileName)) + ParseConfigFile.parseFile(new File(path.toFile, thorpConfigFileName)) } .map(_.foldLeft(ConfigOptions(sources.paths.map(ConfigOption.Source))) { (acc, co) => diff --git a/config/src/test/scala/net/kemitix/thorp/config/ParseConfigFileTest.scala b/config/src/test/scala/net/kemitix/thorp/config/ParseConfigFileTest.scala index c3b62a6..6d38968 100644 --- a/config/src/test/scala/net/kemitix/thorp/config/ParseConfigFileTest.scala +++ b/config/src/test/scala/net/kemitix/thorp/config/ParseConfigFileTest.scala @@ -1,47 +1,58 @@ package net.kemitix.thorp.config -import java.nio.file.{Path, Paths} +import java.io.File +import java.nio.file.Paths +import net.kemitix.thorp.domain.TemporaryFolder import net.kemitix.thorp.filesystem.FileSystem import org.scalatest.FunSpec import zio.DefaultRuntime -class ParseConfigFileTest extends FunSpec { +class ParseConfigFileTest extends FunSpec with TemporaryFolder { private val empty = Right(ConfigOptions()) describe("parse a missing file") { - val filename = Paths.get("/path/to/missing/file") + val file = new File("/path/to/missing/file") it("should return no options") { - assertResult(empty)(invoke(filename)) + assertResult(empty)(invoke(file)) } } describe("parse an empty file") { - val filename = Resource(this, "empty-file").toPath it("should return no options") { - assertResult(empty)(invoke(filename)) + withDirectory(dir => { + val file = writeFile(dir, "empty-file") + assertResult(empty)(invoke(file)) + }) } } describe("parse a file with no valid entries") { - val filename = Resource(this, "invalid-config").toPath it("should return no options") { - assertResult(empty)(invoke(filename)) + withDirectory(dir => { + val file = writeFile(dir, "invalid-config", "no valid = config items") + assertResult(empty)(invoke(file)) + }) } } describe("parse a file with properties") { - val filename = Resource(this, "simple-config").toPath - val expected = Right( - ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source")), - ConfigOption.Bucket("bucket-name")))) it("should return some options") { - assertResult(expected)(invoke(filename)) + val expected = Right( + ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source")), + ConfigOption.Bucket("bucket-name")))) + withDirectory(dir => { + val file = writeFile(dir, + "simple-config", + "source = /path/to/source", + "bucket = bucket-name") + assertResult(expected)(invoke(file)) + }) } } - private def invoke(filename: Path) = { + private def invoke(file: File) = { new DefaultRuntime {}.unsafeRunSync { ParseConfigFile - .parseFile(filename) + .parseFile(file) .provide(FileSystem.Live) }.toEither } diff --git a/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala b/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala index 05a0108..4bc68e4 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala @@ -21,7 +21,7 @@ object ActionGenerator { s3MetaData match { // #1 local exists, remote exists, remote matches - do nothing case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _))) - if localFile.matches(hash) => + if LocalFile.matchesHash(localFile)(hash) => doNothing(bucket, key) // #2 local exists, remote is missing, other matches - copy case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty => @@ -33,7 +33,7 @@ object ActionGenerator { uploadFile(bucket, localFile) // #4 local exists, remote exists, remote no match, other matches - copy case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _))) - if !localFile.matches(hash) && + if !LocalFile.matchesHash(localFile)(hash) && matchByHash.nonEmpty => copyFile(bucket, localFile, matchByHash) // #5 local exists, remote exists, remote no match, other no matches - upload diff --git a/core/src/main/scala/net/kemitix/thorp/core/Filters.scala b/core/src/main/scala/net/kemitix/thorp/core/Filters.scala new file mode 100644 index 0000000..6b21b0f --- /dev/null +++ b/core/src/main/scala/net/kemitix/thorp/core/Filters.scala @@ -0,0 +1,41 @@ +package net.kemitix.thorp.core + +import java.nio.file.Path + +import net.kemitix.thorp.domain.Filter +import net.kemitix.thorp.domain.Filter.{Exclude, Include} + +object Filters { + + def isIncluded(p: Path)(filters: List[Filter]): Boolean = { + sealed trait State + case class Unknown() extends State + case class Accepted() extends State + case class Discarded() extends State + val excluded = isExcludedByFilter(p)(_) + val included = isIncludedByFilter(p)(_) + filters.foldRight(Unknown(): State)((filter, state) => + (filter, state) match { + case (_, Accepted()) => Accepted() + case (_, Discarded()) => Discarded() + case (x: Exclude, _) if excluded(x) => Discarded() + case (i: Include, _) if included(i) => Accepted() + case _ => Unknown() + }) match { + case Accepted() => true + case Discarded() => false + case Unknown() => + filters.forall { + case _: Include => false + case _ => true + } + } + } + + def isIncludedByFilter(path: Path)(filter: Filter): Boolean = + filter.predicate.test(path.toString) + + def isExcludedByFilter(path: Path)(filter: Filter): Boolean = + filter.predicate.test(path.toString) + +} diff --git a/core/src/main/scala/net/kemitix/thorp/core/KeyGenerator.scala b/core/src/main/scala/net/kemitix/thorp/core/KeyGenerator.scala index fb8711f..04bfaf5 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/KeyGenerator.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/KeyGenerator.scala @@ -3,21 +3,17 @@ package net.kemitix.thorp.core import java.nio.file.Path import net.kemitix.thorp.domain.{RemoteKey, Sources} +import zio.Task object KeyGenerator { def generateKey( sources: Sources, prefix: RemoteKey - )(path: Path): RemoteKey = { - val source = sources.forPath(path) - val relativePath = source.relativize(path.toAbsolutePath).toString - RemoteKey( - List( - prefix.key, - relativePath - ).filter(_.nonEmpty) - .mkString("/")) - } + )(path: Path): Task[RemoteKey] = + Sources + .forPath(path)(sources) + .map(p => p.relativize(path.toAbsolutePath).toString) + .map(RemoteKey.resolve(_)(prefix)) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala b/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala index c1972bf..a7e4603 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala @@ -4,9 +4,8 @@ import java.io.File import java.nio.file.Path import net.kemitix.thorp.config.Config -import net.kemitix.thorp.core.KeyGenerator.generateKey import net.kemitix.thorp.core.hasher.Hasher -import net.kemitix.thorp.domain._ +import net.kemitix.thorp.domain.Sources import net.kemitix.thorp.filesystem.FileSystem import zio.{Task, TaskR, ZIO} @@ -48,21 +47,18 @@ object LocalFileStream { .filter({ case (_, included) => included }) .map({ case (path, _) => path }) - private def localFile(path: Path) = { - val file = path.toFile + private def localFile(path: Path) = for { sources <- Config.sources prefix <- Config.prefix + source <- Sources.forPath(path)(sources) hash <- Hasher.hashObject(path) - localFile = LocalFile(file, - sources.forPath(path).toFile, - hash, - generateKey(sources, prefix)(path)) - } yield - LocalFiles(localFiles = Stream(localFile), - count = 1, - totalSizeBytes = file.length) - } + localFile <- LocalFileValidator.validate(path, + source.toFile, + hash, + sources, + prefix) + } yield LocalFiles.one(localFile) private def listFiles(path: Path) = for { @@ -78,6 +74,6 @@ object LocalFileStream { private def isIncluded(path: Path) = for { filters <- Config.filters - } yield Filter.isIncluded(path)(filters) + } yield Filters.isIncluded(path)(filters) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/LocalFileValidator.scala b/core/src/main/scala/net/kemitix/thorp/core/LocalFileValidator.scala new file mode 100644 index 0000000..b1f5592 --- /dev/null +++ b/core/src/main/scala/net/kemitix/thorp/core/LocalFileValidator.scala @@ -0,0 +1,66 @@ +package net.kemitix.thorp.core + +import java.io.File +import java.nio.file.Path + +import net.kemitix.thorp.domain.{ + HashType, + LocalFile, + MD5Hash, + RemoteKey, + Sources +} +import zio.{IO, ZIO} + +object LocalFileValidator { + + def validate( + path: Path, + source: File, + hash: Map[HashType, MD5Hash], + sources: Sources, + prefix: RemoteKey + ): IO[Violation, LocalFile] = + for { + vFile <- validateFile(path.toFile) + remoteKey <- validateRemoteKey(sources, prefix, path) + } yield LocalFile(vFile, source, hash, remoteKey) + + private def validateFile(file: File) = + if (file.isDirectory) + ZIO.fail(Violation.IsNotAFile(file)) + else + ZIO.succeed(file) + + private def validateRemoteKey(sources: Sources, + prefix: RemoteKey, + path: Path) = + KeyGenerator + .generateKey(sources, prefix)(path) + .mapError(e => Violation.InvalidRemoteKey(path, e)) + + sealed trait Violation extends Throwable { + def getMessage: String + } + object Violation { + case class IsNotAFile(file: File) extends Violation { + override def getMessage: String = s"Local File must be a file: ${file}" + } + case class InvalidRemoteKey(path: Path, e: Throwable) extends Violation { + override def getMessage: String = + s"Remote Key for '${path}' is invalid: ${e.getMessage}" + } + } + + def resolve( + path: String, + md5Hashes: Map[HashType, MD5Hash], + source: Path, + sources: Sources, + prefix: RemoteKey + ): IO[Violation, LocalFile] = { + val resolvedPath = source.resolve(path) + validate(resolvedPath, source.toFile, md5Hashes, sources, prefix) + } + +} diff --git a/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala b/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala index c70f0f7..2693abe 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala @@ -13,12 +13,11 @@ case class LocalFiles( count = count + append.count, totalSizeBytes = totalSizeBytes + append.totalSizeBytes ) - } object LocalFiles { - def reduce: Stream[LocalFiles] => LocalFiles = list => list.foldLeft(LocalFiles())((acc, lf) => acc ++ lf) - + def one(localFile: LocalFile): LocalFiles = + LocalFiles(Stream(localFile), 1, localFile.file.length) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/Remote.scala b/core/src/main/scala/net/kemitix/thorp/core/Remote.scala index e9902d6..d9a2b2a 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/Remote.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/Remote.scala @@ -18,7 +18,7 @@ object Remote { remoteKey: RemoteKey ): TaskR[FileSystem, Boolean] = { def existsInSource(source: Path) = - remoteKey.asFile(source, prefix) match { + RemoteKey.asFile(source, prefix)(remoteKey) match { case Some(file) => FileSystem.exists(file) case None => ZIO.succeed(false) } diff --git a/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala index 4e97e2d..d7fc107 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala @@ -2,13 +2,7 @@ package net.kemitix.thorp.core import java.time.Instant -import net.kemitix.thorp.config.{ - Config, - ConfigOption, - ConfigOptions, - ConfigurationBuilder, - Resource -} +import net.kemitix.thorp.config._ import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload} import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain._ @@ -31,8 +25,6 @@ class ActionGeneratorSuite extends FunSpec { ConfigOption.IgnoreUserOptions, ConfigOption.IgnoreGlobalOptions )) - private val fileToKey = - KeyGenerator.generateKey(sources, prefix) _ describe("create actions") { @@ -40,119 +32,156 @@ class ActionGeneratorSuite extends FunSpec { describe("#1 local exists, remote exists, remote matches - do nothing") { val theHash = MD5Hash("the-hash") - val theFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val theRemoteMetadata = - RemoteMetaData(theFile.remoteKey, theHash, lastModified) - val input = - S3MetaData(theFile, // local exists - matchByHash = Set(theRemoteMetadata), // remote matches - matchByKey = Some(theRemoteMetadata) // remote exists + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + theRemoteMetadata = RemoteMetaData(theFile.remoteKey, + theHash, + lastModified) + input = S3MetaData( + theFile, // local exists + matchByHash = Set(theRemoteMetadata), // remote matches + matchByKey = Some(theRemoteMetadata) // remote exists ) + } yield (theFile, input) it("do nothing") { - val expected = - Right( - Stream(DoNothing(bucket, theFile.remoteKey, theFile.file.length))) - val result = invoke(input, previousActions) - assertResult(expected)(result) + env.map({ + case (theFile, input) => { + val expected = + Right(Stream( + DoNothing(bucket, theFile.remoteKey, theFile.file.length + 1))) + val result = invoke(input, previousActions) + assertResult(expected)(result) + } + }) } } describe("#2 local exists, remote is missing, other matches - copy") { val theHash = MD5Hash("the-hash") - val theFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val theRemoteKey = theFile.remoteKey - val otherRemoteKey = prefix.resolve("other-key") - val otherRemoteMetadata = - RemoteMetaData(otherRemoteKey, theHash, lastModified) - val input = - S3MetaData(theFile, // local exists - matchByHash = Set(otherRemoteMetadata), // other matches - matchByKey = None) // remote is missing + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + theRemoteKey = theFile.remoteKey + otherRemoteKey = RemoteKey.resolve("other-key")(prefix) + otherRemoteMetadata = RemoteMetaData(otherRemoteKey, + theHash, + lastModified) + input = S3MetaData( + theFile, // local exists + matchByHash = Set(otherRemoteMetadata), // other matches + matchByKey = None) // remote is missing + } yield (theFile, theRemoteKey, input, otherRemoteKey) it("copy from other key") { - val expected = Right( - Stream( - ToCopy(bucket, - otherRemoteKey, - theHash, - theRemoteKey, - theFile.file.length))) // copy - val result = invoke(input, previousActions) - assertResult(expected)(result) + env.map({ + case (theFile, theRemoteKey, input, otherRemoteKey) => { + val expected = Right( + Stream( + ToCopy(bucket, + otherRemoteKey, + theHash, + theRemoteKey, + theFile.file.length))) // copy + val result = invoke(input, previousActions) + assertResult(expected)(result) + } + }) } - } - describe("#3 local exists, remote is missing, other no matches - upload") { - val theHash = MD5Hash("the-hash") - val theFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val input = S3MetaData(theFile, // local exists + describe("#3 local exists, remote is missing, other no matches - upload") { + val theHash = MD5Hash("the-hash") + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + input = S3MetaData(theFile, // local exists matchByHash = Set.empty, // other no matches matchByKey = None) // remote is missing - it("upload") { - val expected = Right( - Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload - val result = invoke(input, previousActions) - assertResult(expected)(result) + } yield (theFile, input) + it("upload") { + env.map({ + case (theFile, input) => { + val expected = Right(Stream( + ToUpload(bucket, theFile, theFile.file.length))) // upload + val result = invoke(input, previousActions) + assertResult(expected)(result) + } + }) + } } } describe( "#4 local exists, remote exists, remote no match, other matches - copy") { val theHash = MD5Hash("the-hash") - val theFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val theRemoteKey = theFile.remoteKey - val oldHash = MD5Hash("old-hash") - val otherRemoteKey = prefix.resolve("other-key") - val otherRemoteMetadata = - RemoteMetaData(otherRemoteKey, theHash, lastModified) - val oldRemoteMetadata = RemoteMetaData(theRemoteKey, - hash = oldHash, // remote no match - lastModified = lastModified) - val input = - S3MetaData(theFile, // local exists - matchByHash = Set(otherRemoteMetadata), // other matches - matchByKey = Some(oldRemoteMetadata)) // remote exists + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + theRemoteKey = theFile.remoteKey + oldHash = MD5Hash("old-hash") + otherRemoteKey = RemoteKey.resolve("other-key")(prefix) + otherRemoteMetadata = RemoteMetaData(otherRemoteKey, + theHash, + lastModified) + oldRemoteMetadata = RemoteMetaData(theRemoteKey, + hash = oldHash, // remote no match + lastModified = lastModified) + input = S3MetaData( + theFile, // local exists + matchByHash = Set(otherRemoteMetadata), // other matches + matchByKey = Some(oldRemoteMetadata)) // remote exists + } yield (theFile, theRemoteKey, input, otherRemoteKey) it("copy from other key") { - val expected = Right( - Stream( - ToCopy(bucket, - otherRemoteKey, - theHash, - theRemoteKey, - theFile.file.length))) // copy - val result = invoke(input, previousActions) - assertResult(expected)(result) + env.map({ + case (theFile, theRemoteKey, input, otherRemoteKey) => { + val expected = Right( + Stream( + ToCopy(bucket, + otherRemoteKey, + theHash, + theRemoteKey, + theFile.file.length))) // copy + val result = invoke(input, previousActions) + assertResult(expected)(result) + } + }) } } describe( "#5 local exists, remote exists, remote no match, other no matches - upload") { val theHash = MD5Hash("the-hash") - val theFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val theRemoteKey = theFile.remoteKey - val oldHash = MD5Hash("old-hash") - val theRemoteMetadata = - RemoteMetaData(theRemoteKey, oldHash, lastModified) - val input = - S3MetaData(theFile, // local exists - matchByHash = Set.empty, // remote no match, other no match - matchByKey = Some(theRemoteMetadata) // remote exists + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + theRemoteKey = theFile.remoteKey + oldHash = MD5Hash("old-hash") + theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified) + input = S3MetaData( + theFile, // local exists + matchByHash = Set.empty, // remote no match, other no match + matchByKey = Some(theRemoteMetadata) // remote exists ) + } yield (theFile, input) it("upload") { - val expected = Right( - Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload - val result = invoke(input, previousActions) - assertResult(expected)(result) + env.map({ + case (theFile, input) => { + val expected = Right( + Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload + val result = invoke(input, previousActions) + assertResult(expected)(result) + } + }) } } describe("#6 local missing, remote exists - delete") { diff --git a/domain/src/test/scala/net/kemitix/thorp/domain/FiltersSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/FiltersSuite.scala similarity index 79% rename from domain/src/test/scala/net/kemitix/thorp/domain/FiltersSuite.scala rename to core/src/test/scala/net/kemitix/thorp/core/FiltersSuite.scala index 3523c2f..00279da 100644 --- a/domain/src/test/scala/net/kemitix/thorp/domain/FiltersSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/FiltersSuite.scala @@ -1,7 +1,8 @@ -package net.kemitix.thorp.domain +package net.kemitix.thorp.core import java.nio.file.Paths +import net.kemitix.thorp.domain.Filter import net.kemitix.thorp.domain.Filter.{Exclude, Include} import org.scalatest.FunSpec @@ -21,31 +22,32 @@ class FiltersSuite extends FunSpec { describe("default filter") { val include = Include() it("should include files") { - paths.foreach(path => assertResult(true)(include.isIncluded(path))) + paths.foreach(path => + assertResult(true)(Filters.isIncludedByFilter(path)(include))) } } describe("directory exact match include '/upload/subdir/'") { val include = Include("/upload/subdir/") it("include matching directory") { val matching = Paths.get("/upload/subdir/leaf-file") - assertResult(true)(include.isIncluded(matching)) + assertResult(true)(Filters.isIncludedByFilter(matching)(include)) } it("exclude non-matching files") { val nonMatching = Paths.get("/upload/other-file") - assertResult(false)(include.isIncluded(nonMatching)) + assertResult(false)(Filters.isIncludedByFilter(nonMatching)(include)) } } describe("file partial match 'root'") { val include = Include("root") it("include matching file '/upload/root-file") { val matching = Paths.get("/upload/root-file") - assertResult(true)(include.isIncluded(matching)) + assertResult(true)(Filters.isIncludedByFilter(matching)(include)) } it("exclude non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") { val nonMatching1 = Paths.get("/test-file-for-hash.txt") val nonMatching2 = Paths.get("/upload/subdir/leaf-file") - assertResult(false)(include.isIncluded(nonMatching1)) - assertResult(false)(include.isIncluded(nonMatching2)) + assertResult(false)(Filters.isIncludedByFilter(nonMatching1)(include)) + assertResult(false)(Filters.isIncludedByFilter(nonMatching2)(include)) } } } @@ -63,30 +65,30 @@ class FiltersSuite extends FunSpec { val exclude = Exclude("/upload/subdir/") it("exclude matching directory") { val matching = Paths.get("/upload/subdir/leaf-file") - assertResult(true)(exclude.isExcluded(matching)) + assertResult(true)(Filters.isExcludedByFilter(matching)(exclude)) } it("include non-matching files") { val nonMatching = Paths.get("/upload/other-file") - assertResult(false)(exclude.isExcluded(nonMatching)) + assertResult(false)(Filters.isExcludedByFilter(nonMatching)(exclude)) } } describe("file partial match 'root'") { val exclude = Exclude("root") it("exclude matching file '/upload/root-file") { val matching = Paths.get("/upload/root-file") - assertResult(true)(exclude.isExcluded(matching)) + assertResult(true)(Filters.isExcludedByFilter(matching)(exclude)) } it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") { val nonMatching1 = Paths.get("/test-file-for-hash.txt") val nonMatching2 = Paths.get("/upload/subdir/leaf-file") - assertResult(false)(exclude.isExcluded(nonMatching1)) - assertResult(false)(exclude.isExcluded(nonMatching2)) + assertResult(false)(Filters.isExcludedByFilter(nonMatching1)(exclude)) + assertResult(false)(Filters.isExcludedByFilter(nonMatching2)(exclude)) } } } describe("isIncluded") { def invoke(filters: List[Filter]) = { - paths.filter(path => Filter.isIncluded(path)(filters)) + paths.filter(path => Filters.isIncluded(path)(filters)) } describe("when there are no filters") { diff --git a/core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala index f541c20..61f3cfa 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala @@ -1,10 +1,12 @@ package net.kemitix.thorp.core import java.io.File +import java.nio.file.Path import net.kemitix.thorp.config.Resource import net.kemitix.thorp.domain.{RemoteKey, Sources} import org.scalatest.FunSpec +import zio.DefaultRuntime class KeyGeneratorSuite extends FunSpec { @@ -12,26 +14,32 @@ class KeyGeneratorSuite extends FunSpec { private val sourcePath = source.toPath private val prefix = RemoteKey("prefix") private val sources = Sources(List(sourcePath)) - private val fileToKey = - KeyGenerator.generateKey(sources, prefix) _ describe("key generator") { describe("when file is within source") { it("has a valid key") { - val subdir = "subdir" - assertResult(RemoteKey(s"${prefix.key}/$subdir"))( - fileToKey(sourcePath.resolve(subdir))) + val subdir = "subdir" + val expected = Right(RemoteKey(s"${prefix.key}/$subdir")) + val result = invoke(sourcePath.resolve(subdir)) + assertResult(expected)(result) } } describe("when file is deeper within source") { it("has a valid key") { - val subdir = "subdir/deeper/still" - assertResult(RemoteKey(s"${prefix.key}/$subdir"))( - fileToKey(sourcePath.resolve(subdir))) + val subdir = "subdir/deeper/still" + val expected = Right(RemoteKey(s"${prefix.key}/$subdir")) + val result = invoke(sourcePath.resolve(subdir)) + assertResult(expected)(result) } } + + def invoke(path: Path) = { + new DefaultRuntime {}.unsafeRunSync { + KeyGenerator.generateKey(sources, prefix)(path) + }.toEither + } } } diff --git a/core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala index 17b5755..642206d 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala @@ -32,7 +32,7 @@ class LocalFileStreamSuite extends FunSpec { val result = invoke() .map(_.localFiles) - .map(_.map(_.relative.toString)) + .map(_.map(LocalFile.relativeToSource(_).toString)) .map(_.toSet) assertResult(expected)(result) } diff --git a/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala b/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala index c5fe30c..7919635 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala @@ -333,13 +333,17 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { md5Hash: MD5Hash, source: Path, file: File): (String, String, String, String, String) = - ("upload", remoteKey.key, md5Hash.hash, source.toString, file.toString) + ("upload", + remoteKey.key, + MD5Hash.hash(md5Hash), + source.toString, + file.toString) private def toCopy( sourceKey: RemoteKey, md5Hash: MD5Hash, targetKey: RemoteKey): (String, String, String, String, String) = - ("copy", sourceKey.key, md5Hash.hash, targetKey.key, "") + ("copy", sourceKey.key, MD5Hash.hash(md5Hash), targetKey.key, "") private def toDelete( remoteKey: RemoteKey): (String, String, String, String, String) = @@ -386,12 +390,12 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { case ToUpload(_, lf, _) => ("upload", lf.remoteKey.key, - lf.hashes(MD5).hash, + MD5Hash.hash(lf.hashes(MD5)), lf.source.toString, lf.file.toString) case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "") case ToCopy(_, sourceKey, hash, targetKey, _) => - ("copy", sourceKey.key, hash.hash, targetKey.key, "") + ("copy", sourceKey.key, MD5Hash.hash(hash), targetKey.key, "") case DoNothing(_, remoteKey, _) => ("do-nothing", remoteKey.key, "", "", "") case x => ("other", x.toString, "", "", "") diff --git a/core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala index 05eb4a5..b6afcb4 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala @@ -14,8 +14,6 @@ class S3MetaDataEnricherSuite extends FunSpec { private val sourcePath = source.toPath private val sources = Sources(List(sourcePath)) private val prefix = RemoteKey("prefix") - private val fileToKey = - KeyGenerator.generateKey(sources, prefix) _ def getMatchesByKey( status: (Option[HashModified], Set[(MD5Hash, KeyModified)])) @@ -36,138 +34,180 @@ class S3MetaDataEnricherSuite extends FunSpec { describe( "#1a local exists, remote exists, remote matches, other matches - do nothing") { val theHash: MD5Hash = MD5Hash("the-file-hash") - val theFile: LocalFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val theRemoteKey: RemoteKey = theFile.remoteKey - val s3: S3ObjectsData = S3ObjectsData( - byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))), - byKey = Map(theRemoteKey -> HashModified(theHash, lastModified)) - ) - val theRemoteMetadata = - RemoteMetaData(theRemoteKey, theHash, lastModified) + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + theRemoteKey = theFile.remoteKey + s3 = S3ObjectsData( + byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))), + byKey = Map(theRemoteKey -> HashModified(theHash, lastModified)) + ) + theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified) + } yield (theFile, theRemoteMetadata, s3) it("generates valid metadata") { - val expected = S3MetaData(theFile, - matchByHash = Set(theRemoteMetadata), - matchByKey = Some(theRemoteMetadata)) - val result = getMetadata(theFile, s3) - assertResult(expected)(result) + env.map({ + case (theFile, theRemoteMetadata, s3) => { + val expected = S3MetaData(theFile, + matchByHash = Set(theRemoteMetadata), + matchByKey = Some(theRemoteMetadata)) + val result = getMetadata(theFile, s3) + assertResult(expected)(result) + } + }) } } describe( "#1b local exists, remote exists, remote matches, other no matches - do nothing") { val theHash: MD5Hash = MD5Hash("the-file-hash") - val theFile: LocalFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val theRemoteKey: RemoteKey = prefix.resolve("the-file") - val s3: S3ObjectsData = S3ObjectsData( - byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))), - byKey = Map(theRemoteKey -> HashModified(theHash, lastModified)) - ) - val theRemoteMetadata = - RemoteMetaData(theRemoteKey, theHash, lastModified) + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix) + s3: S3ObjectsData = S3ObjectsData( + byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))), + byKey = Map(theRemoteKey -> HashModified(theHash, lastModified)) + ) + theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified) + } yield (theFile, theRemoteMetadata, s3) it("generates valid metadata") { - val expected = S3MetaData(theFile, - matchByHash = Set(theRemoteMetadata), - matchByKey = Some(theRemoteMetadata)) - val result = getMetadata(theFile, s3) - assertResult(expected)(result) + env.map({ + case (theFile, theRemoteMetadata, s3) => { + val expected = S3MetaData(theFile, + matchByHash = Set(theRemoteMetadata), + matchByKey = Some(theRemoteMetadata)) + val result = getMetadata(theFile, s3) + assertResult(expected)(result) + } + }) } } describe( "#2 local exists, remote is missing, remote no match, other matches - copy") { val theHash = MD5Hash("the-hash") - val theFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val otherRemoteKey = RemoteKey("other-key") - val s3: S3ObjectsData = S3ObjectsData( - byHash = Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))), - byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified)) - ) - val otherRemoteMetadata = - RemoteMetaData(otherRemoteKey, theHash, lastModified) + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + otherRemoteKey = RemoteKey("other-key") + s3: S3ObjectsData = S3ObjectsData( + byHash = + Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))), + byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified)) + ) + otherRemoteMetadata = RemoteMetaData(otherRemoteKey, + theHash, + lastModified) + } yield (theFile, otherRemoteMetadata, s3) it("generates valid metadata") { - val expected = S3MetaData(theFile, - matchByHash = Set(otherRemoteMetadata), - matchByKey = None) - val result = getMetadata(theFile, s3) - assertResult(expected)(result) + env.map({ + case (theFile, otherRemoteMetadata, s3) => { + val expected = S3MetaData(theFile, + matchByHash = Set(otherRemoteMetadata), + matchByKey = None) + val result = getMetadata(theFile, s3) + assertResult(expected)(result) + } + }) } } describe( "#3 local exists, remote is missing, remote no match, other no matches - upload") { val theHash = MD5Hash("the-hash") - val theFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val s3: S3ObjectsData = S3ObjectsData() + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + s3: S3ObjectsData = S3ObjectsData() + } yield (theFile, s3) it("generates valid metadata") { - val expected = - S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None) - val result = getMetadata(theFile, s3) - assertResult(expected)(result) + env.map({ + case (theFile, s3) => { + val expected = + S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None) + val result = getMetadata(theFile, s3) + assertResult(expected)(result) + } + }) } } describe( "#4 local exists, remote exists, remote no match, other matches - copy") { val theHash = MD5Hash("the-hash") - val theFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val theRemoteKey = theFile.remoteKey - val oldHash = MD5Hash("old-hash") - val otherRemoteKey = prefix.resolve("other-key") - val s3: S3ObjectsData = S3ObjectsData( - byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)), - theHash -> Set(KeyModified(otherRemoteKey, lastModified))), - byKey = Map( - theRemoteKey -> HashModified(oldHash, lastModified), - otherRemoteKey -> HashModified(theHash, lastModified) + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + theRemoteKey = theFile.remoteKey + oldHash = MD5Hash("old-hash") + otherRemoteKey = RemoteKey.resolve("other-key")(prefix) + s3: S3ObjectsData = S3ObjectsData( + byHash = + Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)), + theHash -> Set(KeyModified(otherRemoteKey, lastModified))), + byKey = Map( + theRemoteKey -> HashModified(oldHash, lastModified), + otherRemoteKey -> HashModified(theHash, lastModified) + ) ) - ) - val theRemoteMetadata = - RemoteMetaData(theRemoteKey, oldHash, lastModified) - val otherRemoteMetadata = - RemoteMetaData(otherRemoteKey, theHash, lastModified) + theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified) + otherRemoteMetadata = RemoteMetaData(otherRemoteKey, + theHash, + lastModified) + } yield (theFile, theRemoteMetadata, otherRemoteMetadata, s3) it("generates valid metadata") { - val expected = S3MetaData(theFile, - matchByHash = Set(otherRemoteMetadata), - matchByKey = Some(theRemoteMetadata)) - val result = getMetadata(theFile, s3) - assertResult(expected)(result) + env.map({ + case (theFile, theRemoteMetadata, otherRemoteMetadata, s3) => { + val expected = S3MetaData(theFile, + matchByHash = Set(otherRemoteMetadata), + matchByKey = Some(theRemoteMetadata)) + val result = getMetadata(theFile, s3) + assertResult(expected)(result) + } + }) } } describe( "#5 local exists, remote exists, remote no match, other no matches - upload") { val theHash = MD5Hash("the-hash") - val theFile = LocalFile.resolve("the-file", - md5HashMap(theHash), - sourcePath, - fileToKey) - val theRemoteKey = theFile.remoteKey - val oldHash = MD5Hash("old-hash") - val s3: S3ObjectsData = S3ObjectsData( - byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)), - theHash -> Set.empty), - byKey = Map( - theRemoteKey -> HashModified(oldHash, lastModified) + val env = for { + theFile <- LocalFileValidator.resolve("the-file", + md5HashMap(theHash), + sourcePath, + sources, + prefix) + theRemoteKey = theFile.remoteKey + oldHash = MD5Hash("old-hash") + s3: S3ObjectsData = S3ObjectsData( + byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)), + theHash -> Set.empty), + byKey = Map( + theRemoteKey -> HashModified(oldHash, lastModified) + ) ) - ) - val theRemoteMetadata = - RemoteMetaData(theRemoteKey, oldHash, lastModified) + theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified) + } yield (theFile, theRemoteMetadata, s3) it("generates valid metadata") { - val expected = S3MetaData(theFile, - matchByHash = Set.empty, - matchByKey = Some(theRemoteMetadata)) - val result = getMetadata(theFile, s3) - assertResult(expected)(result) + env.map({ + case (theFile, theRemoteMetadata, s3) => { + val expected = S3MetaData(theFile, + matchByHash = Set.empty, + matchByKey = Some(theRemoteMetadata)) + val result = getMetadata(theFile, s3) + assertResult(expected)(result) + } + }) } } } @@ -178,71 +218,103 @@ class S3MetaDataEnricherSuite extends FunSpec { describe("getS3Status") { val hash = MD5Hash("hash") - val localFile = - LocalFile.resolve("the-file", md5HashMap(hash), sourcePath, fileToKey) - val key = localFile.remoteKey - val keyOtherKey = LocalFile.resolve("other-key-same-hash", - md5HashMap(hash), - sourcePath, - fileToKey) - val diffHash = MD5Hash("diff") - val keyDiffHash = LocalFile.resolve("other-key-diff-hash", - md5HashMap(diffHash), - sourcePath, - fileToKey) - val lastModified = LastModified(Instant.now) - val s3ObjectsData: S3ObjectsData = S3ObjectsData( - byHash = Map( - hash -> Set(KeyModified(key, lastModified), - KeyModified(keyOtherKey.remoteKey, lastModified)), - diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified)) - ), - byKey = Map( - key -> HashModified(hash, lastModified), - keyOtherKey.remoteKey -> HashModified(hash, lastModified), - keyDiffHash.remoteKey -> HashModified(diffHash, lastModified) + val env = for { + localFile <- LocalFileValidator.resolve("the-file", + md5HashMap(hash), + sourcePath, + sources, + prefix) + key = localFile.remoteKey + keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash", + md5HashMap(hash), + sourcePath, + sources, + prefix) + diffHash = MD5Hash("diff") + keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash", + md5HashMap(diffHash), + sourcePath, + sources, + prefix) + lastModified = LastModified(Instant.now) + s3ObjectsData = S3ObjectsData( + byHash = Map( + hash -> Set(KeyModified(key, lastModified), + KeyModified(keyOtherKey.remoteKey, lastModified)), + diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified)) + ), + byKey = Map( + key -> HashModified(hash, lastModified), + keyOtherKey.remoteKey -> HashModified(hash, lastModified), + keyDiffHash.remoteKey -> HashModified(diffHash, lastModified) + ) ) - ) + } yield (s3ObjectsData, localFile, keyDiffHash, diffHash) - def invoke(localFile: LocalFile) = { + def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) = { getS3Status(localFile, s3ObjectsData) } describe("when remote key exists") { it("should return a result for matching key") { - val result = getMatchesByKey(invoke(localFile)) - assert(result.contains(HashModified(hash, lastModified))) + env.map({ + case (s3ObjectsData, localFile: LocalFile, _, _) => + val result = getMatchesByKey(invoke(localFile, s3ObjectsData)) + assert(result.contains(HashModified(hash, lastModified))) + }) } } describe("when remote key does not exist and no others matches hash") { - val localFile = LocalFile.resolve("missing-file", - md5HashMap(MD5Hash("unique")), - sourcePath, - fileToKey) + val env2 = for { + localFile <- LocalFileValidator.resolve("missing-remote", + md5HashMap(MD5Hash("unique")), + sourcePath, + sources, + prefix) + } yield (localFile) it("should return no matches by key") { - val result = getMatchesByKey(invoke(localFile)) - assert(result.isEmpty) + env.map({ + case (s3ObjectsData, _, _, _) => { + env2.map({ + case (localFile) => { + val result = getMatchesByKey(invoke(localFile, s3ObjectsData)) + assert(result.isEmpty) + } + }) + } + }) } it("should return no matches by hash") { - val result = getMatchesByHash(invoke(localFile)) - assert(result.isEmpty) + env.map({ + case (s3ObjectsData, _, _, _) => { + env2.map({ + case (localFile) => { + val result = getMatchesByHash(invoke(localFile, s3ObjectsData)) + assert(result.isEmpty) + } + }) + } + }) } } describe("when remote key exists and no others match hash") { - it("should return match by key") { - val result = getMatchesByKey(invoke(keyDiffHash)) - assert(result.contains(HashModified(diffHash, lastModified))) - } - it("should return only itself in match by hash") { - val result = getMatchesByHash(invoke(keyDiffHash)) - assert( - result.equals( - Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified))))) - } + env.map({ + case (s3ObjectsData, _, keyDiffHash, diffHash) => { + it("should return match by key") { + val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData)) + assert(result.contains(HashModified(diffHash, lastModified))) + } + it("should return only itself in match by hash") { + val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData)) + assert( + result.equals(Set( + (diffHash, KeyModified(keyDiffHash.remoteKey, lastModified))))) + } + } + }) } - } } diff --git a/core/src/test/scala/net/kemitix/thorp/core/hasher/MD5HashGeneratorTest.scala b/core/src/test/scala/net/kemitix/thorp/core/hasher/MD5HashGeneratorTest.scala index 7237e4f..7ccc624 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/hasher/MD5HashGeneratorTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/hasher/MD5HashGeneratorTest.scala @@ -3,6 +3,7 @@ package net.kemitix.thorp.core.hasher import java.nio.file.Path import net.kemitix.thorp.config.Resource +import net.kemitix.thorp.domain.MD5Hash import net.kemitix.thorp.domain.MD5HashData.{BigFile, Root} import net.kemitix.thorp.filesystem.FileSystem import org.scalatest.FunSpec @@ -42,14 +43,14 @@ class MD5HashGeneratorTest extends FunSpec { val path = Resource(this, "../big-file").toPath it("should generate the correct hash for first chunk of the file") { val part1 = BigFile.Part1 - val expected = Right(part1.hash.hash) - val result = invoke(path, part1.offset, part1.size).map(_.hash) + val expected = Right(MD5Hash.hash(part1.hash)) + val result = invoke(path, part1.offset, part1.size).map(MD5Hash.hash) assertResult(expected)(result) } it("should generate the correct hash for second chunk of the file") { val part2 = BigFile.Part2 - val expected = Right(part2.hash.hash) - val result = invoke(path, part2.offset, part2.size).map(_.hash) + val expected = Right(MD5Hash.hash(part2.hash)) + val result = invoke(path, part2.offset, part2.size).map(MD5Hash.hash) assertResult(expected)(result) } } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala b/domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala index 3a730b8..208d70e 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala @@ -1,53 +1,18 @@ package net.kemitix.thorp.domain -import java.nio.file.Path +import java.util.function.Predicate import java.util.regex.Pattern -sealed trait Filter +sealed trait Filter { + def predicate: Predicate[String] +} object Filter { - - def isIncluded(p: Path)(filters: List[Filter]): Boolean = { - sealed trait State - case class Unknown() extends State - case class Accepted() extends State - case class Discarded() extends State - filters.foldRight(Unknown(): State)((filter, state) => - (filter, state) match { - case (_, Accepted()) => Accepted() - case (_, Discarded()) => Discarded() - case (x: Exclude, _) if x.isExcluded(p) => Discarded() - case (i: Include, _) if i.isIncluded(p) => Accepted() - case _ => Unknown() - }) match { - case Accepted() => true - case Discarded() => false - case Unknown() => - filters.forall { - case _: Include => false - case _ => true - } - } + case class Include(include: String = ".*") extends Filter { + lazy val predicate: Predicate[String] = Pattern.compile(include).asPredicate } - - case class Include( - include: String = ".*" - ) extends Filter { - - private lazy val predicate = Pattern.compile(include).asPredicate - - def isIncluded(path: Path): Boolean = predicate.test(path.toString) - + case class Exclude(exclude: String) extends Filter { + lazy val predicate: Predicate[String] = + Pattern.compile(exclude).asPredicate() } - - case class Exclude( - exclude: String - ) extends Filter { - - private lazy val predicate = Pattern.compile(exclude).asPredicate() - - def isExcluded(path: Path): Boolean = predicate.test(path.toString) - - } - } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala b/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala index 10d8be2..bd47bc3 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala @@ -5,42 +5,22 @@ import java.nio.file.Path import net.kemitix.thorp.domain.HashType.MD5 -final case class LocalFile( +final case class LocalFile private ( file: File, source: File, hashes: Map[HashType, MD5Hash], remoteKey: RemoteKey -) { - - require(!file.isDirectory, s"LocalFile must not be a directory: $file") - - def isDirectory: Boolean = file.isDirectory - - // the path of the file within the source - def relative: Path = source.toPath.relativize(file.toPath) - - def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _) - - def md5base64: Option[String] = hashes.get(MD5).map(_.hash64) - -} +) object LocalFile { - - def resolve( - path: String, - md5Hashes: Map[HashType, MD5Hash], - source: Path, - pathToKey: Path => RemoteKey - ): LocalFile = { - val resolvedPath = source.resolve(path) - LocalFile(resolvedPath.toFile, - source.toFile, - md5Hashes, - pathToKey(resolvedPath)) - } - val remoteKey: SimpleLens[LocalFile, RemoteKey] = SimpleLens[LocalFile, RemoteKey](_.remoteKey, b => a => b.copy(remoteKey = a)) + // the path of the file within the source + def relativeToSource(localFile: LocalFile): Path = + localFile.source.toPath.relativize(localFile.file.toPath) + def matchesHash(localFile: LocalFile)(other: MD5Hash): Boolean = + localFile.hashes.values.exists(other equals _) + def md5base64(localFile: LocalFile): Option[String] = + localFile.hashes.get(MD5).map(MD5Hash.hash64) } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/Logger.scala b/domain/src/main/scala/net/kemitix/thorp/domain/Logger.scala deleted file mode 100644 index 8862d74..0000000 --- a/domain/src/main/scala/net/kemitix/thorp/domain/Logger.scala +++ /dev/null @@ -1,15 +0,0 @@ -package net.kemitix.thorp.domain - -trait Logger { - - // returns an instance of Logger with debug set as indicated - // where the current Logger already matches this state, then - // it returns itself, unmodified - def withDebug(debug: Boolean): Logger - - def debug(message: => String): Unit - def info(message: => String): Unit - def warn(message: String): Unit - def error(message: String): Unit - -} diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala b/domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala index 4210f13..9d21a48 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala @@ -4,20 +4,14 @@ import java.util.Base64 import net.kemitix.thorp.domain.QuoteStripper.stripQuotes -final case class MD5Hash( - in: String -) { - - lazy val hash: String = in filter stripQuotes - - lazy val digest: Array[Byte] = HexEncoder.decode(hash) - - lazy val hash64: String = Base64.getEncoder.encodeToString(digest) -} +final case class MD5Hash(in: String) object MD5Hash { - def fromDigest(digest: Array[Byte]): MD5Hash = { - val hexDigest = (digest map ("%02x" format _)).mkString - MD5Hash(hexDigest) - } + def fromDigest(digest: Array[Byte]): MD5Hash = + MD5Hash((digest map ("%02x" format _)).mkString) + def hash(md5Hash: MD5Hash): String = md5Hash.in.filter(stripQuotes) + def digest(md5Hash: MD5Hash): Array[Byte] = + HexEncoder.decode(MD5Hash.hash(md5Hash)) + def hash64(md5Hash: MD5Hash): String = + Base64.getEncoder.encodeToString(MD5Hash.digest(md5Hash)) } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala b/domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala index 65bcff0..e0d8362 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala @@ -3,31 +3,21 @@ package net.kemitix.thorp.domain import java.io.File import java.nio.file.{Path, Paths} -final case class RemoteKey( - key: String -) { - - def asFile( - source: Path, - prefix: RemoteKey - ): Option[File] = - if (key.length == 0) None - else Some(source.resolve(relativeTo(prefix)).toFile) - - private def relativeTo(prefix: RemoteKey) = { - prefix match { - case RemoteKey("") => Paths.get(key) - case _ => Paths.get(prefix.key).relativize(Paths.get(key)) - } - } - - def resolve(path: String): RemoteKey = - RemoteKey(List(key, path).filterNot(_.isEmpty).mkString("/")) - -} +final case class RemoteKey(key: String) object RemoteKey { val key: SimpleLens[RemoteKey, String] = SimpleLens[RemoteKey, String](_.key, b => a => b.copy(key = a)) - + def asFile(source: Path, prefix: RemoteKey)( + remoteKey: RemoteKey): Option[File] = + if (remoteKey.key.length == 0) None + else Some(source.resolve(RemoteKey.relativeTo(prefix)(remoteKey)).toFile) + def relativeTo(prefix: RemoteKey)(remoteKey: RemoteKey): Path = { + prefix match { + case RemoteKey("") => Paths.get(remoteKey.key) + case _ => Paths.get(prefix.key).relativize(Paths.get(remoteKey.key)) + } + } + def resolve(path: String)(remoteKey: RemoteKey): RemoteKey = + RemoteKey(List(remoteKey.key, path).filterNot(_.isEmpty).mkString("/")) } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala b/domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala index 3bb13ef..0df7aba 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala @@ -2,6 +2,8 @@ package net.kemitix.thorp.domain import java.nio.file.Path +import zio.{Task, ZIO} + /** * The paths to synchronise with target. * @@ -18,18 +20,10 @@ case class Sources( def +(path: Path)(implicit m: Monoid[Sources]): Sources = this ++ List(path) def ++(otherPaths: List[Path])(implicit m: Monoid[Sources]): Sources = m.op(this, Sources(otherPaths)) - - /** - * Returns the source path for the given path. - */ - def forPath(path: Path): Path = - paths.find(source => path.startsWith(source)).get } object Sources { - final val emptySources = Sources(List.empty) - implicit def sourcesAppendMonoid: Monoid[Sources] = new Monoid[Sources] { override def zero: Sources = emptySources override def op(t1: Sources, t2: Sources): Sources = @@ -38,4 +32,12 @@ object Sources { else acc ++ List(path) }) } + + /** + * Returns the source path for the given path. + */ + def forPath(path: Path)(sources: Sources): Task[Path] = + ZIO + .fromOption(sources.paths.find(s => path.startsWith(s))) + .mapError(_ => new Exception("Path is not within any known source")) } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala b/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala index cb6b958..cff644f 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala @@ -1,10 +1,7 @@ package net.kemitix.thorp.domain import net.kemitix.thorp.domain.UploadEvent.RequestEvent -import net.kemitix.thorp.domain.UploadEventLogger.{ - RequestCycle, - logRequestCycle -} +import net.kemitix.thorp.domain.UploadEventLogger.RequestCycle object UploadEventListener { @@ -21,7 +18,7 @@ object UploadEventListener { uploadEvent match { case e: RequestEvent => bytesTransferred += e.transferred - logRequestCycle( + UploadEventLogger( RequestCycle(settings.localFile, bytesTransferred, settings.index, diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventLogger.scala b/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventLogger.scala index 6305212..00521e0 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventLogger.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/UploadEventLogger.scala @@ -14,7 +14,7 @@ object UploadEventLogger { totalBytesSoFar: Long ) - def logRequestCycle(requestCycle: RequestCycle): Unit = { + def apply(requestCycle: RequestCycle): Unit = { val remoteKey = requestCycle.localFile.remoteKey.key val fileLength = requestCycle.localFile.file.length val statusHeight = 7 diff --git a/domain/src/test/scala/net/kemitix/thorp/domain/MD5HashTest.scala b/domain/src/test/scala/net/kemitix/thorp/domain/MD5HashTest.scala index 25c1510..12b5d7a 100644 --- a/domain/src/test/scala/net/kemitix/thorp/domain/MD5HashTest.scala +++ b/domain/src/test/scala/net/kemitix/thorp/domain/MD5HashTest.scala @@ -7,11 +7,11 @@ class MD5HashTest extends FunSpec { describe("recover base64 hash") { it("should recover base 64 #1") { val rootHash = MD5HashData.Root.hash - assertResult(MD5HashData.Root.base64)(rootHash.hash64) + assertResult(MD5HashData.Root.base64)(MD5Hash.hash64(rootHash)) } it("should recover base 64 #2") { val leafHash = MD5HashData.Leaf.hash - assertResult(MD5HashData.Leaf.base64)(leafHash.hash64) + assertResult(MD5HashData.Leaf.base64)(MD5Hash.hash64(leafHash)) } } } diff --git a/domain/src/test/scala/net/kemitix/thorp/domain/RemoteKeyTest.scala b/domain/src/test/scala/net/kemitix/thorp/domain/RemoteKeyTest.scala index b937fce..6bbc0f3 100644 --- a/domain/src/test/scala/net/kemitix/thorp/domain/RemoteKeyTest.scala +++ b/domain/src/test/scala/net/kemitix/thorp/domain/RemoteKeyTest.scala @@ -15,21 +15,21 @@ class RemoteKeyTest extends FreeSpec { val key = emptyKey val path = "path" val expected = RemoteKey("path") - val result = key.resolve(path) + val result = RemoteKey.resolve(path)(key) assertResult(expected)(result) } "when path is empty" in { val key = RemoteKey("key") val path = "" val expected = RemoteKey("key") - val result = key.resolve(path) + val result = RemoteKey.resolve(path)(key) assertResult(expected)(result) } "when key and path are empty" in { val key = emptyKey val path = "" val expected = emptyKey - val result = key.resolve(path) + val result = RemoteKey.resolve(path)(key) assertResult(expected)(result) } } @@ -39,7 +39,7 @@ class RemoteKeyTest extends FreeSpec { val source = Paths.get("source") val prefix = RemoteKey("prefix") val expected = Some(new File("source/key")) - val result = key.asFile(source, prefix) + val result = RemoteKey.asFile(source, prefix)(key) assertResult(expected)(result) } "when prefix is empty" in { @@ -47,7 +47,7 @@ class RemoteKeyTest extends FreeSpec { val source = Paths.get("source") val prefix = emptyKey val expected = Some(new File("source/key")) - val result = key.asFile(source, prefix) + val result = RemoteKey.asFile(source, prefix)(key) assertResult(expected)(result) } "when key is empty" in { @@ -55,7 +55,7 @@ class RemoteKeyTest extends FreeSpec { val source = Paths.get("source") val prefix = RemoteKey("prefix") val expected = None - val result = key.asFile(source, prefix) + val result = RemoteKey.asFile(source, prefix)(key) assertResult(expected)(result) } "when key and prefix are empty" in { @@ -63,7 +63,7 @@ class RemoteKeyTest extends FreeSpec { val source = Paths.get("source") val prefix = emptyKey val expected = None - val result = key.asFile(source, prefix) + val result = RemoteKey.asFile(source, prefix)(key) assertResult(expected)(result) } } diff --git a/domain/src/test/scala/net/kemitix/thorp/domain/TemporaryFolder.scala b/domain/src/test/scala/net/kemitix/thorp/domain/TemporaryFolder.scala index 4b481df..512b7aa 100644 --- a/domain/src/test/scala/net/kemitix/thorp/domain/TemporaryFolder.scala +++ b/domain/src/test/scala/net/kemitix/thorp/domain/TemporaryFolder.scala @@ -38,11 +38,13 @@ trait TemporaryFolder { path.resolve(name).toFile } - def writeFile(directory: Path, name: String, contents: String*): Unit = { + def writeFile(directory: Path, name: String, contents: String*): File = { directory.toFile.mkdirs - val pw = new PrintWriter(directory.resolve(name).toFile, "UTF-8") - contents.foreach(pw.println) - pw.close() + val file = directory.resolve(name).toFile + val writer = new PrintWriter(file, "UTF-8") + contents.foreach(writer.println) + writer.close() + file } } diff --git a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Copier.scala b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Copier.scala index ca0e315..17436f4 100644 --- a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Copier.scala +++ b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Copier.scala @@ -41,7 +41,7 @@ trait Copier { copyRequest.sourceKey.key, copyRequest.bucket.name, copyRequest.targetKey.key - ).withMatchingETagConstraint(copyRequest.hash.hash) + ).withMatchingETagConstraint(MD5Hash.hash(copyRequest.hash)) private def foldFailure( sourceKey: RemoteKey, diff --git a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Uploader.scala b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Uploader.scala index 38cae23..08a618b 100644 --- a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Uploader.scala +++ b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Uploader.scala @@ -68,7 +68,7 @@ trait Uploader { private def metadata: LocalFile => ObjectMetadata = localFile => { val metadata = new ObjectMetadata() - localFile.md5base64.foreach(metadata.setContentMD5) + LocalFile.md5base64(localFile).foreach(metadata.setContentMD5) metadata } diff --git a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala index ada93e4..b3f9265 100644 --- a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala +++ b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala @@ -7,6 +7,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration import com.amazonaws.services.s3.transfer.internal.TransferManagerUtils import net.kemitix.thorp.core.hasher.Hasher import net.kemitix.thorp.domain.HashType.MD5 +import net.kemitix.thorp.domain.MD5Hash import net.kemitix.thorp.filesystem.FileSystem import zio.{TaskR, ZIO} @@ -56,7 +57,7 @@ private trait ETagGenerator { Hasher .hashObjectChunk(path, chunkNumber, chunkSize) .map(_(MD5)) - .map(_.digest) + .map(MD5Hash.digest) def offsets( totalFileSizeBytes: Long, diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala index 4c1805a..0babc7b 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala @@ -32,7 +32,7 @@ class S3ObjectsByHashSuite extends FunSpec { remoteKey: RemoteKey, lastModified: LastModified): S3ObjectSummary = { val summary = new S3ObjectSummary() - summary.setETag(md5Hash.hash) + summary.setETag(MD5Hash.hash(md5Hash)) summary.setKey(remoteKey.key) summary.setLastModified(Date.from(lastModified.when)) summary diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala index 5cec8f3..be21ccd 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala @@ -3,7 +3,7 @@ package net.kemitix.thorp.storage.aws import java.time.Instant import net.kemitix.thorp.config.Resource -import net.kemitix.thorp.core.{KeyGenerator, S3MetaDataEnricher} +import net.kemitix.thorp.core.{LocalFileValidator, S3MetaDataEnricher} import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain._ import org.scalamock.scalatest.MockFactory @@ -15,39 +15,51 @@ class StorageServiceSuite extends FunSpec with MockFactory { private val sourcePath = source.toPath private val sources = Sources(List(sourcePath)) private val prefix = RemoteKey("prefix") - private val fileToKey = - KeyGenerator.generateKey(sources, prefix) _ describe("getS3Status") { val hash = MD5Hash("hash") - val localFile = - LocalFile.resolve("the-file", Map(MD5 -> hash), sourcePath, fileToKey) - val key = localFile.remoteKey - val keyOtherKey = LocalFile.resolve("other-key-same-hash", - Map(MD5 -> hash), - sourcePath, - fileToKey) - val diffHash = MD5Hash("diff") - val keyDiffHash = LocalFile.resolve("other-key-diff-hash", - Map(MD5 -> diffHash), - sourcePath, - fileToKey) - val lastModified = LastModified(Instant.now) - val s3ObjectsData: S3ObjectsData = S3ObjectsData( - byHash = Map( - hash -> Set(KeyModified(key, lastModified), - KeyModified(keyOtherKey.remoteKey, lastModified)), - diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified)) - ), - byKey = Map( - key -> HashModified(hash, lastModified), - keyOtherKey.remoteKey -> HashModified(hash, lastModified), - keyDiffHash.remoteKey -> HashModified(diffHash, lastModified) + val env = for { + localFile <- LocalFileValidator.resolve("the-file", + Map(MD5 -> hash), + sourcePath, + sources, + prefix) + key = localFile.remoteKey + keyOtherKey <- LocalFileValidator.resolve("other-key-same-hash", + Map(MD5 -> hash), + sourcePath, + sources, + prefix) + diffHash = MD5Hash("diff") + keyDiffHash <- LocalFileValidator.resolve("other-key-diff-hash", + Map(MD5 -> diffHash), + sourcePath, + sources, + prefix) + lastModified = LastModified(Instant.now) + s3ObjectsData = S3ObjectsData( + byHash = Map( + hash -> Set(KeyModified(key, lastModified), + KeyModified(keyOtherKey.remoteKey, lastModified)), + diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified)) + ), + byKey = Map( + key -> HashModified(hash, lastModified), + keyOtherKey.remoteKey -> HashModified(hash, lastModified), + keyDiffHash.remoteKey -> HashModified(diffHash, lastModified) + ) ) - ) + } yield + (s3ObjectsData, + localFile: LocalFile, + lastModified, + keyOtherKey, + keyDiffHash, + diffHash, + key) - def invoke(localFile: LocalFile) = + def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) = S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData) def getMatchesByKey( @@ -67,47 +79,94 @@ class StorageServiceSuite extends FunSpec with MockFactory { describe( "when remote key exists, unmodified and other key matches the hash") { it("should return the match by key") { - val result = getMatchesByKey(invoke(localFile)) - assert(result.contains(HashModified(hash, lastModified))) + env.map({ + case (s3ObjectsData, localFile, lastModified, _, _, _, _) => { + val result = getMatchesByKey(invoke(localFile, s3ObjectsData)) + assert(result.contains(HashModified(hash, lastModified))) + } + }) } it("should return both matches for the hash") { - val result = getMatchesByHash(invoke(localFile)) - assertResult( - Set((hash, KeyModified(key, lastModified)), - (hash, KeyModified(keyOtherKey.remoteKey, lastModified))) - )(result) + env.map({ + case (s3ObjectsData, + localFile, + lastModified, + keyOtherKey, + _, + _, + key) => { + val result = getMatchesByHash(invoke(localFile, s3ObjectsData)) + assertResult( + Set((hash, KeyModified(key, lastModified)), + (hash, KeyModified(keyOtherKey.remoteKey, lastModified))) + )(result) + } + }) } } describe("when remote key does not exist and no others matches hash") { - val localFile = LocalFile.resolve("missing-file", - Map(MD5 -> MD5Hash("unique")), - sourcePath, - fileToKey) + val env2 = LocalFileValidator + .resolve("missing-file", + Map(MD5 -> MD5Hash("unique")), + sourcePath, + sources, + prefix) it("should return no matches by key") { - val result = getMatchesByKey(invoke(localFile)) - assert(result.isEmpty) + env2.map(localFile => { + env.map({ + case (s3ObjectsData, _, _, _, _, _, _) => { + val result = getMatchesByKey(invoke(localFile, s3ObjectsData)) + assert(result.isEmpty) + } + }) + }) } it("should return no matches by hash") { - val result = getMatchesByHash(invoke(localFile)) - assert(result.isEmpty) + env2.map(localFile => { + env.map({ + case (s3ObjectsData, _, _, _, _, _, _) => { + val result = getMatchesByHash(invoke(localFile, s3ObjectsData)) + assert(result.isEmpty) + } + }) + }) } } describe("when remote key exists and no others match hash") { - val localFile = keyDiffHash it("should return the match by key") { - val result = getMatchesByKey(invoke(localFile)) - assert(result.contains(HashModified(diffHash, lastModified))) + env.map({ + case (s3ObjectsData, + _, + lastModified, + _, + keyDiffHash, + diffHash, + _) => { + val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData)) + assert(result.contains(HashModified(diffHash, lastModified))) + } + }) } it("should return one match by hash") { - val result = getMatchesByHash(invoke(localFile)) - assertResult( - Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified))) - )(result) + env.map({ + case (s3ObjectsData, + _, + lastModified, + _, + keyDiffHash, + diffHash, + _) => { + val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData)) + assertResult( + Set( + (diffHash, KeyModified(keyDiffHash.remoteKey, lastModified))) + )(result) + } + }) } } - } } diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala index a4891e2..0518e79 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala @@ -29,7 +29,7 @@ class UploaderTest extends FreeSpec with MockFactory { val bucket = Bucket("aBucket") val uploadResult = new UploadResult uploadResult.setKey(remoteKey.key) - uploadResult.setETag(aHash.hash) + uploadResult.setETag(MD5Hash.hash(aHash)) val inProgress = new AmazonUpload.InProgress { override def waitForUploadResult: UploadResult = uploadResult } diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala index 931943a..feb90a5 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala @@ -6,6 +6,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerConfiguration import net.kemitix.thorp.config.Resource import net.kemitix.thorp.core.hasher.Hasher import net.kemitix.thorp.domain.HashType.MD5 +import net.kemitix.thorp.domain.MD5Hash import net.kemitix.thorp.filesystem.FileSystem import org.scalatest.FunSpec import zio.DefaultRuntime @@ -45,7 +46,7 @@ class ETagGeneratorTest extends FunSpec { assertResult(Right(hash))( invoke(bigFilePath, index, chunkSize) .map(_(MD5)) - .map(_.hash)) + .map(MD5Hash.hash)) } } def invoke(path: Path, index: Long, size: Long) =