Refactor ActionGenerator.genAction(S3MetaData,Stream,Bucket) (#143)
* [domain] rename S3MetaData as MatchedMetadata * [domain] rename S3ObjectsData as RemoteObjects * [core] ActionGenerator refactoring * [core] ActionGenerator.createAction renamed and no longer a stream * [core] ActionGenerator refactor * [core] ActionGenerator Usage of head on collections
This commit is contained in:
parent
e41e29127f
commit
07ca6b962f
16 changed files with 185 additions and 177 deletions
|
@ -3,45 +3,53 @@ package net.kemitix.thorp.core
|
||||||
import net.kemitix.thorp.config.Config
|
import net.kemitix.thorp.config.Config
|
||||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain._
|
||||||
import zio.ZIO
|
import zio.RIO
|
||||||
|
|
||||||
object ActionGenerator {
|
object ActionGenerator {
|
||||||
|
|
||||||
def createActions(
|
def createAction(
|
||||||
s3MetaData: S3MetaData,
|
matchedMetadata: MatchedMetadata,
|
||||||
previousActions: Stream[Action]
|
previousActions: Stream[Action]
|
||||||
): ZIO[Config, Nothing, Stream[Action]] =
|
): RIO[Config, Action] =
|
||||||
for {
|
for {
|
||||||
bucket <- Config.bucket
|
bucket <- Config.bucket
|
||||||
} yield genAction(s3MetaData, previousActions, bucket)
|
} yield
|
||||||
|
genAction(formattedMetadata(matchedMetadata, previousActions), bucket)
|
||||||
|
|
||||||
private def genAction(s3MetaData: S3MetaData,
|
private def formattedMetadata(
|
||||||
previousActions: Stream[Action],
|
matchedMetadata: MatchedMetadata,
|
||||||
bucket: Bucket): Stream[Action] = {
|
previousActions: Stream[Action]): TaggedMetadata = {
|
||||||
s3MetaData match {
|
val remoteExists = matchedMetadata.matchByKey.nonEmpty
|
||||||
// #1 local exists, remote exists, remote matches - do nothing
|
val remoteMatches = remoteExists && matchedMetadata.matchByKey.exists(m =>
|
||||||
case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
|
LocalFile.matchesHash(matchedMetadata.localFile)(m.hash))
|
||||||
if LocalFile.matchesHash(localFile)(hash) =>
|
val anyMatches = matchedMetadata.matchByHash.nonEmpty
|
||||||
doNothing(bucket, key)
|
TaggedMetadata(matchedMetadata,
|
||||||
// #2 local exists, remote is missing, other matches - copy
|
previousActions,
|
||||||
case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
|
remoteExists,
|
||||||
copyFile(bucket, localFile, matchByHash)
|
remoteMatches,
|
||||||
// #3 local exists, remote is missing, other no matches - upload
|
anyMatches)
|
||||||
case S3MetaData(localFile, matchByHash, None)
|
}
|
||||||
if matchByHash.isEmpty &&
|
|
||||||
isUploadAlreadyQueued(previousActions)(localFile) =>
|
case class TaggedMetadata(
|
||||||
uploadFile(bucket, localFile)
|
matchedMetadata: MatchedMetadata,
|
||||||
// #4 local exists, remote exists, remote no match, other matches - copy
|
previousActions: Stream[Action],
|
||||||
case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _)))
|
remoteExists: Boolean,
|
||||||
if !LocalFile.matchesHash(localFile)(hash) &&
|
remoteMatches: Boolean,
|
||||||
matchByHash.nonEmpty =>
|
anyMatches: Boolean
|
||||||
copyFile(bucket, localFile, matchByHash)
|
)
|
||||||
// #5 local exists, remote exists, remote no match, other no matches - upload
|
|
||||||
case S3MetaData(localFile, matchByHash, Some(_)) if matchByHash.isEmpty =>
|
private def genAction(taggedMetadata: TaggedMetadata,
|
||||||
uploadFile(bucket, localFile)
|
bucket: Bucket): Action = {
|
||||||
// fallback
|
taggedMetadata match {
|
||||||
case S3MetaData(localFile, _, _) =>
|
case TaggedMetadata(md, _, exists, matches, _) if exists && matches =>
|
||||||
doNothing(bucket, localFile.remoteKey)
|
doNothing(bucket, md.localFile.remoteKey)
|
||||||
|
case TaggedMetadata(md, _, _, _, any) if any =>
|
||||||
|
copyFile(bucket, md.localFile, md.matchByHash.head)
|
||||||
|
case TaggedMetadata(md, previous, _, _, _)
|
||||||
|
if isUploadAlreadyQueued(previous)(md.localFile) =>
|
||||||
|
uploadFile(bucket, md.localFile)
|
||||||
|
case TaggedMetadata(md, _, _, _, _) =>
|
||||||
|
doNothing(bucket, md.localFile.remoteKey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -59,29 +67,22 @@ object ActionGenerator {
|
||||||
private def doNothing(
|
private def doNothing(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
remoteKey: RemoteKey
|
remoteKey: RemoteKey
|
||||||
) =
|
) = DoNothing(bucket, remoteKey, 0L)
|
||||||
Stream(DoNothing(bucket, remoteKey, 0L))
|
|
||||||
|
|
||||||
private def uploadFile(
|
private def uploadFile(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
localFile: LocalFile
|
localFile: LocalFile
|
||||||
) =
|
) = ToUpload(bucket, localFile, localFile.file.length)
|
||||||
Stream(ToUpload(bucket, localFile, localFile.file.length))
|
|
||||||
|
|
||||||
private def copyFile(
|
private def copyFile(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
matchByHash: Set[RemoteMetaData]
|
remoteMetaData: RemoteMetaData
|
||||||
): Stream[Action] =
|
): Action =
|
||||||
matchByHash
|
ToCopy(bucket,
|
||||||
.map { remoteMetaData =>
|
remoteMetaData.remoteKey,
|
||||||
ToCopy(bucket,
|
remoteMetaData.hash,
|
||||||
remoteMetaData.remoteKey,
|
localFile.remoteKey,
|
||||||
remoteMetaData.hash,
|
localFile.file.length)
|
||||||
localFile.remoteKey,
|
|
||||||
localFile.file.length)
|
|
||||||
}
|
|
||||||
.toStream
|
|
||||||
.take(1)
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,22 +29,22 @@ object PlanBuilder {
|
||||||
objects <- Storage.list(bucket, prefix)
|
objects <- Storage.list(bucket, prefix)
|
||||||
} yield objects
|
} yield objects
|
||||||
|
|
||||||
private def assemblePlan(metadata: (S3ObjectsData, LocalFiles)) =
|
private def assemblePlan(metadata: (RemoteObjects, LocalFiles)) =
|
||||||
metadata match {
|
metadata match {
|
||||||
case (remoteData, localData) =>
|
case (remoteObjects, localData) =>
|
||||||
createActions(remoteData, localData.localFiles)
|
createActions(remoteObjects, localData.localFiles)
|
||||||
.map(_.filter(doesSomething).sortBy(SequencePlan.order))
|
.map(_.filter(doesSomething).sortBy(SequencePlan.order))
|
||||||
.map(
|
.map(
|
||||||
SyncPlan(_, SyncTotals(localData.count, localData.totalSizeBytes)))
|
SyncPlan(_, SyncTotals(localData.count, localData.totalSizeBytes)))
|
||||||
}
|
}
|
||||||
|
|
||||||
private def createActions(
|
private def createActions(
|
||||||
remoteData: S3ObjectsData,
|
remoteObjects: RemoteObjects,
|
||||||
localFiles: Stream[LocalFile]
|
localFiles: Stream[LocalFile]
|
||||||
) =
|
) =
|
||||||
for {
|
for {
|
||||||
fileActions <- actionsForLocalFiles(remoteData, localFiles)
|
fileActions <- actionsForLocalFiles(remoteObjects, localFiles)
|
||||||
remoteActions <- actionsForRemoteKeys(remoteData.byKey.keys)
|
remoteActions <- actionsForRemoteKeys(remoteObjects.byKey.keys)
|
||||||
} yield fileActions ++ remoteActions
|
} yield fileActions ++ remoteActions
|
||||||
|
|
||||||
private def doesSomething: Action => Boolean = {
|
private def doesSomething: Action => Boolean = {
|
||||||
|
@ -53,19 +53,19 @@ object PlanBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
private def actionsForLocalFiles(
|
private def actionsForLocalFiles(
|
||||||
remoteData: S3ObjectsData,
|
remoteObjects: RemoteObjects,
|
||||||
localFiles: Stream[LocalFile]
|
localFiles: Stream[LocalFile]
|
||||||
) =
|
) =
|
||||||
ZIO.foldLeft(localFiles)(Stream.empty[Action])((acc, localFile) =>
|
ZIO.foldLeft(localFiles)(Stream.empty[Action])((acc, localFile) =>
|
||||||
createActionFromLocalFile(remoteData, acc, localFile).map(_ ++ acc))
|
createActionFromLocalFile(remoteObjects, acc, localFile).map(_ #:: acc))
|
||||||
|
|
||||||
private def createActionFromLocalFile(
|
private def createActionFromLocalFile(
|
||||||
remoteData: S3ObjectsData,
|
remoteObjects: RemoteObjects,
|
||||||
previousActions: Stream[Action],
|
previousActions: Stream[Action],
|
||||||
localFile: LocalFile
|
localFile: LocalFile
|
||||||
) =
|
) =
|
||||||
ActionGenerator.createActions(
|
ActionGenerator.createAction(
|
||||||
S3MetaDataEnricher.getMetadata(localFile, remoteData),
|
S3MetaDataEnricher.getMetadata(localFile, remoteObjects),
|
||||||
previousActions)
|
previousActions)
|
||||||
|
|
||||||
private def actionsForRemoteKeys(remoteKeys: Iterable[RemoteKey]) =
|
private def actionsForRemoteKeys(remoteKeys: Iterable[RemoteKey]) =
|
||||||
|
|
|
@ -6,10 +6,10 @@ object S3MetaDataEnricher {
|
||||||
|
|
||||||
def getMetadata(
|
def getMetadata(
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
s3ObjectsData: S3ObjectsData
|
remoteObjects: RemoteObjects
|
||||||
): S3MetaData = {
|
): MatchedMetadata = {
|
||||||
val (keyMatches, hashMatches) = getS3Status(localFile, s3ObjectsData)
|
val (keyMatches, hashMatches) = getS3Status(localFile, remoteObjects)
|
||||||
S3MetaData(
|
MatchedMetadata(
|
||||||
localFile,
|
localFile,
|
||||||
matchByKey = keyMatches.map { hm =>
|
matchByKey = keyMatches.map { hm =>
|
||||||
RemoteMetaData(localFile.remoteKey, hm.hash, hm.modified)
|
RemoteMetaData(localFile.remoteKey, hm.hash, hm.modified)
|
||||||
|
@ -22,13 +22,13 @@ object S3MetaDataEnricher {
|
||||||
|
|
||||||
def getS3Status(
|
def getS3Status(
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
s3ObjectsData: S3ObjectsData
|
remoteObjects: RemoteObjects
|
||||||
): (Option[HashModified], Set[(MD5Hash, KeyModified)]) = {
|
): (Option[HashModified], Set[(MD5Hash, KeyModified)]) = {
|
||||||
val matchingByKey = s3ObjectsData.byKey.get(localFile.remoteKey)
|
val matchingByKey = remoteObjects.byKey.get(localFile.remoteKey)
|
||||||
val matchingByHash = localFile.hashes
|
val matchingByHash = localFile.hashes
|
||||||
.map {
|
.map {
|
||||||
case (_, md5Hash) =>
|
case (_, md5Hash) =>
|
||||||
s3ObjectsData.byHash
|
remoteObjects.byHash
|
||||||
.getOrElse(md5Hash, Set())
|
.getOrElse(md5Hash, Set())
|
||||||
.map(km => (md5Hash, km))
|
.map(km => (md5Hash, km))
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,7 +41,7 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
theRemoteMetadata = RemoteMetaData(theFile.remoteKey,
|
theRemoteMetadata = RemoteMetaData(theFile.remoteKey,
|
||||||
theHash,
|
theHash,
|
||||||
lastModified)
|
lastModified)
|
||||||
input = S3MetaData(
|
input = MatchedMetadata(
|
||||||
theFile, // local exists
|
theFile, // local exists
|
||||||
matchByHash = Set(theRemoteMetadata), // remote matches
|
matchByHash = Set(theRemoteMetadata), // remote matches
|
||||||
matchByKey = Some(theRemoteMetadata) // remote exists
|
matchByKey = Some(theRemoteMetadata) // remote exists
|
||||||
|
@ -72,7 +72,7 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
||||||
theHash,
|
theHash,
|
||||||
lastModified)
|
lastModified)
|
||||||
input = S3MetaData(
|
input = MatchedMetadata(
|
||||||
theFile, // local exists
|
theFile, // local exists
|
||||||
matchByHash = Set(otherRemoteMetadata), // other matches
|
matchByHash = Set(otherRemoteMetadata), // other matches
|
||||||
matchByKey = None) // remote is missing
|
matchByKey = None) // remote is missing
|
||||||
|
@ -100,9 +100,9 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
sourcePath,
|
sourcePath,
|
||||||
sources,
|
sources,
|
||||||
prefix)
|
prefix)
|
||||||
input = S3MetaData(theFile, // local exists
|
input = MatchedMetadata(theFile, // local exists
|
||||||
matchByHash = Set.empty, // other no matches
|
matchByHash = Set.empty, // other no matches
|
||||||
matchByKey = None) // remote is missing
|
matchByKey = None) // remote is missing
|
||||||
} yield (theFile, input)
|
} yield (theFile, input)
|
||||||
it("upload") {
|
it("upload") {
|
||||||
env.map({
|
env.map({
|
||||||
|
@ -134,7 +134,7 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
oldRemoteMetadata = RemoteMetaData(theRemoteKey,
|
oldRemoteMetadata = RemoteMetaData(theRemoteKey,
|
||||||
hash = oldHash, // remote no match
|
hash = oldHash, // remote no match
|
||||||
lastModified = lastModified)
|
lastModified = lastModified)
|
||||||
input = S3MetaData(
|
input = MatchedMetadata(
|
||||||
theFile, // local exists
|
theFile, // local exists
|
||||||
matchByHash = Set(otherRemoteMetadata), // other matches
|
matchByHash = Set(otherRemoteMetadata), // other matches
|
||||||
matchByKey = Some(oldRemoteMetadata)) // remote exists
|
matchByKey = Some(oldRemoteMetadata)) // remote exists
|
||||||
|
@ -167,7 +167,7 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
theRemoteKey = theFile.remoteKey
|
theRemoteKey = theFile.remoteKey
|
||||||
oldHash = MD5Hash("old-hash")
|
oldHash = MD5Hash("old-hash")
|
||||||
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
||||||
input = S3MetaData(
|
input = MatchedMetadata(
|
||||||
theFile, // local exists
|
theFile, // local exists
|
||||||
matchByHash = Set.empty, // remote no match, other no match
|
matchByHash = Set.empty, // remote no match, other no match
|
||||||
matchByKey = Some(theRemoteMetadata) // remote exists
|
matchByKey = Some(theRemoteMetadata) // remote exists
|
||||||
|
@ -196,7 +196,7 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
}
|
}
|
||||||
|
|
||||||
private def invoke(
|
private def invoke(
|
||||||
input: S3MetaData,
|
input: MatchedMetadata,
|
||||||
previousActions: Stream[Action]
|
previousActions: Stream[Action]
|
||||||
) = {
|
) = {
|
||||||
type TestEnv = Config with FileSystem
|
type TestEnv = Config with FileSystem
|
||||||
|
@ -206,7 +206,7 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
for {
|
for {
|
||||||
config <- ConfigurationBuilder.buildConfig(configOptions)
|
config <- ConfigurationBuilder.buildConfig(configOptions)
|
||||||
_ <- Config.set(config)
|
_ <- Config.set(config)
|
||||||
actions <- ActionGenerator.createActions(input, previousActions)
|
actions <- ActionGenerator.createAction(input, previousActions)
|
||||||
} yield actions
|
} yield actions
|
||||||
|
|
||||||
new DefaultRuntime {}.unsafeRunSync {
|
new DefaultRuntime {}.unsafeRunSync {
|
||||||
|
|
|
@ -7,7 +7,7 @@ import net.kemitix.thorp.domain._
|
||||||
import net.kemitix.thorp.storage.api.Storage
|
import net.kemitix.thorp.storage.api.Storage
|
||||||
import zio.{RIO, UIO}
|
import zio.{RIO, UIO}
|
||||||
|
|
||||||
case class DummyStorageService(s3ObjectData: S3ObjectsData,
|
case class DummyStorageService(remoteObjects: RemoteObjects,
|
||||||
uploadFiles: Map[File, (RemoteKey, MD5Hash)])
|
uploadFiles: Map[File, (RemoteKey, MD5Hash)])
|
||||||
extends Storage.Service {
|
extends Storage.Service {
|
||||||
|
|
||||||
|
@ -17,8 +17,8 @@ case class DummyStorageService(s3ObjectData: S3ObjectsData,
|
||||||
override def listObjects(
|
override def listObjects(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
prefix: RemoteKey
|
prefix: RemoteKey
|
||||||
): RIO[Console, S3ObjectsData] =
|
): RIO[Console, RemoteObjects] =
|
||||||
RIO(s3ObjectData)
|
RIO(remoteObjects)
|
||||||
|
|
||||||
override def upload(
|
override def upload(
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
|
|
|
@ -56,7 +56,7 @@ class LocalFileStreamSuite extends FunSpec {
|
||||||
with Hasher.Test
|
with Hasher.Test
|
||||||
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live
|
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live
|
||||||
with FileSystem.Live with Hasher.Test {
|
with FileSystem.Live with Hasher.Test {
|
||||||
override def listResult: Task[S3ObjectsData] =
|
override def listResult: Task[RemoteObjects] =
|
||||||
Task.die(new NotImplementedError)
|
Task.die(new NotImplementedError)
|
||||||
override def uploadResult: UIO[StorageQueueEvent] =
|
override def uploadResult: UIO[StorageQueueEvent] =
|
||||||
Task.die(new NotImplementedError)
|
Task.die(new NotImplementedError)
|
||||||
|
|
|
@ -8,12 +8,12 @@ import net.kemitix.thorp.domain.HashType.MD5
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain._
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
|
||||||
class S3MetaDataEnricherSuite extends FunSpec {
|
class MatchedMetadataEnricherSuite extends FunSpec {
|
||||||
val lastModified = LastModified(Instant.now())
|
private val lastModified = LastModified(Instant.now())
|
||||||
private val source = Resource(this, "upload")
|
private val source = Resource(this, "upload")
|
||||||
private val sourcePath = source.toPath
|
private val sourcePath = source.toPath
|
||||||
private val sources = Sources(List(sourcePath))
|
private val sources = Sources(List(sourcePath))
|
||||||
private val prefix = RemoteKey("prefix")
|
private val prefix = RemoteKey("prefix")
|
||||||
|
|
||||||
def getMatchesByKey(
|
def getMatchesByKey(
|
||||||
status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
|
status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
|
||||||
|
@ -41,19 +41,19 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
sources,
|
sources,
|
||||||
prefix)
|
prefix)
|
||||||
theRemoteKey = theFile.remoteKey
|
theRemoteKey = theFile.remoteKey
|
||||||
s3 = S3ObjectsData(
|
remoteObjects = RemoteObjects(
|
||||||
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
|
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
|
||||||
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
|
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
|
||||||
)
|
)
|
||||||
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
|
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
|
||||||
} yield (theFile, theRemoteMetadata, s3)
|
} yield (theFile, theRemoteMetadata, remoteObjects)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
env.map({
|
env.map({
|
||||||
case (theFile, theRemoteMetadata, s3) => {
|
case (theFile, theRemoteMetadata, remoteObjects) => {
|
||||||
val expected = S3MetaData(theFile,
|
val expected = MatchedMetadata(theFile,
|
||||||
matchByHash = Set(theRemoteMetadata),
|
matchByHash = Set(theRemoteMetadata),
|
||||||
matchByKey = Some(theRemoteMetadata))
|
matchByKey = Some(theRemoteMetadata))
|
||||||
val result = getMetadata(theFile, s3)
|
val result = getMetadata(theFile, remoteObjects)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -69,19 +69,19 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
sources,
|
sources,
|
||||||
prefix)
|
prefix)
|
||||||
theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix)
|
theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix)
|
||||||
s3: S3ObjectsData = S3ObjectsData(
|
remoteObjects = RemoteObjects(
|
||||||
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
|
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
|
||||||
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
|
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
|
||||||
)
|
)
|
||||||
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
|
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
|
||||||
} yield (theFile, theRemoteMetadata, s3)
|
} yield (theFile, theRemoteMetadata, remoteObjects)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
env.map({
|
env.map({
|
||||||
case (theFile, theRemoteMetadata, s3) => {
|
case (theFile, theRemoteMetadata, remoteObjects) => {
|
||||||
val expected = S3MetaData(theFile,
|
val expected = MatchedMetadata(theFile,
|
||||||
matchByHash = Set(theRemoteMetadata),
|
matchByHash = Set(theRemoteMetadata),
|
||||||
matchByKey = Some(theRemoteMetadata))
|
matchByKey = Some(theRemoteMetadata))
|
||||||
val result = getMetadata(theFile, s3)
|
val result = getMetadata(theFile, remoteObjects)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -97,7 +97,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
sources,
|
sources,
|
||||||
prefix)
|
prefix)
|
||||||
otherRemoteKey = RemoteKey("other-key")
|
otherRemoteKey = RemoteKey("other-key")
|
||||||
s3: S3ObjectsData = S3ObjectsData(
|
remoteObjects = RemoteObjects(
|
||||||
byHash =
|
byHash =
|
||||||
Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
|
Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
|
||||||
byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
|
byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
|
||||||
|
@ -105,14 +105,15 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
||||||
theHash,
|
theHash,
|
||||||
lastModified)
|
lastModified)
|
||||||
} yield (theFile, otherRemoteMetadata, s3)
|
} yield (theFile, otherRemoteMetadata, remoteObjects)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
env.map({
|
env.map({
|
||||||
case (theFile, otherRemoteMetadata, s3) => {
|
case (theFile, otherRemoteMetadata, remoteObjects) => {
|
||||||
val expected = S3MetaData(theFile,
|
val expected = MatchedMetadata(theFile,
|
||||||
matchByHash = Set(otherRemoteMetadata),
|
matchByHash =
|
||||||
matchByKey = None)
|
Set(otherRemoteMetadata),
|
||||||
val result = getMetadata(theFile, s3)
|
matchByKey = None)
|
||||||
|
val result = getMetadata(theFile, remoteObjects)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -127,14 +128,16 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
sourcePath,
|
sourcePath,
|
||||||
sources,
|
sources,
|
||||||
prefix)
|
prefix)
|
||||||
s3: S3ObjectsData = S3ObjectsData()
|
remoteObjects = RemoteObjects()
|
||||||
} yield (theFile, s3)
|
} yield (theFile, remoteObjects)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
env.map({
|
env.map({
|
||||||
case (theFile, s3) => {
|
case (theFile, remoteObjects) => {
|
||||||
val expected =
|
val expected =
|
||||||
S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None)
|
MatchedMetadata(theFile,
|
||||||
val result = getMetadata(theFile, s3)
|
matchByHash = Set.empty,
|
||||||
|
matchByKey = None)
|
||||||
|
val result = getMetadata(theFile, remoteObjects)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -152,7 +155,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
theRemoteKey = theFile.remoteKey
|
theRemoteKey = theFile.remoteKey
|
||||||
oldHash = MD5Hash("old-hash")
|
oldHash = MD5Hash("old-hash")
|
||||||
otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
|
otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
|
||||||
s3: S3ObjectsData = S3ObjectsData(
|
remoteObjects = RemoteObjects(
|
||||||
byHash =
|
byHash =
|
||||||
Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
|
Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
|
||||||
theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
|
theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
|
||||||
|
@ -165,14 +168,18 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
|
||||||
theHash,
|
theHash,
|
||||||
lastModified)
|
lastModified)
|
||||||
} yield (theFile, theRemoteMetadata, otherRemoteMetadata, s3)
|
} yield (theFile, theRemoteMetadata, otherRemoteMetadata, remoteObjects)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
env.map({
|
env.map({
|
||||||
case (theFile, theRemoteMetadata, otherRemoteMetadata, s3) => {
|
case (theFile,
|
||||||
val expected = S3MetaData(theFile,
|
theRemoteMetadata,
|
||||||
matchByHash = Set(otherRemoteMetadata),
|
otherRemoteMetadata,
|
||||||
matchByKey = Some(theRemoteMetadata))
|
remoteObjects) => {
|
||||||
val result = getMetadata(theFile, s3)
|
val expected = MatchedMetadata(theFile,
|
||||||
|
matchByHash =
|
||||||
|
Set(otherRemoteMetadata),
|
||||||
|
matchByKey = Some(theRemoteMetadata))
|
||||||
|
val result = getMetadata(theFile, remoteObjects)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -189,7 +196,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
prefix)
|
prefix)
|
||||||
theRemoteKey = theFile.remoteKey
|
theRemoteKey = theFile.remoteKey
|
||||||
oldHash = MD5Hash("old-hash")
|
oldHash = MD5Hash("old-hash")
|
||||||
s3: S3ObjectsData = S3ObjectsData(
|
remoteObjects = RemoteObjects(
|
||||||
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
|
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
|
||||||
theHash -> Set.empty),
|
theHash -> Set.empty),
|
||||||
byKey = Map(
|
byKey = Map(
|
||||||
|
@ -197,14 +204,14 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
|
||||||
} yield (theFile, theRemoteMetadata, s3)
|
} yield (theFile, theRemoteMetadata, remoteObjects)
|
||||||
it("generates valid metadata") {
|
it("generates valid metadata") {
|
||||||
env.map({
|
env.map({
|
||||||
case (theFile, theRemoteMetadata, s3) => {
|
case (theFile, theRemoteMetadata, remoteObjects) => {
|
||||||
val expected = S3MetaData(theFile,
|
val expected = MatchedMetadata(theFile,
|
||||||
matchByHash = Set.empty,
|
matchByHash = Set.empty,
|
||||||
matchByKey = Some(theRemoteMetadata))
|
matchByKey = Some(theRemoteMetadata))
|
||||||
val result = getMetadata(theFile, s3)
|
val result = getMetadata(theFile, remoteObjects)
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -237,7 +244,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
sources,
|
sources,
|
||||||
prefix)
|
prefix)
|
||||||
lastModified = LastModified(Instant.now)
|
lastModified = LastModified(Instant.now)
|
||||||
s3ObjectsData = S3ObjectsData(
|
remoteObjects = RemoteObjects(
|
||||||
byHash = Map(
|
byHash = Map(
|
||||||
hash -> Set(KeyModified(key, lastModified),
|
hash -> Set(KeyModified(key, lastModified),
|
||||||
KeyModified(keyOtherKey.remoteKey, lastModified)),
|
KeyModified(keyOtherKey.remoteKey, lastModified)),
|
||||||
|
@ -249,17 +256,17 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
|
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
} yield (s3ObjectsData, localFile, keyDiffHash, diffHash)
|
} yield (remoteObjects, localFile, keyDiffHash, diffHash)
|
||||||
|
|
||||||
def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) = {
|
def invoke(localFile: LocalFile, s3ObjectsData: RemoteObjects) = {
|
||||||
getS3Status(localFile, s3ObjectsData)
|
getS3Status(localFile, s3ObjectsData)
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("when remote key exists") {
|
describe("when remote key exists") {
|
||||||
it("should return a result for matching key") {
|
it("should return a result for matching key") {
|
||||||
env.map({
|
env.map({
|
||||||
case (s3ObjectsData, localFile: LocalFile, _, _) =>
|
case (remoteObjects, localFile: LocalFile, _, _) =>
|
||||||
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
|
val result = getMatchesByKey(invoke(localFile, remoteObjects))
|
||||||
assert(result.contains(HashModified(hash, lastModified)))
|
assert(result.contains(HashModified(hash, lastModified)))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -275,10 +282,10 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
} yield (localFile)
|
} yield (localFile)
|
||||||
it("should return no matches by key") {
|
it("should return no matches by key") {
|
||||||
env.map({
|
env.map({
|
||||||
case (s3ObjectsData, _, _, _) => {
|
case (remoteObjects, _, _, _) => {
|
||||||
env2.map({
|
env2.map({
|
||||||
case (localFile) => {
|
case (localFile) => {
|
||||||
val result = getMatchesByKey(invoke(localFile, s3ObjectsData))
|
val result = getMatchesByKey(invoke(localFile, remoteObjects))
|
||||||
assert(result.isEmpty)
|
assert(result.isEmpty)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -287,10 +294,10 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
}
|
}
|
||||||
it("should return no matches by hash") {
|
it("should return no matches by hash") {
|
||||||
env.map({
|
env.map({
|
||||||
case (s3ObjectsData, _, _, _) => {
|
case (remoteObjects, _, _, _) => {
|
||||||
env2.map({
|
env2.map({
|
||||||
case (localFile) => {
|
case (localFile) => {
|
||||||
val result = getMatchesByHash(invoke(localFile, s3ObjectsData))
|
val result = getMatchesByHash(invoke(localFile, remoteObjects))
|
||||||
assert(result.isEmpty)
|
assert(result.isEmpty)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -301,13 +308,13 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
|
|
||||||
describe("when remote key exists and no others match hash") {
|
describe("when remote key exists and no others match hash") {
|
||||||
env.map({
|
env.map({
|
||||||
case (s3ObjectsData, _, keyDiffHash, diffHash) => {
|
case (remoteObjects, _, keyDiffHash, diffHash) => {
|
||||||
it("should return match by key") {
|
it("should return match by key") {
|
||||||
val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData))
|
val result = getMatchesByKey(invoke(keyDiffHash, remoteObjects))
|
||||||
assert(result.contains(HashModified(diffHash, lastModified)))
|
assert(result.contains(HashModified(diffHash, lastModified)))
|
||||||
}
|
}
|
||||||
it("should return only itself in match by hash") {
|
it("should return only itself in match by hash") {
|
||||||
val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData))
|
val result = getMatchesByHash(invoke(keyDiffHash, remoteObjects))
|
||||||
assert(
|
assert(
|
||||||
result.equals(Set(
|
result.equals(Set(
|
||||||
(diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
|
(diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
|
|
@ -22,7 +22,7 @@ import zio.{DefaultRuntime, Task, UIO}
|
||||||
class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
|
|
||||||
private val lastModified: LastModified = LastModified()
|
private val lastModified: LastModified = LastModified()
|
||||||
private val emptyS3ObjectData = S3ObjectsData()
|
private val emptyRemoteObjects = RemoteObjects()
|
||||||
|
|
||||||
"create a plan" - {
|
"create a plan" - {
|
||||||
|
|
||||||
|
@ -46,7 +46,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
Right(List(toUpload(remoteKey, hash, source, file)))
|
Right(List(toUpload(remoteKey, hash, source, file)))
|
||||||
val result =
|
val result =
|
||||||
invoke(options(source),
|
invoke(options(source),
|
||||||
UIO.succeed(emptyS3ObjectData),
|
UIO.succeed(emptyRemoteObjects),
|
||||||
UIO.succeed(Map(file.toPath -> file)))
|
UIO.succeed(Map(file.toPath -> file)))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
})
|
})
|
||||||
|
@ -62,14 +62,14 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
val aHash = md5Hash(aFile)
|
val aHash = md5Hash(aFile)
|
||||||
val anOtherKey = RemoteKey("other")
|
val anOtherKey = RemoteKey("other")
|
||||||
val expected = Right(List(toCopy(anOtherKey, aHash, remoteKey)))
|
val expected = Right(List(toCopy(anOtherKey, aHash, remoteKey)))
|
||||||
val s3ObjectsData = S3ObjectsData(
|
val remoteObjects = RemoteObjects(
|
||||||
byHash =
|
byHash =
|
||||||
Map(aHash -> Set(KeyModified(anOtherKey, lastModified))),
|
Map(aHash -> Set(KeyModified(anOtherKey, lastModified))),
|
||||||
byKey = Map(anOtherKey -> HashModified(aHash, lastModified))
|
byKey = Map(anOtherKey -> HashModified(aHash, lastModified))
|
||||||
)
|
)
|
||||||
val result =
|
val result =
|
||||||
invoke(options(source),
|
invoke(options(source),
|
||||||
UIO.succeed(s3ObjectsData),
|
UIO.succeed(remoteObjects),
|
||||||
UIO.succeed(Map(aFile.toPath -> aFile,
|
UIO.succeed(Map(aFile.toPath -> aFile,
|
||||||
anOtherFile.toPath -> anOtherFile)))
|
anOtherFile.toPath -> anOtherFile)))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
|
@ -85,14 +85,14 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
val hash = md5Hash(file)
|
val hash = md5Hash(file)
|
||||||
// DoNothing actions should have been filtered out of the plan
|
// DoNothing actions should have been filtered out of the plan
|
||||||
val expected = Right(List())
|
val expected = Right(List())
|
||||||
val s3ObjectsData = S3ObjectsData(
|
val remoteObjects = RemoteObjects(
|
||||||
byHash =
|
byHash =
|
||||||
Map(hash -> Set(KeyModified(remoteKey, lastModified))),
|
Map(hash -> Set(KeyModified(remoteKey, lastModified))),
|
||||||
byKey = Map(remoteKey -> HashModified(hash, lastModified))
|
byKey = Map(remoteKey -> HashModified(hash, lastModified))
|
||||||
)
|
)
|
||||||
val result =
|
val result =
|
||||||
invoke(options(source),
|
invoke(options(source),
|
||||||
UIO.succeed(s3ObjectsData),
|
UIO.succeed(remoteObjects),
|
||||||
UIO.succeed(Map(file.toPath -> file)))
|
UIO.succeed(Map(file.toPath -> file)))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
})
|
})
|
||||||
|
@ -107,7 +107,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
val originalHash = MD5Hash("original-file-content")
|
val originalHash = MD5Hash("original-file-content")
|
||||||
val expected =
|
val expected =
|
||||||
Right(List(toUpload(remoteKey, currentHash, source, file)))
|
Right(List(toUpload(remoteKey, currentHash, source, file)))
|
||||||
val s3ObjectsData = S3ObjectsData(
|
val remoteObjects = RemoteObjects(
|
||||||
byHash = Map(originalHash -> Set(
|
byHash = Map(originalHash -> Set(
|
||||||
KeyModified(remoteKey, lastModified))),
|
KeyModified(remoteKey, lastModified))),
|
||||||
byKey =
|
byKey =
|
||||||
|
@ -115,7 +115,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
)
|
)
|
||||||
val result =
|
val result =
|
||||||
invoke(options(source),
|
invoke(options(source),
|
||||||
UIO.succeed(s3ObjectsData),
|
UIO.succeed(remoteObjects),
|
||||||
UIO.succeed(Map(file.toPath -> file)))
|
UIO.succeed(Map(file.toPath -> file)))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
})
|
})
|
||||||
|
@ -128,14 +128,14 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
val hash = md5Hash(file)
|
val hash = md5Hash(file)
|
||||||
val sourceKey = RemoteKey("other-key")
|
val sourceKey = RemoteKey("other-key")
|
||||||
val expected = Right(List(toCopy(sourceKey, hash, remoteKey)))
|
val expected = Right(List(toCopy(sourceKey, hash, remoteKey)))
|
||||||
val s3ObjectsData = S3ObjectsData(
|
val remoteObjects = RemoteObjects(
|
||||||
byHash =
|
byHash =
|
||||||
Map(hash -> Set(KeyModified(sourceKey, lastModified))),
|
Map(hash -> Set(KeyModified(sourceKey, lastModified))),
|
||||||
byKey = Map()
|
byKey = Map()
|
||||||
)
|
)
|
||||||
val result =
|
val result =
|
||||||
invoke(options(source),
|
invoke(options(source),
|
||||||
UIO.succeed(s3ObjectsData),
|
UIO.succeed(remoteObjects),
|
||||||
UIO.succeed(Map(file.toPath -> file)))
|
UIO.succeed(Map(file.toPath -> file)))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
})
|
})
|
||||||
|
@ -154,13 +154,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
val hash = md5Hash(file)
|
val hash = md5Hash(file)
|
||||||
// DoNothing actions should have been filtered out of the plan
|
// DoNothing actions should have been filtered out of the plan
|
||||||
val expected = Right(List())
|
val expected = Right(List())
|
||||||
val s3ObjectsData = S3ObjectsData(
|
val remoteObjects = RemoteObjects(
|
||||||
byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))),
|
byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))),
|
||||||
byKey = Map(remoteKey -> HashModified(hash, lastModified))
|
byKey = Map(remoteKey -> HashModified(hash, lastModified))
|
||||||
)
|
)
|
||||||
val result =
|
val result =
|
||||||
invoke(options(source),
|
invoke(options(source),
|
||||||
UIO.succeed(s3ObjectsData),
|
UIO.succeed(remoteObjects),
|
||||||
UIO.succeed(Map(file.toPath -> file)))
|
UIO.succeed(Map(file.toPath -> file)))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
})
|
})
|
||||||
|
@ -171,13 +171,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
withDirectory(source => {
|
withDirectory(source => {
|
||||||
val hash = MD5Hash("file-content")
|
val hash = MD5Hash("file-content")
|
||||||
val expected = Right(List(toDelete(remoteKey)))
|
val expected = Right(List(toDelete(remoteKey)))
|
||||||
val s3ObjectsData = S3ObjectsData(
|
val remoteObjects = RemoteObjects(
|
||||||
byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))),
|
byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))),
|
||||||
byKey = Map(remoteKey -> HashModified(hash, lastModified))
|
byKey = Map(remoteKey -> HashModified(hash, lastModified))
|
||||||
)
|
)
|
||||||
val result =
|
val result =
|
||||||
invoke(options(source),
|
invoke(options(source),
|
||||||
UIO.succeed(s3ObjectsData),
|
UIO.succeed(remoteObjects),
|
||||||
UIO.succeed(Map.empty))
|
UIO.succeed(Map.empty))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
})
|
})
|
||||||
|
@ -215,7 +215,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
val result =
|
val result =
|
||||||
invoke(
|
invoke(
|
||||||
options(firstSource)(secondSource),
|
options(firstSource)(secondSource),
|
||||||
UIO.succeed(emptyS3ObjectData),
|
UIO.succeed(emptyRemoteObjects),
|
||||||
UIO.succeed(
|
UIO.succeed(
|
||||||
Map(fileInFirstSource.toPath -> fileInFirstSource,
|
Map(fileInFirstSource.toPath -> fileInFirstSource,
|
||||||
fileInSecondSource.toPath -> fileInSecondSource))
|
fileInSecondSource.toPath -> fileInSecondSource))
|
||||||
|
@ -240,7 +240,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
val result =
|
val result =
|
||||||
invoke(
|
invoke(
|
||||||
options(firstSource)(secondSource),
|
options(firstSource)(secondSource),
|
||||||
UIO.succeed(emptyS3ObjectData),
|
UIO.succeed(emptyRemoteObjects),
|
||||||
UIO.succeed(
|
UIO.succeed(
|
||||||
Map(fileInFirstSource.toPath -> fileInFirstSource,
|
Map(fileInFirstSource.toPath -> fileInFirstSource,
|
||||||
fileInSecondSource.toPath -> fileInSecondSource))
|
fileInSecondSource.toPath -> fileInSecondSource))
|
||||||
|
@ -258,13 +258,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
createFile(secondSource, filename2, "file-2-content")
|
createFile(secondSource, filename2, "file-2-content")
|
||||||
val hash2 = md5Hash(fileInSecondSource)
|
val hash2 = md5Hash(fileInSecondSource)
|
||||||
val expected = Right(List())
|
val expected = Right(List())
|
||||||
val s3ObjectData = S3ObjectsData(
|
val remoteObjects = RemoteObjects(
|
||||||
byHash =
|
byHash =
|
||||||
Map(hash2 -> Set(KeyModified(remoteKey2, lastModified))),
|
Map(hash2 -> Set(KeyModified(remoteKey2, lastModified))),
|
||||||
byKey = Map(remoteKey2 -> HashModified(hash2, lastModified)))
|
byKey = Map(remoteKey2 -> HashModified(hash2, lastModified)))
|
||||||
val result =
|
val result =
|
||||||
invoke(options(firstSource)(secondSource),
|
invoke(options(firstSource)(secondSource),
|
||||||
UIO.succeed(s3ObjectData),
|
UIO.succeed(remoteObjects),
|
||||||
UIO.succeed(
|
UIO.succeed(
|
||||||
Map(fileInSecondSource.toPath -> fileInSecondSource)))
|
Map(fileInSecondSource.toPath -> fileInSecondSource)))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
|
@ -280,13 +280,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
val hash1 = md5Hash(fileInFirstSource)
|
val hash1 = md5Hash(fileInFirstSource)
|
||||||
withDirectory(secondSource => {
|
withDirectory(secondSource => {
|
||||||
val expected = Right(List())
|
val expected = Right(List())
|
||||||
val s3ObjectData = S3ObjectsData(
|
val remoteObjects = RemoteObjects(
|
||||||
byHash =
|
byHash =
|
||||||
Map(hash1 -> Set(KeyModified(remoteKey1, lastModified))),
|
Map(hash1 -> Set(KeyModified(remoteKey1, lastModified))),
|
||||||
byKey = Map(remoteKey1 -> HashModified(hash1, lastModified)))
|
byKey = Map(remoteKey1 -> HashModified(hash1, lastModified)))
|
||||||
val result =
|
val result =
|
||||||
invoke(options(firstSource)(secondSource),
|
invoke(options(firstSource)(secondSource),
|
||||||
UIO.succeed(s3ObjectData),
|
UIO.succeed(remoteObjects),
|
||||||
UIO.succeed(
|
UIO.succeed(
|
||||||
Map(fileInFirstSource.toPath -> fileInFirstSource)))
|
Map(fileInFirstSource.toPath -> fileInFirstSource)))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
|
@ -299,11 +299,11 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
withDirectory(firstSource => {
|
withDirectory(firstSource => {
|
||||||
withDirectory(secondSource => {
|
withDirectory(secondSource => {
|
||||||
val expected = Right(List(toDelete(remoteKey1)))
|
val expected = Right(List(toDelete(remoteKey1)))
|
||||||
val s3ObjectData = S3ObjectsData(byKey =
|
val remoteObjects = RemoteObjects(byKey =
|
||||||
Map(remoteKey1 -> HashModified(MD5Hash(""), lastModified)))
|
Map(remoteKey1 -> HashModified(MD5Hash(""), lastModified)))
|
||||||
val result =
|
val result =
|
||||||
invoke(options(firstSource)(secondSource),
|
invoke(options(firstSource)(secondSource),
|
||||||
UIO.succeed(s3ObjectData),
|
UIO.succeed(remoteObjects),
|
||||||
UIO.succeed(Map.empty))
|
UIO.succeed(Map.empty))
|
||||||
assertResult(expected)(result)
|
assertResult(expected)(result)
|
||||||
})
|
})
|
||||||
|
@ -354,13 +354,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
|
|
||||||
private def invoke(
|
private def invoke(
|
||||||
configOptions: ConfigOptions,
|
configOptions: ConfigOptions,
|
||||||
result: Task[S3ObjectsData],
|
result: Task[RemoteObjects],
|
||||||
files: Task[Map[Path, File]]
|
files: Task[Map[Path, File]]
|
||||||
) = {
|
) = {
|
||||||
type TestEnv = Storage with Console with Config with FileSystem with Hasher
|
type TestEnv = Storage with Console with Config with FileSystem with Hasher
|
||||||
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live
|
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live
|
||||||
with FileSystem.Live with Hasher.Live {
|
with FileSystem.Live with Hasher.Live {
|
||||||
override def listResult: Task[S3ObjectsData] = result
|
override def listResult: Task[RemoteObjects] = result
|
||||||
override def uploadResult: UIO[StorageQueueEvent] =
|
override def uploadResult: UIO[StorageQueueEvent] =
|
||||||
Task.die(new NotImplementedError)
|
Task.die(new NotImplementedError)
|
||||||
override def copyResult: UIO[StorageQueueEvent] =
|
override def copyResult: UIO[StorageQueueEvent] =
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
package net.kemitix.thorp.domain
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
|
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
|
||||||
final case class S3MetaData(
|
final case class MatchedMetadata(
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
matchByHash: Set[RemoteMetaData],
|
matchByHash: Set[RemoteMetaData],
|
||||||
matchByKey: Option[RemoteMetaData]
|
matchByKey: Option[RemoteMetaData]
|
|
@ -3,7 +3,7 @@ package net.kemitix.thorp.domain
|
||||||
/**
|
/**
|
||||||
* A list of objects and their MD5 hash values.
|
* A list of objects and their MD5 hash values.
|
||||||
*/
|
*/
|
||||||
final case class S3ObjectsData(
|
final case class RemoteObjects(
|
||||||
byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty,
|
byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty,
|
||||||
byKey: Map[RemoteKey, HashModified] = Map.empty
|
byKey: Map[RemoteKey, HashModified] = Map.empty
|
||||||
)
|
)
|
|
@ -15,7 +15,7 @@ object Storage {
|
||||||
def listObjects(
|
def listObjects(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
prefix: RemoteKey
|
prefix: RemoteKey
|
||||||
): RIO[Storage with Console, S3ObjectsData]
|
): RIO[Storage with Console, RemoteObjects]
|
||||||
|
|
||||||
def upload(
|
def upload(
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
|
@ -40,7 +40,7 @@ object Storage {
|
||||||
|
|
||||||
trait Test extends Storage {
|
trait Test extends Storage {
|
||||||
|
|
||||||
def listResult: Task[S3ObjectsData]
|
def listResult: Task[RemoteObjects]
|
||||||
def uploadResult: UIO[StorageQueueEvent]
|
def uploadResult: UIO[StorageQueueEvent]
|
||||||
def copyResult: UIO[StorageQueueEvent]
|
def copyResult: UIO[StorageQueueEvent]
|
||||||
def deleteResult: UIO[StorageQueueEvent]
|
def deleteResult: UIO[StorageQueueEvent]
|
||||||
|
@ -50,7 +50,7 @@ object Storage {
|
||||||
|
|
||||||
override def listObjects(
|
override def listObjects(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
prefix: RemoteKey): RIO[Storage with Console, S3ObjectsData] =
|
prefix: RemoteKey): RIO[Storage with Console, RemoteObjects] =
|
||||||
listResult
|
listResult
|
||||||
|
|
||||||
override def upload(
|
override def upload(
|
||||||
|
@ -78,7 +78,7 @@ object Storage {
|
||||||
}
|
}
|
||||||
|
|
||||||
object Test extends Test {
|
object Test extends Test {
|
||||||
override def listResult: Task[S3ObjectsData] =
|
override def listResult: Task[RemoteObjects] =
|
||||||
Task.die(new NotImplementedError)
|
Task.die(new NotImplementedError)
|
||||||
override def uploadResult: UIO[StorageQueueEvent] =
|
override def uploadResult: UIO[StorageQueueEvent] =
|
||||||
Task.die(new NotImplementedError)
|
Task.die(new NotImplementedError)
|
||||||
|
@ -91,7 +91,7 @@ object Storage {
|
||||||
}
|
}
|
||||||
|
|
||||||
final def list(bucket: Bucket,
|
final def list(bucket: Bucket,
|
||||||
prefix: RemoteKey): RIO[Storage with Console, S3ObjectsData] =
|
prefix: RemoteKey): RIO[Storage with Console, RemoteObjects] =
|
||||||
ZIO.accessM(_.storage listObjects (bucket, prefix))
|
ZIO.accessM(_.storage listObjects (bucket, prefix))
|
||||||
|
|
||||||
final def upload(
|
final def upload(
|
||||||
|
|
|
@ -6,7 +6,7 @@ import com.amazonaws.services.s3.model.{
|
||||||
S3ObjectSummary
|
S3ObjectSummary
|
||||||
}
|
}
|
||||||
import net.kemitix.thorp.console._
|
import net.kemitix.thorp.console._
|
||||||
import net.kemitix.thorp.domain.{Bucket, RemoteKey, S3ObjectsData}
|
import net.kemitix.thorp.domain.{Bucket, RemoteKey, RemoteObjects}
|
||||||
import net.kemitix.thorp.storage.aws.S3ObjectsByHash.byHash
|
import net.kemitix.thorp.storage.aws.S3ObjectsByHash.byHash
|
||||||
import net.kemitix.thorp.storage.aws.S3ObjectsByKey.byKey
|
import net.kemitix.thorp.storage.aws.S3ObjectsByKey.byKey
|
||||||
import zio.{Task, RIO}
|
import zio.{Task, RIO}
|
||||||
|
@ -21,7 +21,7 @@ trait Lister {
|
||||||
def listObjects(amazonS3: AmazonS3.Client)(
|
def listObjects(amazonS3: AmazonS3.Client)(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
prefix: RemoteKey
|
prefix: RemoteKey
|
||||||
): RIO[Console, S3ObjectsData] = {
|
): RIO[Console, RemoteObjects] = {
|
||||||
|
|
||||||
def request =
|
def request =
|
||||||
new ListObjectsV2Request()
|
new ListObjectsV2Request()
|
||||||
|
@ -48,7 +48,7 @@ trait Lister {
|
||||||
|
|
||||||
fetch(request)
|
fetch(request)
|
||||||
.map(summaries => {
|
.map(summaries => {
|
||||||
S3ObjectsData(byHash(summaries), byKey(summaries))
|
RemoteObjects(byHash(summaries), byKey(summaries))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -20,7 +20,7 @@ object S3Storage {
|
||||||
AmazonTransferManager(TransferManagerBuilder.defaultTransferManager)
|
AmazonTransferManager(TransferManagerBuilder.defaultTransferManager)
|
||||||
|
|
||||||
override def listObjects(bucket: Bucket,
|
override def listObjects(bucket: Bucket,
|
||||||
prefix: RemoteKey): RIO[Console, S3ObjectsData] =
|
prefix: RemoteKey): RIO[Console, RemoteObjects] =
|
||||||
Lister.listObjects(client)(bucket, prefix)
|
Lister.listObjects(client)(bucket, prefix)
|
||||||
|
|
||||||
override def upload(
|
override def upload(
|
||||||
|
|
|
@ -26,7 +26,7 @@ trait AmazonS3ClientTestFixture extends MockFactory {
|
||||||
override def listObjects(
|
override def listObjects(
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
prefix: RemoteKey
|
prefix: RemoteKey
|
||||||
): RIO[Console, S3ObjectsData] =
|
): RIO[Console, RemoteObjects] =
|
||||||
Lister.listObjects(client)(bucket, prefix)
|
Lister.listObjects(client)(bucket, prefix)
|
||||||
|
|
||||||
override def upload(
|
override def upload(
|
||||||
|
|
|
@ -34,7 +34,7 @@ class ListerTest extends FreeSpec {
|
||||||
RemoteKey(key) -> HashModified(MD5Hash(etag),
|
RemoteKey(key) -> HashModified(MD5Hash(etag),
|
||||||
LastModified(nowInstant))
|
LastModified(nowInstant))
|
||||||
)
|
)
|
||||||
val expected = Right(S3ObjectsData(expectedHashMap, expectedKeyMap))
|
val expected = Right(RemoteObjects(expectedHashMap, expectedKeyMap))
|
||||||
new AmazonS3ClientTestFixture {
|
new AmazonS3ClientTestFixture {
|
||||||
(fixture.amazonS3Client.listObjectsV2 _)
|
(fixture.amazonS3Client.listObjectsV2 _)
|
||||||
.when()
|
.when()
|
||||||
|
@ -65,7 +65,7 @@ class ListerTest extends FreeSpec {
|
||||||
RemoteKey(key2) -> HashModified(MD5Hash(etag2),
|
RemoteKey(key2) -> HashModified(MD5Hash(etag2),
|
||||||
LastModified(nowInstant))
|
LastModified(nowInstant))
|
||||||
)
|
)
|
||||||
val expected = Right(S3ObjectsData(expectedHashMap, expectedKeyMap))
|
val expected = Right(RemoteObjects(expectedHashMap, expectedKeyMap))
|
||||||
new AmazonS3ClientTestFixture {
|
new AmazonS3ClientTestFixture {
|
||||||
(fixture.amazonS3Client.listObjectsV2 _)
|
(fixture.amazonS3Client.listObjectsV2 _)
|
||||||
.when()
|
.when()
|
||||||
|
|
|
@ -38,7 +38,7 @@ class StorageServiceSuite extends FunSpec with MockFactory {
|
||||||
sources,
|
sources,
|
||||||
prefix)
|
prefix)
|
||||||
lastModified = LastModified(Instant.now)
|
lastModified = LastModified(Instant.now)
|
||||||
s3ObjectsData = S3ObjectsData(
|
s3ObjectsData = RemoteObjects(
|
||||||
byHash = Map(
|
byHash = Map(
|
||||||
hash -> Set(KeyModified(key, lastModified),
|
hash -> Set(KeyModified(key, lastModified),
|
||||||
KeyModified(keyOtherKey.remoteKey, lastModified)),
|
KeyModified(keyOtherKey.remoteKey, lastModified)),
|
||||||
|
@ -59,7 +59,7 @@ class StorageServiceSuite extends FunSpec with MockFactory {
|
||||||
diffHash,
|
diffHash,
|
||||||
key)
|
key)
|
||||||
|
|
||||||
def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) =
|
def invoke(localFile: LocalFile, s3ObjectsData: RemoteObjects) =
|
||||||
S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData)
|
S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData)
|
||||||
|
|
||||||
def getMatchesByKey(
|
def getMatchesByKey(
|
||||||
|
|
Loading…
Reference in a new issue