kemitix/thorp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor ActionGenerator.genAction(S3MetaData,Stream,Bucket) (#143)

Browse source 
* [domain] rename S3MetaData as MatchedMetadata

* [domain] rename S3ObjectsData as RemoteObjects

* [core] ActionGenerator refactoring

* [core] ActionGenerator.createAction renamed and no longer a stream

* [core] ActionGenerator refactor

* [core] ActionGenerator Usage of head on collections
This commit is contained in:
Paul Campbell 2019-08-05 12:07:06 +01:00 committed by GitHub
parent e41e29127f
commit 07ca6b962f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 185 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

97
core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala
View file

@ -3,45 +3,53 @@ package net.kemitix.thorp.core
import net.kemitix.thorp.config.Config import net.kemitix.thorp.config.Config
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload} import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain._
import zio.ZIO import zio.RIO
object ActionGenerator { object ActionGenerator {
def createActions( def createAction(
s3MetaData: S3MetaData, matchedMetadata: MatchedMetadata,
previousActions: Stream[Action] previousActions: Stream[Action]
): ZIO[Config, Nothing, Stream[Action]] = ): RIO[Config, Action] =
for { for {
bucket <- Config.bucket bucket <- Config.bucket
} yield genAction(s3MetaData, previousActions, bucket) } yield
genAction(formattedMetadata(matchedMetadata, previousActions), bucket)
private def genAction(s3MetaData: S3MetaData, private def formattedMetadata(
previousActions: Stream[Action], matchedMetadata: MatchedMetadata,
bucket: Bucket): Stream[Action] = { previousActions: Stream[Action]): TaggedMetadata = {
s3MetaData match { val remoteExists = matchedMetadata.matchByKey.nonEmpty
// #1 local exists, remote exists, remote matches - do nothing val remoteMatches = remoteExists && matchedMetadata.matchByKey.exists(m =>
case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _))) LocalFile.matchesHash(matchedMetadata.localFile)(m.hash))
if LocalFile.matchesHash(localFile)(hash) => val anyMatches = matchedMetadata.matchByHash.nonEmpty
doNothing(bucket, key) TaggedMetadata(matchedMetadata,
// #2 local exists, remote is missing, other matches - copy previousActions,
case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty => remoteExists,
copyFile(bucket, localFile, matchByHash) remoteMatches,
// #3 local exists, remote is missing, other no matches - upload anyMatches)
case S3MetaData(localFile, matchByHash, None) }
if matchByHash.isEmpty &&
isUploadAlreadyQueued(previousActions)(localFile) => case class TaggedMetadata(
uploadFile(bucket, localFile) matchedMetadata: MatchedMetadata,
// #4 local exists, remote exists, remote no match, other matches - copy previousActions: Stream[Action],
case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _))) remoteExists: Boolean,
if !LocalFile.matchesHash(localFile)(hash) && remoteMatches: Boolean,
matchByHash.nonEmpty => anyMatches: Boolean
copyFile(bucket, localFile, matchByHash) )
// #5 local exists, remote exists, remote no match, other no matches - upload
case S3MetaData(localFile, matchByHash, Some(_)) if matchByHash.isEmpty => private def genAction(taggedMetadata: TaggedMetadata,
uploadFile(bucket, localFile) bucket: Bucket): Action = {
// fallback taggedMetadata match {
case S3MetaData(localFile, _, _) => case TaggedMetadata(md, _, exists, matches, _) if exists && matches =>
doNothing(bucket, localFile.remoteKey) doNothing(bucket, md.localFile.remoteKey)
case TaggedMetadata(md, _, _, _, any) if any =>
copyFile(bucket, md.localFile, md.matchByHash.head)
case TaggedMetadata(md, previous, _, _, _)
if isUploadAlreadyQueued(previous)(md.localFile) =>
uploadFile(bucket, md.localFile)
case TaggedMetadata(md, _, _, _, _) =>
doNothing(bucket, md.localFile.remoteKey)
} }
} }
@ -59,29 +67,22 @@ object ActionGenerator {
private def doNothing( private def doNothing(
bucket: Bucket, bucket: Bucket,
remoteKey: RemoteKey remoteKey: RemoteKey
) = ) = DoNothing(bucket, remoteKey, 0L)
Stream(DoNothing(bucket, remoteKey, 0L))
private def uploadFile( private def uploadFile(
bucket: Bucket, bucket: Bucket,
localFile: LocalFile localFile: LocalFile
) = ) = ToUpload(bucket, localFile, localFile.file.length)
Stream(ToUpload(bucket, localFile, localFile.file.length))
private def copyFile( private def copyFile(
bucket: Bucket, bucket: Bucket,
localFile: LocalFile, localFile: LocalFile,
matchByHash: Set[RemoteMetaData] remoteMetaData: RemoteMetaData
): Stream[Action] = ): Action =
matchByHash ToCopy(bucket,
.map { remoteMetaData => remoteMetaData.remoteKey,
ToCopy(bucket, remoteMetaData.hash,
remoteMetaData.remoteKey, localFile.remoteKey,
remoteMetaData.hash, localFile.file.length)
localFile.remoteKey,
localFile.file.length)
}
.toStream
.take(1)
} }

22
core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala
View file

@ -29,22 +29,22 @@ object PlanBuilder {
objects <- Storage.list(bucket, prefix) objects <- Storage.list(bucket, prefix)
} yield objects } yield objects
private def assemblePlan(metadata: (S3ObjectsData, LocalFiles)) = private def assemblePlan(metadata: (RemoteObjects, LocalFiles)) =
metadata match { metadata match {
case (remoteData, localData) => case (remoteObjects, localData) =>
createActions(remoteData, localData.localFiles) createActions(remoteObjects, localData.localFiles)
.map(_.filter(doesSomething).sortBy(SequencePlan.order)) .map(_.filter(doesSomething).sortBy(SequencePlan.order))
.map( .map(
SyncPlan(_, SyncTotals(localData.count, localData.totalSizeBytes))) SyncPlan(_, SyncTotals(localData.count, localData.totalSizeBytes)))
} }
private def createActions( private def createActions(
remoteData: S3ObjectsData, remoteObjects: RemoteObjects,
localFiles: Stream[LocalFile] localFiles: Stream[LocalFile]
) = ) =
for { for {
fileActions <- actionsForLocalFiles(remoteData, localFiles) fileActions <- actionsForLocalFiles(remoteObjects, localFiles)
remoteActions <- actionsForRemoteKeys(remoteData.byKey.keys) remoteActions <- actionsForRemoteKeys(remoteObjects.byKey.keys)
} yield fileActions ++ remoteActions } yield fileActions ++ remoteActions
private def doesSomething: Action => Boolean = { private def doesSomething: Action => Boolean = {
@ -53,19 +53,19 @@ object PlanBuilder {
} }
private def actionsForLocalFiles( private def actionsForLocalFiles(
remoteData: S3ObjectsData, remoteObjects: RemoteObjects,
localFiles: Stream[LocalFile] localFiles: Stream[LocalFile]
) = ) =
ZIO.foldLeft(localFiles)(Stream.empty[Action])((acc, localFile) => ZIO.foldLeft(localFiles)(Stream.empty[Action])((acc, localFile) =>
createActionFromLocalFile(remoteData, acc, localFile).map(_ ++ acc)) createActionFromLocalFile(remoteObjects, acc, localFile).map(_ #:: acc))
private def createActionFromLocalFile( private def createActionFromLocalFile(
remoteData: S3ObjectsData, remoteObjects: RemoteObjects,
previousActions: Stream[Action], previousActions: Stream[Action],
localFile: LocalFile localFile: LocalFile
) = ) =
ActionGenerator.createActions( ActionGenerator.createAction(
S3MetaDataEnricher.getMetadata(localFile, remoteData), S3MetaDataEnricher.getMetadata(localFile, remoteObjects),
previousActions) previousActions)
private def actionsForRemoteKeys(remoteKeys: Iterable[RemoteKey]) = private def actionsForRemoteKeys(remoteKeys: Iterable[RemoteKey]) =

14
core/src/main/scala/net/kemitix/thorp/core/S3MetaDataEnricher.scala
View file

@ -6,10 +6,10 @@ object S3MetaDataEnricher {
def getMetadata( def getMetadata(
localFile: LocalFile, localFile: LocalFile,
s3ObjectsData: S3ObjectsData remoteObjects: RemoteObjects
): S3MetaData = { ): MatchedMetadata = {
val (keyMatches, hashMatches) = getS3Status(localFile, s3ObjectsData) val (keyMatches, hashMatches) = getS3Status(localFile, remoteObjects)
S3MetaData( MatchedMetadata(
localFile, localFile,
matchByKey = keyMatches.map { hm => matchByKey = keyMatches.map { hm =>
RemoteMetaData(localFile.remoteKey, hm.hash, hm.modified) RemoteMetaData(localFile.remoteKey, hm.hash, hm.modified)
@ -22,13 +22,13 @@ object S3MetaDataEnricher {
def getS3Status( def getS3Status(
localFile: LocalFile, localFile: LocalFile,
s3ObjectsData: S3ObjectsData remoteObjects: RemoteObjects
): (Option[HashModified], Set[(MD5Hash, KeyModified)]) = { ): (Option[HashModified], Set[(MD5Hash, KeyModified)]) = {
val matchingByKey = s3ObjectsData.byKey.get(localFile.remoteKey) val matchingByKey = remoteObjects.byKey.get(localFile.remoteKey)
val matchingByHash = localFile.hashes val matchingByHash = localFile.hashes
.map { .map {
case (_, md5Hash) => case (_, md5Hash) =>
s3ObjectsData.byHash remoteObjects.byHash
.getOrElse(md5Hash, Set()) .getOrElse(md5Hash, Set())
.map(km => (md5Hash, km)) .map(km => (md5Hash, km))
} }

18
core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala
View file

@ -41,7 +41,7 @@ class ActionGeneratorSuite extends FunSpec {
theRemoteMetadata = RemoteMetaData(theFile.remoteKey, theRemoteMetadata = RemoteMetaData(theFile.remoteKey,
theHash, theHash,
lastModified) lastModified)
input = S3MetaData( input = MatchedMetadata(
theFile, // local exists theFile, // local exists
matchByHash = Set(theRemoteMetadata), // remote matches matchByHash = Set(theRemoteMetadata), // remote matches
matchByKey = Some(theRemoteMetadata) // remote exists matchByKey = Some(theRemoteMetadata) // remote exists
@ -72,7 +72,7 @@ class ActionGeneratorSuite extends FunSpec {
otherRemoteMetadata = RemoteMetaData(otherRemoteKey, otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
theHash, theHash,
lastModified) lastModified)
input = S3MetaData( input = MatchedMetadata(
theFile, // local exists theFile, // local exists
matchByHash = Set(otherRemoteMetadata), // other matches matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = None) // remote is missing matchByKey = None) // remote is missing
@ -100,9 +100,9 @@ class ActionGeneratorSuite extends FunSpec {
sourcePath, sourcePath,
sources, sources,
prefix) prefix)
input = S3MetaData(theFile, // local exists input = MatchedMetadata(theFile, // local exists
matchByHash = Set.empty, // other no matches matchByHash = Set.empty, // other no matches
matchByKey = None) // remote is missing matchByKey = None) // remote is missing
} yield (theFile, input) } yield (theFile, input)
it("upload") { it("upload") {
env.map({ env.map({
@ -134,7 +134,7 @@ class ActionGeneratorSuite extends FunSpec {
oldRemoteMetadata = RemoteMetaData(theRemoteKey, oldRemoteMetadata = RemoteMetaData(theRemoteKey,
hash = oldHash, // remote no match hash = oldHash, // remote no match
lastModified = lastModified) lastModified = lastModified)
input = S3MetaData( input = MatchedMetadata(
theFile, // local exists theFile, // local exists
matchByHash = Set(otherRemoteMetadata), // other matches matchByHash = Set(otherRemoteMetadata), // other matches
matchByKey = Some(oldRemoteMetadata)) // remote exists matchByKey = Some(oldRemoteMetadata)) // remote exists
@ -167,7 +167,7 @@ class ActionGeneratorSuite extends FunSpec {
theRemoteKey = theFile.remoteKey theRemoteKey = theFile.remoteKey
oldHash = MD5Hash("old-hash") oldHash = MD5Hash("old-hash")
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified) theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
input = S3MetaData( input = MatchedMetadata(
theFile, // local exists theFile, // local exists
matchByHash = Set.empty, // remote no match, other no match matchByHash = Set.empty, // remote no match, other no match
matchByKey = Some(theRemoteMetadata) // remote exists matchByKey = Some(theRemoteMetadata) // remote exists
@ -196,7 +196,7 @@ class ActionGeneratorSuite extends FunSpec {
} }
private def invoke( private def invoke(
input: S3MetaData, input: MatchedMetadata,
previousActions: Stream[Action] previousActions: Stream[Action]
) = { ) = {
type TestEnv = Config with FileSystem type TestEnv = Config with FileSystem
@ -206,7 +206,7 @@ class ActionGeneratorSuite extends FunSpec {
for { for {
config <- ConfigurationBuilder.buildConfig(configOptions) config <- ConfigurationBuilder.buildConfig(configOptions)
_ <- Config.set(config) _ <- Config.set(config)
actions <- ActionGenerator.createActions(input, previousActions) actions <- ActionGenerator.createAction(input, previousActions)
} yield actions } yield actions
new DefaultRuntime {}.unsafeRunSync { new DefaultRuntime {}.unsafeRunSync {

6
core/src/test/scala/net/kemitix/thorp/core/DummyStorageService.scala
View file

@ -7,7 +7,7 @@ import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.api.Storage import net.kemitix.thorp.storage.api.Storage
import zio.{RIO, UIO} import zio.{RIO, UIO}
case class DummyStorageService(s3ObjectData: S3ObjectsData, case class DummyStorageService(remoteObjects: RemoteObjects,
uploadFiles: Map[File, (RemoteKey, MD5Hash)]) uploadFiles: Map[File, (RemoteKey, MD5Hash)])
extends Storage.Service { extends Storage.Service {
@ -17,8 +17,8 @@ case class DummyStorageService(s3ObjectData: S3ObjectsData,
override def listObjects( override def listObjects(
bucket: Bucket, bucket: Bucket,
prefix: RemoteKey prefix: RemoteKey
): RIO[Console, S3ObjectsData] = ): RIO[Console, RemoteObjects] =
RIO(s3ObjectData) RIO(remoteObjects)
override def upload( override def upload(
localFile: LocalFile, localFile: LocalFile,

2
core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala
View file

@ -56,7 +56,7 @@ class LocalFileStreamSuite extends FunSpec {
with Hasher.Test with Hasher.Test
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live
with FileSystem.Live with Hasher.Test { with FileSystem.Live with Hasher.Test {
override def listResult: Task[S3ObjectsData] = override def listResult: Task[RemoteObjects] =
Task.die(new NotImplementedError) Task.die(new NotImplementedError)
override def uploadResult: UIO[StorageQueueEvent] = override def uploadResult: UIO[StorageQueueEvent] =
Task.die(new NotImplementedError) Task.die(new NotImplementedError)

123
core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala → core/src/test/scala/net/kemitix/thorp/core/MatchedMetadataEnricherSuite.scala
View file

@ -8,12 +8,12 @@ import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain._
import org.scalatest.FunSpec import org.scalatest.FunSpec
class S3MetaDataEnricherSuite extends FunSpec { class MatchedMetadataEnricherSuite extends FunSpec {
val lastModified = LastModified(Instant.now()) private val lastModified = LastModified(Instant.now())
private val source = Resource(this, "upload") private val source = Resource(this, "upload")
private val sourcePath = source.toPath private val sourcePath = source.toPath
private val sources = Sources(List(sourcePath)) private val sources = Sources(List(sourcePath))
private val prefix = RemoteKey("prefix") private val prefix = RemoteKey("prefix")
def getMatchesByKey( def getMatchesByKey(
status: (Option[HashModified], Set[(MD5Hash, KeyModified)])) status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
@ -41,19 +41,19 @@ class S3MetaDataEnricherSuite extends FunSpec {
sources, sources,
prefix) prefix)
theRemoteKey = theFile.remoteKey theRemoteKey = theFile.remoteKey
s3 = S3ObjectsData( remoteObjects = RemoteObjects(
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))), byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified)) byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
) )
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified) theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
} yield (theFile, theRemoteMetadata, s3) } yield (theFile, theRemoteMetadata, remoteObjects)
it("generates valid metadata") { it("generates valid metadata") {
env.map({ env.map({
case (theFile, theRemoteMetadata, s3) => { case (theFile, theRemoteMetadata, remoteObjects) => {
val expected = S3MetaData(theFile, val expected = MatchedMetadata(theFile,
matchByHash = Set(theRemoteMetadata), matchByHash = Set(theRemoteMetadata),
matchByKey = Some(theRemoteMetadata)) matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3) val result = getMetadata(theFile, remoteObjects)
assertResult(expected)(result) assertResult(expected)(result)
} }
}) })
@ -69,19 +69,19 @@ class S3MetaDataEnricherSuite extends FunSpec {
sources, sources,
prefix) prefix)
theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix) theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix)
s3: S3ObjectsData = S3ObjectsData( remoteObjects = RemoteObjects(
byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))), byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
byKey = Map(theRemoteKey -> HashModified(theHash, lastModified)) byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
) )
theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified) theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
} yield (theFile, theRemoteMetadata, s3) } yield (theFile, theRemoteMetadata, remoteObjects)
it("generates valid metadata") { it("generates valid metadata") {
env.map({ env.map({
case (theFile, theRemoteMetadata, s3) => { case (theFile, theRemoteMetadata, remoteObjects) => {
val expected = S3MetaData(theFile, val expected = MatchedMetadata(theFile,
matchByHash = Set(theRemoteMetadata), matchByHash = Set(theRemoteMetadata),
matchByKey = Some(theRemoteMetadata)) matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3) val result = getMetadata(theFile, remoteObjects)
assertResult(expected)(result) assertResult(expected)(result)
} }
}) })
@ -97,7 +97,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
sources, sources,
prefix) prefix)
otherRemoteKey = RemoteKey("other-key") otherRemoteKey = RemoteKey("other-key")
s3: S3ObjectsData = S3ObjectsData( remoteObjects = RemoteObjects(
byHash = byHash =
Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))), Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified)) byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
@ -105,14 +105,15 @@ class S3MetaDataEnricherSuite extends FunSpec {
otherRemoteMetadata = RemoteMetaData(otherRemoteKey, otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
theHash, theHash,
lastModified) lastModified)
} yield (theFile, otherRemoteMetadata, s3) } yield (theFile, otherRemoteMetadata, remoteObjects)
it("generates valid metadata") { it("generates valid metadata") {
env.map({ env.map({
case (theFile, otherRemoteMetadata, s3) => { case (theFile, otherRemoteMetadata, remoteObjects) => {
val expected = S3MetaData(theFile, val expected = MatchedMetadata(theFile,
matchByHash = Set(otherRemoteMetadata), matchByHash =
matchByKey = None) Set(otherRemoteMetadata),
val result = getMetadata(theFile, s3) matchByKey = None)
val result = getMetadata(theFile, remoteObjects)
assertResult(expected)(result) assertResult(expected)(result)
} }
}) })
@ -127,14 +128,16 @@ class S3MetaDataEnricherSuite extends FunSpec {
sourcePath, sourcePath,
sources, sources,
prefix) prefix)
s3: S3ObjectsData = S3ObjectsData() remoteObjects = RemoteObjects()
} yield (theFile, s3) } yield (theFile, remoteObjects)
it("generates valid metadata") { it("generates valid metadata") {
env.map({ env.map({
case (theFile, s3) => { case (theFile, remoteObjects) => {
val expected = val expected =
S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None) MatchedMetadata(theFile,
val result = getMetadata(theFile, s3) matchByHash = Set.empty,
matchByKey = None)
val result = getMetadata(theFile, remoteObjects)
assertResult(expected)(result) assertResult(expected)(result)
} }
}) })
@ -152,7 +155,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
theRemoteKey = theFile.remoteKey theRemoteKey = theFile.remoteKey
oldHash = MD5Hash("old-hash") oldHash = MD5Hash("old-hash")
otherRemoteKey = RemoteKey.resolve("other-key")(prefix) otherRemoteKey = RemoteKey.resolve("other-key")(prefix)
s3: S3ObjectsData = S3ObjectsData( remoteObjects = RemoteObjects(
byHash = byHash =
Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)), Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
theHash -> Set(KeyModified(otherRemoteKey, lastModified))), theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
@ -165,14 +168,18 @@ class S3MetaDataEnricherSuite extends FunSpec {
otherRemoteMetadata = RemoteMetaData(otherRemoteKey, otherRemoteMetadata = RemoteMetaData(otherRemoteKey,
theHash, theHash,
lastModified) lastModified)
} yield (theFile, theRemoteMetadata, otherRemoteMetadata, s3) } yield (theFile, theRemoteMetadata, otherRemoteMetadata, remoteObjects)
it("generates valid metadata") { it("generates valid metadata") {
env.map({ env.map({
case (theFile, theRemoteMetadata, otherRemoteMetadata, s3) => { case (theFile,
val expected = S3MetaData(theFile, theRemoteMetadata,
matchByHash = Set(otherRemoteMetadata), otherRemoteMetadata,
matchByKey = Some(theRemoteMetadata)) remoteObjects) => {
val result = getMetadata(theFile, s3) val expected = MatchedMetadata(theFile,
matchByHash =
Set(otherRemoteMetadata),
matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, remoteObjects)
assertResult(expected)(result) assertResult(expected)(result)
} }
}) })
@ -189,7 +196,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
prefix) prefix)
theRemoteKey = theFile.remoteKey theRemoteKey = theFile.remoteKey
oldHash = MD5Hash("old-hash") oldHash = MD5Hash("old-hash")
s3: S3ObjectsData = S3ObjectsData( remoteObjects = RemoteObjects(
byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)), byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
theHash -> Set.empty), theHash -> Set.empty),
byKey = Map( byKey = Map(
@ -197,14 +204,14 @@ class S3MetaDataEnricherSuite extends FunSpec {
) )
) )
theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified) theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
} yield (theFile, theRemoteMetadata, s3) } yield (theFile, theRemoteMetadata, remoteObjects)
it("generates valid metadata") { it("generates valid metadata") {
env.map({ env.map({
case (theFile, theRemoteMetadata, s3) => { case (theFile, theRemoteMetadata, remoteObjects) => {
val expected = S3MetaData(theFile, val expected = MatchedMetadata(theFile,
matchByHash = Set.empty, matchByHash = Set.empty,
matchByKey = Some(theRemoteMetadata)) matchByKey = Some(theRemoteMetadata))
val result = getMetadata(theFile, s3) val result = getMetadata(theFile, remoteObjects)
assertResult(expected)(result) assertResult(expected)(result)
} }
}) })
@ -237,7 +244,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
sources, sources,
prefix) prefix)
lastModified = LastModified(Instant.now) lastModified = LastModified(Instant.now)
s3ObjectsData = S3ObjectsData( remoteObjects = RemoteObjects(
byHash = Map( byHash = Map(
hash -> Set(KeyModified(key, lastModified), hash -> Set(KeyModified(key, lastModified),
KeyModified(keyOtherKey.remoteKey, lastModified)), KeyModified(keyOtherKey.remoteKey, lastModified)),
@ -249,17 +256,17 @@ class S3MetaDataEnricherSuite extends FunSpec {
keyDiffHash.remoteKey -> HashModified(diffHash, lastModified) keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
) )
) )
} yield (s3ObjectsData, localFile, keyDiffHash, diffHash) } yield (remoteObjects, localFile, keyDiffHash, diffHash)
def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) = { def invoke(localFile: LocalFile, s3ObjectsData: RemoteObjects) = {
getS3Status(localFile, s3ObjectsData) getS3Status(localFile, s3ObjectsData)
} }
describe("when remote key exists") { describe("when remote key exists") {
it("should return a result for matching key") { it("should return a result for matching key") {
env.map({ env.map({
case (s3ObjectsData, localFile: LocalFile, _, _) => case (remoteObjects, localFile: LocalFile, _, _) =>
val result = getMatchesByKey(invoke(localFile, s3ObjectsData)) val result = getMatchesByKey(invoke(localFile, remoteObjects))
assert(result.contains(HashModified(hash, lastModified))) assert(result.contains(HashModified(hash, lastModified)))
}) })
} }
@ -275,10 +282,10 @@ class S3MetaDataEnricherSuite extends FunSpec {
} yield (localFile) } yield (localFile)
it("should return no matches by key") { it("should return no matches by key") {
env.map({ env.map({
case (s3ObjectsData, _, _, _) => { case (remoteObjects, _, _, _) => {
env2.map({ env2.map({
case (localFile) => { case (localFile) => {
val result = getMatchesByKey(invoke(localFile, s3ObjectsData)) val result = getMatchesByKey(invoke(localFile, remoteObjects))
assert(result.isEmpty) assert(result.isEmpty)
} }
}) })
@ -287,10 +294,10 @@ class S3MetaDataEnricherSuite extends FunSpec {
} }
it("should return no matches by hash") { it("should return no matches by hash") {
env.map({ env.map({
case (s3ObjectsData, _, _, _) => { case (remoteObjects, _, _, _) => {
env2.map({ env2.map({
case (localFile) => { case (localFile) => {
val result = getMatchesByHash(invoke(localFile, s3ObjectsData)) val result = getMatchesByHash(invoke(localFile, remoteObjects))
assert(result.isEmpty) assert(result.isEmpty)
} }
}) })
@ -301,13 +308,13 @@ class S3MetaDataEnricherSuite extends FunSpec {
describe("when remote key exists and no others match hash") { describe("when remote key exists and no others match hash") {
env.map({ env.map({
case (s3ObjectsData, _, keyDiffHash, diffHash) => { case (remoteObjects, _, keyDiffHash, diffHash) => {
it("should return match by key") { it("should return match by key") {
val result = getMatchesByKey(invoke(keyDiffHash, s3ObjectsData)) val result = getMatchesByKey(invoke(keyDiffHash, remoteObjects))
assert(result.contains(HashModified(diffHash, lastModified))) assert(result.contains(HashModified(diffHash, lastModified)))
} }
it("should return only itself in match by hash") { it("should return only itself in match by hash") {
val result = getMatchesByHash(invoke(keyDiffHash, s3ObjectsData)) val result = getMatchesByHash(invoke(keyDiffHash, remoteObjects))
assert( assert(
result.equals(Set( result.equals(Set(
(diffHash, KeyModified(keyDiffHash.remoteKey, lastModified))))) (diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))

48
core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala
View file

@ -22,7 +22,7 @@ import zio.{DefaultRuntime, Task, UIO}
class PlanBuilderTest extends FreeSpec with TemporaryFolder { class PlanBuilderTest extends FreeSpec with TemporaryFolder {
private val lastModified: LastModified = LastModified() private val lastModified: LastModified = LastModified()
private val emptyS3ObjectData = S3ObjectsData() private val emptyRemoteObjects = RemoteObjects()
"create a plan" - { "create a plan" - {
@ -46,7 +46,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
Right(List(toUpload(remoteKey, hash, source, file))) Right(List(toUpload(remoteKey, hash, source, file)))
val result = val result =
invoke(options(source), invoke(options(source),
UIO.succeed(emptyS3ObjectData), UIO.succeed(emptyRemoteObjects),
UIO.succeed(Map(file.toPath -> file))) UIO.succeed(Map(file.toPath -> file)))
assertResult(expected)(result) assertResult(expected)(result)
}) })
@ -62,14 +62,14 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
val aHash = md5Hash(aFile) val aHash = md5Hash(aFile)
val anOtherKey = RemoteKey("other") val anOtherKey = RemoteKey("other")
val expected = Right(List(toCopy(anOtherKey, aHash, remoteKey))) val expected = Right(List(toCopy(anOtherKey, aHash, remoteKey)))
val s3ObjectsData = S3ObjectsData( val remoteObjects = RemoteObjects(
byHash = byHash =
Map(aHash -> Set(KeyModified(anOtherKey, lastModified))), Map(aHash -> Set(KeyModified(anOtherKey, lastModified))),
byKey = Map(anOtherKey -> HashModified(aHash, lastModified)) byKey = Map(anOtherKey -> HashModified(aHash, lastModified))
) )
val result = val result =
invoke(options(source), invoke(options(source),
UIO.succeed(s3ObjectsData), UIO.succeed(remoteObjects),
UIO.succeed(Map(aFile.toPath -> aFile, UIO.succeed(Map(aFile.toPath -> aFile,
anOtherFile.toPath -> anOtherFile))) anOtherFile.toPath -> anOtherFile)))
assertResult(expected)(result) assertResult(expected)(result)
@ -85,14 +85,14 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
val hash = md5Hash(file) val hash = md5Hash(file)
// DoNothing actions should have been filtered out of the plan // DoNothing actions should have been filtered out of the plan
val expected = Right(List()) val expected = Right(List())
val s3ObjectsData = S3ObjectsData( val remoteObjects = RemoteObjects(
byHash = byHash =
Map(hash -> Set(KeyModified(remoteKey, lastModified))), Map(hash -> Set(KeyModified(remoteKey, lastModified))),
byKey = Map(remoteKey -> HashModified(hash, lastModified)) byKey = Map(remoteKey -> HashModified(hash, lastModified))
) )
val result = val result =
invoke(options(source), invoke(options(source),
UIO.succeed(s3ObjectsData), UIO.succeed(remoteObjects),
UIO.succeed(Map(file.toPath -> file))) UIO.succeed(Map(file.toPath -> file)))
assertResult(expected)(result) assertResult(expected)(result)
}) })
@ -107,7 +107,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
val originalHash = MD5Hash("original-file-content") val originalHash = MD5Hash("original-file-content")
val expected = val expected =
Right(List(toUpload(remoteKey, currentHash, source, file))) Right(List(toUpload(remoteKey, currentHash, source, file)))
val s3ObjectsData = S3ObjectsData( val remoteObjects = RemoteObjects(
byHash = Map(originalHash -> Set( byHash = Map(originalHash -> Set(
KeyModified(remoteKey, lastModified))), KeyModified(remoteKey, lastModified))),
byKey = byKey =
@ -115,7 +115,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
) )
val result = val result =
invoke(options(source), invoke(options(source),
UIO.succeed(s3ObjectsData), UIO.succeed(remoteObjects),
UIO.succeed(Map(file.toPath -> file))) UIO.succeed(Map(file.toPath -> file)))
assertResult(expected)(result) assertResult(expected)(result)
}) })
@ -128,14 +128,14 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
val hash = md5Hash(file) val hash = md5Hash(file)
val sourceKey = RemoteKey("other-key") val sourceKey = RemoteKey("other-key")
val expected = Right(List(toCopy(sourceKey, hash, remoteKey))) val expected = Right(List(toCopy(sourceKey, hash, remoteKey)))
val s3ObjectsData = S3ObjectsData( val remoteObjects = RemoteObjects(
byHash = byHash =
Map(hash -> Set(KeyModified(sourceKey, lastModified))), Map(hash -> Set(KeyModified(sourceKey, lastModified))),
byKey = Map() byKey = Map()
) )
val result = val result =
invoke(options(source), invoke(options(source),
UIO.succeed(s3ObjectsData), UIO.succeed(remoteObjects),
UIO.succeed(Map(file.toPath -> file))) UIO.succeed(Map(file.toPath -> file)))
assertResult(expected)(result) assertResult(expected)(result)
}) })
@ -154,13 +154,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
val hash = md5Hash(file) val hash = md5Hash(file)
// DoNothing actions should have been filtered out of the plan // DoNothing actions should have been filtered out of the plan
val expected = Right(List()) val expected = Right(List())
val s3ObjectsData = S3ObjectsData( val remoteObjects = RemoteObjects(
byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))), byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))),
byKey = Map(remoteKey -> HashModified(hash, lastModified)) byKey = Map(remoteKey -> HashModified(hash, lastModified))
) )
val result = val result =
invoke(options(source), invoke(options(source),
UIO.succeed(s3ObjectsData), UIO.succeed(remoteObjects),
UIO.succeed(Map(file.toPath -> file))) UIO.succeed(Map(file.toPath -> file)))
assertResult(expected)(result) assertResult(expected)(result)
}) })
@ -171,13 +171,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
withDirectory(source => { withDirectory(source => {
val hash = MD5Hash("file-content") val hash = MD5Hash("file-content")
val expected = Right(List(toDelete(remoteKey))) val expected = Right(List(toDelete(remoteKey)))
val s3ObjectsData = S3ObjectsData( val remoteObjects = RemoteObjects(
byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))), byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))),
byKey = Map(remoteKey -> HashModified(hash, lastModified)) byKey = Map(remoteKey -> HashModified(hash, lastModified))
) )
val result = val result =
invoke(options(source), invoke(options(source),
UIO.succeed(s3ObjectsData), UIO.succeed(remoteObjects),
UIO.succeed(Map.empty)) UIO.succeed(Map.empty))
assertResult(expected)(result) assertResult(expected)(result)
}) })
@ -215,7 +215,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
val result = val result =
invoke( invoke(
options(firstSource)(secondSource), options(firstSource)(secondSource),
UIO.succeed(emptyS3ObjectData), UIO.succeed(emptyRemoteObjects),
UIO.succeed( UIO.succeed(
Map(fileInFirstSource.toPath -> fileInFirstSource, Map(fileInFirstSource.toPath -> fileInFirstSource,
fileInSecondSource.toPath -> fileInSecondSource)) fileInSecondSource.toPath -> fileInSecondSource))
@ -240,7 +240,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
val result = val result =
invoke( invoke(
options(firstSource)(secondSource), options(firstSource)(secondSource),
UIO.succeed(emptyS3ObjectData), UIO.succeed(emptyRemoteObjects),
UIO.succeed( UIO.succeed(
Map(fileInFirstSource.toPath -> fileInFirstSource, Map(fileInFirstSource.toPath -> fileInFirstSource,
fileInSecondSource.toPath -> fileInSecondSource)) fileInSecondSource.toPath -> fileInSecondSource))
@ -258,13 +258,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
createFile(secondSource, filename2, "file-2-content") createFile(secondSource, filename2, "file-2-content")
val hash2 = md5Hash(fileInSecondSource) val hash2 = md5Hash(fileInSecondSource)
val expected = Right(List()) val expected = Right(List())
val s3ObjectData = S3ObjectsData( val remoteObjects = RemoteObjects(
byHash = byHash =
Map(hash2 -> Set(KeyModified(remoteKey2, lastModified))), Map(hash2 -> Set(KeyModified(remoteKey2, lastModified))),
byKey = Map(remoteKey2 -> HashModified(hash2, lastModified))) byKey = Map(remoteKey2 -> HashModified(hash2, lastModified)))
val result = val result =
invoke(options(firstSource)(secondSource), invoke(options(firstSource)(secondSource),
UIO.succeed(s3ObjectData), UIO.succeed(remoteObjects),
UIO.succeed( UIO.succeed(
Map(fileInSecondSource.toPath -> fileInSecondSource))) Map(fileInSecondSource.toPath -> fileInSecondSource)))
assertResult(expected)(result) assertResult(expected)(result)
@ -280,13 +280,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
val hash1 = md5Hash(fileInFirstSource) val hash1 = md5Hash(fileInFirstSource)
withDirectory(secondSource => { withDirectory(secondSource => {
val expected = Right(List()) val expected = Right(List())
val s3ObjectData = S3ObjectsData( val remoteObjects = RemoteObjects(
byHash = byHash =
Map(hash1 -> Set(KeyModified(remoteKey1, lastModified))), Map(hash1 -> Set(KeyModified(remoteKey1, lastModified))),
byKey = Map(remoteKey1 -> HashModified(hash1, lastModified))) byKey = Map(remoteKey1 -> HashModified(hash1, lastModified)))
val result = val result =
invoke(options(firstSource)(secondSource), invoke(options(firstSource)(secondSource),
UIO.succeed(s3ObjectData), UIO.succeed(remoteObjects),
UIO.succeed( UIO.succeed(
Map(fileInFirstSource.toPath -> fileInFirstSource))) Map(fileInFirstSource.toPath -> fileInFirstSource)))
assertResult(expected)(result) assertResult(expected)(result)
@ -299,11 +299,11 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
withDirectory(firstSource => { withDirectory(firstSource => {
withDirectory(secondSource => { withDirectory(secondSource => {
val expected = Right(List(toDelete(remoteKey1))) val expected = Right(List(toDelete(remoteKey1)))
val s3ObjectData = S3ObjectsData(byKey = val remoteObjects = RemoteObjects(byKey =
Map(remoteKey1 -> HashModified(MD5Hash(""), lastModified))) Map(remoteKey1 -> HashModified(MD5Hash(""), lastModified)))
val result = val result =
invoke(options(firstSource)(secondSource), invoke(options(firstSource)(secondSource),
UIO.succeed(s3ObjectData), UIO.succeed(remoteObjects),
UIO.succeed(Map.empty)) UIO.succeed(Map.empty))
assertResult(expected)(result) assertResult(expected)(result)
}) })
@ -354,13 +354,13 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
private def invoke( private def invoke(
configOptions: ConfigOptions, configOptions: ConfigOptions,
result: Task[S3ObjectsData], result: Task[RemoteObjects],
files: Task[Map[Path, File]] files: Task[Map[Path, File]]
) = { ) = {
type TestEnv = Storage with Console with Config with FileSystem with Hasher type TestEnv = Storage with Console with Config with FileSystem with Hasher
val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live val testEnv: TestEnv = new Storage.Test with Console.Test with Config.Live
with FileSystem.Live with Hasher.Live { with FileSystem.Live with Hasher.Live {
override def listResult: Task[S3ObjectsData] = result override def listResult: Task[RemoteObjects] = result
override def uploadResult: UIO[StorageQueueEvent] = override def uploadResult: UIO[StorageQueueEvent] =
Task.die(new NotImplementedError) Task.die(new NotImplementedError)
override def copyResult: UIO[StorageQueueEvent] = override def copyResult: UIO[StorageQueueEvent] =

2
domain/src/main/scala/net/kemitix/thorp/domain/S3MetaData.scala → domain/src/main/scala/net/kemitix/thorp/domain/MatchedMetadata.scala
View file

@ -1,7 +1,7 @@
package net.kemitix.thorp.domain package net.kemitix.thorp.domain
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key // For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
final case class S3MetaData( final case class MatchedMetadata(
localFile: LocalFile, localFile: LocalFile,
matchByHash: Set[RemoteMetaData], matchByHash: Set[RemoteMetaData],
matchByKey: Option[RemoteMetaData] matchByKey: Option[RemoteMetaData]

2
domain/src/main/scala/net/kemitix/thorp/domain/S3ObjectsData.scala → domain/src/main/scala/net/kemitix/thorp/domain/RemoteObjects.scala
View file

@ -3,7 +3,7 @@ package net.kemitix.thorp.domain
/** /**
* A list of objects and their MD5 hash values. * A list of objects and their MD5 hash values.
*/ */
final case class S3ObjectsData( final case class RemoteObjects(
byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty, byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty,
byKey: Map[RemoteKey, HashModified] = Map.empty byKey: Map[RemoteKey, HashModified] = Map.empty
) )

10
storage-api/src/main/scala/net/kemitix/thorp/storage/api/Storage.scala
View file

@ -15,7 +15,7 @@ object Storage {
def listObjects( def listObjects(
bucket: Bucket, bucket: Bucket,
prefix: RemoteKey prefix: RemoteKey
): RIO[Storage with Console, S3ObjectsData] ): RIO[Storage with Console, RemoteObjects]
def upload( def upload(
localFile: LocalFile, localFile: LocalFile,
@ -40,7 +40,7 @@ object Storage {
trait Test extends Storage { trait Test extends Storage {
def listResult: Task[S3ObjectsData] def listResult: Task[RemoteObjects]
def uploadResult: UIO[StorageQueueEvent] def uploadResult: UIO[StorageQueueEvent]
def copyResult: UIO[StorageQueueEvent] def copyResult: UIO[StorageQueueEvent]
def deleteResult: UIO[StorageQueueEvent] def deleteResult: UIO[StorageQueueEvent]
@ -50,7 +50,7 @@ object Storage {
override def listObjects( override def listObjects(
bucket: Bucket, bucket: Bucket,
prefix: RemoteKey): RIO[Storage with Console, S3ObjectsData] = prefix: RemoteKey): RIO[Storage with Console, RemoteObjects] =
listResult listResult
override def upload( override def upload(
@ -78,7 +78,7 @@ object Storage {
} }
object Test extends Test { object Test extends Test {
override def listResult: Task[S3ObjectsData] = override def listResult: Task[RemoteObjects] =
Task.die(new NotImplementedError) Task.die(new NotImplementedError)
override def uploadResult: UIO[StorageQueueEvent] = override def uploadResult: UIO[StorageQueueEvent] =
Task.die(new NotImplementedError) Task.die(new NotImplementedError)
@ -91,7 +91,7 @@ object Storage {
} }
final def list(bucket: Bucket, final def list(bucket: Bucket,
prefix: RemoteKey): RIO[Storage with Console, S3ObjectsData] = prefix: RemoteKey): RIO[Storage with Console, RemoteObjects] =
ZIO.accessM(_.storage listObjects (bucket, prefix)) ZIO.accessM(_.storage listObjects (bucket, prefix))
final def upload( final def upload(

6
storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Lister.scala
View file

@ -6,7 +6,7 @@ import com.amazonaws.services.s3.model.{
S3ObjectSummary S3ObjectSummary
} }
import net.kemitix.thorp.console._ import net.kemitix.thorp.console._
import net.kemitix.thorp.domain.{Bucket, RemoteKey, S3ObjectsData} import net.kemitix.thorp.domain.{Bucket, RemoteKey, RemoteObjects}
import net.kemitix.thorp.storage.aws.S3ObjectsByHash.byHash import net.kemitix.thorp.storage.aws.S3ObjectsByHash.byHash
import net.kemitix.thorp.storage.aws.S3ObjectsByKey.byKey import net.kemitix.thorp.storage.aws.S3ObjectsByKey.byKey
import zio.{Task, RIO} import zio.{Task, RIO}
@ -21,7 +21,7 @@ trait Lister {
def listObjects(amazonS3: AmazonS3.Client)( def listObjects(amazonS3: AmazonS3.Client)(
bucket: Bucket, bucket: Bucket,
prefix: RemoteKey prefix: RemoteKey
): RIO[Console, S3ObjectsData] = { ): RIO[Console, RemoteObjects] = {
def request = def request =
new ListObjectsV2Request() new ListObjectsV2Request()
@ -48,7 +48,7 @@ trait Lister {
fetch(request) fetch(request)
.map(summaries => { .map(summaries => {
S3ObjectsData(byHash(summaries), byKey(summaries)) RemoteObjects(byHash(summaries), byKey(summaries))
}) })
} }

2
storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3Storage.scala
View file

@ -20,7 +20,7 @@ object S3Storage {
AmazonTransferManager(TransferManagerBuilder.defaultTransferManager) AmazonTransferManager(TransferManagerBuilder.defaultTransferManager)
override def listObjects(bucket: Bucket, override def listObjects(bucket: Bucket,
prefix: RemoteKey): RIO[Console, S3ObjectsData] = prefix: RemoteKey): RIO[Console, RemoteObjects] =
Lister.listObjects(client)(bucket, prefix) Lister.listObjects(client)(bucket, prefix)
override def upload( override def upload(

2
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/AmazonS3ClientTestFixture.scala
View file

@ -26,7 +26,7 @@ trait AmazonS3ClientTestFixture extends MockFactory {
override def listObjects( override def listObjects(
bucket: Bucket, bucket: Bucket,
prefix: RemoteKey prefix: RemoteKey
): RIO[Console, S3ObjectsData] = ): RIO[Console, RemoteObjects] =
Lister.listObjects(client)(bucket, prefix) Lister.listObjects(client)(bucket, prefix)
override def upload( override def upload(

4
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala
View file

@ -34,7 +34,7 @@ class ListerTest extends FreeSpec {
RemoteKey(key) -> HashModified(MD5Hash(etag), RemoteKey(key) -> HashModified(MD5Hash(etag),
LastModified(nowInstant)) LastModified(nowInstant))
) )
val expected = Right(S3ObjectsData(expectedHashMap, expectedKeyMap)) val expected = Right(RemoteObjects(expectedHashMap, expectedKeyMap))
new AmazonS3ClientTestFixture { new AmazonS3ClientTestFixture {
(fixture.amazonS3Client.listObjectsV2 _) (fixture.amazonS3Client.listObjectsV2 _)
.when() .when()
@ -65,7 +65,7 @@ class ListerTest extends FreeSpec {
RemoteKey(key2) -> HashModified(MD5Hash(etag2), RemoteKey(key2) -> HashModified(MD5Hash(etag2),
LastModified(nowInstant)) LastModified(nowInstant))
) )
val expected = Right(S3ObjectsData(expectedHashMap, expectedKeyMap)) val expected = Right(RemoteObjects(expectedHashMap, expectedKeyMap))
new AmazonS3ClientTestFixture { new AmazonS3ClientTestFixture {
(fixture.amazonS3Client.listObjectsV2 _) (fixture.amazonS3Client.listObjectsV2 _)
.when() .when()

4
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala
View file

@ -38,7 +38,7 @@ class StorageServiceSuite extends FunSpec with MockFactory {
sources, sources,
prefix) prefix)
lastModified = LastModified(Instant.now) lastModified = LastModified(Instant.now)
s3ObjectsData = S3ObjectsData( s3ObjectsData = RemoteObjects(
byHash = Map( byHash = Map(
hash -> Set(KeyModified(key, lastModified), hash -> Set(KeyModified(key, lastModified),
KeyModified(keyOtherKey.remoteKey, lastModified)), KeyModified(keyOtherKey.remoteKey, lastModified)),
@ -59,7 +59,7 @@ class StorageServiceSuite extends FunSpec with MockFactory {
diffHash, diffHash,
key) key)
def invoke(localFile: LocalFile, s3ObjectsData: S3ObjectsData) = def invoke(localFile: LocalFile, s3ObjectsData: RemoteObjects) =
S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData) S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData)
def getMatchesByKey( def getMatchesByKey(