kemitix/thorp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[S3MetaDataEnricher,ActionSubmitter] return streams

Browse source 
Help to perpetuate the map/flatMap structure within for-comprehension
in Sync's run method.

Added DoNothing and DoNothingS3Action
This commit is contained in:
Paul Campbell 2019-05-24 07:46:18 +01:00
parent bffc6c032c
commit fa31882e51
6 changed files with 49 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/main/scala/net/kemitix/s3thorp/Action.scala
View file

@ -1,6 +1,7 @@
package net.kemitix.s3thorp
sealed trait Action
case class DoNothing(remoteKey: RemoteKey) extends Action
case class ToUpload(localFile: LocalFile) extends Action
case class ToCopy(sourceKey: RemoteKey,
hash: MD5Hash,

22
src/main/scala/net/kemitix/s3thorp/ActionGenerator.scala
View file

@ -8,9 +8,9 @@ trait ActionGenerator
s3MetaData match {
// #1 local exists, remote exists, remote matches - do nothing
case S3MetaData(localFile, _, Some(RemoteMetaData(_, remoteHash, _)))
case S3MetaData(localFile, _, Some(RemoteMetaData(remoteKey, remoteHash, _)))
if localFile.hash == remoteHash
=> Stream.empty
=> doNothing(remoteKey)
// #2 local exists, remote is missing, other matches - copy
case S3MetaData(localFile, otherMatches, None)
@ -33,16 +33,22 @@ trait ActionGenerator
if hashMatches.isEmpty
=> uploadFile(localFile)
case _ => Stream.empty
}
private def uploadFile(localFile: LocalFile) = Stream(ToUpload(localFile))
private def doNothing(remoteKey: RemoteKey) =
Stream(
DoNothing(remoteKey))
private def copyFile(localFile: LocalFile, matchByHash: Set[RemoteMetaData]): Stream[Action] =
Stream(ToCopy(
private def uploadFile(localFile: LocalFile) =
Stream(
ToUpload(localFile))
private def copyFile(localFile: LocalFile,
matchByHash: Set[RemoteMetaData]): Stream[Action] =
Stream(
ToCopy(
sourceKey = matchByHash.head.remoteKey,
hash = localFile.hash,
targetKey = localFile.remoteKey
))
targetKey = localFile.remoteKey))
}

7
src/main/scala/net/kemitix/s3thorp/ActionSubmitter.scala
View file

@ -8,7 +8,8 @@ trait ActionSubmitter
with Logging {
def submitAction(action: Action)
(implicit c: Config): IO[S3Action] = {
(implicit c: Config): Stream[IO[S3Action]] = {
Stream(
action match {
case ToUpload(file) =>
log4(s" Upload: ${file.relative}")
@ -19,6 +20,8 @@ trait ActionSubmitter
case ToDelete(remoteKey) =>
log4(s" Delete: $remoteKey")
delete(c.bucket, remoteKey)
}
case DoNothing(remoteKey) => IO {
DoNothingS3Action(remoteKey)}
})
}
}

4
src/main/scala/net/kemitix/s3thorp/S3Action.scala
View file

@ -9,6 +9,10 @@ sealed trait S3Action {
}
case class DoNothingS3Action(remoteKey: RemoteKey) extends S3Action {
override val order: Int = 0
}
case class CopyS3Action(remoteKey: RemoteKey) extends S3Action {
override val order: Int = 1
}

12
src/main/scala/net/kemitix/s3thorp/S3MetaDataEnricher.scala
View file

@ -8,14 +8,12 @@ trait S3MetaDataEnricher
def getMetadata(localFile: LocalFile)
(implicit c: Config,
s3ObjectsData: S3ObjectsData): S3MetaData = {
val (keyMatches: Option[HashModified], hashMatches: Set[KeyModified]) = getS3Status(localFile)
s3ObjectsData: S3ObjectsData): Stream[S3MetaData] = {
val (keyMatches, hashMatches) = getS3Status(localFile)
Stream(
S3MetaData(localFile,
matchByKey = keyMatches.map{kmAsRemoteMetaData(localFile.remoteKey)},
matchByHash = hashMatches.map(km => RemoteMetaData(km.key, localFile.hash, km.modified)))
matchByKey = keyMatches map { hm => RemoteMetaData(localFile.remoteKey, hm.hash, hm.modified) },
matchByHash = hashMatches map { km => RemoteMetaData(km.key, localFile.hash, km.modified) }))
}
private def kmAsRemoteMetaData(key: RemoteKey): HashModified => RemoteMetaData = hm => RemoteMetaData(key, hm.hash, hm.modified)
}

16
src/main/scala/net/kemitix/s3thorp/Sync.scala
View file

@ -1,7 +1,5 @@
package net.kemitix.s3thorp
import java.nio.file.Paths
import cats.effect.IO
import cats.implicits._
import net.kemitix.s3thorp.awssdk.{S3Client, S3ObjectsData}
@ -17,18 +15,18 @@ class Sync(s3Client: S3Client)
logRunStart(c).unsafeRunSync
listObjects(c.bucket, c.prefix)
.map { implicit s3ObjectsData => {
val actions = (for {
val actions = for {
file <- findFiles(c.source)
meta = getMetadata(file)
action <- createActions(meta)
ioS3Action = submitAction(action)
} yield ioS3Action).sequence
val sorted = sort(actions)
data <- getMetadata(file)
action <- createActions(data)
s3Action <- submitAction(action)
} yield s3Action
val sorted = sort(actions.sequence)
val list = sorted.unsafeRunSync.toList
val delActions = (for {
key <- s3ObjectsData.byKey.keys
if key.isMissingLocally
ioDelAction = submitAction(ToDelete(key))
ioDelAction <- submitAction(ToDelete(key))
} yield ioDelAction).toStream.sequence
val delList = delActions.unsafeRunSync.toList
logRunFinished(list ++ delList).unsafeRunSync