kemitix/thorp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[core] Extract PlanExecutor

Browse source
This commit is contained in:
Paul Campbell 2019-08-08 20:37:25 +01:00
parent 242804294d
commit 656f3fec30
5 changed files with 77 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
cli/src/main/scala/net/kemitix/thorp/cli/Program.scala
View file

@ -25,9 +25,10 @@ trait Program {
private def execute =
for {
plan <- PlanBuilder.createPlan
archive <- UnversionedMirrorArchive.default(plan.syncTotals)
events <- applyPlan(archive, plan)
_ <- SyncLogging.logRunStart
syncPlan <- PlanBuilder.createPlan
archive <- UnversionedMirrorArchive.default(syncPlan.syncTotals)
events <- PlanExecutor.executePlan(archive, syncPlan)
_ <- SyncLogging.logRunFinished(events)
} yield ()
@ -40,27 +41,6 @@ trait Program {
ZIO.foreach_(errors)(error => Console.putStrLn(s"- $error"))
}
private def applyPlan(archive: ThorpArchive, syncPlan: SyncPlan) =
ZIO
.foldLeft(sequenceActions(syncPlan.actions))(
EventQueue(Stream.empty, syncPlan.syncTotals.totalSizeBytes))(
applyAction(archive)(_, _))
.map(_.events)
private def sequenceActions(actions: Stream[Action]) =
actions.zipWithIndex
.map({ case (a, i) => SequencedAction(a, i) })
private def applyAction(archive: ThorpArchive)(
queue: EventQueue,
action: SequencedAction
) = {
val remainingBytes = queue.bytesInQueue - action.action.size
archive
.update(action, remainingBytes)
.map(events => EventQueue(queue.events ++ Stream(events), remainingBytes))
}
}
object Program extends Program

59
core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala
View file

@ -13,20 +13,14 @@ object PlanBuilder {
def createPlan
: RIO[Storage with Console with Config with FileSystem with Hasher,
SyncPlan] =
SyncLogging.logRunStart *> buildPlan
private def buildPlan =
gatherMetadata >>= assemblePlan
private def gatherMetadata =
fetchRemoteData &&& findLocalFiles
SyncPlan] = (fetchRemoteData <&> findLocalFiles) >>= assemblePlan
private def fetchRemoteData =
for {
bucket <- Config.bucket
prefix <- Config.prefix
objects <- Storage.list(bucket, prefix)
_ <- Console.putStrLn(s"Found ${objects.byKey.size} remote objects")
} yield objects
private def assemblePlan(metadata: (RemoteObjects, LocalFiles)) =
@ -34,17 +28,17 @@ object PlanBuilder {
case (remoteObjects, localData) =>
createActions(remoteObjects, localData.localFiles)
.map(_.filter(doesSomething).sortBy(SequencePlan.order))
.map(
SyncPlan
.create(_,
SyncTotals
.create(localData.count, localData.totalSizeBytes, 0L)))
.map(syncPlan(localData))
}
private def createActions(
remoteObjects: RemoteObjects,
localFiles: Stream[LocalFile]
) =
private def syncPlan(localData: LocalFiles): Stream[Action] => SyncPlan =
SyncPlan.create(_, syncTotal(localData))
private def syncTotal(localData: LocalFiles): SyncTotals =
SyncTotals.create(localData.count, localData.totalSizeBytes, 0L)
private def createActions(remoteObjects: RemoteObjects,
localFiles: Stream[LocalFile]) =
for {
fileActions <- actionsForLocalFiles(remoteObjects, localFiles)
remoteActions <- actionsForRemoteKeys(remoteObjects.byKey.keys)
@ -55,28 +49,30 @@ object PlanBuilder {
case _ => true
}
private def actionsForLocalFiles(
remoteObjects: RemoteObjects,
localFiles: Stream[LocalFile]
) =
ZIO.foldLeft(localFiles)(Stream.empty[Action])((acc, localFile) =>
createActionsFromLocalFile(remoteObjects, acc, localFile).map(_ #::: acc))
private def actionsForLocalFiles(remoteObjects: RemoteObjects,
localFiles: Stream[LocalFile]) =
ZIO.foldLeft(localFiles)(Stream.empty[Action])(
(acc, localFile) =>
createActionsFromLocalFile(remoteObjects, acc, localFile)
.map(_ #::: acc)
)
private def createActionsFromLocalFile(
remoteObjects: RemoteObjects,
private def createActionsFromLocalFile(remoteObjects: RemoteObjects,
previousActions: Stream[Action],
localFile: LocalFile
) =
localFile: LocalFile) =
ActionGenerator.createActions(
S3MetaDataEnricher.getMetadata(localFile, remoteObjects),
previousActions)
previousActions
)
private def actionsForRemoteKeys(remoteKeys: Iterable[RemoteKey]) =
ZIO.foldLeft(remoteKeys)(Stream.empty[Action])((acc, remoteKey) =>
createActionFromRemoteKey(remoteKey).map(_ #:: acc))
ZIO.foldLeft(remoteKeys)(Stream.empty[Action])(
(acc, remoteKey) => createActionFromRemoteKey(remoteKey).map(_ #:: acc)
)
private def createActionFromRemoteKey(
remoteKey: RemoteKey): ZIO[FileSystem with Config, Throwable, Action] =
remoteKey: RemoteKey
): ZIO[FileSystem with Config, Throwable, Action] =
for {
bucket <- Config.bucket
prefix <- Config.prefix
@ -93,6 +89,7 @@ object PlanBuilder {
for {
sources <- Config.sources
found <- ZIO.foreach(sources.paths)(LocalFileStream.findFiles)
_ <- Console.putStrLn(s"Found ${found.flatMap(_.localFiles).size} files")
} yield LocalFiles.reduce(found.toStream)
}

39
core/src/main/scala/net/kemitix/thorp/core/PlanExecutor.scala Normal file
View file

@ -0,0 +1,39 @@
package net.kemitix.thorp.core
import net.kemitix.thorp.config.Config
import net.kemitix.thorp.console.Console
import net.kemitix.thorp.domain.StorageQueueEvent
import net.kemitix.thorp.storage.api.Storage
import zio.{Ref, ZIO}
trait PlanExecutor {
def executePlan(
archive: ThorpArchive,
syncPlan: SyncPlan
): ZIO[Storage with Config with Console,
Throwable,
Seq[
StorageQueueEvent
]] =
for {
actionCounter <- Ref.make(0)
bytesCounter <- Ref.make(0L)
events <- ZIO.foreach(syncPlan.actions) {
updateArchive(archive, actionCounter, bytesCounter)
}
} yield events
private def updateArchive(archive: ThorpArchive,
actionCounterRef: Ref[Int],
bytesCounterRef: Ref[Long])(action: Action) =
for {
actionCounter <- actionCounterRef.update(_ + 1)
bytesCounter <- bytesCounterRef.update(_ + action.size)
event <- archive.update(SequencedAction(action, actionCounter),
bytesCounter)
} yield event
}
object PlanExecutor extends PlanExecutor

2
core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala
View file

@ -30,7 +30,7 @@ trait SyncLogging {
} yield ()
def logRunFinished(
actions: Stream[StorageQueueEvent]
actions: Seq[StorageQueueEvent]
): ZIO[Console, Nothing, Unit] = {
val counters = actions.foldLeft(Counters.empty)(countActivities)
Console.putStrLn(eraseToEndOfScreen) *>

9
core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala
View file

@ -363,23 +363,20 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
override def shutdownResult: UIO[StorageQueueEvent] =
Task.die(new NotImplementedError)
}
def testProgram =
for {
config <- ConfigurationBuilder.buildConfig(configOptions)
_ <- Config.set(config)
plan <- PlanBuilder.createPlan
} yield plan
new DefaultRuntime {}
.unsafeRunSync(testProgram.provide(testEnv))
.toEither
.map(convertResult)
}
private def convertResult
: SyncPlan => List[(String, String, String, String, String)] =
_.actions.toList.map({
private def convertResult(plan: SyncPlan) =
plan.actions.map({
case ToUpload(_, lf, _) =>
("upload",
lf.remoteKey.key,
@ -391,6 +388,6 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
("copy", sourceKey.key, MD5Hash.hash(hash), targetKey.key, "")
case DoNothing(_, remoteKey, _) =>
("do-nothing", remoteKey.key, "", "", "")
case x => ("other", "", "", "", "")
case _ => ("other", "", "", "", "")
})
}