Merge config package object with Config object (#138)
* [config] merge config package object with Config object * [config] Only extract batchMode value where used Don't pull it out early then pass a boolean around
This commit is contained in:
parent
a93781007d
commit
e5de73e705
19 changed files with 139 additions and 138 deletions
|
@ -58,8 +58,8 @@ val zioDependencies = Seq(
|
|||
)
|
||||
|
||||
// cli -> thorp-lib -> storage-aws -> core -> storage-api -> console -> domain
|
||||
// core -> config -> domain
|
||||
// config -> filesystem
|
||||
// storage-api -> config -> domain
|
||||
// config -> filesystem
|
||||
|
||||
lazy val thorp = (project in file("."))
|
||||
.settings(commonSettings)
|
||||
|
@ -102,13 +102,13 @@ lazy val core = (project in file("core"))
|
|||
.settings(testDependencies)
|
||||
.dependsOn(`storage-api`)
|
||||
.dependsOn(domain % "compile->compile;test->test")
|
||||
.dependsOn(config)
|
||||
|
||||
lazy val `storage-api` = (project in file("storage-api"))
|
||||
.settings(commonSettings)
|
||||
.settings(zioDependencies)
|
||||
.settings(assemblyJarName in assembly := "storage-api.jar")
|
||||
.dependsOn(console)
|
||||
.dependsOn(config)
|
||||
|
||||
lazy val console = (project in file("console"))
|
||||
.settings(commonSettings)
|
||||
|
|
|
@ -16,7 +16,7 @@ trait Program {
|
|||
for {
|
||||
cli <- CliArgs.parse(args)
|
||||
config <- ConfigurationBuilder.buildConfig(cli)
|
||||
_ <- setConfiguration(config)
|
||||
_ <- Config.set(config)
|
||||
_ <- ZIO.when(showVersion(cli))(Console.putStrLn(version))
|
||||
_ <- ZIO.when(!showVersion(cli))(execute.catchAll(handleErrors))
|
||||
} yield ()
|
||||
|
@ -27,11 +27,10 @@ trait Program {
|
|||
|
||||
private def execute = {
|
||||
for {
|
||||
plan <- PlanBuilder.createPlan(defaultHashService)
|
||||
batchMode <- isBatchMode
|
||||
archive <- UnversionedMirrorArchive.default(batchMode, plan.syncTotals)
|
||||
events <- applyPlan(archive, plan)
|
||||
_ <- SyncLogging.logRunFinished(events)
|
||||
plan <- PlanBuilder.createPlan(defaultHashService)
|
||||
archive <- UnversionedMirrorArchive.default(plan.syncTotals)
|
||||
events <- applyPlan(archive, plan)
|
||||
_ <- SyncLogging.logRunFinished(events)
|
||||
} yield ()
|
||||
}
|
||||
|
||||
|
|
|
@ -47,4 +47,21 @@ object Config {
|
|||
|
||||
object Live extends Live
|
||||
|
||||
final def set(config: Configuration): ZIO[Config, Nothing, Unit] =
|
||||
ZIO.accessM(_.config setConfiguration config)
|
||||
|
||||
final def batchMode: ZIO[Config, Nothing, Boolean] =
|
||||
ZIO.accessM(_.config isBatchMode)
|
||||
|
||||
final def bucket: ZIO[Config, Nothing, Bucket] =
|
||||
ZIO.accessM(_.config bucket)
|
||||
|
||||
final def prefix: ZIO[Config, Nothing, RemoteKey] =
|
||||
ZIO.accessM(_.config prefix)
|
||||
|
||||
final def sources: ZIO[Config, Nothing, Sources] =
|
||||
ZIO.accessM(_.config sources)
|
||||
|
||||
final def filters: ZIO[Config, Nothing, List[Filter]] =
|
||||
ZIO.accessM(_.config filters)
|
||||
}
|
||||
|
|
|
@ -1,29 +0,0 @@
|
|||
package net.kemitix.thorp
|
||||
|
||||
import net.kemitix.thorp.domain.{Bucket, Filter, RemoteKey, Sources}
|
||||
import zio.ZIO
|
||||
|
||||
package object config {
|
||||
|
||||
final val configService: ZIO[Config, Nothing, Config.Service] =
|
||||
ZIO.access(_.config)
|
||||
|
||||
final def setConfiguration(
|
||||
config: Configuration): ZIO[Config, Nothing, Unit] =
|
||||
ZIO.accessM(_.config setConfiguration config)
|
||||
|
||||
final def isBatchMode: ZIO[Config, Nothing, Boolean] =
|
||||
ZIO.accessM(_.config isBatchMode)
|
||||
|
||||
final def getBucket: ZIO[Config, Nothing, Bucket] =
|
||||
ZIO.accessM(_.config bucket)
|
||||
|
||||
final def getPrefix: ZIO[Config, Nothing, RemoteKey] =
|
||||
ZIO.accessM(_.config prefix)
|
||||
|
||||
final def getSources: ZIO[Config, Nothing, Sources] =
|
||||
ZIO.accessM(_.config sources)
|
||||
|
||||
final def getFilters: ZIO[Config, Nothing, List[Filter]] =
|
||||
ZIO.accessM(_.config filters)
|
||||
}
|
|
@ -1,6 +1,6 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||
import net.kemitix.thorp.domain._
|
||||
import zio.ZIO
|
||||
|
@ -12,7 +12,7 @@ object ActionGenerator {
|
|||
previousActions: Stream[Action]
|
||||
): ZIO[Config, Nothing, Stream[Action]] =
|
||||
for {
|
||||
bucket <- getBucket
|
||||
bucket <- Config.bucket
|
||||
} yield
|
||||
s3MetaData match {
|
||||
// #1 local exists, remote exists, remote matches - do nothing
|
||||
|
|
|
@ -2,7 +2,7 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.nio.file.Path
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.core.KeyGenerator.generateKey
|
||||
import net.kemitix.thorp.domain._
|
||||
import net.kemitix.thorp.filesystem.FileSystem
|
||||
|
@ -56,8 +56,8 @@ object LocalFileStream {
|
|||
private def localFile(hashService: HashService)(path: Path) = {
|
||||
val file = path.toFile
|
||||
for {
|
||||
sources <- getSources
|
||||
prefix <- getPrefix
|
||||
sources <- Config.sources
|
||||
prefix <- Config.prefix
|
||||
hash <- hashService.hashLocalObject(path)
|
||||
localFile = LocalFile(file,
|
||||
sources.forPath(path).toFile,
|
||||
|
@ -78,7 +78,7 @@ object LocalFileStream {
|
|||
|
||||
private def isIncluded(path: Path) =
|
||||
for {
|
||||
filters <- getFilters
|
||||
filters <- Config.filters
|
||||
} yield Filter.isIncluded(filters)(path)
|
||||
|
||||
private def pathToLocalFile(hashService: HashService)(path: Path) =
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.core.Action._
|
||||
import net.kemitix.thorp.domain._
|
||||
|
@ -72,9 +72,9 @@ trait PlanBuilder {
|
|||
|
||||
private def createActionFromRemoteKey(remoteKey: RemoteKey) =
|
||||
for {
|
||||
bucket <- getBucket
|
||||
prefix <- getPrefix
|
||||
sources <- getSources
|
||||
bucket <- Config.bucket
|
||||
prefix <- Config.prefix
|
||||
sources <- Config.sources
|
||||
needsDeleted <- Remote.isMissingLocally(sources, prefix)(remoteKey)
|
||||
} yield
|
||||
if (needsDeleted) ToDelete(bucket, remoteKey, 0L)
|
||||
|
@ -88,8 +88,8 @@ trait PlanBuilder {
|
|||
|
||||
private def fetchRemoteData =
|
||||
for {
|
||||
bucket <- getBucket
|
||||
prefix <- getPrefix
|
||||
bucket <- Config.bucket
|
||||
prefix <- Config.prefix
|
||||
objects <- Storage.list(bucket, prefix)
|
||||
} yield objects
|
||||
|
||||
|
@ -101,7 +101,7 @@ trait PlanBuilder {
|
|||
|
||||
private def findFiles(hashService: HashService) =
|
||||
for {
|
||||
sources <- getSources
|
||||
sources <- Config.sources
|
||||
paths = sources.paths
|
||||
found <- ZIO.foreach(paths)(path =>
|
||||
LocalFileStream.findFiles(hashService)(path))
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent.{
|
||||
CopyQueueEvent,
|
||||
|
@ -16,15 +16,15 @@ trait SyncLogging {
|
|||
|
||||
def logRunStart: ZIO[Console with Config, Nothing, Unit] =
|
||||
for {
|
||||
bucket <- getBucket
|
||||
prefix <- getPrefix
|
||||
sources <- getSources
|
||||
bucket <- Config.bucket
|
||||
prefix <- Config.prefix
|
||||
sources <- Config.sources
|
||||
_ <- Console.putMessageLn(ConsoleOut.ValidConfig(bucket, prefix, sources))
|
||||
} yield ()
|
||||
|
||||
def logFileScan: ZIO[Config with Console, Nothing, Unit] =
|
||||
for {
|
||||
sources <- getSources
|
||||
sources <- Config.sources
|
||||
_ <- Console.putStrLn(
|
||||
s"Scanning local files: ${sources.paths.mkString(", ")}...")
|
||||
} yield ()
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent.{
|
||||
|
@ -22,15 +23,15 @@ trait ThorpArchive {
|
|||
index: Int,
|
||||
action: Action,
|
||||
totalBytesSoFar: Long
|
||||
): TaskR[Storage with Console, StorageQueueEvent]
|
||||
): TaskR[Storage with Console with Config, StorageQueueEvent]
|
||||
|
||||
def logEvent(
|
||||
event: StorageQueueEvent,
|
||||
batchMode: Boolean
|
||||
): TaskR[Console, Unit] =
|
||||
event: StorageQueueEvent
|
||||
): TaskR[Console with Config, Unit] =
|
||||
event match {
|
||||
case UploadQueueEvent(remoteKey, _) =>
|
||||
for {
|
||||
batchMode <- Config.batchMode
|
||||
_ <- TaskR.when(batchMode)(
|
||||
Console.putStrLn(s"Uploaded: ${remoteKey.key}"))
|
||||
_ <- TaskR.when(!batchMode)(
|
||||
|
@ -39,6 +40,7 @@ trait ThorpArchive {
|
|||
} yield ()
|
||||
case CopyQueueEvent(sourceKey, targetKey) =>
|
||||
for {
|
||||
batchMode <- Config.batchMode
|
||||
_ <- TaskR.when(batchMode)(
|
||||
Console.putStrLn(s"Copied: ${sourceKey.key} => ${targetKey.key}"))
|
||||
_ <- TaskR.when(!batchMode)(
|
||||
|
@ -48,13 +50,15 @@ trait ThorpArchive {
|
|||
} yield ()
|
||||
case DeleteQueueEvent(remoteKey) =>
|
||||
for {
|
||||
_ <- TaskR.when(batchMode)(Console.putStrLn(s"Deleted: $remoteKey"))
|
||||
batchMode <- Config.batchMode
|
||||
_ <- TaskR.when(batchMode)(Console.putStrLn(s"Deleted: $remoteKey"))
|
||||
_ <- TaskR.when(!batchMode)(
|
||||
Console.putStrLn(
|
||||
s"${GREEN}Deleted:$RESET ${remoteKey.key}$eraseToEndOfScreen"))
|
||||
} yield ()
|
||||
case ErrorQueueEvent(action, _, e) =>
|
||||
for {
|
||||
batchMode <- Config.batchMode
|
||||
_ <- TaskR.when(batchMode)(
|
||||
Console.putStrLn(
|
||||
s"${action.name} failed: ${action.keys}: ${e.getMessage}"))
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent.DoNothingQueueEvent
|
||||
|
@ -7,31 +8,29 @@ import net.kemitix.thorp.domain._
|
|||
import net.kemitix.thorp.storage.api.Storage
|
||||
import zio.{Task, TaskR}
|
||||
|
||||
case class UnversionedMirrorArchive(
|
||||
batchMode: Boolean,
|
||||
syncTotals: SyncTotals
|
||||
) extends ThorpArchive {
|
||||
case class UnversionedMirrorArchive(syncTotals: SyncTotals)
|
||||
extends ThorpArchive {
|
||||
|
||||
override def update(
|
||||
index: Int,
|
||||
action: Action,
|
||||
totalBytesSoFar: Long
|
||||
): TaskR[Storage with Console, StorageQueueEvent] =
|
||||
): TaskR[Storage with Console with Config, StorageQueueEvent] =
|
||||
action match {
|
||||
case ToUpload(bucket, localFile, _) =>
|
||||
for {
|
||||
event <- doUpload(index, totalBytesSoFar, bucket, localFile)
|
||||
_ <- logEvent(event, batchMode)
|
||||
_ <- logEvent(event)
|
||||
} yield event
|
||||
case ToCopy(bucket, sourceKey, hash, targetKey, _) =>
|
||||
for {
|
||||
event <- Storage.copy(bucket, sourceKey, hash, targetKey)
|
||||
_ <- logEvent(event, batchMode)
|
||||
_ <- logEvent(event)
|
||||
} yield event
|
||||
case ToDelete(bucket, remoteKey, _) =>
|
||||
for {
|
||||
event <- Storage.delete(bucket, remoteKey)
|
||||
_ <- logEvent(event, batchMode)
|
||||
_ <- logEvent(event)
|
||||
} yield event
|
||||
case DoNothing(_, remoteKey, _) =>
|
||||
Task(DoNothingQueueEvent(remoteKey))
|
||||
|
@ -46,17 +45,13 @@ case class UnversionedMirrorArchive(
|
|||
Storage.upload(
|
||||
localFile,
|
||||
bucket,
|
||||
batchMode,
|
||||
UploadEventListener(localFile, index, syncTotals, totalBytesSoFar),
|
||||
1)
|
||||
}
|
||||
|
||||
object UnversionedMirrorArchive {
|
||||
def default(
|
||||
batchMode: Boolean,
|
||||
syncTotals: SyncTotals
|
||||
): Task[ThorpArchive] =
|
||||
def default(syncTotals: SyncTotals): Task[ThorpArchive] =
|
||||
Task {
|
||||
new UnversionedMirrorArchive(batchMode, syncTotals)
|
||||
new UnversionedMirrorArchive(syncTotals)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,7 +2,13 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.time.Instant
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.config.{
|
||||
Config,
|
||||
ConfigOption,
|
||||
ConfigOptions,
|
||||
ConfigurationBuilder,
|
||||
Resource
|
||||
}
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
import net.kemitix.thorp.domain._
|
||||
|
@ -170,7 +176,7 @@ class ActionGeneratorSuite extends FunSpec {
|
|||
def testProgram =
|
||||
for {
|
||||
config <- ConfigurationBuilder.buildConfig(configOptions)
|
||||
_ <- setConfiguration(config)
|
||||
_ <- Config.set(config)
|
||||
actions <- ActionGenerator.createActions(input, previousActions)
|
||||
} yield actions
|
||||
|
||||
|
|
|
@ -22,7 +22,6 @@ case class DummyStorageService(s3ObjectData: S3ObjectsData,
|
|||
|
||||
override def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int): UIO[StorageQueueEvent] = {
|
||||
val (remoteKey, md5Hash) = uploadFiles(localFile.file)
|
||||
|
|
|
@ -2,7 +2,13 @@ package net.kemitix.thorp.core
|
|||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.config.{
|
||||
Config,
|
||||
ConfigOption,
|
||||
ConfigOptions,
|
||||
ConfigurationBuilder,
|
||||
Resource
|
||||
}
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
import net.kemitix.thorp.domain._
|
||||
|
@ -73,7 +79,7 @@ class LocalFileStreamSuite extends FunSpec {
|
|||
def testProgram =
|
||||
for {
|
||||
config <- ConfigurationBuilder.buildConfig(configOptions)
|
||||
_ <- setConfiguration(config)
|
||||
_ <- Config.set(config)
|
||||
files <- LocalFileStream.findFiles(hashService)(sourcePath)
|
||||
} yield files
|
||||
|
||||
|
|
|
@ -3,7 +3,12 @@ package net.kemitix.thorp.core
|
|||
import java.io.File
|
||||
import java.nio.file.Path
|
||||
|
||||
import net.kemitix.thorp.config._
|
||||
import net.kemitix.thorp.config.{
|
||||
Config,
|
||||
ConfigOption,
|
||||
ConfigOptions,
|
||||
ConfigurationBuilder
|
||||
}
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
|
@ -378,7 +383,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
|||
def testProgram =
|
||||
for {
|
||||
config <- ConfigurationBuilder.buildConfig(configOptions)
|
||||
_ <- setConfiguration(config)
|
||||
_ <- Config.set(config)
|
||||
plan <- PlanBuilder.createPlan(hashService)
|
||||
} yield plan
|
||||
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
package net.kemitix.thorp.storage.api
|
||||
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.console.Console
|
||||
import net.kemitix.thorp.domain._
|
||||
import zio.{Task, TaskR, UIO, ZIO}
|
||||
|
@ -19,10 +20,9 @@ object Storage {
|
|||
def upload(
|
||||
localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int
|
||||
): ZIO[Storage, Nothing, StorageQueueEvent]
|
||||
): ZIO[Storage with Config, Nothing, StorageQueueEvent]
|
||||
|
||||
def copy(
|
||||
bucket: Bucket,
|
||||
|
@ -57,7 +57,6 @@ object Storage {
|
|||
override def upload(
|
||||
localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int): ZIO[Storage, Nothing, StorageQueueEvent] =
|
||||
uploadResult
|
||||
|
@ -100,12 +99,11 @@ object Storage {
|
|||
final def upload(
|
||||
localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int
|
||||
): ZIO[Storage, Nothing, StorageQueueEvent] =
|
||||
): ZIO[Storage with Config, Nothing, StorageQueueEvent] =
|
||||
ZIO.accessM(
|
||||
_.storage upload (localFile, bucket, batchMode, uploadEventListener, tryCount))
|
||||
_.storage upload (localFile, bucket, uploadEventListener, tryCount))
|
||||
|
||||
final def copy(
|
||||
bucket: Bucket,
|
||||
|
|
|
@ -2,12 +2,13 @@ package net.kemitix.thorp.storage.aws
|
|||
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerBuilder
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.console.Console
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent.ShutdownQueueEvent
|
||||
import net.kemitix.thorp.domain._
|
||||
import net.kemitix.thorp.storage.api.Storage
|
||||
import net.kemitix.thorp.storage.api.Storage.Service
|
||||
import zio.{TaskR, UIO}
|
||||
import zio.{TaskR, UIO, ZIO}
|
||||
|
||||
object S3Storage {
|
||||
trait Live extends Storage {
|
||||
|
@ -23,14 +24,13 @@ object S3Storage {
|
|||
prefix: RemoteKey): TaskR[Console, S3ObjectsData] =
|
||||
Lister.listObjects(client)(bucket, prefix)
|
||||
|
||||
override def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int): UIO[StorageQueueEvent] =
|
||||
override def upload(
|
||||
localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int): ZIO[Config, Nothing, StorageQueueEvent] =
|
||||
Uploader.upload(transferManager)(localFile,
|
||||
bucket,
|
||||
batchMode,
|
||||
uploadEventListener,
|
||||
1)
|
||||
|
||||
|
|
|
@ -2,6 +2,7 @@ package net.kemitix.thorp.storage.aws
|
|||
|
||||
import com.amazonaws.event.{ProgressEvent, ProgressEventType, ProgressListener}
|
||||
import com.amazonaws.services.s3.model.{ObjectMetadata, PutObjectRequest}
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent.{
|
||||
Action,
|
||||
ErrorQueueEvent,
|
||||
|
@ -13,35 +14,38 @@ import net.kemitix.thorp.domain.UploadEvent.{
|
|||
TransferEvent
|
||||
}
|
||||
import net.kemitix.thorp.domain.{StorageQueueEvent, _}
|
||||
import zio.{Task, UIO}
|
||||
import zio.{UIO, ZIO}
|
||||
|
||||
trait Uploader {
|
||||
|
||||
def upload(transferManager: => AmazonTransferManager)(
|
||||
localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int
|
||||
): UIO[StorageQueueEvent] =
|
||||
transfer(transferManager)(localFile, bucket, batchMode, uploadEventListener)
|
||||
): ZIO[Config, Nothing, StorageQueueEvent] =
|
||||
transfer(transferManager)(localFile, bucket, uploadEventListener)
|
||||
.catchAll(handleError(localFile.remoteKey))
|
||||
|
||||
private def handleError(
|
||||
remoteKey: RemoteKey): Throwable => UIO[ErrorQueueEvent] = { e =>
|
||||
private def handleError(remoteKey: RemoteKey)(e: Throwable) =
|
||||
UIO.succeed(ErrorQueueEvent(Action.Upload(remoteKey.key), remoteKey, e))
|
||||
}
|
||||
|
||||
private def transfer(transferManager: => AmazonTransferManager)(
|
||||
localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
uploadEventListener: UploadEventListener
|
||||
): Task[StorageQueueEvent] = {
|
||||
|
||||
val listener = progressListener(uploadEventListener)
|
||||
val putObjectRequest = request(localFile, bucket, batchMode, listener)
|
||||
) = {
|
||||
val listener = progressListener(uploadEventListener)
|
||||
for {
|
||||
putObjectRequest <- request(localFile, bucket, listener)
|
||||
event <- dispatch(transferManager, putObjectRequest)
|
||||
} yield event
|
||||
}
|
||||
|
||||
private def dispatch(
|
||||
transferManager: AmazonTransferManager,
|
||||
putObjectRequest: PutObjectRequest
|
||||
) = {
|
||||
transferManager
|
||||
.upload(putObjectRequest)
|
||||
.map(_.waitForUploadResult)
|
||||
|
@ -53,14 +57,16 @@ trait Uploader {
|
|||
private def request(
|
||||
localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
listener: ProgressListener
|
||||
): PutObjectRequest = {
|
||||
) = {
|
||||
val request =
|
||||
new PutObjectRequest(bucket.name, localFile.remoteKey.key, localFile.file)
|
||||
.withMetadata(metadata(localFile))
|
||||
if (batchMode) request
|
||||
else request.withGeneralProgressListener(listener)
|
||||
for {
|
||||
batchMode <- Config.batchMode
|
||||
r = if (batchMode) request
|
||||
else request.withGeneralProgressListener(listener)
|
||||
} yield r
|
||||
}
|
||||
|
||||
private def metadata: LocalFile => ObjectMetadata = localFile => {
|
||||
|
|
|
@ -1,11 +1,12 @@
|
|||
package net.kemitix.thorp.storage.aws
|
||||
|
||||
import net.kemitix.thorp.config.Config
|
||||
import net.kemitix.thorp.console.Console
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent.ShutdownQueueEvent
|
||||
import net.kemitix.thorp.domain._
|
||||
import net.kemitix.thorp.storage.api.Storage
|
||||
import org.scalamock.scalatest.MockFactory
|
||||
import zio.{TaskR, UIO}
|
||||
import zio.{TaskR, UIO, ZIO}
|
||||
|
||||
trait AmazonS3ClientTestFixture extends MockFactory {
|
||||
|
||||
|
@ -27,14 +28,13 @@ trait AmazonS3ClientTestFixture extends MockFactory {
|
|||
prefix: RemoteKey): TaskR[Console, S3ObjectsData] =
|
||||
Lister.listObjects(client)(bucket, prefix)
|
||||
|
||||
override def upload(localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int): UIO[StorageQueueEvent] =
|
||||
override def upload(
|
||||
localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int): ZIO[Config, Nothing, StorageQueueEvent] =
|
||||
Uploader.upload(transferManager)(localFile,
|
||||
bucket,
|
||||
batchMode,
|
||||
uploadEventListener,
|
||||
1)
|
||||
|
||||
|
|
|
@ -5,8 +5,7 @@ import java.io.File
|
|||
import com.amazonaws.SdkClientException
|
||||
import com.amazonaws.services.s3.model.AmazonS3Exception
|
||||
import com.amazonaws.services.s3.transfer.model.UploadResult
|
||||
import net.kemitix.thorp.config.Resource
|
||||
import net.kemitix.thorp.console._
|
||||
import net.kemitix.thorp.config.{Config, Resource}
|
||||
import net.kemitix.thorp.domain.HashType.MD5
|
||||
import net.kemitix.thorp.domain.StorageQueueEvent.{
|
||||
Action,
|
||||
|
@ -16,13 +15,10 @@ import net.kemitix.thorp.domain.StorageQueueEvent.{
|
|||
import net.kemitix.thorp.domain._
|
||||
import org.scalamock.scalatest.MockFactory
|
||||
import org.scalatest.FreeSpec
|
||||
import zio.internal.PlatformLive
|
||||
import zio.{Runtime, Task}
|
||||
import zio.{DefaultRuntime, Task}
|
||||
|
||||
class UploaderTest extends FreeSpec with MockFactory {
|
||||
|
||||
private val runtime = Runtime(Console.Live, PlatformLive.Default)
|
||||
|
||||
"upload" - {
|
||||
val aSource: File = Resource(this, "")
|
||||
val aFile: File = Resource(this, "small-file")
|
||||
|
@ -31,7 +27,6 @@ class UploaderTest extends FreeSpec with MockFactory {
|
|||
val remoteKey = RemoteKey("aRemoteKey")
|
||||
val localFile = LocalFile(aFile, aSource, hashes, remoteKey)
|
||||
val bucket = Bucket("aBucket")
|
||||
val batchMode = false
|
||||
val tryCount = 1
|
||||
val uploadResult = new UploadResult
|
||||
uploadResult.setKey(remoteKey.key)
|
||||
|
@ -52,7 +47,6 @@ class UploaderTest extends FreeSpec with MockFactory {
|
|||
invoke(fixture.amazonS3TransferManager)(
|
||||
localFile,
|
||||
bucket,
|
||||
batchMode,
|
||||
uploadEventListener,
|
||||
tryCount
|
||||
)
|
||||
|
@ -72,7 +66,6 @@ class UploaderTest extends FreeSpec with MockFactory {
|
|||
invoke(fixture.amazonS3TransferManager)(
|
||||
localFile,
|
||||
bucket,
|
||||
batchMode,
|
||||
uploadEventListener,
|
||||
tryCount
|
||||
)
|
||||
|
@ -92,7 +85,6 @@ class UploaderTest extends FreeSpec with MockFactory {
|
|||
invoke(fixture.amazonS3TransferManager)(
|
||||
localFile,
|
||||
bucket,
|
||||
batchMode,
|
||||
uploadEventListener,
|
||||
tryCount
|
||||
)
|
||||
|
@ -102,19 +94,22 @@ class UploaderTest extends FreeSpec with MockFactory {
|
|||
def invoke(transferManager: AmazonTransferManager)(
|
||||
localFile: LocalFile,
|
||||
bucket: Bucket,
|
||||
batchMode: Boolean,
|
||||
uploadEventListener: UploadEventListener,
|
||||
tryCount: Int
|
||||
) =
|
||||
runtime.unsafeRunSync {
|
||||
Uploader.upload(transferManager)(
|
||||
localFile,
|
||||
bucket,
|
||||
batchMode,
|
||||
uploadEventListener,
|
||||
tryCount
|
||||
)
|
||||
) = {
|
||||
type TestEnv = Config
|
||||
val testEnv: TestEnv = Config.Live
|
||||
new DefaultRuntime {}.unsafeRunSync {
|
||||
Uploader
|
||||
.upload(transferManager)(
|
||||
localFile,
|
||||
bucket,
|
||||
uploadEventListener,
|
||||
tryCount
|
||||
)
|
||||
.provide(testEnv)
|
||||
}.toEither
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue