[core] Sync simplify the call to run (#61)
This commit is contained in:
parent
21b8917395
commit
dc0c142762
3 changed files with 11 additions and 28 deletions
|
@ -1,28 +1,20 @@
|
||||||
package net.kemitix.s3thorp.cli
|
package net.kemitix.s3thorp.cli
|
||||||
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
import cats.Monad
|
import cats.Monad
|
||||||
import cats.effect.ExitCode
|
import cats.effect.ExitCode
|
||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
import net.kemitix.s3thorp.aws.lib.S3ClientBuilder
|
import net.kemitix.s3thorp.aws.lib.S3ClientBuilder
|
||||||
import net.kemitix.s3thorp.core.MD5HashGenerator.md5File
|
|
||||||
import net.kemitix.s3thorp.core.Sync
|
import net.kemitix.s3thorp.core.Sync
|
||||||
import net.kemitix.s3thorp.domain.{Config, Logger}
|
import net.kemitix.s3thorp.domain.{Config, Logger}
|
||||||
|
|
||||||
object Program {
|
object Program {
|
||||||
|
|
||||||
def apply[M[_]: Monad](config: Config): M[ExitCode] = {
|
def apply[M[_]: Monad](config: Config): M[ExitCode] = {
|
||||||
val logger = new PrintLogger[M](config.verbose)
|
implicit val logger: Logger[M] = new PrintLogger[M](config.verbose)
|
||||||
for {
|
for {
|
||||||
_ <- logger.info("S3Thorp - hashed sync for s3")
|
_ <- logger.info("S3Thorp - hashed sync for s3")
|
||||||
_ <- Sync.run[M](config, S3ClientBuilder.defaultClient, hashGenerator(logger), logger)
|
_ <- Sync.run[M](config, S3ClientBuilder.defaultClient)
|
||||||
} yield ExitCode.Success
|
} yield ExitCode.Success
|
||||||
}
|
}
|
||||||
|
|
||||||
private def hashGenerator[M[_]: Monad](logger: Logger[M]) = {
|
|
||||||
implicit val impLogger: Logger[M] = logger
|
|
||||||
file: File => md5File[M](file)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,5 @@
|
||||||
package net.kemitix.s3thorp.core
|
package net.kemitix.s3thorp.core
|
||||||
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
import cats.Monad
|
import cats.Monad
|
||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client}
|
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client}
|
||||||
|
@ -11,17 +9,15 @@ import net.kemitix.s3thorp.core.ActionSubmitter.submitAction
|
||||||
import net.kemitix.s3thorp.core.LocalFileStream.findFiles
|
import net.kemitix.s3thorp.core.LocalFileStream.findFiles
|
||||||
import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata
|
import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata
|
||||||
import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart}
|
import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart}
|
||||||
import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash, S3MetaData, S3ObjectsData}
|
import net.kemitix.s3thorp.domain._
|
||||||
|
|
||||||
object Sync {
|
object Sync {
|
||||||
|
|
||||||
def run[M[_]: Monad](config: Config,
|
def run[M[_]: Monad](config: Config,
|
||||||
s3Client: S3Client[M],
|
s3Client: S3Client[M])
|
||||||
md5HashGenerator: File => M[MD5Hash],
|
(implicit logger: Logger[M]): M[Unit] = {
|
||||||
logger: Logger[M]): M[Unit] = {
|
|
||||||
|
|
||||||
implicit val c: Config = config
|
implicit val c: Config = config
|
||||||
implicit val implLogger: Logger[M] = logger
|
|
||||||
|
|
||||||
def metaData(s3Data: S3ObjectsData, sFiles: Stream[LocalFile]) =
|
def metaData(s3Data: S3ObjectsData, sFiles: Stream[LocalFile]) =
|
||||||
Monad[M].pure(sFiles.map(file => getMetadata(file, s3Data)))
|
Monad[M].pure(sFiles.map(file => getMetadata(file, s3Data)))
|
||||||
|
@ -34,7 +30,7 @@ object Sync {
|
||||||
|
|
||||||
def copyUploadActions(s3Data: S3ObjectsData): M[Stream[S3Action]] =
|
def copyUploadActions(s3Data: S3ObjectsData): M[Stream[S3Action]] =
|
||||||
(for {
|
(for {
|
||||||
files <- findFiles(c.source, md5HashGenerator)
|
files <- findFiles(c.source, MD5HashGenerator.md5File[M](_))
|
||||||
metaData <- metaData(s3Data, files)
|
metaData <- metaData(s3Data, files)
|
||||||
actions <- actions(metaData)
|
actions <- actions(metaData)
|
||||||
s3Actions <- submit(actions)
|
s3Actions <- submit(actions)
|
||||||
|
|
|
@ -19,11 +19,6 @@ class SyncSuite
|
||||||
val config = Config(Bucket("bucket"), prefix, source = source)
|
val config = Config(Bucket("bucket"), prefix, source = source)
|
||||||
implicit private val logger: Logger[Id] = new DummyLogger[Id]
|
implicit private val logger: Logger[Id] = new DummyLogger[Id]
|
||||||
private val lastModified = LastModified(Instant.now)
|
private val lastModified = LastModified(Instant.now)
|
||||||
private val fileToKey: File => RemoteKey = KeyGenerator.generateKey(source, prefix)
|
|
||||||
private val rootFile = LocalFile.resolve("root-file", rootHash, source, fileToKey)
|
|
||||||
private val leafFile = LocalFile.resolve("subdir/leaf-file", leafHash, source, fileToKey)
|
|
||||||
|
|
||||||
private val md5HashGenerator = MD5HashGenerator.md5File[Id](_)
|
|
||||||
|
|
||||||
def putObjectRequest(bucket: Bucket, remoteKey: RemoteKey, localFile: LocalFile): (String, String, File) =
|
def putObjectRequest(bucket: Bucket, remoteKey: RemoteKey, localFile: LocalFile): (String, String, File) =
|
||||||
(bucket.name, remoteKey.key, localFile.file)
|
(bucket.name, remoteKey.key, localFile.file)
|
||||||
|
@ -37,7 +32,7 @@ class SyncSuite
|
||||||
val s3Client = new RecordingClient(testBucket, S3ObjectsData(
|
val s3Client = new RecordingClient(testBucket, S3ObjectsData(
|
||||||
byHash = Map(),
|
byHash = Map(),
|
||||||
byKey = Map()))
|
byKey = Map()))
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logger)
|
Sync.run(config, s3Client)
|
||||||
it("uploads all files") {
|
it("uploads all files") {
|
||||||
val expectedUploads = Map(
|
val expectedUploads = Map(
|
||||||
"subdir/leaf-file" -> leafRemoteKey,
|
"subdir/leaf-file" -> leafRemoteKey,
|
||||||
|
@ -63,7 +58,7 @@ class SyncSuite
|
||||||
RemoteKey("prefix/root-file") -> HashModified(rootHash, lastModified),
|
RemoteKey("prefix/root-file") -> HashModified(rootHash, lastModified),
|
||||||
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
|
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
|
||||||
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logger)
|
Sync.run(config, s3Client)
|
||||||
it("uploads nothing") {
|
it("uploads nothing") {
|
||||||
val expectedUploads = Map()
|
val expectedUploads = Map()
|
||||||
assertResult(expectedUploads)(s3Client.uploadsRecord)
|
assertResult(expectedUploads)(s3Client.uploadsRecord)
|
||||||
|
@ -87,7 +82,7 @@ class SyncSuite
|
||||||
RemoteKey("prefix/root-file-old") -> HashModified(rootHash, lastModified),
|
RemoteKey("prefix/root-file-old") -> HashModified(rootHash, lastModified),
|
||||||
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
|
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
|
||||||
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logger)
|
Sync.run(config, s3Client)
|
||||||
it("uploads nothing") {
|
it("uploads nothing") {
|
||||||
val expectedUploads = Map()
|
val expectedUploads = Map()
|
||||||
assertResult(expectedUploads)(s3Client.uploadsRecord)
|
assertResult(expectedUploads)(s3Client.uploadsRecord)
|
||||||
|
@ -115,7 +110,7 @@ class SyncSuite
|
||||||
byKey = Map(
|
byKey = Map(
|
||||||
deletedKey -> HashModified(deletedHash, lastModified)))
|
deletedKey -> HashModified(deletedHash, lastModified)))
|
||||||
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logger)
|
Sync.run(config, s3Client)
|
||||||
it("deleted key") {
|
it("deleted key") {
|
||||||
val expectedDeletions = Set(deletedKey)
|
val expectedDeletions = Set(deletedKey)
|
||||||
assertResult(expectedDeletions)(s3Client.deletionsRecord)
|
assertResult(expectedDeletions)(s3Client.deletionsRecord)
|
||||||
|
@ -125,7 +120,7 @@ class SyncSuite
|
||||||
val config: Config = Config(Bucket("bucket"), prefix, source = source, filters = List(Exclude("leaf")))
|
val config: Config = Config(Bucket("bucket"), prefix, source = source, filters = List(Exclude("leaf")))
|
||||||
val s3ObjectsData = S3ObjectsData(Map(), Map())
|
val s3ObjectsData = S3ObjectsData(Map(), Map())
|
||||||
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
val s3Client = new RecordingClient(testBucket, s3ObjectsData)
|
||||||
Sync.run(config, s3Client, md5HashGenerator, logger)
|
Sync.run(config, s3Client)
|
||||||
it("is not uploaded") {
|
it("is not uploaded") {
|
||||||
val expectedUploads = Map(
|
val expectedUploads = Map(
|
||||||
"root-file" -> rootRemoteKey
|
"root-file" -> rootRemoteKey
|
||||||
|
|
Loading…
Reference in a new issue