[core] Sync simplify the call to run (#61)

This commit is contained in:
Paul Campbell 2019-06-14 17:40:36 +01:00 committed by GitHub
parent 21b8917395
commit dc0c142762
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 11 additions and 28 deletions

View file

@ -1,28 +1,20 @@
package net.kemitix.s3thorp.cli package net.kemitix.s3thorp.cli
import java.io.File
import cats.Monad import cats.Monad
import cats.effect.ExitCode import cats.effect.ExitCode
import cats.implicits._ import cats.implicits._
import net.kemitix.s3thorp.aws.lib.S3ClientBuilder import net.kemitix.s3thorp.aws.lib.S3ClientBuilder
import net.kemitix.s3thorp.core.MD5HashGenerator.md5File
import net.kemitix.s3thorp.core.Sync import net.kemitix.s3thorp.core.Sync
import net.kemitix.s3thorp.domain.{Config, Logger} import net.kemitix.s3thorp.domain.{Config, Logger}
object Program { object Program {
def apply[M[_]: Monad](config: Config): M[ExitCode] = { def apply[M[_]: Monad](config: Config): M[ExitCode] = {
val logger = new PrintLogger[M](config.verbose) implicit val logger: Logger[M] = new PrintLogger[M](config.verbose)
for { for {
_ <- logger.info("S3Thorp - hashed sync for s3") _ <- logger.info("S3Thorp - hashed sync for s3")
_ <- Sync.run[M](config, S3ClientBuilder.defaultClient, hashGenerator(logger), logger) _ <- Sync.run[M](config, S3ClientBuilder.defaultClient)
} yield ExitCode.Success } yield ExitCode.Success
} }
private def hashGenerator[M[_]: Monad](logger: Logger[M]) = {
implicit val impLogger: Logger[M] = logger
file: File => md5File[M](file)
}
} }

View file

@ -1,7 +1,5 @@
package net.kemitix.s3thorp.core package net.kemitix.s3thorp.core
import java.io.File
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client} import net.kemitix.s3thorp.aws.api.{S3Action, S3Client}
@ -11,17 +9,15 @@ import net.kemitix.s3thorp.core.ActionSubmitter.submitAction
import net.kemitix.s3thorp.core.LocalFileStream.findFiles import net.kemitix.s3thorp.core.LocalFileStream.findFiles
import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata
import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart} import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart}
import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash, S3MetaData, S3ObjectsData} import net.kemitix.s3thorp.domain._
object Sync { object Sync {
def run[M[_]: Monad](config: Config, def run[M[_]: Monad](config: Config,
s3Client: S3Client[M], s3Client: S3Client[M])
md5HashGenerator: File => M[MD5Hash], (implicit logger: Logger[M]): M[Unit] = {
logger: Logger[M]): M[Unit] = {
implicit val c: Config = config implicit val c: Config = config
implicit val implLogger: Logger[M] = logger
def metaData(s3Data: S3ObjectsData, sFiles: Stream[LocalFile]) = def metaData(s3Data: S3ObjectsData, sFiles: Stream[LocalFile]) =
Monad[M].pure(sFiles.map(file => getMetadata(file, s3Data))) Monad[M].pure(sFiles.map(file => getMetadata(file, s3Data)))
@ -34,7 +30,7 @@ object Sync {
def copyUploadActions(s3Data: S3ObjectsData): M[Stream[S3Action]] = def copyUploadActions(s3Data: S3ObjectsData): M[Stream[S3Action]] =
(for { (for {
files <- findFiles(c.source, md5HashGenerator) files <- findFiles(c.source, MD5HashGenerator.md5File[M](_))
metaData <- metaData(s3Data, files) metaData <- metaData(s3Data, files)
actions <- actions(metaData) actions <- actions(metaData)
s3Actions <- submit(actions) s3Actions <- submit(actions)

View file

@ -19,11 +19,6 @@ class SyncSuite
val config = Config(Bucket("bucket"), prefix, source = source) val config = Config(Bucket("bucket"), prefix, source = source)
implicit private val logger: Logger[Id] = new DummyLogger[Id] implicit private val logger: Logger[Id] = new DummyLogger[Id]
private val lastModified = LastModified(Instant.now) private val lastModified = LastModified(Instant.now)
private val fileToKey: File => RemoteKey = KeyGenerator.generateKey(source, prefix)
private val rootFile = LocalFile.resolve("root-file", rootHash, source, fileToKey)
private val leafFile = LocalFile.resolve("subdir/leaf-file", leafHash, source, fileToKey)
private val md5HashGenerator = MD5HashGenerator.md5File[Id](_)
def putObjectRequest(bucket: Bucket, remoteKey: RemoteKey, localFile: LocalFile): (String, String, File) = def putObjectRequest(bucket: Bucket, remoteKey: RemoteKey, localFile: LocalFile): (String, String, File) =
(bucket.name, remoteKey.key, localFile.file) (bucket.name, remoteKey.key, localFile.file)
@ -37,7 +32,7 @@ class SyncSuite
val s3Client = new RecordingClient(testBucket, S3ObjectsData( val s3Client = new RecordingClient(testBucket, S3ObjectsData(
byHash = Map(), byHash = Map(),
byKey = Map())) byKey = Map()))
Sync.run(config, s3Client, md5HashGenerator, logger) Sync.run(config, s3Client)
it("uploads all files") { it("uploads all files") {
val expectedUploads = Map( val expectedUploads = Map(
"subdir/leaf-file" -> leafRemoteKey, "subdir/leaf-file" -> leafRemoteKey,
@ -63,7 +58,7 @@ class SyncSuite
RemoteKey("prefix/root-file") -> HashModified(rootHash, lastModified), RemoteKey("prefix/root-file") -> HashModified(rootHash, lastModified),
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified))) RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
val s3Client = new RecordingClient(testBucket, s3ObjectsData) val s3Client = new RecordingClient(testBucket, s3ObjectsData)
Sync.run(config, s3Client, md5HashGenerator, logger) Sync.run(config, s3Client)
it("uploads nothing") { it("uploads nothing") {
val expectedUploads = Map() val expectedUploads = Map()
assertResult(expectedUploads)(s3Client.uploadsRecord) assertResult(expectedUploads)(s3Client.uploadsRecord)
@ -87,7 +82,7 @@ class SyncSuite
RemoteKey("prefix/root-file-old") -> HashModified(rootHash, lastModified), RemoteKey("prefix/root-file-old") -> HashModified(rootHash, lastModified),
RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified))) RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified)))
val s3Client = new RecordingClient(testBucket, s3ObjectsData) val s3Client = new RecordingClient(testBucket, s3ObjectsData)
Sync.run(config, s3Client, md5HashGenerator, logger) Sync.run(config, s3Client)
it("uploads nothing") { it("uploads nothing") {
val expectedUploads = Map() val expectedUploads = Map()
assertResult(expectedUploads)(s3Client.uploadsRecord) assertResult(expectedUploads)(s3Client.uploadsRecord)
@ -115,7 +110,7 @@ class SyncSuite
byKey = Map( byKey = Map(
deletedKey -> HashModified(deletedHash, lastModified))) deletedKey -> HashModified(deletedHash, lastModified)))
val s3Client = new RecordingClient(testBucket, s3ObjectsData) val s3Client = new RecordingClient(testBucket, s3ObjectsData)
Sync.run(config, s3Client, md5HashGenerator, logger) Sync.run(config, s3Client)
it("deleted key") { it("deleted key") {
val expectedDeletions = Set(deletedKey) val expectedDeletions = Set(deletedKey)
assertResult(expectedDeletions)(s3Client.deletionsRecord) assertResult(expectedDeletions)(s3Client.deletionsRecord)
@ -125,7 +120,7 @@ class SyncSuite
val config: Config = Config(Bucket("bucket"), prefix, source = source, filters = List(Exclude("leaf"))) val config: Config = Config(Bucket("bucket"), prefix, source = source, filters = List(Exclude("leaf")))
val s3ObjectsData = S3ObjectsData(Map(), Map()) val s3ObjectsData = S3ObjectsData(Map(), Map())
val s3Client = new RecordingClient(testBucket, s3ObjectsData) val s3Client = new RecordingClient(testBucket, s3ObjectsData)
Sync.run(config, s3Client, md5HashGenerator, logger) Sync.run(config, s3Client)
it("is not uploaded") { it("is not uploaded") {
val expectedUploads = Map( val expectedUploads = Map(
"root-file" -> rootRemoteKey "root-file" -> rootRemoteKey