From dc0c142762216fc48fd5924fed0d75e29d7c8bf8 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 14 Jun 2019 17:40:36 +0100 Subject: [PATCH] [core] Sync simplify the call to run (#61) --- .../scala/net/kemitix/s3thorp/cli/Program.scala | 12 ++---------- .../scala/net.kemitix.s3thorp.core/Sync.scala | 12 ++++-------- .../net/kemitix/s3thorp/core/SyncSuite.scala | 15 +++++---------- 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/cli/src/main/scala/net/kemitix/s3thorp/cli/Program.scala b/cli/src/main/scala/net/kemitix/s3thorp/cli/Program.scala index 0f222da..88b1883 100644 --- a/cli/src/main/scala/net/kemitix/s3thorp/cli/Program.scala +++ b/cli/src/main/scala/net/kemitix/s3thorp/cli/Program.scala @@ -1,28 +1,20 @@ package net.kemitix.s3thorp.cli -import java.io.File - import cats.Monad import cats.effect.ExitCode import cats.implicits._ import net.kemitix.s3thorp.aws.lib.S3ClientBuilder -import net.kemitix.s3thorp.core.MD5HashGenerator.md5File import net.kemitix.s3thorp.core.Sync import net.kemitix.s3thorp.domain.{Config, Logger} object Program { def apply[M[_]: Monad](config: Config): M[ExitCode] = { - val logger = new PrintLogger[M](config.verbose) + implicit val logger: Logger[M] = new PrintLogger[M](config.verbose) for { _ <- logger.info("S3Thorp - hashed sync for s3") - _ <- Sync.run[M](config, S3ClientBuilder.defaultClient, hashGenerator(logger), logger) + _ <- Sync.run[M](config, S3ClientBuilder.defaultClient) } yield ExitCode.Success } - private def hashGenerator[M[_]: Monad](logger: Logger[M]) = { - implicit val impLogger: Logger[M] = logger - file: File => md5File[M](file) - } - } diff --git a/core/src/main/scala/net.kemitix.s3thorp.core/Sync.scala b/core/src/main/scala/net.kemitix.s3thorp.core/Sync.scala index f509563..5fd5d89 100644 --- a/core/src/main/scala/net.kemitix.s3thorp.core/Sync.scala +++ b/core/src/main/scala/net.kemitix.s3thorp.core/Sync.scala @@ -1,7 +1,5 @@ package net.kemitix.s3thorp.core -import java.io.File - import cats.Monad import cats.implicits._ import net.kemitix.s3thorp.aws.api.{S3Action, S3Client} @@ -11,17 +9,15 @@ import net.kemitix.s3thorp.core.ActionSubmitter.submitAction import net.kemitix.s3thorp.core.LocalFileStream.findFiles import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart} -import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash, S3MetaData, S3ObjectsData} +import net.kemitix.s3thorp.domain._ object Sync { def run[M[_]: Monad](config: Config, - s3Client: S3Client[M], - md5HashGenerator: File => M[MD5Hash], - logger: Logger[M]): M[Unit] = { + s3Client: S3Client[M]) + (implicit logger: Logger[M]): M[Unit] = { implicit val c: Config = config - implicit val implLogger: Logger[M] = logger def metaData(s3Data: S3ObjectsData, sFiles: Stream[LocalFile]) = Monad[M].pure(sFiles.map(file => getMetadata(file, s3Data))) @@ -34,7 +30,7 @@ object Sync { def copyUploadActions(s3Data: S3ObjectsData): M[Stream[S3Action]] = (for { - files <- findFiles(c.source, md5HashGenerator) + files <- findFiles(c.source, MD5HashGenerator.md5File[M](_)) metaData <- metaData(s3Data, files) actions <- actions(metaData) s3Actions <- submit(actions) diff --git a/core/src/test/scala/net/kemitix/s3thorp/core/SyncSuite.scala b/core/src/test/scala/net/kemitix/s3thorp/core/SyncSuite.scala index 2db8659..a0e3658 100644 --- a/core/src/test/scala/net/kemitix/s3thorp/core/SyncSuite.scala +++ b/core/src/test/scala/net/kemitix/s3thorp/core/SyncSuite.scala @@ -19,11 +19,6 @@ class SyncSuite val config = Config(Bucket("bucket"), prefix, source = source) implicit private val logger: Logger[Id] = new DummyLogger[Id] private val lastModified = LastModified(Instant.now) - private val fileToKey: File => RemoteKey = KeyGenerator.generateKey(source, prefix) - private val rootFile = LocalFile.resolve("root-file", rootHash, source, fileToKey) - private val leafFile = LocalFile.resolve("subdir/leaf-file", leafHash, source, fileToKey) - - private val md5HashGenerator = MD5HashGenerator.md5File[Id](_) def putObjectRequest(bucket: Bucket, remoteKey: RemoteKey, localFile: LocalFile): (String, String, File) = (bucket.name, remoteKey.key, localFile.file) @@ -37,7 +32,7 @@ class SyncSuite val s3Client = new RecordingClient(testBucket, S3ObjectsData( byHash = Map(), byKey = Map())) - Sync.run(config, s3Client, md5HashGenerator, logger) + Sync.run(config, s3Client) it("uploads all files") { val expectedUploads = Map( "subdir/leaf-file" -> leafRemoteKey, @@ -63,7 +58,7 @@ class SyncSuite RemoteKey("prefix/root-file") -> HashModified(rootHash, lastModified), RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified))) val s3Client = new RecordingClient(testBucket, s3ObjectsData) - Sync.run(config, s3Client, md5HashGenerator, logger) + Sync.run(config, s3Client) it("uploads nothing") { val expectedUploads = Map() assertResult(expectedUploads)(s3Client.uploadsRecord) @@ -87,7 +82,7 @@ class SyncSuite RemoteKey("prefix/root-file-old") -> HashModified(rootHash, lastModified), RemoteKey("prefix/subdir/leaf-file") -> HashModified(leafHash, lastModified))) val s3Client = new RecordingClient(testBucket, s3ObjectsData) - Sync.run(config, s3Client, md5HashGenerator, logger) + Sync.run(config, s3Client) it("uploads nothing") { val expectedUploads = Map() assertResult(expectedUploads)(s3Client.uploadsRecord) @@ -115,7 +110,7 @@ class SyncSuite byKey = Map( deletedKey -> HashModified(deletedHash, lastModified))) val s3Client = new RecordingClient(testBucket, s3ObjectsData) - Sync.run(config, s3Client, md5HashGenerator, logger) + Sync.run(config, s3Client) it("deleted key") { val expectedDeletions = Set(deletedKey) assertResult(expectedDeletions)(s3Client.deletionsRecord) @@ -125,7 +120,7 @@ class SyncSuite val config: Config = Config(Bucket("bucket"), prefix, source = source, filters = List(Exclude("leaf"))) val s3ObjectsData = S3ObjectsData(Map(), Map()) val s3Client = new RecordingClient(testBucket, s3ObjectsData) - Sync.run(config, s3Client, md5HashGenerator, logger) + Sync.run(config, s3Client) it("is not uploaded") { val expectedUploads = Map( "root-file" -> rootRemoteKey