diff --git a/CHANGELOG.org b/CHANGELOG.org index 445fc68..8ba203c 100644 --- a/CHANGELOG.org +++ b/CHANGELOG.org @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]], and this project adheres to [[https://semver.org/spec/v2.0.0.html][Semantic Versioning]]. -* [0.3.0] - ??? +* [0.3.0] - 2019-05-23 ** Added diff --git a/README.org b/README.org index 9f3509b..b36aa7e 100644 --- a/README.org +++ b/README.org @@ -14,11 +14,11 @@ hash of the file contents. s3thorp Usage: s3thorp [options] - -s, --source Source directory to sync to S3 - -b, --bucket S3 bucket name - -p, --prefix Prefix within the S3 Bucket - -f, --filter Exclude matching paths - -v, --verbose Verbosity level (1-5) + -s, --source Source directory to sync to S3 + -b, --bucket S3 bucket name + -p, --prefix Prefix within the S3 Bucket + -f, --filters [,]Exclude matching paths + -v, --verbose Verbosity level (1-5) #+end_example * Behaviour diff --git a/src/main/scala/net/kemitix/s3thorp/Config.scala b/src/main/scala/net/kemitix/s3thorp/Config.scala index aa382a2..9eb7350 100644 --- a/src/main/scala/net/kemitix/s3thorp/Config.scala +++ b/src/main/scala/net/kemitix/s3thorp/Config.scala @@ -5,7 +5,7 @@ import java.io.File case class Config(bucket: Bucket = Bucket(""), prefix: RemoteKey = RemoteKey(""), verbose: Int = 1, - filter: Filter = Filter(), + filters: Seq[Filter] = List(), source: File ) { require(source.isDirectory, s"Source must be a directory: $source") diff --git a/src/main/scala/net/kemitix/s3thorp/LocalFileStream.scala b/src/main/scala/net/kemitix/s3thorp/LocalFileStream.scala index 70916f1..d0867c9 100644 --- a/src/main/scala/net/kemitix/s3thorp/LocalFileStream.scala +++ b/src/main/scala/net/kemitix/s3thorp/LocalFileStream.scala @@ -10,7 +10,7 @@ trait LocalFileStream (implicit c: Config): Stream[LocalFile] = { log5(s"- Entering: $file") val files = for { - f <- dirPaths(file) filter { f => c.filter isIncluded f.toPath } + f <- dirPaths(file) filter { f => c.filters.forall { filter => filter isIncluded f.toPath } } fs <- recurseIntoSubDirectories(f) } yield fs log5(s"- Leaving: $file") diff --git a/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala b/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala index 6bcc0d5..eedc4b9 100644 --- a/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala +++ b/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala @@ -25,8 +25,8 @@ object ParseArgs { opt[String]('p', "prefix") .action((str, c) => c.copy(prefix = RemoteKey(str))) .text("Prefix within the S3 Bucket"), - opt[String]('f', "filter") - .action((str,c) => c.copy(filter = Filter(str))) + opt[Seq[String]]('f', "filters") + .action((str,c) => c.copy(filters = str.map(Filter))) .text("Exclude matching paths"), opt[Int]('v', "verbose") .validate(i => diff --git a/src/main/scala/net/kemitix/s3thorp/SyncLogging.scala b/src/main/scala/net/kemitix/s3thorp/SyncLogging.scala index c2374f1..f2eb83f 100644 --- a/src/main/scala/net/kemitix/s3thorp/SyncLogging.scala +++ b/src/main/scala/net/kemitix/s3thorp/SyncLogging.scala @@ -6,7 +6,8 @@ import cats.effect.IO trait SyncLogging extends Logging { def logRunStart(c: Config): IO[Unit] = IO { - log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}")(c) + log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, " + + s"Filter: ${c.filters.map{f => f.filter}.mkString(""", """)}")(c) } def logRunFinished(actions: List[S3Action]) diff --git a/src/main/scala/net/kemitix/s3thorp/awssdk/S3ClientLogging.scala b/src/main/scala/net/kemitix/s3thorp/awssdk/S3ClientLogging.scala index a9b0e33..4be7f7e 100644 --- a/src/main/scala/net/kemitix/s3thorp/awssdk/S3ClientLogging.scala +++ b/src/main/scala/net/kemitix/s3thorp/awssdk/S3ClientLogging.scala @@ -28,7 +28,7 @@ trait S3ClientLogging bucket: Bucket) (implicit c: Config): PutObjectResponse => IO[PutObjectResponse] = { in => IO { - log4(s"Uploading: ${bucket.name}:${localFile.remoteKey}") + log4(s"Uploading: ${bucket.name}:${localFile.remoteKey.key}") in } } @@ -37,7 +37,7 @@ trait S3ClientLogging bucket: Bucket) (implicit c: Config): PutObjectResponse => IO[Unit] = { in =>IO { - log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey}") + log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}") } } diff --git a/src/test/scala/net/kemitix/s3thorp/ActionGeneratorSuite.scala b/src/test/scala/net/kemitix/s3thorp/ActionGeneratorSuite.scala index ec7dd00..17680ea 100644 --- a/src/test/scala/net/kemitix/s3thorp/ActionGeneratorSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/ActionGeneratorSuite.scala @@ -2,8 +2,6 @@ package net.kemitix.s3thorp import java.time.Instant -import org.scalatest.FunSpec - class ActionGeneratorSuite extends UnitTest with KeyGenerator { diff --git a/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala b/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala index daaa876..f0399d2 100644 --- a/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala @@ -155,7 +155,7 @@ class SyncSuite } } describe("when a file is file is excluded") { - val filteredConfig = config.copy(filter = Filter("leaf"), verbose = 5) + val filteredConfig = config.copy(filters = List(Filter("leaf"))) val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(Map(), Map())) sync.run(filteredConfig).unsafeRunSync it("is not uploaded") { diff --git a/src/test/scala/net/kemitix/s3thorp/awssdk/ThorpS3ClientSuite.scala b/src/test/scala/net/kemitix/s3thorp/awssdk/ThorpS3ClientSuite.scala index 7eb29d9..c2b3016 100644 --- a/src/test/scala/net/kemitix/s3thorp/awssdk/ThorpS3ClientSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/awssdk/ThorpS3ClientSuite.scala @@ -1,17 +1,17 @@ package net.kemitix.s3thorp.awssdk import java.time.Instant -import java.time.temporal.ChronoUnit -import scala.collection.JavaConverters._ import cats.effect.IO import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient -import net.kemitix.s3thorp.{Bucket, Config, HashModified, KeyModified, LastModified, MD5Hash, Main, RemoteKey, Resource} +import net.kemitix.s3thorp._ import org.scalatest.FunSpec import software.amazon.awssdk.services.s3 import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, ListObjectsV2Response, S3Object} +import scala.collection.JavaConverters._ + class ThorpS3ClientSuite extends FunSpec { describe("listObjectsInPrefix") {