Support multiple filters (#18)

* Support multiple filters

* Clean up imports

* [S3ClientLogging] log the remote key value

* Update changelog, readme and long arg name

* [SyncSuite] update test

CHANGELOG.org

@@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file.
 The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]], and this project adheres to
 [[https://semver.org/spec/v2.0.0.html][Semantic Versioning]].
 
-* [0.3.0] - ???
+* [0.3.0] - 2019-05-23
 
 ** Added
 

README.org

@@ -14,11 +14,11 @@ hash of the file contents.
     s3thorp
     Usage: s3thorp [options]
 
-      -s, --source <value>   Source directory to sync to S3
-      -b, --bucket <value>   S3 bucket name
-      -p, --prefix <value>   Prefix within the S3 Bucket
-      -f, --filter <value>   Exclude matching paths
-      -v, --verbose <value>  Verbosity level (1-5)
+      -s, --source <value>             Source directory to sync to S3
+      -b, --bucket <value>             S3 bucket name
+      -p, --prefix <value>             Prefix within the S3 Bucket
+      -f, --filters <value>[,<values>]Exclude matching paths
+      -v, --verbose <value>            Verbosity level (1-5)
   #+end_example
 
 * Behaviour

src/main/scala/net/kemitix/s3thorp/Config.scala

@@ -5,7 +5,7 @@ import java.io.File
 case class Config(bucket: Bucket = Bucket(""),
                   prefix: RemoteKey = RemoteKey(""),
                   verbose: Int = 1,
-                  filter: Filter = Filter(),
+                  filters: Seq[Filter] = List(),
                   source: File
                ) {
   require(source.isDirectory, s"Source must be a directory: $source")

src/main/scala/net/kemitix/s3thorp/LocalFileStream.scala

@@ -10,7 +10,7 @@ trait LocalFileStream
                (implicit c: Config): Stream[LocalFile] = {
     log5(s"- Entering: $file")
     val files = for {
-      f <- dirPaths(file) filter { f => c.filter isIncluded f.toPath }
+      f <- dirPaths(file) filter { f => c.filters.forall { filter => filter isIncluded f.toPath } }
       fs <- recurseIntoSubDirectories(f)
       } yield fs
     log5(s"-  Leaving: $file")

src/main/scala/net/kemitix/s3thorp/ParseArgs.scala

@@ -25,8 +25,8 @@ object ParseArgs {
       opt[String]('p', "prefix")
         .action((str, c) => c.copy(prefix = RemoteKey(str)))
         .text("Prefix within the S3 Bucket"),
-      opt[String]('f', "filter")
-        .action((str,c) => c.copy(filter = Filter(str)))
+      opt[Seq[String]]('f', "filters")
+        .action((str,c) => c.copy(filters = str.map(Filter)))
         .text("Exclude matching paths"),
       opt[Int]('v', "verbose")
         .validate(i =>

src/main/scala/net/kemitix/s3thorp/SyncLogging.scala

@@ -6,7 +6,8 @@ import cats.effect.IO
 trait SyncLogging extends Logging {
 
   def logRunStart(c: Config): IO[Unit] = IO {
-    log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}")(c)
+    log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, " +
+      s"Filter: ${c.filters.map{f => f.filter}.mkString(""", """)}")(c)
   }
 
   def logRunFinished(actions: List[S3Action])

src/main/scala/net/kemitix/s3thorp/awssdk/S3ClientLogging.scala

@@ -28,7 +28,7 @@ trait S3ClientLogging
                      bucket: Bucket)
                     (implicit c: Config): PutObjectResponse => IO[PutObjectResponse] = {
     in => IO {
-      log4(s"Uploading: ${bucket.name}:${localFile.remoteKey}")
+      log4(s"Uploading: ${bucket.name}:${localFile.remoteKey.key}")
       in
     }
   }
@@ -37,7 +37,7 @@ trait S3ClientLogging
                       bucket: Bucket)
                      (implicit c: Config): PutObjectResponse => IO[Unit] = {
     in =>IO {
-      log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey}")
+      log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}")
     }
   }
 

src/test/scala/net/kemitix/s3thorp/ActionGeneratorSuite.scala

@@ -2,8 +2,6 @@ package net.kemitix.s3thorp
 
 import java.time.Instant
 
-import org.scalatest.FunSpec
-
 class ActionGeneratorSuite
   extends UnitTest
     with KeyGenerator {

src/test/scala/net/kemitix/s3thorp/SyncSuite.scala

@@ -155,7 +155,7 @@ class SyncSuite
       }
     }
     describe("when a file is file is excluded") {
-      val filteredConfig = config.copy(filter = Filter("leaf"), verbose = 5)
+      val filteredConfig = config.copy(filters = List(Filter("leaf")))
       val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(Map(), Map()))
       sync.run(filteredConfig).unsafeRunSync
       it("is not uploaded") {

src/test/scala/net/kemitix/s3thorp/awssdk/ThorpS3ClientSuite.scala

@@ -1,17 +1,17 @@
 package net.kemitix.s3thorp.awssdk
 
 import java.time.Instant
-import java.time.temporal.ChronoUnit
 
-import scala.collection.JavaConverters._
 import cats.effect.IO
 import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
 import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
-import net.kemitix.s3thorp.{Bucket, Config, HashModified, KeyModified, LastModified, MD5Hash, Main, RemoteKey, Resource}
+import net.kemitix.s3thorp._
 import org.scalatest.FunSpec
 import software.amazon.awssdk.services.s3
 import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, ListObjectsV2Response, S3Object}
 
+import scala.collection.JavaConverters._
+
 class ThorpS3ClientSuite extends FunSpec {
 
   describe("listObjectsInPrefix") {