Support multiple filters (#18)

* Support multiple filters

* Clean up imports

* [S3ClientLogging] log the remote key value

* Update changelog, readme and long arg name

* [SyncSuite] update test
This commit is contained in:
Paul Campbell 2019-05-23 19:35:48 +01:00 committed by GitHub
parent 37ac41093e
commit bffc6c032c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 18 additions and 19 deletions

View file

@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file.
The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]], and this project adheres to
[[https://semver.org/spec/v2.0.0.html][Semantic Versioning]].
* [0.3.0] - ???
* [0.3.0] - 2019-05-23
** Added

View file

@ -14,11 +14,11 @@ hash of the file contents.
s3thorp
Usage: s3thorp [options]
-s, --source <value> Source directory to sync to S3
-b, --bucket <value> S3 bucket name
-p, --prefix <value> Prefix within the S3 Bucket
-f, --filter <value> Exclude matching paths
-v, --verbose <value> Verbosity level (1-5)
-s, --source <value> Source directory to sync to S3
-b, --bucket <value> S3 bucket name
-p, --prefix <value> Prefix within the S3 Bucket
-f, --filters <value>[,<values>]Exclude matching paths
-v, --verbose <value> Verbosity level (1-5)
#+end_example
* Behaviour

View file

@ -5,7 +5,7 @@ import java.io.File
case class Config(bucket: Bucket = Bucket(""),
prefix: RemoteKey = RemoteKey(""),
verbose: Int = 1,
filter: Filter = Filter(),
filters: Seq[Filter] = List(),
source: File
) {
require(source.isDirectory, s"Source must be a directory: $source")

View file

@ -10,7 +10,7 @@ trait LocalFileStream
(implicit c: Config): Stream[LocalFile] = {
log5(s"- Entering: $file")
val files = for {
f <- dirPaths(file) filter { f => c.filter isIncluded f.toPath }
f <- dirPaths(file) filter { f => c.filters.forall { filter => filter isIncluded f.toPath } }
fs <- recurseIntoSubDirectories(f)
} yield fs
log5(s"- Leaving: $file")

View file

@ -25,8 +25,8 @@ object ParseArgs {
opt[String]('p', "prefix")
.action((str, c) => c.copy(prefix = RemoteKey(str)))
.text("Prefix within the S3 Bucket"),
opt[String]('f', "filter")
.action((str,c) => c.copy(filter = Filter(str)))
opt[Seq[String]]('f', "filters")
.action((str,c) => c.copy(filters = str.map(Filter)))
.text("Exclude matching paths"),
opt[Int]('v', "verbose")
.validate(i =>

View file

@ -6,7 +6,8 @@ import cats.effect.IO
trait SyncLogging extends Logging {
def logRunStart(c: Config): IO[Unit] = IO {
log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}")(c)
log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, " +
s"Filter: ${c.filters.map{f => f.filter}.mkString(""", """)}")(c)
}
def logRunFinished(actions: List[S3Action])

View file

@ -28,7 +28,7 @@ trait S3ClientLogging
bucket: Bucket)
(implicit c: Config): PutObjectResponse => IO[PutObjectResponse] = {
in => IO {
log4(s"Uploading: ${bucket.name}:${localFile.remoteKey}")
log4(s"Uploading: ${bucket.name}:${localFile.remoteKey.key}")
in
}
}
@ -37,7 +37,7 @@ trait S3ClientLogging
bucket: Bucket)
(implicit c: Config): PutObjectResponse => IO[Unit] = {
in =>IO {
log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey}")
log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}")
}
}

View file

@ -2,8 +2,6 @@ package net.kemitix.s3thorp
import java.time.Instant
import org.scalatest.FunSpec
class ActionGeneratorSuite
extends UnitTest
with KeyGenerator {

View file

@ -155,7 +155,7 @@ class SyncSuite
}
}
describe("when a file is file is excluded") {
val filteredConfig = config.copy(filter = Filter("leaf"), verbose = 5)
val filteredConfig = config.copy(filters = List(Filter("leaf")))
val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(Map(), Map()))
sync.run(filteredConfig).unsafeRunSync
it("is not uploaded") {

View file

@ -1,17 +1,17 @@
package net.kemitix.s3thorp.awssdk
import java.time.Instant
import java.time.temporal.ChronoUnit
import scala.collection.JavaConverters._
import cats.effect.IO
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
import net.kemitix.s3thorp.{Bucket, Config, HashModified, KeyModified, LastModified, MD5Hash, Main, RemoteKey, Resource}
import net.kemitix.s3thorp._
import org.scalatest.FunSpec
import software.amazon.awssdk.services.s3
import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, ListObjectsV2Response, S3Object}
import scala.collection.JavaConverters._
class ThorpS3ClientSuite extends FunSpec {
describe("listObjectsInPrefix") {