Support multiple filters (#18)
* Support multiple filters * Clean up imports * [S3ClientLogging] log the remote key value * Update changelog, readme and long arg name * [SyncSuite] update test
This commit is contained in:
parent
37ac41093e
commit
bffc6c032c
10 changed files with 18 additions and 19 deletions
|
@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file.
|
|||
The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]], and this project adheres to
|
||||
[[https://semver.org/spec/v2.0.0.html][Semantic Versioning]].
|
||||
|
||||
* [0.3.0] - ???
|
||||
* [0.3.0] - 2019-05-23
|
||||
|
||||
** Added
|
||||
|
||||
|
|
10
README.org
10
README.org
|
@ -14,11 +14,11 @@ hash of the file contents.
|
|||
s3thorp
|
||||
Usage: s3thorp [options]
|
||||
|
||||
-s, --source <value> Source directory to sync to S3
|
||||
-b, --bucket <value> S3 bucket name
|
||||
-p, --prefix <value> Prefix within the S3 Bucket
|
||||
-f, --filter <value> Exclude matching paths
|
||||
-v, --verbose <value> Verbosity level (1-5)
|
||||
-s, --source <value> Source directory to sync to S3
|
||||
-b, --bucket <value> S3 bucket name
|
||||
-p, --prefix <value> Prefix within the S3 Bucket
|
||||
-f, --filters <value>[,<values>]Exclude matching paths
|
||||
-v, --verbose <value> Verbosity level (1-5)
|
||||
#+end_example
|
||||
|
||||
* Behaviour
|
||||
|
|
|
@ -5,7 +5,7 @@ import java.io.File
|
|||
case class Config(bucket: Bucket = Bucket(""),
|
||||
prefix: RemoteKey = RemoteKey(""),
|
||||
verbose: Int = 1,
|
||||
filter: Filter = Filter(),
|
||||
filters: Seq[Filter] = List(),
|
||||
source: File
|
||||
) {
|
||||
require(source.isDirectory, s"Source must be a directory: $source")
|
||||
|
|
|
@ -10,7 +10,7 @@ trait LocalFileStream
|
|||
(implicit c: Config): Stream[LocalFile] = {
|
||||
log5(s"- Entering: $file")
|
||||
val files = for {
|
||||
f <- dirPaths(file) filter { f => c.filter isIncluded f.toPath }
|
||||
f <- dirPaths(file) filter { f => c.filters.forall { filter => filter isIncluded f.toPath } }
|
||||
fs <- recurseIntoSubDirectories(f)
|
||||
} yield fs
|
||||
log5(s"- Leaving: $file")
|
||||
|
|
|
@ -25,8 +25,8 @@ object ParseArgs {
|
|||
opt[String]('p', "prefix")
|
||||
.action((str, c) => c.copy(prefix = RemoteKey(str)))
|
||||
.text("Prefix within the S3 Bucket"),
|
||||
opt[String]('f', "filter")
|
||||
.action((str,c) => c.copy(filter = Filter(str)))
|
||||
opt[Seq[String]]('f', "filters")
|
||||
.action((str,c) => c.copy(filters = str.map(Filter)))
|
||||
.text("Exclude matching paths"),
|
||||
opt[Int]('v', "verbose")
|
||||
.validate(i =>
|
||||
|
|
|
@ -6,7 +6,8 @@ import cats.effect.IO
|
|||
trait SyncLogging extends Logging {
|
||||
|
||||
def logRunStart(c: Config): IO[Unit] = IO {
|
||||
log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}")(c)
|
||||
log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, " +
|
||||
s"Filter: ${c.filters.map{f => f.filter}.mkString(""", """)}")(c)
|
||||
}
|
||||
|
||||
def logRunFinished(actions: List[S3Action])
|
||||
|
|
|
@ -28,7 +28,7 @@ trait S3ClientLogging
|
|||
bucket: Bucket)
|
||||
(implicit c: Config): PutObjectResponse => IO[PutObjectResponse] = {
|
||||
in => IO {
|
||||
log4(s"Uploading: ${bucket.name}:${localFile.remoteKey}")
|
||||
log4(s"Uploading: ${bucket.name}:${localFile.remoteKey.key}")
|
||||
in
|
||||
}
|
||||
}
|
||||
|
@ -37,7 +37,7 @@ trait S3ClientLogging
|
|||
bucket: Bucket)
|
||||
(implicit c: Config): PutObjectResponse => IO[Unit] = {
|
||||
in =>IO {
|
||||
log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey}")
|
||||
log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -2,8 +2,6 @@ package net.kemitix.s3thorp
|
|||
|
||||
import java.time.Instant
|
||||
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class ActionGeneratorSuite
|
||||
extends UnitTest
|
||||
with KeyGenerator {
|
||||
|
|
|
@ -155,7 +155,7 @@ class SyncSuite
|
|||
}
|
||||
}
|
||||
describe("when a file is file is excluded") {
|
||||
val filteredConfig = config.copy(filter = Filter("leaf"), verbose = 5)
|
||||
val filteredConfig = config.copy(filters = List(Filter("leaf")))
|
||||
val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(Map(), Map()))
|
||||
sync.run(filteredConfig).unsafeRunSync
|
||||
it("is not uploaded") {
|
||||
|
|
|
@ -1,17 +1,17 @@
|
|||
package net.kemitix.s3thorp.awssdk
|
||||
|
||||
import java.time.Instant
|
||||
import java.time.temporal.ChronoUnit
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import cats.effect.IO
|
||||
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
|
||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||
import net.kemitix.s3thorp.{Bucket, Config, HashModified, KeyModified, LastModified, MD5Hash, Main, RemoteKey, Resource}
|
||||
import net.kemitix.s3thorp._
|
||||
import org.scalatest.FunSpec
|
||||
import software.amazon.awssdk.services.s3
|
||||
import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, ListObjectsV2Response, S3Object}
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
class ThorpS3ClientSuite extends FunSpec {
|
||||
|
||||
describe("listObjectsInPrefix") {
|
||||
|
|
Loading…
Reference in a new issue