Support multiple filters (#18)
* Support multiple filters * Clean up imports * [S3ClientLogging] log the remote key value * Update changelog, readme and long arg name * [SyncSuite] update test
This commit is contained in:
parent
37ac41093e
commit
bffc6c032c
10 changed files with 18 additions and 19 deletions
|
@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file.
|
||||||
The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]], and this project adheres to
|
The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]], and this project adheres to
|
||||||
[[https://semver.org/spec/v2.0.0.html][Semantic Versioning]].
|
[[https://semver.org/spec/v2.0.0.html][Semantic Versioning]].
|
||||||
|
|
||||||
* [0.3.0] - ???
|
* [0.3.0] - 2019-05-23
|
||||||
|
|
||||||
** Added
|
** Added
|
||||||
|
|
||||||
|
|
|
@ -17,7 +17,7 @@ hash of the file contents.
|
||||||
-s, --source <value> Source directory to sync to S3
|
-s, --source <value> Source directory to sync to S3
|
||||||
-b, --bucket <value> S3 bucket name
|
-b, --bucket <value> S3 bucket name
|
||||||
-p, --prefix <value> Prefix within the S3 Bucket
|
-p, --prefix <value> Prefix within the S3 Bucket
|
||||||
-f, --filter <value> Exclude matching paths
|
-f, --filters <value>[,<values>]Exclude matching paths
|
||||||
-v, --verbose <value> Verbosity level (1-5)
|
-v, --verbose <value> Verbosity level (1-5)
|
||||||
#+end_example
|
#+end_example
|
||||||
|
|
||||||
|
|
|
@ -5,7 +5,7 @@ import java.io.File
|
||||||
case class Config(bucket: Bucket = Bucket(""),
|
case class Config(bucket: Bucket = Bucket(""),
|
||||||
prefix: RemoteKey = RemoteKey(""),
|
prefix: RemoteKey = RemoteKey(""),
|
||||||
verbose: Int = 1,
|
verbose: Int = 1,
|
||||||
filter: Filter = Filter(),
|
filters: Seq[Filter] = List(),
|
||||||
source: File
|
source: File
|
||||||
) {
|
) {
|
||||||
require(source.isDirectory, s"Source must be a directory: $source")
|
require(source.isDirectory, s"Source must be a directory: $source")
|
||||||
|
|
|
@ -10,7 +10,7 @@ trait LocalFileStream
|
||||||
(implicit c: Config): Stream[LocalFile] = {
|
(implicit c: Config): Stream[LocalFile] = {
|
||||||
log5(s"- Entering: $file")
|
log5(s"- Entering: $file")
|
||||||
val files = for {
|
val files = for {
|
||||||
f <- dirPaths(file) filter { f => c.filter isIncluded f.toPath }
|
f <- dirPaths(file) filter { f => c.filters.forall { filter => filter isIncluded f.toPath } }
|
||||||
fs <- recurseIntoSubDirectories(f)
|
fs <- recurseIntoSubDirectories(f)
|
||||||
} yield fs
|
} yield fs
|
||||||
log5(s"- Leaving: $file")
|
log5(s"- Leaving: $file")
|
||||||
|
|
|
@ -25,8 +25,8 @@ object ParseArgs {
|
||||||
opt[String]('p', "prefix")
|
opt[String]('p', "prefix")
|
||||||
.action((str, c) => c.copy(prefix = RemoteKey(str)))
|
.action((str, c) => c.copy(prefix = RemoteKey(str)))
|
||||||
.text("Prefix within the S3 Bucket"),
|
.text("Prefix within the S3 Bucket"),
|
||||||
opt[String]('f', "filter")
|
opt[Seq[String]]('f', "filters")
|
||||||
.action((str,c) => c.copy(filter = Filter(str)))
|
.action((str,c) => c.copy(filters = str.map(Filter)))
|
||||||
.text("Exclude matching paths"),
|
.text("Exclude matching paths"),
|
||||||
opt[Int]('v', "verbose")
|
opt[Int]('v', "verbose")
|
||||||
.validate(i =>
|
.validate(i =>
|
||||||
|
|
|
@ -6,7 +6,8 @@ import cats.effect.IO
|
||||||
trait SyncLogging extends Logging {
|
trait SyncLogging extends Logging {
|
||||||
|
|
||||||
def logRunStart(c: Config): IO[Unit] = IO {
|
def logRunStart(c: Config): IO[Unit] = IO {
|
||||||
log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}")(c)
|
log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, " +
|
||||||
|
s"Filter: ${c.filters.map{f => f.filter}.mkString(""", """)}")(c)
|
||||||
}
|
}
|
||||||
|
|
||||||
def logRunFinished(actions: List[S3Action])
|
def logRunFinished(actions: List[S3Action])
|
||||||
|
|
|
@ -28,7 +28,7 @@ trait S3ClientLogging
|
||||||
bucket: Bucket)
|
bucket: Bucket)
|
||||||
(implicit c: Config): PutObjectResponse => IO[PutObjectResponse] = {
|
(implicit c: Config): PutObjectResponse => IO[PutObjectResponse] = {
|
||||||
in => IO {
|
in => IO {
|
||||||
log4(s"Uploading: ${bucket.name}:${localFile.remoteKey}")
|
log4(s"Uploading: ${bucket.name}:${localFile.remoteKey.key}")
|
||||||
in
|
in
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -37,7 +37,7 @@ trait S3ClientLogging
|
||||||
bucket: Bucket)
|
bucket: Bucket)
|
||||||
(implicit c: Config): PutObjectResponse => IO[Unit] = {
|
(implicit c: Config): PutObjectResponse => IO[Unit] = {
|
||||||
in =>IO {
|
in =>IO {
|
||||||
log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey}")
|
log3(s"Uploaded: ${bucket.name}:${localFile.remoteKey.key}")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,8 +2,6 @@ package net.kemitix.s3thorp
|
||||||
|
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
import org.scalatest.FunSpec
|
|
||||||
|
|
||||||
class ActionGeneratorSuite
|
class ActionGeneratorSuite
|
||||||
extends UnitTest
|
extends UnitTest
|
||||||
with KeyGenerator {
|
with KeyGenerator {
|
||||||
|
|
|
@ -155,7 +155,7 @@ class SyncSuite
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("when a file is file is excluded") {
|
describe("when a file is file is excluded") {
|
||||||
val filteredConfig = config.copy(filter = Filter("leaf"), verbose = 5)
|
val filteredConfig = config.copy(filters = List(Filter("leaf")))
|
||||||
val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(Map(), Map()))
|
val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(Map(), Map()))
|
||||||
sync.run(filteredConfig).unsafeRunSync
|
sync.run(filteredConfig).unsafeRunSync
|
||||||
it("is not uploaded") {
|
it("is not uploaded") {
|
||||||
|
|
|
@ -1,17 +1,17 @@
|
||||||
package net.kemitix.s3thorp.awssdk
|
package net.kemitix.s3thorp.awssdk
|
||||||
|
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
import java.time.temporal.ChronoUnit
|
|
||||||
|
|
||||||
import scala.collection.JavaConverters._
|
|
||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
|
import com.github.j5ik2o.reactive.aws.s3.S3AsyncClient
|
||||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||||
import net.kemitix.s3thorp.{Bucket, Config, HashModified, KeyModified, LastModified, MD5Hash, Main, RemoteKey, Resource}
|
import net.kemitix.s3thorp._
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
import software.amazon.awssdk.services.s3
|
import software.amazon.awssdk.services.s3
|
||||||
import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, ListObjectsV2Response, S3Object}
|
import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, ListObjectsV2Response, S3Object}
|
||||||
|
|
||||||
|
import scala.collection.JavaConverters._
|
||||||
|
|
||||||
class ThorpS3ClientSuite extends FunSpec {
|
class ThorpS3ClientSuite extends FunSpec {
|
||||||
|
|
||||||
describe("listObjectsInPrefix") {
|
describe("listObjectsInPrefix") {
|
||||||
|
|
Loading…
Reference in a new issue