Rename filter as exclude (#23)
* [Filter => Exclude] rename class * [Config] rename filters field as excludes * [readme,ParseArgs] change commandline arg f to x and filters to excludes * [SyncSuite] rename val * [ExcludeSuite] rename vars * [SyncLogging] Update message
This commit is contained in:
parent
082babb94d
commit
35e85702aa
8 changed files with 23 additions and 23 deletions
|
@ -17,7 +17,7 @@ hash of the file contents.
|
||||||
-s, --source <value> Source directory to sync to S3
|
-s, --source <value> Source directory to sync to S3
|
||||||
-b, --bucket <value> S3 bucket name
|
-b, --bucket <value> S3 bucket name
|
||||||
-p, --prefix <value> Prefix within the S3 Bucket
|
-p, --prefix <value> Prefix within the S3 Bucket
|
||||||
-f, --filters <value>[,<values>]Exclude matching paths
|
-x, --exclude <value>[,<values>] Exclude matching paths
|
||||||
-v, --verbose <value> Verbosity level (1-5)
|
-v, --verbose <value> Verbosity level (1-5)
|
||||||
#+end_example
|
#+end_example
|
||||||
|
|
||||||
|
|
|
@ -5,7 +5,7 @@ import java.io.File
|
||||||
case class Config(bucket: Bucket = Bucket(""),
|
case class Config(bucket: Bucket = Bucket(""),
|
||||||
prefix: RemoteKey = RemoteKey(""),
|
prefix: RemoteKey = RemoteKey(""),
|
||||||
verbose: Int = 1,
|
verbose: Int = 1,
|
||||||
filters: Seq[Filter] = List(),
|
excludes: Seq[Exclude] = List(),
|
||||||
multiPartThreshold: Long = 1024 * 1024 * 5,
|
multiPartThreshold: Long = 1024 * 1024 * 5,
|
||||||
maxRetries: Int = 3,
|
maxRetries: Int = 3,
|
||||||
source: File
|
source: File
|
||||||
|
|
|
@ -4,9 +4,9 @@ import java.nio.file.Path
|
||||||
import java.util.function.Predicate
|
import java.util.function.Predicate
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
case class Filter(filter: String = "!.*") {
|
case class Exclude(exclude: String = "!.*") {
|
||||||
|
|
||||||
lazy val predicate: Predicate[String] = Pattern.compile(filter).asPredicate()
|
lazy val predicate: Predicate[String] = Pattern.compile(exclude).asPredicate()
|
||||||
|
|
||||||
def isIncluded(path: Path): Boolean = !isExcluded(path)
|
def isIncluded(path: Path): Boolean = !isExcluded(path)
|
||||||
|
|
|
@ -10,7 +10,7 @@ trait LocalFileStream
|
||||||
(implicit c: Config): Stream[LocalFile] = {
|
(implicit c: Config): Stream[LocalFile] = {
|
||||||
log5(s"- Entering: $file")
|
log5(s"- Entering: $file")
|
||||||
val files = for {
|
val files = for {
|
||||||
f <- dirPaths(file) filter { f => c.filters.forall { filter => filter isIncluded f.toPath } }
|
f <- dirPaths(file) filter { f => c.excludes.forall { filter => filter isIncluded f.toPath } }
|
||||||
fs <- recurseIntoSubDirectories(f)
|
fs <- recurseIntoSubDirectories(f)
|
||||||
} yield fs
|
} yield fs
|
||||||
log5(s"- Leaving: $file")
|
log5(s"- Leaving: $file")
|
||||||
|
|
|
@ -25,8 +25,8 @@ object ParseArgs {
|
||||||
opt[String]('p', "prefix")
|
opt[String]('p', "prefix")
|
||||||
.action((str, c) => c.copy(prefix = RemoteKey(str)))
|
.action((str, c) => c.copy(prefix = RemoteKey(str)))
|
||||||
.text("Prefix within the S3 Bucket"),
|
.text("Prefix within the S3 Bucket"),
|
||||||
opt[Seq[String]]('f', "filters")
|
opt[Seq[String]]('x', "exclude")
|
||||||
.action((str,c) => c.copy(filters = str.map(Filter)))
|
.action((str,c) => c.copy(excludes = str.map(Exclude)))
|
||||||
.text("Exclude matching paths"),
|
.text("Exclude matching paths"),
|
||||||
opt[Int]('v', "verbose")
|
opt[Int]('v', "verbose")
|
||||||
.validate(i =>
|
.validate(i =>
|
||||||
|
|
|
@ -7,7 +7,7 @@ trait SyncLogging extends Logging {
|
||||||
|
|
||||||
def logRunStart(implicit c: Config): Unit =
|
def logRunStart(implicit c: Config): Unit =
|
||||||
log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, " +
|
log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, " +
|
||||||
s"Filter: ${c.filters.map{f => f.filter}.mkString(""", """)}")(c)
|
s"Exclude: ${c.excludes.map{ f => f.exclude}.mkString(""", """)}")(c)
|
||||||
|
|
||||||
def logFileScan(implicit c: Config): Unit =
|
def logFileScan(implicit c: Config): Unit =
|
||||||
log1(s"Scanning local files: ${c.source}...")
|
log1(s"Scanning local files: ${c.source}...")
|
||||||
|
|
|
@ -2,42 +2,42 @@ package net.kemitix.s3thorp
|
||||||
|
|
||||||
import java.nio.file.{Path, Paths}
|
import java.nio.file.{Path, Paths}
|
||||||
|
|
||||||
class FilterSuite extends UnitTest {
|
class ExcludeSuite extends UnitTest {
|
||||||
|
|
||||||
describe("default filter") {
|
describe("default exclude") {
|
||||||
val filter = Filter()
|
val exclude = Exclude()
|
||||||
val paths: List[Path] = List("/a-file", "a-file", "path/to/a/file", "/path/to/a/file",
|
val paths: List[Path] = List("/a-file", "a-file", "path/to/a/file", "/path/to/a/file",
|
||||||
"/home/pcampbell/repos/kemitix/s3thorp/target/scala-2.12/test-classes/net/kemitix/s3thorp/upload/subdir"
|
"/home/pcampbell/repos/kemitix/s3thorp/target/scala-2.12/test-classes/net/kemitix/s3thorp/upload/subdir"
|
||||||
) map { p => Paths.get(p)}
|
) map { p => Paths.get(p)}
|
||||||
it("should not exclude files") {
|
it("should not exclude files") {
|
||||||
paths.foreach(path => { assertResult(false)(filter.isExcluded(path)) })
|
paths.foreach(path => { assertResult(false)(exclude.isExcluded(path)) })
|
||||||
}
|
}
|
||||||
it("should include files") {
|
it("should include files") {
|
||||||
paths.foreach(path => assertResult(true)(filter.isIncluded(path)))
|
paths.foreach(path => assertResult(true)(exclude.isIncluded(path)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("directory exact match filter '/upload/subdir/'") {
|
describe("directory exact match exclude '/upload/subdir/'") {
|
||||||
val filter = Filter("/upload/subdir/")
|
val exclude = Exclude("/upload/subdir/")
|
||||||
it("exclude matching directory") {
|
it("exclude matching directory") {
|
||||||
val matching = Paths.get("/upload/subdir/leaf-file")
|
val matching = Paths.get("/upload/subdir/leaf-file")
|
||||||
assertResult(true)(filter.isExcluded(matching))
|
assertResult(true)(exclude.isExcluded(matching))
|
||||||
}
|
}
|
||||||
it("include non-matching files") {
|
it("include non-matching files") {
|
||||||
val nonMatching = Paths.get("/upload/other-file")
|
val nonMatching = Paths.get("/upload/other-file")
|
||||||
assertResult(true)(filter.isIncluded(nonMatching))
|
assertResult(true)(exclude.isIncluded(nonMatching))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("file partial match 'root'") {
|
describe("file partial match 'root'") {
|
||||||
val filter = Filter("root")
|
val exclude = Exclude("root")
|
||||||
it("exclude matching file '/upload/root-file") {
|
it("exclude matching file '/upload/root-file") {
|
||||||
val matching = Paths.get("/upload/root-file")
|
val matching = Paths.get("/upload/root-file")
|
||||||
assertResult(true)(filter.isExcluded(matching))
|
assertResult(true)(exclude.isExcluded(matching))
|
||||||
}
|
}
|
||||||
it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
|
it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") {
|
||||||
val nonMatching1 = Paths.get("/test-file-for-hash.txt")
|
val nonMatching1 = Paths.get("/test-file-for-hash.txt")
|
||||||
val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
|
val nonMatching2 = Paths.get("/upload/subdir/leaf-file")
|
||||||
assertResult(true)(filter.isIncluded(nonMatching1))
|
assertResult(true)(exclude.isIncluded(nonMatching1))
|
||||||
assertResult(true)(filter.isIncluded(nonMatching2))
|
assertResult(true)(exclude.isIncluded(nonMatching2))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -158,9 +158,9 @@ class SyncSuite
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
describe("when a file is file is excluded") {
|
describe("when a file is file is excluded") {
|
||||||
val filteredConfig = config.copy(filters = List(Filter("leaf")))
|
val configWithExclusion = config.copy(excludes = List(Exclude("leaf")))
|
||||||
val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(Map(), Map()))
|
val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(Map(), Map()))
|
||||||
sync.run(filteredConfig).unsafeRunSync
|
sync.run(configWithExclusion).unsafeRunSync
|
||||||
it("is not uploaded") {
|
it("is not uploaded") {
|
||||||
val expectedUploads = Map(
|
val expectedUploads = Map(
|
||||||
"root-file" -> rootRemoteKey
|
"root-file" -> rootRemoteKey
|
||||||
|
|
Loading…
Reference in a new issue