diff --git a/README.org b/README.org index b36aa7e..4bd4125 100644 --- a/README.org +++ b/README.org @@ -17,7 +17,7 @@ hash of the file contents. -s, --source Source directory to sync to S3 -b, --bucket S3 bucket name -p, --prefix Prefix within the S3 Bucket - -f, --filters [,]Exclude matching paths + -x, --exclude [,] Exclude matching paths -v, --verbose Verbosity level (1-5) #+end_example diff --git a/src/main/scala/net/kemitix/s3thorp/Config.scala b/src/main/scala/net/kemitix/s3thorp/Config.scala index b9a93a9..2f61aa5 100644 --- a/src/main/scala/net/kemitix/s3thorp/Config.scala +++ b/src/main/scala/net/kemitix/s3thorp/Config.scala @@ -5,7 +5,7 @@ import java.io.File case class Config(bucket: Bucket = Bucket(""), prefix: RemoteKey = RemoteKey(""), verbose: Int = 1, - filters: Seq[Filter] = List(), + excludes: Seq[Exclude] = List(), multiPartThreshold: Long = 1024 * 1024 * 5, maxRetries: Int = 3, source: File diff --git a/src/main/scala/net/kemitix/s3thorp/Filter.scala b/src/main/scala/net/kemitix/s3thorp/Exclude.scala similarity index 66% rename from src/main/scala/net/kemitix/s3thorp/Filter.scala rename to src/main/scala/net/kemitix/s3thorp/Exclude.scala index 339484c..818dc62 100644 --- a/src/main/scala/net/kemitix/s3thorp/Filter.scala +++ b/src/main/scala/net/kemitix/s3thorp/Exclude.scala @@ -4,9 +4,9 @@ import java.nio.file.Path import java.util.function.Predicate import java.util.regex.Pattern -case class Filter(filter: String = "!.*") { +case class Exclude(exclude: String = "!.*") { - lazy val predicate: Predicate[String] = Pattern.compile(filter).asPredicate() + lazy val predicate: Predicate[String] = Pattern.compile(exclude).asPredicate() def isIncluded(path: Path): Boolean = !isExcluded(path) diff --git a/src/main/scala/net/kemitix/s3thorp/LocalFileStream.scala b/src/main/scala/net/kemitix/s3thorp/LocalFileStream.scala index d0867c9..5aa4665 100644 --- a/src/main/scala/net/kemitix/s3thorp/LocalFileStream.scala +++ b/src/main/scala/net/kemitix/s3thorp/LocalFileStream.scala @@ -10,7 +10,7 @@ trait LocalFileStream (implicit c: Config): Stream[LocalFile] = { log5(s"- Entering: $file") val files = for { - f <- dirPaths(file) filter { f => c.filters.forall { filter => filter isIncluded f.toPath } } + f <- dirPaths(file) filter { f => c.excludes.forall { filter => filter isIncluded f.toPath } } fs <- recurseIntoSubDirectories(f) } yield fs log5(s"- Leaving: $file") diff --git a/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala b/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala index eedc4b9..052a8dd 100644 --- a/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala +++ b/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala @@ -25,8 +25,8 @@ object ParseArgs { opt[String]('p', "prefix") .action((str, c) => c.copy(prefix = RemoteKey(str))) .text("Prefix within the S3 Bucket"), - opt[Seq[String]]('f', "filters") - .action((str,c) => c.copy(filters = str.map(Filter))) + opt[Seq[String]]('x', "exclude") + .action((str,c) => c.copy(excludes = str.map(Exclude))) .text("Exclude matching paths"), opt[Int]('v', "verbose") .validate(i => diff --git a/src/main/scala/net/kemitix/s3thorp/SyncLogging.scala b/src/main/scala/net/kemitix/s3thorp/SyncLogging.scala index 695ba0e..b8758b4 100644 --- a/src/main/scala/net/kemitix/s3thorp/SyncLogging.scala +++ b/src/main/scala/net/kemitix/s3thorp/SyncLogging.scala @@ -7,7 +7,7 @@ trait SyncLogging extends Logging { def logRunStart(implicit c: Config): Unit = log1(s"Bucket: ${c.bucket.name}, Prefix: ${c.prefix.key}, Source: ${c.source}, " + - s"Filter: ${c.filters.map{f => f.filter}.mkString(""", """)}")(c) + s"Exclude: ${c.excludes.map{ f => f.exclude}.mkString(""", """)}")(c) def logFileScan(implicit c: Config): Unit = log1(s"Scanning local files: ${c.source}...") diff --git a/src/test/scala/net/kemitix/s3thorp/FilterSuite.scala b/src/test/scala/net/kemitix/s3thorp/ExcludeSuite.scala similarity index 58% rename from src/test/scala/net/kemitix/s3thorp/FilterSuite.scala rename to src/test/scala/net/kemitix/s3thorp/ExcludeSuite.scala index 204158c..fc060b5 100644 --- a/src/test/scala/net/kemitix/s3thorp/FilterSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/ExcludeSuite.scala @@ -2,42 +2,42 @@ package net.kemitix.s3thorp import java.nio.file.{Path, Paths} -class FilterSuite extends UnitTest { +class ExcludeSuite extends UnitTest { - describe("default filter") { - val filter = Filter() + describe("default exclude") { + val exclude = Exclude() val paths: List[Path] = List("/a-file", "a-file", "path/to/a/file", "/path/to/a/file", "/home/pcampbell/repos/kemitix/s3thorp/target/scala-2.12/test-classes/net/kemitix/s3thorp/upload/subdir" ) map { p => Paths.get(p)} it("should not exclude files") { - paths.foreach(path => { assertResult(false)(filter.isExcluded(path)) }) + paths.foreach(path => { assertResult(false)(exclude.isExcluded(path)) }) } it("should include files") { - paths.foreach(path => assertResult(true)(filter.isIncluded(path))) + paths.foreach(path => assertResult(true)(exclude.isIncluded(path))) } } - describe("directory exact match filter '/upload/subdir/'") { - val filter = Filter("/upload/subdir/") + describe("directory exact match exclude '/upload/subdir/'") { + val exclude = Exclude("/upload/subdir/") it("exclude matching directory") { val matching = Paths.get("/upload/subdir/leaf-file") - assertResult(true)(filter.isExcluded(matching)) + assertResult(true)(exclude.isExcluded(matching)) } it("include non-matching files") { val nonMatching = Paths.get("/upload/other-file") - assertResult(true)(filter.isIncluded(nonMatching)) + assertResult(true)(exclude.isIncluded(nonMatching)) } } describe("file partial match 'root'") { - val filter = Filter("root") + val exclude = Exclude("root") it("exclude matching file '/upload/root-file") { val matching = Paths.get("/upload/root-file") - assertResult(true)(filter.isExcluded(matching)) + assertResult(true)(exclude.isExcluded(matching)) } it("include non-matching files 'test-file-for-hash.txt' & '/upload/subdir/leaf-file'") { val nonMatching1 = Paths.get("/test-file-for-hash.txt") val nonMatching2 = Paths.get("/upload/subdir/leaf-file") - assertResult(true)(filter.isIncluded(nonMatching1)) - assertResult(true)(filter.isIncluded(nonMatching2)) + assertResult(true)(exclude.isIncluded(nonMatching1)) + assertResult(true)(exclude.isIncluded(nonMatching2)) } } diff --git a/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala b/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala index 723d9e8..8da1e58 100644 --- a/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala @@ -158,9 +158,9 @@ class SyncSuite } } describe("when a file is file is excluded") { - val filteredConfig = config.copy(filters = List(Filter("leaf"))) + val configWithExclusion = config.copy(excludes = List(Exclude("leaf"))) val sync = new RecordingSync(testBucket, new DummyS3Client {}, S3ObjectsData(Map(), Map())) - sync.run(filteredConfig).unsafeRunSync + sync.run(configWithExclusion).unsafeRunSync it("is not uploaded") { val expectedUploads = Map( "root-file" -> rootRemoteKey