diff --git a/README.org b/README.org index 1faa682..7228d84 100644 --- a/README.org +++ b/README.org @@ -12,11 +12,12 @@ hash of the file contents. #+begin_example s3thorp - Usage: S3Thorp [options] + Usage: s3thorp [options] - -s, --source Source directory to sync to S3 - -b, --bucket S3 bucket name - -p, --prefix Prefix within the S3 Bucket + -s, --source Source directory to sync to S3 + -b, --bucket S3 bucket name + -p, --prefix Prefix within the S3 Bucket + -v, --verbose Verbosity level (1-5) #+end_example * Creating Native Images diff --git a/src/main/scala/net/kemitix/s3thorp/Config.scala b/src/main/scala/net/kemitix/s3thorp/Config.scala index 91a2fc2..b610b76 100644 --- a/src/main/scala/net/kemitix/s3thorp/Config.scala +++ b/src/main/scala/net/kemitix/s3thorp/Config.scala @@ -7,6 +7,7 @@ import net.kemitix.s3thorp.Sync.{Bucket, LocalFile} case class Config(bucket: Bucket = "", prefix: String = "", + verbose: Int = 1, source: LocalFile ) { diff --git a/src/main/scala/net/kemitix/s3thorp/Logging.scala b/src/main/scala/net/kemitix/s3thorp/Logging.scala index 5689c12..38dd6ea 100644 --- a/src/main/scala/net/kemitix/s3thorp/Logging.scala +++ b/src/main/scala/net/kemitix/s3thorp/Logging.scala @@ -4,4 +4,18 @@ import com.typesafe.scalalogging.LazyLogging trait Logging extends LazyLogging { + def log1(message: String)(implicit config: Config): Unit = if (config.verbose >= 1) logger.info(message) + + def log2(message: String)(implicit config: Config): Unit = if (config.verbose >= 2) logger.info(message) + + def log3(message: String)(implicit config: Config): Unit = if (config.verbose >= 3) logger.info(message) + + def log4(message: String)(implicit config: Config): Unit = if (config.verbose >= 4) logger.info(message) + + def log5(message: String)(implicit config: Config): Unit = if (config.verbose >= 5) logger.info(message) + + def warn(message: String): Unit = logger.warn(message) + + def error(message: String): Unit = logger.error(message) + } diff --git a/src/main/scala/net/kemitix/s3thorp/Main.scala b/src/main/scala/net/kemitix/s3thorp/Main.scala index 98cac12..dd23b2e 100644 --- a/src/main/scala/net/kemitix/s3thorp/Main.scala +++ b/src/main/scala/net/kemitix/s3thorp/Main.scala @@ -9,14 +9,14 @@ import net.kemitix.s3thorp.awssdk.S3Client object Main extends IOApp with Logging { val defaultConfig: Config = - Config("(none)", "", Paths.get(".").toFile) + Config(source = Paths.get(".").toFile) val sync = new Sync(S3Client.defaultClient) def program(args: List[String]): IO[ExitCode] = for { - _ <- IO(logger.info("S3Thorp - hashed sync for s3")) a <- ParseArgs(args, defaultConfig) + _ <- IO(log1("S3Thorp - hashed sync for s3")(a)) _ <- sync.run(a) } yield ExitCode.Success diff --git a/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala b/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala index ccdbd23..5141eb5 100644 --- a/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala +++ b/src/main/scala/net/kemitix/s3thorp/ParseArgs.scala @@ -12,7 +12,7 @@ object ParseArgs { val parserBuilder = builder[Config] import parserBuilder._ sequence( - programName("S3Thorp"), + programName("s3thorp"), head("s3thorp"), opt[String]('s', "source") .action((str, c) => c.copy(source = Paths.get(str).toFile)) @@ -24,7 +24,13 @@ object ParseArgs { .text("S3 bucket name"), opt[String]('p', "prefix") .action((str, c) => c.copy(prefix = str)) - .text("Prefix within the S3 Bucket") + .text("Prefix within the S3 Bucket"), + opt[Int]('v', "verbose") + .validate(i => + if (i >= 1 && i <= 5) Right(Unit) + else Left("Verbosity level must be between 1 and 5")) + .action((i, c) => c.copy(verbose = i)) + .text("Verbosity level (1-5)") ) } diff --git a/src/main/scala/net/kemitix/s3thorp/S3MetaDataEnricher.scala b/src/main/scala/net/kemitix/s3thorp/S3MetaDataEnricher.scala index a05c52d..c4f6b6c 100644 --- a/src/main/scala/net/kemitix/s3thorp/S3MetaDataEnricher.scala +++ b/src/main/scala/net/kemitix/s3thorp/S3MetaDataEnricher.scala @@ -13,7 +13,7 @@ trait S3MetaDataEnricher def enrichWithS3MetaData(c: Config)(implicit hashLookup: HashLookup): File => Either[File, S3MetaData] = { val remoteKey = generateKey(c)_ file => { - logger.info(s"- Consider: ${c.relativePath(file)}") + log3(s"- Consider: ${c.relativePath(file)}")(c) val key = remoteKey(file) objectHead(key).map { hlm: (MD5Hash, LastModified) => { diff --git a/src/main/scala/net/kemitix/s3thorp/S3Uploader.scala b/src/main/scala/net/kemitix/s3thorp/S3Uploader.scala index 0fd1fe0..2c7c594 100644 --- a/src/main/scala/net/kemitix/s3thorp/S3Uploader.scala +++ b/src/main/scala/net/kemitix/s3thorp/S3Uploader.scala @@ -16,7 +16,7 @@ trait S3Uploader file => { val key = remoteKey(file) val shortFile = c.relativePath(file) - logger.info(s" Upload: $shortFile") + log4(s" Upload: $shortFile")(c) (file, upload(file, c.bucket, key)) } } diff --git a/src/main/scala/net/kemitix/s3thorp/Sync.scala b/src/main/scala/net/kemitix/s3thorp/Sync.scala index 5d7067e..be2ae92 100644 --- a/src/main/scala/net/kemitix/s3thorp/Sync.scala +++ b/src/main/scala/net/kemitix/s3thorp/Sync.scala @@ -18,7 +18,8 @@ class Sync(s3Client: S3Client) s3Client.upload(localFile, bucket, remoteKey) def run(c: Config): IO[Unit] = { - logger.info(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}") + implicit val config: Config = c + log1(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}") s3Client.listObjects(c.bucket, c.prefix).map { hashLookup => { val stream: Stream[(File, IO[Either[Throwable, MD5Hash]])] = streamDirectoryPaths(c.source).map( enrichWithS3MetaData(c)(hashLookup)).flatMap( @@ -26,10 +27,10 @@ class Sync(s3Client: S3Client) performUpload(c)) val count: Int = stream.foldLeft(0)((a: Int, io) => { io._2.unsafeRunSync - logger.info(s"- Done: ${io._1}") + log1(s"- Done: ${io._1}") a + 1 }) - logger.info(s"Uploaded $count files") + log1(s"Uploaded $count files") }} } diff --git a/src/main/scala/net/kemitix/s3thorp/UploadSelectionFilter.scala b/src/main/scala/net/kemitix/s3thorp/UploadSelectionFilter.scala index c72dae4..97e27a7 100644 --- a/src/main/scala/net/kemitix/s3thorp/UploadSelectionFilter.scala +++ b/src/main/scala/net/kemitix/s3thorp/UploadSelectionFilter.scala @@ -20,13 +20,13 @@ trait UploadSelectionFilter def uploadRequiredFilter(c: Config): Either[File, S3MetaData] => Stream[File] = { case Left(file) => { - logger.info(s" Created: ${c.relativePath(file)}") + log5(s" Created: ${c.relativePath(file)}")(c) Stream(file) } case Right(s3Metadata) => { val localHash: MD5Hash = md5File(s3Metadata.localFile) if (localHash != s3Metadata.remoteHash) { - logger.info(s" Updated: ${c.relativePath(s3Metadata.localFile)}") + log5(s" Updated: ${c.relativePath(s3Metadata.localFile)}")(c) Stream(s3Metadata.localFile) } else Stream.empty diff --git a/src/test/scala/net/kemitix/s3thorp/S3MetaDataEnricherSuite.scala b/src/test/scala/net/kemitix/s3thorp/S3MetaDataEnricherSuite.scala index 9fd34c4..8ea5445 100644 --- a/src/test/scala/net/kemitix/s3thorp/S3MetaDataEnricherSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/S3MetaDataEnricherSuite.scala @@ -12,7 +12,7 @@ class S3MetaDataEnricherSuite extends FunSpec { private val sourcePath = "/root/from/here/" private val source = Paths.get(sourcePath).toFile private val prefix = "prefix" - private val config = Config("bucket", prefix, source) + private val config = Config("bucket", prefix, source = source) new S3MetaDataEnricher with DummyS3Client { describe("key generator") { diff --git a/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala b/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala index 8000d81..f2cf7b9 100644 --- a/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/SyncSuite.scala @@ -35,7 +35,7 @@ class SyncSuite extends FunSpec { val testBucket = "bucket" val source = Resource(this, "upload") // source contains the files root-file and subdir/leaf-file - val config = Config("bucket", "prefix", source) + val config = Config("bucket", "prefix", source = source) describe("when all files should be uploaded") { var uploadsRecord: Map[String, RemoteKey] = Map() val sync = new Sync(new DummyS3Client{ diff --git a/src/test/scala/net/kemitix/s3thorp/UploadSelectionFilterSuite.scala b/src/test/scala/net/kemitix/s3thorp/UploadSelectionFilterSuite.scala index 007e363..edae2e9 100644 --- a/src/test/scala/net/kemitix/s3thorp/UploadSelectionFilterSuite.scala +++ b/src/test/scala/net/kemitix/s3thorp/UploadSelectionFilterSuite.scala @@ -11,7 +11,7 @@ class UploadSelectionFilterSuite extends FunSpec { describe("uploadRequiredFilter") { val localFile = Resource(this, "test-file-for-hash.txt") val localHash = "0cbfe978783bd7950d5da4ff85e4af37" - val config = Config("bucket", "prefix", localFile.getParentFile) + val config = Config("bucket", "prefix", source = localFile.getParentFile) def invokeSubject(input: Either[File, S3MetaData]) = uploadRequiredFilter(config)(input).toList describe("when supplied a file") {