Add configurable logging levels, selected from command line argument (#12)

* [config,parseargs] Accept v/verbose command line argument

* [parseargs] lowercase program name

* [logging] Log messages based on command line argument

* [readme] update usage
This commit is contained in:
Paul Campbell 2019-05-16 21:59:40 +01:00 committed by GitHub
parent 74afb288cc
commit 00743c425c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 41 additions and 18 deletions

View file

@ -12,11 +12,12 @@ hash of the file contents.
#+begin_example
s3thorp
Usage: S3Thorp [options]
Usage: s3thorp [options]
-s, --source <value> Source directory to sync to S3
-b, --bucket <value> S3 bucket name
-p, --prefix <value> Prefix within the S3 Bucket
-s, --source <value> Source directory to sync to S3
-b, --bucket <value> S3 bucket name
-p, --prefix <value> Prefix within the S3 Bucket
-v, --verbose <value> Verbosity level (1-5)
#+end_example
* Creating Native Images

View file

@ -7,6 +7,7 @@ import net.kemitix.s3thorp.Sync.{Bucket, LocalFile}
case class Config(bucket: Bucket = "",
prefix: String = "",
verbose: Int = 1,
source: LocalFile
) {

View file

@ -4,4 +4,18 @@ import com.typesafe.scalalogging.LazyLogging
trait Logging extends LazyLogging {
def log1(message: String)(implicit config: Config): Unit = if (config.verbose >= 1) logger.info(message)
def log2(message: String)(implicit config: Config): Unit = if (config.verbose >= 2) logger.info(message)
def log3(message: String)(implicit config: Config): Unit = if (config.verbose >= 3) logger.info(message)
def log4(message: String)(implicit config: Config): Unit = if (config.verbose >= 4) logger.info(message)
def log5(message: String)(implicit config: Config): Unit = if (config.verbose >= 5) logger.info(message)
def warn(message: String): Unit = logger.warn(message)
def error(message: String): Unit = logger.error(message)
}

View file

@ -9,14 +9,14 @@ import net.kemitix.s3thorp.awssdk.S3Client
object Main extends IOApp with Logging {
val defaultConfig: Config =
Config("(none)", "", Paths.get(".").toFile)
Config(source = Paths.get(".").toFile)
val sync = new Sync(S3Client.defaultClient)
def program(args: List[String]): IO[ExitCode] =
for {
_ <- IO(logger.info("S3Thorp - hashed sync for s3"))
a <- ParseArgs(args, defaultConfig)
_ <- IO(log1("S3Thorp - hashed sync for s3")(a))
_ <- sync.run(a)
} yield ExitCode.Success

View file

@ -12,7 +12,7 @@ object ParseArgs {
val parserBuilder = builder[Config]
import parserBuilder._
sequence(
programName("S3Thorp"),
programName("s3thorp"),
head("s3thorp"),
opt[String]('s', "source")
.action((str, c) => c.copy(source = Paths.get(str).toFile))
@ -24,7 +24,13 @@ object ParseArgs {
.text("S3 bucket name"),
opt[String]('p', "prefix")
.action((str, c) => c.copy(prefix = str))
.text("Prefix within the S3 Bucket")
.text("Prefix within the S3 Bucket"),
opt[Int]('v', "verbose")
.validate(i =>
if (i >= 1 && i <= 5) Right(Unit)
else Left("Verbosity level must be between 1 and 5"))
.action((i, c) => c.copy(verbose = i))
.text("Verbosity level (1-5)")
)
}

View file

@ -13,7 +13,7 @@ trait S3MetaDataEnricher
def enrichWithS3MetaData(c: Config)(implicit hashLookup: HashLookup): File => Either[File, S3MetaData] = {
val remoteKey = generateKey(c)_
file => {
logger.info(s"- Consider: ${c.relativePath(file)}")
log3(s"- Consider: ${c.relativePath(file)}")(c)
val key = remoteKey(file)
objectHead(key).map {
hlm: (MD5Hash, LastModified) => {

View file

@ -16,7 +16,7 @@ trait S3Uploader
file => {
val key = remoteKey(file)
val shortFile = c.relativePath(file)
logger.info(s" Upload: $shortFile")
log4(s" Upload: $shortFile")(c)
(file, upload(file, c.bucket, key))
}
}

View file

@ -18,7 +18,8 @@ class Sync(s3Client: S3Client)
s3Client.upload(localFile, bucket, remoteKey)
def run(c: Config): IO[Unit] = {
logger.info(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
implicit val config: Config = c
log1(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
s3Client.listObjects(c.bucket, c.prefix).map { hashLookup => {
val stream: Stream[(File, IO[Either[Throwable, MD5Hash]])] = streamDirectoryPaths(c.source).map(
enrichWithS3MetaData(c)(hashLookup)).flatMap(
@ -26,10 +27,10 @@ class Sync(s3Client: S3Client)
performUpload(c))
val count: Int = stream.foldLeft(0)((a: Int, io) => {
io._2.unsafeRunSync
logger.info(s"- Done: ${io._1}")
log1(s"- Done: ${io._1}")
a + 1
})
logger.info(s"Uploaded $count files")
log1(s"Uploaded $count files")
}}
}

View file

@ -20,13 +20,13 @@ trait UploadSelectionFilter
def uploadRequiredFilter(c: Config): Either[File, S3MetaData] => Stream[File] = {
case Left(file) => {
logger.info(s" Created: ${c.relativePath(file)}")
log5(s" Created: ${c.relativePath(file)}")(c)
Stream(file)
}
case Right(s3Metadata) => {
val localHash: MD5Hash = md5File(s3Metadata.localFile)
if (localHash != s3Metadata.remoteHash) {
logger.info(s" Updated: ${c.relativePath(s3Metadata.localFile)}")
log5(s" Updated: ${c.relativePath(s3Metadata.localFile)}")(c)
Stream(s3Metadata.localFile)
}
else Stream.empty

View file

@ -12,7 +12,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
private val sourcePath = "/root/from/here/"
private val source = Paths.get(sourcePath).toFile
private val prefix = "prefix"
private val config = Config("bucket", prefix, source)
private val config = Config("bucket", prefix, source = source)
new S3MetaDataEnricher with DummyS3Client {
describe("key generator") {

View file

@ -35,7 +35,7 @@ class SyncSuite extends FunSpec {
val testBucket = "bucket"
val source = Resource(this, "upload")
// source contains the files root-file and subdir/leaf-file
val config = Config("bucket", "prefix", source)
val config = Config("bucket", "prefix", source = source)
describe("when all files should be uploaded") {
var uploadsRecord: Map[String, RemoteKey] = Map()
val sync = new Sync(new DummyS3Client{

View file

@ -11,7 +11,7 @@ class UploadSelectionFilterSuite extends FunSpec {
describe("uploadRequiredFilter") {
val localFile = Resource(this, "test-file-for-hash.txt")
val localHash = "0cbfe978783bd7950d5da4ff85e4af37"
val config = Config("bucket", "prefix", localFile.getParentFile)
val config = Config("bucket", "prefix", source = localFile.getParentFile)
def invokeSubject(input: Either[File, S3MetaData]) =
uploadRequiredFilter(config)(input).toList
describe("when supplied a file") {