Add configurable logging levels, selected from command line argument (#12)
* [config,parseargs] Accept v/verbose command line argument * [parseargs] lowercase program name * [logging] Log messages based on command line argument * [readme] update usage
This commit is contained in:
parent
74afb288cc
commit
00743c425c
12 changed files with 41 additions and 18 deletions
|
@ -12,11 +12,12 @@ hash of the file contents.
|
|||
|
||||
#+begin_example
|
||||
s3thorp
|
||||
Usage: S3Thorp [options]
|
||||
Usage: s3thorp [options]
|
||||
|
||||
-s, --source <value> Source directory to sync to S3
|
||||
-b, --bucket <value> S3 bucket name
|
||||
-p, --prefix <value> Prefix within the S3 Bucket
|
||||
-s, --source <value> Source directory to sync to S3
|
||||
-b, --bucket <value> S3 bucket name
|
||||
-p, --prefix <value> Prefix within the S3 Bucket
|
||||
-v, --verbose <value> Verbosity level (1-5)
|
||||
#+end_example
|
||||
|
||||
* Creating Native Images
|
||||
|
|
|
@ -7,6 +7,7 @@ import net.kemitix.s3thorp.Sync.{Bucket, LocalFile}
|
|||
|
||||
case class Config(bucket: Bucket = "",
|
||||
prefix: String = "",
|
||||
verbose: Int = 1,
|
||||
source: LocalFile
|
||||
) {
|
||||
|
||||
|
|
|
@ -4,4 +4,18 @@ import com.typesafe.scalalogging.LazyLogging
|
|||
|
||||
trait Logging extends LazyLogging {
|
||||
|
||||
def log1(message: String)(implicit config: Config): Unit = if (config.verbose >= 1) logger.info(message)
|
||||
|
||||
def log2(message: String)(implicit config: Config): Unit = if (config.verbose >= 2) logger.info(message)
|
||||
|
||||
def log3(message: String)(implicit config: Config): Unit = if (config.verbose >= 3) logger.info(message)
|
||||
|
||||
def log4(message: String)(implicit config: Config): Unit = if (config.verbose >= 4) logger.info(message)
|
||||
|
||||
def log5(message: String)(implicit config: Config): Unit = if (config.verbose >= 5) logger.info(message)
|
||||
|
||||
def warn(message: String): Unit = logger.warn(message)
|
||||
|
||||
def error(message: String): Unit = logger.error(message)
|
||||
|
||||
}
|
||||
|
|
|
@ -9,14 +9,14 @@ import net.kemitix.s3thorp.awssdk.S3Client
|
|||
object Main extends IOApp with Logging {
|
||||
|
||||
val defaultConfig: Config =
|
||||
Config("(none)", "", Paths.get(".").toFile)
|
||||
Config(source = Paths.get(".").toFile)
|
||||
|
||||
val sync = new Sync(S3Client.defaultClient)
|
||||
|
||||
def program(args: List[String]): IO[ExitCode] =
|
||||
for {
|
||||
_ <- IO(logger.info("S3Thorp - hashed sync for s3"))
|
||||
a <- ParseArgs(args, defaultConfig)
|
||||
_ <- IO(log1("S3Thorp - hashed sync for s3")(a))
|
||||
_ <- sync.run(a)
|
||||
} yield ExitCode.Success
|
||||
|
||||
|
|
|
@ -12,7 +12,7 @@ object ParseArgs {
|
|||
val parserBuilder = builder[Config]
|
||||
import parserBuilder._
|
||||
sequence(
|
||||
programName("S3Thorp"),
|
||||
programName("s3thorp"),
|
||||
head("s3thorp"),
|
||||
opt[String]('s', "source")
|
||||
.action((str, c) => c.copy(source = Paths.get(str).toFile))
|
||||
|
@ -24,7 +24,13 @@ object ParseArgs {
|
|||
.text("S3 bucket name"),
|
||||
opt[String]('p', "prefix")
|
||||
.action((str, c) => c.copy(prefix = str))
|
||||
.text("Prefix within the S3 Bucket")
|
||||
.text("Prefix within the S3 Bucket"),
|
||||
opt[Int]('v', "verbose")
|
||||
.validate(i =>
|
||||
if (i >= 1 && i <= 5) Right(Unit)
|
||||
else Left("Verbosity level must be between 1 and 5"))
|
||||
.action((i, c) => c.copy(verbose = i))
|
||||
.text("Verbosity level (1-5)")
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
@ -13,7 +13,7 @@ trait S3MetaDataEnricher
|
|||
def enrichWithS3MetaData(c: Config)(implicit hashLookup: HashLookup): File => Either[File, S3MetaData] = {
|
||||
val remoteKey = generateKey(c)_
|
||||
file => {
|
||||
logger.info(s"- Consider: ${c.relativePath(file)}")
|
||||
log3(s"- Consider: ${c.relativePath(file)}")(c)
|
||||
val key = remoteKey(file)
|
||||
objectHead(key).map {
|
||||
hlm: (MD5Hash, LastModified) => {
|
||||
|
|
|
@ -16,7 +16,7 @@ trait S3Uploader
|
|||
file => {
|
||||
val key = remoteKey(file)
|
||||
val shortFile = c.relativePath(file)
|
||||
logger.info(s" Upload: $shortFile")
|
||||
log4(s" Upload: $shortFile")(c)
|
||||
(file, upload(file, c.bucket, key))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -18,7 +18,8 @@ class Sync(s3Client: S3Client)
|
|||
s3Client.upload(localFile, bucket, remoteKey)
|
||||
|
||||
def run(c: Config): IO[Unit] = {
|
||||
logger.info(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
|
||||
implicit val config: Config = c
|
||||
log1(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
|
||||
s3Client.listObjects(c.bucket, c.prefix).map { hashLookup => {
|
||||
val stream: Stream[(File, IO[Either[Throwable, MD5Hash]])] = streamDirectoryPaths(c.source).map(
|
||||
enrichWithS3MetaData(c)(hashLookup)).flatMap(
|
||||
|
@ -26,10 +27,10 @@ class Sync(s3Client: S3Client)
|
|||
performUpload(c))
|
||||
val count: Int = stream.foldLeft(0)((a: Int, io) => {
|
||||
io._2.unsafeRunSync
|
||||
logger.info(s"- Done: ${io._1}")
|
||||
log1(s"- Done: ${io._1}")
|
||||
a + 1
|
||||
})
|
||||
logger.info(s"Uploaded $count files")
|
||||
log1(s"Uploaded $count files")
|
||||
}}
|
||||
}
|
||||
|
||||
|
|
|
@ -20,13 +20,13 @@ trait UploadSelectionFilter
|
|||
|
||||
def uploadRequiredFilter(c: Config): Either[File, S3MetaData] => Stream[File] = {
|
||||
case Left(file) => {
|
||||
logger.info(s" Created: ${c.relativePath(file)}")
|
||||
log5(s" Created: ${c.relativePath(file)}")(c)
|
||||
Stream(file)
|
||||
}
|
||||
case Right(s3Metadata) => {
|
||||
val localHash: MD5Hash = md5File(s3Metadata.localFile)
|
||||
if (localHash != s3Metadata.remoteHash) {
|
||||
logger.info(s" Updated: ${c.relativePath(s3Metadata.localFile)}")
|
||||
log5(s" Updated: ${c.relativePath(s3Metadata.localFile)}")(c)
|
||||
Stream(s3Metadata.localFile)
|
||||
}
|
||||
else Stream.empty
|
||||
|
|
|
@ -12,7 +12,7 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
|||
private val sourcePath = "/root/from/here/"
|
||||
private val source = Paths.get(sourcePath).toFile
|
||||
private val prefix = "prefix"
|
||||
private val config = Config("bucket", prefix, source)
|
||||
private val config = Config("bucket", prefix, source = source)
|
||||
|
||||
new S3MetaDataEnricher with DummyS3Client {
|
||||
describe("key generator") {
|
||||
|
|
|
@ -35,7 +35,7 @@ class SyncSuite extends FunSpec {
|
|||
val testBucket = "bucket"
|
||||
val source = Resource(this, "upload")
|
||||
// source contains the files root-file and subdir/leaf-file
|
||||
val config = Config("bucket", "prefix", source)
|
||||
val config = Config("bucket", "prefix", source = source)
|
||||
describe("when all files should be uploaded") {
|
||||
var uploadsRecord: Map[String, RemoteKey] = Map()
|
||||
val sync = new Sync(new DummyS3Client{
|
||||
|
|
|
@ -11,7 +11,7 @@ class UploadSelectionFilterSuite extends FunSpec {
|
|||
describe("uploadRequiredFilter") {
|
||||
val localFile = Resource(this, "test-file-for-hash.txt")
|
||||
val localHash = "0cbfe978783bd7950d5da4ff85e4af37"
|
||||
val config = Config("bucket", "prefix", localFile.getParentFile)
|
||||
val config = Config("bucket", "prefix", source = localFile.getParentFile)
|
||||
def invokeSubject(input: Either[File, S3MetaData]) =
|
||||
uploadRequiredFilter(config)(input).toList
|
||||
describe("when supplied a file") {
|
||||
|
|
Loading…
Reference in a new issue