Apply scalafmt (#108)

* [scalafmt] Add .scalafmt

* [sbt] Add unused import warning and make them errors

* [core] ConfigurationBuilder remove unused import

* [core] ConfigurationBuilder apply scalafmt

* [domain] reformat

* [storage-api] reformat

* [core] reformat and some refactoring

* [storage-aws] reformat

* [cli] reformat

* [core] post rebase fix up

* [storage-aws] UploaderSuite tidy up

* [domain] MD5Hash tweak

Without doing this we have a file that we don't create a valid md5
hash for. See #103

* [storage-aws] UploaderSuite remove unused import

* [storage-aws] StorageServiceSuite reformatted

* [sbt] consistent settings for all modules

Can't enable as stubbing TransferManager attempts to use the default
constructor for TransferManager.

* [storage-aws] Add AmazonTransferManager

This gives us an interface we can safely stub in unit tests without
the compiler failing the build because TransferManager's default
constructor is deprecated.

* [storage-aws] Add AmazonUpload

Prevents deprecation errors due to deprecated parameters on Transfer.

* [sbt] mode import to top of file

.scalafmt.conf

@@ -0,0 +1,2 @@
+maxColumn = 80
+align = more

build.sbt

@@ -1,3 +1,5 @@
+import sbtassembly.AssemblyPlugin.defaultShellScript
+
 inThisBuild(List(
   organization := "net.kemitix.thorp",
   homepage := Some(url("https://github.com/kemitix/thorp")),
@@ -15,6 +17,15 @@ inThisBuild(List(
 val commonSettings = Seq(
   sonatypeProfileName := "net.kemitix",
   scalaVersion := "2.12.8",
+  scalacOptions ++= Seq(
+    "-Ywarn-unused-import",
+    "-Xfatal-warnings",
+    "-feature",
+    "-deprecation",
+    "-unchecked",
+    "-language:postfixOps",
+    "-language:higherKinds",
+    "-Ypartial-unification"),
   test in assembly := {}
 )
 
@@ -43,15 +54,7 @@ val awsSdkDependencies = Seq(
 val catsEffectsSettings = Seq(
   libraryDependencies ++=  Seq(
     "org.typelevel" %% "cats-effect" % "1.3.1"
-  ),
+  )
-  // recommended for cats-effects
-  scalacOptions ++= Seq(
-    "-feature",
-    "-deprecation",
-    "-unchecked",
-    "-language:postfixOps",
-    "-language:higherKinds",
-    "-Ypartial-unification")
 )
 
 // cli -> thorp-lib -> storage-aws -> core -> storage-api -> domain
@@ -60,7 +63,6 @@ lazy val thorp = (project in file("."))
   .settings(commonSettings)
   .aggregate(cli, `thorp-lib`, `storage-aws`, core, `storage-api`, domain)
 
-import sbtassembly.AssemblyPlugin.defaultShellScript
 lazy val cli = (project in file("cli"))
   .settings(commonSettings)
   .settings(mainClass in assembly := Some("net.kemitix.thorp.cli.Main"))

cli/src/main/scala/net/kemitix/thorp/cli/Main.scala

@@ -11,9 +11,9 @@ object Main extends IOApp {
       .map(Program.run)
       .getOrElse(IO(ExitCode.Error))
       .guaranteeCase {
-          case Canceled => exitCaseLogger.warn("Interrupted")
+        case Canceled  => exitCaseLogger.warn("Interrupted")
-          case Error(e) => exitCaseLogger.error(e.getMessage)
+        case Error(e)  => exitCaseLogger.error(e.getMessage)
-          case Completed => IO.unit
+        case Completed => IO.unit
       }
   }
 

cli/src/main/scala/net/kemitix/thorp/cli/ParseArgs.scala

@@ -35,7 +35,7 @@ object ParseArgs {
         .text("Include only matching paths"),
       opt[String]('x', "exclude")
         .unbounded()
-        .action((str,cos) => ConfigOption.Exclude(str) :: cos)
+        .action((str, cos) => ConfigOption.Exclude(str) :: cos)
         .text("Exclude matching paths"),
       opt[Unit]('d', "debug")
         .action((_, cos) => ConfigOption.Debug() :: cos)
@@ -50,7 +50,8 @@ object ParseArgs {
   }
 
   def apply(args: List[String]): Option[ConfigOptions] =
-    OParser.parse(configParser, args, List())
+    OParser
+      .parse(configParser, args, List())
       .map(ConfigOptions)
 
 }

cli/src/main/scala/net/kemitix/thorp/cli/PrintLogger.scala

@@ -9,11 +9,14 @@ class PrintLogger(isDebug: Boolean = false) extends Logger {
     if (isDebug) IO(println(s"[ DEBUG] $message"))
     else IO.unit
 
-  override def info(message: => String): IO[Unit] = IO(println(s"[  INFO] $message"))
+  override def info(message: => String): IO[Unit] =
+    IO(println(s"[  INFO] $message"))
 
-  override def warn(message: String): IO[Unit] = IO(println(s"[  WARN] $message"))
+  override def warn(message: String): IO[Unit] =
+    IO(println(s"[  WARN] $message"))
 
-  override def error(message: String): IO[Unit] = IO(println(s"[ ERROR] $message"))
+  override def error(message: String): IO[Unit] =
+    IO(println(s"[ ERROR] $message"))
 
   override def withDebug(debug: Boolean): Logger =
     if (isDebug == debug) this

cli/src/main/scala/net/kemitix/thorp/cli/Program.scala

@@ -17,45 +17,50 @@ trait Program extends PlanBuilder {
       } yield ExitCode.Success
     else
       for {
-        syncPlan <- createPlan(defaultStorageService, defaultHashService, cliOptions).valueOrF(handleErrors)
+        syncPlan <- createPlan(
+          defaultStorageService,
+          defaultHashService,
+          cliOptions
+        ).valueOrF(handleErrors)
         archive <- thorpArchive(cliOptions, syncPlan)
-        events <- handleActions(archive, syncPlan)
+        events  <- handleActions(archive, syncPlan)
-        _ <- defaultStorageService.shutdown
+        _       <- defaultStorageService.shutdown
-        _ <- SyncLogging.logRunFinished(events)
+        _       <- SyncLogging.logRunFinished(events)
       } yield ExitCode.Success
   }
 
-  def thorpArchive(cliOptions: ConfigOptions,
+  def thorpArchive(
-                           syncPlan: SyncPlan): IO[ThorpArchive] =
+      cliOptions: ConfigOptions,
+      syncPlan: SyncPlan
+  ): IO[ThorpArchive] =
     IO.pure(
       UnversionedMirrorArchive.default(
         defaultStorageService,
         ConfigQuery.batchMode(cliOptions),
         syncPlan.syncTotals
-    ))
+      ))
 
-  private def handleErrors(implicit logger: Logger): List[String] => IO[SyncPlan] = {
+  private def handleErrors(
-    errors => {
+      implicit logger: Logger
-      for {
+  ): List[String] => IO[SyncPlan] = errors => {
-        _ <- logger.error("There were errors:")
+    for {
-        _ <- errors.map(error => logger.error(s" - $error")).sequence
+      _ <- logger.error("There were errors:")
-      } yield SyncPlan()
+      _ <- errors.map(error => logger.error(s" - $error")).sequence
-    }
+    } yield SyncPlan()
   }
 
-  private def handleActions(archive: ThorpArchive,
+  private def handleActions(
-                            syncPlan: SyncPlan)
+      archive: ThorpArchive,
-                           (implicit l: Logger): IO[Stream[StorageQueueEvent]] = {
+      syncPlan: SyncPlan
+  )(implicit l: Logger): IO[Stream[StorageQueueEvent]] = {
     type Accumulator = (Stream[IO[StorageQueueEvent]], Long)
     val zero: Accumulator = (Stream(), syncPlan.syncTotals.totalSizeBytes)
-    val (actions, _) = syncPlan.actions
+    val (actions, _) = syncPlan.actions.zipWithIndex.reverse
-      .zipWithIndex
+      .foldLeft(zero) { (acc: Accumulator, indexedAction) =>
-      .reverse
+        {
-      .foldLeft(zero) {
-        (acc: Accumulator, indexedAction) => {
           val (stream, bytesToDo) = acc
-          val (action, index) = indexedAction
+          val (action, index)     = indexedAction
-          val remainingBytes = bytesToDo - action.size
+          val remainingBytes      = bytesToDo - action.size
           (
             archive.update(index, action, remainingBytes) ++ stream,
             remainingBytes

cli/src/test/scala/net/kemitix/thorp/cli/ParseArgsTest.scala

@@ -24,17 +24,15 @@ class ParseArgsTest extends FunSpec {
     }
     describe("when source is a relative path to a directory") {
       val result = invokeWithSource(pathTo("."))
-      it("should succeed") {pending}
+      it("should succeed") { pending }
     }
     describe("when there are multiple sources") {
-      val args = List(
+      val args =
-        "--source", "path1",
+        List("--source", "path1", "--source", "path2", "--bucket", "bucket")
-        "--source", "path2",
-        "--bucket", "bucket")
       it("should get multiple sources") {
-        val expected = Some(Set("path1", "path2").map(Paths.get(_)))
+        val expected      = Some(Set("path1", "path2").map(Paths.get(_)))
         val configOptions = ParseArgs(args)
-        val result = configOptions.map(ConfigQuery.sources(_).paths.toSet)
+        val result        = configOptions.map(ConfigQuery.sources(_).paths.toSet)
         assertResult(expected)(result)
       }
     }
@@ -43,7 +41,7 @@ class ParseArgsTest extends FunSpec {
   describe("parse - debug") {
     def invokeWithArgument(arg: String): ConfigOptions = {
       val strings = List("--source", pathTo("."), "--bucket", "bucket", arg)
-          .filter(_ != "")
+        .filter(_ != "")
       val maybeOptions = ParseArgs(strings)
       maybeOptions.getOrElse(ConfigOptions())
     }

cli/src/test/scala/net/kemitix/thorp/cli/ProgramTest.scala

@@ -13,18 +13,23 @@ import org.scalatest.FunSpec
 
 class ProgramTest extends FunSpec {
 
-  val source: File = Resource(this, ".")
+  val source: File     = Resource(this, ".")
   val sourcePath: Path = source.toPath
-  val bucket: Bucket = Bucket("aBucket")
+  val bucket: Bucket   = Bucket("aBucket")
-  val hash: MD5Hash = MD5Hash("aHash")
+  val hash: MD5Hash    = MD5Hash("aHash")
-  val copyAction: Action = ToCopy(bucket, RemoteKey("copy-me"), hash, RemoteKey("overwrite-me"), 17L)
+  val copyAction: Action =
-  val uploadAction: Action = ToUpload(bucket, LocalFile.resolve("aFile", Map(), sourcePath, _ => RemoteKey("upload-me")), 23L)
+    ToCopy(bucket, RemoteKey("copy-me"), hash, RemoteKey("overwrite-me"), 17L)
+  val uploadAction: Action = ToUpload(
+    bucket,
+    LocalFile.resolve("aFile", Map(), sourcePath, _ => RemoteKey("upload-me")),
+    23L)
   val deleteAction: Action = ToDelete(bucket, RemoteKey("delete-me"), 0L)
 
-  val configOptions: ConfigOptions = ConfigOptions(options = List(
+  val configOptions: ConfigOptions = ConfigOptions(
-    ConfigOption.IgnoreGlobalOptions,
+    options = List(
-    ConfigOption.IgnoreUserOptions
+      ConfigOption.IgnoreGlobalOptions,
-  ))
+      ConfigOption.IgnoreUserOptions
+    ))
 
   describe("upload, copy and delete actions in plan") {
     val archive = TestProgram.thorpArchive
@@ -36,31 +41,30 @@ class ProgramTest extends FunSpec {
     }
   }
 
-  object TestProgram extends Program with TestPlanBuilder {
-    val thorpArchive: ActionCaptureArchive = new ActionCaptureArchive
-    override def thorpArchive(cliOptions: ConfigOptions, syncPlan: SyncPlan): IO[ThorpArchive] =
-      IO.pure(thorpArchive)
-  }
-
   trait TestPlanBuilder extends PlanBuilder {
     override def createPlan(storageService: StorageService,
                             hashService: HashService,
-                            configOptions: ConfigOptions)
+                            configOptions: ConfigOptions)(
-                           (implicit l: Logger): EitherT[IO, List[String], SyncPlan] = {
+        implicit l: Logger): EitherT[IO, List[String], SyncPlan] = {
-      EitherT.right(IO(SyncPlan(Stream(copyAction, uploadAction, deleteAction))))
+      EitherT.right(
+        IO(SyncPlan(Stream(copyAction, uploadAction, deleteAction))))
     }
   }
 
   class ActionCaptureArchive extends ThorpArchive {
     var actions: List[Action] = List[Action]()
-    override def update(index: Int,
+    override def update(index: Int, action: Action, totalBytesSoFar: Long)(
-                        action: Action,
+        implicit l: Logger): Stream[IO[StorageQueueEvent]] = {
-                        totalBytesSoFar: Long)
-                       (implicit l: Logger): Stream[IO[StorageQueueEvent]] = {
       actions = action :: actions
       Stream()
     }
   }
 
-}
+  object TestProgram extends Program with TestPlanBuilder {
+    val thorpArchive: ActionCaptureArchive = new ActionCaptureArchive
+    override def thorpArchive(cliOptions: ConfigOptions,
+                              syncPlan: SyncPlan): IO[ThorpArchive] =
+      IO.pure(thorpArchive)
+  }
 
+}

core/src/main/scala/net/kemitix/thorp/core/Action.scala

@@ -8,22 +8,30 @@ sealed trait Action {
 }
 object Action {
 
-  final case class DoNothing(bucket: Bucket,
+  final case class DoNothing(
-                             remoteKey: RemoteKey,
+      bucket: Bucket,
-                             size: Long) extends Action
+      remoteKey: RemoteKey,
+      size: Long
+  ) extends Action
 
-  final case class ToUpload(bucket: Bucket,
+  final case class ToUpload(
-                            localFile: LocalFile,
+      bucket: Bucket,
-                            size: Long) extends Action
+      localFile: LocalFile,
+      size: Long
+  ) extends Action
 
-  final case class ToCopy(bucket: Bucket,
+  final case class ToCopy(
-                          sourceKey: RemoteKey,
+      bucket: Bucket,
-                          hash: MD5Hash,
+      sourceKey: RemoteKey,
-                          targetKey: RemoteKey,
+      hash: MD5Hash,
-                          size: Long) extends Action
+      targetKey: RemoteKey,
+      size: Long
+  ) extends Action
 
-  final case class ToDelete(bucket: Bucket,
+  final case class ToDelete(
-                            remoteKey: RemoteKey,
+      bucket: Bucket,
-                            size: Long) extends Action
+      remoteKey: RemoteKey,
+      size: Long
+  ) extends Action
 
 }

core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala

@@ -5,70 +5,73 @@ import net.kemitix.thorp.domain._
 
 object ActionGenerator {
 
-  def remoteNameNotAlreadyQueued(localFile: LocalFile,
+  def createActions(
-                                 previousActions: Stream[Action]): Boolean = {
+      s3MetaData: S3MetaData,
-    val key = localFile.remoteKey.key
+      previousActions: Stream[Action]
-    !previousActions.exists {
+  )(implicit c: Config): Stream[Action] =
-      case ToUpload(_, lf, _) => lf.remoteKey.key equals key
-      case _ => false
-    }
-  }
-
-  def createActions(s3MetaData: S3MetaData,
-                    previousActions: Stream[Action])
-                   (implicit c: Config): Stream[Action] =
     s3MetaData match {
-
       // #1 local exists, remote exists, remote matches - do nothing
-      case S3MetaData(localFile, _, Some(RemoteMetaData(remoteKey, remoteHash, _)))
+      case S3MetaData(localFile, _, Some(RemoteMetaData(key, hash, _)))
-        if localFile.matches(remoteHash)
+          if localFile.matches(hash) =>
-      => doNothing(c.bucket, remoteKey)
+        doNothing(c.bucket, key)
-
       // #2 local exists, remote is missing, other matches - copy
-      case S3MetaData(localFile, otherMatches, None)
+      case S3MetaData(localFile, matchByHash, None) if matchByHash.nonEmpty =>
-        if otherMatches.nonEmpty
+        copyFile(c.bucket, localFile, matchByHash)
-      => copyFile(c.bucket, localFile, otherMatches)
-
       // #3 local exists, remote is missing, other no matches - upload
-      case S3MetaData(localFile, otherMatches, None)
+      case S3MetaData(localFile, matchByHash, None)
-        if otherMatches.isEmpty &&
+          if matchByHash.isEmpty &&
-          remoteNameNotAlreadyQueued(localFile, previousActions)
+            isUploadAlreadyQueued(previousActions)(localFile) =>
-      => uploadFile(c.bucket, localFile)
+        uploadFile(c.bucket, localFile)
-
       // #4 local exists, remote exists, remote no match, other matches - copy
-      case S3MetaData(localFile, otherMatches, Some(RemoteMetaData(_, remoteHash, _)))
+      case S3MetaData(localFile, matchByHash, Some(RemoteMetaData(_, hash, _)))
-        if !localFile.matches(remoteHash) &&
+          if !localFile.matches(hash) &&
-          otherMatches.nonEmpty
+            matchByHash.nonEmpty =>
-      => copyFile(c.bucket, localFile, otherMatches)
+        copyFile(c.bucket, localFile, matchByHash)
-
       // #5 local exists, remote exists, remote no match, other no matches - upload
-      case S3MetaData(localFile, hashMatches, Some(_))
+      case S3MetaData(localFile, matchByHash, Some(_)) if matchByHash.isEmpty =>
-        if hashMatches.isEmpty
+        uploadFile(c.bucket, localFile)
-      => uploadFile(c.bucket, localFile)
+      // fallback
-
       case S3MetaData(localFile, _, _) =>
         doNothing(c.bucket, localFile.remoteKey)
     }
 
-  private def doNothing(bucket: Bucket,
+  def isUploadAlreadyQueued(
-                        remoteKey: RemoteKey) =
+      previousActions: Stream[Action]
-    Stream(
+  )(
-      DoNothing(bucket, remoteKey, 0L))
+      localFile: LocalFile
-
+  ): Boolean = {
-  private def uploadFile(bucket: Bucket,
+    !previousActions.exists {
-                         localFile: LocalFile) =
+      case ToUpload(_, lf, _) => lf.remoteKey.key equals localFile.remoteKey.key
-    Stream(
+      case _                  => false
-      ToUpload(bucket, localFile, localFile.file.length))
-
-  private def copyFile(bucket: Bucket,
-                       localFile: LocalFile,
-                       matchByHash: Set[RemoteMetaData]): Stream[Action] = {
-    val headOption = matchByHash.headOption
-    headOption.toStream.map { remoteMetaData =>
-      val sourceKey = remoteMetaData.remoteKey
-      val hash = remoteMetaData.hash
-      ToCopy(bucket, sourceKey, hash, localFile.remoteKey, localFile.file.length)
     }
   }
 
+  private def doNothing(
+      bucket: Bucket,
+      remoteKey: RemoteKey
+  ) =
+    Stream(DoNothing(bucket, remoteKey, 0L))
+
+  private def uploadFile(
+      bucket: Bucket,
+      localFile: LocalFile
+  ) =
+    Stream(ToUpload(bucket, localFile, localFile.file.length))
+
+  private def copyFile(
+      bucket: Bucket,
+      localFile: LocalFile,
+      matchByHash: Set[RemoteMetaData]
+  ): Stream[Action] =
+    matchByHash
+      .map { remoteMetaData =>
+        ToCopy(bucket,
+               remoteMetaData.remoteKey,
+               remoteMetaData.hash,
+               localFile.remoteKey,
+               localFile.file.length)
+      }
+      .toStream
+      .take(1)
+
 }

core/src/main/scala/net/kemitix/thorp/core/ConfigOption.scala

@@ -10,15 +10,11 @@ sealed trait ConfigOption {
 }
 
 object ConfigOption {
-  case object Version extends ConfigOption {
-    override def update(config: Config): Config = config
-  }
-  case object BatchMode extends ConfigOption {
-    override def update(config: Config): Config = config.copy(batchMode = true)
-  }
   case class Source(path: Path) extends ConfigOption {
-    override def update(config: Config): Config = config.copy(sources = config.sources ++ path)
+    override def update(config: Config): Config =
+      config.copy(sources = config.sources ++ path)
   }
+
   case class Bucket(name: String) extends ConfigOption {
     override def update(config: Config): Config =
       if (config.bucket.name.isEmpty)
@@ -26,6 +22,7 @@ object ConfigOption {
       else
         config
   }
+
   case class Prefix(path: String) extends ConfigOption {
     override def update(config: Config): Config =
       if (config.prefix.key.isEmpty)
@@ -33,15 +30,29 @@ object ConfigOption {
       else
         config
   }
+
   case class Include(pattern: String) extends ConfigOption {
-    override def update(config: Config): Config = config.copy(filters = domain.Filter.Include(pattern) :: config.filters)
+    override def update(config: Config): Config =
+      config.copy(filters = domain.Filter.Include(pattern) :: config.filters)
   }
+
   case class Exclude(pattern: String) extends ConfigOption {
-    override def update(config: Config): Config = config.copy(filters = domain.Filter.Exclude(pattern) :: config.filters)
+    override def update(config: Config): Config =
+      config.copy(filters = domain.Filter.Exclude(pattern) :: config.filters)
   }
+
   case class Debug() extends ConfigOption {
     override def update(config: Config): Config = config.copy(debug = true)
   }
+
+  case object Version extends ConfigOption {
+    override def update(config: Config): Config = config
+  }
+
+  case object BatchMode extends ConfigOption {
+    override def update(config: Config): Config = config.copy(batchMode = true)
+  }
+
   case object IgnoreUserOptions extends ConfigOption {
     override def update(config: Config): Config = config
   }

core/src/main/scala/net/kemitix/thorp/core/ConfigOptions.scala

@@ -2,10 +2,14 @@ package net.kemitix.thorp.core
 
 import cats.Semigroup
 
-case class ConfigOptions(options: List[ConfigOption] = List())
+case class ConfigOptions(
-    extends Semigroup[ConfigOptions] {
+    options: List[ConfigOption] = List()
+) extends Semigroup[ConfigOptions] {
 
-  override def combine(x: ConfigOptions, y: ConfigOptions): ConfigOptions =
+  override def combine(
+      x: ConfigOptions,
+      y: ConfigOptions
+  ): ConfigOptions =
     x ++ y
 
   def ++(other: ConfigOptions): ConfigOptions =

core/src/main/scala/net/kemitix/thorp/core/ConfigQuery.scala

@@ -19,15 +19,16 @@ trait ConfigQuery {
     configOptions contains ConfigOption.IgnoreGlobalOptions
 
   def sources(configOptions: ConfigOptions): Sources = {
-    val paths = configOptions.options.flatMap( {
+    val paths = configOptions.options.flatMap {
       case ConfigOption.Source(sourcePath) => Some(sourcePath)
       case _ => None
-    })
+    }
     Sources(paths match {
       case List() => List(Paths.get(System.getenv("PWD")))
       case _ => paths
     })
   }
+
 }
 
 object ConfigQuery extends ConfigQuery

core/src/main/scala/net/kemitix/thorp/core/ConfigValidator.scala

@@ -10,18 +10,12 @@ sealed trait ConfigValidator {
 
   type ValidationResult[A] = ValidatedNec[ConfigValidation, A]
 
-  def validateSourceIsDirectory(source: Path): ValidationResult[Path] =
+  def validateConfig(
-    if(source.toFile.isDirectory) source.validNec
+      config: Config): Validated[NonEmptyChain[ConfigValidation], Config] =
-    else ConfigValidation.SourceIsNotADirectory.invalidNec
+    (
-
+      validateSources(config.sources),
-  def validateSourceIsReadable(source: Path): ValidationResult[Path] =
+      validateBucket(config.bucket)
-    if(source.toFile.canRead) source.validNec
+    ).mapN((_, _) => config)
-    else ConfigValidation.SourceIsNotReadable.invalidNec
-
-  def validateSource(source: Path): ValidationResult[Path] =
-    validateSourceIsDirectory(source)
-      .andThen(s =>
-        validateSourceIsReadable(s))
 
   def validateBucket(bucket: Bucket): ValidationResult[Bucket] =
     if (bucket.name.isEmpty) ConfigValidation.BucketNameIsMissing.invalidNec
@@ -29,14 +23,21 @@ sealed trait ConfigValidator {
 
   def validateSources(sources: Sources): ValidationResult[Sources] =
     sources.paths
-      .map(validateSource).sequence
+      .map(validateSource)
+      .sequence
       .map(_ => sources)
 
-  def validateConfig(config: Config): Validated[NonEmptyChain[ConfigValidation], Config] =
+  def validateSource(source: Path): ValidationResult[Path] =
-    (
+    validateSourceIsDirectory(source)
-      validateSources(config.sources),
+      .andThen(s => validateSourceIsReadable(s))
-      validateBucket(config.bucket)
+
-    ).mapN((_, _) => config)
+  def validateSourceIsDirectory(source: Path): ValidationResult[Path] =
+    if (source.toFile.isDirectory) source.validNec
+    else ConfigValidation.SourceIsNotADirectory.invalidNec
+
+  def validateSourceIsReadable(source: Path): ValidationResult[Path] =
+    if (source.toFile.canRead) source.validNec
+    else ConfigValidation.SourceIsNotReadable.invalidNec
 }
 
 object ConfigValidator extends ConfigValidator

core/src/main/scala/net/kemitix/thorp/core/ConfigurationBuilder.scala

@@ -1,13 +1,12 @@
 package net.kemitix.thorp.core
 
-import java.nio.file.{Files, Path, Paths}
+import java.nio.file.{Path, Paths}
 
 import cats.data.NonEmptyChain
 import cats.effect.IO
-import cats.implicits._
 import net.kemitix.thorp.core.ConfigValidator.validateConfig
 import net.kemitix.thorp.core.ParseConfigFile.parseFile
-import net.kemitix.thorp.domain.{Config, Sources}
+import net.kemitix.thorp.domain.Config
 
 /**
   * Builds a configuration from settings in a file within the
@@ -15,33 +14,42 @@ import net.kemitix.thorp.domain.{Config, Sources}
   */
 trait ConfigurationBuilder {
 
-  def buildConfig(priorityOptions: ConfigOptions): IO[Either[NonEmptyChain[ConfigValidation], Config]] = {
+  private val sourceConfigFilename = ".thorp.config"
-    val sources = ConfigQuery.sources(priorityOptions)
+  private val userConfigFilename   = ".config/thorp.conf"
+  private val globalConfig         = Paths.get("/etc/thorp.conf")
+  private val userHome             = Paths.get(System.getProperty("user.home"))
+  private val pwd                  = Paths.get(System.getenv("PWD"))
+
+  def buildConfig(priorityOpts: ConfigOptions)
+      : IO[Either[NonEmptyChain[ConfigValidation], Config]] = {
+    val sources = ConfigQuery.sources(priorityOpts)
     for {
       sourceOptions <- SourceConfigLoader.loadSourceConfigs(sources)
-      userOptions <- userOptions(priorityOptions ++ sourceOptions)
+      userOptions   <- userOptions(priorityOpts ++ sourceOptions)
-      globalOptions <- globalOptions(priorityOptions ++ sourceOptions ++ userOptions)
+      globalOptions <- globalOptions(priorityOpts ++ sourceOptions ++ userOptions)
-      collected = priorityOptions ++ sourceOptions ++ userOptions ++ globalOptions
+      collected = priorityOpts ++ sourceOptions ++ userOptions ++ globalOptions
-      config = collateOptions(collected)
+      config    = collateOptions(collected)
     } yield validateConfig(config).toEither
   }
 
-  private def userOptions(higherPriorityOptions: ConfigOptions): IO[ConfigOptions] =
+  private val emptyConfig = IO(ConfigOptions())
-    if (ConfigQuery.ignoreUserOptions(higherPriorityOptions)) IO(ConfigOptions())
-    else readFile(userHome, ".config/thorp.conf")
 
-  private def globalOptions(higherPriorityOptions: ConfigOptions): IO[ConfigOptions] =
+  private def userOptions(priorityOpts: ConfigOptions) =
-    if (ConfigQuery.ignoreGlobalOptions(higherPriorityOptions)) IO(ConfigOptions())
+    if (ConfigQuery.ignoreUserOptions(priorityOpts)) emptyConfig
-    else parseFile(Paths.get("/etc/thorp.conf"))
+    else readFile(userHome, userConfigFilename)
 
-  private def userHome = Paths.get(System.getProperty("user.home"))
+  private def globalOptions(priorityOpts: ConfigOptions) =
+    if (ConfigQuery.ignoreGlobalOptions(priorityOpts)) emptyConfig
+    else parseFile(globalConfig)
 
-  private def readFile(source: Path, filename: String): IO[ConfigOptions] =
+  private def readFile(
+      source: Path,
+      filename: String
+  ) =
     parseFile(source.resolve(filename))
 
-  private def collateOptions(configOptions: ConfigOptions): Config = {
+  private def collateOptions(configOptions: ConfigOptions): Config =
     configOptions.options.foldLeft(Config())((c, co) => co.update(c))
-  }
 
 }
 

core/src/main/scala/net/kemitix/thorp/core/Counters.scala

@@ -1,6 +1,8 @@
 package net.kemitix.thorp.core
 
-final case class Counters(uploaded: Int = 0,
+final case class Counters(
-                          deleted: Int = 0,
+    uploaded: Int = 0,
-                          copied: Int = 0,
+    deleted: Int = 0,
-                          errors: Int = 0)
+    copied: Int = 0,
+    errors: Int = 0
+)

core/src/main/scala/net/kemitix/thorp/core/KeyGenerator.scala

@@ -6,13 +6,17 @@ import net.kemitix.thorp.domain.{RemoteKey, Sources}
 
 object KeyGenerator {
 
-  def generateKey(sources: Sources,
+  def generateKey(
-                  prefix: RemoteKey)
+      sources: Sources,
-                 (path: Path): RemoteKey = {
+      prefix: RemoteKey
-    val source = sources.forPath(path)
+  )(path: Path): RemoteKey = {
-    val relativePath = source.relativize(path.toAbsolutePath)
+    val source       = sources.forPath(path)
-    RemoteKey(List(prefix.key, relativePath.toString)
+    val relativePath = source.relativize(path.toAbsolutePath).toString
-        .filterNot(_.isEmpty)
+    RemoteKey(
+      List(
+        prefix.key,
+        relativePath
+      ).filter(_.nonEmpty)
         .mkString("/"))
   }
 

core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala

@@ -1,6 +1,5 @@
 package net.kemitix.thorp.core
 
-import java.io.File
 import java.nio.file.Path
 
 import cats.effect.IO
@@ -11,66 +10,80 @@ import net.kemitix.thorp.storage.api.HashService
 
 object LocalFileStream {
 
-  def findFiles(source: Path,
+  private val emptyIOLocalFiles = IO.pure(LocalFiles())
-                hashService: HashService)
+
-               (implicit c: Config,
+  def findFiles(
-                logger: Logger): IO[LocalFiles] = {
+      source: Path,
+      hashService: HashService
+  )(
+      implicit c: Config,
+      logger: Logger
+  ): IO[LocalFiles] = {
 
     val isIncluded: Path => Boolean = Filter.isIncluded(c.filters)
 
+    val pathToLocalFile: Path => IO[LocalFiles] = path =>
+      localFile(hashService, logger, c)(path)
+
     def loop(path: Path): IO[LocalFiles] = {
 
-      def dirPaths(path: Path): IO[Stream[Path]] =
+      def dirPaths(path: Path) =
-        IO(listFiles(path))
+        listFiles(path)
-          .map(fs =>
+          .map(_.filter(isIncluded))
-            Stream(fs: _*)
-              .map(_.toPath)
-              .filter(isIncluded))
 
-      def recurseIntoSubDirectories(path: Path): IO[LocalFiles] =
+      def recurseIntoSubDirectories(path: Path) =
         path.toFile match {
           case f if f.isDirectory => loop(path)
-          case _ => localFile(hashService, path)
+          case _                  => pathToLocalFile(path)
         }
 
-      def recurse(paths: Stream[Path]): IO[LocalFiles] =
+      def recurse(paths: Stream[Path]) =
-        paths.foldLeft(IO.pure(LocalFiles()))((acc, path) =>
+        paths.foldLeft(emptyIOLocalFiles)(
-          recurseIntoSubDirectories(path)
+          (acc, path) =>
-            .flatMap(localFiles => acc.map(accLocalFiles => accLocalFiles ++ localFiles)))
+            recurseIntoSubDirectories(path)
+              .flatMap(localFiles =>
+                acc.map(accLocalFiles => accLocalFiles ++ localFiles)))
 
       for {
-        _ <- logger.debug(s"- Entering: $path")
+        _          <- logger.debug(s"- Entering: $path")
-        fs <- dirPaths(path)
+        paths      <- dirPaths(path)
-        lfs <- recurse(fs)
+        localFiles <- recurse(paths)
-        _ <- logger.debug(s"- Leaving : $path")
+        _          <- logger.debug(s"- Leaving : $path")
-      } yield lfs
+      } yield localFiles
     }
 
     loop(source)
   }
 
-  private def localFile(hashService: HashService,
+  def localFile(
-                        path: Path)
+      hashService: HashService,
-                       (implicit l: Logger, c: Config) = {
+      l: Logger,
-    val file = path.toFile
+      c: Config
-    val source = c.sources.forPath(path)
+  ): Path => IO[LocalFiles] =
-    for {
+    path => {
-      hash <- hashService.hashLocalObject(path)
+      val file   = path.toFile
-    } yield
+      val source = c.sources.forPath(path)
-      LocalFiles(
+      for {
-        localFiles = Stream(
+        hash <- hashService.hashLocalObject(path)(l)
-          domain.LocalFile(
+      } yield
-            file,
+        LocalFiles(localFiles = Stream(
-            source.toFile,
+                     domain.LocalFile(file,
-            hash,
+                                      source.toFile,
-            generateKey(c.sources, c.prefix)(path))),
+                                      hash,
-        count = 1,
+                                      generateKey(c.sources, c.prefix)(path))),
-        totalSizeBytes = file.length)
+                   count = 1,
-  }
+                   totalSizeBytes = file.length)
+    }
 
-  //TODO: Change this to return an Either[IllegalArgumentException, Array[File]]
+  //TODO: Change this to return an Either[IllegalArgumentException, Stream[Path]]
-  private def listFiles(path: Path): Array[File] = {
+  private def listFiles(path: Path) = {
-    Option(path.toFile.listFiles)
+    IO(
-      .getOrElse(throw new IllegalArgumentException(s"Directory not found $path"))
+      Option(path.toFile.listFiles)
+        .map { fs =>
+          Stream(fs: _*)
+            .map(_.toPath)
+        }
+        .getOrElse(
+          throw new IllegalArgumentException(s"Directory not found $path")))
   }
 }

core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala

@@ -2,13 +2,16 @@ package net.kemitix.thorp.core
 
 import net.kemitix.thorp.domain.LocalFile
 
-case class LocalFiles(localFiles: Stream[LocalFile] = Stream(),
+case class LocalFiles(
-                      count: Long = 0,
+    localFiles: Stream[LocalFile] = Stream(),
-                      totalSizeBytes: Long = 0) {
+    count: Long = 0,
+    totalSizeBytes: Long = 0
+) {
   def ++(append: LocalFiles): LocalFiles =
-    copy(localFiles = localFiles ++ append.localFiles,
+    copy(
+      localFiles = localFiles ++ append.localFiles,
       count = count + append.count,
-      totalSizeBytes = totalSizeBytes + append.totalSizeBytes)
+      totalSizeBytes = totalSizeBytes + append.totalSizeBytes
+    )
 
 }
-

core/src/main/scala/net/kemitix/thorp/core/MD5HashGenerator.scala

@@ -29,7 +29,36 @@ object MD5HashGenerator {
   def md5File(path: Path)(implicit logger: Logger): IO[MD5Hash] =
     md5FileChunk(path, 0, path.toFile.length)
 
-  private def openFile(file: File, offset: Long) = IO {
+  def md5FileChunk(
+      path: Path,
+      offset: Long,
+      size: Long
+  )(implicit logger: Logger): IO[MD5Hash] = {
+    val file      = path.toFile
+    val endOffset = Math.min(offset + size, file.length)
+    for {
+      _      <- logger.debug(s"md5:reading:size ${file.length}:$path")
+      digest <- readFile(file, offset, endOffset)
+      hash = MD5Hash.fromDigest(digest)
+      _ <- logger.debug(s"md5:generated:${hash.hash}:$path")
+    } yield hash
+  }
+
+  private def readFile(
+      file: File,
+      offset: Long,
+      endOffset: Long
+  ) =
+    for {
+      fis    <- openFile(file, offset)
+      digest <- digestFile(fis, offset, endOffset)
+      _      <- closeFile(fis)
+    } yield digest
+
+  private def openFile(
+      file: File,
+      offset: Long
+  ) = IO {
     val stream = new FileInputStream(file)
     stream skip offset
     stream
@@ -37,24 +66,24 @@ object MD5HashGenerator {
 
   private def closeFile(fis: FileInputStream) = IO(fis.close())
 
-  private def readFile(file: File, offset: Long, endOffset: Long) =
+  private def digestFile(
-    for {
+      fis: FileInputStream,
-      fis <- openFile(file, offset)
+      offset: Long,
-      digest <- digestFile(fis, offset, endOffset)
+      endOffset: Long
-      _ <- closeFile(fis)
+  ) =
-    } yield digest
-
-  private def digestFile(fis: FileInputStream, offset: Long, endOffset: Long) =
     IO {
       val md5 = MessageDigest getInstance "MD5"
       NumericRange(offset, endOffset, maxBufferSize)
-        .foreach(currentOffset => md5 update readToBuffer(fis, currentOffset, endOffset))
+        .foreach(currentOffset =>
+          md5 update readToBuffer(fis, currentOffset, endOffset))
       md5.digest
     }
 
-  private def readToBuffer(fis: FileInputStream,
+  private def readToBuffer(
-                   currentOffset: Long,
+      fis: FileInputStream,
-                   endOffset: Long) = {
+      currentOffset: Long,
+      endOffset: Long
+  ) = {
     val buffer =
       if (nextBufferSize(currentOffset, endOffset) < maxBufferSize)
         new Array[Byte](nextBufferSize(currentOffset, endOffset))
@@ -63,24 +92,12 @@ object MD5HashGenerator {
     buffer
   }
 
-  private def nextBufferSize(currentOffset: Long, endOffset: Long) = {
+  private def nextBufferSize(
+      currentOffset: Long,
+      endOffset: Long
+  ) = {
     val toRead = endOffset - currentOffset
-    val result = Math.min(maxBufferSize, toRead)
+    Math.min(maxBufferSize, toRead).toInt
-    result.toInt
-  }
-
-  def md5FileChunk(path: Path,
-                   offset: Long,
-                   size: Long)
-                  (implicit logger: Logger): IO[MD5Hash] = {
-    val file = path.toFile
-    val endOffset = Math.min(offset + size, file.length)
-    for {
-      _ <- logger.debug(s"md5:reading:size ${file.length}:$path")
-      digest <- readFile(file, offset, endOffset)
-      hash = MD5Hash.fromDigest(digest)
-      _ <- logger.debug(s"md5:generated:${hash.hash}:$path")
-    } yield hash
   }
 
 }

core/src/main/scala/net/kemitix/thorp/core/ParseConfigFile.scala

@@ -9,7 +9,8 @@ import scala.collection.JavaConverters._
 trait ParseConfigFile {
 
   def parseFile(filename: Path): IO[ConfigOptions] =
-    readFile(filename).map(ParseConfigLines.parseLines)
+    readFile(filename)
+      .map(ParseConfigLines.parseLines)
 
   private def readFile(filename: Path) = {
     if (Files.exists(filename)) readFileThatExists(filename)
@@ -25,4 +26,4 @@ trait ParseConfigFile {
 
 }
 
-object ParseConfigFile extends ParseConfigFile
+object ParseConfigFile extends ParseConfigFile

core/src/main/scala/net/kemitix/thorp/core/ParseConfigLines.scala

@@ -7,35 +7,38 @@ import net.kemitix.thorp.core.ConfigOption._
 
 trait ParseConfigLines {
 
+  private val pattern = "^\\s*(?<key>\\S*)\\s*=\\s*(?<value>\\S*)\\s*$"
+  private val format  = Pattern.compile(pattern)
+
   def parseLines(lines: List[String]): ConfigOptions =
     ConfigOptions(lines.flatMap(parseLine))
 
-  private val pattern = "^\\s*(?<key>\\S*)\\s*=\\s*(?<value>\\S*)\\s*$"
-  private val format = Pattern.compile(pattern)
-
   private def parseLine(str: String) =
     format.matcher(str) match {
       case m if m.matches => parseKeyValue(m.group("key"), m.group("value"))
-      case _ =>None
+      case _              => None
+    }
+
+  private def parseKeyValue(
+      key: String,
+      value: String
+  ): Option[ConfigOption] =
+    key.toLowerCase match {
+      case "source"  => Some(Source(Paths.get(value)))
+      case "bucket"  => Some(Bucket(value))
+      case "prefix"  => Some(Prefix(value))
+      case "include" => Some(Include(value))
+      case "exclude" => Some(Exclude(value))
+      case "debug"   => if (truthy(value)) Some(Debug()) else None
+      case _         => None
     }
 
   def truthy(value: String): Boolean =
     value.toLowerCase match {
-      case "true" => true
+      case "true"    => true
-      case "yes" => true
+      case "yes"     => true
       case "enabled" => true
-      case _ => false
+      case _         => false
-    }
-
-  private def parseKeyValue(key: String, value: String): Option[ConfigOption] =
-    key.toLowerCase match {
-      case "source" => Some(Source(Paths.get(value)))
-      case "bucket" => Some(Bucket(value))
-      case "prefix" => Some(Prefix(value))
-      case "include" => Some(Include(value))
-      case "exclude" => Some(Exclude(value))
-      case "debug" => if (truthy(value)) Some(Debug()) else None
-      case _ => None
     }
 
 }

core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala

@@ -9,10 +9,11 @@ import net.kemitix.thorp.storage.api.{HashService, StorageService}
 
 trait PlanBuilder {
 
-  def createPlan(storageService: StorageService,
+  def createPlan(
-            hashService: HashService,
+      storageService: StorageService,
-            configOptions: ConfigOptions)
+      hashService: HashService,
-           (implicit l: Logger): EitherT[IO, List[String], SyncPlan] =
+      configOptions: ConfigOptions
+  )(implicit l: Logger): EitherT[IO, List[String], SyncPlan] =
     EitherT(ConfigurationBuilder.buildConfig(configOptions))
       .leftMap(errorMessages)
       .flatMap(config => useValidConfig(storageService, hashService)(config, l))
@@ -20,90 +21,113 @@ trait PlanBuilder {
   def errorMessages(errors: NonEmptyChain[ConfigValidation]): List[String] =
     errors.map(cv => cv.errorMessage).toList
 
-  def removeDoNothing: Action => Boolean = {
+  def useValidConfig(
-    case _: DoNothing => false
+      storageService: StorageService,
-    case _ => true
+      hashService: HashService
-  }
+  )(implicit c: Config, l: Logger): EitherT[IO, List[String], SyncPlan] =
-
-  def assemblePlan(implicit c: Config): ((S3ObjectsData, LocalFiles)) => SyncPlan = {
-    case (remoteData, localData) => {
-      val actions =
-        (actionsForLocalFiles(localData, remoteData) ++
-          actionsForRemoteKeys(remoteData))
-          .filter(removeDoNothing)
-      SyncPlan(
-        actions = actions,
-        syncTotals = SyncTotals(
-          count = localData.count,
-          totalSizeBytes = localData.totalSizeBytes))
-    }
-  }
-
-  def useValidConfig(storageService: StorageService,
-                     hashService: HashService)
-                    (implicit c: Config, l: Logger): EitherT[IO, List[String], SyncPlan] = {
     for {
-      _ <- EitherT.liftF(SyncLogging.logRunStart(c.bucket, c.prefix, c.sources))
+      _       <- EitherT.liftF(SyncLogging.logRunStart(c.bucket, c.prefix, c.sources))
-      actions <- gatherMetadata(storageService, hashService)
+      actions <- buildPlan(storageService, hashService)
-        .leftMap(error => List(error))
-        .map(assemblePlan)
     } yield actions
+
+  private def buildPlan(
+      storageService: StorageService,
+      hashService: HashService
+  )(implicit c: Config, l: Logger) =
+    gatherMetadata(storageService, hashService)
+      .leftMap(List(_))
+      .map(assemblePlan)
+
+  def assemblePlan(
+      implicit c: Config): ((S3ObjectsData, LocalFiles)) => SyncPlan = {
+    case (remoteData, localData) =>
+      SyncPlan(
+        actions = createActions(remoteData, localData).filter(doesSomething),
+        syncTotals = SyncTotals(count = localData.count,
+                                totalSizeBytes = localData.totalSizeBytes)
+      )
   }
 
-  private def gatherMetadata(storageService: StorageService,
+  private def createActions(
-                             hashService: HashService)
+      remoteData: S3ObjectsData,
-                            (implicit l: Logger,
+      localData: LocalFiles
-                             c: Config): EitherT[IO, String, (S3ObjectsData, LocalFiles)] =
+  )(implicit c: Config): Stream[Action] =
+    actionsForLocalFiles(localData, remoteData) ++
+      actionsForRemoteKeys(remoteData)
+
+  def doesSomething: Action => Boolean = {
+    case _: DoNothing => false
+    case _            => true
+  }
+
+  private val emptyActionStream = Stream[Action]()
+
+  private def actionsForLocalFiles(
+      localData: LocalFiles,
+      remoteData: S3ObjectsData
+  )(implicit c: Config) =
+    localData.localFiles.foldLeft(emptyActionStream)((acc, lf) =>
+      createActionFromLocalFile(lf, remoteData, acc) ++ acc)
+
+  private def createActionFromLocalFile(
+      lf: LocalFile,
+      remoteData: S3ObjectsData,
+      previousActions: Stream[Action]
+  )(implicit c: Config) =
+    ActionGenerator.createActions(
+      S3MetaDataEnricher.getMetadata(lf, remoteData),
+      previousActions)
+
+  private def actionsForRemoteKeys(remoteData: S3ObjectsData)(
+      implicit c: Config) =
+    remoteData.byKey.keys.foldLeft(emptyActionStream)((acc, rk) =>
+      createActionFromRemoteKey(rk) #:: acc)
+
+  private def createActionFromRemoteKey(rk: RemoteKey)(implicit c: Config) =
+    if (rk.isMissingLocally(c.sources, c.prefix))
+      Action.ToDelete(c.bucket, rk, 0L)
+    else DoNothing(c.bucket, rk, 0L)
+
+  private def gatherMetadata(
+      storageService: StorageService,
+      hashService: HashService
+  )(implicit l: Logger,
+    c: Config): EitherT[IO, String, (S3ObjectsData, LocalFiles)] =
     for {
       remoteData <- fetchRemoteData(storageService)
-      localData <- EitherT.liftF(findLocalFiles(hashService))
+      localData  <- EitherT.liftF(findLocalFiles(hashService))
     } yield (remoteData, localData)
 
-  private def actionsForLocalFiles(localData: LocalFiles, remoteData: S3ObjectsData)
+  private def fetchRemoteData(
-                                  (implicit c: Config) =
+      storageService: StorageService
-    localData.localFiles.foldLeft(Stream[Action]())((acc, lf) => createActionFromLocalFile(lf, remoteData, acc) ++ acc)
+  )(implicit c: Config, l: Logger) =
-
-  private def actionsForRemoteKeys(remoteData: S3ObjectsData)
-                                  (implicit c: Config) =
-    remoteData.byKey.keys.foldLeft(Stream[Action]())((acc, rk) => createActionFromRemoteKey(rk) #:: acc)
-
-  private def fetchRemoteData(storageService: StorageService)
-                             (implicit c: Config, l: Logger) =
     storageService.listObjects(c.bucket, c.prefix)
 
-  private def findLocalFiles(hashService: HashService)
+  private def findLocalFiles(
-                            (implicit config: Config, l: Logger) =
+      hashService: HashService
+  )(implicit config: Config, l: Logger) =
     for {
-      _ <- SyncLogging.logFileScan
+      _          <- SyncLogging.logFileScan
       localFiles <- findFiles(hashService)
     } yield localFiles
 
-  private def findFiles(hashService: HashService)
+  private def findFiles(
-                       (implicit c: Config, l: Logger): IO[LocalFiles] = {
+      hashService: HashService
+  )(implicit c: Config, l: Logger) = {
     val ioListLocalFiles = (for {
       source <- c.sources.paths
     } yield LocalFileStream.findFiles(source, hashService)).sequence
     for {
       listLocalFiles <- ioListLocalFiles
-      localFiles = listLocalFiles.foldRight(LocalFiles()){
+      localFiles = listLocalFiles.foldRight(LocalFiles()) {
-        (acc, moreLocalFiles) => {
+        (acc, moreLocalFiles) =>
-          acc ++ moreLocalFiles
+          {
-        }
+            acc ++ moreLocalFiles
+          }
       }
     } yield localFiles
   }
 
-  private def createActionFromLocalFile(lf: LocalFile,
-                                        remoteData: S3ObjectsData,
-                                        previousActions: Stream[Action])
-                                       (implicit c: Config) =
-    ActionGenerator.createActions(S3MetaDataEnricher.getMetadata(lf, remoteData), previousActions)
-
-  private def createActionFromRemoteKey(rk: RemoteKey)
-                                       (implicit c: Config) =
-    if (rk.isMissingLocally(c.sources, c.prefix)) Action.ToDelete(c.bucket, rk, 0L)
-    else DoNothing(c.bucket, rk, 0L)
-
 }
 
 object PlanBuilder extends PlanBuilder

core/src/main/scala/net/kemitix/thorp/core/Resource.scala

@@ -6,10 +6,11 @@ import scala.util.Try
 
 object Resource {
 
-  def apply(base: AnyRef,
+  def apply(
-            name: String): File = {
+      base: AnyRef,
-    Try{
+      name: String
+  ): File =
+    Try {
       new File(base.getClass.getResource(name).getPath)
     }.getOrElse(throw new FileNotFoundException(name))
-  }
 }

core/src/main/scala/net/kemitix/thorp/core/S3MetaDataEnricher.scala

@@ -4,23 +4,36 @@ import net.kemitix.thorp.domain._
 
 object S3MetaDataEnricher {
 
-  def getMetadata(localFile: LocalFile,
+  def getMetadata(
-                  s3ObjectsData: S3ObjectsData)
+      localFile: LocalFile,
-                 (implicit c: Config): S3MetaData = {
+      s3ObjectsData: S3ObjectsData
+  )(implicit c: Config): S3MetaData = {
     val (keyMatches, hashMatches) = getS3Status(localFile, s3ObjectsData)
-    S3MetaData(localFile,
+    S3MetaData(
-      matchByKey = keyMatches map { hm => RemoteMetaData(localFile.remoteKey, hm.hash, hm.modified) },
+      localFile,
-      matchByHash = hashMatches map { case (hash, km) => RemoteMetaData(km.key, hash, km.modified) })
+      matchByKey = keyMatches.map { hm =>
+        RemoteMetaData(localFile.remoteKey, hm.hash, hm.modified)
+      },
+      matchByHash = hashMatches.map {
+        case (hash, km) => RemoteMetaData(km.key, hash, km.modified)
+      }
+    )
   }
 
-  def getS3Status(localFile: LocalFile,
+  def getS3Status(
-                  s3ObjectsData: S3ObjectsData): (Option[HashModified], Set[(MD5Hash, KeyModified)]) = {
+      localFile: LocalFile,
+      s3ObjectsData: S3ObjectsData
+  ): (Option[HashModified], Set[(MD5Hash, KeyModified)]) = {
     val matchingByKey = s3ObjectsData.byKey.get(localFile.remoteKey)
     val matchingByHash = localFile.hashes
-      .map { case(_, md5Hash) =>
+      .map {
-        s3ObjectsData.byHash.getOrElse(md5Hash, Set())
+        case (_, md5Hash) =>
-          .map(km => (md5Hash, km))
+          s3ObjectsData.byHash
-      }.flatten.toSet
+            .getOrElse(md5Hash, Set())
+            .map(km => (md5Hash, km))
+      }
+      .flatten
+      .toSet
     (matchingByKey, matchingByHash)
   }
 

core/src/main/scala/net/kemitix/thorp/core/SimpleHashService.scala

@@ -8,12 +8,11 @@ import net.kemitix.thorp.storage.api.HashService
 
 case class SimpleHashService() extends HashService {
 
-  override def hashLocalObject(path: Path)
+  override def hashLocalObject(
-                              (implicit l: Logger): IO[Map[String, MD5Hash]] =
+      path: Path
+  )(implicit l: Logger): IO[Map[String, MD5Hash]] =
     for {
       md5 <- MD5HashGenerator.md5File(path)
-    } yield Map(
+    } yield Map("md5" -> md5)
-      "md5" -> md5
-    )
 
 }

core/src/main/scala/net/kemitix/thorp/core/SourceConfigLoader.scala

@@ -1,7 +1,5 @@
 package net.kemitix.thorp.core
 
-import java.nio.file.{Files, Path}
-
 import cats.effect.IO
 import cats.implicits._
 import net.kemitix.thorp.domain.Sources

core/src/main/scala/net/kemitix/thorp/core/SyncLogging.scala

@@ -2,15 +2,21 @@ package net.kemitix.thorp.core
 
 import cats.effect.IO
 import cats.implicits._
-import net.kemitix.thorp.domain.StorageQueueEvent.{CopyQueueEvent, DeleteQueueEvent, ErrorQueueEvent, UploadQueueEvent}
+import net.kemitix.thorp.domain.StorageQueueEvent.{
+  CopyQueueEvent,
+  DeleteQueueEvent,
+  ErrorQueueEvent,
+  UploadQueueEvent
+}
 import net.kemitix.thorp.domain._
 
 trait SyncLogging {
 
-  def logRunStart(bucket: Bucket,
+  def logRunStart(
-                  prefix: RemoteKey,
+      bucket: Bucket,
-                  sources: Sources)
+      prefix: RemoteKey,
-                 (implicit logger: Logger): IO[Unit] = {
+      sources: Sources
+  )(implicit logger: Logger): IO[Unit] = {
     val sourcesList = sources.paths.mkString(", ")
     logger.info(s"Bucket: ${bucket.name}, Prefix: ${prefix.key}, Source: $sourcesList")
   }
@@ -19,17 +25,9 @@ trait SyncLogging {
                   logger: Logger): IO[Unit] =
     logger.info(s"Scanning local files: ${c.sources.paths.mkString(", ")}...")
 
-  def logErrors(actions: Stream[StorageQueueEvent])
+  def logRunFinished(
-               (implicit logger: Logger): IO[Unit] =
+      actions: Stream[StorageQueueEvent]
-    for {
+  )(implicit logger: Logger): IO[Unit] = {
-      _ <- actions.map {
-        case ErrorQueueEvent(k, e) => logger.warn(s"${k.key}: ${e.getMessage}")
-        case _ => IO.unit
-      }.sequence
-    } yield ()
-
-  def logRunFinished(actions: Stream[StorageQueueEvent])
-                    (implicit logger: Logger): IO[Unit] = {
     val counters = actions.foldLeft(Counters())(countActivities)
     for {
       _ <- logger.info(s"Uploaded ${counters.uploaded} files")
@@ -40,6 +38,16 @@ trait SyncLogging {
     } yield ()
   }
 
+  def logErrors(
+      actions: Stream[StorageQueueEvent]
+  )(implicit logger: Logger): IO[Unit] =
+    for {
+      _ <- actions.map {
+        case ErrorQueueEvent(k, e) => logger.warn(s"${k.key}: ${e.getMessage}")
+        case _                     => IO.unit
+      }.sequence
+    } yield ()
+
   private def countActivities: (Counters, StorageQueueEvent) => Counters =
     (counters: Counters, s3Action: StorageQueueEvent) => {
       s3Action match {

core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala

@@ -2,5 +2,7 @@ package net.kemitix.thorp.core
 
 import net.kemitix.thorp.domain.SyncTotals
 
-case class SyncPlan(actions: Stream[Action] = Stream(),
+case class SyncPlan(
-                    syncTotals: SyncTotals = SyncTotals())
+    actions: Stream[Action] = Stream(),
+    syncTotals: SyncTotals = SyncTotals()
+)

core/src/main/scala/net/kemitix/thorp/core/ThorpArchive.scala

@@ -5,14 +5,17 @@ import net.kemitix.thorp.domain.{LocalFile, Logger, StorageQueueEvent}
 
 trait ThorpArchive {
 
-  def update(index: Int,
+  def update(
-             action: Action,
+      index: Int,
-             totalBytesSoFar: Long)
+      action: Action,
-            (implicit l: Logger): Stream[IO[StorageQueueEvent]]
+      totalBytesSoFar: Long
+  )(implicit l: Logger): Stream[IO[StorageQueueEvent]]
 
-  def fileUploaded(localFile: LocalFile,
+  def logFileUploaded(
-                   batchMode: Boolean)
+      localFile: LocalFile,
-                  (implicit l: Logger): IO[Unit] =
+      batchMode: Boolean
-    if (batchMode) l.info(s"Uploaded: ${localFile.remoteKey.key}") else IO.unit
+  )(implicit l: Logger): IO[Unit] =
+    if (batchMode) l.info(s"Uploaded: ${localFile.remoteKey.key}")
+    else IO.unit
 
 }

core/src/main/scala/net/kemitix/thorp/core/UnversionedMirrorArchive.scala

@@ -3,41 +3,64 @@ package net.kemitix.thorp.core
 import cats.effect.IO
 import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
 import net.kemitix.thorp.domain.StorageQueueEvent.DoNothingQueueEvent
-import net.kemitix.thorp.domain.{Logger, StorageQueueEvent, SyncTotals, UploadEventListener}
+import net.kemitix.thorp.domain.{
+  Bucket,
+  LocalFile,
+  Logger,
+  StorageQueueEvent,
+  SyncTotals,
+  UploadEventListener
+}
 import net.kemitix.thorp.storage.api.StorageService
 
-case class UnversionedMirrorArchive(storageService: StorageService,
+case class UnversionedMirrorArchive(
-                                    batchMode: Boolean,
+    storageService: StorageService,
-                                    syncTotals: SyncTotals) extends ThorpArchive {
+    batchMode: Boolean,
-  override def update(index: Int,
+    syncTotals: SyncTotals
-                      action: Action,
+) extends ThorpArchive {
-                      totalBytesSoFar: Long)
-                     (implicit l: Logger): Stream[IO[StorageQueueEvent]] =
-    Stream(
-      action match {
-        case ToUpload(bucket, localFile, size) =>
-          for {
-            event <- storageService.upload(localFile, bucket, batchMode,
-              new UploadEventListener(localFile, index, syncTotals, totalBytesSoFar), 1)
-            _ <- fileUploaded(localFile, batchMode)
-          } yield event
-        case ToCopy(bucket, sourceKey, hash, targetKey, size) =>
-          for {
-            event <- storageService.copy(bucket, sourceKey, hash, targetKey)
-          } yield event
-        case ToDelete(bucket, remoteKey, size) =>
-          for {
-            event <- storageService.delete(bucket, remoteKey)
-          } yield event
-        case DoNothing(_, remoteKey, size) =>
-          IO.pure(DoNothingQueueEvent(remoteKey))
-      })
 
+  override def update(
+      index: Int,
+      action: Action,
+      totalBytesSoFar: Long
+  )(implicit l: Logger): Stream[IO[StorageQueueEvent]] =
+    Stream(action match {
+      case ToUpload(bucket, localFile, _) =>
+        for {
+          event <- doUpload(index, totalBytesSoFar, bucket, localFile)
+          _     <- logFileUploaded(localFile, batchMode)
+        } yield event
+      case ToCopy(bucket, sourceKey, hash, targetKey, _) =>
+        for {
+          event <- storageService.copy(bucket, sourceKey, hash, targetKey)
+        } yield event
+      case ToDelete(bucket, remoteKey, _) =>
+        for {
+          event <- storageService.delete(bucket, remoteKey)
+        } yield event
+      case DoNothing(_, remoteKey, _) =>
+        IO.pure(DoNothingQueueEvent(remoteKey))
+    })
+
+  private def doUpload(
+      index: Int,
+      totalBytesSoFar: Long,
+      bucket: Bucket,
+      localFile: LocalFile
+  ) =
+    storageService.upload(
+      localFile,
+      bucket,
+      batchMode,
+      new UploadEventListener(localFile, index, syncTotals, totalBytesSoFar),
+      1)
 }
 
 object UnversionedMirrorArchive {
-  def default(storageService: StorageService,
+  def default(
-              batchMode: Boolean,
+      storageService: StorageService,
-              syncTotals: SyncTotals): ThorpArchive =
+      batchMode: Boolean,
+      syncTotals: SyncTotals
+  ): ThorpArchive =
     new UnversionedMirrorArchive(storageService, batchMode, syncTotals)
 }

core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala

@@ -6,105 +6,142 @@ import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
 import net.kemitix.thorp.domain._
 import org.scalatest.FunSpec
 
-class ActionGeneratorSuite
+class ActionGeneratorSuite extends FunSpec {
-  extends FunSpec {
+  val lastModified       = LastModified(Instant.now())
-
+  private val source     = Resource(this, "upload")
-  private val source = Resource(this, "upload")
   private val sourcePath = source.toPath
-  private val prefix = RemoteKey("prefix")
+  private val prefix     = RemoteKey("prefix")
-  private val bucket = Bucket("bucket")
+  private val bucket     = Bucket("bucket")
-  implicit private val config: Config = Config(bucket, prefix, sources = Sources(List(sourcePath)))
+  implicit private val config: Config =
-  private val fileToKey = KeyGenerator.generateKey(config.sources, config.prefix) _
+    Config(bucket, prefix, sources = Sources(List(sourcePath)))
-  val lastModified = LastModified(Instant.now())
+  private val fileToKey =
+    KeyGenerator.generateKey(config.sources, config.prefix) _
 
-    describe("create actions") {
+  describe("create actions") {
 
-      val previousActions = Stream.empty[Action]
+    val previousActions = Stream.empty[Action]
 
-      def invoke(input: S3MetaData) = ActionGenerator.createActions(input, previousActions).toList
+    def invoke(input: S3MetaData) =
+      ActionGenerator.createActions(input, previousActions).toList
 
-      describe("#1 local exists, remote exists, remote matches - do nothing") {
+    describe("#1 local exists, remote exists, remote matches - do nothing") {
-        val theHash = MD5Hash("the-hash")
+      val theHash = MD5Hash("the-hash")
-        val theFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
+      val theFile = LocalFile.resolve("the-file",
-        val theRemoteMetadata = RemoteMetaData(theFile.remoteKey, theHash, lastModified)
+                                      md5HashMap(theHash),
-        val input = S3MetaData(theFile, // local exists
+                                      sourcePath,
-          matchByHash = Set(theRemoteMetadata), // remote matches
+                                      fileToKey)
-          matchByKey = Some(theRemoteMetadata) // remote exists
+      val theRemoteMetadata =
-          )
+        RemoteMetaData(theFile.remoteKey, theHash, lastModified)
-        it("do nothing") {
+      val input =
-          val expected = List(DoNothing(bucket, theFile.remoteKey, theFile.file.length))
+        S3MetaData(theFile, // local exists
-          val result = invoke(input)
+                   matchByHash = Set(theRemoteMetadata), // remote matches
-          assertResult(expected)(result)
+                   matchByKey = Some(theRemoteMetadata) // remote exists
-        }
-      }
-      describe("#2 local exists, remote is missing, other matches - copy") {
-        val theHash = MD5Hash("the-hash")
-        val theFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
-        val theRemoteKey = theFile.remoteKey
-        val otherRemoteKey = prefix.resolve("other-key")
-        val otherRemoteMetadata = RemoteMetaData(otherRemoteKey, theHash, lastModified)
-        val input = S3MetaData(theFile, // local exists
-          matchByHash = Set(otherRemoteMetadata), // other matches
-          matchByKey = None) // remote is missing
-        it("copy from other key") {
-          val expected = List(ToCopy(bucket, otherRemoteKey, theHash, theRemoteKey, theFile.file.length)) // copy
-          val result = invoke(input)
-          assertResult(expected)(result)
-        }
-      }
-      describe("#3 local exists, remote is missing, other no matches - upload") {
-        val theHash = MD5Hash("the-hash")
-        val theFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
-        val input = S3MetaData(theFile, // local exists
-          matchByHash = Set.empty, // other no matches
-          matchByKey = None) // remote is missing
-        it("upload") {
-          val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload
-          val result = invoke(input)
-          assertResult(expected)(result)
-        }
-      }
-      describe("#4 local exists, remote exists, remote no match, other matches - copy") {
-        val theHash = MD5Hash("the-hash")
-        val theFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
-        val theRemoteKey = theFile.remoteKey
-        val oldHash = MD5Hash("old-hash")
-        val otherRemoteKey = prefix.resolve("other-key")
-        val otherRemoteMetadata = RemoteMetaData(otherRemoteKey, theHash, lastModified)
-        val oldRemoteMetadata = RemoteMetaData(theRemoteKey,
-          hash = oldHash, // remote no match
-          lastModified = lastModified)
-        val input = S3MetaData(theFile, // local exists
-          matchByHash = Set(otherRemoteMetadata), // other matches
-          matchByKey = Some(oldRemoteMetadata)) // remote exists
-        it("copy from other key") {
-          val expected = List(ToCopy(bucket, otherRemoteKey, theHash, theRemoteKey, theFile.file.length)) // copy
-          val result = invoke(input)
-          assertResult(expected)(result)
-        }
-      }
-      describe("#5 local exists, remote exists, remote no match, other no matches - upload") {
-        val theHash = MD5Hash("the-hash")
-        val theFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
-        val theRemoteKey = theFile.remoteKey
-        val oldHash = MD5Hash("old-hash")
-        val theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
-        val input = S3MetaData(theFile, // local exists
-          matchByHash = Set.empty, // remote no match, other no match
-          matchByKey = Some(theRemoteMetadata) // remote exists
         )
-        it("upload") {
+      it("do nothing") {
-          val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload
+        val expected =
-          val result = invoke(input)
+          List(DoNothing(bucket, theFile.remoteKey, theFile.file.length))
-          assertResult(expected)(result)
+        val result = invoke(input)
-        }
+        assertResult(expected)(result)
-      }
-      describe("#6 local missing, remote exists - delete") {
-        it("TODO") {
-          pending
-        }
       }
     }
+    describe("#2 local exists, remote is missing, other matches - copy") {
+      val theHash = MD5Hash("the-hash")
+      val theFile = LocalFile.resolve("the-file",
+                                      md5HashMap(theHash),
+                                      sourcePath,
+                                      fileToKey)
+      val theRemoteKey   = theFile.remoteKey
+      val otherRemoteKey = prefix.resolve("other-key")
+      val otherRemoteMetadata =
+        RemoteMetaData(otherRemoteKey, theHash, lastModified)
+      val input =
+        S3MetaData(theFile, // local exists
+                   matchByHash = Set(otherRemoteMetadata), // other matches
+                   matchByKey = None) // remote is missing
+      it("copy from other key") {
+        val expected = List(
+          ToCopy(bucket,
+                 otherRemoteKey,
+                 theHash,
+                 theRemoteKey,
+                 theFile.file.length)) // copy
+        val result = invoke(input)
+        assertResult(expected)(result)
+      }
+    }
+    describe("#3 local exists, remote is missing, other no matches - upload") {
+      val theHash = MD5Hash("the-hash")
+      val theFile = LocalFile.resolve("the-file",
+                                      md5HashMap(theHash),
+                                      sourcePath,
+                                      fileToKey)
+      val input = S3MetaData(theFile, // local exists
+                             matchByHash = Set.empty, // other no matches
+                             matchByKey = None) // remote is missing
+      it("upload") {
+        val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload
+        val result   = invoke(input)
+        assertResult(expected)(result)
+      }
+    }
+    describe(
+      "#4 local exists, remote exists, remote no match, other matches - copy") {
+      val theHash = MD5Hash("the-hash")
+      val theFile = LocalFile.resolve("the-file",
+                                      md5HashMap(theHash),
+                                      sourcePath,
+                                      fileToKey)
+      val theRemoteKey   = theFile.remoteKey
+      val oldHash        = MD5Hash("old-hash")
+      val otherRemoteKey = prefix.resolve("other-key")
+      val otherRemoteMetadata =
+        RemoteMetaData(otherRemoteKey, theHash, lastModified)
+      val oldRemoteMetadata = RemoteMetaData(theRemoteKey,
+                                             hash = oldHash, // remote no match
+                                             lastModified = lastModified)
+      val input =
+        S3MetaData(theFile, // local exists
+                   matchByHash = Set(otherRemoteMetadata), // other matches
+                   matchByKey = Some(oldRemoteMetadata)) // remote exists
+      it("copy from other key") {
+        val expected = List(
+          ToCopy(bucket,
+                 otherRemoteKey,
+                 theHash,
+                 theRemoteKey,
+                 theFile.file.length)) // copy
+        val result = invoke(input)
+        assertResult(expected)(result)
+      }
+    }
+    describe(
+      "#5 local exists, remote exists, remote no match, other no matches - upload") {
+      val theHash = MD5Hash("the-hash")
+      val theFile = LocalFile.resolve("the-file",
+                                      md5HashMap(theHash),
+                                      sourcePath,
+                                      fileToKey)
+      val theRemoteKey = theFile.remoteKey
+      val oldHash      = MD5Hash("old-hash")
+      val theRemoteMetadata =
+        RemoteMetaData(theRemoteKey, oldHash, lastModified)
+      val input =
+        S3MetaData(theFile, // local exists
+                   matchByHash = Set.empty, // remote no match, other no match
+                   matchByKey = Some(theRemoteMetadata) // remote exists
+        )
+      it("upload") {
+        val expected = List(ToUpload(bucket, theFile, theFile.file.length)) // upload
+        val result   = invoke(input)
+        assertResult(expected)(result)
+      }
+    }
+    describe("#6 local missing, remote exists - delete") {
+      it("TODO") {
+        pending
+      }
+    }
+  }
 
   private def md5HashMap(theHash: MD5Hash) = {
     Map("md5" -> theHash)

core/src/test/scala/net/kemitix/thorp/core/ConfigOptionTest.scala

@@ -9,15 +9,16 @@ class ConfigOptionTest extends FunSpec with TemporaryFolder {
     it("should preserve their order") {
       withDirectory(path1 => {
         withDirectory(path2 => {
-          val configOptions = ConfigOptions(List(
+          val configOptions = ConfigOptions(
-            ConfigOption.Source(path1),
+            List(
-            ConfigOption.Source(path2),
+              ConfigOption.Source(path1),
-            ConfigOption.Bucket("bucket"),
+              ConfigOption.Source(path2),
-            ConfigOption.IgnoreGlobalOptions,
+              ConfigOption.Bucket("bucket"),
-            ConfigOption.IgnoreUserOptions
+              ConfigOption.IgnoreGlobalOptions,
-          ))
+              ConfigOption.IgnoreUserOptions
+            ))
           val expected = Sources(List(path1, path2))
-          val result = invoke(configOptions)
+          val result   = invoke(configOptions)
           assert(result.isRight, result)
           assertResult(expected)(ConfigQuery.sources(configOptions))
         })

core/src/test/scala/net/kemitix/thorp/core/ConfigurationBuilderTest.scala

@@ -6,17 +6,16 @@ import net.kemitix.thorp.domain.Filter.{Exclude, Include}
 import net.kemitix.thorp.domain._
 import org.scalatest.FunSpec
 
-import scala.language.postfixOps
-
 class ConfigurationBuilderTest extends FunSpec with TemporaryFolder {
 
-  private val pwd: Path = Paths.get(System.getenv("PWD"))
+  private val pwd: Path                     = Paths.get(System.getenv("PWD"))
-  private val aBucket = Bucket("aBucket")
+  private val aBucket                       = Bucket("aBucket")
   private val coBucket: ConfigOption.Bucket = ConfigOption.Bucket(aBucket.name)
   private val thorpConfigFileName           = ".thorp.conf"
 
   private def configOptions(options: ConfigOption*): ConfigOptions =
-    ConfigOptions(List(
+    ConfigOptions(
+      List(
         ConfigOption.IgnoreUserOptions,
         ConfigOption.IgnoreGlobalOptions
       ) ++ options)
@@ -59,8 +58,8 @@ class ConfigurationBuilderTest extends FunSpec with TemporaryFolder {
     it("should only include the source once") {
       withDirectory(aSource => {
         val expected = Right(Sources(List(aSource)))
-        val options = configOptions(ConfigOption.Source(aSource), coBucket)
+        val options  = configOptions(ConfigOption.Source(aSource), coBucket)
-        val result = invoke(options).map(_.sources)
+        val result   = invoke(options).map(_.sources)
         assertResult(expected)(result)
       })
     }
@@ -70,10 +69,9 @@ class ConfigurationBuilderTest extends FunSpec with TemporaryFolder {
       withDirectory(currentSource => {
         withDirectory(previousSource => {
           val expected = Right(List(currentSource, previousSource))
-          val options = configOptions(
+          val options = configOptions(ConfigOption.Source(currentSource),
-            ConfigOption.Source(currentSource),
+                                      ConfigOption.Source(previousSource),
-            ConfigOption.Source(previousSource),
+                                      coBucket)
-            coBucket)
           val result = invoke(options).map(_.sources.paths)
           assertResult(expected)(result)
         })
@@ -84,12 +82,12 @@ class ConfigurationBuilderTest extends FunSpec with TemporaryFolder {
     it("should include both sources in order") {
       withDirectory(currentSource => {
         withDirectory(previousSource => {
-          writeFile(currentSource, thorpConfigFileName,
+          writeFile(currentSource,
-            s"source = $previousSource")
+                    thorpConfigFileName,
+                    s"source = $previousSource")
           val expected = Right(List(currentSource, previousSource))
-          val options = configOptions(
+          val options =
-            ConfigOption.Source(currentSource),
+            configOptions(ConfigOption.Source(currentSource), coBucket)
-            coBucket)
           val result = invoke(options).map(_.sources.paths)
           assertResult(expected)(result)
         })
@@ -99,19 +97,24 @@ class ConfigurationBuilderTest extends FunSpec with TemporaryFolder {
       it("should include settings from only current") {
         withDirectory(previousSource => {
           withDirectory(currentSource => {
-            writeFile(currentSource, thorpConfigFileName,
+            writeFile(
+              currentSource,
+              thorpConfigFileName,
               s"source = $previousSource",
               "bucket = current-bucket",
               "prefix = current-prefix",
               "include = current-include",
-              "exclude = current-exclude")
+              "exclude = current-exclude"
-            writeFile(previousSource, thorpConfigFileName,
+            )
-              "bucket = previous-bucket",
+            writeFile(previousSource,
-              "prefix = previous-prefix",
+                      thorpConfigFileName,
-              "include = previous-include",
+                      "bucket = previous-bucket",
-              "exclude = previous-exclude")
+                      "prefix = previous-prefix",
+                      "include = previous-include",
+                      "exclude = previous-exclude")
             // should have both sources in order
-            val expectedSources = Right(Sources(List(currentSource, previousSource)))
+            val expectedSources =
+              Right(Sources(List(currentSource, previousSource)))
             // should have bucket from current only
             val expectedBuckets = Right(Bucket("current-bucket"))
             // should have prefix from current only
@@ -121,7 +124,7 @@ class ConfigurationBuilderTest extends FunSpec with TemporaryFolder {
               Filter.Exclude("current-exclude"),
               Filter.Include("current-include")))
             val options = configOptions(ConfigOption.Source(currentSource))
-            val result = invoke(options)
+            val result  = invoke(options)
             assertResult(expectedSources)(result.map(_.sources))
             assertResult(expectedBuckets)(result.map(_.bucket))
             assertResult(expectedPrefixes)(result.map(_.prefix))
@@ -136,7 +139,9 @@ class ConfigurationBuilderTest extends FunSpec with TemporaryFolder {
     it("should only include first two sources") {
       withDirectory(currentSource => {
         withDirectory(parentSource => {
-          writeFile(currentSource, thorpConfigFileName, s"source = $parentSource")
+          writeFile(currentSource,
+                    thorpConfigFileName,
+                    s"source = $parentSource")
           withDirectory(grandParentSource => {
             writeFile(parentSource, thorpConfigFileName, s"source = $grandParentSource")
             val expected = Right(List(currentSource, parentSource))

core/src/test/scala/net/kemitix/thorp/core/DummyHashService.scala

@@ -7,10 +7,10 @@ import net.kemitix.thorp.domain.{Logger, MD5Hash}
 import net.kemitix.thorp.storage.api.HashService
 
 case class DummyHashService(hashes: Map[Path, Map[String, MD5Hash]])
-  extends HashService {
+    extends HashService {
 
-  override def hashLocalObject(path: Path)
+  override def hashLocalObject(path: Path)(
-                              (implicit l: Logger): IO[Map[String, MD5Hash]] =
+      implicit l: Logger): IO[Map[String, MD5Hash]] =
     IO.pure(hashes(path))
 
 }

core/src/test/scala/net/kemitix/thorp/core/DummyLogger.scala

@@ -7,7 +7,7 @@ class DummyLogger extends Logger {
 
   override def debug(message: => String): IO[Unit] = IO.unit
 
-  override def info(message: =>String): IO[Unit] = IO.unit
+  override def info(message: => String): IO[Unit] = IO.unit
 
   override def warn(message: String): IO[Unit] = IO.unit
 

core/src/test/scala/net/kemitix/thorp/core/DummyStorageService.scala

@@ -9,14 +9,13 @@ import net.kemitix.thorp.storage.api.StorageService
 
 case class DummyStorageService(s3ObjectData: S3ObjectsData,
                                uploadFiles: Map[File, (RemoteKey, MD5Hash)])
-  extends StorageService {
+    extends StorageService {
 
   override def shutdown: IO[StorageQueueEvent] =
     IO.pure(StorageQueueEvent.ShutdownQueueEvent())
 
-  override def listObjects(bucket: Bucket,
+  override def listObjects(bucket: Bucket, prefix: RemoteKey)(
-                           prefix: RemoteKey)
+      implicit l: Logger): EitherT[IO, String, S3ObjectsData] =
-                          (implicit l: Logger): EitherT[IO, String, S3ObjectsData] =
     EitherT.liftF(IO.pure(s3ObjectData))
 
   override def upload(localFile: LocalFile,

core/src/test/scala/net/kemitix/thorp/core/KeyGeneratorSuite.scala

@@ -8,26 +8,30 @@ import org.scalatest.FunSpec
 class KeyGeneratorSuite extends FunSpec {
 
   private val source: File = Resource(this, "upload")
-  private val sourcePath = source.toPath
+  private val sourcePath   = source.toPath
-  private val prefix = RemoteKey("prefix")
+  private val prefix       = RemoteKey("prefix")
-  implicit private val config: Config = Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
+  implicit private val config: Config =
-  private val fileToKey = KeyGenerator.generateKey(config.sources, config.prefix) _
+    Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
+  private val fileToKey =
+    KeyGenerator.generateKey(config.sources, config.prefix) _
 
   describe("key generator") {
 
     describe("when file is within source") {
       it("has a valid key") {
         val subdir = "subdir"
-        assertResult(RemoteKey(s"${prefix.key}/$subdir"))(fileToKey(sourcePath.resolve(subdir)))
+        assertResult(RemoteKey(s"${prefix.key}/$subdir"))(
+          fileToKey(sourcePath.resolve(subdir)))
       }
     }
 
     describe("when file is deeper within source") {
       it("has a valid key") {
         val subdir = "subdir/deeper/still"
-        assertResult(RemoteKey(s"${prefix.key}/$subdir"))(fileToKey(sourcePath.resolve(subdir)))
+        assertResult(RemoteKey(s"${prefix.key}/$subdir"))(
+          fileToKey(sourcePath.resolve(subdir)))
       }
     }
   }
 
-}
+}

core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala

@@ -2,30 +2,34 @@ package net.kemitix.thorp.core
 
 import java.nio.file.Paths
 
-import net.kemitix.thorp.domain.{Config, LocalFile, Logger, MD5HashData, Sources}
+import net.kemitix.thorp.domain._
 import net.kemitix.thorp.storage.api.HashService
 import org.scalatest.FunSpec
 
 class LocalFileStreamSuite extends FunSpec {
 
-  private val source = Resource(this, "upload")
+  private val source     = Resource(this, "upload")
   private val sourcePath = source.toPath
-  private val hashService: HashService = DummyHashService(Map(
+  private val hashService: HashService = DummyHashService(
-    file("root-file") -> Map("md5" -> MD5HashData.Root.hash),
+    Map(
-    file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash)
+      file("root-file")        -> Map("md5" -> MD5HashData.Root.hash),
-  ))
+      file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash)
+    ))
 
   private def file(filename: String) =
     sourcePath.resolve(Paths.get(filename))
 
-  implicit private val config: Config = Config(sources = Sources(List(sourcePath)))
+  implicit private val config: Config = Config(
+    sources = Sources(List(sourcePath)))
   implicit private val logger: Logger = new DummyLogger
 
   describe("findFiles") {
     it("should find all files") {
       val result: Set[String] =
         invoke.localFiles.toSet
-          .map { x: LocalFile => x.relative.toString }
+          .map { x: LocalFile =>
+            x.relative.toString
+          }
       assertResult(Set("subdir/leaf-file", "root-file"))(result)
     }
     it("should count all files") {

core/src/test/scala/net/kemitix/thorp/core/MD5HashGeneratorTest.scala

@@ -6,10 +6,11 @@ import org.scalatest.FunSpec
 
 class MD5HashGeneratorTest extends FunSpec {
 
-  private val source = Resource(this, "upload")
+  private val source     = Resource(this, "upload")
   private val sourcePath = source.toPath
-  private val prefix = RemoteKey("prefix")
+  private val prefix     = RemoteKey("prefix")
-  implicit private val config: Config = Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
+  implicit private val config: Config =
+    Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
   implicit private val logger: Logger = new DummyLogger
 
   describe("read a small file (smaller than buffer)") {
@@ -23,22 +24,26 @@ class MD5HashGeneratorTest extends FunSpec {
     val path = Resource(this, "big-file").toPath
     it("should generate the correct hash") {
       val expected = MD5HashData.BigFile.hash
-      val result = MD5HashGenerator.md5File(path).unsafeRunSync
+      val result   = MD5HashGenerator.md5File(path).unsafeRunSync
       assertResult(expected)(result)
     }
   }
   describe("read chunks of file") {
     val path = Resource(this, "big-file").toPath
     it("should generate the correct hash for first chunk of the file") {
-      val part1 = MD5HashData.BigFile.Part1
+      val part1    = MD5HashData.BigFile.Part1
       val expected = part1.hash
-      val result = MD5HashGenerator.md5FileChunk(path, part1.offset, part1.size).unsafeRunSync
+      val result = MD5HashGenerator
+        .md5FileChunk(path, part1.offset, part1.size)
+        .unsafeRunSync
       assertResult(expected)(result)
     }
     it("should generate the correcy hash for second chunk of the file") {
-      val part2 = MD5HashData.BigFile.Part2
+      val part2    = MD5HashData.BigFile.Part2
       val expected = part2.hash
-      val result = MD5HashGenerator.md5FileChunk(path, part2.offset, part2.size).unsafeRunSync
+      val result = MD5HashGenerator
+        .md5FileChunk(path, part2.offset, part2.size)
+        .unsafeRunSync
       assertResult(expected)(result)
     }
   }

core/src/test/scala/net/kemitix/thorp/core/ParseConfigFileTest.scala

@@ -6,9 +6,11 @@ import org.scalatest.FunSpec
 
 class ParseConfigFileTest extends FunSpec {
 
-  private def invoke(filename: Path) = ParseConfigFile.parseFile(filename).unsafeRunSync
   private val empty = ConfigOptions()
 
+  private def invoke(filename: Path) =
+    ParseConfigFile.parseFile(filename).unsafeRunSync
+
   describe("parse a missing file") {
     val filename = Paths.get("/path/to/missing/file")
     it("should return no options") {
@@ -29,9 +31,9 @@ class ParseConfigFileTest extends FunSpec {
   }
   describe("parse a file with properties") {
     val filename = Resource(this, "simple-config").toPath
-    val expected = ConfigOptions(List(
+    val expected = ConfigOptions(
-      ConfigOption.Source(Paths.get("/path/to/source")),
+      List(ConfigOption.Source(Paths.get("/path/to/source")),
-      ConfigOption.Bucket("bucket-name")))
+           ConfigOption.Bucket("bucket-name")))
     it("should return some options") {
       assertResult(expected)(invoke(filename))
     }

core/src/test/scala/net/kemitix/thorp/core/ParseConfigLinesTest.scala

@@ -9,64 +9,72 @@ class ParseConfigLinesTest extends FunSpec {
   describe("parse single lines") {
     describe("source") {
       it("should parse") {
-        val expected = ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source"))))
+        val expected =
-        val result = ParseConfigLines.parseLines(List("source = /path/to/source"))
+          ConfigOptions(List(ConfigOption.Source(Paths.get("/path/to/source"))))
+        val result =
+          ParseConfigLines.parseLines(List("source = /path/to/source"))
         assertResult(expected)(result)
       }
     }
     describe("bucket") {
       it("should parse") {
         val expected = ConfigOptions(List(ConfigOption.Bucket("bucket-name")))
-        val result = ParseConfigLines.parseLines(List("bucket = bucket-name"))
+        val result   = ParseConfigLines.parseLines(List("bucket = bucket-name"))
         assertResult(expected)(result)
       }
     }
     describe("prefix") {
       it("should parse") {
-        val expected = ConfigOptions(List(ConfigOption.Prefix("prefix/to/files")))
+        val expected =
-        val result = ParseConfigLines.parseLines(List("prefix = prefix/to/files"))
+          ConfigOptions(List(ConfigOption.Prefix("prefix/to/files")))
+        val result =
+          ParseConfigLines.parseLines(List("prefix = prefix/to/files"))
         assertResult(expected)(result)
       }
     }
     describe("include") {
       it("should parse") {
-        val expected = ConfigOptions(List(ConfigOption.Include("path/to/include")))
+        val expected =
-        val result = ParseConfigLines.parseLines(List("include = path/to/include"))
+          ConfigOptions(List(ConfigOption.Include("path/to/include")))
+        val result =
+          ParseConfigLines.parseLines(List("include = path/to/include"))
         assertResult(expected)(result)
       }
     }
     describe("exclude") {
       it("should parse") {
-        val expected = ConfigOptions(List(ConfigOption.Exclude("path/to/exclude")))
+        val expected =
-        val result = ParseConfigLines.parseLines(List("exclude = path/to/exclude"))
+          ConfigOptions(List(ConfigOption.Exclude("path/to/exclude")))
+        val result =
+          ParseConfigLines.parseLines(List("exclude = path/to/exclude"))
         assertResult(expected)(result)
       }
     }
     describe("debug - true") {
       it("should parse") {
         val expected = ConfigOptions(List(ConfigOption.Debug()))
-        val result = ParseConfigLines.parseLines(List("debug = true"))
+        val result   = ParseConfigLines.parseLines(List("debug = true"))
         assertResult(expected)(result)
       }
     }
     describe("debug - false") {
       it("should parse") {
         val expected = ConfigOptions()
-        val result = ParseConfigLines.parseLines(List("debug = false"))
+        val result   = ParseConfigLines.parseLines(List("debug = false"))
         assertResult(expected)(result)
       }
     }
     describe("comment line") {
       it("should be ignored") {
         val expected = ConfigOptions()
-        val result = ParseConfigLines.parseLines(List("# ignore me"))
+        val result   = ParseConfigLines.parseLines(List("# ignore me"))
         assertResult(expected)(result)
       }
     }
     describe("unrecognised option") {
       it("should be ignored") {
         val expected = ConfigOptions()
-        val result = ParseConfigLines.parseLines(List("unsupported = option"))
+        val result   = ParseConfigLines.parseLines(List("unsupported = option"))
         assertResult(expected)(result)
       }
     }

core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala

@@ -9,12 +9,10 @@ import net.kemitix.thorp.storage.api.{HashService, StorageService}
 import org.scalatest.FreeSpec
 
 class PlanBuilderTest extends FreeSpec with TemporaryFolder {
-
+  val lastModified: LastModified      = LastModified()
-  private val planBuilder = new PlanBuilder {}
+  private val planBuilder             = new PlanBuilder {}
-  private val emptyS3ObjectData = S3ObjectsData()
   private implicit val logger: Logger = new DummyLogger
-
+  private val emptyS3ObjectData       = S3ObjectsData()
-  val lastModified: LastModified = LastModified()
 
   "create a plan" - {
 
@@ -22,7 +20,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
 
     "one source" - {
       "a file" - {
-        val filename = "aFile"
+        val filename  = "aFile"
         val remoteKey = RemoteKey(filename)
         "with no matching remote key" - {
           "with no other remote key with matching hash" - {
@@ -31,17 +29,22 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
                 val file = createFile(source, filename, "file-content")
                 val hash = md5Hash(file)
 
-                val expected = Right(List(
+                val expected = Right(
-                  toUpload(remoteKey, hash, source, file)
+                  List(
-                ))
+                    toUpload(remoteKey, hash, source, file)
+                  ))
 
-                val storageService = DummyStorageService(emptyS3ObjectData, Map(
+                val storageService =
-                  file -> (remoteKey, hash)
+                  DummyStorageService(emptyS3ObjectData,
-                ))
+                                      Map(
+                                        file -> (remoteKey, hash)
+                                      ))
 
-                val result = invoke(storageService, hashService, configOptions(
+                val result =
-                  ConfigOption.Source(source),
+                  invoke(storageService,
-                  ConfigOption.Bucket("a-bucket")))
+                         hashService,
+                         configOptions(ConfigOption.Source(source),
+                                       ConfigOption.Bucket("a-bucket")))
 
                 assertResult(expected)(result)
               })
@@ -51,29 +54,35 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
             "copy file" in {
               withDirectory(source => {
                 val anOtherFilename = "other"
-                val content = "file-content"
+                val content         = "file-content"
-                val aFile = createFile(source, filename, content)
+                val aFile           = createFile(source, filename, content)
-                val anOtherFile = createFile(source, anOtherFilename, content)
+                val anOtherFile     = createFile(source, anOtherFilename, content)
-                val aHash = md5Hash(aFile)
+                val aHash           = md5Hash(aFile)
 
                 val anOtherKey = RemoteKey("other")
 
-                val expected = Right(List(
+                val expected = Right(
-                  toCopy(anOtherKey, aHash, remoteKey)
+                  List(
-                ))
+                    toCopy(anOtherKey, aHash, remoteKey)
+                  ))
 
                 val s3ObjectsData = S3ObjectsData(
-                  byHash = Map(aHash -> Set(KeyModified(anOtherKey, lastModified))),
+                  byHash =
+                    Map(aHash            -> Set(KeyModified(anOtherKey, lastModified))),
                   byKey = Map(anOtherKey -> HashModified(aHash, lastModified))
                 )
 
-                val storageService = DummyStorageService(s3ObjectsData, Map(
+                val storageService =
-                  aFile -> (remoteKey, aHash)
+                  DummyStorageService(s3ObjectsData,
-                ))
+                                      Map(
+                                        aFile -> (remoteKey, aHash)
+                                      ))
 
-                val result = invoke(storageService, hashService, configOptions(
+                val result =
-                  ConfigOption.Source(source),
+                  invoke(storageService,
-                  ConfigOption.Bucket("a-bucket")))
+                         hashService,
+                         configOptions(ConfigOption.Source(source),
+                                       ConfigOption.Bucket("a-bucket")))
 
                 assertResult(expected)(result)
               })
@@ -91,17 +100,22 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
                 val expected = Right(List())
 
                 val s3ObjectsData = S3ObjectsData(
-                  byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))),
+                  byHash =
+                    Map(hash            -> Set(KeyModified(remoteKey, lastModified))),
                   byKey = Map(remoteKey -> HashModified(hash, lastModified))
                 )
 
-                val storageService = DummyStorageService(s3ObjectsData, Map(
+                val storageService =
-                  file -> (remoteKey, hash)
+                  DummyStorageService(s3ObjectsData,
-                ))
+                                      Map(
+                                        file -> (remoteKey, hash)
+                                      ))
 
-                val result = invoke(storageService, hashService, configOptions(
+                val result =
-                  ConfigOption.Source(source),
+                  invoke(storageService,
-                  ConfigOption.Bucket("a-bucket")))
+                         hashService,
+                         configOptions(ConfigOption.Source(source),
+                                       ConfigOption.Bucket("a-bucket")))
 
                 assertResult(expected)(result)
               })
@@ -111,26 +125,33 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
             "with no matching remote hash" - {
               "upload file" in {
                 withDirectory(source => {
-                  val file = createFile(source, filename, "file-content")
+                  val file         = createFile(source, filename, "file-content")
-                  val currentHash = md5Hash(file)
+                  val currentHash  = md5Hash(file)
                   val originalHash = MD5Hash("original-file-content")
 
-                  val expected = Right(List(
+                  val expected = Right(
-                    toUpload(remoteKey, currentHash, source, file)
+                    List(
-                  ))
+                      toUpload(remoteKey, currentHash, source, file)
+                    ))
 
                   val s3ObjectsData = S3ObjectsData(
-                    byHash = Map(originalHash -> Set(KeyModified(remoteKey, lastModified))),
+                    byHash = Map(originalHash -> Set(
-                    byKey = Map(remoteKey -> HashModified(originalHash, lastModified))
+                      KeyModified(remoteKey, lastModified))),
+                    byKey =
+                      Map(remoteKey -> HashModified(originalHash, lastModified))
                   )
 
-                  val storageService = DummyStorageService(s3ObjectsData, Map(
+                  val storageService =
-                    file -> (remoteKey, currentHash)
+                    DummyStorageService(s3ObjectsData,
-                  ))
+                                        Map(
+                                          file -> (remoteKey, currentHash)
+                                        ))
 
-                  val result = invoke(storageService, hashService, configOptions(
+                  val result =
-                    ConfigOption.Source(source),
+                    invoke(storageService,
-                    ConfigOption.Bucket("a-bucket")))
+                           hashService,
+                           configOptions(ConfigOption.Source(source),
+                                         ConfigOption.Bucket("a-bucket")))
 
                   assertResult(expected)(result)
                 })
@@ -139,26 +160,32 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
             "with matching remote hash" - {
               "copy file" in {
                 withDirectory(source => {
-                  val file = createFile(source, filename, "file-content")
+                  val file      = createFile(source, filename, "file-content")
-                  val hash = md5Hash(file)
+                  val hash      = md5Hash(file)
                   val sourceKey = RemoteKey("other-key")
 
-                  val expected = Right(List(
+                  val expected = Right(
-                    toCopy(sourceKey, hash, remoteKey)
+                    List(
-                  ))
+                      toCopy(sourceKey, hash, remoteKey)
+                    ))
 
                   val s3ObjectsData = S3ObjectsData(
-                    byHash = Map(hash -> Set(KeyModified(sourceKey, lastModified))),
+                    byHash =
+                      Map(hash -> Set(KeyModified(sourceKey, lastModified))),
                     byKey = Map()
                   )
 
-                  val storageService = DummyStorageService(s3ObjectsData, Map(
+                  val storageService =
-                    file -> (remoteKey, hash)
+                    DummyStorageService(s3ObjectsData,
-                  ))
+                                        Map(
+                                          file -> (remoteKey, hash)
+                                        ))
 
-                  val result = invoke(storageService, hashService, configOptions(
+                  val result =
-                    ConfigOption.Source(source),
+                    invoke(storageService,
-                    ConfigOption.Bucket("a-bucket")))
+                           hashService,
+                           configOptions(ConfigOption.Source(source),
+                                         ConfigOption.Bucket("a-bucket")))
 
                   assertResult(expected)(result)
                 })
@@ -168,7 +195,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
         }
       }
       "a remote key" - {
-        val filename = "aFile"
+        val filename  = "aFile"
         val remoteKey = RemoteKey(filename)
         "with a matching local file" - {
           "do nothing" in {
@@ -180,17 +207,21 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
               val expected = Right(List())
 
               val s3ObjectsData = S3ObjectsData(
-                byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))),
+                byHash = Map(hash     -> Set(KeyModified(remoteKey, lastModified))),
                 byKey = Map(remoteKey -> HashModified(hash, lastModified))
               )
 
-              val storageService = DummyStorageService(s3ObjectsData, Map(
+              val storageService =
-                file -> (remoteKey, hash)
+                DummyStorageService(s3ObjectsData,
-              ))
+                                    Map(
+                                      file -> (remoteKey, hash)
+                                    ))
 
-              val result = invoke(storageService, hashService, configOptions(
+              val result =
-                ConfigOption.Source(source),
+                invoke(storageService,
-                ConfigOption.Bucket("a-bucket")))
+                       hashService,
+                       configOptions(ConfigOption.Source(source),
+                                     ConfigOption.Bucket("a-bucket")))
 
               assertResult(expected)(result)
             })
@@ -201,20 +232,23 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
             withDirectory(source => {
               val hash = MD5Hash("file-content")
 
-              val expected = Right(List(
+              val expected = Right(
-                toDelete(remoteKey)
+                List(
-              ))
+                  toDelete(remoteKey)
+                ))
 
               val s3ObjectsData = S3ObjectsData(
-                byHash = Map(hash -> Set(KeyModified(remoteKey, lastModified))),
+                byHash = Map(hash     -> Set(KeyModified(remoteKey, lastModified))),
                 byKey = Map(remoteKey -> HashModified(hash, lastModified))
               )
 
               val storageService = DummyStorageService(s3ObjectsData, Map.empty)
 
-              val result = invoke(storageService, hashService, configOptions(
+              val result =
-                ConfigOption.Source(source),
+                invoke(storageService,
-                ConfigOption.Bucket("a-bucket")))
+                       hashService,
+                       configOptions(ConfigOption.Source(source),
+                                     ConfigOption.Bucket("a-bucket")))
 
               assertResult(expected)(result)
             })
@@ -224,33 +258,39 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
     }
 
     "two sources" - {
-      val filename1 = "file-1"
+      val filename1  = "file-1"
-      val filename2 = "file-2"
+      val filename2  = "file-2"
       val remoteKey1 = RemoteKey(filename1)
       val remoteKey2 = RemoteKey(filename2)
       "unique files in both" - {
         "upload all files" in {
           withDirectory(firstSource => {
-            val fileInFirstSource = createFile(firstSource, filename1, "file-1-content")
+            val fileInFirstSource =
+              createFile(firstSource, filename1, "file-1-content")
             val hash1 = md5Hash(fileInFirstSource)
 
             withDirectory(secondSource => {
-              val fileInSecondSource = createFile(secondSource, filename2, "file-2-content")
+              val fileInSecondSource =
+                createFile(secondSource, filename2, "file-2-content")
               val hash2 = md5Hash(fileInSecondSource)
 
-              val expected = Right(List(
+              val expected = Right(
-                toUpload(remoteKey2, hash2, secondSource, fileInSecondSource),
+                List(
-                toUpload(remoteKey1, hash1, firstSource, fileInFirstSource)
+                  toUpload(remoteKey2, hash2, secondSource, fileInSecondSource),
-              ))
+                  toUpload(remoteKey1, hash1, firstSource, fileInFirstSource)
+                ))
 
-              val storageService = DummyStorageService(emptyS3ObjectData, Map(
+              val storageService = DummyStorageService(
-                fileInFirstSource -> (remoteKey1, hash1),
+                emptyS3ObjectData,
-                fileInSecondSource -> (remoteKey2, hash2)))
+                Map(fileInFirstSource  -> (remoteKey1, hash1),
+                    fileInSecondSource -> (remoteKey2, hash2)))
 
-              val result = invoke(storageService, hashService, configOptions(
+              val result =
-                ConfigOption.Source(firstSource),
+                invoke(storageService,
-                ConfigOption.Source(secondSource),
+                       hashService,
-                ConfigOption.Bucket("a-bucket")))
+                       configOptions(ConfigOption.Source(firstSource),
+                                     ConfigOption.Source(secondSource),
+                                     ConfigOption.Bucket("a-bucket")))
 
               assertResult(expected)(result)
             })
@@ -260,52 +300,63 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
       "same filename in both" - {
         "only upload file in first source" in {
           withDirectory(firstSource => {
-            val fileInFirstSource: File = createFile(firstSource, filename1, "file-1-content")
+            val fileInFirstSource: File =
+              createFile(firstSource, filename1, "file-1-content")
             val hash1 = md5Hash(fileInFirstSource)
 
             withDirectory(secondSource => {
-              val fileInSecondSource: File = createFile(secondSource, filename1, "file-2-content")
+              val fileInSecondSource: File =
+                createFile(secondSource, filename1, "file-2-content")
               val hash2 = md5Hash(fileInSecondSource)
 
-              val expected = Right(List(
+              val expected = Right(
-                toUpload(remoteKey1, hash1, firstSource, fileInFirstSource)
+                List(
-              ))
+                  toUpload(remoteKey1, hash1, firstSource, fileInFirstSource)
+                ))
 
-              val storageService = DummyStorageService(emptyS3ObjectData, Map(
+              val storageService = DummyStorageService(
-                fileInFirstSource -> (remoteKey1, hash1),
+                emptyS3ObjectData,
-                fileInSecondSource -> (remoteKey2, hash2)))
+                Map(fileInFirstSource  -> (remoteKey1, hash1),
+                    fileInSecondSource -> (remoteKey2, hash2)))
 
-              val result = invoke(storageService, hashService, configOptions(
+              val result =
-                ConfigOption.Source(firstSource),
+                invoke(storageService,
-                ConfigOption.Source(secondSource),
+                       hashService,
-                ConfigOption.Bucket("a-bucket")))
+                       configOptions(ConfigOption.Source(firstSource),
+                                     ConfigOption.Source(secondSource),
+                                     ConfigOption.Bucket("a-bucket")))
 
               assertResult(expected)(result)
             })
           })
         }
       }
-      "with a remote file only present in second source" -  {
+      "with a remote file only present in second source" - {
         "do not delete it " in {
           withDirectory(firstSource => {
 
             withDirectory(secondSource => {
-              val fileInSecondSource = createFile(secondSource, filename2, "file-2-content")
+              val fileInSecondSource =
+                createFile(secondSource, filename2, "file-2-content")
               val hash2 = md5Hash(fileInSecondSource)
 
               val expected = Right(List())
 
               val s3ObjectData = S3ObjectsData(
-                byHash = Map(hash2 -> Set(KeyModified(remoteKey2, lastModified))),
+                byHash =
+                  Map(hash2            -> Set(KeyModified(remoteKey2, lastModified))),
                 byKey = Map(remoteKey2 -> HashModified(hash2, lastModified)))
 
-              val storageService = DummyStorageService(s3ObjectData, Map(
+              val storageService = DummyStorageService(
-                fileInSecondSource -> (remoteKey2, hash2)))
+                s3ObjectData,
+                Map(fileInSecondSource -> (remoteKey2, hash2)))
 
-              val result = invoke(storageService, hashService, configOptions(
+              val result =
-                ConfigOption.Source(firstSource),
+                invoke(storageService,
-                ConfigOption.Source(secondSource),
+                       hashService,
-                ConfigOption.Bucket("a-bucket")))
+                       configOptions(ConfigOption.Source(firstSource),
+                                     ConfigOption.Source(secondSource),
+                                     ConfigOption.Bucket("a-bucket")))
 
               assertResult(expected)(result)
             })
@@ -315,7 +366,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
       "with remote file only present in first source" - {
         "do not delete it" in {
           withDirectory(firstSource => {
-            val fileInFirstSource: File = createFile(firstSource, filename1, "file-1-content")
+            val fileInFirstSource: File =
+              createFile(firstSource, filename1, "file-1-content")
             val hash1 = md5Hash(fileInFirstSource)
 
             withDirectory(secondSource => {
@@ -323,16 +375,20 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
               val expected = Right(List())
 
               val s3ObjectData = S3ObjectsData(
-                byHash = Map(hash1 -> Set(KeyModified(remoteKey1, lastModified))),
+                byHash =
+                  Map(hash1            -> Set(KeyModified(remoteKey1, lastModified))),
                 byKey = Map(remoteKey1 -> HashModified(hash1, lastModified)))
 
-              val storageService = DummyStorageService(s3ObjectData, Map(
+              val storageService = DummyStorageService(
-                fileInFirstSource -> (remoteKey1, hash1)))
+                s3ObjectData,
+                Map(fileInFirstSource -> (remoteKey1, hash1)))
 
-              val result = invoke(storageService, hashService, configOptions(
+              val result =
-                ConfigOption.Source(firstSource),
+                invoke(storageService,
-                ConfigOption.Source(secondSource),
+                       hashService,
-                ConfigOption.Bucket("a-bucket")))
+                       configOptions(ConfigOption.Source(firstSource),
+                                     ConfigOption.Source(secondSource),
+                                     ConfigOption.Bucket("a-bucket")))
 
               assertResult(expected)(result)
             })
@@ -345,19 +401,22 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
 
             withDirectory(secondSource => {
 
-              val expected = Right(List(
+              val expected = Right(
-                toDelete(remoteKey1)
+                List(
-              ))
+                  toDelete(remoteKey1)
+                ))
 
-              val s3ObjectData = S3ObjectsData(
+              val s3ObjectData = S3ObjectsData(byKey =
-                byKey = Map(remoteKey1 -> HashModified(MD5Hash(""), lastModified)))
+                Map(remoteKey1 -> HashModified(MD5Hash(""), lastModified)))
 
               val storageService = DummyStorageService(s3ObjectData, Map())
 
-              val result = invoke(storageService, hashService, configOptions(
+              val result =
-                ConfigOption.Source(firstSource),
+                invoke(storageService,
-                ConfigOption.Source(secondSource),
+                       hashService,
-                ConfigOption.Bucket("a-bucket")))
+                       configOptions(ConfigOption.Source(firstSource),
+                                     ConfigOption.Source(secondSource),
+                                     ConfigOption.Bucket("a-bucket")))
 
               assertResult(expected)(result)
             })
@@ -378,27 +437,40 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
                        file: File): (String, String, String, String, String) =
     ("upload", remoteKey.key, md5Hash.hash, source.toString, file.toString)
 
-  private def toCopy(sourceKey: RemoteKey,
+  private def toCopy(
-                     md5Hash: MD5Hash,
+      sourceKey: RemoteKey,
-                     targetKey: RemoteKey): (String, String, String, String, String) =
+      md5Hash: MD5Hash,
+      targetKey: RemoteKey): (String, String, String, String, String) =
     ("copy", sourceKey.key, md5Hash.hash, targetKey.key, "")
 
-  private def toDelete(remoteKey: RemoteKey): (String, String, String, String, String) =
+  private def toDelete(
+      remoteKey: RemoteKey): (String, String, String, String, String) =
     ("delete", remoteKey.key, "", "", "")
 
   private def configOptions(configOptions: ConfigOption*): ConfigOptions =
-    ConfigOptions(List(configOptions:_*))
+    ConfigOptions(List(configOptions: _*))
 
   private def invoke(storageService: StorageService,
                      hashService: HashService,
-                     configOptions: ConfigOptions): Either[List[String], List[(String, String, String, String, String)]] =
+                     configOptions: ConfigOptions)
-    planBuilder.createPlan(storageService, hashService, configOptions)
+    : Either[List[String], List[(String, String, String, String, String)]] =
-      .value.unsafeRunSync().map(_.actions.toList.map({
+    planBuilder
-      case ToUpload(_, lf, _) => ("upload", lf.remoteKey.key, lf.hashes("md5").hash, lf.source.toString, lf.file.toString)
+      .createPlan(storageService, hashService, configOptions)
-      case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")
+      .value
-      case ToCopy(_, sourceKey, hash, targetKey, _) => ("copy", sourceKey.key, hash.hash, targetKey.key, "")
+      .unsafeRunSync()
-      case DoNothing(_, remoteKey, _) => ("do-nothing", remoteKey.key, "", "", "")
+      .map(_.actions.toList.map({
-      case x => ("other", x.toString, "", "", "")
+        case ToUpload(_, lf, _) =>
-    }))
+          ("upload",
+           lf.remoteKey.key,
+           lf.hashes("md5").hash,
+           lf.source.toString,
+           lf.file.toString)
+        case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")
+        case ToCopy(_, sourceKey, hash, targetKey, _) =>
+          ("copy", sourceKey.key, hash.hash, targetKey.key, "")
+        case DoNothing(_, remoteKey, _) =>
+          ("do-nothing", remoteKey.key, "", "", "")
+        case x => ("other", x.toString, "", "", "")
+      }))
 
 }

core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala

@@ -6,138 +6,169 @@ import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
 import net.kemitix.thorp.domain._
 import org.scalatest.FunSpec
 
-class S3MetaDataEnricherSuite
+class S3MetaDataEnricherSuite extends FunSpec {
-  extends FunSpec {
+  val lastModified       = LastModified(Instant.now())
-
+  private val source     = Resource(this, "upload")
-  private val source = Resource(this, "upload")
   private val sourcePath = source.toPath
-  private val prefix = RemoteKey("prefix")
+  private val prefix     = RemoteKey("prefix")
-  implicit private val config: Config = Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
+  implicit private val config: Config =
-  private val fileToKey = KeyGenerator.generateKey(config.sources, config.prefix) _
+    Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
-  val lastModified = LastModified(Instant.now())
+  private val fileToKey =
+    KeyGenerator.generateKey(config.sources, config.prefix) _
 
-  def getMatchesByKey(status: (Option[HashModified], Set[(MD5Hash, KeyModified)])): Option[HashModified] = {
+  def getMatchesByKey(
+      status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
+    : Option[HashModified] = {
     val (byKey, _) = status
     byKey
   }
 
-  def getMatchesByHash(status: (Option[HashModified], Set[(MD5Hash, KeyModified)])): Set[(MD5Hash, KeyModified)] = {
+  def getMatchesByHash(
+      status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
+    : Set[(MD5Hash, KeyModified)] = {
     val (_, byHash) = status
     byHash
   }
 
   describe("enrich with metadata") {
 
-      describe("#1a local exists, remote exists, remote matches, other matches - do nothing") {
+    describe(
-        val theHash: MD5Hash = MD5Hash("the-file-hash")
+      "#1a local exists, remote exists, remote matches, other matches - do nothing") {
-        val theFile: LocalFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
+      val theHash: MD5Hash = MD5Hash("the-file-hash")
-        val theRemoteKey: RemoteKey = theFile.remoteKey
+      val theFile: LocalFile = LocalFile.resolve("the-file",
-        val s3: S3ObjectsData = S3ObjectsData(
+                                                 md5HashMap(theHash),
-          byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
+                                                 sourcePath,
-          byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
+                                                 fileToKey)
+      val theRemoteKey: RemoteKey = theFile.remoteKey
+      val s3: S3ObjectsData = S3ObjectsData(
+        byHash = Map(theHash     -> Set(KeyModified(theRemoteKey, lastModified))),
+        byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
+      )
+      val theRemoteMetadata =
+        RemoteMetaData(theRemoteKey, theHash, lastModified)
+      it("generates valid metadata") {
+        val expected = S3MetaData(theFile,
+                                  matchByHash = Set(theRemoteMetadata),
+                                  matchByKey = Some(theRemoteMetadata))
+        val result = getMetadata(theFile, s3)
+        assertResult(expected)(result)
+      }
+    }
+    describe(
+      "#1b local exists, remote exists, remote matches, other no matches - do nothing") {
+      val theHash: MD5Hash = MD5Hash("the-file-hash")
+      val theFile: LocalFile = LocalFile.resolve("the-file",
+                                                 md5HashMap(theHash),
+                                                 sourcePath,
+                                                 fileToKey)
+      val theRemoteKey: RemoteKey = prefix.resolve("the-file")
+      val s3: S3ObjectsData = S3ObjectsData(
+        byHash = Map(theHash     -> Set(KeyModified(theRemoteKey, lastModified))),
+        byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
+      )
+      val theRemoteMetadata =
+        RemoteMetaData(theRemoteKey, theHash, lastModified)
+      it("generates valid metadata") {
+        val expected = S3MetaData(theFile,
+                                  matchByHash = Set(theRemoteMetadata),
+                                  matchByKey = Some(theRemoteMetadata))
+        val result = getMetadata(theFile, s3)
+        assertResult(expected)(result)
+      }
+    }
+    describe(
+      "#2 local exists, remote is missing, remote no match, other matches - copy") {
+      val theHash = MD5Hash("the-hash")
+      val theFile = LocalFile.resolve("the-file",
+                                      md5HashMap(theHash),
+                                      sourcePath,
+                                      fileToKey)
+      val otherRemoteKey = RemoteKey("other-key")
+      val s3: S3ObjectsData = S3ObjectsData(
+        byHash = Map(theHash       -> Set(KeyModified(otherRemoteKey, lastModified))),
+        byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
+      )
+      val otherRemoteMetadata =
+        RemoteMetaData(otherRemoteKey, theHash, lastModified)
+      it("generates valid metadata") {
+        val expected = S3MetaData(theFile,
+                                  matchByHash = Set(otherRemoteMetadata),
+                                  matchByKey = None)
+        val result = getMetadata(theFile, s3)
+        assertResult(expected)(result)
+      }
+    }
+    describe(
+      "#3 local exists, remote is missing, remote no match, other no matches - upload") {
+      val theHash = MD5Hash("the-hash")
+      val theFile = LocalFile.resolve("the-file",
+                                      md5HashMap(theHash),
+                                      sourcePath,
+                                      fileToKey)
+      val s3: S3ObjectsData = S3ObjectsData()
+      it("generates valid metadata") {
+        val expected =
+          S3MetaData(theFile, matchByHash = Set.empty, matchByKey = None)
+        val result = getMetadata(theFile, s3)
+        assertResult(expected)(result)
+      }
+    }
+    describe(
+      "#4 local exists, remote exists, remote no match, other matches - copy") {
+      val theHash = MD5Hash("the-hash")
+      val theFile = LocalFile.resolve("the-file",
+                                      md5HashMap(theHash),
+                                      sourcePath,
+                                      fileToKey)
+      val theRemoteKey   = theFile.remoteKey
+      val oldHash        = MD5Hash("old-hash")
+      val otherRemoteKey = prefix.resolve("other-key")
+      val s3: S3ObjectsData = S3ObjectsData(
+        byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
+                     theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
+        byKey = Map(
+          theRemoteKey   -> HashModified(oldHash, lastModified),
+          otherRemoteKey -> HashModified(theHash, lastModified)
         )
-        val theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
+      )
-        it("generates valid metadata") {
+      val theRemoteMetadata =
-          val expected = S3MetaData(theFile,
+        RemoteMetaData(theRemoteKey, oldHash, lastModified)
-            matchByHash = Set(theRemoteMetadata),
+      val otherRemoteMetadata =
-            matchByKey = Some(theRemoteMetadata))
+        RemoteMetaData(otherRemoteKey, theHash, lastModified)
-          val result = getMetadata(theFile, s3)
+      it("generates valid metadata") {
-          assertResult(expected)(result)
+        val expected = S3MetaData(theFile,
-        }
+                                  matchByHash = Set(otherRemoteMetadata),
+                                  matchByKey = Some(theRemoteMetadata))
+        val result = getMetadata(theFile, s3)
+        assertResult(expected)(result)
       }
-      describe("#1b local exists, remote exists, remote matches, other no matches - do nothing") {
+    }
-        val theHash: MD5Hash = MD5Hash("the-file-hash")
+    describe(
-        val theFile: LocalFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
+      "#5 local exists, remote exists, remote no match, other no matches - upload") {
-        val theRemoteKey: RemoteKey = prefix.resolve("the-file")
+      val theHash = MD5Hash("the-hash")
-        val s3: S3ObjectsData = S3ObjectsData(
+      val theFile = LocalFile.resolve("the-file",
-          byHash = Map(theHash -> Set(KeyModified(theRemoteKey, lastModified))),
+                                      md5HashMap(theHash),
-          byKey = Map(theRemoteKey -> HashModified(theHash, lastModified))
+                                      sourcePath,
+                                      fileToKey)
+      val theRemoteKey = theFile.remoteKey
+      val oldHash      = MD5Hash("old-hash")
+      val s3: S3ObjectsData = S3ObjectsData(
+        byHash = Map(oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
+                     theHash -> Set.empty),
+        byKey = Map(
+          theRemoteKey -> HashModified(oldHash, lastModified)
         )
-        val theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash, lastModified)
+      )
-        it("generates valid metadata") {
+      val theRemoteMetadata =
-          val expected = S3MetaData(theFile,
+        RemoteMetaData(theRemoteKey, oldHash, lastModified)
-            matchByHash = Set(theRemoteMetadata),
+      it("generates valid metadata") {
-            matchByKey = Some(theRemoteMetadata))
+        val expected = S3MetaData(theFile,
-          val result = getMetadata(theFile, s3)
+                                  matchByHash = Set.empty,
-          assertResult(expected)(result)
+                                  matchByKey = Some(theRemoteMetadata))
-        }
+        val result = getMetadata(theFile, s3)
-      }
+        assertResult(expected)(result)
-      describe("#2 local exists, remote is missing, remote no match, other matches - copy") {
-        val theHash = MD5Hash("the-hash")
-        val theFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
-        val otherRemoteKey = RemoteKey("other-key")
-        val s3: S3ObjectsData = S3ObjectsData(
-          byHash = Map(theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
-          byKey = Map(otherRemoteKey -> HashModified(theHash, lastModified))
-        )
-        val otherRemoteMetadata = RemoteMetaData(otherRemoteKey, theHash, lastModified)
-        it("generates valid metadata") {
-          val expected = S3MetaData(theFile,
-            matchByHash = Set(otherRemoteMetadata),
-            matchByKey = None)
-          val result = getMetadata(theFile, s3)
-          assertResult(expected)(result)
-        }
-      }
-      describe("#3 local exists, remote is missing, remote no match, other no matches - upload") {
-        val theHash = MD5Hash("the-hash")
-        val theFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
-        val s3: S3ObjectsData = S3ObjectsData()
-        it("generates valid metadata") {
-          val expected = S3MetaData(theFile,
-            matchByHash = Set.empty,
-            matchByKey = None)
-          val result = getMetadata(theFile, s3)
-          assertResult(expected)(result)
-        }
-      }
-      describe("#4 local exists, remote exists, remote no match, other matches - copy") {
-        val theHash = MD5Hash("the-hash")
-        val theFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
-        val theRemoteKey = theFile.remoteKey
-        val oldHash = MD5Hash("old-hash")
-        val otherRemoteKey = prefix.resolve("other-key")
-        val s3: S3ObjectsData = S3ObjectsData(
-          byHash = Map(
-            oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
-            theHash -> Set(KeyModified(otherRemoteKey, lastModified))),
-          byKey = Map(
-            theRemoteKey -> HashModified(oldHash, lastModified),
-            otherRemoteKey -> HashModified(theHash, lastModified)
-          )
-        )
-        val theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
-        val otherRemoteMetadata = RemoteMetaData(otherRemoteKey, theHash, lastModified)
-        it("generates valid metadata") {
-          val expected = S3MetaData(theFile,
-            matchByHash = Set(otherRemoteMetadata),
-            matchByKey = Some(theRemoteMetadata))
-          val result = getMetadata(theFile, s3)
-          assertResult(expected)(result)
-        }
-      }
-      describe("#5 local exists, remote exists, remote no match, other no matches - upload") {
-        val theHash = MD5Hash("the-hash")
-        val theFile = LocalFile.resolve("the-file", md5HashMap(theHash), sourcePath, fileToKey)
-        val theRemoteKey = theFile.remoteKey
-        val oldHash = MD5Hash("old-hash")
-        val s3: S3ObjectsData = S3ObjectsData(
-          byHash = Map(
-            oldHash -> Set(KeyModified(theRemoteKey, lastModified)),
-            theHash -> Set.empty),
-          byKey = Map(
-            theRemoteKey -> HashModified(oldHash, lastModified)
-          )
-        )
-        val theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash, lastModified)
-        it("generates valid metadata") {
-          val expected = S3MetaData(theFile,
-            matchByHash = Set.empty,
-            matchByKey = Some(theRemoteMetadata))
-          val result = getMetadata(theFile, s3)
-          assertResult(expected)(result)
-        }
       }
+    }
   }
 
   private def md5HashMap(theHash: MD5Hash) = {
@@ -146,20 +177,31 @@ class S3MetaDataEnricherSuite
 
   describe("getS3Status") {
     val hash = MD5Hash("hash")
-    val localFile = LocalFile.resolve("the-file", md5HashMap(hash), sourcePath, fileToKey)
+    val localFile =
+      LocalFile.resolve("the-file", md5HashMap(hash), sourcePath, fileToKey)
     val key = localFile.remoteKey
-    val keyOtherKey = LocalFile.resolve("other-key-same-hash", md5HashMap(hash), sourcePath, fileToKey)
+    val keyOtherKey = LocalFile.resolve("other-key-same-hash",
+                                        md5HashMap(hash),
+                                        sourcePath,
+                                        fileToKey)
     val diffHash = MD5Hash("diff")
-    val keyDiffHash = LocalFile.resolve("other-key-diff-hash", md5HashMap(diffHash), sourcePath, fileToKey)
+    val keyDiffHash = LocalFile.resolve("other-key-diff-hash",
+                                        md5HashMap(diffHash),
+                                        sourcePath,
+                                        fileToKey)
     val lastModified = LastModified(Instant.now)
     val s3ObjectsData: S3ObjectsData = S3ObjectsData(
       byHash = Map(
-        hash -> Set(KeyModified(key, lastModified), KeyModified(keyOtherKey.remoteKey, lastModified)),
+        hash -> Set(KeyModified(key, lastModified),
-        diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))),
+                    KeyModified(keyOtherKey.remoteKey, lastModified)),
+        diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
+      ),
       byKey = Map(
-        key -> HashModified(hash, lastModified),
+        key                   -> HashModified(hash, lastModified),
         keyOtherKey.remoteKey -> HashModified(hash, lastModified),
-        keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)))
+        keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
+      )
+    )
 
     def invoke(localFile: LocalFile) = {
       getS3Status(localFile, s3ObjectsData)
@@ -173,7 +215,10 @@ class S3MetaDataEnricherSuite
     }
 
     describe("when remote key does not exist and no others matches hash") {
-      val localFile = LocalFile.resolve("missing-file", md5HashMap(MD5Hash("unique")), sourcePath, fileToKey)
+      val localFile = LocalFile.resolve("missing-file",
+                                        md5HashMap(MD5Hash("unique")),
+                                        sourcePath,
+                                        fileToKey)
       it("should return no matches by key") {
         val result = getMatchesByKey(invoke(localFile))
         assert(result.isEmpty)
@@ -191,7 +236,9 @@ class S3MetaDataEnricherSuite
       }
       it("should return only itself in match by hash") {
         val result = getMatchesByHash(invoke(keyDiffHash))
-        assert(result.equals(Set((diffHash, KeyModified(keyDiffHash.remoteKey,lastModified)))))
+        assert(
+          result.equals(
+            Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))))
       }
     }
 

core/src/test/scala/net/kemitix/thorp/core/StorageQueueEventSuite.scala

@@ -1,6 +1,10 @@
 package net.kemitix.thorp.core
 
-import net.kemitix.thorp.domain.StorageQueueEvent.{CopyQueueEvent, DeleteQueueEvent, UploadQueueEvent}
+import net.kemitix.thorp.domain.StorageQueueEvent.{
+  CopyQueueEvent,
+  DeleteQueueEvent,
+  UploadQueueEvent
+}
 import net.kemitix.thorp.domain.{MD5Hash, RemoteKey}
 import org.scalatest.FunSpec
 
@@ -8,13 +12,13 @@ class StorageQueueEventSuite extends FunSpec {
 
   describe("Ordering of types") {
     val remoteKey = RemoteKey("remote-key")
-    val md5Hash = MD5Hash("md5hash")
+    val md5Hash   = MD5Hash("md5hash")
-    val copy = CopyQueueEvent(remoteKey)
+    val copy      = CopyQueueEvent(remoteKey)
-    val upload = UploadQueueEvent(remoteKey, md5Hash)
+    val upload    = UploadQueueEvent(remoteKey, md5Hash)
-    val delete = DeleteQueueEvent(remoteKey)
+    val delete    = DeleteQueueEvent(remoteKey)
-    val unsorted = List(delete, copy, upload)
+    val unsorted  = List(delete, copy, upload)
     it("should sort as copy < upload < delete ") {
-      val result = unsorted.sorted
+      val result   = unsorted.sorted
       val expected = List(copy, upload, delete)
       assertResult(expected)(result)
     }

core/src/test/scala/net/kemitix/thorp/core/SyncSuite.scala

@@ -8,72 +8,89 @@ import cats.data.EitherT
 import cats.effect.IO
 import net.kemitix.thorp.core.Action.{ToCopy, ToDelete, ToUpload}
 import net.kemitix.thorp.domain.MD5HashData.{Leaf, Root}
-import net.kemitix.thorp.domain.StorageQueueEvent.{CopyQueueEvent, DeleteQueueEvent, ShutdownQueueEvent, UploadQueueEvent}
+import net.kemitix.thorp.domain.StorageQueueEvent.{
+  CopyQueueEvent,
+  DeleteQueueEvent,
+  ShutdownQueueEvent,
+  UploadQueueEvent
+}
 import net.kemitix.thorp.domain._
 import net.kemitix.thorp.storage.api.{HashService, StorageService}
 import org.scalatest.FunSpec
 
-class SyncSuite
+class SyncSuite extends FunSpec {
-  extends FunSpec {
+  private val testBucket = Bucket("bucket")
-
+  private val source     = Resource(this, "upload")
-  private val source = Resource(this, "upload")
   private val sourcePath = source.toPath
-  private val prefix = RemoteKey("prefix")
+  // source contains the files root-file and subdir/leaf-file
-  private val configOptions =
+  private val rootRemoteKey = RemoteKey("prefix/root-file")
-    ConfigOptions(List(
+  private val leafRemoteKey = RemoteKey("prefix/subdir/leaf-file")
-      ConfigOption.Source(sourcePath),
+  private val rootFile: LocalFile =
-      ConfigOption.Bucket("bucket"),
+    LocalFile.resolve("root-file",
-      ConfigOption.Prefix("prefix"),
+                      md5HashMap(Root.hash),
-      ConfigOption.IgnoreGlobalOptions,
+                      sourcePath,
-      ConfigOption.IgnoreUserOptions
+                      _ => rootRemoteKey)
-    ))
   implicit private val logger: Logger = new DummyLogger
+  private val leafFile: LocalFile =
+    LocalFile.resolve("subdir/leaf-file",
+                      md5HashMap(Leaf.hash),
+                      sourcePath,
+                      _ => leafRemoteKey)
+  private val hashService =
+    DummyHashService(
+      Map(
+        file("root-file")        -> Map("md5" -> MD5HashData.Root.hash),
+        file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash)
+      ))
+  private val configOptions =
+    ConfigOptions(
+      List(
+        ConfigOption.Source(sourcePath),
+        ConfigOption.Bucket("bucket"),
+        ConfigOption.Prefix("prefix"),
+        ConfigOption.IgnoreGlobalOptions,
+        ConfigOption.IgnoreUserOptions
+      ))
   private val lastModified = LastModified(Instant.now)
 
-  def putObjectRequest(bucket: Bucket, remoteKey: RemoteKey, localFile: LocalFile): (String, String, File) =
+  def putObjectRequest(bucket: Bucket,
+                       remoteKey: RemoteKey,
+                       localFile: LocalFile): (String, String, File) =
     (bucket.name, remoteKey.key, localFile.file)
 
-  val testBucket = Bucket("bucket")
+  def invokeSubjectForActions(
-  // source contains the files root-file and subdir/leaf-file
+      storageService: StorageService,
-  val rootRemoteKey = RemoteKey("prefix/root-file")
+      hashService: HashService,
-  val leafRemoteKey = RemoteKey("prefix/subdir/leaf-file")
+      configOptions: ConfigOptions): Either[List[String], Stream[Action]] = {
-  val rootFile: LocalFile =
-    LocalFile.resolve("root-file", md5HashMap(Root.hash), sourcePath, _ => rootRemoteKey)
-  val leafFile: LocalFile =
-    LocalFile.resolve("subdir/leaf-file", md5HashMap(Leaf.hash), sourcePath, _ => leafRemoteKey)
-
-  private def md5HashMap(md5Hash: MD5Hash): Map[String, MD5Hash] =
-    Map("md5" -> md5Hash)
-
-  val hashService =
-    DummyHashService(Map(
-      file("root-file") -> Map("md5" -> MD5HashData.Root.hash),
-      file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash)
-    ))
-
-  private def file(filename: String) =
-    sourcePath.resolve(Paths.get(filename))
-
-  def invokeSubject(storageService: StorageService,
-                    hashService: HashService,
-                    configOptions: ConfigOptions): Either[List[String], SyncPlan] = {
-    PlanBuilder.createPlan(storageService, hashService, configOptions).value.unsafeRunSync
-  }
-
-  def invokeSubjectForActions(storageService: StorageService,
-                              hashService: HashService,
-                              configOptions: ConfigOptions): Either[List[String], Stream[Action]] = {
     invokeSubject(storageService, hashService, configOptions)
       .map(_.actions)
   }
 
+  def invokeSubject(
+      storageService: StorageService,
+      hashService: HashService,
+      configOptions: ConfigOptions): Either[List[String], SyncPlan] = {
+    PlanBuilder
+      .createPlan(storageService, hashService, configOptions)
+      .value
+      .unsafeRunSync
+  }
+
+  private def md5HashMap(md5Hash: MD5Hash): Map[String, MD5Hash] =
+    Map("md5" -> md5Hash)
+
+  private def file(filename: String) =
+    sourcePath.resolve(Paths.get(filename))
+
   describe("when all files should be uploaded") {
-    val storageService = new RecordingStorageService(testBucket, S3ObjectsData())
+    val storageService =
+      new RecordingStorageService(testBucket, S3ObjectsData())
     it("uploads all files") {
-      val expected = Right(Set(
+      val expected = Right(
-        ToUpload(testBucket, rootFile, rootFile.file.length),
+        Set(ToUpload(testBucket, rootFile, rootFile.file.length),
-        ToUpload(testBucket, leafFile, leafFile.file.length)))
+            ToUpload(testBucket, leafFile, leafFile.file.length)))
-      val result = invokeSubjectForActions(storageService, hashService, configOptions)
+      val result =
+        invokeSubjectForActions(storageService, hashService, configOptions)
       assertResult(expected)(result.map(_.toSet))
     }
   }
@@ -81,15 +98,21 @@ class SyncSuite
   describe("when no files should be uploaded") {
     val s3ObjectsData = S3ObjectsData(
       byHash = Map(
-        Root.hash -> Set(KeyModified(RemoteKey("prefix/root-file"), lastModified)),
+        Root.hash -> Set(
-        Leaf.hash -> Set(KeyModified(RemoteKey("prefix/subdir/leaf-file"), lastModified))),
+          KeyModified(RemoteKey("prefix/root-file"), lastModified)),
+        Leaf.hash -> Set(
+          KeyModified(RemoteKey("prefix/subdir/leaf-file"), lastModified))
+      ),
       byKey = Map(
         RemoteKey("prefix/root-file") -> HashModified(Root.hash, lastModified),
-        RemoteKey("prefix/subdir/leaf-file") -> HashModified(Leaf.hash, lastModified)))
+        RemoteKey("prefix/subdir/leaf-file") -> HashModified(Leaf.hash,
+                                                             lastModified))
+    )
     val storageService = new RecordingStorageService(testBucket, s3ObjectsData)
     it("no actions") {
       val expected = Stream()
-      val result = invokeSubjectForActions(storageService, hashService, configOptions)
+      val result =
+        invokeSubjectForActions(storageService, hashService, configOptions)
       assert(result.isRight)
       assertResult(expected)(result.right.get)
     }
@@ -99,19 +122,27 @@ class SyncSuite
     val targetKey = RemoteKey("prefix/root-file")
     // 'root-file-old' should be renamed as 'root-file'
     val s3ObjectsData = S3ObjectsData(
-      byHash = Map(
+      byHash =
-        Root.hash -> Set(KeyModified(sourceKey, lastModified)),
+        Map(Root.hash -> Set(KeyModified(sourceKey, lastModified)),
-        Leaf.hash -> Set(KeyModified(RemoteKey("prefix/subdir/leaf-file"), lastModified))),
+            Leaf.hash -> Set(
-      byKey = Map(
+              KeyModified(RemoteKey("prefix/subdir/leaf-file"), lastModified))),
-        sourceKey -> HashModified(Root.hash, lastModified),
+      byKey =
-        RemoteKey("prefix/subdir/leaf-file") -> HashModified(Leaf.hash, lastModified)))
+        Map(sourceKey -> HashModified(Root.hash, lastModified),
+            RemoteKey("prefix/subdir/leaf-file") -> HashModified(Leaf.hash,
+                                                                 lastModified))
+    )
     val storageService = new RecordingStorageService(testBucket, s3ObjectsData)
     it("copies the file and deletes the original") {
       val expected = Stream(
-        ToCopy(testBucket,  sourceKey, Root.hash, targetKey, rootFile.file.length),
+        ToCopy(testBucket,
+               sourceKey,
+               Root.hash,
+               targetKey,
+               rootFile.file.length),
         ToDelete(testBucket, sourceKey, 0L)
       )
-      val result = invokeSubjectForActions(storageService, hashService, configOptions)
+      val result =
+        invokeSubjectForActions(storageService, hashService, configOptions)
       assert(result.isRight)
       assertResult(expected)(result.right.get)
     }
@@ -123,22 +154,30 @@ class SyncSuite
   }
   describe("when a file is deleted locally it is deleted from S3") {
     val deletedHash = MD5Hash("deleted-hash")
-    val deletedKey = RemoteKey("prefix/deleted-file")
+    val deletedKey  = RemoteKey("prefix/deleted-file")
     val s3ObjectsData = S3ObjectsData(
       byHash = Map(
-        Root.hash -> Set(KeyModified(RemoteKey("prefix/root-file"), lastModified)),
+        Root.hash -> Set(
-        Leaf.hash -> Set(KeyModified(RemoteKey("prefix/subdir/leaf-file"), lastModified)),
+          KeyModified(RemoteKey("prefix/root-file"), lastModified)),
-        deletedHash -> Set(KeyModified(RemoteKey("prefix/deleted-file"), lastModified))),
+        Leaf.hash -> Set(
+          KeyModified(RemoteKey("prefix/subdir/leaf-file"), lastModified)),
+        deletedHash -> Set(
+          KeyModified(RemoteKey("prefix/deleted-file"), lastModified))
+      ),
       byKey = Map(
         RemoteKey("prefix/root-file") -> HashModified(Root.hash, lastModified),
-        RemoteKey("prefix/subdir/leaf-file") -> HashModified(Leaf.hash, lastModified),
+        RemoteKey("prefix/subdir/leaf-file") -> HashModified(Leaf.hash,
-        deletedKey -> HashModified(deletedHash, lastModified)))
+                                                             lastModified),
+        deletedKey -> HashModified(deletedHash, lastModified)
+      )
+    )
     val storageService = new RecordingStorageService(testBucket, s3ObjectsData)
     it("deleted key") {
       val expected = Stream(
         ToDelete(testBucket, deletedKey, 0L)
       )
-      val result = invokeSubjectForActions(storageService,hashService, configOptions)
+      val result =
+        invokeSubjectForActions(storageService, hashService, configOptions)
       assert(result.isRight)
       assertResult(expected)(result.right.get)
     }
@@ -146,15 +185,23 @@ class SyncSuite
   describe("when a file is excluded") {
     val s3ObjectsData = S3ObjectsData(
       byHash = Map(
-        Root.hash -> Set(KeyModified(RemoteKey("prefix/root-file"), lastModified)),
+        Root.hash -> Set(
-        Leaf.hash -> Set(KeyModified(RemoteKey("prefix/subdir/leaf-file"), lastModified))),
+          KeyModified(RemoteKey("prefix/root-file"), lastModified)),
+        Leaf.hash -> Set(
+          KeyModified(RemoteKey("prefix/subdir/leaf-file"), lastModified))
+      ),
       byKey = Map(
         RemoteKey("prefix/root-file") -> HashModified(Root.hash, lastModified),
-        RemoteKey("prefix/subdir/leaf-file") -> HashModified(Leaf.hash, lastModified)))
+        RemoteKey("prefix/subdir/leaf-file") -> HashModified(Leaf.hash,
+                                                             lastModified))
+    )
     val storageService = new RecordingStorageService(testBucket, s3ObjectsData)
     it("is not uploaded") {
       val expected = Stream()
-      val result = invokeSubjectForActions(storageService, hashService, ConfigOption.Exclude("leaf") :: configOptions)
+      val result =
+        invokeSubjectForActions(storageService,
+                                hashService,
+                                ConfigOption.Exclude("leaf") :: configOptions)
       assert(result.isRight)
       assertResult(expected)(result.right.get)
     }
@@ -162,11 +209,10 @@ class SyncSuite
 
   class RecordingStorageService(testBucket: Bucket,
                                 s3ObjectsData: S3ObjectsData)
-    extends StorageService {
+      extends StorageService {
 
-    override def listObjects(bucket: Bucket,
+    override def listObjects(bucket: Bucket, prefix: RemoteKey)(
-                             prefix: RemoteKey)
+        implicit l: Logger): EitherT[IO, String, S3ObjectsData] =
-                            (implicit l: Logger): EitherT[IO, String, S3ObjectsData] =
       EitherT.liftF(IO.pure(s3ObjectsData))
 
     override def upload(localFile: LocalFile,

core/src/test/scala/net/kemitix/thorp/core/TemporaryFolder.scala

@@ -10,22 +10,32 @@ trait TemporaryFolder {
 
   def withDirectory(testCode: Path => Any): Any = {
     val dir: Path = Files.createTempDirectory("thorp-temp")
-    val t = Try(testCode(dir))
+    val t         = Try(testCode(dir))
     remove(dir)
     t.get
   }
 
   def remove(root: Path): Unit = {
-    Files.walkFileTree(root, new SimpleFileVisitor[Path] {
+    Files.walkFileTree(
-      override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
+      root,
-        Files.delete(file)
+      new SimpleFileVisitor[Path] {
-        FileVisitResult.CONTINUE
+        override def visitFile(file: Path,
+                               attrs: BasicFileAttributes): FileVisitResult = {
+          Files.delete(file)
+          FileVisitResult.CONTINUE
+        }
+        override def postVisitDirectory(dir: Path,
+                                        exc: IOException): FileVisitResult = {
+          Files.delete(dir)
+          FileVisitResult.CONTINUE
+        }
       }
-      override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult = {
+    )
-        Files.delete(dir)
+  }
-        FileVisitResult.CONTINUE
+
-      }
+  def createFile(path: Path, name: String, content: String*): File = {
-    })
+    writeFile(path, name, content: _*)
+    path.resolve(name).toFile
   }
 
   def writeFile(directory: Path, name: String, contents: String*): Unit = {
@@ -35,9 +45,4 @@ trait TemporaryFolder {
     pw.close()
   }
 
-  def createFile(path: Path, name: String, content: String*): File = {
+}
-    writeFile(path, name, content:_*)
-    path.resolve(name).toFile
-  }
-
-}

domain/src/main/scala/net/kemitix/thorp/domain/Bucket.scala

@@ -1,3 +1,5 @@
 package net.kemitix.thorp.domain
 
-final case class Bucket(name: String)
+final case class Bucket(
+    name: String
+)

domain/src/main/scala/net/kemitix/thorp/domain/Config.scala

@@ -1,8 +1,10 @@
 package net.kemitix.thorp.domain
 
-final case class Config(bucket: Bucket = Bucket(""),
+final case class Config(
-                        prefix: RemoteKey = RemoteKey(""),
+    bucket: Bucket = Bucket(""),
-                        filters: List[Filter] = List(),
+    prefix: RemoteKey = RemoteKey(""),
-                        debug: Boolean = false,
+    filters: List[Filter] = List(),
-                        batchMode: Boolean = false,
+    debug: Boolean = false,
-                        sources: Sources = Sources(List()))
+    batchMode: Boolean = false,
+    sources: Sources = Sources(List())
+)

domain/src/main/scala/net/kemitix/thorp/domain/Filter.scala

@@ -7,7 +7,32 @@ sealed trait Filter
 
 object Filter {
 
-  case class Include(include: String = ".*") extends Filter {
+  def isIncluded(filters: List[Filter])(p: Path): Boolean = {
+    sealed trait State
+    case class Unknown()   extends State
+    case class Accepted()  extends State
+    case class Discarded() extends State
+    filters.foldRight(Unknown(): State)((filter, state) =>
+      (filter, state) match {
+        case (_, Accepted())                    => Accepted()
+        case (_, Discarded())                   => Discarded()
+        case (x: Exclude, _) if x.isExcluded(p) => Discarded()
+        case (i: Include, _) if i.isIncluded(p) => Accepted()
+        case _                                  => Unknown()
+    }) match {
+      case Accepted()  => true
+      case Discarded() => false
+      case Unknown() =>
+        filters.forall {
+          case _: Include => false
+          case _          => true
+        }
+    }
+  }
+
+  case class Include(
+      include: String = ".*"
+  ) extends Filter {
 
     private lazy val predicate = Pattern.compile(include).asPredicate
 
@@ -15,7 +40,9 @@ object Filter {
 
   }
 
-  case class Exclude(exclude: String) extends Filter {
+  case class Exclude(
+      exclude: String
+  ) extends Filter {
 
     private lazy val predicate = Pattern.compile(exclude).asPredicate()
 
@@ -23,26 +50,4 @@ object Filter {
 
   }
 
-  def isIncluded(filters: List[Filter])(p: Path): Boolean = {
-    sealed trait State
-    case class Unknown() extends State
-    case class Accepted() extends State
-    case class Discarded() extends State
-    filters.foldRight(Unknown(): State)((filter, state) =>
-      (filter, state) match {
-        case (_, Accepted()) => Accepted()
-        case (_, Discarded()) => Discarded()
-        case (x: Exclude, _) if x.isExcluded(p) => Discarded()
-        case (i: Include, _) if i.isIncluded(p) => Accepted()
-        case _ => Unknown()
-      }) match {
-      case Accepted() => true
-      case Discarded() => false
-      case Unknown() => filters.forall {
-        case _: Include => false
-        case _ => true
-      }
-    }
-  }
-
 }

domain/src/main/scala/net/kemitix/thorp/domain/HashModified.scala

@@ -1,4 +1,6 @@
 package net.kemitix.thorp.domain
 
-final case class HashModified(hash: MD5Hash,
+final case class HashModified(
-                              modified: LastModified)
+    hash: MD5Hash,
+    modified: LastModified
+)

domain/src/main/scala/net/kemitix/thorp/domain/KeyModified.scala

@@ -1,4 +1,6 @@
 package net.kemitix.thorp.domain
 
-final case class KeyModified(key: RemoteKey,
+final case class KeyModified(
-                             modified: LastModified)
+    key: RemoteKey,
+    modified: LastModified
+)

domain/src/main/scala/net/kemitix/thorp/domain/LastModified.scala

@@ -2,4 +2,6 @@ package net.kemitix.thorp.domain
 
 import java.time.Instant
 
-final case class LastModified(when: Instant = Instant.now)
+final case class LastModified(
+    when: Instant = Instant.now
+)

domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala

@@ -3,7 +3,12 @@ package net.kemitix.thorp.domain
 import java.io.File
 import java.nio.file.Path
 
-final case class LocalFile(file: File, source: File, hashes: Map[String, MD5Hash], remoteKey: RemoteKey) {
+final case class LocalFile(
+    file: File,
+    source: File,
+    hashes: Map[String, MD5Hash],
+    remoteKey: RemoteKey
+) {
 
   require(!file.isDirectory, s"LocalFile must not be a directory: $file")
 
@@ -19,11 +24,18 @@ final case class LocalFile(file: File, source: File, hashes: Map[String, MD5Hash
 }
 
 object LocalFile {
-  def resolve(path: String,
+
-              md5Hashes: Map[String, MD5Hash],
+  def resolve(
-              source: Path,
+      path: String,
-              pathToKey: Path => RemoteKey): LocalFile = {
+      md5Hashes: Map[String, MD5Hash],
+      source: Path,
+      pathToKey: Path => RemoteKey
+  ): LocalFile = {
     val resolvedPath = source.resolve(path)
-    LocalFile(resolvedPath.toFile, source.toFile, md5Hashes, pathToKey(resolvedPath))
+    LocalFile(resolvedPath.toFile,
+              source.toFile,
+              md5Hashes,
+              pathToKey(resolvedPath))
   }
+
 }

domain/src/main/scala/net/kemitix/thorp/domain/MD5Hash.scala

@@ -4,7 +4,9 @@ import java.util.Base64
 
 import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
 
-final case class MD5Hash(in: String) {
+final case class MD5Hash(
+    in: String
+) {
 
   lazy val hash: String = in filter stripQuotes
 

domain/src/main/scala/net/kemitix/thorp/domain/RemoteKey.scala

@@ -3,25 +3,34 @@ package net.kemitix.thorp.domain
 import java.io.File
 import java.nio.file.{Path, Paths}
 
-final case class RemoteKey(key: String) {
+final case class RemoteKey(
+    key: String
+) {
 
-  def asFile(source: Path, prefix: RemoteKey): Option[File] =
+  def isMissingLocally(
+      sources: Sources,
+      prefix: RemoteKey
+  ): Boolean =
+    !sources.paths.exists(source =>
+      asFile(source, prefix) match {
+        case Some(file) => file.exists
+        case None       => false
+    })
+
+  def asFile(
+      source: Path,
+      prefix: RemoteKey
+  ): Option[File] =
     if (key.length == 0) None
     else Some(source.resolve(relativeTo(prefix)).toFile)
 
   private def relativeTo(prefix: RemoteKey) = {
     prefix match {
       case RemoteKey("") => Paths.get(key)
-      case _ => Paths.get(prefix.key).relativize(Paths.get(key))
+      case _             => Paths.get(prefix.key).relativize(Paths.get(key))
     }
   }
 
-  def isMissingLocally(sources: Sources, prefix: RemoteKey): Boolean =
-    !sources.paths.exists(source => asFile(source, prefix) match {
-      case Some(file) => file.exists
-      case None => false
-    })
-
   def resolve(path: String): RemoteKey =
     RemoteKey(List(key, path).filterNot(_.isEmpty).mkString("/"))
 

domain/src/main/scala/net/kemitix/thorp/domain/RemoteMetaData.scala

@@ -1,5 +1,7 @@
 package net.kemitix.thorp.domain
 
-final case class RemoteMetaData(remoteKey: RemoteKey,
+final case class RemoteMetaData(
-                                hash: MD5Hash,
+    remoteKey: RemoteKey,
-                                lastModified: LastModified)
+    hash: MD5Hash,
+    lastModified: LastModified
+)

domain/src/main/scala/net/kemitix/thorp/domain/S3MetaData.scala

@@ -2,6 +2,7 @@ package net.kemitix.thorp.domain
 
 // For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
 final case class S3MetaData(
-  localFile: LocalFile,
+    localFile: LocalFile,
-  matchByHash: Set[RemoteMetaData],
+    matchByHash: Set[RemoteMetaData],
-  matchByKey: Option[RemoteMetaData])
+    matchByKey: Option[RemoteMetaData]
+)

domain/src/main/scala/net/kemitix/thorp/domain/S3ObjectsData.scala

@@ -3,5 +3,7 @@ package net.kemitix.thorp.domain
 /**
   * A list of objects and their MD5 hash values.
   */
-final case class S3ObjectsData(byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty,
+final case class S3ObjectsData(
-                               byKey: Map[RemoteKey, HashModified] = Map.empty)
+    byHash: Map[MD5Hash, Set[KeyModified]] = Map.empty,
+    byKey: Map[RemoteKey, HashModified] = Map.empty
+)

domain/src/main/scala/net/kemitix/thorp/domain/SizeTranslation.scala

@@ -8,11 +8,10 @@ object SizeTranslation {
 
   def sizeInEnglish(length: Long): String =
     length.toDouble match {
-      case bytes if bytes > gbLimit => f"${bytes / 1024 / 1024 /1024}%.3fGb"
+      case bytes if bytes > gbLimit => f"${bytes / 1024 / 1024 / 1024}%.3fGb"
       case bytes if bytes > mbLimit => f"${bytes / 1024 / 1024}%.2fMb"
       case bytes if bytes > kbLimit => f"${bytes / 1024}%.0fKb"
-      case bytes => s"${length}b"
+      case bytes                    => s"${length}b"
     }
 
-
 }

domain/src/main/scala/net/kemitix/thorp/domain/Sources.scala

@@ -12,11 +12,15 @@ import java.nio.file.Path
   *
   * A path should only occur once in paths.
   */
-case class Sources(paths: List[Path]) {
+case class Sources(
-  def ++(path: Path): Sources = this ++ List(path)
+    paths: List[Path]
+) {
+  def ++(path: Path): Sources             = this ++ List(path)
   def ++(otherPaths: List[Path]): Sources = Sources(
-    otherPaths.foldLeft(paths)((acc, path) => if (acc.contains(path)) acc else acc ++ List(path))
+    otherPaths.foldLeft(paths) { (acc, path) =>
-  )
+      if (acc.contains(path)) acc
+      else acc ++ List(path)
+    })
 
   /**
     * Returns the source path for the given path.

domain/src/main/scala/net/kemitix/thorp/domain/StorageQueueEvent.scala

@@ -8,24 +8,35 @@ sealed trait StorageQueueEvent {
 
 object StorageQueueEvent {
 
-  final case class DoNothingQueueEvent(remoteKey: RemoteKey) extends StorageQueueEvent {
+  final case class DoNothingQueueEvent(
+      remoteKey: RemoteKey
+  ) extends StorageQueueEvent {
     override val order: Int = 0
   }
 
-  final case class CopyQueueEvent(remoteKey: RemoteKey) extends StorageQueueEvent {
+  final case class CopyQueueEvent(
+      remoteKey: RemoteKey
+  ) extends StorageQueueEvent {
     override val order: Int = 1
   }
 
-  final case class UploadQueueEvent(remoteKey: RemoteKey,
+  final case class UploadQueueEvent(
-                                    md5Hash: MD5Hash) extends StorageQueueEvent {
+      remoteKey: RemoteKey,
+      md5Hash: MD5Hash
+  ) extends StorageQueueEvent {
     override val order: Int = 2
   }
 
-  final case class DeleteQueueEvent(remoteKey: RemoteKey) extends StorageQueueEvent {
+  final case class DeleteQueueEvent(
+      remoteKey: RemoteKey
+  ) extends StorageQueueEvent {
     override val order: Int = 3
   }
 
-  final case class ErrorQueueEvent(remoteKey: RemoteKey, e: Throwable) extends StorageQueueEvent {
+  final case class ErrorQueueEvent(
+      remoteKey: RemoteKey,
+      e: Throwable
+  ) extends StorageQueueEvent {
     override val order: Int = 10
   }
 

domain/src/main/scala/net/kemitix/thorp/domain/SyncTotals.scala

@@ -1,5 +1,7 @@
 package net.kemitix.thorp.domain
 
-case class SyncTotals(count: Long = 0L,
+case class SyncTotals(
-                      totalSizeBytes: Long = 0L,
+    count: Long = 0L,
-                      sizeUploadedBytes: Long = 0L)
+    totalSizeBytes: Long = 0L,
+    sizeUploadedBytes: Long = 0L
+)

domain/src/main/scala/net/kemitix/thorp/domain/Terminal.scala

@@ -2,55 +2,8 @@ package net.kemitix.thorp.domain
 
 object Terminal {
 
-  private val esc = "\u001B"
+  val esc: String = "\u001B"
-  private val csi = esc + "["
+  val csi: String = esc + "["
-
-  /**
-    * Move the cursor up, default 1 line.
-    *
-    * Stops at the edge of the screen.
-    */
-  def cursorUp(lines: Int = 1): String = csi + lines + "A"
-
-  /**
-    * Move the cursor down, default 1 line.
-    *
-    * Stops at the edge of the screen.
-    */
-  def cursorDown(lines: Int = 1): String = csi + lines + "B"
-
-  /**
-    * Move the cursor forward, default 1 column.
-    *
-    * Stops at the edge of the screen.
-    */
-  def cursorForward(cols: Int = 1): String = csi + cols + "C"
-
-  /**
-    * Move the cursor back, default 1 column,
-    *
-    * Stops at the edge of the screen.
-    */
-  def cursorBack(cols: Int = 1): String = csi + cols + "D"
-
-  /**
-    * Move the cursor to the beginning of the line, default 1, down.
-    */
-  def cursorNextLine(lines: Int = 1): String = csi + lines + "E"
-  /**
-    * Move the cursor to the beginning of the line, default 1, up.
-    */
-  def cursorPrevLine(lines: Int = 1): String = csi + lines + "F"
-
-  /**
-    * Move the cursor to the column on the current line.
-    */
-  def cursorHorizAbs(col: Int): String = csi + col + "G"
-
-  /**
-    * Move the cursor to the position on screen (1,1 is the top-left).
-    */
-  def cursorPosition(row: Int, col: Int): String = csi + row + ";" + col + "H"
 
   /**
     * Clear from cursor to end of screen.
@@ -97,6 +50,74 @@ object Terminal {
     */
   val eraseLine: String = csi + "2K"
 
+  /**
+    * Saves the cursor position/state.
+    */
+  val saveCursorPosition: String = csi + "s"
+
+  /**
+    * Restores the cursor position/state.
+    */
+  val restoreCursorPosition: String  = csi + "u"
+  val enableAlternateBuffer: String  = csi + "?1049h"
+  val disableAlternateBuffer: String = csi + "?1049l"
+  private val subBars = Map(0 -> " ",
+                            1 -> "▏",
+                            2 -> "▎",
+                            3 -> "▍",
+                            4 -> "▌",
+                            5 -> "▋",
+                            6 -> "▊",
+                            7 -> "▉")
+
+  /**
+    * Move the cursor up, default 1 line.
+    *
+    * Stops at the edge of the screen.
+    */
+  def cursorUp(lines: Int = 1): String = csi + lines + "A"
+
+  /**
+    * Move the cursor down, default 1 line.
+    *
+    * Stops at the edge of the screen.
+    */
+  def cursorDown(lines: Int = 1): String = csi + lines + "B"
+
+  /**
+    * Move the cursor forward, default 1 column.
+    *
+    * Stops at the edge of the screen.
+    */
+  def cursorForward(cols: Int = 1): String = csi + cols + "C"
+
+  /**
+    * Move the cursor back, default 1 column,
+    *
+    * Stops at the edge of the screen.
+    */
+  def cursorBack(cols: Int = 1): String = csi + cols + "D"
+
+  /**
+    * Move the cursor to the beginning of the line, default 1, down.
+    */
+  def cursorNextLine(lines: Int = 1): String = csi + lines + "E"
+
+  /**
+    * Move the cursor to the beginning of the line, default 1, up.
+    */
+  def cursorPrevLine(lines: Int = 1): String = csi + lines + "F"
+
+  /**
+    * Move the cursor to the column on the current line.
+    */
+  def cursorHorizAbs(col: Int): String = csi + col + "G"
+
+  /**
+    * Move the cursor to the position on screen (1,1 is the top-left).
+    */
+  def cursorPosition(row: Int, col: Int): String = csi + row + ";" + col + "H"
+
   /**
     * Scroll page up, default 1, lines.
     */
@@ -107,20 +128,6 @@ object Terminal {
     */
   def scrollDown(lines: Int = 1): String = csi + lines + "T"
 
-  /**
-    * Saves the cursor position/state.
-    */
-  val saveCursorPosition: String = csi + "s"
-
-  /**
-    * Restores the cursor position/state.
-    */
-  val restoreCursorPosition: String = csi + "u"
-
-  val enableAlternateBuffer: String = csi + "?1049h"
-
-  val disableAlternateBuffer: String = csi + "?1049l"
-
   /**
     * The Width of the terminal, as reported by the COLUMNS environment variable.
     *
@@ -135,28 +142,22 @@ object Terminal {
       .getOrElse(80)
   }
 
-  private val subBars = Map(
+  def progressBar(
-    0 -> " ",
+      pos: Double,
-    1 -> "▏",
+      max: Double,
-    2 -> "▎",
+      width: Int
-    3 -> "▍",
+  ): String = {
-    4 -> "▌",
+    val barWidth          = width - 2
-    5 -> "▋",
+    val phases            = subBars.values.size
-    6 -> "▊",
+    val pxWidth           = barWidth * phases
-    7 -> "▉")
+    val ratio             = pos / max
-
+    val pxDone            = pxWidth * ratio
-  def progressBar(pos: Double, max: Double, width: Int): String = {
-    val barWidth = width - 2
-    val phases = subBars.values.size
-    val pxWidth = barWidth * phases
-    val ratio = pos / max
-    val pxDone = pxWidth * ratio
     val fullHeadSize: Int = (pxDone / phases).toInt
-    val part = (pxDone % phases).toInt
+    val part              = (pxDone % phases).toInt
-    val partial = if (part != 0) subBars.getOrElse(part, "") else ""
+    val partial           = if (part != 0) subBars.getOrElse(part, "") else ""
-    val head = ("█" * fullHeadSize) + partial
+    val head              = ("█" * fullHeadSize) + partial
-    val tailSize = barWidth - head.length
+    val tailSize          = barWidth - head.length
-    val tail = " " * tailSize
+    val tail              = " " * tailSize
     s"[$head$tail]"
   }
 

domain/src/main/scala/net/kemitix/thorp/domain/UploadEvent.scala

@@ -6,12 +6,18 @@ sealed trait UploadEvent {
 
 object UploadEvent {
 
-  final case class TransferEvent(name: String) extends UploadEvent
+  final case class TransferEvent(
+      name: String
+  ) extends UploadEvent
 
-  final case class RequestEvent(name: String,
+  final case class RequestEvent(
-                                bytes: Long,
+      name: String,
-                                transferred: Long) extends UploadEvent
+      bytes: Long,
+      transferred: Long
+  ) extends UploadEvent
 
-  final case class ByteTransferEvent(name: String) extends UploadEvent
+  final case class ByteTransferEvent(
+      name: String
+  ) extends UploadEvent
 
 }

domain/src/main/scala/net/kemitix/thorp/domain/UploadEventListener.scala

@@ -3,17 +3,24 @@ package net.kemitix.thorp.domain
 import net.kemitix.thorp.domain.UploadEvent.RequestEvent
 import net.kemitix.thorp.domain.UploadEventLogger.logRequestCycle
 
-class UploadEventListener(localFile: LocalFile,
+class UploadEventListener(
-                          index: Int,
+    localFile: LocalFile,
-                          syncTotals: SyncTotals,
+    index: Int,
-                          totalBytesSoFar: Long) {
+    syncTotals: SyncTotals,
+    totalBytesSoFar: Long
+) {
 
   var bytesTransferred = 0L
 
   def listener: UploadEvent => Unit = {
-      case e: RequestEvent =>
+    case e: RequestEvent =>
-        bytesTransferred += e.transferred
+      bytesTransferred += e.transferred
-        logRequestCycle(localFile, e, bytesTransferred, index, syncTotals, totalBytesSoFar)
+      logRequestCycle(localFile,
+                      e,
+                      bytesTransferred,
+                      index,
+                      syncTotals,
+                      totalBytesSoFar)
     case _ => ()
-    }
+  }
 }

domain/src/main/scala/net/kemitix/thorp/domain/UploadEventLogger.scala

@@ -8,35 +8,42 @@ import scala.io.AnsiColor._
 
 trait UploadEventLogger {
 
-  def logRequestCycle(localFile: LocalFile,
+  def logRequestCycle(
-                      event: RequestEvent,
+      localFile: LocalFile,
-                      bytesTransferred: Long,
+      event: RequestEvent,
-                      index: Int,
+      bytesTransferred: Long,
-                      syncTotals: SyncTotals,
+      index: Int,
-                      totalBytesSoFar: Long): Unit = {
+      syncTotals: SyncTotals,
-    val remoteKey = localFile.remoteKey.key
+      totalBytesSoFar: Long
-    val fileLength = localFile.file.length
+  ): Unit = {
+    val remoteKey    = localFile.remoteKey.key
+    val fileLength   = localFile.file.length
     val statusHeight = 7
     if (bytesTransferred < fileLength) {
       println(
         s"${GREEN}Uploading:$RESET $remoteKey$eraseToEndOfScreen\n" +
           statusWithBar(" File", sizeInEnglish, bytesTransferred, fileLength) +
           statusWithBar("Files", l => l.toString, index, syncTotals.count) +
-          statusWithBar(" Size", sizeInEnglish, bytesTransferred + totalBytesSoFar, syncTotals.totalSizeBytes) +
+          statusWithBar(" Size",
+                        sizeInEnglish,
+                        bytesTransferred + totalBytesSoFar,
+                        syncTotals.totalSizeBytes) +
           s"${Terminal.cursorPrevLine(statusHeight)}")
     } else
       println(s"${GREEN}Uploaded:$RESET $remoteKey$eraseToEndOfScreen")
   }
 
-  private def statusWithBar(label: String,
+  private def statusWithBar(
-                            format: Long => String,
+      label: String,
-                            current: Long,
+      format: Long => String,
-                            max: Long,
+      current: Long,
-                            pre: Long = 0): String = {
+      max: Long,
+      pre: Long = 0
+  ): String = {
     val percent = f"${(current * 100) / max}%2d"
     s"$GREEN$label:$RESET ($percent%) ${format(current)} of ${format(max)}" +
       (if (pre > 0) s" (pre-synced ${format(pre)}"
-      else "") + s"$eraseLineForward\n" +
+       else "") + s"$eraseLineForward\n" +
       progressBar(current, max, Terminal.width)
   }
 }

domain/src/test/scala/net/kemitix/thorp/domain/FiltersSuite.scala

@@ -13,13 +13,14 @@ class FiltersSuite extends FunSpec {
   private val path4 = "/path/to/another/file"
   private val path5 = "/home/pcampbell/repos/kemitix/s3thorp"
   private val path6 = "/kemitix/s3thorp/upload/subdir"
-  private val paths = List(path1, path2, path3, path4, path5, path6).map(Paths.get(_))
+  private val paths =
+    List(path1, path2, path3, path4, path5, path6).map(Paths.get(_))
 
   describe("Include") {
 
     describe("default filter") {
       val include = Include()
-       it("should include files") {
+      it("should include files") {
         paths.foreach(path => assertResult(true)(include.isIncluded(path)))
       }
     }
@@ -92,7 +93,7 @@ class FiltersSuite extends FunSpec {
       val filters = List[Filter]()
       it("should accept all files") {
         val expected = paths
-        val result = invoke(filters)
+        val result   = invoke(filters)
         assertResult(expected)(result)
       }
     }
@@ -100,7 +101,7 @@ class FiltersSuite extends FunSpec {
       val filters = List(Include(".txt"))
       it("should only include two matching paths") {
         val expected = List(path2, path3).map(Paths.get(_))
-        val result = invoke(filters)
+        val result   = invoke(filters)
         assertResult(expected)(result)
       }
     }
@@ -108,7 +109,7 @@ class FiltersSuite extends FunSpec {
       val filters = List(Exclude("path"))
       it("should include only other paths") {
         val expected = List(path1, path2, path5, path6).map(Paths.get(_))
-        val result = invoke(filters)
+        val result   = invoke(filters)
         assertResult(expected)(result)
       }
     }
@@ -116,7 +117,7 @@ class FiltersSuite extends FunSpec {
       val filters = List(Include(".txt"), Exclude(".*"))
       it("should include nothing") {
         val expected = List()
-        val result = invoke(filters)
+        val result   = invoke(filters)
         assertResult(expected)(result)
       }
     }
@@ -124,7 +125,7 @@ class FiltersSuite extends FunSpec {
       val filters = List(Exclude(".*"), Include(".txt"))
       it("should include only the .txt files") {
         val expected = List(path2, path3).map(Paths.get(_))
-        val result = invoke(filters)
+        val result   = invoke(filters)
         assertResult(expected)(result)
       }
     }

domain/src/test/scala/net/kemitix/thorp/domain/MD5HashData.scala

@@ -3,25 +3,24 @@ package net.kemitix.thorp.domain
 object MD5HashData {
 
   object Root {
-    val hash = MD5Hash("a3a6ac11a0eb577b81b3bb5c95cc8a6e")
+    val hash   = MD5Hash("a3a6ac11a0eb577b81b3bb5c95cc8a6e")
     val base64 = "o6asEaDrV3uBs7tclcyKbg=="
   }
-
   object Leaf {
-    val hash = MD5Hash("208386a650bdec61cfcd7bd8dcb6b542")
+    val hash   = MD5Hash("208386a650bdec61cfcd7bd8dcb6b542")
     val base64 = "IIOGplC97GHPzXvY3La1Qg=="
   }
   object BigFile {
     val hash = MD5Hash("b1ab1f7680138e6db7309200584e35d8")
     object Part1 {
       val offset = 0
-      val size = 1048576
+      val size   = 1048576
-      val hash = MD5Hash("39d4a9c78b9cfddf6d241a201a4ab726")
+      val hash   = MD5Hash("39d4a9c78b9cfddf6d241a201a4ab726")
     }
     object Part2 {
       val offset = 1048576
-      val size = 1048576
+      val size   = 1048576
-      val hash = MD5Hash("af5876f3a3bc6e66f4ae96bb93d8dae0")
+      val hash   = MD5Hash("af5876f3a3bc6e66f4ae96bb93d8dae0")
     }
   }
 

domain/src/test/scala/net/kemitix/thorp/domain/RemoteKeyTest.scala

@@ -12,58 +12,58 @@ class RemoteKeyTest extends FreeSpec {
   "create a RemoteKey" - {
     "can resolve a path" - {
       "when key is empty" in {
-        val key = emptyKey
+        val key      = emptyKey
-        val path = "path"
+        val path     = "path"
         val expected = RemoteKey("path")
-        val result = key.resolve(path)
+        val result   = key.resolve(path)
         assertResult(expected)(result)
       }
-      "when path is empty" in  {
+      "when path is empty" in {
-        val key = RemoteKey("key")
+        val key      = RemoteKey("key")
-        val path = ""
+        val path     = ""
         val expected = RemoteKey("key")
-        val result = key.resolve(path)
+        val result   = key.resolve(path)
         assertResult(expected)(result)
       }
       "when key and path are empty" in {
-        val key = emptyKey
+        val key      = emptyKey
-        val path = ""
+        val path     = ""
         val expected = emptyKey
-        val result = key.resolve(path)
+        val result   = key.resolve(path)
         assertResult(expected)(result)
       }
     }
     "asFile" - {
       "when key and prefix are non-empty" in {
-        val key = RemoteKey("prefix/key")
+        val key      = RemoteKey("prefix/key")
-        val source = Paths.get("source")
+        val source   = Paths.get("source")
-        val prefix = RemoteKey("prefix")
+        val prefix   = RemoteKey("prefix")
         val expected = Some(new File("source/key"))
-        val result = key.asFile(source, prefix)
+        val result   = key.asFile(source, prefix)
         assertResult(expected)(result)
       }
       "when prefix is empty" in {
-        val key = RemoteKey("key")
+        val key      = RemoteKey("key")
-        val source = Paths.get("source")
+        val source   = Paths.get("source")
-        val prefix = emptyKey
+        val prefix   = emptyKey
         val expected = Some(new File("source/key"))
-        val result = key.asFile(source, prefix)
+        val result   = key.asFile(source, prefix)
         assertResult(expected)(result)
       }
       "when key is empty" in {
-        val key = emptyKey
+        val key      = emptyKey
-        val source = Paths.get("source")
+        val source   = Paths.get("source")
-        val prefix = RemoteKey("prefix")
+        val prefix   = RemoteKey("prefix")
         val expected = None
-        val result = key.asFile(source, prefix)
+        val result   = key.asFile(source, prefix)
         assertResult(expected)(result)
       }
       "when key and prefix are empty" in {
-        val key = emptyKey
+        val key      = emptyKey
-        val source = Paths.get("source")
+        val source   = Paths.get("source")
-        val prefix = emptyKey
+        val prefix   = emptyKey
         val expected = None
-        val result = key.asFile(source, prefix)
+        val result   = key.asFile(source, prefix)
         assertResult(expected)(result)
       }
     }

domain/src/test/scala/net/kemitix/thorp/domain/SizeTranslationTest.scala

@@ -27,7 +27,8 @@ class SizeTranslationTest extends FunSpec {
     }
     describe("when size is over 10Gb") {
       it("should be in Gb with three decimal place") {
-        assertResult("5468.168Gb")(SizeTranslation.sizeInEnglish(5871400857278L))
+        assertResult("5468.168Gb")(
+          SizeTranslation.sizeInEnglish(5871400857278L))
       }
     }
   }

storage-api/src/main/scala/net/kemitix/thorp/storage/api/HashService.scala

@@ -10,7 +10,6 @@ import net.kemitix.thorp.domain.{Logger, MD5Hash}
   */
 trait HashService {
 
-  def hashLocalObject(path: Path)
+  def hashLocalObject(path: Path)(implicit l: Logger): IO[Map[String, MD5Hash]]
-                     (implicit l: Logger): IO[Map[String, MD5Hash]]
 
 }

storage-api/src/main/scala/net/kemitix/thorp/storage/api/StorageService.scala

@@ -8,22 +8,29 @@ trait StorageService {
 
   def shutdown: IO[StorageQueueEvent]
 
-  def listObjects(bucket: Bucket,
+  def listObjects(
-                  prefix: RemoteKey)
+      bucket: Bucket,
-                 (implicit l: Logger): EitherT[IO, String, S3ObjectsData]
+      prefix: RemoteKey
+  )(implicit l: Logger): EitherT[IO, String, S3ObjectsData]
 
-  def upload(localFile: LocalFile,
+  def upload(
-             bucket: Bucket,
+      localFile: LocalFile,
-             batchMode: Boolean,
+      bucket: Bucket,
-             uploadEventListener: UploadEventListener,
+      batchMode: Boolean,
-             tryCount: Int): IO[StorageQueueEvent]
+      uploadEventListener: UploadEventListener,
+      tryCount: Int
+  ): IO[StorageQueueEvent]
 
-  def copy(bucket: Bucket,
+  def copy(
-           sourceKey: RemoteKey,
+      bucket: Bucket,
-           hash: MD5Hash,
+      sourceKey: RemoteKey,
-           targetKey: RemoteKey): IO[StorageQueueEvent]
+      hash: MD5Hash,
+      targetKey: RemoteKey
+  ): IO[StorageQueueEvent]
 
-  def delete(bucket: Bucket,
+  def delete(
-             remoteKey: RemoteKey): IO[StorageQueueEvent]
+      bucket: Bucket,
+      remoteKey: RemoteKey
+  ): IO[StorageQueueEvent]
 
 }

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/AmazonTransferManager.scala

@@ -0,0 +1,11 @@
+package net.kemitix.thorp.storage.aws
+
+import com.amazonaws.services.s3.model.PutObjectRequest
+import com.amazonaws.services.s3.transfer.TransferManager
+
+case class AmazonTransferManager(transferManager: TransferManager) {
+  def shutdownNow(now: Boolean): Unit = transferManager.shutdownNow(now)
+
+  def upload(putObjectRequest: PutObjectRequest): AmazonUpload =
+    AmazonUpload(transferManager.upload(putObjectRequest))
+}

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/AmazonUpload.scala

@@ -0,0 +1,8 @@
+package net.kemitix.thorp.storage.aws
+
+import com.amazonaws.services.s3.transfer.Upload
+import com.amazonaws.services.s3.transfer.model.UploadResult
+
+case class AmazonUpload(upload: Upload) {
+  def waitForUploadResult: UploadResult = upload.waitForUploadResult()
+}

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Copier.scala

@@ -8,21 +8,29 @@ import net.kemitix.thorp.domain._
 
 class Copier(amazonS3: AmazonS3) {
 
-  def copy(bucket: Bucket,
+  def copy(
-           sourceKey: RemoteKey,
+      bucket: Bucket,
-           hash: MD5Hash,
+      sourceKey: RemoteKey,
-           targetKey: RemoteKey): IO[StorageQueueEvent] =
+      hash: MD5Hash,
+      targetKey: RemoteKey
+  ): IO[StorageQueueEvent] =
     for {
       _ <- copyObject(bucket, sourceKey, hash, targetKey)
     } yield CopyQueueEvent(targetKey)
 
-  private def copyObject(bucket: Bucket,
+  private def copyObject(
-                         sourceKey: RemoteKey,
+      bucket: Bucket,
-                         hash: MD5Hash,
+      sourceKey: RemoteKey,
-                         targetKey: RemoteKey) = {
+      hash: MD5Hash,
+      targetKey: RemoteKey
+  ) = {
     val request =
-      new CopyObjectRequest(bucket.name, sourceKey.key, bucket.name, targetKey.key)
+      new CopyObjectRequest(
-        .withMatchingETagConstraint(hash.hash)
+        bucket.name,
+        sourceKey.key,
+        bucket.name,
+        targetKey.key
+      ).withMatchingETagConstraint(hash.hash)
     IO(amazonS3.copyObject(request))
   }
 

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Deleter.scala

@@ -4,17 +4,24 @@ import cats.effect.IO
 import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.s3.model.DeleteObjectRequest
 import net.kemitix.thorp.domain.StorageQueueEvent.DeleteQueueEvent
-import net.kemitix.thorp.domain.{Bucket, RemoteKey}
+import net.kemitix.thorp.domain.{Bucket, RemoteKey, StorageQueueEvent}
 
 class Deleter(amazonS3: AmazonS3) {
 
-  def delete(bucket: Bucket,
+  def delete(
-             remoteKey: RemoteKey): IO[DeleteQueueEvent] =
+      bucket: Bucket,
+      remoteKey: RemoteKey
+  ): IO[StorageQueueEvent] =
     for {
       _ <- deleteObject(bucket, remoteKey)
     } yield DeleteQueueEvent(remoteKey)
 
-  private def deleteObject(bucket: Bucket, remoteKey: RemoteKey) =
+  private def deleteObject(
-    IO(amazonS3.deleteObject(new DeleteObjectRequest(bucket.name, remoteKey.key)))
+      bucket: Bucket,
+      remoteKey: RemoteKey
+  ) = {
+    val request = new DeleteObjectRequest(bucket.name, remoteKey.key)
+    IO(amazonS3.deleteObject(request))
+  }
 
 }

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/ETagGenerator.scala

@@ -12,11 +12,14 @@ import net.kemitix.thorp.domain.{Logger, MD5Hash}
 
 trait ETagGenerator {
 
-  def eTag(path: Path)(implicit l: Logger): IO[String]= {
+  def eTag(
+      path: Path
+  )(implicit l: Logger): IO[String] = {
     val partSize = calculatePartSize(path)
-    val parts = numParts(path.toFile.length, partSize)
+    val parts    = numParts(path.toFile.length, partSize)
     partsIndex(parts)
-      .map(digestChunk(path, partSize)).sequence
+      .map(digestChunk(path, partSize))
+      .sequence
       .map(concatenateDigests)
       .map(MD5HashGenerator.hex)
       .map(hash => s"$hash-$parts")
@@ -29,25 +32,42 @@ trait ETagGenerator {
     lab => lab.foldLeft(Array[Byte]())((acc, ab) => acc ++ ab)
 
   private def calculatePartSize(path: Path) = {
-    val request = new PutObjectRequest("", "", path.toFile)
+    val request       = new PutObjectRequest("", "", path.toFile)
     val configuration = new TransferManagerConfiguration
     TransferManagerUtils.calculateOptimalPartSize(request, configuration)
   }
 
-  private def numParts(fileLength: Long, optimumPartSize: Long) = {
+  private def numParts(
+      fileLength: Long,
+      optimumPartSize: Long
+  ) = {
     val fullParts = Math.floorDiv(fileLength, optimumPartSize)
-    val incompletePart = if (Math.floorMod(fileLength, optimumPartSize) > 0) 1 else 0
+    val incompletePart =
+      if (Math.floorMod(fileLength, optimumPartSize) > 0) 1
+      else 0
     fullParts + incompletePart
   }
 
-  def offsets(totalFileSizeBytes: Long, optimalPartSize: Long): List[Long] =
+  def digestChunk(
-    Range.Long(0, totalFileSizeBytes, optimalPartSize).toList
+      path: Path,
-
+      chunkSize: Long
-  def digestChunk(path: Path, chunkSize: Long)(chunkNumber: Long)(implicit l: Logger): IO[Array[Byte]] =
+  )(
+      chunkNumber: Long
+  )(implicit l: Logger): IO[Array[Byte]] =
     hashChunk(path, chunkNumber, chunkSize).map(_.digest)
 
-  def hashChunk(path: Path, chunkNumber: Long, chunkSize: Long)(implicit l: Logger): IO[MD5Hash] =
+  def hashChunk(
+      path: Path,
+      chunkNumber: Long,
+      chunkSize: Long
+  )(implicit l: Logger): IO[MD5Hash] =
     MD5HashGenerator.md5FileChunk(path, chunkNumber * chunkSize, chunkSize)
+
+  def offsets(
+      totalFileSizeBytes: Long,
+      optimalPartSize: Long
+  ): List[Long] =
+    Range.Long(0, totalFileSizeBytes, optimalPartSize).toList
 }
 
 object ETagGenerator extends ETagGenerator

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Lister.scala

@@ -2,6 +2,7 @@ package net.kemitix.thorp.storage.aws
 
 import cats.data.EitherT
 import cats.effect.IO
+import cats.implicits._
 import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.s3.model.{ListObjectsV2Request, S3ObjectSummary}
 import net.kemitix.thorp.domain
@@ -17,31 +18,37 @@ class Lister(amazonS3: AmazonS3) {
   private type Token = String
   private type Batch = (Stream[S3ObjectSummary], Option[Token])
 
-  def listObjects(bucket: Bucket,
+  def listObjects(
-                  prefix: RemoteKey)
+      bucket: Bucket,
-                 (implicit l: Logger): EitherT[IO, String, S3ObjectsData] = {
+      prefix: RemoteKey
+  )(implicit l: Logger): EitherT[IO, String, S3ObjectsData] = {
 
-    val requestMore = (token: Token) => new ListObjectsV2Request()
+    val requestMore = (token: Token) =>
-      .withBucketName(bucket.name)
+      new ListObjectsV2Request()
-      .withPrefix(prefix.key)
+        .withBucketName(bucket.name)
-      .withContinuationToken(token)
+        .withPrefix(prefix.key)
+        .withContinuationToken(token)
 
     def fetchBatch: ListObjectsV2Request => EitherT[IO, String, Batch] =
-      request => EitherT {
+      request =>
-        for {
+        EitherT {
-          _ <- ListerLogger.logFetchBatch
+          for {
-          batch <- tryFetchBatch(request)
+            _     <- ListerLogger.logFetchBatch
-        } yield batch
+            batch <- tryFetchBatch(request)
+          } yield batch
       }
 
-    def fetchMore(more: Option[Token]): EitherT[IO, String, Stream[S3ObjectSummary]] = {
+    def fetchMore(
+        more: Option[Token]
+    ): EitherT[IO, String, Stream[S3ObjectSummary]] = {
       more match {
-        case None => EitherT.right(IO.pure(Stream.empty))
+        case None        => EitherT.right(IO.pure(Stream.empty))
         case Some(token) => fetch(requestMore(token))
       }
     }
 
-    def fetch: ListObjectsV2Request => EitherT[IO, String, Stream[S3ObjectSummary]] =
+    def fetch
+      : ListObjectsV2Request => EitherT[IO, String, Stream[S3ObjectSummary]] =
       request => {
         for {
           batch <- fetchBatch(request)
@@ -51,11 +58,16 @@ class Lister(amazonS3: AmazonS3) {
       }
 
     for {
-      summaries <- fetch(new ListObjectsV2Request().withBucketName(bucket.name).withPrefix(prefix.key))
+      summaries <- fetch(
+        new ListObjectsV2Request()
+          .withBucketName(bucket.name)
+          .withPrefix(prefix.key))
     } yield domain.S3ObjectsData(byHash(summaries), byKey(summaries))
   }
 
-  private def tryFetchBatch(request: ListObjectsV2Request): IO[Either[String, (Stream[S3ObjectSummary], Option[Token])]] = {
+  private def tryFetchBatch(
+      request: ListObjectsV2Request
+  ): IO[Either[String, (Stream[S3ObjectSummary], Option[Token])]] = {
     IO {
       Try(amazonS3.listObjectsV2(request))
         .map { result =>
@@ -63,7 +75,9 @@ class Lister(amazonS3: AmazonS3) {
             if (result.isTruncated) Some(result.getNextContinuationToken)
             else None
           (result.getObjectSummaries.asScala.toStream, more)
-        }.toEither.swap.map(e => e.getMessage).swap
+        }
+        .toEither
+        .leftMap(e => e.getMessage)
     }
   }
 }

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/ListerLogger.scala

@@ -4,6 +4,7 @@ import cats.effect.IO
 import net.kemitix.thorp.domain.Logger
 
 trait ListerLogger {
-  def logFetchBatch(implicit l: Logger): IO[Unit] = l.info("Fetching remote summaries...")
+  def logFetchBatch(implicit l: Logger): IO[Unit] =
+    l.info("Fetching remote summaries...")
 }
 object ListerLogger extends ListerLogger

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3HashService.scala

@@ -15,15 +15,17 @@ trait S3HashService extends HashService {
     * @param path the local path to scan
     * @return a set of hash values
     */
-  override def hashLocalObject(path: Path)
+  override def hashLocalObject(
-                              (implicit l: Logger): IO[Map[String, MD5Hash]] =
+      path: Path
+  )(implicit l: Logger): IO[Map[String, MD5Hash]] =
     for {
-      md5 <- MD5HashGenerator.md5File(path)
+      md5  <- MD5HashGenerator.md5File(path)
       etag <- ETagGenerator.eTag(path).map(MD5Hash(_))
-    } yield Map(
+    } yield
-      "md5" -> md5,
+      Map(
-      "etag" -> etag
+        "md5"  -> md5,
-    )
+        "etag" -> etag
+      )
 
 }
 

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHash.scala

@@ -5,14 +5,19 @@ import net.kemitix.thorp.domain.{KeyModified, LastModified, MD5Hash, RemoteKey}
 
 object S3ObjectsByHash {
 
-  def byHash(os: Stream[S3ObjectSummary]): Map[MD5Hash, Set[KeyModified]] = {
+  def byHash(
+      os: Stream[S3ObjectSummary]
+  ): Map[MD5Hash, Set[KeyModified]] = {
     val mD5HashToS3Objects: Map[MD5Hash, Stream[S3ObjectSummary]] =
-      os.groupBy(o => MD5Hash(o.getETag.filter{c => c != '"'}))
+      os.groupBy(o => MD5Hash(o.getETag.filter(_ != '"')))
-    val hashToModifieds: Map[MD5Hash, Set[KeyModified]] =
+    mD5HashToS3Objects.mapValues { os =>
-      mD5HashToS3Objects.mapValues { os =>
+      os.map { o =>
-        os.map { o =>
+        KeyModified(
-          KeyModified(RemoteKey(o.getKey), LastModified(o.getLastModified.toInstant))}.toSet }
+          RemoteKey(o.getKey),
-    hashToModifieds
+          LastModified(o.getLastModified.toInstant)
+        )
+      }.toSet
+    }
   }
 
 }

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByKey.scala

@@ -5,12 +5,14 @@ import net.kemitix.thorp.domain.{HashModified, LastModified, MD5Hash, RemoteKey}
 
 object S3ObjectsByKey {
 
-  def byKey(os: Stream[S3ObjectSummary]) =
+  def byKey(os: Stream[S3ObjectSummary]): Map[RemoteKey, HashModified] =
-    os.map { o => {
+    os.map { o =>
-      val remoteKey = RemoteKey(o.getKey)
+      {
-      val hash = MD5Hash(o.getETag)
+        val remoteKey    = RemoteKey(o.getKey)
-      val lastModified = LastModified(o.getLastModified.toInstant)
+        val hash         = MD5Hash(o.getETag)
-      (remoteKey, HashModified(hash, lastModified))
+        val lastModified = LastModified(o.getLastModified.toInstant)
-    }}.toMap
+        (remoteKey, HashModified(hash, lastModified))
+      }
+    }.toMap
 
 }

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3StorageService.scala

@@ -3,45 +3,52 @@ package net.kemitix.thorp.storage.aws
 import cats.data.EitherT
 import cats.effect.IO
 import com.amazonaws.services.s3.AmazonS3
-import com.amazonaws.services.s3.transfer.TransferManager
 import net.kemitix.thorp.domain.StorageQueueEvent.ShutdownQueueEvent
 import net.kemitix.thorp.domain._
 import net.kemitix.thorp.storage.api.StorageService
 
-class S3StorageService(amazonS3Client: => AmazonS3,
+class S3StorageService(
-                       amazonS3TransferManager: => TransferManager)
+    amazonS3Client: => AmazonS3,
-  extends StorageService {
+    amazonTransferManager: => AmazonTransferManager
+) extends StorageService {
 
   lazy val objectLister = new Lister(amazonS3Client)
-  lazy val copier = new Copier(amazonS3Client)
+  lazy val copier       = new Copier(amazonS3Client)
-  lazy val uploader = new Uploader(amazonS3TransferManager)
+  lazy val uploader     = new Uploader(amazonTransferManager)
-  lazy val deleter = new Deleter(amazonS3Client)
+  lazy val deleter      = new Deleter(amazonS3Client)
 
-  override def listObjects(bucket: Bucket,
+  override def listObjects(
-                           prefix: RemoteKey)
+      bucket: Bucket,
-                          (implicit l: Logger): EitherT[IO, String, S3ObjectsData] =
+      prefix: RemoteKey
+  )(implicit l: Logger): EitherT[IO, String, S3ObjectsData] =
     objectLister.listObjects(bucket, prefix)
 
-  override def copy(bucket: Bucket,
+  override def copy(
-                    sourceKey: RemoteKey,
+      bucket: Bucket,
-                    hash: MD5Hash,
+      sourceKey: RemoteKey,
-                    targetKey: RemoteKey): IO[StorageQueueEvent] =
+      hash: MD5Hash,
-    copier.copy(bucket, sourceKey,hash, targetKey)
+      targetKey: RemoteKey
+  ): IO[StorageQueueEvent] =
+    copier.copy(bucket, sourceKey, hash, targetKey)
 
-  override def upload(localFile: LocalFile,
+  override def upload(
-                      bucket: Bucket,
+      localFile: LocalFile,
-                      batchMode: Boolean,
+      bucket: Bucket,
-                      uploadEventListener: UploadEventListener,
+      batchMode: Boolean,
-                      tryCount: Int): IO[StorageQueueEvent] =
+      uploadEventListener: UploadEventListener,
+      tryCount: Int
+  ): IO[StorageQueueEvent] =
     uploader.upload(localFile, bucket, batchMode, uploadEventListener, 1)
 
-  override def delete(bucket: Bucket,
+  override def delete(
-                      remoteKey: RemoteKey): IO[StorageQueueEvent] =
+      bucket: Bucket,
+      remoteKey: RemoteKey
+  ): IO[StorageQueueEvent] =
     deleter.delete(bucket, remoteKey)
 
   override def shutdown: IO[StorageQueueEvent] =
     IO {
-      amazonS3TransferManager.shutdownNow(true)
+      amazonTransferManager.shutdownNow(true)
       amazonS3Client.shutdown()
       ShutdownQueueEvent()
     }

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3StorageServiceBuilder.scala

@@ -1,18 +1,24 @@
 package net.kemitix.thorp.storage.aws
 
-import com.amazonaws.services.s3.transfer.{TransferManager, TransferManagerBuilder}
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder
 import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
 import net.kemitix.thorp.storage.api.StorageService
 
 object S3StorageServiceBuilder {
 
-  def createService(amazonS3Client: AmazonS3,
+  def createService(
-                    amazonS3TransferManager: TransferManager): StorageService =
+      amazonS3Client: AmazonS3,
-    new S3StorageService(amazonS3Client, amazonS3TransferManager)
+      amazonTransferManager: AmazonTransferManager
+  ): StorageService =
+    new S3StorageService(
+      amazonS3Client,
+      amazonTransferManager
+    )
 
   lazy val defaultStorageService: StorageService =
     createService(
       AmazonS3ClientBuilder.defaultClient,
-      TransferManagerBuilder.defaultTransferManager)
+      AmazonTransferManager(TransferManagerBuilder.defaultTransferManager)
+    )
 
 }

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Uploader.scala

@@ -4,35 +4,45 @@ import cats.effect.IO
 import com.amazonaws.event.{ProgressEvent, ProgressEventType, ProgressListener}
 import com.amazonaws.services.s3.model.{ObjectMetadata, PutObjectRequest}
 import com.amazonaws.services.s3.transfer.model.UploadResult
-import com.amazonaws.services.s3.transfer.{TransferManager => AmazonTransferManager}
+import net.kemitix.thorp.domain.StorageQueueEvent.{
-import net.kemitix.thorp.domain.StorageQueueEvent.{ErrorQueueEvent, UploadQueueEvent}
+  ErrorQueueEvent,
-import net.kemitix.thorp.domain.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent}
+  UploadQueueEvent
+}
+import net.kemitix.thorp.domain.UploadEvent.{
+  ByteTransferEvent,
+  RequestEvent,
+  TransferEvent
+}
 import net.kemitix.thorp.domain.{StorageQueueEvent, _}
 
 import scala.util.Try
 
 class Uploader(transferManager: => AmazonTransferManager) {
 
-  def upload(localFile: LocalFile,
+  def upload(
-             bucket: Bucket,
+      localFile: LocalFile,
-             batchMode: Boolean,
+      bucket: Bucket,
-             uploadEventListener: UploadEventListener,
+      batchMode: Boolean,
-             tryCount: Int): IO[StorageQueueEvent] =
+      uploadEventListener: UploadEventListener,
+      tryCount: Int
+  ): IO[StorageQueueEvent] =
     for {
       upload <- transfer(localFile, bucket, batchMode, uploadEventListener)
       action = upload match {
-        case Right(r) => UploadQueueEvent(RemoteKey(r.getKey), MD5Hash(r.getETag))
+        case Right(r) =>
+          UploadQueueEvent(RemoteKey(r.getKey), MD5Hash(r.getETag))
         case Left(e) => ErrorQueueEvent(localFile.remoteKey, e)
       }
     } yield action
 
-  private def transfer(localFile: LocalFile,
+  private def transfer(
-                       bucket: Bucket,
+      localFile: LocalFile,
-                       batchMode: Boolean,
+      bucket: Bucket,
-                       uploadEventListener: UploadEventListener,
+      batchMode: Boolean,
-                      ): IO[Either[Throwable, UploadResult]] = {
+      uploadEventListener: UploadEventListener
+  ): IO[Either[Throwable, UploadResult]] = {
     val listener: ProgressListener = progressListener(uploadEventListener)
-    val putObjectRequest = request(localFile, bucket, batchMode, listener)
+    val putObjectRequest           = request(localFile, bucket, batchMode, listener)
     IO {
       Try(transferManager.upload(putObjectRequest))
         .map(_.waitForUploadResult)
@@ -40,23 +50,25 @@ class Uploader(transferManager: => AmazonTransferManager) {
     }
   }
 
-  private def request(localFile: LocalFile,
+  private def request(
-                      bucket: Bucket,
+      localFile: LocalFile,
-                      batchMode: Boolean,
+      bucket: Bucket,
-                      listener: ProgressListener): PutObjectRequest = {
+      batchMode: Boolean,
+      listener: ProgressListener
+  ): PutObjectRequest = {
     val metadata = new ObjectMetadata()
     localFile.md5base64.foreach(metadata.setContentMD5)
-    val request = new PutObjectRequest(bucket.name, localFile.remoteKey.key, localFile.file)
+    val request =
-      .withMetadata(metadata)
+      new PutObjectRequest(bucket.name, localFile.remoteKey.key, localFile.file)
+        .withMetadata(metadata)
     if (batchMode) request
     else request.withGeneralProgressListener(listener)
   }
 
   private def progressListener(uploadEventListener: UploadEventListener) =
     new ProgressListener {
-      override def progressChanged(progressEvent: ProgressEvent): Unit = {
+      override def progressChanged(progressEvent: ProgressEvent): Unit =
         uploadEventListener.listener(eventHandler(progressEvent))
-      }
 
       private def eventHandler(progressEvent: ProgressEvent) = {
         progressEvent match {

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/DummyLogger.scala

@@ -7,7 +7,7 @@ class DummyLogger extends Logger {
 
   override def debug(message: => String): IO[Unit] = IO.unit
 
-  override def info(message: =>String): IO[Unit] = IO.unit
+  override def info(message: => String): IO[Unit] = IO.unit
 
   override def warn(message: String): IO[Unit] = IO.unit
 

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ETagGeneratorTest.scala

@@ -7,18 +7,21 @@ import org.scalatest.FunSpec
 
 class ETagGeneratorTest extends FunSpec {
 
-  private val bigFile = Resource(this, "big-file")
+  private val bigFile       = Resource(this, "big-file")
-  private val bigFilePath = bigFile.toPath
+  private val bigFilePath   = bigFile.toPath
   private val configuration = new TransferManagerConfiguration
-  private val chunkSize = 1200000
+  private val chunkSize     = 1200000
   configuration.setMinimumUploadPartSize(chunkSize)
   private val logger = new DummyLogger
 
   describe("Create offsets") {
     it("should create offsets") {
-      val offsets = ETagGenerator.offsets(bigFile.length, chunkSize)
+      val offsets = ETagGenerator
+        .offsets(bigFile.length, chunkSize)
         .foldRight(List[Long]())((l: Long, a: List[Long]) => l :: a)
-      assertResult(List(0, chunkSize, chunkSize * 2, chunkSize * 3, chunkSize * 4))(offsets)
+      assertResult(
+        List(0, chunkSize, chunkSize * 2, chunkSize * 3, chunkSize * 4))(
+        offsets)
     }
   }
 
@@ -35,8 +38,12 @@ class ETagGeneratorTest extends FunSpec {
         "5bd6e10a99fef100fe7bf5eaa0a42384",
         "8a0c1d0778ac8fcf4ca2010eba4711eb"
       ).zipWithIndex
-      md5Hashes.foreach { case (hash, index) =>
+      md5Hashes.foreach {
-        test(hash, ETagGenerator.hashChunk(bigFilePath, index, chunkSize)(logger).unsafeRunSync)
+        case (hash, index) =>
+          test(hash,
+               ETagGenerator
+                 .hashChunk(bigFilePath, index, chunkSize)(logger)
+                 .unsafeRunSync)
       }
     }
   }

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala

@@ -10,22 +10,23 @@ import org.scalatest.FunSpec
 
 class S3ObjectsByHashSuite extends FunSpec {
 
-    describe("grouping s3 object together by their hash values") {
+  describe("grouping s3 object together by their hash values") {
-      val hash = MD5Hash("hash")
+    val hash         = MD5Hash("hash")
-      val key1 = RemoteKey("key-1")
+    val key1         = RemoteKey("key-1")
-      val key2 = RemoteKey("key-2")
+    val key2         = RemoteKey("key-2")
-      val lastModified = LastModified(Instant.now.truncatedTo(ChronoUnit.MILLIS))
+    val lastModified = LastModified(Instant.now.truncatedTo(ChronoUnit.MILLIS))
-      val o1 = s3object(hash, key1, lastModified)
+    val o1           = s3object(hash, key1, lastModified)
-      val o2 = s3object(hash, key2, lastModified)
+    val o2           = s3object(hash, key2, lastModified)
-      val os = Stream(o1, o2)
+    val os           = Stream(o1, o2)
-      it("should group by the hash value") {
+    it("should group by the hash value") {
-        val expected: Map[MD5Hash, Set[KeyModified]] = Map(
+      val expected: Map[MD5Hash, Set[KeyModified]] = Map(
-          hash -> Set(KeyModified(key1, lastModified), KeyModified(key2, lastModified))
+        hash -> Set(KeyModified(key1, lastModified),
-        )
+                    KeyModified(key2, lastModified))
-        val result = S3ObjectsByHash.byHash(os)
+      )
-        assertResult(expected)(result)
+      val result = S3ObjectsByHash.byHash(os)
-      }
+      assertResult(expected)(result)
     }
+  }
 
   private def s3object(md5Hash: MD5Hash,
                        remoteKey: RemoteKey,

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3StorageServiceSuite.scala

@@ -5,22 +5,24 @@ import java.time.temporal.ChronoUnit
 import java.util.Date
 
 import com.amazonaws.services.s3.AmazonS3
-import com.amazonaws.services.s3.model.{ListObjectsV2Request, ListObjectsV2Result, S3ObjectSummary}
+import com.amazonaws.services.s3.model.{
-import com.amazonaws.services.s3.transfer.TransferManager
+  ListObjectsV2Request,
+  ListObjectsV2Result,
+  S3ObjectSummary
+}
 import net.kemitix.thorp.core.Resource
 import net.kemitix.thorp.domain._
 import org.scalamock.scalatest.MockFactory
 import org.scalatest.FunSpec
 
-class S3StorageServiceSuite
+class S3StorageServiceSuite extends FunSpec with MockFactory {
-  extends FunSpec
-    with MockFactory {
 
   describe("listObjectsInPrefix") {
-    val source = Resource(this, "upload")
+    val source     = Resource(this, "upload")
     val sourcePath = source.toPath
-    val prefix = RemoteKey("prefix")
+    val prefix     = RemoteKey("prefix")
-    implicit val config: Config = Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
+    implicit val config: Config =
+      Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
     implicit val implLogger: Logger = new DummyLogger
 
     val lm = LastModified(Instant.now.truncatedTo(ChronoUnit.MILLIS))
@@ -29,7 +31,9 @@ class S3StorageServiceSuite
 
     val k1a = RemoteKey("key1a")
 
-    def objectSummary(hash: MD5Hash, remoteKey: RemoteKey, lastModified: LastModified) = {
+    def objectSummary(hash: MD5Hash,
+                      remoteKey: RemoteKey,
+                      lastModified: LastModified) = {
       val summary = new S3ObjectSummary()
       summary.setETag(hash.hash)
       summary.setKey(remoteKey.key)
@@ -46,27 +50,33 @@ class S3StorageServiceSuite
     val k2 = RemoteKey("key2")
     val o2 = objectSummary(h2, k2, lm)
 
-    val amazonS3 = stub[AmazonS3]
+    val amazonS3                = stub[AmazonS3]
-    val amazonS3TransferManager = stub[TransferManager]
+    val amazonS3TransferManager = stub[AmazonTransferManager]
-    val storageService = new S3StorageService(amazonS3, amazonS3TransferManager)
+    val storageService          = new S3StorageService(amazonS3, amazonS3TransferManager)
 
     val myFakeResponse = new ListObjectsV2Result()
-    val summaries = myFakeResponse.getObjectSummaries
+    val summaries      = myFakeResponse.getObjectSummaries
     summaries.add(o1a)
     summaries.add(o1b)
     summaries.add(o2)
-    (amazonS3 listObjectsV2 (_: ListObjectsV2Request)).when(*).returns(myFakeResponse)
+    (amazonS3 listObjectsV2 (_: ListObjectsV2Request))
+      .when(*)
+      .returns(myFakeResponse)
 
-    it("should build list of hash lookups, with duplicate objects grouped by hash") {
+    it(
-      val expected = Right(S3ObjectsData(
+      "should build list of hash lookups, with duplicate objects grouped by hash") {
-        byHash = Map(
+      val expected = Right(
-          h1 -> Set(KeyModified(k1a, lm), KeyModified(k1b, lm)),
+        S3ObjectsData(
-          h2 -> Set(KeyModified(k2, lm))),
+          byHash = Map(h1 -> Set(KeyModified(k1a, lm), KeyModified(k1b, lm)),
-        byKey = Map(
+                       h2 -> Set(KeyModified(k2, lm))),
-          k1a -> HashModified(h1, lm),
+          byKey = Map(k1a -> HashModified(h1, lm),
-          k1b -> HashModified(h1, lm),
+                      k1b -> HashModified(h1, lm),
-          k2 -> HashModified(h2, lm))))
+                      k2  -> HashModified(h2, lm))
-      val result = storageService.listObjects(Bucket("bucket"), RemoteKey("prefix")).value.unsafeRunSync
+        ))
+      val result = storageService
+        .listObjects(Bucket("bucket"), RemoteKey("prefix"))
+        .value
+        .unsafeRunSync
       assertResult(expected)(result)
     }
   }

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala

@@ -5,7 +5,6 @@ import java.time.Instant
 import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.s3.model.PutObjectRequest
 import com.amazonaws.services.s3.transfer.model.UploadResult
-import com.amazonaws.services.s3.transfer.{TransferManager, Upload}
 import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
 import net.kemitix.thorp.domain.MD5HashData.Root
 import net.kemitix.thorp.domain.StorageQueueEvent.UploadQueueEvent
@@ -13,49 +12,65 @@ import net.kemitix.thorp.domain._
 import org.scalamock.scalatest.MockFactory
 import org.scalatest.FunSpec
 
-class StorageServiceSuite
+class StorageServiceSuite extends FunSpec with MockFactory {
-  extends FunSpec
-    with MockFactory {
 
-  val source = Resource(this, "upload")
+  private val source     = Resource(this, "upload")
-  val sourcePath = source.toPath
+  private val sourcePath = source.toPath
 
   private val prefix = RemoteKey("prefix")
-  implicit private val config: Config = Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
+  implicit private val config: Config =
+    Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
   implicit private val implLogger: Logger = new DummyLogger
-  private val fileToKey = KeyGenerator.generateKey(config.sources, config.prefix) _
+  private val fileToKey =
+    KeyGenerator.generateKey(config.sources, config.prefix) _
 
   describe("getS3Status") {
     val hash = MD5Hash("hash")
-    val localFile = LocalFile.resolve("the-file", md5HashMap(hash), sourcePath, fileToKey)
+    val localFile =
+      LocalFile.resolve("the-file", md5HashMap(hash), sourcePath, fileToKey)
     val key = localFile.remoteKey
-    val keyOtherKey = LocalFile.resolve("other-key-same-hash", md5HashMap(hash), sourcePath, fileToKey)
+    val keyOtherKey = LocalFile.resolve("other-key-same-hash",
+                                        md5HashMap(hash),
+                                        sourcePath,
+                                        fileToKey)
     val diffHash = MD5Hash("diff")
-    val keyDiffHash = LocalFile.resolve("other-key-diff-hash", md5HashMap(diffHash), sourcePath, fileToKey)
+    val keyDiffHash = LocalFile.resolve("other-key-diff-hash",
+                                        md5HashMap(diffHash),
+                                        sourcePath,
+                                        fileToKey)
     val lastModified = LastModified(Instant.now)
     val s3ObjectsData: S3ObjectsData = S3ObjectsData(
       byHash = Map(
-        hash -> Set(KeyModified(key, lastModified), KeyModified(keyOtherKey.remoteKey, lastModified)),
+        hash -> Set(KeyModified(key, lastModified),
-        diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))),
+                    KeyModified(keyOtherKey.remoteKey, lastModified)),
+        diffHash -> Set(KeyModified(keyDiffHash.remoteKey, lastModified))
+      ),
       byKey = Map(
-        key -> HashModified(hash, lastModified),
+        key                   -> HashModified(hash, lastModified),
         keyOtherKey.remoteKey -> HashModified(hash, lastModified),
-        keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)))
+        keyDiffHash.remoteKey -> HashModified(diffHash, lastModified)
+      )
+    )
 
     def invoke(localFile: LocalFile) =
       S3MetaDataEnricher.getS3Status(localFile, s3ObjectsData)
 
-    def getMatchesByKey(status: (Option[HashModified], Set[(MD5Hash, KeyModified)])): Option[HashModified] = {
+    def getMatchesByKey(
+        status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
+      : Option[HashModified] = {
       val (byKey, _) = status
       byKey
     }
 
-    def getMatchesByHash(status: (Option[HashModified], Set[(MD5Hash, KeyModified)])): Set[(MD5Hash, KeyModified)] = {
+    def getMatchesByHash(
+        status: (Option[HashModified], Set[(MD5Hash, KeyModified)]))
+      : Set[(MD5Hash, KeyModified)] = {
       val (_, byHash) = status
       byHash
     }
 
-    describe("when remote key exists, unmodified and other key matches the hash") {
+    describe(
+      "when remote key exists, unmodified and other key matches the hash") {
       it("should return the match by key") {
         val result = getMatchesByKey(invoke(localFile))
         assert(result.contains(HashModified(hash, lastModified)))
@@ -63,15 +78,17 @@ class StorageServiceSuite
       it("should return both matches for the hash") {
         val result = getMatchesByHash(invoke(localFile))
         assertResult(
-          Set(
+          Set((hash, KeyModified(key, lastModified)),
-            (hash, KeyModified(key, lastModified)),
+              (hash, KeyModified(keyOtherKey.remoteKey, lastModified)))
-            (hash, KeyModified(keyOtherKey.remoteKey, lastModified)))
         )(result)
       }
     }
 
     describe("when remote key does not exist and no others matches hash") {
-      val localFile = LocalFile.resolve("missing-file", md5HashMap(MD5Hash("unique")), sourcePath, fileToKey)
+      val localFile = LocalFile.resolve("missing-file",
+                                        md5HashMap(MD5Hash("unique")),
+                                        sourcePath,
+                                        fileToKey)
       it("should return no matches by key") {
         val result = getMatchesByKey(invoke(localFile))
         assert(result.isEmpty)
@@ -91,36 +108,41 @@ class StorageServiceSuite
       it("should return one match by hash") {
         val result = getMatchesByHash(invoke(localFile))
         assertResult(
-          Set(
+          Set((diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))
-            (diffHash, KeyModified(keyDiffHash.remoteKey, lastModified)))
         )(result)
       }
     }
 
   }
 
-  private def md5HashMap(hash: MD5Hash) = {
+  private def md5HashMap(hash: MD5Hash) =
     Map("md5" -> hash)
-  }
 
   val batchMode: Boolean = true
 
   describe("upload") {
 
     describe("when uploading a file") {
-      val amazonS3 = stub[AmazonS3]
+      val amazonS3              = stub[AmazonS3]
-      val amazonS3TransferManager = stub[TransferManager]
+      val amazonTransferManager = stub[AmazonTransferManager]
-      val storageService = new S3StorageService(amazonS3, amazonS3TransferManager)
+      val storageService =
+        new S3StorageService(amazonS3, amazonTransferManager)
 
       val prefix = RemoteKey("prefix")
       val localFile =
-        LocalFile.resolve("root-file", md5HashMap(Root.hash), sourcePath, KeyGenerator.generateKey(config.sources, prefix))
+        LocalFile.resolve("root-file",
-      val bucket = Bucket("a-bucket")
+                          md5HashMap(Root.hash),
+                          sourcePath,
+                          KeyGenerator.generateKey(config.sources, prefix))
+      val bucket    = Bucket("a-bucket")
       val remoteKey = RemoteKey("prefix/root-file")
-      val uploadEventListener = new UploadEventListener(localFile, 1, SyncTotals(), 0L)
+      val uploadEventListener =
+        new UploadEventListener(localFile, 1, SyncTotals(), 0L)
 
-      val upload = stub[Upload]
+      val upload = stub[AmazonUpload]
-      (amazonS3TransferManager upload (_: PutObjectRequest)).when(*).returns(upload)
+      (amazonTransferManager upload (_: PutObjectRequest))
+        .when(*)
+        .returns(upload)
       val uploadResult = stub[UploadResult]
       (upload.waitForUploadResult _).when().returns(uploadResult)
       (uploadResult.getETag _).when().returns(Root.hash.hash)
@@ -130,7 +152,12 @@ class StorageServiceSuite
         pending
         //FIXME: works okay on its own, but fails when run with others
         val expected = UploadQueueEvent(remoteKey, Root.hash)
-        val result = storageService.upload(localFile, bucket, batchMode, uploadEventListener, 1)
+        val result =
+          storageService.upload(localFile,
+                                bucket,
+                                batchMode,
+                                uploadEventListener,
+                                1)
         assertResult(expected)(result)
       }
     }

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderSuite.scala

@@ -1,7 +1,5 @@
 package net.kemitix.thorp.storage.aws
 
-import java.time.Instant
-
 import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.s3.transfer._
 import net.kemitix.thorp.core.KeyGenerator.generateKey
@@ -11,24 +9,18 @@ import net.kemitix.thorp.domain.{UploadEventListener, _}
 import org.scalamock.scalatest.MockFactory
 import org.scalatest.FunSpec
 
-class UploaderSuite
+class UploaderSuite extends FunSpec with MockFactory {
-  extends FunSpec
-    with MockFactory {
 
-  private val source = Resource(this, ".")
+  private val batchMode: Boolean = true
-  private val sourcePath = source.toPath
+  private val source             = Resource(this, ".")
-  private val prefix = RemoteKey("prefix")
+  private val sourcePath         = source.toPath
-  implicit private val config: Config = Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
+  private val prefix             = RemoteKey("prefix")
+  implicit private val config: Config =
+    Config(Bucket("bucket"), prefix, sources = Sources(List(sourcePath)))
+  private val fileToKey                   = generateKey(config.sources, config.prefix) _
   implicit private val implLogger: Logger = new DummyLogger
-  private val fileToKey = generateKey(config.sources, config.prefix) _
-  val lastModified = LastModified(Instant.now())
 
-  def md5HashMap(hash: MD5Hash): Map[String, MD5Hash] =
+  def md5HashMap(hash: MD5Hash): Map[String, MD5Hash] = Map("md5" -> hash)
-    Map(
-      "md5" -> hash
-    )
-
-  val batchMode: Boolean = true
 
   describe("S3ClientMultiPartTransferManagerSuite") {
     describe("upload") {
@@ -37,16 +29,26 @@ class UploaderSuite
       // Should we just test that the correct parameters are passed to initiate, or will this test
       // just collapse and die if the amazonS3 doesn't respond properly to TransferManager input
       // dies when putObject is called
-      val returnedKey = RemoteKey("returned-key")
+      val returnedKey  = RemoteKey("returned-key")
       val returnedHash = MD5Hash("returned-hash")
-      val bigFile = LocalFile.resolve("small-file", md5HashMap(MD5Hash("the-hash")), sourcePath, fileToKey)
+      val bigFile = LocalFile.resolve("small-file",
-      val uploadEventListener = new UploadEventListener(bigFile, 1, SyncTotals(), 0L)
+                                      md5HashMap(MD5Hash("the-hash")),
+                                      sourcePath,
+                                      fileToKey)
+      val uploadEventListener =
+        new UploadEventListener(bigFile, 1, SyncTotals(), 0L)
       val amazonS3 = mock[AmazonS3]
-      val amazonS3TransferManager = TransferManagerBuilder.standard().withS3Client(amazonS3).build
+      val amazonTransferManager =
-      val uploader = new Uploader(amazonS3TransferManager)
+        AmazonTransferManager(
+          TransferManagerBuilder.standard().withS3Client(amazonS3).build)
+      val uploader = new Uploader(amazonTransferManager)
       it("should upload") {
         val expected = UploadQueueEvent(returnedKey, returnedHash)
-        val result = uploader.upload(bigFile, config.bucket, batchMode, uploadEventListener, 1)
+        val result = uploader.upload(bigFile,
+                                     config.bucket,
+                                     batchMode,
+                                     uploadEventListener,
+                                     1)
         assertResult(expected)(result)
       }
     }