Rename project to Thorp (#75)

* [sbt] change application name

* [cli] rename package

* [cli] Change displayed application name and description

* [domain] rename package

* [core] fix bad package directory structure

* [core] rename package

* [aws-lib] rename package

* [aws-api] rename package

* [cli] rename programe for usage message

* [bin] rename and update script

* [gitignore] update

* [readme] update
This commit is contained in:
Paul Campbell 2019-06-17 15:33:49 +01:00 committed by GitHub
parent 6450bbeff5
commit 9196dd623f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
78 changed files with 190 additions and 189 deletions

2
.gitignore vendored
View file

@ -28,5 +28,5 @@ project/plugins/project/
### Metals ### ### Metals ###
.bloop/ .bloop/
.metals/ .metals/
/s3thorp /thorp
*.zip *.zip

View file

@ -1,8 +1,8 @@
* s3thorp * thorp
Synchronisation of files with S3 using the hash of the file contents. Synchronisation of files with S3 using the hash of the file contents.
[[https://www.codacy.com/app/kemitix/s3thorp][file:https://img.shields.io/codacy/grade/14ea6ad0825249c994a27a82d3485180.svg?style=for-the-badge]] [[https://www.codacy.com/app/kemitix/thorp][file:https://img.shields.io/codacy/grade/14ea6ad0825249c994a27a82d3485180.svg?style=for-the-badge]]
Originally based on Alex Kudlick's [[https://github.com/akud/aws-s3-sync-by-hash][aws-s3-sync-by-hash]]. Originally based on Alex Kudlick's [[https://github.com/akud/aws-s3-sync-by-hash][aws-s3-sync-by-hash]].
@ -13,8 +13,8 @@ hash of the file contents.
* Usage * Usage
#+begin_example #+begin_example
s3thorp thorp
Usage: s3thorp [options] Usage: thorp [options]
-s, --source <value> Source directory to sync to S3 -s, --source <value> Source directory to sync to S3
-b, --bucket <value> S3 bucket name -b, --bucket <value> S3 bucket name
@ -46,8 +46,8 @@ When considering a local file, the following table governs what should happen:
To build as an executable jar, perform `sbt assembly` To build as an executable jar, perform `sbt assembly`
This will create the file This will create the file
`cli/target/scala-2.12/s3thorp-assembly-$VERSION.jar` (where $VERSION `cli/target/scala-2.12/thorp-assembly-$VERSION.jar` (where $VERSION
is substituted) is substituted)
Copy and rename this file as `s3thorp.jar` into the same directory as Copy and rename this file as `thorp.jar` into the same directory as
the `bin/s3throp` shell script. the `bin/throp` shell script.

View file

@ -1,6 +1,6 @@
package net.kemitix.s3thorp.aws.api package net.kemitix.thorp.aws.api
import net.kemitix.s3thorp.domain.{MD5Hash, RemoteKey} import net.kemitix.thorp.domain.{MD5Hash, RemoteKey}
sealed trait S3Action { sealed trait S3Action {

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.aws.api package net.kemitix.thorp.aws.api
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action} import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action}
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey, S3ObjectsData} import net.kemitix.thorp.domain._
trait S3Client[M[_]] { trait S3Client[M[_]] {

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.aws.api package net.kemitix.thorp.aws.api
sealed trait UploadEvent { sealed trait UploadEvent {
def name: String def name: String

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.aws.api package net.kemitix.thorp.aws.api
import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent} import net.kemitix.thorp.aws.api.UploadEvent.RequestEvent
import net.kemitix.s3thorp.domain.LocalFile import net.kemitix.thorp.domain.LocalFile
class UploadProgressListener(localFile: LocalFile) class UploadProgressListener(localFile: LocalFile)
extends UploadProgressLogging { extends UploadProgressLogging {

View file

@ -1,9 +1,9 @@
package net.kemitix.s3thorp.aws.api package net.kemitix.thorp.aws.api
import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent} import net.kemitix.thorp.aws.api.UploadEvent.RequestEvent
import net.kemitix.s3thorp.domain.Terminal.{clearLine, returnToPreviousLine} import net.kemitix.thorp.domain.SizeTranslation.sizeInEnglish
import net.kemitix.s3thorp.domain.{LocalFile, Terminal} import net.kemitix.thorp.domain.Terminal.{clearLine, returnToPreviousLine}
import net.kemitix.s3thorp.domain.SizeTranslation.sizeInEnglish import net.kemitix.thorp.domain.{LocalFile, Terminal}
import scala.io.AnsiColor._ import scala.io.AnsiColor._

View file

@ -1,9 +1,9 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import cats.Monad import cats.Monad
import com.amazonaws.services.s3.transfer.{TransferManager, TransferManagerBuilder} import com.amazonaws.services.s3.transfer.{TransferManager, TransferManagerBuilder}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder} import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import net.kemitix.s3thorp.aws.api.S3Client import net.kemitix.thorp.aws.api.S3Client
object S3ClientBuilder { object S3ClientBuilder {

View file

@ -1,12 +1,12 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.CopyObjectRequest import com.amazonaws.services.s3.model.CopyObjectRequest
import net.kemitix.s3thorp.aws.api.S3Action.CopyS3Action import net.kemitix.thorp.aws.api.S3Action.CopyS3Action
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logCopyFinish, logCopyStart} import net.kemitix.thorp.aws.lib.S3ClientLogging.{logCopyFinish, logCopyStart}
import net.kemitix.s3thorp.domain.{Bucket, Logger, MD5Hash, RemoteKey} import net.kemitix.thorp.domain.{Bucket, Logger, MD5Hash, RemoteKey}
class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) { class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) {

View file

@ -1,12 +1,12 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.DeleteObjectRequest import com.amazonaws.services.s3.model.DeleteObjectRequest
import net.kemitix.s3thorp.aws.api.S3Action.DeleteS3Action import net.kemitix.thorp.aws.api.S3Action.DeleteS3Action
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logDeleteFinish, logDeleteStart} import net.kemitix.thorp.aws.lib.S3ClientLogging.{logDeleteFinish, logDeleteStart}
import net.kemitix.s3thorp.domain.{Bucket, Logger, RemoteKey} import net.kemitix.thorp.domain.{Bucket, Logger, RemoteKey}
class S3ClientDeleter[M[_]: Monad](amazonS3: AmazonS3) { class S3ClientDeleter[M[_]: Monad](amazonS3: AmazonS3) {

View file

@ -1,8 +1,7 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import cats.Monad import cats.Monad
import com.amazonaws.services.s3.model.PutObjectResult import net.kemitix.thorp.domain.{Bucket, Logger, RemoteKey}
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, RemoteKey}
object S3ClientLogging { object S3ClientLogging {

View file

@ -1,13 +1,14 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.{ListObjectsV2Request, S3ObjectSummary} import com.amazonaws.services.s3.model.{ListObjectsV2Request, S3ObjectSummary}
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logListObjectsFinish, logListObjectsStart} import net.kemitix.thorp.aws.lib.S3ClientLogging.{logListObjectsFinish, logListObjectsStart}
import net.kemitix.s3thorp.aws.lib.S3ObjectsByHash.byHash import net.kemitix.thorp.aws.lib.S3ObjectsByHash.byHash
import net.kemitix.s3thorp.aws.lib.S3ObjectsByKey.byKey import net.kemitix.thorp.aws.lib.S3ObjectsByKey.byKey
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain
import net.kemitix.thorp.domain.{Bucket, Logger, RemoteKey, S3ObjectsData}
import scala.collection.JavaConverters._ import scala.collection.JavaConverters._
@ -54,7 +55,7 @@ class S3ClientObjectLister[M[_]: Monad](amazonS3: AmazonS3) {
r = new ListObjectsV2Request().withBucketName(bucket.name).withPrefix(prefix.key) r = new ListObjectsV2Request().withBucketName(bucket.name).withPrefix(prefix.key)
summaries <- fetch(r) summaries <- fetch(r)
_ <- logListObjectsFinish[M](bucket, prefix) _ <- logListObjectsFinish[M](bucket, prefix)
} yield S3ObjectsData(byHash(summaries), byKey(summaries)) } yield domain.S3ObjectsData(byHash(summaries), byKey(summaries))
} }
} }

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import com.amazonaws.services.s3.model.S3ObjectSummary import com.amazonaws.services.s3.model.S3ObjectSummary
import net.kemitix.s3thorp.domain.{KeyModified, LastModified, MD5Hash, RemoteKey} import net.kemitix.thorp.domain.{KeyModified, LastModified, MD5Hash, RemoteKey}
object S3ObjectsByHash { object S3ObjectsByHash {

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import com.amazonaws.services.s3.model.S3ObjectSummary import com.amazonaws.services.s3.model.S3ObjectSummary
import net.kemitix.s3thorp.domain.{HashModified, LastModified, MD5Hash, RemoteKey} import net.kemitix.thorp.domain.{HashModified, LastModified, MD5Hash, RemoteKey}
object S3ObjectsByKey { object S3ObjectsByKey {

View file

@ -1,11 +1,11 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import cats.Monad import cats.Monad
import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.transfer.TransferManager import com.amazonaws.services.s3.transfer.TransferManager
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action} import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action}
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client, UploadProgressListener} import net.kemitix.thorp.aws.api.{S3Action, S3Client, UploadProgressListener}
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
class ThorpS3Client[M[_]: Monad](amazonS3Client: => AmazonS3, class ThorpS3Client[M[_]: Monad](amazonS3Client: => AmazonS3,
amazonS3TransferManager: => TransferManager) amazonS3TransferManager: => TransferManager)

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
@ -6,11 +6,11 @@ import com.amazonaws.event.{ProgressEvent, ProgressEventType, ProgressListener}
import com.amazonaws.services.s3.model.PutObjectRequest import com.amazonaws.services.s3.model.PutObjectRequest
import com.amazonaws.services.s3.transfer.model.UploadResult import com.amazonaws.services.s3.transfer.model.UploadResult
import com.amazonaws.services.s3.transfer.{TransferManager => AmazonTransferManager} import com.amazonaws.services.s3.transfer.{TransferManager => AmazonTransferManager}
import net.kemitix.s3thorp.aws.api.S3Action.{ErroredS3Action, UploadS3Action} import net.kemitix.thorp.aws.api.S3Action.{ErroredS3Action, UploadS3Action}
import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent} import net.kemitix.thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent}
import net.kemitix.s3thorp.aws.api.{S3Action, UploadProgressListener} import net.kemitix.thorp.aws.api.{S3Action, UploadProgressListener}
import net.kemitix.s3thorp.aws.lib.UploaderLogging.{logMultiPartUploadFinished, logMultiPartUploadStart} import net.kemitix.thorp.aws.lib.UploaderLogging.{logMultiPartUploadFinished, logMultiPartUploadStart}
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey} import net.kemitix.thorp.domain._
import scala.util.Try import scala.util.Try

View file

@ -1,9 +1,9 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import cats.Monad import cats.Monad
import net.kemitix.s3thorp.domain.Terminal.clearLine import net.kemitix.thorp.domain.SizeTranslation.sizeInEnglish
import net.kemitix.s3thorp.domain.SizeTranslation.sizeInEnglish import net.kemitix.thorp.domain.Terminal.clearLine
import net.kemitix.s3thorp.domain.{LocalFile, Logger} import net.kemitix.thorp.domain.{LocalFile, Logger}
object UploaderLogging { object UploaderLogging {

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.aws.lib
import cats.Monad import cats.Monad
import net.kemitix.s3thorp.domain.Logger import net.kemitix.thorp.domain.Logger
class DummyLogger[M[_]: Monad] extends Logger[M] { class DummyLogger[M[_]: Monad] extends Logger[M] {

View file

@ -1,6 +1,6 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.aws.lib
import net.kemitix.s3thorp.domain.MD5Hash import net.kemitix.thorp.domain.MD5Hash
object MD5HashData { object MD5HashData {

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import java.time.Instant import java.time.Instant
@ -7,11 +7,11 @@ import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.PutObjectRequest import com.amazonaws.services.s3.model.PutObjectRequest
import com.amazonaws.services.s3.transfer.model.UploadResult import com.amazonaws.services.s3.transfer.model.UploadResult
import com.amazonaws.services.s3.transfer.{TransferManager, Upload} import com.amazonaws.services.s3.transfer.{TransferManager, Upload}
import net.kemitix.s3thorp.aws.api.S3Action.UploadS3Action import net.kemitix.thorp.aws.api.S3Action.UploadS3Action
import net.kemitix.s3thorp.aws.api.{S3Client, UploadProgressListener} import net.kemitix.thorp.aws.api.{S3Client, UploadProgressListener}
import net.kemitix.s3thorp.aws.lib.MD5HashData.rootHash import net.kemitix.thorp.aws.lib.MD5HashData.rootHash
import net.kemitix.s3thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher} import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
import org.scalamock.scalatest.MockFactory import org.scalamock.scalatest.MockFactory
import org.scalatest.FunSpec import org.scalatest.FunSpec

View file

@ -1,11 +1,11 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import java.time.Instant import java.time.Instant
import java.time.temporal.ChronoUnit import java.time.temporal.ChronoUnit
import java.util.Date import java.util.Date
import com.amazonaws.services.s3.model.S3ObjectSummary import com.amazonaws.services.s3.model.S3ObjectSummary
import net.kemitix.s3thorp.domain.{KeyModified, LastModified, MD5Hash, RemoteKey} import net.kemitix.thorp.domain.{KeyModified, LastModified, MD5Hash, RemoteKey}
import org.scalatest.FunSpec import org.scalatest.FunSpec
class S3ObjectsByHashSuite extends FunSpec { class S3ObjectsByHashSuite extends FunSpec {

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import java.time.Instant import java.time.Instant
import java.time.temporal.ChronoUnit import java.time.temporal.ChronoUnit
@ -8,8 +8,8 @@ import cats.Id
import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.{ListObjectsV2Request, ListObjectsV2Result, S3ObjectSummary} import com.amazonaws.services.s3.model.{ListObjectsV2Request, ListObjectsV2Result, S3ObjectSummary}
import com.amazonaws.services.s3.transfer.TransferManager import com.amazonaws.services.s3.transfer.TransferManager
import net.kemitix.s3thorp.core.Resource import net.kemitix.thorp.core.Resource
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
import org.scalamock.scalatest.MockFactory import org.scalamock.scalatest.MockFactory
import org.scalatest.FunSpec import org.scalatest.FunSpec

View file

@ -1,15 +1,15 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.aws.lib
import java.time.Instant import java.time.Instant
import cats.Id import cats.Id
import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.transfer._ import com.amazonaws.services.s3.transfer._
import net.kemitix.s3thorp.aws.api.S3Action.UploadS3Action import net.kemitix.thorp.aws.api.S3Action.UploadS3Action
import net.kemitix.s3thorp.aws.api.UploadProgressListener import net.kemitix.thorp.aws.api.UploadProgressListener
import net.kemitix.s3thorp.core.KeyGenerator.generateKey import net.kemitix.thorp.core.KeyGenerator.generateKey
import net.kemitix.s3thorp.core.Resource import net.kemitix.thorp.core.Resource
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
import org.scalamock.scalatest.MockFactory import org.scalamock.scalatest.MockFactory
import org.scalatest.FunSpec import org.scalatest.FunSpec

View file

@ -1,4 +1,4 @@
#!/usr/bin/env sh #!/usr/bin/env sh
THORP_HOME=`dirname $0` THORP_HOME=`dirname $0`
eval `resize` eval `resize`
java -jar $THORP_HOME/s3thorp.jar $* java -jar $THORP_HOME/thorp.jar $*

View file

@ -6,7 +6,7 @@ val commonSettings = Seq(
) )
val applicationSettings = Seq( val applicationSettings = Seq(
name := "s3thorp", name := "thorp",
) )
val testDependencies = Seq( val testDependencies = Seq(
libraryDependencies ++= Seq( libraryDependencies ++= Seq(
@ -58,7 +58,7 @@ val catsEffectsSettings = Seq(
lazy val cli = (project in file("cli")) lazy val cli = (project in file("cli"))
.settings(commonSettings) .settings(commonSettings)
.settings(mainClass in assembly := Some("net.kemitix.s3thorp.cli.Main")) .settings(mainClass in assembly := Some("net.kemitix.thorp.cli.Main"))
.settings(applicationSettings) .settings(applicationSettings)
.settings(catsEffectsSettings) .settings(catsEffectsSettings)
.aggregate(`aws-lib`, core, `aws-api`, domain) .aggregate(`aws-lib`, core, `aws-api`, domain)

View file

@ -1,10 +1,10 @@
package net.kemitix.s3thorp.cli package net.kemitix.thorp.cli
import java.nio.file.Paths import java.nio.file.Paths
import cats.effect.ExitCase.{Canceled, Completed, Error} import cats.effect.ExitCase.{Canceled, Completed, Error}
import cats.effect.{ExitCode, IO, IOApp} import cats.effect.{ExitCode, IO, IOApp}
import net.kemitix.s3thorp.domain.Config import net.kemitix.thorp.domain.Config
object Main extends IOApp { object Main extends IOApp {

View file

@ -1,10 +1,10 @@
package net.kemitix.s3thorp.cli package net.kemitix.thorp.cli
import java.io.File import java.io.File
import java.nio.file.Paths import java.nio.file.Paths
import net.kemitix.s3thorp.domain.Filter.{Exclude, Include} import net.kemitix.thorp.domain.Filter.{Exclude, Include}
import net.kemitix.s3thorp.domain.{Bucket, Config, RemoteKey} import net.kemitix.thorp.domain.{Bucket, Config, RemoteKey}
import scopt.OParser import scopt.OParser
object ParseArgs { object ParseArgs {
@ -13,8 +13,8 @@ object ParseArgs {
val parserBuilder = OParser.builder[Config] val parserBuilder = OParser.builder[Config]
import parserBuilder._ import parserBuilder._
OParser.sequence( OParser.sequence(
programName("s3thorp"), programName("thorp"),
head("s3thorp"), head("thorp"),
opt[String]('s', "source") opt[String]('s', "source")
.action((str, c) => c.copy(source = Paths.get(str).toFile)) .action((str, c) => c.copy(source = Paths.get(str).toFile))
.validate(s => if (new File(s).isDirectory) Right(()) else Left("Source is not a directory")) .validate(s => if (new File(s).isDirectory) Right(()) else Left("Source is not a directory"))

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.cli package net.kemitix.thorp.cli
import cats.Monad import cats.Monad
import net.kemitix.s3thorp.domain.Logger import net.kemitix.thorp.domain.Logger
class PrintLogger[M[_]: Monad](isDebug: Boolean) extends Logger[M] { class PrintLogger[M[_]: Monad](isDebug: Boolean) extends Logger[M] {

View file

@ -1,18 +1,18 @@
package net.kemitix.s3thorp.cli package net.kemitix.thorp.cli
import cats.Monad import cats.Monad
import cats.effect.ExitCode import cats.effect.ExitCode
import cats.implicits._ import cats.implicits._
import net.kemitix.s3thorp.aws.lib.S3ClientBuilder import net.kemitix.thorp.aws.lib.S3ClientBuilder
import net.kemitix.s3thorp.core.Sync import net.kemitix.thorp.core.Sync
import net.kemitix.s3thorp.domain.{Config, Logger} import net.kemitix.thorp.domain.{Config, Logger}
object Program { object Program {
def apply[M[_]: Monad](config: Config): M[ExitCode] = { def apply[M[_]: Monad](config: Config): M[ExitCode] = {
implicit val logger: Logger[M] = new PrintLogger[M](config.debug) implicit val logger: Logger[M] = new PrintLogger[M](config.debug)
for { for {
_ <- logger.info("S3Thorp - hashed sync for s3") _ <- logger.info("Thorp - hashed sync for cloud storage")
_ <- Sync.run[M](config, S3ClientBuilder.defaultClient) _ <- Sync.run[M](config, S3ClientBuilder.defaultClient)
} yield ExitCode.Success } yield ExitCode.Success
} }

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.cli package net.kemitix.thorp.cli
import net.kemitix.s3thorp.core.Resource import net.kemitix.thorp.core.Resource
import net.kemitix.s3thorp.domain.{Bucket, Config} import net.kemitix.thorp.domain.Config
import org.scalatest.FunSpec import org.scalatest.FunSpec
import scala.util.Try import scala.util.Try

View file

@ -1,6 +1,6 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey} import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
sealed trait Action { sealed trait Action {
def bucket: Bucket def bucket: Bucket

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToUpload} import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
object ActionGenerator { object ActionGenerator {

View file

@ -1,11 +1,11 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
import net.kemitix.s3thorp.aws.api.S3Action.DoNothingS3Action import net.kemitix.thorp.aws.api.S3Action.DoNothingS3Action
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client, UploadProgressListener} import net.kemitix.thorp.aws.api.{S3Action, S3Client, UploadProgressListener}
import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload} import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
import net.kemitix.s3thorp.domain.{Config, Logger} import net.kemitix.thorp.domain.{Config, Logger}
object ActionSubmitter { object ActionSubmitter {

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
final case class Counters(uploaded: Int = 0, final case class Counters(uploaded: Int = 0,
deleted: Int = 0, deleted: Int = 0,

View file

@ -1,8 +1,8 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import java.io.File import java.io.File
import net.kemitix.s3thorp.domain.RemoteKey import net.kemitix.thorp.domain.RemoteKey
object KeyGenerator { object KeyGenerator {

View file

@ -1,12 +1,13 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import java.io.File import java.io.File
import java.nio.file.Path import java.nio.file.Path
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
import net.kemitix.s3thorp.core.KeyGenerator.generateKey import net.kemitix.thorp.core.KeyGenerator.generateKey
import net.kemitix.s3thorp.domain.{Config, Filter, LocalFile, Logger, MD5Hash} import net.kemitix.thorp.domain
import net.kemitix.thorp.domain._
object LocalFileStream { object LocalFileStream {
@ -32,7 +33,7 @@ object LocalFileStream {
file match { file match {
case f if f.isDirectory => loop(file) case f if f.isDirectory => loop(file)
case _ => for(hash <- md5HashGenerator(file)) case _ => for(hash <- md5HashGenerator(file))
yield Stream(LocalFile(file, c.source, hash, generateKey(c.source, c.prefix))) yield Stream(domain.LocalFile(file, c.source, hash, generateKey(c.source, c.prefix)))
} }
def recurse(fs: Stream[File]): M[Stream[LocalFile]] = def recurse(fs: Stream[File]): M[Stream[LocalFile]] =

View file

@ -1,11 +1,11 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import java.io.{File, FileInputStream} import java.io.{File, FileInputStream}
import java.security.MessageDigest import java.security.MessageDigest
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
import net.kemitix.s3thorp.domain.{Logger, MD5Hash} import net.kemitix.thorp.domain.{Logger, MD5Hash}
import scala.collection.immutable.NumericRange import scala.collection.immutable.NumericRange

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import java.io.{File, FileNotFoundException} import java.io.{File, FileNotFoundException}

View file

@ -1,6 +1,6 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
object S3MetaDataEnricher { object S3MetaDataEnricher {

View file

@ -1,15 +1,15 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client} import net.kemitix.thorp.aws.api.{S3Action, S3Client}
import net.kemitix.s3thorp.core.Action.ToDelete import net.kemitix.thorp.core.Action.ToDelete
import net.kemitix.s3thorp.core.ActionGenerator.createActions import net.kemitix.thorp.core.ActionGenerator.createActions
import net.kemitix.s3thorp.core.ActionSubmitter.submitAction import net.kemitix.thorp.core.ActionSubmitter.submitAction
import net.kemitix.s3thorp.core.LocalFileStream.findFiles import net.kemitix.thorp.core.LocalFileStream.findFiles
import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata import net.kemitix.thorp.core.S3MetaDataEnricher.getMetadata
import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart} import net.kemitix.thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart}
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
object Sync { object Sync {

View file

@ -1,10 +1,10 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import cats.Monad import cats.Monad
import cats.implicits._ import cats.implicits._
import net.kemitix.s3thorp.aws.api.S3Action import net.kemitix.thorp.aws.api.S3Action
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, ErroredS3Action, UploadS3Action} import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, ErroredS3Action, UploadS3Action}
import net.kemitix.s3thorp.domain.{Config, Logger} import net.kemitix.thorp.domain.{Config, Logger}
// Logging for the Sync class // Logging for the Sync class
object SyncLogging { object SyncLogging {

View file

@ -1,9 +1,9 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import java.time.Instant import java.time.Instant
import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToUpload} import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
import org.scalatest.FunSpec import org.scalatest.FunSpec
class ActionGeneratorSuite class ActionGeneratorSuite

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.core
import cats.Monad import cats.Monad
import net.kemitix.s3thorp.domain.Logger import net.kemitix.thorp.domain.Logger
class DummyLogger[M[_]: Monad] extends Logger[M] { class DummyLogger[M[_]: Monad] extends Logger[M] {

View file

@ -1,8 +1,8 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import java.io.File import java.io.File
import net.kemitix.s3thorp.domain.{Bucket, Config, RemoteKey} import net.kemitix.thorp.domain.{Bucket, Config, RemoteKey}
import org.scalatest.FunSpec import org.scalatest.FunSpec
class KeyGeneratorSuite extends FunSpec { class KeyGeneratorSuite extends FunSpec {

View file

@ -1,9 +1,9 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import java.io.File import java.io.File
import cats.Id import cats.Id
import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash} import net.kemitix.thorp.domain.{Config, LocalFile, Logger, MD5Hash}
import org.scalatest.FunSpec import org.scalatest.FunSpec
class LocalFileStreamSuite extends FunSpec { class LocalFileStreamSuite extends FunSpec {

View file

@ -1,6 +1,6 @@
package net.kemitix.s3thorp.aws.lib package net.kemitix.thorp.core
import net.kemitix.s3thorp.domain.MD5Hash import net.kemitix.thorp.domain.MD5Hash
object MD5HashData { object MD5HashData {

View file

@ -1,8 +1,8 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import cats.Id import cats.Id
import net.kemitix.s3thorp.core.MD5HashData.rootHash import net.kemitix.thorp.core.MD5HashData.rootHash
import net.kemitix.s3thorp.domain.{Bucket, Config, Logger, MD5Hash, RemoteKey} import net.kemitix.thorp.domain._
import org.scalatest.FunSpec import org.scalatest.FunSpec
class MD5HashGeneratorTest extends FunSpec { class MD5HashGeneratorTest extends FunSpec {

View file

@ -1,7 +1,7 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, UploadS3Action} import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, UploadS3Action}
import net.kemitix.s3thorp.domain.{MD5Hash, RemoteKey} import net.kemitix.thorp.domain.{MD5Hash, RemoteKey}
import org.scalatest.FunSpec import org.scalatest.FunSpec
class S3ActionSuite extends FunSpec { class S3ActionSuite extends FunSpec {

View file

@ -1,9 +1,9 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import java.time.Instant import java.time.Instant
import net.kemitix.s3thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status} import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
import org.scalatest.FunSpec import org.scalatest.FunSpec
class S3MetaDataEnricherSuite class S3MetaDataEnricherSuite

View file

@ -1,14 +1,14 @@
package net.kemitix.s3thorp.core package net.kemitix.thorp.core
import java.io.File import java.io.File
import java.time.Instant import java.time.Instant
import cats.Id import cats.Id
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, UploadS3Action} import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, UploadS3Action}
import net.kemitix.s3thorp.aws.api.{S3Client, UploadProgressListener} import net.kemitix.thorp.aws.api.{S3Client, UploadProgressListener}
import net.kemitix.s3thorp.core.MD5HashData.{leafHash, rootHash} import net.kemitix.thorp.core.MD5HashData.{leafHash, rootHash}
import net.kemitix.s3thorp.domain.Filter.Exclude import net.kemitix.thorp.domain.Filter.Exclude
import net.kemitix.s3thorp.domain._ import net.kemitix.thorp.domain._
import org.scalatest.FunSpec import org.scalatest.FunSpec
class SyncSuite class SyncSuite

View file

@ -1,9 +0,0 @@
package net.kemitix.s3thorp.domain
import net.kemitix.s3thorp.domain.QuoteStripper.stripQuotes
final case class MD5Hash(in: String) {
lazy val hash: String = in filter stripQuotes
}

View file

@ -1,3 +1,3 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
final case class Bucket(name: String) final case class Bucket(name: String)

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
import java.io.File import java.io.File

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
import java.nio.file.Path import java.nio.file.Path
import java.util.regex.Pattern import java.util.regex.Pattern

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
final case class HashModified(hash: MD5Hash, final case class HashModified(hash: MD5Hash,
modified: LastModified) modified: LastModified)

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
final case class KeyModified(key: RemoteKey, final case class KeyModified(key: RemoteKey,
modified: LastModified) modified: LastModified)

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
import java.time.Instant import java.time.Instant

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
import java.io.File import java.io.File
import java.nio.file.Path import java.nio.file.Path

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
trait Logger[M[_]] { trait Logger[M[_]] {

View file

@ -0,0 +1,9 @@
package net.kemitix.thorp.domain
import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
final case class MD5Hash(in: String) {
lazy val hash: String = in filter stripQuotes
}

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
object QuoteStripper { object QuoteStripper {

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
import java.io.File import java.io.File
import java.nio.file.Paths import java.nio.file.Paths

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
final case class RemoteMetaData(remoteKey: RemoteKey, final case class RemoteMetaData(remoteKey: RemoteKey,
hash: MD5Hash, hash: MD5Hash,

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key // For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
final case class S3MetaData( final case class S3MetaData(

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
/** /**
* A list of objects and their MD5 hash values. * A list of objects and their MD5 hash values.

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
object SizeTranslation { object SizeTranslation {

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
object Terminal { object Terminal {

View file

@ -1,8 +1,8 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
import java.nio.file.{Path, Paths} import java.nio.file.Paths
import net.kemitix.s3thorp.domain.Filter.{Exclude, Include} import net.kemitix.thorp.domain.Filter.{Exclude, Include}
import org.scalatest.FunSpec import org.scalatest.FunSpec
class FiltersSuite extends FunSpec { class FiltersSuite extends FunSpec {

View file

@ -1,4 +1,4 @@
package net.kemitix.s3thorp.domain package net.kemitix.thorp.domain
import org.scalatest.FunSpec import org.scalatest.FunSpec