Rename project to Thorp (#75)
* [sbt] change application name * [cli] rename package * [cli] Change displayed application name and description * [domain] rename package * [core] fix bad package directory structure * [core] rename package * [aws-lib] rename package * [aws-api] rename package * [cli] rename programe for usage message * [bin] rename and update script * [gitignore] update * [readme] update
This commit is contained in:
parent
6450bbeff5
commit
9196dd623f
78 changed files with 190 additions and 189 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -28,5 +28,5 @@ project/plugins/project/
|
|||
### Metals ###
|
||||
.bloop/
|
||||
.metals/
|
||||
/s3thorp
|
||||
/thorp
|
||||
*.zip
|
||||
|
|
14
README.org
14
README.org
|
@ -1,8 +1,8 @@
|
|||
* s3thorp
|
||||
* thorp
|
||||
|
||||
Synchronisation of files with S3 using the hash of the file contents.
|
||||
|
||||
[[https://www.codacy.com/app/kemitix/s3thorp][file:https://img.shields.io/codacy/grade/14ea6ad0825249c994a27a82d3485180.svg?style=for-the-badge]]
|
||||
[[https://www.codacy.com/app/kemitix/thorp][file:https://img.shields.io/codacy/grade/14ea6ad0825249c994a27a82d3485180.svg?style=for-the-badge]]
|
||||
|
||||
Originally based on Alex Kudlick's [[https://github.com/akud/aws-s3-sync-by-hash][aws-s3-sync-by-hash]].
|
||||
|
||||
|
@ -13,8 +13,8 @@ hash of the file contents.
|
|||
* Usage
|
||||
|
||||
#+begin_example
|
||||
s3thorp
|
||||
Usage: s3thorp [options]
|
||||
thorp
|
||||
Usage: thorp [options]
|
||||
|
||||
-s, --source <value> Source directory to sync to S3
|
||||
-b, --bucket <value> S3 bucket name
|
||||
|
@ -46,8 +46,8 @@ When considering a local file, the following table governs what should happen:
|
|||
To build as an executable jar, perform `sbt assembly`
|
||||
|
||||
This will create the file
|
||||
`cli/target/scala-2.12/s3thorp-assembly-$VERSION.jar` (where $VERSION
|
||||
`cli/target/scala-2.12/thorp-assembly-$VERSION.jar` (where $VERSION
|
||||
is substituted)
|
||||
|
||||
Copy and rename this file as `s3thorp.jar` into the same directory as
|
||||
the `bin/s3throp` shell script.
|
||||
Copy and rename this file as `thorp.jar` into the same directory as
|
||||
the `bin/throp` shell script.
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
package net.kemitix.s3thorp.aws.api
|
||||
package net.kemitix.thorp.aws.api
|
||||
|
||||
import net.kemitix.s3thorp.domain.{MD5Hash, RemoteKey}
|
||||
import net.kemitix.thorp.domain.{MD5Hash, RemoteKey}
|
||||
|
||||
sealed trait S3Action {
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.aws.api
|
||||
package net.kemitix.thorp.aws.api
|
||||
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action}
|
||||
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey, S3ObjectsData}
|
||||
import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action}
|
||||
import net.kemitix.thorp.domain._
|
||||
|
||||
trait S3Client[M[_]] {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.aws.api
|
||||
package net.kemitix.thorp.aws.api
|
||||
|
||||
sealed trait UploadEvent {
|
||||
def name: String
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.aws.api
|
||||
package net.kemitix.thorp.aws.api
|
||||
|
||||
import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent}
|
||||
import net.kemitix.s3thorp.domain.LocalFile
|
||||
import net.kemitix.thorp.aws.api.UploadEvent.RequestEvent
|
||||
import net.kemitix.thorp.domain.LocalFile
|
||||
|
||||
class UploadProgressListener(localFile: LocalFile)
|
||||
extends UploadProgressLogging {
|
|
@ -1,9 +1,9 @@
|
|||
package net.kemitix.s3thorp.aws.api
|
||||
package net.kemitix.thorp.aws.api
|
||||
|
||||
import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent}
|
||||
import net.kemitix.s3thorp.domain.Terminal.{clearLine, returnToPreviousLine}
|
||||
import net.kemitix.s3thorp.domain.{LocalFile, Terminal}
|
||||
import net.kemitix.s3thorp.domain.SizeTranslation.sizeInEnglish
|
||||
import net.kemitix.thorp.aws.api.UploadEvent.RequestEvent
|
||||
import net.kemitix.thorp.domain.SizeTranslation.sizeInEnglish
|
||||
import net.kemitix.thorp.domain.Terminal.{clearLine, returnToPreviousLine}
|
||||
import net.kemitix.thorp.domain.{LocalFile, Terminal}
|
||||
|
||||
import scala.io.AnsiColor._
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import cats.Monad
|
||||
import com.amazonaws.services.s3.transfer.{TransferManager, TransferManagerBuilder}
|
||||
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
|
||||
import net.kemitix.s3thorp.aws.api.S3Client
|
||||
import net.kemitix.thorp.aws.api.S3Client
|
||||
|
||||
object S3ClientBuilder {
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import cats.Monad
|
||||
import cats.implicits._
|
||||
import com.amazonaws.services.s3.AmazonS3
|
||||
import com.amazonaws.services.s3.model.CopyObjectRequest
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.CopyS3Action
|
||||
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logCopyFinish, logCopyStart}
|
||||
import net.kemitix.s3thorp.domain.{Bucket, Logger, MD5Hash, RemoteKey}
|
||||
import net.kemitix.thorp.aws.api.S3Action.CopyS3Action
|
||||
import net.kemitix.thorp.aws.lib.S3ClientLogging.{logCopyFinish, logCopyStart}
|
||||
import net.kemitix.thorp.domain.{Bucket, Logger, MD5Hash, RemoteKey}
|
||||
|
||||
class S3ClientCopier[M[_]: Monad](amazonS3: AmazonS3) {
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import cats.Monad
|
||||
import cats.implicits._
|
||||
import com.amazonaws.services.s3.AmazonS3
|
||||
import com.amazonaws.services.s3.model.DeleteObjectRequest
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.DeleteS3Action
|
||||
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logDeleteFinish, logDeleteStart}
|
||||
import net.kemitix.s3thorp.domain.{Bucket, Logger, RemoteKey}
|
||||
import net.kemitix.thorp.aws.api.S3Action.DeleteS3Action
|
||||
import net.kemitix.thorp.aws.lib.S3ClientLogging.{logDeleteFinish, logDeleteStart}
|
||||
import net.kemitix.thorp.domain.{Bucket, Logger, RemoteKey}
|
||||
|
||||
class S3ClientDeleter[M[_]: Monad](amazonS3: AmazonS3) {
|
||||
|
|
@ -1,8 +1,7 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import cats.Monad
|
||||
import com.amazonaws.services.s3.model.PutObjectResult
|
||||
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, RemoteKey}
|
||||
import net.kemitix.thorp.domain.{Bucket, Logger, RemoteKey}
|
||||
|
||||
object S3ClientLogging {
|
||||
|
|
@ -1,13 +1,14 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import cats.Monad
|
||||
import cats.implicits._
|
||||
import com.amazonaws.services.s3.AmazonS3
|
||||
import com.amazonaws.services.s3.model.{ListObjectsV2Request, S3ObjectSummary}
|
||||
import net.kemitix.s3thorp.aws.lib.S3ClientLogging.{logListObjectsFinish, logListObjectsStart}
|
||||
import net.kemitix.s3thorp.aws.lib.S3ObjectsByHash.byHash
|
||||
import net.kemitix.s3thorp.aws.lib.S3ObjectsByKey.byKey
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.aws.lib.S3ClientLogging.{logListObjectsFinish, logListObjectsStart}
|
||||
import net.kemitix.thorp.aws.lib.S3ObjectsByHash.byHash
|
||||
import net.kemitix.thorp.aws.lib.S3ObjectsByKey.byKey
|
||||
import net.kemitix.thorp.domain
|
||||
import net.kemitix.thorp.domain.{Bucket, Logger, RemoteKey, S3ObjectsData}
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
|
@ -54,7 +55,7 @@ class S3ClientObjectLister[M[_]: Monad](amazonS3: AmazonS3) {
|
|||
r = new ListObjectsV2Request().withBucketName(bucket.name).withPrefix(prefix.key)
|
||||
summaries <- fetch(r)
|
||||
_ <- logListObjectsFinish[M](bucket, prefix)
|
||||
} yield S3ObjectsData(byHash(summaries), byKey(summaries))
|
||||
} yield domain.S3ObjectsData(byHash(summaries), byKey(summaries))
|
||||
}
|
||||
|
||||
}
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary
|
||||
import net.kemitix.s3thorp.domain.{KeyModified, LastModified, MD5Hash, RemoteKey}
|
||||
import net.kemitix.thorp.domain.{KeyModified, LastModified, MD5Hash, RemoteKey}
|
||||
|
||||
object S3ObjectsByHash {
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary
|
||||
import net.kemitix.s3thorp.domain.{HashModified, LastModified, MD5Hash, RemoteKey}
|
||||
import net.kemitix.thorp.domain.{HashModified, LastModified, MD5Hash, RemoteKey}
|
||||
|
||||
object S3ObjectsByKey {
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import cats.Monad
|
||||
import com.amazonaws.services.s3.AmazonS3
|
||||
import com.amazonaws.services.s3.transfer.TransferManager
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action}
|
||||
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client, UploadProgressListener}
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action}
|
||||
import net.kemitix.thorp.aws.api.{S3Action, S3Client, UploadProgressListener}
|
||||
import net.kemitix.thorp.domain._
|
||||
|
||||
class ThorpS3Client[M[_]: Monad](amazonS3Client: => AmazonS3,
|
||||
amazonS3TransferManager: => TransferManager)
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import cats.Monad
|
||||
import cats.implicits._
|
||||
|
@ -6,11 +6,11 @@ import com.amazonaws.event.{ProgressEvent, ProgressEventType, ProgressListener}
|
|||
import com.amazonaws.services.s3.model.PutObjectRequest
|
||||
import com.amazonaws.services.s3.transfer.model.UploadResult
|
||||
import com.amazonaws.services.s3.transfer.{TransferManager => AmazonTransferManager}
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.{ErroredS3Action, UploadS3Action}
|
||||
import net.kemitix.s3thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent}
|
||||
import net.kemitix.s3thorp.aws.api.{S3Action, UploadProgressListener}
|
||||
import net.kemitix.s3thorp.aws.lib.UploaderLogging.{logMultiPartUploadFinished, logMultiPartUploadStart}
|
||||
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, Logger, MD5Hash, RemoteKey}
|
||||
import net.kemitix.thorp.aws.api.S3Action.{ErroredS3Action, UploadS3Action}
|
||||
import net.kemitix.thorp.aws.api.UploadEvent.{ByteTransferEvent, RequestEvent, TransferEvent}
|
||||
import net.kemitix.thorp.aws.api.{S3Action, UploadProgressListener}
|
||||
import net.kemitix.thorp.aws.lib.UploaderLogging.{logMultiPartUploadFinished, logMultiPartUploadStart}
|
||||
import net.kemitix.thorp.domain._
|
||||
|
||||
import scala.util.Try
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import cats.Monad
|
||||
import net.kemitix.s3thorp.domain.Terminal.clearLine
|
||||
import net.kemitix.s3thorp.domain.SizeTranslation.sizeInEnglish
|
||||
import net.kemitix.s3thorp.domain.{LocalFile, Logger}
|
||||
import net.kemitix.thorp.domain.SizeTranslation.sizeInEnglish
|
||||
import net.kemitix.thorp.domain.Terminal.clearLine
|
||||
import net.kemitix.thorp.domain.{LocalFile, Logger}
|
||||
|
||||
object UploaderLogging {
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import cats.Monad
|
||||
import net.kemitix.s3thorp.domain.Logger
|
||||
import net.kemitix.thorp.domain.Logger
|
||||
|
||||
class DummyLogger[M[_]: Monad] extends Logger[M] {
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import net.kemitix.s3thorp.domain.MD5Hash
|
||||
import net.kemitix.thorp.domain.MD5Hash
|
||||
|
||||
object MD5HashData {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
|
@ -7,11 +7,11 @@ import com.amazonaws.services.s3.AmazonS3
|
|||
import com.amazonaws.services.s3.model.PutObjectRequest
|
||||
import com.amazonaws.services.s3.transfer.model.UploadResult
|
||||
import com.amazonaws.services.s3.transfer.{TransferManager, Upload}
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.UploadS3Action
|
||||
import net.kemitix.s3thorp.aws.api.{S3Client, UploadProgressListener}
|
||||
import net.kemitix.s3thorp.aws.lib.MD5HashData.rootHash
|
||||
import net.kemitix.s3thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.aws.api.S3Action.UploadS3Action
|
||||
import net.kemitix.thorp.aws.api.{S3Client, UploadProgressListener}
|
||||
import net.kemitix.thorp.aws.lib.MD5HashData.rootHash
|
||||
import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalamock.scalatest.MockFactory
|
||||
import org.scalatest.FunSpec
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import java.time.Instant
|
||||
import java.time.temporal.ChronoUnit
|
||||
import java.util.Date
|
||||
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary
|
||||
import net.kemitix.s3thorp.domain.{KeyModified, LastModified, MD5Hash, RemoteKey}
|
||||
import net.kemitix.thorp.domain.{KeyModified, LastModified, MD5Hash, RemoteKey}
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class S3ObjectsByHashSuite extends FunSpec {
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import java.time.Instant
|
||||
import java.time.temporal.ChronoUnit
|
||||
|
@ -8,8 +8,8 @@ import cats.Id
|
|||
import com.amazonaws.services.s3.AmazonS3
|
||||
import com.amazonaws.services.s3.model.{ListObjectsV2Request, ListObjectsV2Result, S3ObjectSummary}
|
||||
import com.amazonaws.services.s3.transfer.TransferManager
|
||||
import net.kemitix.s3thorp.core.Resource
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.core.Resource
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalamock.scalatest.MockFactory
|
||||
import org.scalatest.FunSpec
|
||||
|
|
@ -1,15 +1,15 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.aws.lib
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
import cats.Id
|
||||
import com.amazonaws.services.s3.AmazonS3
|
||||
import com.amazonaws.services.s3.transfer._
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.UploadS3Action
|
||||
import net.kemitix.s3thorp.aws.api.UploadProgressListener
|
||||
import net.kemitix.s3thorp.core.KeyGenerator.generateKey
|
||||
import net.kemitix.s3thorp.core.Resource
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.aws.api.S3Action.UploadS3Action
|
||||
import net.kemitix.thorp.aws.api.UploadProgressListener
|
||||
import net.kemitix.thorp.core.KeyGenerator.generateKey
|
||||
import net.kemitix.thorp.core.Resource
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalamock.scalatest.MockFactory
|
||||
import org.scalatest.FunSpec
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env sh
|
||||
THORP_HOME=`dirname $0`
|
||||
eval `resize`
|
||||
java -jar $THORP_HOME/s3thorp.jar $*
|
||||
java -jar $THORP_HOME/thorp.jar $*
|
|
@ -6,7 +6,7 @@ val commonSettings = Seq(
|
|||
)
|
||||
|
||||
val applicationSettings = Seq(
|
||||
name := "s3thorp",
|
||||
name := "thorp",
|
||||
)
|
||||
val testDependencies = Seq(
|
||||
libraryDependencies ++= Seq(
|
||||
|
@ -58,7 +58,7 @@ val catsEffectsSettings = Seq(
|
|||
|
||||
lazy val cli = (project in file("cli"))
|
||||
.settings(commonSettings)
|
||||
.settings(mainClass in assembly := Some("net.kemitix.s3thorp.cli.Main"))
|
||||
.settings(mainClass in assembly := Some("net.kemitix.thorp.cli.Main"))
|
||||
.settings(applicationSettings)
|
||||
.settings(catsEffectsSettings)
|
||||
.aggregate(`aws-lib`, core, `aws-api`, domain)
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
package net.kemitix.s3thorp.cli
|
||||
package net.kemitix.thorp.cli
|
||||
|
||||
import java.nio.file.Paths
|
||||
|
||||
import cats.effect.ExitCase.{Canceled, Completed, Error}
|
||||
import cats.effect.{ExitCode, IO, IOApp}
|
||||
import net.kemitix.s3thorp.domain.Config
|
||||
import net.kemitix.thorp.domain.Config
|
||||
|
||||
object Main extends IOApp {
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
package net.kemitix.s3thorp.cli
|
||||
package net.kemitix.thorp.cli
|
||||
|
||||
import java.io.File
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.s3thorp.domain.Filter.{Exclude, Include}
|
||||
import net.kemitix.s3thorp.domain.{Bucket, Config, RemoteKey}
|
||||
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
|
||||
import net.kemitix.thorp.domain.{Bucket, Config, RemoteKey}
|
||||
import scopt.OParser
|
||||
|
||||
object ParseArgs {
|
||||
|
@ -13,8 +13,8 @@ object ParseArgs {
|
|||
val parserBuilder = OParser.builder[Config]
|
||||
import parserBuilder._
|
||||
OParser.sequence(
|
||||
programName("s3thorp"),
|
||||
head("s3thorp"),
|
||||
programName("thorp"),
|
||||
head("thorp"),
|
||||
opt[String]('s', "source")
|
||||
.action((str, c) => c.copy(source = Paths.get(str).toFile))
|
||||
.validate(s => if (new File(s).isDirectory) Right(()) else Left("Source is not a directory"))
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.cli
|
||||
package net.kemitix.thorp.cli
|
||||
|
||||
import cats.Monad
|
||||
import net.kemitix.s3thorp.domain.Logger
|
||||
import net.kemitix.thorp.domain.Logger
|
||||
|
||||
class PrintLogger[M[_]: Monad](isDebug: Boolean) extends Logger[M] {
|
||||
|
|
@ -1,18 +1,18 @@
|
|||
package net.kemitix.s3thorp.cli
|
||||
package net.kemitix.thorp.cli
|
||||
|
||||
import cats.Monad
|
||||
import cats.effect.ExitCode
|
||||
import cats.implicits._
|
||||
import net.kemitix.s3thorp.aws.lib.S3ClientBuilder
|
||||
import net.kemitix.s3thorp.core.Sync
|
||||
import net.kemitix.s3thorp.domain.{Config, Logger}
|
||||
import net.kemitix.thorp.aws.lib.S3ClientBuilder
|
||||
import net.kemitix.thorp.core.Sync
|
||||
import net.kemitix.thorp.domain.{Config, Logger}
|
||||
|
||||
object Program {
|
||||
|
||||
def apply[M[_]: Monad](config: Config): M[ExitCode] = {
|
||||
implicit val logger: Logger[M] = new PrintLogger[M](config.debug)
|
||||
for {
|
||||
_ <- logger.info("S3Thorp - hashed sync for s3")
|
||||
_ <- logger.info("Thorp - hashed sync for cloud storage")
|
||||
_ <- Sync.run[M](config, S3ClientBuilder.defaultClient)
|
||||
} yield ExitCode.Success
|
||||
}
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.cli
|
||||
package net.kemitix.thorp.cli
|
||||
|
||||
import net.kemitix.s3thorp.core.Resource
|
||||
import net.kemitix.s3thorp.domain.{Bucket, Config}
|
||||
import net.kemitix.thorp.core.Resource
|
||||
import net.kemitix.thorp.domain.Config
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
import scala.util.Try
|
|
@ -1,6 +1,6 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
|
||||
import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
|
||||
|
||||
sealed trait Action {
|
||||
def bucket: Bucket
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||
import net.kemitix.thorp.domain._
|
||||
|
||||
object ActionGenerator {
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import cats.Monad
|
||||
import cats.implicits._
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.DoNothingS3Action
|
||||
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client, UploadProgressListener}
|
||||
import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
||||
import net.kemitix.s3thorp.domain.{Config, Logger}
|
||||
import net.kemitix.thorp.aws.api.S3Action.DoNothingS3Action
|
||||
import net.kemitix.thorp.aws.api.{S3Action, S3Client, UploadProgressListener}
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
||||
import net.kemitix.thorp.domain.{Config, Logger}
|
||||
|
||||
object ActionSubmitter {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
final case class Counters(uploaded: Int = 0,
|
||||
deleted: Int = 0,
|
|
@ -1,8 +1,8 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import java.io.File
|
||||
|
||||
import net.kemitix.s3thorp.domain.RemoteKey
|
||||
import net.kemitix.thorp.domain.RemoteKey
|
||||
|
||||
object KeyGenerator {
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import java.io.File
|
||||
import java.nio.file.Path
|
||||
|
||||
import cats.Monad
|
||||
import cats.implicits._
|
||||
import net.kemitix.s3thorp.core.KeyGenerator.generateKey
|
||||
import net.kemitix.s3thorp.domain.{Config, Filter, LocalFile, Logger, MD5Hash}
|
||||
import net.kemitix.thorp.core.KeyGenerator.generateKey
|
||||
import net.kemitix.thorp.domain
|
||||
import net.kemitix.thorp.domain._
|
||||
|
||||
object LocalFileStream {
|
||||
|
||||
|
@ -32,7 +33,7 @@ object LocalFileStream {
|
|||
file match {
|
||||
case f if f.isDirectory => loop(file)
|
||||
case _ => for(hash <- md5HashGenerator(file))
|
||||
yield Stream(LocalFile(file, c.source, hash, generateKey(c.source, c.prefix)))
|
||||
yield Stream(domain.LocalFile(file, c.source, hash, generateKey(c.source, c.prefix)))
|
||||
}
|
||||
|
||||
def recurse(fs: Stream[File]): M[Stream[LocalFile]] =
|
|
@ -1,11 +1,11 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import java.io.{File, FileInputStream}
|
||||
import java.security.MessageDigest
|
||||
|
||||
import cats.Monad
|
||||
import cats.implicits._
|
||||
import net.kemitix.s3thorp.domain.{Logger, MD5Hash}
|
||||
import net.kemitix.thorp.domain.{Logger, MD5Hash}
|
||||
|
||||
import scala.collection.immutable.NumericRange
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import java.io.{File, FileNotFoundException}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.domain._
|
||||
|
||||
object S3MetaDataEnricher {
|
||||
|
|
@ -1,15 +1,15 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import cats.Monad
|
||||
import cats.implicits._
|
||||
import net.kemitix.s3thorp.aws.api.{S3Action, S3Client}
|
||||
import net.kemitix.s3thorp.core.Action.ToDelete
|
||||
import net.kemitix.s3thorp.core.ActionGenerator.createActions
|
||||
import net.kemitix.s3thorp.core.ActionSubmitter.submitAction
|
||||
import net.kemitix.s3thorp.core.LocalFileStream.findFiles
|
||||
import net.kemitix.s3thorp.core.S3MetaDataEnricher.getMetadata
|
||||
import net.kemitix.s3thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart}
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.aws.api.{S3Action, S3Client}
|
||||
import net.kemitix.thorp.core.Action.ToDelete
|
||||
import net.kemitix.thorp.core.ActionGenerator.createActions
|
||||
import net.kemitix.thorp.core.ActionSubmitter.submitAction
|
||||
import net.kemitix.thorp.core.LocalFileStream.findFiles
|
||||
import net.kemitix.thorp.core.S3MetaDataEnricher.getMetadata
|
||||
import net.kemitix.thorp.core.SyncLogging.{logFileScan, logRunFinished, logRunStart}
|
||||
import net.kemitix.thorp.domain._
|
||||
|
||||
object Sync {
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import cats.Monad
|
||||
import cats.implicits._
|
||||
import net.kemitix.s3thorp.aws.api.S3Action
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, ErroredS3Action, UploadS3Action}
|
||||
import net.kemitix.s3thorp.domain.{Config, Logger}
|
||||
import net.kemitix.thorp.aws.api.S3Action
|
||||
import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, ErroredS3Action, UploadS3Action}
|
||||
import net.kemitix.thorp.domain.{Config, Logger}
|
||||
|
||||
// Logging for the Sync class
|
||||
object SyncLogging {
|
|
@ -1,9 +1,9 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
import net.kemitix.s3thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class ActionGeneratorSuite
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import cats.Monad
|
||||
import net.kemitix.s3thorp.domain.Logger
|
||||
import net.kemitix.thorp.domain.Logger
|
||||
|
||||
class DummyLogger[M[_]: Monad] extends Logger[M] {
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import java.io.File
|
||||
|
||||
import net.kemitix.s3thorp.domain.{Bucket, Config, RemoteKey}
|
||||
import net.kemitix.thorp.domain.{Bucket, Config, RemoteKey}
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class KeyGeneratorSuite extends FunSpec {
|
|
@ -1,9 +1,9 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import java.io.File
|
||||
|
||||
import cats.Id
|
||||
import net.kemitix.s3thorp.domain.{Config, LocalFile, Logger, MD5Hash}
|
||||
import net.kemitix.thorp.domain.{Config, LocalFile, Logger, MD5Hash}
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class LocalFileStreamSuite extends FunSpec {
|
|
@ -1,6 +1,6 @@
|
|||
package net.kemitix.s3thorp.aws.lib
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.s3thorp.domain.MD5Hash
|
||||
import net.kemitix.thorp.domain.MD5Hash
|
||||
|
||||
object MD5HashData {
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import cats.Id
|
||||
import net.kemitix.s3thorp.core.MD5HashData.rootHash
|
||||
import net.kemitix.s3thorp.domain.{Bucket, Config, Logger, MD5Hash, RemoteKey}
|
||||
import net.kemitix.thorp.core.MD5HashData.rootHash
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class MD5HashGeneratorTest extends FunSpec {
|
|
@ -1,7 +1,7 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, UploadS3Action}
|
||||
import net.kemitix.s3thorp.domain.{MD5Hash, RemoteKey}
|
||||
import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, UploadS3Action}
|
||||
import net.kemitix.thorp.domain.{MD5Hash, RemoteKey}
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class S3ActionSuite extends FunSpec {
|
|
@ -1,9 +1,9 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
import net.kemitix.s3thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class S3MetaDataEnricherSuite
|
|
@ -1,14 +1,14 @@
|
|||
package net.kemitix.s3thorp.core
|
||||
package net.kemitix.thorp.core
|
||||
|
||||
import java.io.File
|
||||
import java.time.Instant
|
||||
|
||||
import cats.Id
|
||||
import net.kemitix.s3thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, UploadS3Action}
|
||||
import net.kemitix.s3thorp.aws.api.{S3Client, UploadProgressListener}
|
||||
import net.kemitix.s3thorp.core.MD5HashData.{leafHash, rootHash}
|
||||
import net.kemitix.s3thorp.domain.Filter.Exclude
|
||||
import net.kemitix.s3thorp.domain._
|
||||
import net.kemitix.thorp.aws.api.S3Action.{CopyS3Action, DeleteS3Action, UploadS3Action}
|
||||
import net.kemitix.thorp.aws.api.{S3Client, UploadProgressListener}
|
||||
import net.kemitix.thorp.core.MD5HashData.{leafHash, rootHash}
|
||||
import net.kemitix.thorp.domain.Filter.Exclude
|
||||
import net.kemitix.thorp.domain._
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class SyncSuite
|
|
@ -1,9 +0,0 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
|
||||
import net.kemitix.s3thorp.domain.QuoteStripper.stripQuotes
|
||||
|
||||
final case class MD5Hash(in: String) {
|
||||
|
||||
lazy val hash: String = in filter stripQuotes
|
||||
|
||||
}
|
|
@ -1,3 +1,3 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
final case class Bucket(name: String)
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
import java.io.File
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.util.regex.Pattern
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
final case class HashModified(hash: MD5Hash,
|
||||
modified: LastModified)
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
final case class KeyModified(key: RemoteKey,
|
||||
modified: LastModified)
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
import java.time.Instant
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
import java.io.File
|
||||
import java.nio.file.Path
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
trait Logger[M[_]] {
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
package net.kemitix.thorp.domain
|
||||
|
||||
import net.kemitix.thorp.domain.QuoteStripper.stripQuotes
|
||||
|
||||
final case class MD5Hash(in: String) {
|
||||
|
||||
lazy val hash: String = in filter stripQuotes
|
||||
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
object QuoteStripper {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
import java.io.File
|
||||
import java.nio.file.Paths
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
final case class RemoteMetaData(remoteKey: RemoteKey,
|
||||
hash: MD5Hash,
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
|
||||
final case class S3MetaData(
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
/**
|
||||
* A list of objects and their MD5 hash values.
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
object SizeTranslation {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
object Terminal {
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
import java.nio.file.{Path, Paths}
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.s3thorp.domain.Filter.{Exclude, Include}
|
||||
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class FiltersSuite extends FunSpec {
|
|
@ -1,4 +1,4 @@
|
|||
package net.kemitix.s3thorp.domain
|
||||
package net.kemitix.thorp.domain
|
||||
|
||||
import org.scalatest.FunSpec
|
||||
|
Loading…
Reference in a new issue