Final case classes (#32)
* case classes are final * [Counters] Extract to it's own file * [LocalFile] allow overriding Hash
This commit is contained in:
parent
0386fde322
commit
bb283a12d4
12 changed files with 56 additions and 42 deletions
|
@ -1,9 +1,10 @@
|
||||||
package net.kemitix.s3thorp
|
package net.kemitix.s3thorp
|
||||||
|
|
||||||
sealed trait Action
|
sealed trait Action
|
||||||
case class DoNothing(remoteKey: RemoteKey) extends Action
|
final case class DoNothing(remoteKey: RemoteKey) extends Action
|
||||||
case class ToUpload(localFile: LocalFile) extends Action
|
final case class ToUpload(localFile: LocalFile) extends Action
|
||||||
case class ToCopy(sourceKey: RemoteKey,
|
final case class ToCopy(
|
||||||
|
sourceKey: RemoteKey,
|
||||||
hash: MD5Hash,
|
hash: MD5Hash,
|
||||||
targetKey: RemoteKey) extends Action
|
targetKey: RemoteKey) extends Action
|
||||||
case class ToDelete(remoteKey: RemoteKey) extends Action
|
final case class ToDelete(remoteKey: RemoteKey) extends Action
|
||||||
|
|
|
@ -2,7 +2,8 @@ package net.kemitix.s3thorp
|
||||||
|
|
||||||
import java.io.File
|
import java.io.File
|
||||||
|
|
||||||
case class Config(bucket: Bucket = Bucket(""),
|
final case class Config(
|
||||||
|
bucket: Bucket = Bucket(""),
|
||||||
prefix: RemoteKey = RemoteKey(""),
|
prefix: RemoteKey = RemoteKey(""),
|
||||||
verbose: Int = 1,
|
verbose: Int = 1,
|
||||||
filters: Seq[Filter] = List(),
|
filters: Seq[Filter] = List(),
|
||||||
|
@ -10,7 +11,7 @@ case class Config(bucket: Bucket = Bucket(""),
|
||||||
multiPartThreshold: Long = 1024 * 1024 * 5,
|
multiPartThreshold: Long = 1024 * 1024 * 5,
|
||||||
maxRetries: Int = 3,
|
maxRetries: Int = 3,
|
||||||
source: File
|
source: File
|
||||||
) {
|
) {
|
||||||
require(source.isDirectory, s"Source must be a directory: $source")
|
require(source.isDirectory, s"Source must be a directory: $source")
|
||||||
require(multiPartThreshold >= 1024 * 1024 * 5, s"Threshold for multi-part upload is 5Mb: '$multiPartThreshold'")
|
require(multiPartThreshold >= 1024 * 1024 * 5, s"Threshold for multi-part upload is 5Mb: '$multiPartThreshold'")
|
||||||
}
|
}
|
||||||
|
|
6
src/main/scala/net/kemitix/s3thorp/Counters.scala
Normal file
6
src/main/scala/net/kemitix/s3thorp/Counters.scala
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
package net.kemitix.s3thorp
|
||||||
|
|
||||||
|
|
||||||
|
final case class Counters(uploaded: Int = 0,
|
||||||
|
deleted: Int = 0,
|
||||||
|
copied: Int = 0)
|
|
@ -4,7 +4,7 @@ import java.nio.file.Path
|
||||||
import java.util.function.Predicate
|
import java.util.function.Predicate
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
case class Exclude(exclude: String = "!.*") {
|
final case class Exclude(exclude: String = "!.*") {
|
||||||
|
|
||||||
lazy val predicate: Predicate[String] = Pattern.compile(exclude).asPredicate()
|
lazy val predicate: Predicate[String] = Pattern.compile(exclude).asPredicate()
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ import java.nio.file.Path
|
||||||
import java.util.function.Predicate
|
import java.util.function.Predicate
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
case class Filter(filter: String = ".*") {
|
final case class Filter(filter: String = ".*") {
|
||||||
|
|
||||||
lazy val predicate: Predicate[String] = Pattern.compile(filter).asPredicate.negate
|
lazy val predicate: Predicate[String] = Pattern.compile(filter).asPredicate.negate
|
||||||
|
|
||||||
|
|
|
@ -3,15 +3,17 @@ package net.kemitix.s3thorp
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
case class LocalFile(file: File,
|
final case class LocalFile(
|
||||||
|
file: File,
|
||||||
source: File,
|
source: File,
|
||||||
keyGenerator: File => RemoteKey)
|
keyGenerator: File => RemoteKey,
|
||||||
|
suppliedHash: Option[MD5Hash] = None)
|
||||||
(implicit c: Config)
|
(implicit c: Config)
|
||||||
extends MD5HashGenerator {
|
extends MD5HashGenerator {
|
||||||
|
|
||||||
require(!file.isDirectory, s"LocalFile must not be a directory: $file")
|
require(!file.isDirectory, s"LocalFile must not be a directory: $file")
|
||||||
|
|
||||||
private lazy val myhash = md5File(file)
|
private lazy val myhash = suppliedHash.getOrElse(md5File(file))
|
||||||
|
|
||||||
def hash: MD5Hash = myhash
|
def hash: MD5Hash = myhash
|
||||||
|
|
||||||
|
|
|
@ -9,22 +9,25 @@ sealed trait S3Action {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
case class DoNothingS3Action(remoteKey: RemoteKey) extends S3Action {
|
final case class DoNothingS3Action(remoteKey: RemoteKey) extends S3Action {
|
||||||
override val order: Int = 0
|
override val order: Int = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
case class CopyS3Action(remoteKey: RemoteKey) extends S3Action {
|
final case class CopyS3Action(remoteKey: RemoteKey) extends S3Action {
|
||||||
override val order: Int = 1
|
override val order: Int = 1
|
||||||
}
|
}
|
||||||
case class UploadS3Action(remoteKey: RemoteKey,
|
|
||||||
|
final case class UploadS3Action(
|
||||||
|
remoteKey: RemoteKey,
|
||||||
md5Hash: MD5Hash) extends S3Action {
|
md5Hash: MD5Hash) extends S3Action {
|
||||||
override val order: Int = 2
|
override val order: Int = 2
|
||||||
}
|
}
|
||||||
case class DeleteS3Action(remoteKey: RemoteKey) extends S3Action {
|
|
||||||
|
final case class DeleteS3Action(remoteKey: RemoteKey) extends S3Action {
|
||||||
override val order: Int = 3
|
override val order: Int = 3
|
||||||
}
|
}
|
||||||
|
|
||||||
case class ErroredS3Action(remoteKey: RemoteKey, e: Throwable) extends S3Action {
|
final case class ErroredS3Action(remoteKey: RemoteKey, e: Throwable) extends S3Action {
|
||||||
override val order: Int = 10
|
override val order: Int = 10
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package net.kemitix.s3thorp
|
package net.kemitix.s3thorp
|
||||||
|
|
||||||
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
|
// For the LocalFile, the set of matching S3 objects with the same MD5Hash, and any S3 object with the same remote key
|
||||||
case class S3MetaData(localFile: LocalFile,
|
final case class S3MetaData(
|
||||||
|
localFile: LocalFile,
|
||||||
matchByHash: Set[RemoteMetaData],
|
matchByHash: Set[RemoteMetaData],
|
||||||
matchByKey: Option[RemoteMetaData])
|
matchByKey: Option[RemoteMetaData])
|
|
@ -35,8 +35,4 @@ trait SyncLogging extends Logging {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
case class Counters(uploaded: Int = 0,
|
|
||||||
deleted: Int = 0,
|
|
||||||
copied: Int = 0)
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
package net.kemitix.s3thorp.awssdk
|
package net.kemitix.s3thorp.awssdk
|
||||||
|
|
||||||
case class CancellableMultiPartUpload(e: Throwable,
|
final case class CancellableMultiPartUpload(
|
||||||
|
e: Throwable,
|
||||||
uploadId: String) extends Exception(e)
|
uploadId: String) extends Exception(e)
|
||||||
|
|
|
@ -5,7 +5,8 @@ import net.kemitix.s3thorp.{HashModified, KeyModified, MD5Hash, RemoteKey}
|
||||||
/**
|
/**
|
||||||
* A list of objects and their MD5 hash values.
|
* A list of objects and their MD5 hash values.
|
||||||
*/
|
*/
|
||||||
case class S3ObjectsData(byHash: Map[MD5Hash, Set[KeyModified]],
|
final case class S3ObjectsData(
|
||||||
|
byHash: Map[MD5Hash, Set[KeyModified]],
|
||||||
byKey: Map[RemoteKey, HashModified]) {
|
byKey: Map[RemoteKey, HashModified]) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,9 +8,11 @@ abstract class UnitTest extends FunSpec {
|
||||||
|
|
||||||
def aLocalFile(path: String, myHash: MD5Hash, source: File, fileToKey: File => RemoteKey)
|
def aLocalFile(path: String, myHash: MD5Hash, source: File, fileToKey: File => RemoteKey)
|
||||||
(implicit c: Config): LocalFile =
|
(implicit c: Config): LocalFile =
|
||||||
new LocalFile(source.toPath.resolve(path).toFile, source, fileToKey) {
|
LocalFile(
|
||||||
override def hash: MD5Hash = myHash
|
file = source.toPath.resolve(path).toFile,
|
||||||
}
|
source = source,
|
||||||
|
keyGenerator = fileToKey,
|
||||||
|
suppliedHash = Some(myHash))
|
||||||
|
|
||||||
def aRemoteKey(prefix: RemoteKey, path: String): RemoteKey =
|
def aRemoteKey(prefix: RemoteKey, path: String): RemoteKey =
|
||||||
RemoteKey(prefix.key + "/" + path)
|
RemoteKey(prefix.key + "/" + path)
|
||||||
|
|
Loading…
Reference in a new issue