Don't use String as key in Map for hashes (#124)
* Don't use String as key in Map for hashes * [domain] HashType remote redundant braces
This commit is contained in:
parent
ad0f848bed
commit
985cc9f147
15 changed files with 69 additions and 40 deletions
|
@ -18,6 +18,7 @@ The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]
|
||||||
|
|
||||||
- Replace cats-effect with zio (#117)
|
- Replace cats-effect with zio (#117)
|
||||||
- Replace Monocle with local SimpleLens implementation (#121)
|
- Replace Monocle with local SimpleLens implementation (#121)
|
||||||
|
- Don't use String as key in Map for hashes (#124)
|
||||||
|
|
||||||
** Dependencies
|
** Dependencies
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,8 @@ package net.kemitix.thorp.core
|
||||||
|
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.domain.MD5Hash
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
|
import net.kemitix.thorp.domain.{HashType, MD5Hash}
|
||||||
import net.kemitix.thorp.storage.api.HashService
|
import net.kemitix.thorp.storage.api.HashService
|
||||||
import zio.Task
|
import zio.Task
|
||||||
|
|
||||||
|
@ -10,9 +11,9 @@ case class SimpleHashService() extends HashService {
|
||||||
|
|
||||||
override def hashLocalObject(
|
override def hashLocalObject(
|
||||||
path: Path
|
path: Path
|
||||||
): Task[Map[String, MD5Hash]] =
|
): Task[Map[HashType, MD5Hash]] =
|
||||||
for {
|
for {
|
||||||
md5 <- MD5HashGenerator.md5File(path)
|
md5 <- MD5HashGenerator.md5File(path)
|
||||||
} yield Map("md5" -> md5)
|
} yield Map(MD5 -> md5)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ package net.kemitix.thorp.core
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
|
||||||
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain._
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
|
||||||
|
@ -143,7 +144,7 @@ class ActionGeneratorSuite extends FunSpec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def md5HashMap(theHash: MD5Hash) = {
|
private def md5HashMap(theHash: MD5Hash): Map[HashType, MD5Hash] = {
|
||||||
Map("md5" -> theHash)
|
Map(MD5 -> theHash)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,14 +2,14 @@ package net.kemitix.thorp.core
|
||||||
|
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.domain.MD5Hash
|
import net.kemitix.thorp.domain.{HashType, MD5Hash}
|
||||||
import net.kemitix.thorp.storage.api.HashService
|
import net.kemitix.thorp.storage.api.HashService
|
||||||
import zio.Task
|
import zio.Task
|
||||||
|
|
||||||
case class DummyHashService(hashes: Map[Path, Map[String, MD5Hash]])
|
case class DummyHashService(hashes: Map[Path, Map[HashType, MD5Hash]])
|
||||||
extends HashService {
|
extends HashService {
|
||||||
|
|
||||||
override def hashLocalObject(path: Path): Task[Map[String, MD5Hash]] =
|
override def hashLocalObject(path: Path): Task[Map[HashType, MD5Hash]] =
|
||||||
Task(hashes(path))
|
Task(hashes(path))
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@ package net.kemitix.thorp.core
|
||||||
|
|
||||||
import java.nio.file.Paths
|
import java.nio.file.Paths
|
||||||
|
|
||||||
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain._
|
||||||
import net.kemitix.thorp.storage.api.HashService
|
import net.kemitix.thorp.storage.api.HashService
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
@ -13,8 +14,8 @@ class LocalFileStreamSuite extends FunSpec {
|
||||||
private val sourcePath = source.toPath
|
private val sourcePath = source.toPath
|
||||||
private val hashService: HashService = DummyHashService(
|
private val hashService: HashService = DummyHashService(
|
||||||
Map(
|
Map(
|
||||||
file("root-file") -> Map("md5" -> MD5HashData.Root.hash),
|
file("root-file") -> Map(MD5 -> MD5HashData.Root.hash),
|
||||||
file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash)
|
file("subdir/leaf-file") -> Map(MD5 -> MD5HashData.Leaf.hash)
|
||||||
))
|
))
|
||||||
|
|
||||||
private def file(filename: String) =
|
private def file(filename: String) =
|
||||||
|
|
|
@ -5,6 +5,7 @@ import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.console._
|
import net.kemitix.thorp.console._
|
||||||
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
|
||||||
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain._
|
||||||
import net.kemitix.thorp.storage.api.{HashService, StorageService}
|
import net.kemitix.thorp.storage.api.{HashService, StorageService}
|
||||||
import org.scalatest.FreeSpec
|
import org.scalatest.FreeSpec
|
||||||
|
@ -433,7 +434,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
def md5Hash(file: File) = {
|
def md5Hash(file: File) = {
|
||||||
runtime
|
runtime
|
||||||
.unsafeRunSync {
|
.unsafeRunSync {
|
||||||
hashService.hashLocalObject(file.toPath).map(_.get("md5"))
|
hashService.hashLocalObject(file.toPath).map(_.get(MD5))
|
||||||
}
|
}
|
||||||
.toEither
|
.toEither
|
||||||
.toOption
|
.toOption
|
||||||
|
@ -477,7 +478,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
|
||||||
case ToUpload(_, lf, _) =>
|
case ToUpload(_, lf, _) =>
|
||||||
("upload",
|
("upload",
|
||||||
lf.remoteKey.key,
|
lf.remoteKey.key,
|
||||||
lf.hashes("md5").hash,
|
lf.hashes(MD5).hash,
|
||||||
lf.source.toString,
|
lf.source.toString,
|
||||||
lf.file.toString)
|
lf.file.toString)
|
||||||
case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")
|
case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")
|
||||||
|
|
|
@ -3,6 +3,7 @@ package net.kemitix.thorp.core
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
|
import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
|
||||||
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain._
|
||||||
import org.scalatest.FunSpec
|
import org.scalatest.FunSpec
|
||||||
|
|
||||||
|
@ -171,8 +172,8 @@ class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def md5HashMap(theHash: MD5Hash) = {
|
private def md5HashMap(theHash: MD5Hash): Map[HashType, MD5Hash] = {
|
||||||
Map("md5" -> theHash)
|
Map(MD5 -> theHash)
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("getS3Status") {
|
describe("getS3Status") {
|
||||||
|
|
|
@ -3,22 +3,28 @@ package net.kemitix.thorp.core
|
||||||
import java.io.File
|
import java.io.File
|
||||||
|
|
||||||
import net.kemitix.thorp.core.Action._
|
import net.kemitix.thorp.core.Action._
|
||||||
import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
|
import net.kemitix.thorp.domain.{
|
||||||
|
Bucket,
|
||||||
|
HashType,
|
||||||
|
LocalFile,
|
||||||
|
MD5Hash,
|
||||||
|
RemoteKey
|
||||||
|
}
|
||||||
import org.scalatest.FreeSpec
|
import org.scalatest.FreeSpec
|
||||||
|
|
||||||
class SequencePlanTest extends FreeSpec {
|
class SequencePlanTest extends FreeSpec {
|
||||||
|
|
||||||
"sort" - {
|
"sort" - {
|
||||||
"a list of assorted actions" - {
|
"a list of assorted actions" - {
|
||||||
val bucket = Bucket("aBucket")
|
val bucket = Bucket("aBucket")
|
||||||
val remoteKey1 = RemoteKey("remoteKey1")
|
val remoteKey1 = RemoteKey("remoteKey1")
|
||||||
val remoteKey2 = RemoteKey("targetHash")
|
val remoteKey2 = RemoteKey("targetHash")
|
||||||
val hash = MD5Hash("aHash")
|
val hash = MD5Hash("aHash")
|
||||||
val hashes: Map[String, MD5Hash] = Map()
|
val hashes = Map[HashType, MD5Hash]()
|
||||||
val size = 1024
|
val size = 1024
|
||||||
val file1 = new File("aFile")
|
val file1 = new File("aFile")
|
||||||
val file2 = new File("aFile")
|
val file2 = new File("aFile")
|
||||||
val source = new File("source")
|
val source = new File("source")
|
||||||
val localFile1 =
|
val localFile1 =
|
||||||
LocalFile(file1, source, hashes, remoteKey1)
|
LocalFile(file1, source, hashes, remoteKey1)
|
||||||
val localFile2 =
|
val localFile2 =
|
||||||
|
|
|
@ -7,6 +7,7 @@ import java.time.Instant
|
||||||
import net.kemitix.thorp.console
|
import net.kemitix.thorp.console
|
||||||
import net.kemitix.thorp.console.MyConsole
|
import net.kemitix.thorp.console.MyConsole
|
||||||
import net.kemitix.thorp.core.Action.{ToCopy, ToDelete, ToUpload}
|
import net.kemitix.thorp.core.Action.{ToCopy, ToDelete, ToUpload}
|
||||||
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
import net.kemitix.thorp.domain.MD5HashData.{Leaf, Root}
|
import net.kemitix.thorp.domain.MD5HashData.{Leaf, Root}
|
||||||
import net.kemitix.thorp.domain.StorageQueueEvent.{
|
import net.kemitix.thorp.domain.StorageQueueEvent.{
|
||||||
CopyQueueEvent,
|
CopyQueueEvent,
|
||||||
|
@ -43,8 +44,8 @@ class SyncSuite extends FunSpec {
|
||||||
private val hashService =
|
private val hashService =
|
||||||
DummyHashService(
|
DummyHashService(
|
||||||
Map(
|
Map(
|
||||||
file("root-file") -> Map("md5" -> MD5HashData.Root.hash),
|
file("root-file") -> Map(MD5 -> MD5HashData.Root.hash),
|
||||||
file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash)
|
file("subdir/leaf-file") -> Map(MD5 -> MD5HashData.Leaf.hash)
|
||||||
))
|
))
|
||||||
private val configOptions =
|
private val configOptions =
|
||||||
ConfigOptions(
|
ConfigOptions(
|
||||||
|
@ -62,8 +63,8 @@ class SyncSuite extends FunSpec {
|
||||||
localFile: LocalFile): (String, String, File) =
|
localFile: LocalFile): (String, String, File) =
|
||||||
(bucket.name, remoteKey.key, localFile.file)
|
(bucket.name, remoteKey.key, localFile.file)
|
||||||
|
|
||||||
private def md5HashMap(md5Hash: MD5Hash): Map[String, MD5Hash] =
|
private def md5HashMap(md5Hash: MD5Hash): Map[HashType, MD5Hash] =
|
||||||
Map("md5" -> md5Hash)
|
Map(MD5 -> md5Hash)
|
||||||
|
|
||||||
private def file(filename: String) =
|
private def file(filename: String) =
|
||||||
sourcePath.resolve(Paths.get(filename))
|
sourcePath.resolve(Paths.get(filename))
|
||||||
|
@ -207,7 +208,7 @@ class SyncSuite extends FunSpec {
|
||||||
batchMode: Boolean,
|
batchMode: Boolean,
|
||||||
uploadEventListener: UploadEventListener,
|
uploadEventListener: UploadEventListener,
|
||||||
tryCount: Int): Task[UploadQueueEvent] =
|
tryCount: Int): Task[UploadQueueEvent] =
|
||||||
Task(UploadQueueEvent(localFile.remoteKey, localFile.hashes("md5")))
|
Task(UploadQueueEvent(localFile.remoteKey, localFile.hashes(MD5)))
|
||||||
|
|
||||||
override def copy(bucket: Bucket,
|
override def copy(bucket: Bucket,
|
||||||
sourceKey: RemoteKey,
|
sourceKey: RemoteKey,
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
package net.kemitix.thorp.domain
|
||||||
|
|
||||||
|
trait HashType
|
||||||
|
|
||||||
|
object HashType {
|
||||||
|
case object MD5 extends HashType
|
||||||
|
}
|
|
@ -3,10 +3,12 @@ package net.kemitix.thorp.domain
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
|
|
||||||
final case class LocalFile(
|
final case class LocalFile(
|
||||||
file: File,
|
file: File,
|
||||||
source: File,
|
source: File,
|
||||||
hashes: Map[String, MD5Hash],
|
hashes: Map[HashType, MD5Hash],
|
||||||
remoteKey: RemoteKey
|
remoteKey: RemoteKey
|
||||||
) {
|
) {
|
||||||
|
|
||||||
|
@ -19,7 +21,7 @@ final case class LocalFile(
|
||||||
|
|
||||||
def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _)
|
def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _)
|
||||||
|
|
||||||
def md5base64: Option[String] = hashes.get("md5").map(_.hash64)
|
def md5base64: Option[String] = hashes.get(MD5).map(_.hash64)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -27,7 +29,7 @@ object LocalFile {
|
||||||
|
|
||||||
def resolve(
|
def resolve(
|
||||||
path: String,
|
path: String,
|
||||||
md5Hashes: Map[String, MD5Hash],
|
md5Hashes: Map[HashType, MD5Hash],
|
||||||
source: Path,
|
source: Path,
|
||||||
pathToKey: Path => RemoteKey
|
pathToKey: Path => RemoteKey
|
||||||
): LocalFile = {
|
): LocalFile = {
|
||||||
|
|
|
@ -2,7 +2,7 @@ package net.kemitix.thorp.storage.api
|
||||||
|
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.domain.MD5Hash
|
import net.kemitix.thorp.domain.{HashType, MD5Hash}
|
||||||
import zio.Task
|
import zio.Task
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -10,6 +10,6 @@ import zio.Task
|
||||||
*/
|
*/
|
||||||
trait HashService {
|
trait HashService {
|
||||||
|
|
||||||
def hashLocalObject(path: Path): Task[Map[String, MD5Hash]]
|
def hashLocalObject(path: Path): Task[Map[HashType, MD5Hash]]
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
package net.kemitix.thorp.storage.aws
|
||||||
|
|
||||||
|
import net.kemitix.thorp.domain.HashType
|
||||||
|
|
||||||
|
object ETag extends HashType
|
|
@ -3,7 +3,8 @@ package net.kemitix.thorp.storage.aws
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
|
|
||||||
import net.kemitix.thorp.core.MD5HashGenerator
|
import net.kemitix.thorp.core.MD5HashGenerator
|
||||||
import net.kemitix.thorp.domain.MD5Hash
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
|
import net.kemitix.thorp.domain.{HashType, MD5Hash}
|
||||||
import net.kemitix.thorp.storage.api.HashService
|
import net.kemitix.thorp.storage.api.HashService
|
||||||
import zio.Task
|
import zio.Task
|
||||||
|
|
||||||
|
@ -17,14 +18,14 @@ trait S3HashService extends HashService {
|
||||||
*/
|
*/
|
||||||
override def hashLocalObject(
|
override def hashLocalObject(
|
||||||
path: Path
|
path: Path
|
||||||
): Task[Map[String, MD5Hash]] =
|
): Task[Map[HashType, MD5Hash]] =
|
||||||
for {
|
for {
|
||||||
md5 <- MD5HashGenerator.md5File(path)
|
md5 <- MD5HashGenerator.md5File(path)
|
||||||
etag <- ETagGenerator.eTag(path).map(MD5Hash(_))
|
etag <- ETagGenerator.eTag(path).map(MD5Hash(_))
|
||||||
} yield
|
} yield
|
||||||
Map(
|
Map(
|
||||||
"md5" -> md5,
|
MD5 -> md5,
|
||||||
"etag" -> etag
|
ETag -> etag
|
||||||
)
|
)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,6 +5,7 @@ import java.time.Instant
|
||||||
import com.amazonaws.services.s3.model.PutObjectRequest
|
import com.amazonaws.services.s3.model.PutObjectRequest
|
||||||
import com.amazonaws.services.s3.transfer.model.UploadResult
|
import com.amazonaws.services.s3.transfer.model.UploadResult
|
||||||
import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
|
import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
|
||||||
|
import net.kemitix.thorp.domain.HashType.MD5
|
||||||
import net.kemitix.thorp.domain.MD5HashData.Root
|
import net.kemitix.thorp.domain.MD5HashData.Root
|
||||||
import net.kemitix.thorp.domain.StorageQueueEvent.UploadQueueEvent
|
import net.kemitix.thorp.domain.StorageQueueEvent.UploadQueueEvent
|
||||||
import net.kemitix.thorp.domain._
|
import net.kemitix.thorp.domain._
|
||||||
|
@ -113,8 +114,8 @@ class StorageServiceSuite extends FunSpec with MockFactory {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private def md5HashMap(hash: MD5Hash) =
|
private def md5HashMap(hash: MD5Hash): Map[HashType, MD5Hash] =
|
||||||
Map("md5" -> hash)
|
Map(MD5 -> hash)
|
||||||
|
|
||||||
val batchMode: Boolean = true
|
val batchMode: Boolean = true
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue