Don't use String as key in Map for hashes (#124)

* Don't use String as key in Map for hashes

* [domain] HashType remote redundant braces
This commit is contained in:
Paul Campbell 2019-07-24 19:50:28 +01:00 committed by GitHub
parent ad0f848bed
commit 985cc9f147
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 69 additions and 40 deletions

View file

@ -18,6 +18,7 @@ The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]
- Replace cats-effect with zio (#117) - Replace cats-effect with zio (#117)
- Replace Monocle with local SimpleLens implementation (#121) - Replace Monocle with local SimpleLens implementation (#121)
- Don't use String as key in Map for hashes (#124)
** Dependencies ** Dependencies

View file

@ -2,7 +2,8 @@ package net.kemitix.thorp.core
import java.nio.file.Path import java.nio.file.Path
import net.kemitix.thorp.domain.MD5Hash import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.{HashType, MD5Hash}
import net.kemitix.thorp.storage.api.HashService import net.kemitix.thorp.storage.api.HashService
import zio.Task import zio.Task
@ -10,9 +11,9 @@ case class SimpleHashService() extends HashService {
override def hashLocalObject( override def hashLocalObject(
path: Path path: Path
): Task[Map[String, MD5Hash]] = ): Task[Map[HashType, MD5Hash]] =
for { for {
md5 <- MD5HashGenerator.md5File(path) md5 <- MD5HashGenerator.md5File(path)
} yield Map("md5" -> md5) } yield Map(MD5 -> md5)
} }

View file

@ -3,6 +3,7 @@ package net.kemitix.thorp.core
import java.time.Instant import java.time.Instant
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload} import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain._
import org.scalatest.FunSpec import org.scalatest.FunSpec
@ -143,7 +144,7 @@ class ActionGeneratorSuite extends FunSpec {
} }
} }
private def md5HashMap(theHash: MD5Hash) = { private def md5HashMap(theHash: MD5Hash): Map[HashType, MD5Hash] = {
Map("md5" -> theHash) Map(MD5 -> theHash)
} }
} }

View file

@ -2,14 +2,14 @@ package net.kemitix.thorp.core
import java.nio.file.Path import java.nio.file.Path
import net.kemitix.thorp.domain.MD5Hash import net.kemitix.thorp.domain.{HashType, MD5Hash}
import net.kemitix.thorp.storage.api.HashService import net.kemitix.thorp.storage.api.HashService
import zio.Task import zio.Task
case class DummyHashService(hashes: Map[Path, Map[String, MD5Hash]]) case class DummyHashService(hashes: Map[Path, Map[HashType, MD5Hash]])
extends HashService { extends HashService {
override def hashLocalObject(path: Path): Task[Map[String, MD5Hash]] = override def hashLocalObject(path: Path): Task[Map[HashType, MD5Hash]] =
Task(hashes(path)) Task(hashes(path))
} }

View file

@ -2,6 +2,7 @@ package net.kemitix.thorp.core
import java.nio.file.Paths import java.nio.file.Paths
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.api.HashService import net.kemitix.thorp.storage.api.HashService
import org.scalatest.FunSpec import org.scalatest.FunSpec
@ -13,8 +14,8 @@ class LocalFileStreamSuite extends FunSpec {
private val sourcePath = source.toPath private val sourcePath = source.toPath
private val hashService: HashService = DummyHashService( private val hashService: HashService = DummyHashService(
Map( Map(
file("root-file") -> Map("md5" -> MD5HashData.Root.hash), file("root-file") -> Map(MD5 -> MD5HashData.Root.hash),
file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash) file("subdir/leaf-file") -> Map(MD5 -> MD5HashData.Leaf.hash)
)) ))
private def file(filename: String) = private def file(filename: String) =

View file

@ -5,6 +5,7 @@ import java.nio.file.Path
import net.kemitix.thorp.console._ import net.kemitix.thorp.console._
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload} import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.api.{HashService, StorageService} import net.kemitix.thorp.storage.api.{HashService, StorageService}
import org.scalatest.FreeSpec import org.scalatest.FreeSpec
@ -433,7 +434,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
def md5Hash(file: File) = { def md5Hash(file: File) = {
runtime runtime
.unsafeRunSync { .unsafeRunSync {
hashService.hashLocalObject(file.toPath).map(_.get("md5")) hashService.hashLocalObject(file.toPath).map(_.get(MD5))
} }
.toEither .toEither
.toOption .toOption
@ -477,7 +478,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
case ToUpload(_, lf, _) => case ToUpload(_, lf, _) =>
("upload", ("upload",
lf.remoteKey.key, lf.remoteKey.key,
lf.hashes("md5").hash, lf.hashes(MD5).hash,
lf.source.toString, lf.source.toString,
lf.file.toString) lf.file.toString)
case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "") case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")

View file

@ -3,6 +3,7 @@ package net.kemitix.thorp.core
import java.time.Instant import java.time.Instant
import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status} import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain._
import org.scalatest.FunSpec import org.scalatest.FunSpec
@ -171,8 +172,8 @@ class S3MetaDataEnricherSuite extends FunSpec {
} }
} }
private def md5HashMap(theHash: MD5Hash) = { private def md5HashMap(theHash: MD5Hash): Map[HashType, MD5Hash] = {
Map("md5" -> theHash) Map(MD5 -> theHash)
} }
describe("getS3Status") { describe("getS3Status") {

View file

@ -3,22 +3,28 @@ package net.kemitix.thorp.core
import java.io.File import java.io.File
import net.kemitix.thorp.core.Action._ import net.kemitix.thorp.core.Action._
import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey} import net.kemitix.thorp.domain.{
Bucket,
HashType,
LocalFile,
MD5Hash,
RemoteKey
}
import org.scalatest.FreeSpec import org.scalatest.FreeSpec
class SequencePlanTest extends FreeSpec { class SequencePlanTest extends FreeSpec {
"sort" - { "sort" - {
"a list of assorted actions" - { "a list of assorted actions" - {
val bucket = Bucket("aBucket") val bucket = Bucket("aBucket")
val remoteKey1 = RemoteKey("remoteKey1") val remoteKey1 = RemoteKey("remoteKey1")
val remoteKey2 = RemoteKey("targetHash") val remoteKey2 = RemoteKey("targetHash")
val hash = MD5Hash("aHash") val hash = MD5Hash("aHash")
val hashes: Map[String, MD5Hash] = Map() val hashes = Map[HashType, MD5Hash]()
val size = 1024 val size = 1024
val file1 = new File("aFile") val file1 = new File("aFile")
val file2 = new File("aFile") val file2 = new File("aFile")
val source = new File("source") val source = new File("source")
val localFile1 = val localFile1 =
LocalFile(file1, source, hashes, remoteKey1) LocalFile(file1, source, hashes, remoteKey1)
val localFile2 = val localFile2 =

View file

@ -7,6 +7,7 @@ import java.time.Instant
import net.kemitix.thorp.console import net.kemitix.thorp.console
import net.kemitix.thorp.console.MyConsole import net.kemitix.thorp.console.MyConsole
import net.kemitix.thorp.core.Action.{ToCopy, ToDelete, ToUpload} import net.kemitix.thorp.core.Action.{ToCopy, ToDelete, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.MD5HashData.{Leaf, Root} import net.kemitix.thorp.domain.MD5HashData.{Leaf, Root}
import net.kemitix.thorp.domain.StorageQueueEvent.{ import net.kemitix.thorp.domain.StorageQueueEvent.{
CopyQueueEvent, CopyQueueEvent,
@ -43,8 +44,8 @@ class SyncSuite extends FunSpec {
private val hashService = private val hashService =
DummyHashService( DummyHashService(
Map( Map(
file("root-file") -> Map("md5" -> MD5HashData.Root.hash), file("root-file") -> Map(MD5 -> MD5HashData.Root.hash),
file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash) file("subdir/leaf-file") -> Map(MD5 -> MD5HashData.Leaf.hash)
)) ))
private val configOptions = private val configOptions =
ConfigOptions( ConfigOptions(
@ -62,8 +63,8 @@ class SyncSuite extends FunSpec {
localFile: LocalFile): (String, String, File) = localFile: LocalFile): (String, String, File) =
(bucket.name, remoteKey.key, localFile.file) (bucket.name, remoteKey.key, localFile.file)
private def md5HashMap(md5Hash: MD5Hash): Map[String, MD5Hash] = private def md5HashMap(md5Hash: MD5Hash): Map[HashType, MD5Hash] =
Map("md5" -> md5Hash) Map(MD5 -> md5Hash)
private def file(filename: String) = private def file(filename: String) =
sourcePath.resolve(Paths.get(filename)) sourcePath.resolve(Paths.get(filename))
@ -207,7 +208,7 @@ class SyncSuite extends FunSpec {
batchMode: Boolean, batchMode: Boolean,
uploadEventListener: UploadEventListener, uploadEventListener: UploadEventListener,
tryCount: Int): Task[UploadQueueEvent] = tryCount: Int): Task[UploadQueueEvent] =
Task(UploadQueueEvent(localFile.remoteKey, localFile.hashes("md5"))) Task(UploadQueueEvent(localFile.remoteKey, localFile.hashes(MD5)))
override def copy(bucket: Bucket, override def copy(bucket: Bucket,
sourceKey: RemoteKey, sourceKey: RemoteKey,

View file

@ -0,0 +1,7 @@
package net.kemitix.thorp.domain
trait HashType
object HashType {
case object MD5 extends HashType
}

View file

@ -3,10 +3,12 @@ package net.kemitix.thorp.domain
import java.io.File import java.io.File
import java.nio.file.Path import java.nio.file.Path
import net.kemitix.thorp.domain.HashType.MD5
final case class LocalFile( final case class LocalFile(
file: File, file: File,
source: File, source: File,
hashes: Map[String, MD5Hash], hashes: Map[HashType, MD5Hash],
remoteKey: RemoteKey remoteKey: RemoteKey
) { ) {
@ -19,7 +21,7 @@ final case class LocalFile(
def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _) def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _)
def md5base64: Option[String] = hashes.get("md5").map(_.hash64) def md5base64: Option[String] = hashes.get(MD5).map(_.hash64)
} }
@ -27,7 +29,7 @@ object LocalFile {
def resolve( def resolve(
path: String, path: String,
md5Hashes: Map[String, MD5Hash], md5Hashes: Map[HashType, MD5Hash],
source: Path, source: Path,
pathToKey: Path => RemoteKey pathToKey: Path => RemoteKey
): LocalFile = { ): LocalFile = {

View file

@ -2,7 +2,7 @@ package net.kemitix.thorp.storage.api
import java.nio.file.Path import java.nio.file.Path
import net.kemitix.thorp.domain.MD5Hash import net.kemitix.thorp.domain.{HashType, MD5Hash}
import zio.Task import zio.Task
/** /**
@ -10,6 +10,6 @@ import zio.Task
*/ */
trait HashService { trait HashService {
def hashLocalObject(path: Path): Task[Map[String, MD5Hash]] def hashLocalObject(path: Path): Task[Map[HashType, MD5Hash]]
} }

View file

@ -0,0 +1,5 @@
package net.kemitix.thorp.storage.aws
import net.kemitix.thorp.domain.HashType
object ETag extends HashType

View file

@ -3,7 +3,8 @@ package net.kemitix.thorp.storage.aws
import java.nio.file.Path import java.nio.file.Path
import net.kemitix.thorp.core.MD5HashGenerator import net.kemitix.thorp.core.MD5HashGenerator
import net.kemitix.thorp.domain.MD5Hash import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.{HashType, MD5Hash}
import net.kemitix.thorp.storage.api.HashService import net.kemitix.thorp.storage.api.HashService
import zio.Task import zio.Task
@ -17,14 +18,14 @@ trait S3HashService extends HashService {
*/ */
override def hashLocalObject( override def hashLocalObject(
path: Path path: Path
): Task[Map[String, MD5Hash]] = ): Task[Map[HashType, MD5Hash]] =
for { for {
md5 <- MD5HashGenerator.md5File(path) md5 <- MD5HashGenerator.md5File(path)
etag <- ETagGenerator.eTag(path).map(MD5Hash(_)) etag <- ETagGenerator.eTag(path).map(MD5Hash(_))
} yield } yield
Map( Map(
"md5" -> md5, MD5 -> md5,
"etag" -> etag ETag -> etag
) )
} }

View file

@ -5,6 +5,7 @@ import java.time.Instant
import com.amazonaws.services.s3.model.PutObjectRequest import com.amazonaws.services.s3.model.PutObjectRequest
import com.amazonaws.services.s3.transfer.model.UploadResult import com.amazonaws.services.s3.transfer.model.UploadResult
import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher} import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.MD5HashData.Root import net.kemitix.thorp.domain.MD5HashData.Root
import net.kemitix.thorp.domain.StorageQueueEvent.UploadQueueEvent import net.kemitix.thorp.domain.StorageQueueEvent.UploadQueueEvent
import net.kemitix.thorp.domain._ import net.kemitix.thorp.domain._
@ -113,8 +114,8 @@ class StorageServiceSuite extends FunSpec with MockFactory {
} }
private def md5HashMap(hash: MD5Hash) = private def md5HashMap(hash: MD5Hash): Map[HashType, MD5Hash] =
Map("md5" -> hash) Map(MD5 -> hash)
val batchMode: Boolean = true val batchMode: Boolean = true