kemitix/thorp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Don't use String as key in Map for hashes (#124)

Browse source 
* Don't use String as key in Map for hashes

* [domain] HashType remote redundant braces
This commit is contained in:
Paul Campbell 2019-07-24 19:50:28 +01:00 committed by GitHub
parent ad0f848bed
commit 985cc9f147
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 69 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.org
View file

@ -18,6 +18,7 @@ The format is based on [[https://keepachangelog.com/en/1.0.0/][Keep a Changelog]
- Replace cats-effect with zio (#117)
- Replace Monocle with local SimpleLens implementation (#121)
- Don't use String as key in Map for hashes (#124)
** Dependencies

7
core/src/main/scala/net/kemitix/thorp/core/SimpleHashService.scala
View file

@ -2,7 +2,8 @@ package net.kemitix.thorp.core
import java.nio.file.Path
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.{HashType, MD5Hash}
import net.kemitix.thorp.storage.api.HashService
import zio.Task
@ -10,9 +11,9 @@ case class SimpleHashService() extends HashService {
override def hashLocalObject(
path: Path
): Task[Map[String, MD5Hash]] =
): Task[Map[HashType, MD5Hash]] =
for {
md5 <- MD5HashGenerator.md5File(path)
} yield Map("md5" -> md5)
} yield Map(MD5 -> md5)
}

5
core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala
View file

@ -3,6 +3,7 @@ package net.kemitix.thorp.core
import java.time.Instant
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
import org.scalatest.FunSpec
@ -143,7 +144,7 @@ class ActionGeneratorSuite extends FunSpec {
}
}
private def md5HashMap(theHash: MD5Hash) = {
Map("md5" -> theHash)
private def md5HashMap(theHash: MD5Hash): Map[HashType, MD5Hash] = {
Map(MD5 -> theHash)
}
}

6
core/src/test/scala/net/kemitix/thorp/core/DummyHashService.scala
View file

@ -2,14 +2,14 @@ package net.kemitix.thorp.core
import java.nio.file.Path
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.domain.{HashType, MD5Hash}
import net.kemitix.thorp.storage.api.HashService
import zio.Task
case class DummyHashService(hashes: Map[Path, Map[String, MD5Hash]])
case class DummyHashService(hashes: Map[Path, Map[HashType, MD5Hash]])
extends HashService {
override def hashLocalObject(path: Path): Task[Map[String, MD5Hash]] =
override def hashLocalObject(path: Path): Task[Map[HashType, MD5Hash]] =
Task(hashes(path))
}

5
core/src/test/scala/net/kemitix/thorp/core/LocalFileStreamSuite.scala
View file

@ -2,6 +2,7 @@ package net.kemitix.thorp.core
import java.nio.file.Paths
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.api.HashService
import org.scalatest.FunSpec
@ -13,8 +14,8 @@ class LocalFileStreamSuite extends FunSpec {
private val sourcePath = source.toPath
private val hashService: HashService = DummyHashService(
Map(
file("root-file") -> Map("md5" -> MD5HashData.Root.hash),
file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash)
file("root-file") -> Map(MD5 -> MD5HashData.Root.hash),
file("subdir/leaf-file") -> Map(MD5 -> MD5HashData.Leaf.hash)
))
private def file(filename: String) =

5
core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala
View file

@ -5,6 +5,7 @@ import java.nio.file.Path
import net.kemitix.thorp.console._
import net.kemitix.thorp.core.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.api.{HashService, StorageService}
import org.scalatest.FreeSpec
@ -433,7 +434,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
def md5Hash(file: File) = {
runtime
.unsafeRunSync {
hashService.hashLocalObject(file.toPath).map(_.get("md5"))
hashService.hashLocalObject(file.toPath).map(_.get(MD5))
}
.toEither
.toOption
@ -477,7 +478,7 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder {
case ToUpload(_, lf, _) =>
("upload",
lf.remoteKey.key,
lf.hashes("md5").hash,
lf.hashes(MD5).hash,
lf.source.toString,
lf.file.toString)
case ToDelete(_, remoteKey, _) => ("delete", remoteKey.key, "", "", "")

5
core/src/test/scala/net/kemitix/thorp/core/S3MetaDataEnricherSuite.scala
View file

@ -3,6 +3,7 @@ package net.kemitix.thorp.core
import java.time.Instant
import net.kemitix.thorp.core.S3MetaDataEnricher.{getMetadata, getS3Status}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain._
import org.scalatest.FunSpec
@ -171,8 +172,8 @@ class S3MetaDataEnricherSuite extends FunSpec {
}
}
private def md5HashMap(theHash: MD5Hash) = {
Map("md5" -> theHash)
private def md5HashMap(theHash: MD5Hash): Map[HashType, MD5Hash] = {
Map(MD5 -> theHash)
}
describe("getS3Status") {

26
core/src/test/scala/net/kemitix/thorp/core/SequencePlanTest.scala
View file

@ -3,22 +3,28 @@ package net.kemitix.thorp.core
import java.io.File
import net.kemitix.thorp.core.Action._
import net.kemitix.thorp.domain.{Bucket, LocalFile, MD5Hash, RemoteKey}
import net.kemitix.thorp.domain.{
Bucket,
HashType,
LocalFile,
MD5Hash,
RemoteKey
}
import org.scalatest.FreeSpec
class SequencePlanTest extends FreeSpec {
"sort" - {
"a list of assorted actions" - {
val bucket = Bucket("aBucket")
val remoteKey1 = RemoteKey("remoteKey1")
val remoteKey2 = RemoteKey("targetHash")
val hash = MD5Hash("aHash")
val hashes: Map[String, MD5Hash] = Map()
val size = 1024
val file1 = new File("aFile")
val file2 = new File("aFile")
val source = new File("source")
val bucket = Bucket("aBucket")
val remoteKey1 = RemoteKey("remoteKey1")
val remoteKey2 = RemoteKey("targetHash")
val hash = MD5Hash("aHash")
val hashes = Map[HashType, MD5Hash]()
val size = 1024
val file1 = new File("aFile")
val file2 = new File("aFile")
val source = new File("source")
val localFile1 =
LocalFile(file1, source, hashes, remoteKey1)
val localFile2 =

11
core/src/test/scala/net/kemitix/thorp/core/SyncSuite.scala
View file

@ -7,6 +7,7 @@ import java.time.Instant
import net.kemitix.thorp.console
import net.kemitix.thorp.console.MyConsole
import net.kemitix.thorp.core.Action.{ToCopy, ToDelete, ToUpload}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.MD5HashData.{Leaf, Root}
import net.kemitix.thorp.domain.StorageQueueEvent.{
CopyQueueEvent,
@ -43,8 +44,8 @@ class SyncSuite extends FunSpec {
private val hashService =
DummyHashService(
Map(
file("root-file") -> Map("md5" -> MD5HashData.Root.hash),
file("subdir/leaf-file") -> Map("md5" -> MD5HashData.Leaf.hash)
file("root-file") -> Map(MD5 -> MD5HashData.Root.hash),
file("subdir/leaf-file") -> Map(MD5 -> MD5HashData.Leaf.hash)
))
private val configOptions =
ConfigOptions(
@ -62,8 +63,8 @@ class SyncSuite extends FunSpec {
localFile: LocalFile): (String, String, File) =
(bucket.name, remoteKey.key, localFile.file)
private def md5HashMap(md5Hash: MD5Hash): Map[String, MD5Hash] =
Map("md5" -> md5Hash)
private def md5HashMap(md5Hash: MD5Hash): Map[HashType, MD5Hash] =
Map(MD5 -> md5Hash)
private def file(filename: String) =
sourcePath.resolve(Paths.get(filename))
@ -207,7 +208,7 @@ class SyncSuite extends FunSpec {
batchMode: Boolean,
uploadEventListener: UploadEventListener,
tryCount: Int): Task[UploadQueueEvent] =
Task(UploadQueueEvent(localFile.remoteKey, localFile.hashes("md5")))
Task(UploadQueueEvent(localFile.remoteKey, localFile.hashes(MD5)))
override def copy(bucket: Bucket,
sourceKey: RemoteKey,

7
domain/src/main/scala/net/kemitix/thorp/domain/HashType.scala Normal file
View file

@ -0,0 +1,7 @@
package net.kemitix.thorp.domain
trait HashType
object HashType {
case object MD5 extends HashType
}

8
domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala
View file

@ -3,10 +3,12 @@ package net.kemitix.thorp.domain
import java.io.File
import java.nio.file.Path
import net.kemitix.thorp.domain.HashType.MD5
final case class LocalFile(
file: File,
source: File,
hashes: Map[String, MD5Hash],
hashes: Map[HashType, MD5Hash],
remoteKey: RemoteKey
) {
@ -19,7 +21,7 @@ final case class LocalFile(
def matches(other: MD5Hash): Boolean = hashes.values.exists(other equals _)
def md5base64: Option[String] = hashes.get("md5").map(_.hash64)
def md5base64: Option[String] = hashes.get(MD5).map(_.hash64)
}
@ -27,7 +29,7 @@ object LocalFile {
def resolve(
path: String,
md5Hashes: Map[String, MD5Hash],
md5Hashes: Map[HashType, MD5Hash],
source: Path,
pathToKey: Path => RemoteKey
): LocalFile = {

4
storage-api/src/main/scala/net/kemitix/thorp/storage/api/HashService.scala
View file

@ -2,7 +2,7 @@ package net.kemitix.thorp.storage.api
import java.nio.file.Path
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.domain.{HashType, MD5Hash}
import zio.Task
/**
@ -10,6 +10,6 @@ import zio.Task
*/
trait HashService {
def hashLocalObject(path: Path): Task[Map[String, MD5Hash]]
def hashLocalObject(path: Path): Task[Map[HashType, MD5Hash]]
}

5
storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/ETag.scala Normal file
View file

@ -0,0 +1,5 @@
package net.kemitix.thorp.storage.aws
import net.kemitix.thorp.domain.HashType
object ETag extends HashType

9
storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3HashService.scala
View file

@ -3,7 +3,8 @@ package net.kemitix.thorp.storage.aws
import java.nio.file.Path
import net.kemitix.thorp.core.MD5HashGenerator
import net.kemitix.thorp.domain.MD5Hash
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.{HashType, MD5Hash}
import net.kemitix.thorp.storage.api.HashService
import zio.Task
@ -17,14 +18,14 @@ trait S3HashService extends HashService {
*/
override def hashLocalObject(
path: Path
): Task[Map[String, MD5Hash]] =
): Task[Map[HashType, MD5Hash]] =
for {
md5 <- MD5HashGenerator.md5File(path)
etag <- ETagGenerator.eTag(path).map(MD5Hash(_))
} yield
Map(
"md5" -> md5,
"etag" -> etag
MD5 -> md5,
ETag -> etag
)
}

5
storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala
View file

@ -5,6 +5,7 @@ import java.time.Instant
import com.amazonaws.services.s3.model.PutObjectRequest
import com.amazonaws.services.s3.transfer.model.UploadResult
import net.kemitix.thorp.core.{KeyGenerator, Resource, S3MetaDataEnricher}
import net.kemitix.thorp.domain.HashType.MD5
import net.kemitix.thorp.domain.MD5HashData.Root
import net.kemitix.thorp.domain.StorageQueueEvent.UploadQueueEvent
import net.kemitix.thorp.domain._
@ -113,8 +114,8 @@ class StorageServiceSuite extends FunSpec with MockFactory {
}
private def md5HashMap(hash: MD5Hash) =
Map("md5" -> hash)
private def md5HashMap(hash: MD5Hash): Map[HashType, MD5Hash] =
Map(MD5 -> hash)
val batchMode: Boolean = true