From 3d4c238030a2f6f689d32e9560b78ffd0f936b89 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sun, 8 Sep 2019 18:59:43 +0100 Subject: [PATCH] Add more tests (#192) * [storage-aws] Refactoring * [lib] Add test for FileScanner * [lib] Add LocalFileSystemTest for scanCopyUpload * [lib] Add LocalFileSystem tests for scanDelete Also send a UIEvent for ActionChosen(ToDelete) * [lib] LocalFileSystemTest can handle files in any order * [lib] LocalFileSystemTest can handle files in any order #2 Don't include accountCounter or byteCounter are the order can change. * [domain] Remove LocalFile.relativeToSource * [sbt] Add missing modules to aggregate --- build.sbt | 2 +- .../net/kemitix/thorp/domain/Action.scala | 9 +- .../net/kemitix/thorp/domain/LocalFile.scala | 6 +- .../kemitix/thorp/domain/MD5HashData.scala | 4 + .../kemitix/thorp/lib/LocalFileSystem.scala | 22 +- .../kemitix/thorp/lib/FileScannerTest.scala | 62 ++++ .../thorp/lib/LocalFileSystemTest.scala | 341 ++++++++++++++++++ .../storage/aws/hasher/ETagGenerator.scala | 24 +- .../aws/hasher/ETagGeneratorTest.scala | 43 +-- 9 files changed, 459 insertions(+), 54 deletions(-) create mode 100644 lib/src/test/scala/net/kemitix/thorp/lib/FileScannerTest.scala create mode 100644 lib/src/test/scala/net/kemitix/thorp/lib/LocalFileSystemTest.scala diff --git a/build.sbt b/build.sbt index d30cfda..701b512 100644 --- a/build.sbt +++ b/build.sbt @@ -77,7 +77,7 @@ val eipDependencies = Seq( lazy val thorp = (project in file(".")) .settings(commonSettings) - .aggregate(app, cli, `storage-aws`, lib, `storage`, domain) + .aggregate(app, cli, config, console, domain, filesystem, lib, storage, `storage-aws`, uishell) lazy val app = (project in file("app")) .settings(commonSettings) diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/Action.scala b/domain/src/main/scala/net/kemitix/thorp/domain/Action.scala index aa30550..3858fe7 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/Action.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/Action.scala @@ -3,6 +3,7 @@ package net.kemitix.thorp.domain sealed trait Action { def bucket: Bucket def size: Long + def remoteKey: RemoteKey } object Action { @@ -16,7 +17,9 @@ object Action { bucket: Bucket, localFile: LocalFile, size: Long - ) extends Action + ) extends Action { + override def remoteKey: RemoteKey = localFile.remoteKey + } final case class ToCopy( bucket: Bucket, @@ -24,7 +27,9 @@ object Action { hash: MD5Hash, targetKey: RemoteKey, size: Long - ) extends Action + ) extends Action { + override def remoteKey: RemoteKey = targetKey + } final case class ToDelete( bucket: Bucket, diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala b/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala index dad6864..fbb9bb8 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/LocalFile.scala @@ -1,10 +1,9 @@ package net.kemitix.thorp.domain import java.io.File -import java.nio.file.Path import net.kemitix.thorp.domain.HashType.MD5 -import Implicits._ +import net.kemitix.thorp.domain.Implicits._ final case class LocalFile private ( file: File, @@ -18,9 +17,6 @@ object LocalFile { val remoteKey: SimpleLens[LocalFile, RemoteKey] = SimpleLens[LocalFile, RemoteKey](_.remoteKey, b => a => b.copy(remoteKey = a)) - // the path of the file within the source - def relativeToSource(localFile: LocalFile): Path = - localFile.source.toPath.relativize(localFile.file.toPath) def matchesHash(localFile: LocalFile)(other: MD5Hash): Boolean = localFile.hashes.values.exists(other === _) def md5base64(localFile: LocalFile): Option[String] = diff --git a/domain/src/test/scala/net/kemitix/thorp/domain/MD5HashData.scala b/domain/src/test/scala/net/kemitix/thorp/domain/MD5HashData.scala index 4c5dd1d..8c714b0 100644 --- a/domain/src/test/scala/net/kemitix/thorp/domain/MD5HashData.scala +++ b/domain/src/test/scala/net/kemitix/thorp/domain/MD5HashData.scala @@ -5,10 +5,14 @@ object MD5HashData { object Root { val hash: MD5Hash = MD5Hash("a3a6ac11a0eb577b81b3bb5c95cc8a6e") val base64: String = "o6asEaDrV3uBs7tclcyKbg==" + val remoteKey = RemoteKey("root-file") + val size: Long = 55 } object Leaf { val hash: MD5Hash = MD5Hash("208386a650bdec61cfcd7bd8dcb6b542") val base64: String = "IIOGplC97GHPzXvY3La1Qg==" + val remoteKey = RemoteKey("subdir/leaf-file") + val size: Long = 58 } object BigFile { val hash: MD5Hash = MD5Hash("b1ab1f7680138e6db7309200584e35d8") diff --git a/lib/src/main/scala/net/kemitix/thorp/lib/LocalFileSystem.scala b/lib/src/main/scala/net/kemitix/thorp/lib/LocalFileSystem.scala index 7a68be0..6c3e800 100644 --- a/lib/src/main/scala/net/kemitix/thorp/lib/LocalFileSystem.scala +++ b/lib/src/main/scala/net/kemitix/thorp/lib/LocalFileSystem.scala @@ -30,7 +30,7 @@ trait LocalFileSystem { def scanDelete( uiChannel: UChannel[Any, UIEvent], remoteData: RemoteObjects, - archive: UnversionedMirrorArchive.type + archive: ThorpArchive ): RIO[Clock with Config with FileSystem with Storage, Seq[StorageEvent]] } @@ -63,7 +63,7 @@ object LocalFileSystem extends LocalFileSystem { override def scanDelete( uiChannel: UChannel[Any, UIEvent], remoteData: RemoteObjects, - archive: UnversionedMirrorArchive.type + archive: ThorpArchive ): RIO[Clock with Config with FileSystem with Storage, Seq[StorageEvent]] = for { actionCounter <- Ref.make(0) @@ -92,18 +92,23 @@ object LocalFileSystem extends LocalFileSystem { UIO { message => val localFile = message.body for { - _ <- uiFileFound(uiChannel)(localFile) - action <- chooseAction(remoteObjects, uploads, uiChannel)(localFile) - actionCounter <- actionCounterRef.update(_ + 1) - bytesCounter <- bytesCounterRef.update(_ + action.size) - actionChosenMessage <- Message.create(UIEvent.ActionChosen(action)) - _ <- MessageChannel.send(uiChannel)(actionChosenMessage) + _ <- uiFileFound(uiChannel)(localFile) + action <- chooseAction(remoteObjects, uploads, uiChannel)(localFile) + actionCounter <- actionCounterRef.update(_ + 1) + bytesCounter <- bytesCounterRef.update(_ + action.size) + _ <- uiActionChosen(uiChannel)(action) sequencedAction = SequencedAction(action, actionCounter) event <- archive.update(sequencedAction, bytesCounter) _ <- eventsRef.update(list => event :: list) _ <- uiActionFinished(uiChannel)(action, actionCounter, bytesCounter) } yield () } + + private def uiActionChosen(uiChannel: MessageChannel.UChannel[Any, UIEvent])( + action: Action) = + Message.create(UIEvent.ActionChosen(action)) >>= + MessageChannel.send(uiChannel) + private def uiActionFinished(uiChannel: UChannel[Any, UIEvent])( action: Action, actionCounter: Int, @@ -234,6 +239,7 @@ object LocalFileSystem extends LocalFileSystem { actionCounter <- actionCounterRef.update(_ + 1) bucket <- Config.bucket action = ToDelete(bucket, remoteKey, 0L) + _ <- uiActionChosen(uiChannel)(action) bytesCounter <- bytesCounterRef.update(_ + action.size) sequencedAction = SequencedAction(action, actionCounter) event <- archive.update(sequencedAction, 0L) diff --git a/lib/src/test/scala/net/kemitix/thorp/lib/FileScannerTest.scala b/lib/src/test/scala/net/kemitix/thorp/lib/FileScannerTest.scala new file mode 100644 index 0000000..cfcd04e --- /dev/null +++ b/lib/src/test/scala/net/kemitix/thorp/lib/FileScannerTest.scala @@ -0,0 +1,62 @@ +package net.kemitix.thorp.lib + +import java.util.concurrent.atomic.AtomicReference + +import net.kemitix.eip.zio.MessageChannel +import net.kemitix.thorp.config.{ + Config, + ConfigOption, + ConfigOptions, + ConfigurationBuilder +} +import net.kemitix.thorp.domain.{LocalFile, RemoteKey} +import net.kemitix.thorp.filesystem.{FileSystem, Hasher, Resource} +import net.kemitix.thorp.lib.FileScanner.ScannedFile +import org.scalatest.FreeSpec +import zio.clock.Clock +import zio.{DefaultRuntime, Ref, UIO} + +class FileScannerTest extends FreeSpec { + + "scanSources" - { + "creates a FileSender for files in resources" in { + def receiver(scanned: Ref[List[RemoteKey]]) + : UIO[MessageChannel.UReceiver[Any, ScannedFile]] = UIO { message => + for { + _ <- scanned.update(l => LocalFile.remoteKey.get(message.body) :: l) + } yield () + } + val scannedFiles = + new AtomicReference[List[RemoteKey]](List.empty) + val sourcePath = Resource(this, "upload").toPath + val configOptions: List[ConfigOption] = + List[ConfigOption](ConfigOption.Source(sourcePath), + ConfigOption.Bucket("bucket"), + ConfigOption.IgnoreGlobalOptions, + ConfigOption.IgnoreUserOptions) + val program = for { + config <- ConfigurationBuilder.buildConfig(ConfigOptions(configOptions)) + _ <- Config.set(config) + scanner <- FileScanner.scanSources + scannedRef <- Ref.make[List[RemoteKey]](List.empty) + receiver <- receiver(scannedRef) + _ <- MessageChannel.pointToPoint(scanner)(receiver).runDrain + scanned <- scannedRef.get + _ <- UIO(scannedFiles.set(scanned)) + } yield () + object TestEnv + extends FileScanner.Live + with Clock.Live + with Hasher.Live + with FileSystem.Live + with Config.Live + val completed = + new DefaultRuntime {}.unsafeRunSync(program.provide(TestEnv)).toEither + assert(completed.isRight) + assertResult(Set(RemoteKey("root-file"), RemoteKey("subdir/leaf-file")))( + scannedFiles.get.toSet) + } + + } + +} diff --git a/lib/src/test/scala/net/kemitix/thorp/lib/LocalFileSystemTest.scala b/lib/src/test/scala/net/kemitix/thorp/lib/LocalFileSystemTest.scala new file mode 100644 index 0000000..7eb8c19 --- /dev/null +++ b/lib/src/test/scala/net/kemitix/thorp/lib/LocalFileSystemTest.scala @@ -0,0 +1,341 @@ +package net.kemitix.thorp.lib + +import java.util.concurrent.atomic.AtomicReference + +import net.kemitix.eip.zio.MessageChannel +import net.kemitix.thorp.config.ConfigOption.{ + IgnoreGlobalOptions, + IgnoreUserOptions +} +import net.kemitix.thorp.config.{ + Config, + ConfigOption, + ConfigOptions, + ConfigurationBuilder +} +import net.kemitix.thorp.domain.Action.{DoNothing, ToCopy, ToDelete, ToUpload} +import net.kemitix.thorp.domain._ +import net.kemitix.thorp.filesystem.{FileSystem, Hasher, Resource} +import net.kemitix.thorp.storage.Storage +import net.kemitix.throp.uishell.UIEvent +import net.kemitix.throp.uishell.UIEvent._ +import org.scalatest.FreeSpec +import org.scalatest.Matchers._ +import zio.clock.Clock +import zio.{DefaultRuntime, UIO} + +import scala.collection.MapView + +class LocalFileSystemTest extends FreeSpec { + + private val source = Resource(this, "upload") + private val sourcePath = source.toPath + private val sourceOption = ConfigOption.Source(sourcePath) + private val bucket = Bucket("bucket") + private val bucketOption = ConfigOption.Bucket(bucket.name) + private val configOptions = ConfigOptions( + List[ConfigOption]( + sourceOption, + bucketOption, + IgnoreGlobalOptions, + IgnoreUserOptions + )) + + private val uiEvents = new AtomicReference[List[UIEvent]](List.empty) + private val actions = new AtomicReference[List[SequencedAction]](List.empty) + + private def archive: ThorpArchive = + (sequencedAction: SequencedAction, _) => + UIO { + actions.updateAndGet(l => sequencedAction :: l) + StorageEvent.DoNothingEvent(sequencedAction.action.remoteKey) + } + + private val runtime = new DefaultRuntime {} + + private object TestEnv + extends Clock.Live + with Hasher.Live + with FileSystem.Live + with Config.Live + with FileScanner.Live + with Storage.Test + + "scanCopyUpload" - { + def sender(objects: RemoteObjects): UIO[MessageChannel.ESender[ + Clock with Hasher with FileSystem with Config with FileScanner with Config with Storage, + Throwable, + UIEvent]] = + UIO { uiChannel => + (for { + _ <- LocalFileSystem.scanCopyUpload(uiChannel, objects, archive) + } yield ()) <* MessageChannel.endChannel(uiChannel) + } + def receiver(): UIO[MessageChannel.UReceiver[Any, UIEvent]] = + UIO { message => + val uiEvent = message.body + uiEvents.updateAndGet(l => uiEvent :: l) + UIO(()) + } + def program(remoteObjects: RemoteObjects) = + for { + config <- ConfigurationBuilder.buildConfig(configOptions) + _ <- Config.set(config) + sender <- sender(remoteObjects) + receiver <- receiver() + _ <- MessageChannel.pointToPoint(sender)(receiver).runDrain + } yield () + "where remote has no objects" - { + val remoteObjects = RemoteObjects.empty + "upload all files" - { + "update archive with upload actions" in { + actions.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val actionList: Set[Action] = actions.get.map(_.action).toSet + actionList.filter(_.isInstanceOf[ToUpload]) should have size 2 + actionList.map(_.remoteKey) shouldEqual Set( + MD5HashData.Root.remoteKey, + MD5HashData.Leaf.remoteKey) + } + "ui is updated" in { + uiEvents.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val summary = uiEventsSummary + summary should have size 6 + summary should contain inOrderElementsOf List( + "file found : root-file", + "action chosen : root-file : ToUpload", + "action finished : root-file : ToUpload") + summary should contain inOrderElementsOf List( + "file found : subdir/leaf-file", + "action chosen : subdir/leaf-file : ToUpload", + "action finished : subdir/leaf-file : ToUpload" + ) + } + } + } + "where remote has all object" - { + val remoteObjects = + RemoteObjects( + byHash = MapView(MD5HashData.Root.hash -> MD5HashData.Root.remoteKey, + MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey), + byKey = MapView(MD5HashData.Root.remoteKey -> MD5HashData.Root.hash, + MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash) + ) + "do nothing for all files" - { + "all archive actions do nothing" in { + actions.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val actionList: Set[Action] = actions.get.map(_.action).toSet + actionList should have size 2 + actionList.filter(_.isInstanceOf[DoNothing]) should have size 2 + } + "ui is updated" in { + uiEvents.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val summary = uiEventsSummary + summary should have size 6 + summary should contain inOrderElementsOf List( + "file found : root-file", + "action chosen : root-file : DoNothing", + "action finished : root-file : DoNothing") + summary should contain inOrderElementsOf List( + "file found : subdir/leaf-file", + "action chosen : subdir/leaf-file : DoNothing", + "action finished : subdir/leaf-file : DoNothing" + ) + } + } + } + "where remote has some objects" - { + val remoteObjects = + RemoteObjects( + byHash = MapView(MD5HashData.Root.hash -> MD5HashData.Root.remoteKey), + byKey = MapView(MD5HashData.Root.remoteKey -> MD5HashData.Root.hash) + ) + "upload leaf, do nothing for root" - { + "archive actions upload leaf" in { + actions.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val actionList: Set[Action] = actions.get.map(_.action).toSet + actionList + .filter(_.isInstanceOf[DoNothing]) + .map(_.remoteKey) shouldEqual Set(MD5HashData.Root.remoteKey) + actionList + .filter(_.isInstanceOf[ToUpload]) + .map(_.remoteKey) shouldEqual Set(MD5HashData.Leaf.remoteKey) + } + "ui is updated" in { + uiEvents.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val summary = uiEventsSummary + summary should contain inOrderElementsOf List( + "file found : root-file", + "action chosen : root-file : DoNothing", + "action finished : root-file : DoNothing") + summary should contain inOrderElementsOf List( + "file found : subdir/leaf-file", + "action chosen : subdir/leaf-file : ToUpload", + "action finished : subdir/leaf-file : ToUpload" + ) + } + } + } + "where remote objects are swapped" ignore { + val remoteObjects = + RemoteObjects( + byHash = MapView(MD5HashData.Root.hash -> MD5HashData.Leaf.remoteKey, + MD5HashData.Leaf.hash -> MD5HashData.Root.remoteKey), + byKey = MapView(MD5HashData.Root.remoteKey -> MD5HashData.Leaf.hash, + MD5HashData.Leaf.remoteKey -> MD5HashData.Root.hash) + ) + "copy files" - { + "archive swaps objects" ignore { + // TODO this is not supported + } + } + } + "where file has been renamed" - { + // renamed from "other/root" to "root-file" + val otherRootKey = RemoteKey("other/root") + val remoteObjects = + RemoteObjects( + byHash = MapView(MD5HashData.Root.hash -> otherRootKey, + MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey), + byKey = MapView(otherRootKey -> MD5HashData.Root.hash, + MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash) + ) + "copy object and delete original" in { + actions.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val actionList: Set[Action] = actions.get.map(_.action).toSet + actionList should have size 2 + actionList + .filter(_.isInstanceOf[DoNothing]) + .map(_.remoteKey) shouldEqual Set(MD5HashData.Leaf.remoteKey) + actionList + .filter(_.isInstanceOf[ToCopy]) + .map(_.remoteKey) shouldEqual Set(MD5HashData.Root.remoteKey) + } + "ui is updated" in { + uiEvents.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val summary = uiEventsSummary + summary should contain inOrderElementsOf List( + "file found : root-file", + "action chosen : root-file : ToCopy", + "action finished : root-file : ToCopy") + summary should contain inOrderElementsOf List( + "file found : subdir/leaf-file", + "action chosen : subdir/leaf-file : DoNothing", + "action finished : subdir/leaf-file : DoNothing" + ) + } + } + } + + "scanDelete" - { + def sender(objects: RemoteObjects): UIO[ + MessageChannel.ESender[Clock with Config with FileSystem with Storage, + Throwable, + UIEvent]] = + UIO { uiChannel => + (for { + _ <- LocalFileSystem.scanDelete(uiChannel, objects, archive) + } yield ()) <* MessageChannel.endChannel(uiChannel) + } + def receiver(): UIO[MessageChannel.UReceiver[Any, UIEvent]] = + UIO { message => + val uiEvent = message.body + uiEvents.updateAndGet(l => uiEvent :: l) + UIO(()) + } + def program(remoteObjects: RemoteObjects) = { + for { + config <- ConfigurationBuilder.buildConfig(configOptions) + _ <- Config.set(config) + sender <- sender(remoteObjects) + receiver <- receiver() + _ <- MessageChannel.pointToPoint(sender)(receiver).runDrain + } yield () + } + "where remote has no extra objects" - { + val remoteObjects = RemoteObjects( + byHash = MapView(MD5HashData.Root.hash -> MD5HashData.Root.remoteKey, + MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey), + byKey = MapView(MD5HashData.Root.remoteKey -> MD5HashData.Root.hash, + MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash) + ) + "do nothing for all files" - { + "no archive actions" in { + actions.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val actionList: Set[Action] = actions.get.map(_.action).toSet + actionList should have size 0 + } + "ui is updated" in { + uiEvents.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + uiEventsSummary shouldEqual List("key found: root-file", + "key found: subdir/leaf-file") + } + } + } + "where remote has extra objects" - { + val extraHash = MD5Hash("extra") + val extraObject = RemoteKey("extra") + val remoteObjects = RemoteObjects( + byHash = MapView(MD5HashData.Root.hash -> MD5HashData.Root.remoteKey, + MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey, + extraHash -> extraObject), + byKey = MapView(MD5HashData.Root.remoteKey -> MD5HashData.Root.hash, + MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash, + extraObject -> extraHash) + ) + "remove the extra object" - { + "archive delete action" in { + actions.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + val actionList: Set[Action] = actions.get.map(_.action).toSet + actionList should have size 1 + actionList + .filter(_.isInstanceOf[ToDelete]) + .map(_.remoteKey) shouldEqual Set(extraObject) + } + "ui is updated" in { + uiEvents.set(List.empty) + runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv)) + uiEventsSummary shouldEqual List( + "key found: root-file", + "key found: subdir/leaf-file", + "key found: extra", + "action chosen : extra : ToDelete", + "action finished : extra : ToDelete" + ) + } + } + } + } + + private def uiEventsSummary: List[String] = { + uiEvents + .get() + .reverse + .map { + case FileFound(localFile) => + String.format("file found : %s", localFile.remoteKey.key) + case ActionChosen(action) => + String.format("action chosen : %s : %s", + action.remoteKey.key, + action.getClass.getSimpleName) + case ActionFinished(action, actionCounter, bytesCounter) => + String.format("action finished : %s : %s", + action.remoteKey.key, + action.getClass.getSimpleName) + case KeyFound(remoteKey) => + String.format("key found: %s", remoteKey.key) + case x => String.format("unknown : %s", x.getClass.getSimpleName) + } + } + +} diff --git a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala index fb9301e..2d4a87a 100644 --- a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala +++ b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/hasher/ETagGenerator.scala @@ -12,15 +12,25 @@ import zio.{RIO, ZIO} private trait ETagGenerator { - def eTag( - path: Path - ): RIO[Hasher with FileSystem, String] = { + def eTag(path: Path): RIO[Hasher with FileSystem, String] + + def offsets(totalFileSizeBytes: Long, optimalPartSize: Long): List[Long] + +} + +private object ETagGenerator extends ETagGenerator { + + override def eTag(path: Path): RIO[Hasher with FileSystem, String] = { val partSize = calculatePartSize(path) val parts = numParts(path.toFile.length, partSize) eTagHex(path, partSize, parts) .map(hash => s"$hash-$parts") } + override def offsets(totalFileSizeBytes: Long, + optimalPartSize: Long): List[Long] = + Range.Long(0, totalFileSizeBytes, optimalPartSize).toList + private def eTagHex(path: Path, partSize: Long, parts: Long) = ZIO .foreach(partsIndex(parts))(digestChunk(path, partSize)) @@ -57,12 +67,4 @@ private trait ETagGenerator { .hashObjectChunk(path, chunkNumber, chunkSize) .map(_(MD5)) .map(MD5Hash.digest) - - def offsets( - totalFileSizeBytes: Long, - optimalPartSize: Long - ): List[Long] = - Range.Long(0, totalFileSizeBytes, optimalPartSize).toList } - -private object ETagGenerator extends ETagGenerator diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala index 876e5b2..6d57187 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/hasher/ETagGeneratorTest.scala @@ -1,17 +1,13 @@ package net.kemitix.thorp.storage.aws.hasher -import java.nio.file.Path - import com.amazonaws.services.s3.transfer.TransferManagerConfiguration import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain.MD5Hash import net.kemitix.thorp.filesystem.{FileSystem, Hasher, Resource} -import org.scalatest.FunSpec +import org.scalatest.FreeSpec import zio.DefaultRuntime -class ETagGeneratorTest extends FunSpec { - - object TestEnv extends Hasher.Live with FileSystem.Live +class ETagGeneratorTest extends FreeSpec { private val bigFile = Resource(this, "../big-file") private val bigFilePath = bigFile.toPath @@ -19,8 +15,8 @@ class ETagGeneratorTest extends FunSpec { private val chunkSize = 1200000 configuration.setMinimumUploadPartSize(chunkSize) - describe("Create offsets") { - it("should create offsets") { + "Create offsets" - { + "should create offsets" in { val offsets = ETagGenerator .offsets(bigFile.length, chunkSize) .foldRight(List[Long]())((l: Long, a: List[Long]) => l :: a) @@ -30,8 +26,11 @@ class ETagGeneratorTest extends FunSpec { } } - describe("create md5 hash for each chunk") { - it("should create expected hash for chunks") { + private val runtime: DefaultRuntime = new DefaultRuntime {} + object TestEnv extends Hasher.Live with FileSystem.Live + + "create md5 hash for each chunk" - { + "should create expected hash for chunks" in { val md5Hashes = List( "68b7d37e6578297621e06f01800204f1", "973475b14a7bda6ad8864a7f9913a947", @@ -41,33 +40,23 @@ class ETagGeneratorTest extends FunSpec { ).zipWithIndex md5Hashes.foreach { case (hash, index) => + val program = Hasher.hashObjectChunk(bigFilePath, index, chunkSize) + val result = runtime.unsafeRunSync(program.provide(TestEnv)).toEither assertResult(Right(hash))( - invoke(bigFilePath, index, chunkSize) + result .map(_(MD5)) .map(MD5Hash.hash)) } } - def invoke(path: Path, index: Long, size: Long) = - new DefaultRuntime {}.unsafeRunSync { - Hasher - .hashObjectChunk(path, index, size) - .provide(TestEnv) - }.toEither } - describe("create etag for whole file") { + "create etag for whole file" - { val expected = "f14327c90ad105244c446c498bfe9a7d-2" - it("should match aws etag for the file") { - val result = invoke(bigFilePath) + "should match aws etag for the file" in { + val program = ETagGenerator.eTag(bigFilePath) + val result = runtime.unsafeRunSync(program.provide(TestEnv)).toEither assertResult(Right(expected))(result) } - def invoke(path: Path) = { - new DefaultRuntime {}.unsafeRunSync { - ETagGenerator - .eTag(path) - .provide(TestEnv) - }.toEither - } } }