From 321773f04c3169f73d5c1211961c54974c17aff9 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sun, 1 Sep 2019 21:30:16 +0100 Subject: [PATCH] Upgrade to Scala 2.13.0 (#176) --- .travis.yml | 2 +- build.sbt | 14 ++++-- .../kemitix/thorp/core/ActionGenerator.scala | 20 ++++---- .../net/kemitix/thorp/core/EventQueue.scala | 2 +- .../kemitix/thorp/core/LocalFileStream.scala | 11 +++-- .../net/kemitix/thorp/core/LocalFiles.scala | 8 ++-- .../net/kemitix/thorp/core/PlanBuilder.scala | 14 +++--- .../net/kemitix/thorp/core/PlanExecutor.scala | 6 +-- .../net/kemitix/thorp/core/SyncPlan.scala | 6 +-- .../thorp/core/ActionGeneratorSuite.scala | 16 +++---- .../core/MatchedMetadataEnricherSuite.scala | 28 ++++++----- .../kemitix/thorp/core/PlanBuilderTest.scala | 38 ++++++++------- .../kemitix/thorp/core/PlanExecutorTest.scala | 12 +++-- .../net/kemitix/thorp/domain/HexEncoder.scala | 2 + .../net/kemitix/thorp/domain/NonUnit.scala | 2 +- .../kemitix/thorp/domain/RemoteObjects.scala | 12 +++-- .../kemitix/thorp/filesystem/FileSystem.scala | 2 +- .../kemitix/thorp/storage/aws/Lister.scala | 14 +++--- .../thorp/storage/aws/S3ObjectsByHash.scala | 10 ++-- .../thorp/storage/aws/S3ObjectsByKey.scala | 16 ++++--- .../thorp/storage/aws/ListerTest.scala | 48 +++++++++++-------- .../storage/aws/S3ObjectsByHashSuite.scala | 4 +- .../storage/aws/StorageServiceSuite.scala | 6 ++- 23 files changed, 162 insertions(+), 131 deletions(-) diff --git a/.travis.yml b/.travis.yml index f19613c..f5af1ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: scala scala: - - 2.12.8 + - 2.13.0 jdk: - openjdk8 - openjdk11 diff --git a/build.sbt b/build.sbt index c68f9bc..f1bd087 100644 --- a/build.sbt +++ b/build.sbt @@ -16,17 +16,23 @@ inThisBuild(List( val commonSettings = Seq( sonatypeProfileName := "net.kemitix", - scalaVersion := "2.12.8", + scalaVersion := "2.13.0", scalacOptions ++= Seq( - "-Ywarn-unused-import", + "-Ywarn-unused:imports", "-Xfatal-warnings", "-feature", "-deprecation", "-unchecked", "-language:postfixOps", "-language:higherKinds", - "-Ypartial-unification"), - wartremoverErrors ++= Warts.unsafe.filterNot(wart => List(Wart.Any, Wart.Nothing, Wart.Serializable).contains(wart)), + "-language:higherKinds"), + wartremoverErrors ++= Warts.unsafe.filterNot(wart => List( + Wart.Any, + Wart.Nothing, + Wart.Serializable, + Wart.NonUnitStatements, + Wart.StringPlusAny + ).contains(wart)), test in assembly := {} ) diff --git a/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala b/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala index d813ab9..7faa7da 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/ActionGenerator.scala @@ -10,8 +10,8 @@ object ActionGenerator { def createActions( matchedMetadata: MatchedMetadata, - previousActions: Stream[Action] - ): RIO[Config, Stream[Action]] = + previousActions: LazyList[Action] + ): RIO[Config, LazyList[Action]] = for { bucket <- Config.bucket } yield @@ -19,7 +19,7 @@ object ActionGenerator { private def formattedMetadata( matchedMetadata: MatchedMetadata, - previousActions: Stream[Action]): TaggedMetadata = { + previousActions: LazyList[Action]): TaggedMetadata = { val remoteExists = matchedMetadata.matchByKey.nonEmpty val remoteMatches = remoteExists && matchedMetadata.matchByKey.exists(m => LocalFile.matchesHash(matchedMetadata.localFile)(m.hash)) @@ -33,14 +33,14 @@ object ActionGenerator { final case class TaggedMetadata( matchedMetadata: MatchedMetadata, - previousActions: Stream[Action], + previousActions: LazyList[Action], remoteExists: Boolean, remoteMatches: Boolean, anyMatches: Boolean ) private def genAction(taggedMetadata: TaggedMetadata, - bucket: Bucket): Stream[Action] = { + bucket: Bucket): LazyList[Action] = { taggedMetadata match { case TaggedMetadata(md, _, remoteExists, remoteMatches, _) if remoteExists && remoteMatches => @@ -58,7 +58,7 @@ object ActionGenerator { private def key = LocalFile.remoteKey ^|-> RemoteKey.key def isNotUploadAlreadyQueued( - previousActions: Stream[Action] + previousActions: LazyList[Action] )( localFile: LocalFile ): Boolean = !previousActions.exists { @@ -69,21 +69,21 @@ object ActionGenerator { private def doNothing( bucket: Bucket, remoteKey: RemoteKey - ) = Stream(DoNothing(bucket, remoteKey, 0L)) + ) = LazyList(DoNothing(bucket, remoteKey, 0L)) private def uploadFile( bucket: Bucket, localFile: LocalFile - ) = Stream(ToUpload(bucket, localFile, localFile.file.length)) + ) = LazyList(ToUpload(bucket, localFile, localFile.file.length)) private def copyFile( bucket: Bucket, localFile: LocalFile, remoteMetaData: Set[RemoteMetaData] ) = - remoteMetaData + LazyList + .from(remoteMetaData) .take(1) - .toStream .map( other => ToCopy(bucket, diff --git a/core/src/main/scala/net/kemitix/thorp/core/EventQueue.scala b/core/src/main/scala/net/kemitix/thorp/core/EventQueue.scala index 0c7aeda..22adae1 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/EventQueue.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/EventQueue.scala @@ -3,6 +3,6 @@ package net.kemitix.thorp.core import net.kemitix.thorp.domain.StorageQueueEvent final case class EventQueue( - events: Stream[StorageQueueEvent], + events: LazyList[StorageQueueEvent], bytesInQueue: Long ) diff --git a/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala b/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala index c9a03d4..e2b86e7 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/LocalFileStream.scala @@ -22,11 +22,11 @@ object LocalFileStream { case _ => localFile(path) } - def recurse(paths: Stream[Path]) + def recurse(paths: LazyList[Path]) : RIO[Config with FileSystem with Hasher, LocalFiles] = for { recursed <- ZIO.foreach(paths)(path => recurseIntoSubDirectories(path)) - } yield LocalFiles.reduce(recursed.toStream) + } yield LocalFiles.reduce(LazyList.from(recursed)) def loop(path: Path): RIO[Config with FileSystem with Hasher, LocalFiles] = dirPaths(path) >>= recurse @@ -37,12 +37,13 @@ object LocalFileStream { private def dirPaths(path: Path) = listFiles(path) >>= includedDirPaths - private def includedDirPaths(paths: Stream[Path]) = + private def includedDirPaths(paths: LazyList[Path]) = for { flaggedPaths <- RIO.foreach(paths)(path => isIncluded(path).map((path, _))) } yield - flaggedPaths.toStream + LazyList + .from(flaggedPaths) .filter({ case (_, included) => included }) .map({ case (path, _) => path }) @@ -63,7 +64,7 @@ object LocalFileStream { for { files <- Task(path.toFile.listFiles) _ <- filesMustExist(path, files) - } yield Stream(files: _*).map(_.toPath) + } yield LazyList.from(files.toIndexedSeq).map(_.toPath) private def filesMustExist(path: Path, files: Array[File]) = Task { diff --git a/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala b/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala index 6981e4a..6b9b0af 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/LocalFiles.scala @@ -3,7 +3,7 @@ package net.kemitix.thorp.core import net.kemitix.thorp.domain.LocalFile final case class LocalFiles( - localFiles: Stream[LocalFile], + localFiles: LazyList[LocalFile], count: Long, totalSizeBytes: Long ) { @@ -16,9 +16,9 @@ final case class LocalFiles( } object LocalFiles { - val empty: LocalFiles = LocalFiles(Stream.empty, 0L, 0L) - def reduce: Stream[LocalFiles] => LocalFiles = + val empty: LocalFiles = LocalFiles(LazyList.empty, 0L, 0L) + def reduce: LazyList[LocalFiles] => LocalFiles = list => list.foldLeft(LocalFiles.empty)((acc, lf) => acc ++ lf) def one(localFile: LocalFile): LocalFiles = - LocalFiles(Stream(localFile), 1, localFile.file.length) + LocalFiles(LazyList(localFile), 1, localFile.file.length) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala b/core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala index fe20b9c..638a61e 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/PlanBuilder.scala @@ -31,14 +31,14 @@ object PlanBuilder { .map(syncPlan(localData)) } - private def syncPlan(localData: LocalFiles): Stream[Action] => SyncPlan = + private def syncPlan(localData: LocalFiles): LazyList[Action] => SyncPlan = SyncPlan.create(_, syncTotal(localData)) private def syncTotal(localData: LocalFiles): SyncTotals = SyncTotals.create(localData.count, localData.totalSizeBytes, 0L) private def createActions(remoteObjects: RemoteObjects, - localFiles: Stream[LocalFile]) = + localFiles: LazyList[LocalFile]) = for { fileActions <- actionsForLocalFiles(remoteObjects, localFiles) remoteActions <- actionsForRemoteKeys(remoteObjects.byKey.keys) @@ -50,15 +50,15 @@ object PlanBuilder { } private def actionsForLocalFiles(remoteObjects: RemoteObjects, - localFiles: Stream[LocalFile]) = - ZIO.foldLeft(localFiles)(Stream.empty[Action])( + localFiles: LazyList[LocalFile]) = + ZIO.foldLeft(localFiles)(LazyList.empty[Action])( (acc, localFile) => createActionsFromLocalFile(remoteObjects, acc, localFile) .map(_ #::: acc) ) private def createActionsFromLocalFile(remoteObjects: RemoteObjects, - previousActions: Stream[Action], + previousActions: LazyList[Action], localFile: LocalFile) = ActionGenerator.createActions( S3MetaDataEnricher.getMetadata(localFile, remoteObjects), @@ -66,7 +66,7 @@ object PlanBuilder { ) private def actionsForRemoteKeys(remoteKeys: Iterable[RemoteKey]) = - ZIO.foldLeft(remoteKeys)(Stream.empty[Action])( + ZIO.foldLeft(remoteKeys)(LazyList.empty[Action])( (acc, remoteKey) => createActionFromRemoteKey(remoteKey).map(_ #:: acc) ) @@ -90,6 +90,6 @@ object PlanBuilder { sources <- Config.sources found <- ZIO.foreach(sources.paths)(LocalFileStream.findFiles) _ <- Console.putStrLn(s"Found ${found.flatMap(_.localFiles).size} files") - } yield LocalFiles.reduce(found.toStream) + } yield LocalFiles.reduce(LazyList.from(found)) } diff --git a/core/src/main/scala/net/kemitix/thorp/core/PlanExecutor.scala b/core/src/main/scala/net/kemitix/thorp/core/PlanExecutor.scala index b54406f..7562d73 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/PlanExecutor.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/PlanExecutor.scala @@ -29,9 +29,9 @@ trait PlanExecutor { bytesCounter: Ref[Long] ): ZIO[Storage with Console with Config, Throwable, - Stream[StorageQueueEvent]] = { - ZIO.foldLeft(syncPlan.actions)(Stream.empty[StorageQueueEvent]) { - (stream: Stream[StorageQueueEvent], action) => + LazyList[StorageQueueEvent]] = { + ZIO.foldLeft(syncPlan.actions)(LazyList.empty[StorageQueueEvent]) { + (stream: LazyList[StorageQueueEvent], action) => val result: ZIO[Storage with Console with Config, Throwable, StorageQueueEvent] = diff --git a/core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala b/core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala index c404692..1815f59 100644 --- a/core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala +++ b/core/src/main/scala/net/kemitix/thorp/core/SyncPlan.scala @@ -3,12 +3,12 @@ package net.kemitix.thorp.core import net.kemitix.thorp.domain.SyncTotals final case class SyncPlan private ( - actions: Stream[Action], + actions: LazyList[Action], syncTotals: SyncTotals ) object SyncPlan { - val empty: SyncPlan = SyncPlan(Stream.empty, SyncTotals.empty) - def create(actions: Stream[Action], syncTotals: SyncTotals): SyncPlan = + val empty: SyncPlan = SyncPlan(LazyList.empty, SyncTotals.empty) + def create(actions: LazyList[Action], syncTotals: SyncTotals): SyncPlan = SyncPlan(actions, syncTotals) } diff --git a/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala index c90297d..a466392 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/ActionGeneratorSuite.scala @@ -25,7 +25,7 @@ class ActionGeneratorSuite extends FunSpec { describe("create actions") { - val previousActions = Stream.empty[Action] + val previousActions = LazyList.empty[Action] describe("#1 local exists, remote exists, remote matches - do nothing") { val theHash = MD5Hash("the-hash") @@ -46,7 +46,7 @@ class ActionGeneratorSuite extends FunSpec { env.map({ case (theFile, input) => { val expected = - Right(Stream( + Right(LazyList( DoNothing(bucket, theFile.remoteKey, theFile.file.length + 1))) val result = invoke(input, previousActions) assertResult(expected)(result) @@ -74,7 +74,7 @@ class ActionGeneratorSuite extends FunSpec { env.map({ case (theFile, theRemoteKey, input, otherRemoteKey) => { val expected = Right( - Stream( + LazyList( ToCopy(bucket, otherRemoteKey, theHash, @@ -100,7 +100,7 @@ class ActionGeneratorSuite extends FunSpec { it("upload") { env.map({ case (theFile, input) => { - val expected = Right(Stream( + val expected = Right(LazyList( ToUpload(bucket, theFile, theFile.file.length))) // upload val result = invoke(input, previousActions) assertResult(expected)(result) @@ -134,7 +134,7 @@ class ActionGeneratorSuite extends FunSpec { env.map({ case (theFile, theRemoteKey, input, otherRemoteKey) => { val expected = Right( - Stream( + LazyList( ToCopy(bucket, otherRemoteKey, theHash, @@ -167,8 +167,8 @@ class ActionGeneratorSuite extends FunSpec { it("upload") { env.map({ case (theFile, input) => { - val expected = Right( - Stream(ToUpload(bucket, theFile, theFile.file.length))) // upload + val expected = Right(LazyList( + ToUpload(bucket, theFile, theFile.file.length))) // upload val result = invoke(input, previousActions) assertResult(expected)(result) } @@ -188,7 +188,7 @@ class ActionGeneratorSuite extends FunSpec { private def invoke( input: MatchedMetadata, - previousActions: Stream[Action] + previousActions: LazyList[Action] ) = { type TestEnv = Config with FileSystem val testEnv: TestEnv = new Config.Live with FileSystem.Live {} diff --git a/core/src/test/scala/net/kemitix/thorp/core/MatchedMetadataEnricherSuite.scala b/core/src/test/scala/net/kemitix/thorp/core/MatchedMetadataEnricherSuite.scala index 3ffe419..53f8cfc 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/MatchedMetadataEnricherSuite.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/MatchedMetadataEnricherSuite.scala @@ -6,6 +6,8 @@ import net.kemitix.thorp.domain.HashType.MD5 import net.kemitix.thorp.domain._ import org.scalatest.FunSpec +import scala.collection.MapView + class MatchedMetadataEnricherSuite extends FunSpec { private val source = Resource(this, "upload") private val sourcePath = source.toPath @@ -37,8 +39,8 @@ class MatchedMetadataEnricherSuite extends FunSpec { prefix) theRemoteKey = theFile.remoteKey remoteObjects = RemoteObjects( - byHash = Map(theHash -> Set(theRemoteKey)), - byKey = Map(theRemoteKey -> theHash) + byHash = MapView(theHash -> Set(theRemoteKey)), + byKey = MapView(theRemoteKey -> theHash) ) theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash) } yield (theFile, theRemoteMetadata, remoteObjects) @@ -65,8 +67,8 @@ class MatchedMetadataEnricherSuite extends FunSpec { prefix) theRemoteKey: RemoteKey = RemoteKey.resolve("the-file")(prefix) remoteObjects = RemoteObjects( - byHash = Map(theHash -> Set(theRemoteKey)), - byKey = Map(theRemoteKey -> theHash) + byHash = MapView(theHash -> Set(theRemoteKey)), + byKey = MapView(theRemoteKey -> theHash) ) theRemoteMetadata = RemoteMetaData(theRemoteKey, theHash) } yield (theFile, theRemoteMetadata, remoteObjects) @@ -93,8 +95,8 @@ class MatchedMetadataEnricherSuite extends FunSpec { prefix) otherRemoteKey = RemoteKey("other-key") remoteObjects = RemoteObjects( - byHash = Map(theHash -> Set(otherRemoteKey)), - byKey = Map(otherRemoteKey -> theHash) + byHash = MapView(theHash -> Set(otherRemoteKey)), + byKey = MapView(otherRemoteKey -> theHash) ) otherRemoteMetadata = RemoteMetaData(otherRemoteKey, theHash) } yield (theFile, otherRemoteMetadata, remoteObjects) @@ -148,9 +150,9 @@ class MatchedMetadataEnricherSuite extends FunSpec { oldHash = MD5Hash("old-hash") otherRemoteKey = RemoteKey.resolve("other-key")(prefix) remoteObjects = RemoteObjects( - byHash = - Map(oldHash -> Set(theRemoteKey), theHash -> Set(otherRemoteKey)), - byKey = Map( + byHash = MapView(oldHash -> Set(theRemoteKey), + theHash -> Set(otherRemoteKey)), + byKey = MapView( theRemoteKey -> oldHash, otherRemoteKey -> theHash ) @@ -186,8 +188,8 @@ class MatchedMetadataEnricherSuite extends FunSpec { theRemoteKey = theFile.remoteKey oldHash = MD5Hash("old-hash") remoteObjects = RemoteObjects( - byHash = Map(oldHash -> Set(theRemoteKey), theHash -> Set.empty), - byKey = Map(theRemoteKey -> oldHash) + byHash = MapView(oldHash -> Set(theRemoteKey), theHash -> Set.empty), + byKey = MapView(theRemoteKey -> oldHash) ) theRemoteMetadata = RemoteMetaData(theRemoteKey, oldHash) } yield (theFile, theRemoteMetadata, remoteObjects) @@ -230,11 +232,11 @@ class MatchedMetadataEnricherSuite extends FunSpec { sources, prefix) remoteObjects = RemoteObjects( - byHash = Map( + byHash = MapView( hash -> Set(key, keyOtherKey.remoteKey), diffHash -> Set(keyDiffHash.remoteKey) ), - byKey = Map( + byKey = MapView( key -> hash, keyOtherKey.remoteKey -> hash, keyDiffHash.remoteKey -> diffHash diff --git a/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala b/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala index 70587ce..1fcef9c 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/PlanBuilderTest.scala @@ -19,6 +19,8 @@ import net.kemitix.thorp.storage.api.Storage import org.scalatest.FreeSpec import zio.{DefaultRuntime, Task, UIO} +import scala.collection.MapView + class PlanBuilderTest extends FreeSpec with TemporaryFolder { private val emptyRemoteObjects = RemoteObjects.empty @@ -62,8 +64,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { val anOtherKey = RemoteKey("other") val expected = Right(List(toCopy(anOtherKey, aHash, remoteKey))) val remoteObjects = RemoteObjects( - byHash = Map(aHash -> Set(anOtherKey)), - byKey = Map(anOtherKey -> aHash) + byHash = MapView(aHash -> Set(anOtherKey)), + byKey = MapView(anOtherKey -> aHash) ) val result = invoke(options(source), @@ -84,8 +86,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { // DoNothing actions should have been filtered out of the plan val expected = Right(List()) val remoteObjects = RemoteObjects( - byHash = Map(hash -> Set(remoteKey)), - byKey = Map(remoteKey -> hash) + byHash = MapView(hash -> Set(remoteKey)), + byKey = MapView(remoteKey -> hash) ) val result = invoke(options(source), @@ -105,8 +107,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { val expected = Right(List(toUpload(remoteKey, currentHash, source, file))) val remoteObjects = RemoteObjects( - byHash = Map(originalHash -> Set(remoteKey)), - byKey = Map(remoteKey -> originalHash) + byHash = MapView(originalHash -> Set(remoteKey)), + byKey = MapView(remoteKey -> originalHash) ) val result = invoke(options(source), @@ -124,8 +126,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { val sourceKey = RemoteKey("other-key") val expected = Right(List(toCopy(sourceKey, hash, remoteKey))) val remoteObjects = RemoteObjects( - byHash = Map(hash -> Set(sourceKey)), - byKey = Map.empty + byHash = MapView(hash -> Set(sourceKey)), + byKey = MapView.empty ) val result = invoke(options(source), @@ -149,8 +151,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { // DoNothing actions should have been filtered out of the plan val expected = Right(List()) val remoteObjects = RemoteObjects( - byHash = Map(hash -> Set(remoteKey)), - byKey = Map(remoteKey -> hash) + byHash = MapView(hash -> Set(remoteKey)), + byKey = MapView(remoteKey -> hash) ) val result = invoke(options(source), @@ -166,8 +168,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { val hash = MD5Hash("file-content") val expected = Right(List(toDelete(remoteKey))) val remoteObjects = RemoteObjects( - byHash = Map(hash -> Set(remoteKey)), - byKey = Map(remoteKey -> hash) + byHash = MapView(hash -> Set(remoteKey)), + byKey = MapView(remoteKey -> hash) ) val result = invoke(options(source), @@ -253,8 +255,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { val hash2 = md5Hash(fileInSecondSource) val expected = Right(List()) val remoteObjects = - RemoteObjects(byHash = Map(hash2 -> Set(remoteKey2)), - byKey = Map(remoteKey2 -> hash2)) + RemoteObjects(byHash = MapView(hash2 -> Set(remoteKey2)), + byKey = MapView(remoteKey2 -> hash2)) val result = invoke(options(firstSource)(secondSource), UIO.succeed(remoteObjects), @@ -274,8 +276,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { withDirectory(secondSource => { val expected = Right(List()) val remoteObjects = - RemoteObjects(byHash = Map(hash1 -> Set(remoteKey1)), - byKey = Map(remoteKey1 -> hash1)) + RemoteObjects(byHash = MapView(hash1 -> Set(remoteKey1)), + byKey = MapView(remoteKey1 -> hash1)) val result = invoke(options(firstSource)(secondSource), UIO.succeed(remoteObjects), @@ -292,8 +294,8 @@ class PlanBuilderTest extends FreeSpec with TemporaryFolder { withDirectory(secondSource => { val expected = Right(List(toDelete(remoteKey1))) val remoteObjects = - RemoteObjects(byHash = Map.empty, - byKey = Map(remoteKey1 -> MD5Hash(""))) + RemoteObjects(byHash = MapView.empty, + byKey = MapView(remoteKey1 -> MD5Hash(""))) val result = invoke(options(firstSource)(secondSource), UIO.succeed(remoteObjects), diff --git a/core/src/test/scala/net/kemitix/thorp/core/PlanExecutorTest.scala b/core/src/test/scala/net/kemitix/thorp/core/PlanExecutorTest.scala index a3c1c03..18858bc 100644 --- a/core/src/test/scala/net/kemitix/thorp/core/PlanExecutorTest.scala +++ b/core/src/test/scala/net/kemitix/thorp/core/PlanExecutorTest.scala @@ -15,11 +15,11 @@ import zio.{DefaultRuntime, ZIO} class PlanExecutorTest extends FreeSpec { - private def subject(in: Stream[Int]): ZIO[Any, Throwable, Stream[Int]] = - ZIO.foldLeft(in)(Stream.empty[Int])((s, i) => ZIO(i #:: s)).map(_.reverse) + private def subject(in: LazyList[Int]): ZIO[Any, Throwable, LazyList[Int]] = + ZIO.foldLeft(in)(LazyList.empty[Int])((s, i) => ZIO(i #:: s)).map(_.reverse) "zio foreach on a stream can be a stream" in { - val input = (1 to 1000000).toStream + val input = LazyList.from(1 to 1000000) val program = subject(input) val result = new DefaultRuntime {}.unsafeRunSync(program).toEither assertResult(Right(input))(result) @@ -29,7 +29,8 @@ class PlanExecutorTest extends FreeSpec { val nActions = 100000 val bucket = Bucket("bucket") val remoteKey = RemoteKey("remoteKey") - val input = (1 to nActions).toStream.map(DoNothing(bucket, remoteKey, _)) + val input = + LazyList.from(1 to nActions).map(DoNothing(bucket, remoteKey, _)) val syncTotals = SyncTotals.empty val archiveTask = UnversionedMirrorArchive.default(syncTotals) @@ -45,7 +46,8 @@ class PlanExecutorTest extends FreeSpec { new DefaultRuntime {}.unsafeRunSync(program.provide(TestEnv)).toEither val expected = Right( - (1 to nActions).toStream + LazyList + .from(1 to nActions) .map(_ => StorageQueueEvent.DoNothingQueueEvent(remoteKey))) assertResult(expected)(result) } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/HexEncoder.scala b/domain/src/main/scala/net/kemitix/thorp/domain/HexEncoder.scala index 74d5293..c35a7c0 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/HexEncoder.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/HexEncoder.scala @@ -12,7 +12,9 @@ trait HexEncoder { def decode(hexString: String): Array[Byte] = hexString .replaceAll("[^0-9A-Fa-f]", "") + .toSeq .sliding(2, 2) + .map(_.unwrap) .toArray .map(Integer.parseInt(_, 16).toByte) diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/NonUnit.scala b/domain/src/main/scala/net/kemitix/thorp/domain/NonUnit.scala index 82398e6..f532de1 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/NonUnit.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/NonUnit.scala @@ -2,7 +2,7 @@ package net.kemitix.thorp.domain object NonUnit { @specialized def ~*[A](evaluateForSideEffectOnly: A): Unit = { - val _: A = evaluateForSideEffectOnly + val _ = evaluateForSideEffectOnly () //Return unit to prevent warning due to discarding value } } diff --git a/domain/src/main/scala/net/kemitix/thorp/domain/RemoteObjects.scala b/domain/src/main/scala/net/kemitix/thorp/domain/RemoteObjects.scala index ecc5ba1..abe4dc0 100644 --- a/domain/src/main/scala/net/kemitix/thorp/domain/RemoteObjects.scala +++ b/domain/src/main/scala/net/kemitix/thorp/domain/RemoteObjects.scala @@ -1,16 +1,18 @@ package net.kemitix.thorp.domain +import scala.collection.MapView + /** * A list of objects and their MD5 hash values. */ final case class RemoteObjects private ( - byHash: Map[MD5Hash, Set[RemoteKey]], - byKey: Map[RemoteKey, MD5Hash] + byHash: MapView[MD5Hash, Set[RemoteKey]], + byKey: MapView[RemoteKey, MD5Hash] ) object RemoteObjects { - val empty: RemoteObjects = RemoteObjects(Map.empty, Map.empty) - def create(byHash: Map[MD5Hash, Set[RemoteKey]], - byKey: Map[RemoteKey, MD5Hash]): RemoteObjects = + val empty: RemoteObjects = RemoteObjects(MapView.empty, MapView.empty) + def create(byHash: MapView[MD5Hash, Set[RemoteKey]], + byKey: MapView[RemoteKey, MD5Hash]): RemoteObjects = RemoteObjects(byHash, byKey) } diff --git a/filesystem/src/main/scala/net/kemitix/thorp/filesystem/FileSystem.scala b/filesystem/src/main/scala/net/kemitix/thorp/filesystem/FileSystem.scala index 2cbdcd9..b09c535 100644 --- a/filesystem/src/main/scala/net/kemitix/thorp/filesystem/FileSystem.scala +++ b/filesystem/src/main/scala/net/kemitix/thorp/filesystem/FileSystem.scala @@ -6,7 +6,7 @@ import java.util.stream import zio.{Task, RIO, UIO, ZIO, ZManaged} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ trait FileSystem { val filesystem: FileSystem.Service diff --git a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Lister.scala b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Lister.scala index 6e451da..906de76 100644 --- a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Lister.scala +++ b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Lister.scala @@ -11,12 +11,12 @@ import net.kemitix.thorp.storage.aws.S3ObjectsByHash.byHash import net.kemitix.thorp.storage.aws.S3ObjectsByKey.byKey import zio.{Task, RIO} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ trait Lister { private type Token = String - case class Batch(summaries: Stream[S3ObjectSummary], more: Option[Token]) + case class Batch(summaries: LazyList[S3ObjectSummary], more: Option[Token]) def listObjects(amazonS3: AmazonS3.Client)( bucket: Bucket, @@ -34,12 +34,12 @@ trait Lister { def fetchBatch: ListObjectsV2Request => RIO[Console, Batch] = request => ListerLogger.logFetchBatch *> tryFetchBatch(amazonS3)(request) - def fetchMore: Option[Token] => RIO[Console, Stream[S3ObjectSummary]] = { - case None => RIO.succeed(Stream.empty) + def fetchMore: Option[Token] => RIO[Console, LazyList[S3ObjectSummary]] = { + case None => RIO.succeed(LazyList.empty) case Some(token) => fetch(requestMore(token)) } - def fetch: ListObjectsV2Request => RIO[Console, Stream[S3ObjectSummary]] = + def fetch: ListObjectsV2Request => RIO[Console, LazyList[S3ObjectSummary]] = request => for { batch <- fetchBatch(request) @@ -60,8 +60,8 @@ trait Lister { .map(result => Batch(objectSummaries(result), moreToken(result))) private def objectSummaries( - result: ListObjectsV2Result): Stream[S3ObjectSummary] = - result.getObjectSummaries.asScala.toStream + result: ListObjectsV2Result): LazyList[S3ObjectSummary] = + LazyList.from(result.getObjectSummaries.asScala) private def moreToken(result: ListObjectsV2Result): Option[String] = if (result.isTruncated) Some(result.getNextContinuationToken) diff --git a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHash.scala b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHash.scala index 6e63c0f..0eee832 100644 --- a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHash.scala +++ b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHash.scala @@ -3,14 +3,16 @@ package net.kemitix.thorp.storage.aws import com.amazonaws.services.s3.model.S3ObjectSummary import net.kemitix.thorp.domain.{MD5Hash, RemoteKey} +import scala.collection.MapView + object S3ObjectsByHash { def byHash( - os: Stream[S3ObjectSummary] - ): Map[MD5Hash, Set[RemoteKey]] = { - val mD5HashToS3Objects: Map[MD5Hash, Stream[S3ObjectSummary]] = + os: LazyList[S3ObjectSummary] + ): MapView[MD5Hash, Set[RemoteKey]] = { + val mD5HashToS3Objects: Map[MD5Hash, LazyList[S3ObjectSummary]] = os.groupBy(o => MD5Hash(o.getETag.filter(_ != '"'))) - mD5HashToS3Objects.mapValues { os => + mD5HashToS3Objects.view.mapValues { os => os.map(_.getKey).map(RemoteKey(_)).toSet } } diff --git a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByKey.scala b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByKey.scala index 96c4710..5a1580d 100644 --- a/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByKey.scala +++ b/storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3ObjectsByKey.scala @@ -3,15 +3,19 @@ package net.kemitix.thorp.storage.aws import com.amazonaws.services.s3.model.S3ObjectSummary import net.kemitix.thorp.domain.{MD5Hash, RemoteKey} +import scala.collection.MapView + object S3ObjectsByKey { - def byKey(os: Stream[S3ObjectSummary]): Map[RemoteKey, MD5Hash] = + def byKey(os: LazyList[S3ObjectSummary]): MapView[RemoteKey, MD5Hash] = os.map { o => - { - val remoteKey = RemoteKey(o.getKey) - val hash = MD5Hash(o.getETag) - (remoteKey, hash) + { + val remoteKey = RemoteKey(o.getKey) + val hash = MD5Hash(o.getETag) + (remoteKey, hash) + } } - }.toMap + .toMap + .view } diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala index 8668ae9..c2078c2 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/ListerTest.scala @@ -12,6 +12,7 @@ import net.kemitix.thorp.console._ import net.kemitix.thorp.domain.NonUnit.~* import net.kemitix.thorp.domain._ import org.scalatest.FreeSpec +import org.scalatest.Matchers._ import zio.internal.PlatformLive import zio.{Runtime, Task, UIO} @@ -29,16 +30,17 @@ class ListerTest extends FreeSpec { val etag = "etag" val expectedHashMap = Map(MD5Hash(etag) -> Set(RemoteKey(key))) val expectedKeyMap = Map(RemoteKey(key) -> MD5Hash(etag)) - val expected = Right(RemoteObjects(expectedHashMap, expectedKeyMap)) new AmazonS3ClientTestFixture { - ~*( - (fixture.amazonS3Client.listObjectsV2 _) - .when() - .returns(_ => { - UIO.succeed(objectResults(nowDate, key, etag, false)) - })) - private val result = invoke(fixture.amazonS3Client)(bucket, prefix) - ~*(assertResult(expected)(result)) + (fixture.amazonS3Client.listObjectsV2 _) + .when() + .returns(_ => { + UIO.succeed(objectResults(nowDate, key, etag, truncated = false)) + }) + private val result = invoke(fixture.amazonS3Client)(bucket, prefix) + private val hashMap = result.map(_.byHash).map(m => Map.from(m)) + private val keyMap = result.map(_.byKey).map(m => Map.from(m)) + hashMap should be(Right(expectedHashMap)) + keyMap should be(Right(expectedKeyMap)) } } @@ -56,19 +58,23 @@ class ListerTest extends FreeSpec { RemoteKey(key1) -> MD5Hash(etag1), RemoteKey(key2) -> MD5Hash(etag2) ) - val expected = Right(RemoteObjects(expectedHashMap, expectedKeyMap)) new AmazonS3ClientTestFixture { - ~*( - (fixture.amazonS3Client.listObjectsV2 _) - .when() - .returns(_ => UIO(objectResults(nowDate, key1, etag1, true))) - .noMoreThanOnce()) - ~*( - (fixture.amazonS3Client.listObjectsV2 _) - .when() - .returns(_ => UIO(objectResults(nowDate, key2, etag2, false)))) - private val result = invoke(fixture.amazonS3Client)(bucket, prefix) - ~*(assertResult(expected)(result)) + + (fixture.amazonS3Client.listObjectsV2 _) + .when() + .returns(_ => + UIO(objectResults(nowDate, key1, etag1, truncated = true))) + .noMoreThanOnce() + + (fixture.amazonS3Client.listObjectsV2 _) + .when() + .returns(_ => + UIO(objectResults(nowDate, key2, etag2, truncated = false))) + private val result = invoke(fixture.amazonS3Client)(bucket, prefix) + private val hashMap = result.map(_.byHash).map(m => Map.from(m)) + private val keyMap = result.map(_.byKey).map(m => Map.from(m)) + hashMap should be(Right(expectedHashMap)) + keyMap should be(Right(expectedKeyMap)) } } diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala index b01ff8d..14ca5c2 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/S3ObjectsByHashSuite.scala @@ -12,12 +12,12 @@ class S3ObjectsByHashSuite extends FunSpec { val key2 = RemoteKey("key-2") val o1 = s3object(hash, key1) val o2 = s3object(hash, key2) - val os = Stream(o1, o2) + val os = LazyList(o1, o2) it("should group by the hash value") { val expected: Map[MD5Hash, Set[RemoteKey]] = Map( hash -> Set(key1, key2) ) - val result = S3ObjectsByHash.byHash(os) + val result = Map.from(S3ObjectsByHash.byHash(os)) assertResult(expected)(result) } } diff --git a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala index 2706ae1..c049ba3 100644 --- a/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala +++ b/storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/StorageServiceSuite.scala @@ -7,6 +7,8 @@ import net.kemitix.thorp.domain._ import org.scalamock.scalatest.MockFactory import org.scalatest.FunSpec +import scala.collection.MapView + class StorageServiceSuite extends FunSpec with MockFactory { private val source = Resource(this, "upload") @@ -36,11 +38,11 @@ class StorageServiceSuite extends FunSpec with MockFactory { sources, prefix) s3ObjectsData = RemoteObjects( - byHash = Map( + byHash = MapView( hash -> Set(key, keyOtherKey.remoteKey), diffHash -> Set(keyDiffHash.remoteKey) ), - byKey = Map( + byKey = MapView( key -> hash, keyOtherKey.remoteKey -> hash, keyDiffHash.remoteKey -> diffHash