Refactor Uploader.upload(LocalFile,Bucket,Boolean,UploadEventListener,Int) (#129)

* [storate-api] Remove unused tryCount

* [storage-aws] Uploader Reduce the number of parameters

core/src/main/scala/net/kemitix/thorp/core/UnversionedMirrorArchive.scala

@@ -36,8 +36,7 @@ case class UnversionedMirrorArchive(syncTotals: SyncTotals)
     Storage.upload(
       localFile,
       bucket,
-      UploadEventListener(localFile, index, syncTotals, totalBytesSoFar),
+      UploadEventListener(localFile, index, syncTotals, totalBytesSoFar))
-      1)
 }
 
 object UnversionedMirrorArchive {

core/src/test/scala/net/kemitix/thorp/core/DummyStorageService.scala

@@ -20,10 +20,11 @@ case class DummyStorageService(s3ObjectData: S3ObjectsData,
   ): TaskR[Console, S3ObjectsData] =
     TaskR(s3ObjectData)
 
-  override def upload(localFile: LocalFile,
+  override def upload(
+      localFile: LocalFile,
       bucket: Bucket,
       uploadEventListener: UploadEventListener,
-                      tryCount: Int): UIO[StorageQueueEvent] = {
+  ): UIO[StorageQueueEvent] = {
     val (remoteKey, md5Hash) = uploadFiles(localFile.file)
     UIO(StorageQueueEvent.UploadQueueEvent(remoteKey, md5Hash))
   }

storage-api/src/main/scala/net/kemitix/thorp/storage/api/Storage.scala

@@ -21,7 +21,6 @@ object Storage {
         localFile: LocalFile,
         bucket: Bucket,
         uploadEventListener: UploadEventListener,
-        tryCount: Int
     ): ZIO[Storage with Config, Nothing, StorageQueueEvent]
 
     def copy(
@@ -57,8 +56,8 @@ object Storage {
       override def upload(
           localFile: LocalFile,
           bucket: Bucket,
-          uploadEventListener: UploadEventListener,
+          uploadEventListener: UploadEventListener
-          tryCount: Int): ZIO[Storage, Nothing, StorageQueueEvent] =
+      ): ZIO[Storage, Nothing, StorageQueueEvent] =
         uploadResult
 
       override def copy(
@@ -99,11 +98,9 @@ object Storage {
   final def upload(
       localFile: LocalFile,
       bucket: Bucket,
-      uploadEventListener: UploadEventListener,
+      uploadEventListener: UploadEventListener
-      tryCount: Int
   ): ZIO[Storage with Config, Nothing, StorageQueueEvent] =
-    ZIO.accessM(
+    ZIO.accessM(_.storage upload (localFile, bucket, uploadEventListener))
-      _.storage upload (localFile, bucket, uploadEventListener, tryCount))
 
   final def copy(
       bucket: Bucket,

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/S3Storage.scala

@@ -28,11 +28,9 @@ object S3Storage {
           localFile: LocalFile,
           bucket: Bucket,
           uploadEventListener: UploadEventListener,
-          tryCount: Int): ZIO[Config, Nothing, StorageQueueEvent] =
+      ): ZIO[Config, Nothing, StorageQueueEvent] =
-        Uploader.upload(transferManager)(localFile,
+        Uploader.upload(transferManager)(
-                                         bucket,
+          Uploader.Request(localFile, bucket, uploadEventListener))
-                                         uploadEventListener,
-                                         1)
 
       override def copy(bucket: Bucket,
                         sourceKey: RemoteKey,

storage-aws/src/main/scala/net/kemitix/thorp/storage/aws/Uploader.scala

@@ -18,24 +18,24 @@ import zio.{UIO, ZIO}
 
 trait Uploader {
 
-  def upload(transferManager: => AmazonTransferManager)(
+  case class Request(
       localFile: LocalFile,
       bucket: Bucket,
-      uploadEventListener: UploadEventListener,
+      uploadEventListener: UploadEventListener
-      tryCount: Int
+  )
-  ): ZIO[Config, Nothing, StorageQueueEvent] =
+
-    transfer(transferManager)(localFile, bucket, uploadEventListener)
+  def upload(transferManager: => AmazonTransferManager)(
-      .catchAll(handleError(localFile.remoteKey))
+      request: Request): ZIO[Config, Nothing, StorageQueueEvent] =
+    transfer(transferManager)(request)
+      .catchAll(handleError(request.localFile.remoteKey))
 
   private def handleError(remoteKey: RemoteKey)(e: Throwable) =
     UIO(ErrorQueueEvent(Action.Upload(remoteKey.key), remoteKey, e))
 
   private def transfer(transferManager: => AmazonTransferManager)(
-      localFile: LocalFile,
+      request: Request
-      bucket: Bucket,
-      uploadEventListener: UploadEventListener
   ) =
-    request(localFile, bucket, progressListener(uploadEventListener)) >>=
+    putObjectRequest(request) >>=
       dispatch(transferManager)
 
   private def dispatch(transferManager: AmazonTransferManager)(
@@ -49,18 +49,20 @@ trait Uploader {
                          MD5Hash(uploadResult.getETag)))
   }
 
-  private def request(
+  private def putObjectRequest(
-      localFile: LocalFile,
+      request: Request
-      bucket: Bucket,
-      listener: ProgressListener
   ) = {
-    val request =
+    val putRequest =
-      new PutObjectRequest(bucket.name, localFile.remoteKey.key, localFile.file)
+      new PutObjectRequest(request.bucket.name,
-        .withMetadata(metadata(localFile))
+                           request.localFile.remoteKey.key,
+                           request.localFile.file)
+        .withMetadata(metadata(request.localFile))
     for {
       batchMode <- Config.batchMode
-      r = if (batchMode) request
+      r = if (batchMode) putRequest
-      else request.withGeneralProgressListener(listener)
+      else
+        putRequest.withGeneralProgressListener(
+          progressListener(request.uploadEventListener))
     } yield r
   }
 

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/AmazonS3ClientTestFixture.scala

@@ -25,28 +25,31 @@ trait AmazonS3ClientTestFixture extends MockFactory {
 
         override def listObjects(
             bucket: Bucket,
-            prefix: RemoteKey): TaskR[Console, S3ObjectsData] =
+            prefix: RemoteKey
+        ): TaskR[Console, S3ObjectsData] =
           Lister.listObjects(client)(bucket, prefix)
 
         override def upload(
             localFile: LocalFile,
             bucket: Bucket,
             uploadEventListener: UploadEventListener,
-            tryCount: Int): ZIO[Config, Nothing, StorageQueueEvent] =
+        ): ZIO[Config, Nothing, StorageQueueEvent] =
-          Uploader.upload(transferManager)(localFile,
+          Uploader.upload(transferManager)(
-                                           bucket,
+            Uploader.Request(localFile, bucket, uploadEventListener))
-                                           uploadEventListener,
-                                           1)
 
-        override def copy(bucket: Bucket,
+        override def copy(
+            bucket: Bucket,
             sourceKey: RemoteKey,
             hash: MD5Hash,
-                          targetKey: RemoteKey): UIO[StorageQueueEvent] =
+            targetKey: RemoteKey
+        ): UIO[StorageQueueEvent] =
           Copier.copy(client)(
             Copier.Request(bucket, sourceKey, hash, targetKey))
 
-        override def delete(bucket: Bucket,
+        override def delete(
-                            remoteKey: RemoteKey): UIO[StorageQueueEvent] =
+            bucket: Bucket,
+            remoteKey: RemoteKey
+        ): UIO[StorageQueueEvent] =
           Deleter.delete(client)(bucket, remoteKey)
 
         override def shutdown: UIO[StorageQueueEvent] = {

storage-aws/src/test/scala/net/kemitix/thorp/storage/aws/UploaderTest.scala

@@ -27,7 +27,6 @@ class UploaderTest extends FreeSpec with MockFactory {
     val remoteKey     = RemoteKey("aRemoteKey")
     val localFile     = LocalFile(aFile, aSource, hashes, remoteKey)
     val bucket        = Bucket("aBucket")
-    val tryCount      = 1
     val uploadResult  = new UploadResult
     uploadResult.setKey(remoteKey.key)
     uploadResult.setETag(aHash.hash)
@@ -47,8 +46,7 @@ class UploaderTest extends FreeSpec with MockFactory {
           invoke(fixture.amazonS3TransferManager)(
             localFile,
             bucket,
-            uploadEventListener,
+            uploadEventListener
-            tryCount
           )
         assertResult(expected)(result)
       }
@@ -66,8 +64,7 @@ class UploaderTest extends FreeSpec with MockFactory {
           invoke(fixture.amazonS3TransferManager)(
             localFile,
             bucket,
-            uploadEventListener,
+            uploadEventListener
-            tryCount
           )
         assertResult(expected)(result)
       }
@@ -85,8 +82,7 @@ class UploaderTest extends FreeSpec with MockFactory {
           invoke(fixture.amazonS3TransferManager)(
             localFile,
             bucket,
-            uploadEventListener,
+            uploadEventListener
-            tryCount
           )
         assertResult(expected)(result)
       }
@@ -94,19 +90,14 @@ class UploaderTest extends FreeSpec with MockFactory {
     def invoke(transferManager: AmazonTransferManager)(
         localFile: LocalFile,
         bucket: Bucket,
-        uploadEventListener: UploadEventListener,
+        uploadEventListener: UploadEventListener
-        tryCount: Int
     ) = {
       type TestEnv = Config
       val testEnv: TestEnv = Config.Live
       new DefaultRuntime {}.unsafeRunSync {
         Uploader
           .upload(transferManager)(
-            localFile,
+            Uploader.Request(localFile, bucket, uploadEventListener))
-            bucket,
-            uploadEventListener,
-            tryCount
-          )
           .provide(testEnv)
       }.toEither
     }