Improve Code Quality (#37)
* [core] convert QuoteStripper to an object and move to core * [aws-lib]S3ClientUploader: use case matching instead of else if blocks * [aws-lib] put imports at top of file * [domain] remove redundant braces after class definition * [aws-lib] remove redundant braces after class definition * [core] avoid using head on a collection
This commit is contained in:
parent
f54c50aaf3
commit
8ec667343a
16 changed files with 30 additions and 51 deletions
|
@ -1,7 +0,0 @@
|
||||||
package net.kemitix.s3thorp.aws.lib
|
|
||||||
|
|
||||||
trait QuoteStripper {
|
|
||||||
|
|
||||||
def stripQuotes: Char => Boolean = _ != '"'
|
|
||||||
|
|
||||||
}
|
|
|
@ -6,6 +6,7 @@ import com.amazonaws.services.s3.AmazonS3
|
||||||
import com.amazonaws.services.s3.model._
|
import com.amazonaws.services.s3.model._
|
||||||
import net.kemitix.s3thorp.aws.api.S3Action.{ErroredS3Action, UploadS3Action}
|
import net.kemitix.s3thorp.aws.api.S3Action.{ErroredS3Action, UploadS3Action}
|
||||||
import net.kemitix.s3thorp.aws.api.{S3Action, UploadProgressListener}
|
import net.kemitix.s3thorp.aws.api.{S3Action, UploadProgressListener}
|
||||||
|
import net.kemitix.s3thorp.core.QuoteStripper.stripQuotes
|
||||||
import net.kemitix.s3thorp.core.MD5HashGenerator.md5FilePart
|
import net.kemitix.s3thorp.core.MD5HashGenerator.md5FilePart
|
||||||
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash}
|
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash}
|
||||||
|
|
||||||
|
@ -14,8 +15,7 @@ import scala.util.control.NonFatal
|
||||||
|
|
||||||
private class S3ClientMultiPartUploader(s3Client: AmazonS3)
|
private class S3ClientMultiPartUploader(s3Client: AmazonS3)
|
||||||
extends S3ClientUploader
|
extends S3ClientUploader
|
||||||
with S3ClientMultiPartUploaderLogging
|
with S3ClientMultiPartUploaderLogging {
|
||||||
with QuoteStripper {
|
|
||||||
|
|
||||||
def accepts(localFile: LocalFile)
|
def accepts(localFile: LocalFile)
|
||||||
(implicit multiPartThreshold: Long): Boolean =
|
(implicit multiPartThreshold: Long): Boolean =
|
||||||
|
|
|
@ -2,6 +2,7 @@ package net.kemitix.s3thorp.aws.lib
|
||||||
|
|
||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
import com.github.j5ik2o.reactive.aws.s3.cats.S3CatsIOClient
|
||||||
|
import net.kemitix.s3thorp.core.QuoteStripper.stripQuotes
|
||||||
import net.kemitix.s3thorp.domain._
|
import net.kemitix.s3thorp.domain._
|
||||||
import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, S3Object}
|
import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, S3Object}
|
||||||
|
|
||||||
|
@ -9,8 +10,7 @@ import scala.collection.JavaConverters._
|
||||||
|
|
||||||
class S3ClientObjectLister(s3Client: S3CatsIOClient)
|
class S3ClientObjectLister(s3Client: S3CatsIOClient)
|
||||||
extends S3ClientLogging
|
extends S3ClientLogging
|
||||||
with S3ObjectsByHash
|
with S3ObjectsByHash {
|
||||||
with QuoteStripper {
|
|
||||||
|
|
||||||
def listObjects(bucket: Bucket,
|
def listObjects(bucket: Bucket,
|
||||||
prefix: RemoteKey)
|
prefix: RemoteKey)
|
||||||
|
|
|
@ -5,12 +5,12 @@ import com.amazonaws.services.s3.AmazonS3
|
||||||
import com.amazonaws.services.s3.model.PutObjectRequest
|
import com.amazonaws.services.s3.model.PutObjectRequest
|
||||||
import net.kemitix.s3thorp.aws.api.S3Action.UploadS3Action
|
import net.kemitix.s3thorp.aws.api.S3Action.UploadS3Action
|
||||||
import net.kemitix.s3thorp.aws.api.UploadProgressListener
|
import net.kemitix.s3thorp.aws.api.UploadProgressListener
|
||||||
|
import net.kemitix.s3thorp.core.QuoteStripper.stripQuotes
|
||||||
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash}
|
import net.kemitix.s3thorp.domain.{Bucket, LocalFile, MD5Hash}
|
||||||
|
|
||||||
class S3ClientPutObjectUploader(amazonS3: => AmazonS3)
|
class S3ClientPutObjectUploader(amazonS3: => AmazonS3)
|
||||||
extends S3ClientUploader
|
extends S3ClientUploader
|
||||||
with S3ClientLogging
|
with S3ClientLogging {
|
||||||
with QuoteStripper {
|
|
||||||
|
|
||||||
override def accepts(localFile: LocalFile)(implicit multiPartThreshold: Long): Boolean = true
|
override def accepts(localFile: LocalFile)(implicit multiPartThreshold: Long): Boolean = true
|
||||||
|
|
||||||
|
|
|
@ -23,12 +23,11 @@ trait S3ClientUploader {
|
||||||
def progressListener(uploadProgressListener: UploadProgressListener): ProgressListener = {
|
def progressListener(uploadProgressListener: UploadProgressListener): ProgressListener = {
|
||||||
new ProgressListener {
|
new ProgressListener {
|
||||||
override def progressChanged(event: ProgressEvent): Unit = {
|
override def progressChanged(event: ProgressEvent): Unit = {
|
||||||
if (event.getEventType.isTransferEvent)
|
event match {
|
||||||
TransferEvent(event.getEventType.name)
|
case e if e.getEventType.isTransferEvent => TransferEvent(e.getEventType.name)
|
||||||
else if (event.getEventType equals ProgressEventType.RESPONSE_BYTE_TRANSFER_EVENT)
|
case e if e.getEventType equals ProgressEventType.RESPONSE_BYTE_TRANSFER_EVENT => ByteTransferEvent(e.getEventType.name)
|
||||||
ByteTransferEvent(event.getEventType.name)
|
case e => RequestEvent(e.getEventType.name, e.getBytes, e.getBytesTransferred)
|
||||||
else
|
}
|
||||||
RequestEvent(event.getEventType.name, event.getBytes, event.getBytesTransferred)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,8 +13,7 @@ class ThorpS3Client(ioS3Client: S3CatsIOClient,
|
||||||
amazonS3Client: => AmazonS3,
|
amazonS3Client: => AmazonS3,
|
||||||
amazonS3TransferManager: => TransferManager)
|
amazonS3TransferManager: => TransferManager)
|
||||||
extends S3Client
|
extends S3Client
|
||||||
with S3ClientLogging
|
with S3ClientLogging {
|
||||||
with QuoteStripper {
|
|
||||||
|
|
||||||
// lazy val amazonS3Client = AmazonS3ClientBuilder.defaultClient
|
// lazy val amazonS3Client = AmazonS3ClientBuilder.defaultClient
|
||||||
// lazy val amazonS3TransferManager = TransferManagerBuilder.defaultTransferManager
|
// lazy val amazonS3TransferManager = TransferManagerBuilder.defaultTransferManager
|
||||||
|
|
|
@ -4,12 +4,12 @@ import java.io.File
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
import com.amazonaws.AmazonClientException
|
import com.amazonaws.AmazonClientException
|
||||||
|
import com.amazonaws.event.ProgressListener
|
||||||
import com.amazonaws.services.s3.model
|
import com.amazonaws.services.s3.model
|
||||||
import com.amazonaws.services.s3.transfer.model.UploadResult
|
import com.amazonaws.services.s3.transfer.model.UploadResult
|
||||||
import com.amazonaws.services.s3.transfer._
|
import com.amazonaws.services.s3.transfer._
|
||||||
import net.kemitix.s3thorp.aws.api.S3Action.UploadS3Action
|
import net.kemitix.s3thorp.aws.api.S3Action.UploadS3Action
|
||||||
import net.kemitix.s3thorp.aws.api.UploadProgressListener
|
import net.kemitix.s3thorp.aws.api.UploadProgressListener
|
||||||
import net.kemitix.s3thorp.aws.lib.S3ClientMultiPartTransferManager
|
|
||||||
import net.kemitix.s3thorp.core.KeyGenerator.generateKey
|
import net.kemitix.s3thorp.core.KeyGenerator.generateKey
|
||||||
import net.kemitix.s3thorp.core.{MD5HashGenerator, Resource}
|
import net.kemitix.s3thorp.core.{MD5HashGenerator, Resource}
|
||||||
import net.kemitix.s3thorp.domain._
|
import net.kemitix.s3thorp.domain._
|
||||||
|
@ -115,8 +115,6 @@ class S3ClientMultiPartTransferManagerSuite
|
||||||
|
|
||||||
override def getProgress: TransferProgress = ???
|
override def getProgress: TransferProgress = ???
|
||||||
|
|
||||||
import com.amazonaws.event.ProgressListener
|
|
||||||
|
|
||||||
override def addProgressListener(listener: ProgressListener): Unit = ???
|
override def addProgressListener(listener: ProgressListener): Unit = ???
|
||||||
|
|
||||||
override def removeProgressListener(listener: ProgressListener): Unit = ???
|
override def removeProgressListener(listener: ProgressListener): Unit = ???
|
||||||
|
|
|
@ -283,7 +283,7 @@ class S3ClientMultiPartUploaderSuite
|
||||||
override def abortMultipartUpload(abortMultipartUploadRequest: AbortMultipartUploadRequest): Unit = {
|
override def abortMultipartUpload(abortMultipartUploadRequest: AbortMultipartUploadRequest): Unit = {
|
||||||
canceled set true
|
canceled set true
|
||||||
}
|
}
|
||||||
}) {}
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
private def invoke(uploader: S3ClientMultiPartUploader, theFile: LocalFile, progressListener: UploadProgressListener) = {
|
private def invoke(uploader: S3ClientMultiPartUploader, theFile: LocalFile, progressListener: UploadProgressListener) = {
|
||||||
|
|
|
@ -50,11 +50,7 @@ object ActionGenerator {
|
||||||
private def copyFile(bucket: Bucket,
|
private def copyFile(bucket: Bucket,
|
||||||
localFile: LocalFile,
|
localFile: LocalFile,
|
||||||
matchByHash: Set[RemoteMetaData]): Stream[Action] =
|
matchByHash: Set[RemoteMetaData]): Stream[Action] =
|
||||||
Stream(
|
matchByHash.headOption.map(_.remoteKey).toStream.map {sourceKey =>
|
||||||
ToCopy(
|
ToCopy(bucket, sourceKey, localFile.hash, localFile.remoteKey)}
|
||||||
bucket,
|
|
||||||
sourceKey = matchByHash.head.remoteKey,
|
|
||||||
hash = localFile.hash,
|
|
||||||
targetKey = localFile.remoteKey))
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
package net.kemitix.s3thorp.core
|
||||||
|
|
||||||
|
object QuoteStripper {
|
||||||
|
|
||||||
|
def stripQuotes: Char => Boolean = _ != '"'
|
||||||
|
|
||||||
|
}
|
|
@ -1,5 +1,3 @@
|
||||||
package net.kemitix.s3thorp.domain
|
package net.kemitix.s3thorp.domain
|
||||||
|
|
||||||
final case class Bucket(name: String) {
|
final case class Bucket(name: String)
|
||||||
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,6 +1,4 @@
|
||||||
package net.kemitix.s3thorp.domain
|
package net.kemitix.s3thorp.domain
|
||||||
|
|
||||||
final case class HashModified(hash: MD5Hash,
|
final case class HashModified(hash: MD5Hash,
|
||||||
modified: LastModified) {
|
modified: LastModified)
|
||||||
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,6 +1,4 @@
|
||||||
package net.kemitix.s3thorp.domain
|
package net.kemitix.s3thorp.domain
|
||||||
|
|
||||||
final case class KeyModified(key: RemoteKey,
|
final case class KeyModified(key: RemoteKey,
|
||||||
modified: LastModified) {
|
modified: LastModified)
|
||||||
|
|
||||||
}
|
|
||||||
|
|
|
@ -2,6 +2,4 @@ package net.kemitix.s3thorp.domain
|
||||||
|
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
final case class LastModified(when: Instant) {
|
final case class LastModified(when: Instant)
|
||||||
|
|
||||||
}
|
|
||||||
|
|
|
@ -2,6 +2,4 @@ package net.kemitix.s3thorp.domain
|
||||||
|
|
||||||
final case class RemoteMetaData(remoteKey: RemoteKey,
|
final case class RemoteMetaData(remoteKey: RemoteKey,
|
||||||
hash: MD5Hash,
|
hash: MD5Hash,
|
||||||
lastModified: LastModified) {
|
lastModified: LastModified)
|
||||||
|
|
||||||
}
|
|
||||||
|
|
|
@ -3,8 +3,5 @@ package net.kemitix.s3thorp.domain
|
||||||
/**
|
/**
|
||||||
* A list of objects and their MD5 hash values.
|
* A list of objects and their MD5 hash values.
|
||||||
*/
|
*/
|
||||||
final case class S3ObjectsData(
|
final case class S3ObjectsData(byHash: Map[MD5Hash, Set[KeyModified]],
|
||||||
byHash: Map[MD5Hash, Set[KeyModified]],
|
byKey: Map[RemoteKey, HashModified])
|
||||||
byKey: Map[RemoteKey, HashModified]) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
Loading…
Reference in a new issue