[s3metadataenricher] generate key for file and stub headobject
This commit is contained in:
parent
b87495cfb5
commit
c5044e2ff7
3 changed files with 53 additions and 8 deletions
|
@ -1,7 +1,6 @@
|
|||
package net.kemitix.s3thorp
|
||||
|
||||
import java.io.File
|
||||
import java.time.Instant
|
||||
|
||||
import fs2.Stream
|
||||
import cats.effect.IO
|
||||
|
@ -9,11 +8,18 @@ import Main.putStrLn
|
|||
|
||||
trait S3MetaDataEnricher extends S3Client {
|
||||
|
||||
def enrichWithS3MetaData: File => Stream[IO, S3MetaData] =
|
||||
file => Stream.eval(for {
|
||||
_ <- putStrLn(s"enrich: $file")
|
||||
// HEAD(bucket, prefix, relative(file))
|
||||
// create blank S3MetaData records (sealed trait?)
|
||||
} yield S3MetaData(file, "", "", Instant.now()))
|
||||
|
||||
def generateKey(c: Config)(file: File): String = {
|
||||
s"${c.prefix}/${c.source.toPath.relativize(file.toPath)}"
|
||||
}
|
||||
|
||||
def enrichWithS3MetaData(c: Config): File => Stream[IO, S3MetaData] = {
|
||||
val fileToString = generateKey(c)_
|
||||
file =>
|
||||
Stream.eval(for {
|
||||
_ <- putStrLn(s"enrich: $file")
|
||||
key = fileToString(file)
|
||||
head <- IO(objectHead(c.bucket, key))
|
||||
(hash, lastModified) = head
|
||||
} yield S3MetaData(file, key, hash, lastModified))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -15,7 +15,7 @@ object Sync extends LocalFileStream with S3MetaDataEnricher {
|
|||
_ <- putStrLn(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
|
||||
_ <- {
|
||||
streamDirectoryPaths(c.source).flatMap(
|
||||
enrichWithS3MetaData).flatMap(
|
||||
enrichWithS3MetaData(c)).flatMap(
|
||||
uploadRequiredFilter).flatMap(
|
||||
performUpload).compile.drain
|
||||
}
|
||||
|
|
|
@ -0,0 +1,39 @@
|
|||
package net.kemitix.s3thorp
|
||||
|
||||
import java.io.File
|
||||
import java.nio.file.Paths
|
||||
|
||||
import net.kemitix.s3thorp.Sync.{Hash, LastModified}
|
||||
import org.scalatest.FunSpec
|
||||
|
||||
class S3MetaDataEnricherSuite extends FunSpec {
|
||||
|
||||
new S3MetaDataEnricher {
|
||||
describe("key generator") {
|
||||
val path = "/root/from/here"
|
||||
val source = Paths.get(path).toFile
|
||||
val prefix = "prefix"
|
||||
val config = Config("bucket", prefix, source)
|
||||
val subject = generateKey(config)_
|
||||
|
||||
def resolve(subdir: String): File = {
|
||||
source.toPath.resolve(subdir).toFile
|
||||
}
|
||||
|
||||
describe("when file is within source") {
|
||||
it("has a valid key") {
|
||||
val subdir = "subdir"
|
||||
assertResult(s"$prefix/$subdir")(subject(resolve(subdir)))
|
||||
}
|
||||
}
|
||||
|
||||
describe("when file is deeper within source") {
|
||||
it("has a valid key") {
|
||||
val subdir = "subdir/deeper/still"
|
||||
assertResult(s"$prefix/$subdir")(subject(resolve(subdir)))
|
||||
}
|
||||
}
|
||||
}
|
||||
override def objectHead(bucket: String, key: String): (Hash, LastModified) = ???
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue