[s3metadataenricher] generate key for file and stub headobject

This commit is contained in:
Paul Campbell 2019-05-08 07:01:39 +01:00
parent b87495cfb5
commit c5044e2ff7
3 changed files with 53 additions and 8 deletions

View file

@ -1,7 +1,6 @@
package net.kemitix.s3thorp package net.kemitix.s3thorp
import java.io.File import java.io.File
import java.time.Instant
import fs2.Stream import fs2.Stream
import cats.effect.IO import cats.effect.IO
@ -9,11 +8,18 @@ import Main.putStrLn
trait S3MetaDataEnricher extends S3Client { trait S3MetaDataEnricher extends S3Client {
def enrichWithS3MetaData: File => Stream[IO, S3MetaData] = def generateKey(c: Config)(file: File): String = {
file => Stream.eval(for { s"${c.prefix}/${c.source.toPath.relativize(file.toPath)}"
_ <- putStrLn(s"enrich: $file") }
// HEAD(bucket, prefix, relative(file))
// create blank S3MetaData records (sealed trait?)
} yield S3MetaData(file, "", "", Instant.now()))
def enrichWithS3MetaData(c: Config): File => Stream[IO, S3MetaData] = {
val fileToString = generateKey(c)_
file =>
Stream.eval(for {
_ <- putStrLn(s"enrich: $file")
key = fileToString(file)
head <- IO(objectHead(c.bucket, key))
(hash, lastModified) = head
} yield S3MetaData(file, key, hash, lastModified))
}
} }

View file

@ -15,7 +15,7 @@ object Sync extends LocalFileStream with S3MetaDataEnricher {
_ <- putStrLn(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}") _ <- putStrLn(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
_ <- { _ <- {
streamDirectoryPaths(c.source).flatMap( streamDirectoryPaths(c.source).flatMap(
enrichWithS3MetaData).flatMap( enrichWithS3MetaData(c)).flatMap(
uploadRequiredFilter).flatMap( uploadRequiredFilter).flatMap(
performUpload).compile.drain performUpload).compile.drain
} }

View file

@ -0,0 +1,39 @@
package net.kemitix.s3thorp
import java.io.File
import java.nio.file.Paths
import net.kemitix.s3thorp.Sync.{Hash, LastModified}
import org.scalatest.FunSpec
class S3MetaDataEnricherSuite extends FunSpec {
new S3MetaDataEnricher {
describe("key generator") {
val path = "/root/from/here"
val source = Paths.get(path).toFile
val prefix = "prefix"
val config = Config("bucket", prefix, source)
val subject = generateKey(config)_
def resolve(subdir: String): File = {
source.toPath.resolve(subdir).toFile
}
describe("when file is within source") {
it("has a valid key") {
val subdir = "subdir"
assertResult(s"$prefix/$subdir")(subject(resolve(subdir)))
}
}
describe("when file is deeper within source") {
it("has a valid key") {
val subdir = "subdir/deeper/still"
assertResult(s"$prefix/$subdir")(subject(resolve(subdir)))
}
}
}
override def objectHead(bucket: String, key: String): (Hash, LastModified) = ???
}
}