[s3metadataenricher] generate key for file and stub headobject
This commit is contained in:
parent
b87495cfb5
commit
c5044e2ff7
3 changed files with 53 additions and 8 deletions
|
@ -1,7 +1,6 @@
|
||||||
package net.kemitix.s3thorp
|
package net.kemitix.s3thorp
|
||||||
|
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.time.Instant
|
|
||||||
|
|
||||||
import fs2.Stream
|
import fs2.Stream
|
||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
|
@ -9,11 +8,18 @@ import Main.putStrLn
|
||||||
|
|
||||||
trait S3MetaDataEnricher extends S3Client {
|
trait S3MetaDataEnricher extends S3Client {
|
||||||
|
|
||||||
def enrichWithS3MetaData: File => Stream[IO, S3MetaData] =
|
def generateKey(c: Config)(file: File): String = {
|
||||||
file => Stream.eval(for {
|
s"${c.prefix}/${c.source.toPath.relativize(file.toPath)}"
|
||||||
_ <- putStrLn(s"enrich: $file")
|
}
|
||||||
// HEAD(bucket, prefix, relative(file))
|
|
||||||
// create blank S3MetaData records (sealed trait?)
|
|
||||||
} yield S3MetaData(file, "", "", Instant.now()))
|
|
||||||
|
|
||||||
|
def enrichWithS3MetaData(c: Config): File => Stream[IO, S3MetaData] = {
|
||||||
|
val fileToString = generateKey(c)_
|
||||||
|
file =>
|
||||||
|
Stream.eval(for {
|
||||||
|
_ <- putStrLn(s"enrich: $file")
|
||||||
|
key = fileToString(file)
|
||||||
|
head <- IO(objectHead(c.bucket, key))
|
||||||
|
(hash, lastModified) = head
|
||||||
|
} yield S3MetaData(file, key, hash, lastModified))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,7 +15,7 @@ object Sync extends LocalFileStream with S3MetaDataEnricher {
|
||||||
_ <- putStrLn(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
|
_ <- putStrLn(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}")
|
||||||
_ <- {
|
_ <- {
|
||||||
streamDirectoryPaths(c.source).flatMap(
|
streamDirectoryPaths(c.source).flatMap(
|
||||||
enrichWithS3MetaData).flatMap(
|
enrichWithS3MetaData(c)).flatMap(
|
||||||
uploadRequiredFilter).flatMap(
|
uploadRequiredFilter).flatMap(
|
||||||
performUpload).compile.drain
|
performUpload).compile.drain
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,39 @@
|
||||||
|
package net.kemitix.s3thorp
|
||||||
|
|
||||||
|
import java.io.File
|
||||||
|
import java.nio.file.Paths
|
||||||
|
|
||||||
|
import net.kemitix.s3thorp.Sync.{Hash, LastModified}
|
||||||
|
import org.scalatest.FunSpec
|
||||||
|
|
||||||
|
class S3MetaDataEnricherSuite extends FunSpec {
|
||||||
|
|
||||||
|
new S3MetaDataEnricher {
|
||||||
|
describe("key generator") {
|
||||||
|
val path = "/root/from/here"
|
||||||
|
val source = Paths.get(path).toFile
|
||||||
|
val prefix = "prefix"
|
||||||
|
val config = Config("bucket", prefix, source)
|
||||||
|
val subject = generateKey(config)_
|
||||||
|
|
||||||
|
def resolve(subdir: String): File = {
|
||||||
|
source.toPath.resolve(subdir).toFile
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("when file is within source") {
|
||||||
|
it("has a valid key") {
|
||||||
|
val subdir = "subdir"
|
||||||
|
assertResult(s"$prefix/$subdir")(subject(resolve(subdir)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("when file is deeper within source") {
|
||||||
|
it("has a valid key") {
|
||||||
|
val subdir = "subdir/deeper/still"
|
||||||
|
assertResult(s"$prefix/$subdir")(subject(resolve(subdir)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
override def objectHead(bucket: String, key: String): (Hash, LastModified) = ???
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue