diff --git a/src/main/scala/net/kemitix/s3thorp/S3MetaDataEnricher.scala b/src/main/scala/net/kemitix/s3thorp/S3MetaDataEnricher.scala index 09eeec6..6ec12ac 100644 --- a/src/main/scala/net/kemitix/s3thorp/S3MetaDataEnricher.scala +++ b/src/main/scala/net/kemitix/s3thorp/S3MetaDataEnricher.scala @@ -1,7 +1,6 @@ package net.kemitix.s3thorp import java.io.File -import java.time.Instant import fs2.Stream import cats.effect.IO @@ -9,11 +8,18 @@ import Main.putStrLn trait S3MetaDataEnricher extends S3Client { - def enrichWithS3MetaData: File => Stream[IO, S3MetaData] = - file => Stream.eval(for { - _ <- putStrLn(s"enrich: $file") - // HEAD(bucket, prefix, relative(file)) - // create blank S3MetaData records (sealed trait?) - } yield S3MetaData(file, "", "", Instant.now())) + def generateKey(c: Config)(file: File): String = { + s"${c.prefix}/${c.source.toPath.relativize(file.toPath)}" + } + def enrichWithS3MetaData(c: Config): File => Stream[IO, S3MetaData] = { + val fileToString = generateKey(c)_ + file => + Stream.eval(for { + _ <- putStrLn(s"enrich: $file") + key = fileToString(file) + head <- IO(objectHead(c.bucket, key)) + (hash, lastModified) = head + } yield S3MetaData(file, key, hash, lastModified)) + } } diff --git a/src/main/scala/net/kemitix/s3thorp/Sync.scala b/src/main/scala/net/kemitix/s3thorp/Sync.scala index 81fb3f2..a3c8aae 100644 --- a/src/main/scala/net/kemitix/s3thorp/Sync.scala +++ b/src/main/scala/net/kemitix/s3thorp/Sync.scala @@ -15,7 +15,7 @@ object Sync extends LocalFileStream with S3MetaDataEnricher { _ <- putStrLn(s"Bucket: ${c.bucket}, Prefix: ${c.prefix}, Source: ${c.source}") _ <- { streamDirectoryPaths(c.source).flatMap( - enrichWithS3MetaData).flatMap( + enrichWithS3MetaData(c)).flatMap( uploadRequiredFilter).flatMap( performUpload).compile.drain } diff --git a/src/test/scala/net/kemitix/s3thorp/S3MetaDataEnricherSuite.scala b/src/test/scala/net/kemitix/s3thorp/S3MetaDataEnricherSuite.scala new file mode 100644 index 0000000..cd9a830 --- /dev/null +++ b/src/test/scala/net/kemitix/s3thorp/S3MetaDataEnricherSuite.scala @@ -0,0 +1,39 @@ +package net.kemitix.s3thorp + +import java.io.File +import java.nio.file.Paths + +import net.kemitix.s3thorp.Sync.{Hash, LastModified} +import org.scalatest.FunSpec + +class S3MetaDataEnricherSuite extends FunSpec { + + new S3MetaDataEnricher { + describe("key generator") { + val path = "/root/from/here" + val source = Paths.get(path).toFile + val prefix = "prefix" + val config = Config("bucket", prefix, source) + val subject = generateKey(config)_ + + def resolve(subdir: String): File = { + source.toPath.resolve(subdir).toFile + } + + describe("when file is within source") { + it("has a valid key") { + val subdir = "subdir" + assertResult(s"$prefix/$subdir")(subject(resolve(subdir))) + } + } + + describe("when file is deeper within source") { + it("has a valid key") { + val subdir = "subdir/deeper/still" + assertResult(s"$prefix/$subdir")(subject(resolve(subdir))) + } + } + } + override def objectHead(bucket: String, key: String): (Hash, LastModified) = ??? + } +}