Remove ZIO (#479)

* [BROKEN]Remove ZIO and EIP-ZIO

* [BROKEN]uishell.UIShell.receiver: implement

* [BROKEN]domain.Channel: new implementation replacing MessageChannel

* [BROKEN] use domain.Channel

* Shutdown filescanner channel when finished

* start the uiChannel

* domain.Channel: channel completes once shutdown and queue empty

* uishell: down use eraseLineForward in batch mode

* lib: set file runner before adding listener

* uishell: don’t log do nothing events when not in batch mode

* domain.Channel: if exception in child thread then shutdown channel

* uishell: use correct line endings for showing chosen actions

* domain.Channel: don’t wait for shutdown if not running

* domain: remove legacy MessageChannel

* domain.Channel: don’t hold thread array

* lib.LocalFileSystem: restore delete scanner

* lib.LocalFileSystem: shutdown deletetion channel

* domain.Channel: improved shutdown logic

* clean up
This commit is contained in:
Paul Campbell 2020-06-24 22:39:16 +01:00 committed by GitHub
parent ec4ed099a5
commit a4bd24ebce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 981 additions and 1153 deletions

View file

@ -51,12 +51,6 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</dependency>
<!-- eip-zio -->
<dependency>
<groupId>net.kemitix</groupId>
<artifactId>eip-zio_2.13</artifactId>
</dependency>
</dependencies>
<build>

View file

@ -1,17 +1,8 @@
package net.kemitix.thorp
import net.kemitix.thorp.lib.FileScanner
import zio.clock.Clock
import zio.{App, ZEnv, ZIO}
object Main {
object Main extends App {
object LiveThorpApp extends Clock.Live with FileScanner.Live
override def run(args: List[String]): ZIO[ZEnv, Nothing, Int] =
Program
.run(args)
.provide(LiveThorpApp)
.fold(_ => 1, _ => 0)
def main(args: Array[String]): Unit =
Program.run(args.toList)
}

View file

@ -1,7 +1,5 @@
package net.kemitix.thorp
import net.kemitix.eip.zio.MessageChannel.UChannel
import net.kemitix.eip.zio.{Message, MessageChannel}
import net.kemitix.thorp.cli.CliArgs
import net.kemitix.thorp.config._
import net.kemitix.thorp.console._
@ -11,12 +9,10 @@ import net.kemitix.thorp.domain.StorageEvent.{
ErrorEvent,
UploadEvent
}
import net.kemitix.thorp.domain.{Counters, RemoteObjects, StorageEvent}
import net.kemitix.thorp.lib._
import net.kemitix.thorp.domain.{Channel, Counters, StorageEvent}
import net.kemitix.thorp.lib.{LocalFileSystem, UnversionedMirrorArchive}
import net.kemitix.thorp.storage.Storage
import net.kemitix.thorp.uishell.{UIEvent, UIShell}
import zio.clock.Clock
import zio.{IO, RIO, UIO, ZIO}
import scala.io.AnsiColor.{RESET, WHITE}
import scala.jdk.CollectionConverters._
@ -26,91 +22,68 @@ trait Program {
val version = "0.11.0"
lazy val versionLabel = s"${WHITE}Thorp v$version$RESET"
def run(args: List[String]): ZIO[Clock with FileScanner, Nothing, Unit] = {
(for {
cli <- UIO(CliArgs.parse(args.toArray))
config <- IO(ConfigurationBuilder.buildConfig(cli))
_ <- UIO(Console.putStrLn(versionLabel))
_ <- ZIO.when(!showVersion(cli))(
executeWithUI(config).catchAll(handleErrors)
)
} yield ())
.catchAll(e => {
Console.putStrLn("An ERROR occurred:")
Console.putStrLn(e.getMessage)
UIO.unit
})
def run(args: List[String]): Unit = {
val cli = CliArgs.parse(args.toArray)
val config = ConfigurationBuilder.buildConfig(cli)
Console.putStrLn(versionLabel)
if (!showVersion(cli)) {
executeWithUI(config)
}
}
private def showVersion: ConfigOptions => Boolean =
cli => ConfigQuery.showVersion(cli)
private def executeWithUI(
configuration: Configuration
): ZIO[Clock with FileScanner, Throwable, Unit] =
for {
uiEventSender <- execute(configuration)
uiEventReceiver <- UIShell.receiver(configuration)
_ <- MessageChannel.pointToPoint(uiEventSender)(uiEventReceiver).runDrain
} yield ()
private def executeWithUI(configuration: Configuration): Unit = {
val uiChannel: Channel[UIEvent] = Channel.create("thorp-ui")
uiChannel.addListener(UIShell.receiver(configuration))
uiChannel.run(sink => execute(configuration, sink), "thorp-main")
uiChannel.start()
uiChannel.waitForShutdown()
}
type UIChannel = UChannel[Any, UIEvent]
private def execute(configuration: Configuration,
uiSink: Channel.Sink[UIEvent]) = {
showValidConfig(uiSink)
val remoteObjects =
fetchRemoteData(configuration, uiSink)
val archive = UnversionedMirrorArchive
val storageEvents = LocalFileSystem
.scanCopyUpload(configuration, uiSink, remoteObjects, archive)
val deleteEvents = LocalFileSystem
.scanDelete(configuration, uiSink, remoteObjects, archive)
showSummary(uiSink)(storageEvents ++ deleteEvents)
uiSink.shutdown();
}
private def execute(
configuration: Configuration
): UIO[MessageChannel.ESender[Clock with FileScanner, Throwable, UIEvent]] =
UIO { uiChannel =>
(for {
_ <- showValidConfig(uiChannel)
remoteData <- fetchRemoteData(configuration, uiChannel)
archive <- UIO(UnversionedMirrorArchive)
copyUploadEvents <- LocalFileSystem
.scanCopyUpload(configuration, uiChannel, remoteData, archive)
deleteEvents <- LocalFileSystem
.scanDelete(configuration, uiChannel, remoteData, archive)
_ <- showSummary(uiChannel)(copyUploadEvents ++ deleteEvents)
} yield ()) <* MessageChannel.endChannel(uiChannel)
}
private def showValidConfig(uiSink: Channel.Sink[UIEvent]): Unit =
uiSink.accept(UIEvent.showValidConfig)
private def showValidConfig(uiChannel: UIChannel) =
Message.create(UIEvent.showValidConfig) >>= MessageChannel.send(uiChannel)
private def fetchRemoteData(
configuration: Configuration,
uiChannel: UIChannel
): ZIO[Clock, Throwable, RemoteObjects] = {
private def fetchRemoteData(configuration: Configuration,
uiSink: Channel.Sink[UIEvent]) = {
val bucket = configuration.bucket
val prefix = configuration.prefix
val objects = Storage.getInstance().list(bucket, prefix)
for {
_ <- Message.create(UIEvent.remoteDataFetched(objects.byKey.size)) >>= MessageChannel
.send(uiChannel)
} yield objects
uiSink.accept(UIEvent.remoteDataFetched(objects.byKey.size))
objects
}
private def handleErrors(throwable: Throwable) =
UIO(Console.putStrLn("There were errors:")) *> logValidationErrors(
throwable
)
//TODO not called
private def logValidationErrors(throwable: Throwable) =
throwable match {
case validateError: ConfigValidationException =>
ZIO.foreach_(validateError.getErrors.asScala)(
error => UIO(Console.putStrLn(s"- $error"))
)
validateError.getErrors.asScala
.map(error => Console.putStrLn(s"- $error"))
}
private def showSummary(
uiChannel: UIChannel
)(events: Seq[StorageEvent]): RIO[Clock, Unit] = {
uiSink: Channel.Sink[UIEvent]
)(events: Seq[StorageEvent]): Unit = {
val counters = events.foldLeft(Counters.empty)(countActivities)
Message.create(UIEvent.showSummary(counters)) >>=
MessageChannel.send(uiChannel)
uiSink.accept(UIEvent.showSummary(counters))
}
private def countActivities: (Counters, StorageEvent) => Counters =
private def countActivities =
(counters: Counters, s3Action: StorageEvent) => {
s3Action match {
case _: UploadEvent => counters.incrementUploaded()
@ -120,7 +93,6 @@ trait Program {
case _ => counters
}
}
}
object Program extends Program

View file

@ -3,7 +3,6 @@ package net.kemitix.thorp.config;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.List;
public interface ParseConfigFile {
static ConfigOptions parseFile(File file) throws IOException {

View file

@ -15,9 +15,8 @@ public interface SourceConfigLoader {
ConfigOptions.create(
sources.paths()
.stream()
.peek(path -> {
System.out.println("Using source: " + path);
})
.peek(path ->
System.out.println("Using source: " + path))
.map(ConfigOption::source)
.collect(Collectors.toList()));
// add settings from each source as options

Binary file not shown.

Before

Width:  |  Height:  |  Size: 233 KiB

After

Width:  |  Height:  |  Size: 219 KiB

View file

@ -0,0 +1,188 @@
package net.kemitix.thorp.domain;
import lombok.RequiredArgsConstructor;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedTransferQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
public interface Channel<T> {
static <T> Channel<T> create(String name) {
return new ChannelImpl<T>(name);
}
void start();
Channel<T> add(T item);
Channel<T> addAll(Collection<T> items);
Channel<T> addListener(Listener<T> listener);
Channel<T> removeListener(Listener<T> listener);
Channel<T> run(Consumer<Sink<T>> program, String name);
void shutdown();
void shutdownNow() throws InterruptedException;
void waitForShutdown() throws InterruptedException;
class ChannelImpl<T> implements Channel<T> {
private final BlockingQueue<T> queue = new LinkedTransferQueue<>();
private final Runner<T> runner;
private final Thread thread;
public ChannelImpl(String name) {
runner = new Runner<T>(name, queue);
thread = new Thread(runner, name);
}
@Override
public void start() {
thread.start();
}
@Override
public Channel<T> add(T item) {
queue.add(item);
return this;
}
@Override
public Channel<T> addAll(Collection<T> items) {
queue.addAll(items);
return this;
}
@Override
public Channel<T> addListener(Listener<T> listener) {
runner.addListener(listener);
return this;
}
@Override
public Channel<T> removeListener(Listener<T> listener) {
runner.removeListener(listener);
return this;
}
@Override
public Channel<T> run(Consumer<Sink<T>> program, String name) {
return spawn(() -> program.accept(runner), name);
}
private Channel<T> spawn(Runnable runnable, String name) {
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
runnable.run();
} catch (Exception e) {
shutdown();
}
}
}, name);
thread.start();
return this;
}
@Override
public void shutdown() {
runner.shutdown();
}
@Override
public void shutdownNow() throws InterruptedException {
runner.shutdownNow();
}
@Override
public void waitForShutdown() throws InterruptedException {
runner.waitForShutdown();
}
}
@RequiredArgsConstructor
class Runner<T> implements Runnable, Sink<T> {
private final String name;
private final BlockingQueue<T> queue;
private final AtomicBoolean shutdown = new AtomicBoolean(false);
private final List<Listener<T>> listeners = new ArrayList<>();
private final CountDownLatch shutdownLatch = new CountDownLatch(1);
private Thread runnerThread;
@Override
public void run() {
runnerThread = Thread.currentThread();
while(isRunning()) {
takeItem()
.ifPresent(item -> {
listeners.forEach(listener -> {
listener.accept(item);
});
});
}
shutdownLatch.countDown();
}
public void addListener(Listener<T> listener) {
listeners.add(listener);
}
public void removeListener(Listener<T> listener) {
listeners.remove(listener);
}
public Optional<T> takeItem() {
try {
return Optional.of(queue.take());
} catch (InterruptedException e) {
shutdown();
}
return Optional.empty();
}
private boolean isRunning() {
return !isShutdown();
}
private boolean isShutdown() {
return shutdown.get() && queue.isEmpty();
}
@Override
public void accept(T item) {
queue.add(item);
}
@Override
public void shutdown() {
if (isRunning()) {
shutdown.set(true);
}
if (queue.isEmpty() && runnerThread != null) {
runnerThread.interrupt();
}
}
public void shutdownNow() throws InterruptedException {
shutdown();
waitForShutdown();
}
public void waitForShutdown() throws InterruptedException {
if (isRunning())
shutdownLatch.await();
}
}
interface Listener<T> {
void accept(T item);
}
interface Sink<T> {
void accept(T item);
void shutdown();
}
}

View file

@ -1,71 +0,0 @@
package net.kemitix.thorp.domain;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedTransferQueue;
import java.util.concurrent.atomic.AtomicBoolean;
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public class MessageChannel<T> {
private final MessageSupplier<T> messageSupplier;
private final List<MessageConsumer<T>> messageConsumers;
private final Thread channelThread;
static <T> MessageChannel<T> create(MessageSupplier<T> supplier) {
List<MessageConsumer<T>> consumers = new ArrayList<>();
return new MessageChannel<T>(supplier, consumers,
new Thread(new ChannelRunner<T>(supplier, consumers)));
}
public static <T> BlockingQueue<T> createMessageSupplier(Class<T> messageClass) {
return new LinkedTransferQueue<>();
}
public void addMessageConsumer(MessageConsumer<T> consumer) {
messageConsumers.add(consumer);
}
public void startChannel() {
channelThread.start();
}
public void shutdownChannel() {
channelThread.interrupt();
}
public interface MessageSupplier<T> {
T take() throws InterruptedException;
boolean isComplete();
}
public interface MessageConsumer<T> {
void accept(T message);
}
@RequiredArgsConstructor
private static class ChannelRunner<T> implements Runnable {
AtomicBoolean shutdownTrigger = new AtomicBoolean(false);
private final MessageSupplier<T> supplier;
private final List<MessageConsumer<T>> consumers;
@Override
public void run() {
while (!shutdownTrigger.get()) {
try {
T message = supplier.take();
for (MessageConsumer<T> consumer : consumers) {
consumer.accept(message);
}
if (supplier.isComplete()) {
shutdownTrigger.set(true);
}
} catch (InterruptedException e) {
shutdownTrigger.set(true);
}
}
}
}
}

View file

@ -16,6 +16,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<!-- thorp -->

View file

@ -0,0 +1,80 @@
package net.kemitix.thorp.lib;
import net.kemitix.thorp.config.Configuration;
import net.kemitix.thorp.domain.*;
import net.kemitix.thorp.filesystem.FileSystem;
import net.kemitix.thorp.filesystem.PathCache;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.security.NoSuchAlgorithmException;
import java.util.List;
public interface FileScanner {
static void scanSources(
Configuration configuration,
Channel.Sink<LocalFile> fileSink
) {
configuration.sources.paths()
.forEach(path ->
scanSource(configuration, fileSink, path));
fileSink.shutdown();
}
static void scanSource(
Configuration configuration,
Channel.Sink<LocalFile> fileSink,
Path sourcePath
) {
scanPath(configuration, fileSink, sourcePath);
}
static void scanPath(
Configuration configuration,
Channel.Sink<LocalFile> fileSink,
Path path
) {
// dirs
FileSystem.listDirs(path).forEach(dir ->
scanPath(configuration, fileSink, dir));
// files
List<File> files = FileSystem.listFiles(path);
files.forEach(file -> handleFile(configuration, fileSink, file));
}
static void handleFile(
Configuration configuration,
Channel.Sink<LocalFile> fileSink,
File file
) {
boolean isIncluded = Filters.isIncluded(configuration, file);
if (isIncluded) {
File source = configuration.sources.forPath(file.toPath()).toFile();
Hashes hashes = hashObject(file);
RemoteKey remoteKey =
RemoteKey.from(source.toPath(), configuration.prefix, file);
LocalFile localFile =
LocalFile.create(
file, source, hashes, remoteKey, file.length());
fileSink.accept(localFile);
}
}
static Hashes hashObject(File file) {
try {
return HashGenerator.hashObject(file.toPath());
} catch (IOException | NoSuchAlgorithmException e) {
throw new RuntimeException("Error hashing object: " + file, e);
}
}
static PathCache findCache(Path sourcePath) {
try {
return FileSystem.findCache(sourcePath);
} catch (IOException e) {
throw new RuntimeException(
"Error finding source cache for source: " + sourcePath, e);
}
}
}

View file

@ -1,160 +0,0 @@
package net.kemitix.thorp.lib
import java.io.File
import java.nio.file.Path
import scala.jdk.CollectionConverters._
import net.kemitix.eip.zio.MessageChannel.{EChannel, ESender}
import net.kemitix.eip.zio.{Message, MessageChannel}
import net.kemitix.thorp.config.Configuration
import net.kemitix.thorp.domain._
import net.kemitix.thorp.filesystem._
import zio.clock.Clock
import zio.{RIO, UIO, ZIO}
trait FileScanner {
val fileScanner: FileScanner.Service
}
object FileScanner {
type RemoteHashes = Map[MD5Hash, RemoteKey]
type ScannedFile = LocalFile
type FileSender =
ESender[Clock with FileScanner, Throwable, ScannedFile]
type ScannerChannel = EChannel[Any, Throwable, ScannedFile]
type CacheData = (Path, FileData)
type CacheChannel = EChannel[Any, Throwable, CacheData]
type CacheSender =
ESender[Clock with FileScanner, Throwable, CacheData]
final def scanSources(
configuration: Configuration): RIO[FileScanner, FileSender] =
ZIO.accessM(_.fileScanner.scanSources(configuration))
trait Service {
def scanSources(configuration: Configuration): RIO[FileScanner, FileSender]
}
trait Live extends FileScanner {
val fileScanner: Service = new Service {
override def scanSources(
configuration: Configuration): RIO[FileScanner, FileSender] =
RIO {
fileChannel: EChannel[Clock with FileScanner,
Throwable,
ScannedFile] =>
{
val sources = configuration.sources
(for {
_ <- ZIO.foreach(sources.paths.asScala) { sourcePath =>
for {
cacheSender <- scanSource(configuration, fileChannel)(
sourcePath)
cacheReceiver <- cacheReceiver(sourcePath)
_ <- MessageChannel
.pointToPoint(cacheSender)(cacheReceiver)
.runDrain
_ = FileSystem.moveFile(
sourcePath.resolve(PathCache.tempFileName),
sourcePath.resolve(PathCache.fileName))
} yield ()
}
} yield ()) <* MessageChannel.endChannel(fileChannel)
}
}
private def scanSource(configuration: Configuration,
fileChannel: ScannerChannel)(
sourcePath: Path): RIO[FileScanner, CacheSender] =
RIO { cacheChannel =>
(for {
cache <- UIO(FileSystem.findCache(sourcePath))
_ <- scanPath(configuration, fileChannel, cacheChannel)(sourcePath,
cache)
} yield ()) <* MessageChannel.endChannel(cacheChannel)
}
private def scanPath(configuration: Configuration,
fileChannel: ScannerChannel,
cacheChannel: CacheChannel)(
path: Path,
cache: PathCache): ZIO[Clock with FileScanner, Throwable, Unit] =
for {
dirs <- UIO(FileSystem.listDirs(path))
_ <- ZIO.foreach(dirs.asScala)(
scanPath(configuration, fileChannel, cacheChannel)(_, cache))
files = FileSystem.listFiles(path).asScala.toList
_ <- handleFiles(configuration,
fileChannel,
cacheChannel,
cache,
files)
} yield ()
private def handleFiles(
configuration: Configuration,
fileChannel: ScannerChannel,
cacheChannel: CacheChannel,
pathCache: PathCache,
files: List[File]
): ZIO[Clock, Throwable, List[Unit]] =
ZIO.foreach(files) {
handleFile(configuration, fileChannel, cacheChannel, pathCache)
}
private def handleFile(
configuration: Configuration,
fileChannel: ScannerChannel,
cacheChannel: CacheChannel,
cache: PathCache
)(file: File): ZIO[Clock, Throwable, Unit] =
for {
isIncluded <- Filters.isIncluded(configuration, file)
_ <- ZIO.when(isIncluded) {
sendHashedFile(configuration, fileChannel, cacheChannel)(file,
cache)
}
} yield ()
private def sendHashedFile(
configuration: Configuration,
fileChannel: ScannerChannel,
cacheChannel: CacheChannel
)(file: File, pathCache: PathCache) = {
val sources = configuration.sources
val source = sources.forPath(file.toPath)
val prefix = configuration.prefix
val path = source.relativize(file.toPath)
val hashes = HashGenerator.hashObject(file.toPath)
val remoteKey = RemoteKey.from(source, prefix, file)
val size = file.length()
for {
fileMsg <- Message.create(
LocalFile.create(file, source.toFile, hashes, remoteKey, size))
_ <- MessageChannel.send(fileChannel)(fileMsg)
modified <- UIO(FileSystem.lastModified(file))
cacheMsg <- Message.create(
path -> FileData.create(hashes, LastModified.at(modified)))
_ <- MessageChannel.send(cacheChannel)(cacheMsg)
} yield ()
}
def cacheReceiver(
sourcePath: Path): UIO[MessageChannel.UReceiver[Any, CacheData]] = {
val tempFile = sourcePath.resolve(PathCache.tempFileName).toFile
UIO { message =>
val (path, fileData) = message.body
for {
line <- UIO(PathCache.export(path, fileData).asScala)
_ <- UIO(FileSystem.appendLines(line.toList.asJava, tempFile))
} yield ()
}
}
}
}
object Live extends Live
}

View file

@ -6,30 +6,31 @@ import java.nio.file.Path
import net.kemitix.thorp.config.Configuration
import net.kemitix.thorp.domain.Filter
import net.kemitix.thorp.domain.Filter.{Exclude, Include}
import zio.UIO
import scala.jdk.CollectionConverters._
object Filters {
def isIncluded(configuration: Configuration, file: File): UIO[Boolean] =
UIO(isIncluded(file.toPath)(configuration.filters.asScala.toList))
def isIncluded(configuration: Configuration, file: File): Boolean =
isIncluded(file.toPath)(configuration.filters.asScala.toList)
def isIncluded(p: Path)(filters: List[Filter]): Boolean = {
sealed trait State
final case class Unknown() extends State
final case class Accepted() extends State
final case class Unknown() extends State
final case class Accepted() extends State
final case class Discarded() extends State
val excluded = isExcludedByFilter(p)(_)
val included = isIncludedByFilter(p)(_)
filters.foldRight(Unknown(): State)((filter, state) =>
(filter, state) match {
case (_, Accepted()) => Accepted()
case (_, Discarded()) => Discarded()
case (x: Exclude, _) if excluded(x) => Discarded()
case (i: Include, _) if included(i) => Accepted()
case _ => Unknown()
}) match {
filters.foldRight(Unknown(): State)(
(filter, state) =>
(filter, state) match {
case (_, Accepted()) => Accepted()
case (_, Discarded()) => Discarded()
case (x: Exclude, _) if excluded(x) => Discarded()
case (i: Include, _) if included(i) => Accepted()
case _ => Unknown()
}
) match {
case Accepted() => true
case Discarded() => false
case Unknown() =>

View file

@ -1,190 +1,179 @@
package net.kemitix.thorp.lib
import scala.jdk.OptionConverters._
import scala.jdk.CollectionConverters._
import net.kemitix.eip.zio.MessageChannel.UChannel
import net.kemitix.eip.zio.{Message, MessageChannel}
import java.util
import java.util.concurrent.atomic.{AtomicInteger, AtomicLong}
import net.kemitix.thorp.config.Configuration
import net.kemitix.thorp.domain.RemoteObjects
import net.kemitix.thorp.domain._
import net.kemitix.thorp.domain.Channel.{Listener, Sink}
import net.kemitix.thorp.domain.{RemoteObjects, _}
import net.kemitix.thorp.filesystem.FileSystem
import net.kemitix.thorp.storage.Storage
import net.kemitix.thorp.uishell.UIEvent
import zio._
import zio.clock.Clock
import scala.jdk.CollectionConverters._
import scala.jdk.OptionConverters._
trait LocalFileSystem {
def scanCopyUpload(
configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
remoteObjects: RemoteObjects,
archive: ThorpArchive
): RIO[Clock with FileScanner with Storage, Seq[StorageEvent]]
def scanCopyUpload(configuration: Configuration,
uiSink: Channel.Sink[UIEvent],
remoteObjects: RemoteObjects,
archive: ThorpArchive): Seq[StorageEvent]
def scanDelete(
configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
remoteData: RemoteObjects,
archive: ThorpArchive
): RIO[Clock with Storage, Seq[StorageEvent]]
def scanDelete(configuration: Configuration,
uiSink: Channel.Sink[UIEvent],
remoteData: RemoteObjects,
archive: ThorpArchive): Seq[StorageEvent]
}
object LocalFileSystem extends LocalFileSystem {
override def scanCopyUpload(
configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
remoteObjects: RemoteObjects,
archive: ThorpArchive
): RIO[Clock with FileScanner, Seq[StorageEvent]] =
for {
actionCounter <- Ref.make(0)
bytesCounter <- Ref.make(0L)
uploads <- Ref.make(Map.empty[MD5Hash, Promise[Throwable, RemoteKey]])
eventsRef <- Ref.make(List.empty[StorageEvent])
fileSender <- FileScanner.scanSources(configuration)
fileReceiver <- fileReceiver(
override def scanCopyUpload(configuration: Configuration,
uiSink: Channel.Sink[UIEvent],
remoteObjects: RemoteObjects,
archive: ThorpArchive): Seq[StorageEvent] = {
val fileChannel: Channel[LocalFile] = Channel.create("files")
// state for the file receiver
val actionCounter = new AtomicInteger()
val bytesCounter = new AtomicLong()
val uploads = Map.empty[MD5Hash, RemoteKey]
val events = new util.LinkedList[StorageEvent]
fileChannel.addListener(
fileReceiver(
configuration,
uiChannel,
uiSink,
remoteObjects,
archive,
uploads,
actionCounter,
bytesCounter,
eventsRef
events
)
parallel = configuration.parallel
_ <- MessageChannel
.pointToPointPar(parallel)(fileSender)(fileReceiver)
.runDrain
events <- eventsRef.get
} yield events
)
override def scanDelete(
configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
remoteData: RemoteObjects,
archive: ThorpArchive
): RIO[Clock, Seq[StorageEvent]] =
for {
actionCounter <- Ref.make(0)
bytesCounter <- Ref.make(0L)
eventsRef <- Ref.make(List.empty[StorageEvent])
keySender <- keySender(remoteData.byKey.keys.asScala)
keyReceiver <- keyReceiver(
fileChannel.run(
sink => FileScanner.scanSources(configuration, sink),
"scan-sources"
)
fileChannel.start()
fileChannel.waitForShutdown()
events.asScala.toList
}
override def scanDelete(configuration: Configuration,
uiSink: Channel.Sink[UIEvent],
remoteData: RemoteObjects,
archive: ThorpArchive): Seq[StorageEvent] = {
val deletionsChannel: Channel[RemoteKey] = Channel.create("deletions")
// state for the file receiver
val actionCounter = new AtomicInteger()
val bytesCounter = new AtomicLong()
val events = new util.LinkedList[StorageEvent]
deletionsChannel.addListener(
keyReceiver(
configuration,
uiChannel,
uiSink,
archive,
actionCounter,
bytesCounter,
eventsRef
events
)
parallel = configuration.parallel
_ <- MessageChannel
.pointToPointPar(parallel)(keySender)(keyReceiver)
.runDrain
events <- eventsRef.get
} yield events
)
deletionsChannel.run(sink => {
remoteData.byKey.keys().forEach(key => sink.accept(key))
sink.shutdown()
}, "delete-source")
deletionsChannel.start()
deletionsChannel.waitForShutdown()
events.asScala.toList
}
private def fileReceiver(
configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
uiSink: Channel.Sink[UIEvent],
remoteObjects: RemoteObjects,
archive: ThorpArchive,
uploads: Ref[Map[MD5Hash, Promise[Throwable, RemoteKey]]],
actionCounterRef: Ref[Int],
bytesCounterRef: Ref[Long],
eventsRef: Ref[List[StorageEvent]]
): UIO[MessageChannel.UReceiver[Clock, FileScanner.ScannedFile]] =
UIO { message =>
val localFile = message.body
for {
_ <- uiFileFound(uiChannel)(localFile)
action <- chooseAction(
configuration,
remoteObjects,
uploads,
uiChannel
)(localFile)
actionCounter <- actionCounterRef.update(_ + 1)
bytesCounter <- bytesCounterRef.update(_ + action.size)
_ <- uiActionChosen(uiChannel)(action)
sequencedAction = SequencedAction(action, actionCounter)
event <- archive.update(
configuration,
uiChannel,
sequencedAction,
bytesCounter
)
_ <- eventsRef.update(list => event :: list)
_ <- uiActionFinished(uiChannel)(
action,
actionCounter,
bytesCounter,
event
)
} yield ()
uploads: Map[MD5Hash, RemoteKey],
actionCounter: AtomicInteger,
bytesCounter: AtomicLong,
events: util.Deque[StorageEvent]
): Listener[LocalFile] = { (localFile: LocalFile) =>
{
uiFileFound(uiSink)(localFile)
val action =
chooseAction(configuration, remoteObjects, uploads, uiSink)(localFile)
actionCounter.incrementAndGet()
bytesCounter.addAndGet(action.size)
uiActionChosen(uiSink)(action)
val sequencedAction = SequencedAction(action, actionCounter.get())
val event = archive
.update(configuration, uiSink, sequencedAction, bytesCounter.get)
events.addFirst(event)
uiActionFinished(uiSink)(
action,
actionCounter.get,
bytesCounter.get,
event
)
}
}
private def uiActionChosen(
uiChannel: MessageChannel.UChannel[Any, UIEvent]
)(action: Action) =
Message.create(UIEvent.actionChosen(action)) >>=
MessageChannel.send(uiChannel)
uiSink: Channel.Sink[UIEvent]
)(action: Action): Unit =
uiSink.accept(UIEvent.actionChosen(action))
private def uiActionFinished(uiChannel: UChannel[Any, UIEvent])(
private def uiActionFinished(uiSink: Channel.Sink[UIEvent])(
action: Action,
actionCounter: Int,
bytesCounter: Long,
event: StorageEvent
) =
Message.create(
): Unit =
uiSink.accept(
UIEvent.actionFinished(action, actionCounter, bytesCounter, event)
) >>=
MessageChannel.send(uiChannel)
)
private def uiFileFound(
uiChannel: UChannel[Any, UIEvent]
)(localFile: LocalFile) =
Message.create(UIEvent.fileFound(localFile)) >>=
MessageChannel.send(uiChannel)
uiSink: Channel.Sink[UIEvent]
)(localFile: LocalFile): Unit =
uiSink.accept(UIEvent.fileFound(localFile))
private def chooseAction(
configuration: Configuration,
remoteObjects: RemoteObjects,
uploads: Ref[Map[MD5Hash, Promise[Throwable, RemoteKey]]],
uiChannel: UChannel[Any, UIEvent],
)(localFile: LocalFile): ZIO[Clock, Nothing, Action] = {
for {
remoteExists <- UIO(remoteObjects.remoteKeyExists(localFile.remoteKey))
remoteMatches <- UIO(remoteObjects.remoteMatchesLocalFile(localFile))
remoteForHash <- UIO(
remoteObjects.remoteHasHash(localFile.hashes).toScala
)
previous <- uploads.get
bucket = configuration.bucket
action <- if (remoteExists && remoteMatches)
doNothing(localFile, bucket)
else {
remoteForHash match {
case pair: Some[Tuple[RemoteKey, MD5Hash]] =>
val sourceKey = pair.value.a
val hash = pair.value.b
doCopy(localFile, bucket, sourceKey, hash)
case _ if matchesPreviousUpload(previous, localFile.hashes) =>
doCopyWithPreviousUpload(localFile, bucket, previous, uiChannel)
case _ =>
doUpload(localFile, bucket)
}
private def chooseAction(configuration: Configuration,
remoteObjects: RemoteObjects,
uploads: Map[MD5Hash, RemoteKey],
uiSink: Channel.Sink[UIEvent],
)(localFile: LocalFile): Action = {
val remoteExists = remoteObjects.remoteKeyExists(localFile.remoteKey)
val remoteMatches = remoteObjects.remoteMatchesLocalFile(localFile)
val remoteForHash = remoteObjects.remoteHasHash(localFile.hashes).toScala
val previous = uploads
val bucket = configuration.bucket
val action = if (remoteExists && remoteMatches) {
doNothing(localFile, bucket)
} else {
remoteForHash match {
case pair: Some[Tuple[RemoteKey, MD5Hash]] =>
val sourceKey = pair.value.a
val hash = pair.value.b
doCopy(localFile, bucket, sourceKey, hash)
case _ if matchesPreviousUpload(previous, localFile.hashes) =>
doCopyWithPreviousUpload(localFile, bucket, previous, uiSink)
case _ =>
doUpload(localFile, bucket)
}
} yield action
}
action
}
private def matchesPreviousUpload(
previous: Map[MD5Hash, Promise[Throwable, RemoteKey]],
hashes: Hashes
): Boolean =
private def matchesPreviousUpload(previous: Map[MD5Hash, RemoteKey],
hashes: Hashes) =
hashes
.values()
.stream()
@ -192,25 +181,21 @@ object LocalFileSystem extends LocalFileSystem {
previous.contains(hash)
})
private def doNothing(localFile: LocalFile, bucket: Bucket): UIO[Action] =
UIO {
Action.doNothing(bucket, localFile.remoteKey, localFile.length)
}
private def doNothing(localFile: LocalFile, bucket: Bucket) =
Action.doNothing(bucket, localFile.remoteKey, localFile.length)
private def doCopy(localFile: LocalFile,
bucket: Bucket,
sourceKey: RemoteKey,
hash: MD5Hash): UIO[Action] = UIO {
hash: MD5Hash) =
Action
.toCopy(bucket, sourceKey, hash, localFile.remoteKey, localFile.length)
}
private def doCopyWithPreviousUpload(
localFile: LocalFile,
bucket: Bucket,
previous: Map[MD5Hash, Promise[Throwable, RemoteKey]],
uiChannel: UChannel[Any, UIEvent],
): ZIO[Clock, Nothing, Action] = {
private def doCopyWithPreviousUpload(localFile: LocalFile,
bucket: Bucket,
previous: Map[MD5Hash, RemoteKey],
uiSink: Channel.Sink[UIEvent],
) = {
localFile.hashes
.values()
.stream()
@ -220,91 +205,58 @@ object LocalFileSystem extends LocalFileSystem {
.findFirst()
.toScala
.map({ hash =>
for {
awaitingMessage <- Message.create(
UIEvent.awaitingAnotherUpload(localFile.remoteKey, hash)
{
uiSink
.accept(UIEvent.awaitingAnotherUpload(localFile.remoteKey, hash))
val action = Action.toCopy(
bucket,
previous(hash),
hash,
localFile.remoteKey,
localFile.length
)
_ <- MessageChannel.send(uiChannel)(awaitingMessage)
action <- previous(hash).await.map(
remoteKey =>
Action.toCopy(
bucket,
remoteKey,
hash,
localFile.remoteKey,
localFile.length
)
)
waitFinishedMessage <- Message.create(
UIEvent.anotherUploadWaitComplete(action)
)
_ <- MessageChannel.send(uiChannel)(waitFinishedMessage)
} yield action
uiSink.accept(UIEvent.anotherUploadWaitComplete(action))
action
}
})
.getOrElse(doUpload(localFile, bucket))
.refineToOrDie[Nothing]
}
private def doUpload(localFile: LocalFile, bucket: Bucket): UIO[Action] = {
UIO(Action.toUpload(bucket, localFile, localFile.length))
}
private def doUpload(localFile: LocalFile, bucket: Bucket) =
Action.toUpload(bucket, localFile, localFile.length)
def keySender(
keys: Iterable[RemoteKey]
): UIO[MessageChannel.Sender[Clock, RemoteKey]] =
UIO { channel =>
ZIO.foreach(keys) { key =>
Message.create(key) >>= MessageChannel.send(channel)
} *> MessageChannel.endChannel(channel)
}
def keyReceiver(
configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
archive: ThorpArchive,
actionCounterRef: Ref[Int],
bytesCounterRef: Ref[Long],
eventsRef: Ref[List[StorageEvent]]
): UIO[MessageChannel.UReceiver[Clock, RemoteKey]] =
UIO { message =>
def keyReceiver(configuration: Configuration,
uiSink: Channel.Sink[UIEvent],
archive: ThorpArchive,
actionCounter: AtomicInteger,
bytesCounter: AtomicLong,
events: util.Deque[StorageEvent]): Listener[RemoteKey] = {
(remoteKey: RemoteKey) =>
{
val remoteKey = message.body
for {
_ <- uiKeyFound(uiChannel)(remoteKey)
sources = configuration.sources
prefix = configuration.prefix
exists = FileSystem.hasLocalFile(sources, prefix, remoteKey)
_ <- ZIO.when(!exists) {
for {
actionCounter <- actionCounterRef.update(_ + 1)
bucket = configuration.bucket
action = Action.toDelete(bucket, remoteKey, 0L)
_ <- uiActionChosen(uiChannel)(action)
bytesCounter <- bytesCounterRef.update(_ + action.size)
sequencedAction = SequencedAction(action, actionCounter)
event <- archive.update(
configuration,
uiChannel,
sequencedAction,
0L
)
_ <- eventsRef.update(list => event :: list)
_ <- uiActionFinished(uiChannel)(
action,
actionCounter,
bytesCounter,
event
)
} yield ()
}
} yield ()
uiKeyFound(uiSink)(remoteKey)
val sources = configuration.sources
val prefix = configuration.prefix
val exists = FileSystem.hasLocalFile(sources, prefix, remoteKey)
if (!exists) {
actionCounter.incrementAndGet()
val bucket = configuration.bucket
val action = Action.toDelete(bucket, remoteKey, 0L)
uiActionChosen(uiSink)(action)
bytesCounter.addAndGet(action.size)
val sequencedAction = SequencedAction(action, actionCounter.get())
val event = archive.update(configuration, uiSink, sequencedAction, 0L)
events.addFirst(event)
uiActionFinished(uiSink)(
action,
actionCounter.get(),
bytesCounter.get(),
event
)
}
}
}
}
private def uiKeyFound(
uiChannel: UChannel[Any, UIEvent]
)(remoteKey: RemoteKey) =
Message.create(UIEvent.keyFound(remoteKey)) >>=
MessageChannel.send(uiChannel)
private def uiKeyFound(uiSink: Sink[UIEvent])(remoteKey: RemoteKey): Unit =
uiSink.accept(UIEvent.keyFound(remoteKey))
}

View file

@ -1,66 +1,44 @@
package net.kemitix.thorp.lib
import net.kemitix.eip.zio.MessageChannel.UChannel
import net.kemitix.thorp.config.Configuration
import net.kemitix.thorp.console._
import net.kemitix.thorp.domain.StorageEvent
import net.kemitix.thorp.domain.StorageEvent._
import net.kemitix.thorp.domain.{Channel, StorageEvent}
import net.kemitix.thorp.uishell.UIEvent
import zio.UIO
trait ThorpArchive {
def update(configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
uiSink: Channel.Sink[UIEvent],
sequencedAction: SequencedAction,
totalBytesSoFar: Long): UIO[StorageEvent]
totalBytesSoFar: Long): StorageEvent
def logEvent(configuration: Configuration,
event: StorageEvent): UIO[StorageEvent] = {
event: StorageEvent): StorageEvent = {
val batchMode = configuration.batchMode
for {
sqe <- event match {
case uploadEvent: UploadEvent =>
val remoteKey = uploadEvent.remoteKey
UIO(event) <* {
Console.putMessageLnB(
ConsoleOut.uploadComplete(remoteKey),
batchMode
)
UIO.unit
}
case copyEvent: CopyEvent =>
val sourceKey = copyEvent.sourceKey
val targetKey = copyEvent.targetKey
UIO(event) <* {
Console.putMessageLnB(
ConsoleOut.copyComplete(sourceKey, targetKey),
batchMode
)
UIO.unit
}
case deleteEvent: DeleteEvent =>
val remoteKey = deleteEvent.remoteKey
UIO(event) <* {
Console.putMessageLnB(
ConsoleOut.deleteComplete(remoteKey),
batchMode
)
UIO.unit
}
case errorEvent: ErrorEvent =>
val action = errorEvent.action
val e = errorEvent.e
UIO(event) <* {
Console.putMessageLnB(
ConsoleOut.errorQueueEventOccurred(action, e),
batchMode
)
UIO.unit
}
case _ => UIO(event)
}
} yield sqe
event match {
case uploadEvent: UploadEvent =>
val remoteKey = uploadEvent.remoteKey
Console.putMessageLnB(ConsoleOut.uploadComplete(remoteKey), batchMode)
case copyEvent: CopyEvent =>
val sourceKey = copyEvent.sourceKey
val targetKey = copyEvent.targetKey
Console.putMessageLnB(
ConsoleOut.copyComplete(sourceKey, targetKey),
batchMode
)
case deleteEvent: DeleteEvent =>
val remoteKey = deleteEvent.remoteKey
Console.putMessageLnB(ConsoleOut.deleteComplete(remoteKey), batchMode)
case errorEvent: ErrorEvent =>
val action = errorEvent.action
val e = errorEvent.e
Console.putMessageLnB(
ConsoleOut.errorQueueEventOccurred(action, e),
batchMode
)
}
event
}
}

View file

@ -1,59 +1,49 @@
package net.kemitix.thorp.lib
import net.kemitix.eip.zio.MessageChannel.UChannel
import net.kemitix.thorp.config.Configuration
import net.kemitix.thorp.domain.Action.{ToCopy, ToDelete, ToUpload}
import net.kemitix.thorp.domain._
import net.kemitix.thorp.storage.Storage
import net.kemitix.thorp.uishell.{UIEvent, UploadEventListener}
import zio.UIO
trait UnversionedMirrorArchive extends ThorpArchive {
override def update(configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
uiSink: Channel.Sink[UIEvent],
sequencedAction: SequencedAction,
totalBytesSoFar: Long): UIO[StorageEvent] = {
totalBytesSoFar: Long): StorageEvent = {
val action = sequencedAction.action
val index = sequencedAction.index
val bucket = action.bucket
action match {
case upload: ToUpload =>
val localFile = upload.localFile
UIO {
doUpload(
configuration,
uiChannel,
index,
totalBytesSoFar,
bucket,
localFile
)
}
doUpload(
configuration,
uiSink,
index,
totalBytesSoFar,
bucket,
localFile
)
case toCopy: ToCopy =>
val sourceKey = toCopy.sourceKey
val hash = toCopy.hash
val targetKey = toCopy.targetKey
UIO {
Storage
.getInstance()
.copy(bucket, sourceKey, hash, targetKey)
}
Storage
.getInstance()
.copy(bucket, sourceKey, hash, targetKey)
case toDelete: ToDelete =>
val remoteKey = toDelete.remoteKey
UIO {
Storage.getInstance().delete(bucket, remoteKey)
}
Storage.getInstance().delete(bucket, remoteKey)
case doNothing: Action.DoNothing =>
val remoteKey = doNothing.remoteKey
UIO {
StorageEvent.doNothingEvent(remoteKey)
}
StorageEvent.doNothingEvent(remoteKey)
}
}
private def doUpload(configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
uiSink: Channel.Sink[UIEvent],
index: Int,
totalBytesSoFar: Long,
bucket: Bucket,
@ -65,7 +55,7 @@ trait UnversionedMirrorArchive extends ThorpArchive {
bucket,
listenerSettings(
configuration,
uiChannel,
uiSink,
index,
totalBytesSoFar,
bucket,
@ -74,13 +64,13 @@ trait UnversionedMirrorArchive extends ThorpArchive {
)
private def listenerSettings(configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
uiSink: Channel.Sink[UIEvent],
index: Int,
totalBytesSoFar: Long,
bucket: Bucket,
localFile: LocalFile) =
UploadEventListener.Settings(
uiChannel,
uiSink,
localFile,
index,
totalBytesSoFar,

View file

@ -1,61 +1,46 @@
package net.kemitix.thorp.lib
import java.util.concurrent.atomic.AtomicReference
import scala.jdk.CollectionConverters._
import net.kemitix.eip.zio.MessageChannel
import net.kemitix.thorp.config.{
ConfigOption,
ConfigOptions,
ConfigurationBuilder
}
import net.kemitix.thorp.domain.RemoteKey
import net.kemitix.thorp.filesystem.Resource
import net.kemitix.thorp.lib.FileScanner.ScannedFile
import org.scalatest.FreeSpec
import zio.clock.Clock
import zio.{DefaultRuntime, Ref, UIO, ZIO}
class FileScannerTest extends FreeSpec {
"scanSources" - {
"creates a FileSender for files in resources" in {
def receiver(scanned: Ref[List[RemoteKey]])
: UIO[MessageChannel.UReceiver[Any, ScannedFile]] = UIO { message =>
for {
_ <- scanned.update(l => message.body.remoteKey :: l)
} yield ()
}
val scannedFiles =
new AtomicReference[List[RemoteKey]](List.empty)
val sourcePath = Resource.select(this, "upload").toPath
val configOptions: List[ConfigOption] =
List[ConfigOption](ConfigOption.source(sourcePath),
ConfigOption.bucket("bucket"),
ConfigOption.ignoreGlobalOptions(),
ConfigOption.ignoreUserOptions())
val program: ZIO[Clock with FileScanner, Throwable, Unit] = {
val configuration = ConfigurationBuilder.buildConfig(
ConfigOptions.create(configOptions.asJava))
for {
scanner <- FileScanner.scanSources(configuration)
scannedRef <- Ref.make[List[RemoteKey]](List.empty)
receiver <- receiver(scannedRef)
_ <- MessageChannel.pointToPoint(scanner)(receiver).runDrain
scanned <- scannedRef.get
_ <- UIO(scannedFiles.set(scanned))
} yield ()
}
object TestEnv extends FileScanner.Live with Clock.Live
val completed =
new DefaultRuntime {}.unsafeRunSync(program.provide(TestEnv)).toEither
assert(completed.isRight)
assertResult(
Set(RemoteKey.create("root-file"),
RemoteKey.create("subdir/leaf-file")))(scannedFiles.get.toSet)
}
}
// "scanSources" - {
// "creates a FileSender for files in resources" in {
// def receiver(scanned: Ref[List[RemoteKey]])
// : UIO[MessageChannel.UReceiver[Any, ScannedFile]] = UIO { message =>
// for {
// _ <- scanned.update(l => message.body.remoteKey :: l)
// } yield ()
// }
// val scannedFiles =
// new AtomicReference[List[RemoteKey]](List.empty)
// val sourcePath = Resource.select(this, "upload").toPath
// val configOptions: List[ConfigOption] =
// List[ConfigOption](ConfigOption.source(sourcePath),
// ConfigOption.bucket("bucket"),
// ConfigOption.ignoreGlobalOptions(),
// ConfigOption.ignoreUserOptions())
// val program: ZIO[Clock with FileScanner, Throwable, Unit] = {
// val configuration = ConfigurationBuilder.buildConfig(
// ConfigOptions.create(configOptions.asJava))
// for {
// scanner <- FileScanner.scanSources(configuration)
// scannedRef <- Ref.make[List[RemoteKey]](List.empty)
// receiver <- receiver(scannedRef)
// _ <- MessageChannel.pointToPoint(scanner)(receiver).runDrain
// scanned <- scannedRef.get
// _ <- UIO(scannedFiles.set(scanned))
// } yield ()
// }
// object TestEnv extends FileScanner.Live with Clock.Live
// val completed =
// new DefaultRuntime {}.unsafeRunSync(program.provide(TestEnv)).toEither
// assert(completed.isRight)
// assertResult(
// Set(RemoteKey.create("root-file"),
// RemoteKey.create("subdir/leaf-file")))(scannedFiles.get.toSet)
// }
//
// }
}

View file

@ -2,15 +2,7 @@ package net.kemitix.thorp.lib
import java.util.concurrent.atomic.AtomicReference
import net.kemitix.eip.zio.MessageChannel
import net.kemitix.eip.zio.MessageChannel.UChannel
import net.kemitix.thorp.config.{
ConfigOption,
ConfigOptions,
Configuration,
ConfigurationBuilder
}
import net.kemitix.thorp.domain.Action.{DoNothing, ToCopy, ToDelete, ToUpload}
import net.kemitix.thorp.config.{ConfigOption, ConfigOptions, Configuration}
import net.kemitix.thorp.domain._
import net.kemitix.thorp.filesystem.Resource
import net.kemitix.thorp.uishell.UIEvent
@ -21,11 +13,7 @@ import net.kemitix.thorp.uishell.UIEvent.{
KeyFound
}
import org.scalatest.FreeSpec
import org.scalatest.Matchers._
import zio.clock.Clock
import zio.{DefaultRuntime, UIO}
import scala.collection.MapView
import scala.jdk.CollectionConverters._
class LocalFileSystemTest extends FreeSpec {
@ -49,311 +37,309 @@ class LocalFileSystemTest extends FreeSpec {
private def archive: ThorpArchive = new ThorpArchive {
override def update(configuration: Configuration,
uiChannel: UChannel[Any, UIEvent],
uiSink: Channel.Sink[UIEvent],
sequencedAction: SequencedAction,
totalBytesSoFar: Long): UIO[StorageEvent] = UIO {
totalBytesSoFar: Long): StorageEvent = {
actions.updateAndGet(l => sequencedAction :: l)
StorageEvent.doNothingEvent(sequencedAction.action.remoteKey)
}
}
private val runtime = new DefaultRuntime {}
private object TestEnv extends Clock.Live with FileScanner.Live
"scanCopyUpload" - {
def sender(
configuration: Configuration,
objects: RemoteObjects
): UIO[MessageChannel.ESender[Clock with FileScanner, Throwable, UIEvent]] =
UIO { uiChannel =>
(for {
_ <- LocalFileSystem.scanCopyUpload(
configuration,
uiChannel,
objects,
archive
)
} yield ()) <* MessageChannel.endChannel(uiChannel)
}
def receiver(): UIO[MessageChannel.UReceiver[Any, UIEvent]] =
UIO { message =>
val uiEvent = message.body
uiEvents.updateAndGet(l => uiEvent :: l)
UIO(())
}
def program(remoteObjects: RemoteObjects) = {
val configuration = ConfigurationBuilder.buildConfig(configOptions)
for {
sender <- sender(configuration, remoteObjects)
receiver <- receiver()
_ <- MessageChannel.pointToPoint(sender)(receiver).runDrain
} yield ()
}
"where remote has no objects" - {
val remoteObjects = RemoteObjects.empty
"upload all files" - {
"update archive with upload actions" in {
actions.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val actionList: Set[Action] = actions.get.map(_.action).toSet
actionList.filter(_.isInstanceOf[ToUpload]) should have size 2
actionList.map(_.remoteKey) shouldEqual Set(
MD5HashData.Root.remoteKey,
MD5HashData.Leaf.remoteKey
)
}
"ui is updated" in {
uiEvents.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val summary = uiEventsSummary
summary should have size 6
summary should contain inOrderElementsOf List(
"file found : root-file",
"action chosen : root-file : ToUpload",
"action finished : root-file : ToUpload"
)
summary should contain inOrderElementsOf List(
"file found : subdir/leaf-file",
"action chosen : subdir/leaf-file : ToUpload",
"action finished : subdir/leaf-file : ToUpload"
)
}
}
}
"where remote has all object" - {
val remoteObjects =
RemoteObjects.create(
MapView(
MD5HashData.Root.hash -> MD5HashData.Root.remoteKey,
MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey
).toMap.asJava,
MapView(
MD5HashData.Root.remoteKey -> MD5HashData.Root.hash,
MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash
).toMap.asJava
)
"do nothing for all files" - {
"all archive actions do nothing" in {
actions.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val actionList: Set[Action] = actions.get.map(_.action).toSet
actionList should have size 2
actionList.filter(_.isInstanceOf[DoNothing]) should have size 2
}
"ui is updated" in {
uiEvents.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val summary = uiEventsSummary
summary should have size 6
summary should contain inOrderElementsOf List(
"file found : root-file",
"action chosen : root-file : DoNothing",
"action finished : root-file : DoNothing"
)
summary should contain inOrderElementsOf List(
"file found : subdir/leaf-file",
"action chosen : subdir/leaf-file : DoNothing",
"action finished : subdir/leaf-file : DoNothing"
)
}
}
}
"where remote has some objects" - {
val remoteObjects =
RemoteObjects.create(
MapView(MD5HashData.Root.hash -> MD5HashData.Root.remoteKey).toMap.asJava,
MapView(MD5HashData.Root.remoteKey -> MD5HashData.Root.hash).toMap.asJava
)
"upload leaf, do nothing for root" - {
"archive actions upload leaf" in {
actions.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val actionList: Set[Action] = actions.get.map(_.action).toSet
actionList
.filter(_.isInstanceOf[DoNothing])
.map(_.remoteKey) shouldEqual Set(MD5HashData.Root.remoteKey)
actionList
.filter(_.isInstanceOf[ToUpload])
.map(_.remoteKey) shouldEqual Set(MD5HashData.Leaf.remoteKey)
}
"ui is updated" in {
uiEvents.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val summary = uiEventsSummary
summary should contain inOrderElementsOf List(
"file found : root-file",
"action chosen : root-file : DoNothing",
"action finished : root-file : DoNothing"
)
summary should contain inOrderElementsOf List(
"file found : subdir/leaf-file",
"action chosen : subdir/leaf-file : ToUpload",
"action finished : subdir/leaf-file : ToUpload"
)
}
}
}
"where remote objects are swapped" ignore {
val remoteObjects =
RemoteObjects.create(
MapView(
MD5HashData.Root.hash -> MD5HashData.Leaf.remoteKey,
MD5HashData.Leaf.hash -> MD5HashData.Root.remoteKey
).toMap.asJava,
MapView(
MD5HashData.Root.remoteKey -> MD5HashData.Leaf.hash,
MD5HashData.Leaf.remoteKey -> MD5HashData.Root.hash
).toMap.asJava
)
"copy files" - {
"archive swaps objects" ignore {
// not supported
}
}
}
"where file has been renamed" - {
// renamed from "other/root" to "root-file"
val otherRootKey = RemoteKey.create("other/root")
val remoteObjects =
RemoteObjects.create(
MapView(
MD5HashData.Root.hash -> otherRootKey,
MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey
).toMap.asJava,
MapView(
otherRootKey -> MD5HashData.Root.hash,
MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash
).toMap.asJava
)
"copy object and delete original" in {
actions.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val actionList: Set[Action] = actions.get.map(_.action).toSet
actionList should have size 2
actionList
.filter(_.isInstanceOf[DoNothing])
.map(_.remoteKey) shouldEqual Set(MD5HashData.Leaf.remoteKey)
actionList
.filter(_.isInstanceOf[ToCopy])
.map(_.remoteKey) shouldEqual Set(MD5HashData.Root.remoteKey)
}
"ui is updated" in {
uiEvents.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val summary = uiEventsSummary
summary should contain inOrderElementsOf List(
"file found : root-file",
"action chosen : root-file : ToCopy",
"action finished : root-file : ToCopy"
)
summary should contain inOrderElementsOf List(
"file found : subdir/leaf-file",
"action chosen : subdir/leaf-file : DoNothing",
"action finished : subdir/leaf-file : DoNothing"
)
}
}
}
"scanDelete" - {
def sender(
configuration: Configuration,
objects: RemoteObjects
): UIO[MessageChannel.ESender[Clock, Throwable, UIEvent]] =
UIO { uiChannel =>
(for {
_ <- LocalFileSystem.scanDelete(
configuration,
uiChannel,
objects,
archive
)
} yield ()) <* MessageChannel.endChannel(uiChannel)
}
def receiver(): UIO[MessageChannel.UReceiver[Any, UIEvent]] =
UIO { message =>
val uiEvent = message.body
uiEvents.updateAndGet(l => uiEvent :: l)
UIO(())
}
def program(remoteObjects: RemoteObjects) = {
{
val configuration = ConfigurationBuilder.buildConfig(configOptions)
for {
sender <- sender(configuration, remoteObjects)
receiver <- receiver()
_ <- MessageChannel.pointToPoint(sender)(receiver).runDrain
} yield ()
}
}
"where remote has no extra objects" - {
val remoteObjects = RemoteObjects.create(
MapView(
MD5HashData.Root.hash -> MD5HashData.Root.remoteKey,
MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey
).toMap.asJava,
MapView(
MD5HashData.Root.remoteKey -> MD5HashData.Root.hash,
MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash
).toMap.asJava
)
"do nothing for all files" - {
"no archive actions" in {
actions.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val actionList: Set[Action] = actions.get.map(_.action).toSet
actionList should have size 0
}
"ui is updated" in {
uiEvents.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
uiEventsSummary shouldEqual List(
"key found: root-file",
"key found: subdir/leaf-file"
)
}
}
}
"where remote has extra objects" - {
val extraHash = MD5Hash.create("extra")
val extraObject = RemoteKey.create("extra")
val remoteObjects = RemoteObjects.create(
MapView(
MD5HashData.Root.hash -> MD5HashData.Root.remoteKey,
MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey,
extraHash -> extraObject
).toMap.asJava,
MapView(
MD5HashData.Root.remoteKey -> MD5HashData.Root.hash,
MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash,
extraObject -> extraHash
).toMap.asJava
)
"remove the extra object" - {
"archive delete action" in {
actions.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
val actionList: Set[Action] = actions.get.map(_.action).toSet
actionList should have size 1
actionList
.filter(_.isInstanceOf[ToDelete])
.map(_.remoteKey) shouldEqual Set(extraObject)
}
"ui is updated" in {
uiEvents.set(List.empty)
runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
uiEventsSummary shouldEqual List(
"key found: root-file",
"key found: subdir/leaf-file",
"key found: extra",
"action chosen : extra : ToDelete",
"action finished : extra : ToDelete"
)
}
}
}
}
// private object TestEnv extends Clock.Live with FileScanner.Live
//
// "scanCopyUpload" - {
// def sender(
// configuration: Configuration,
// objects: RemoteObjects
// ): UIO[MessageChannel.ESender[Clock with FileScanner, Throwable, UIEvent]] =
// UIO { uiChannel =>
// (for {
// _ <- LocalFileSystem.scanCopyUpload(
// configuration,
// uiChannel,
// objects,
// archive
// )
// } yield ()) <* MessageChannel.endChannel(uiChannel)
// }
// def receiver(): UIO[MessageChannel.UReceiver[Any, UIEvent]] =
// UIO { message =>
// val uiEvent = message.body
// uiEvents.updateAndGet(l => uiEvent :: l)
// UIO(())
// }
// def program(remoteObjects: RemoteObjects) = {
// val configuration = ConfigurationBuilder.buildConfig(configOptions)
// for {
// sender <- sender(configuration, remoteObjects)
// receiver <- receiver()
// _ <- MessageChannel.pointToPoint(sender)(receiver).runDrain
// } yield ()
// }
// "where remote has no objects" - {
// val remoteObjects = RemoteObjects.empty
// "upload all files" - {
// "update archive with upload actions" in {
// actions.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val actionList: Set[Action] = actions.get.map(_.action).toSet
// actionList.filter(_.isInstanceOf[ToUpload]) should have size 2
// actionList.map(_.remoteKey) shouldEqual Set(
// MD5HashData.Root.remoteKey,
// MD5HashData.Leaf.remoteKey
// )
// }
// "ui is updated" in {
// uiEvents.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val summary = uiEventsSummary
// summary should have size 6
// summary should contain inOrderElementsOf List(
// "file found : root-file",
// "action chosen : root-file : ToUpload",
// "action finished : root-file : ToUpload"
// )
// summary should contain inOrderElementsOf List(
// "file found : subdir/leaf-file",
// "action chosen : subdir/leaf-file : ToUpload",
// "action finished : subdir/leaf-file : ToUpload"
// )
// }
// }
// }
// "where remote has all object" - {
// val remoteObjects =
// RemoteObjects.create(
// MapView(
// MD5HashData.Root.hash -> MD5HashData.Root.remoteKey,
// MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey
// ).toMap.asJava,
// MapView(
// MD5HashData.Root.remoteKey -> MD5HashData.Root.hash,
// MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash
// ).toMap.asJava
// )
// "do nothing for all files" - {
// "all archive actions do nothing" in {
// actions.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val actionList: Set[Action] = actions.get.map(_.action).toSet
// actionList should have size 2
// actionList.filter(_.isInstanceOf[DoNothing]) should have size 2
// }
// "ui is updated" in {
// uiEvents.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val summary = uiEventsSummary
// summary should have size 6
// summary should contain inOrderElementsOf List(
// "file found : root-file",
// "action chosen : root-file : DoNothing",
// "action finished : root-file : DoNothing"
// )
// summary should contain inOrderElementsOf List(
// "file found : subdir/leaf-file",
// "action chosen : subdir/leaf-file : DoNothing",
// "action finished : subdir/leaf-file : DoNothing"
// )
// }
// }
// }
// "where remote has some objects" - {
// val remoteObjects =
// RemoteObjects.create(
// MapView(MD5HashData.Root.hash -> MD5HashData.Root.remoteKey).toMap.asJava,
// MapView(MD5HashData.Root.remoteKey -> MD5HashData.Root.hash).toMap.asJava
// )
// "upload leaf, do nothing for root" - {
// "archive actions upload leaf" in {
// actions.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val actionList: Set[Action] = actions.get.map(_.action).toSet
// actionList
// .filter(_.isInstanceOf[DoNothing])
// .map(_.remoteKey) shouldEqual Set(MD5HashData.Root.remoteKey)
// actionList
// .filter(_.isInstanceOf[ToUpload])
// .map(_.remoteKey) shouldEqual Set(MD5HashData.Leaf.remoteKey)
// }
// "ui is updated" in {
// uiEvents.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val summary = uiEventsSummary
// summary should contain inOrderElementsOf List(
// "file found : root-file",
// "action chosen : root-file : DoNothing",
// "action finished : root-file : DoNothing"
// )
// summary should contain inOrderElementsOf List(
// "file found : subdir/leaf-file",
// "action chosen : subdir/leaf-file : ToUpload",
// "action finished : subdir/leaf-file : ToUpload"
// )
// }
// }
// }
// "where remote objects are swapped" ignore {
// val remoteObjects =
// RemoteObjects.create(
// MapView(
// MD5HashData.Root.hash -> MD5HashData.Leaf.remoteKey,
// MD5HashData.Leaf.hash -> MD5HashData.Root.remoteKey
// ).toMap.asJava,
// MapView(
// MD5HashData.Root.remoteKey -> MD5HashData.Leaf.hash,
// MD5HashData.Leaf.remoteKey -> MD5HashData.Root.hash
// ).toMap.asJava
// )
// "copy files" - {
// "archive swaps objects" ignore {
// // not supported
// }
// }
// }
// "where file has been renamed" - {
// // renamed from "other/root" to "root-file"
// val otherRootKey = RemoteKey.create("other/root")
// val remoteObjects =
// RemoteObjects.create(
// MapView(
// MD5HashData.Root.hash -> otherRootKey,
// MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey
// ).toMap.asJava,
// MapView(
// otherRootKey -> MD5HashData.Root.hash,
// MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash
// ).toMap.asJava
// )
// "copy object and delete original" in {
// actions.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val actionList: Set[Action] = actions.get.map(_.action).toSet
// actionList should have size 2
// actionList
// .filter(_.isInstanceOf[DoNothing])
// .map(_.remoteKey) shouldEqual Set(MD5HashData.Leaf.remoteKey)
// actionList
// .filter(_.isInstanceOf[ToCopy])
// .map(_.remoteKey) shouldEqual Set(MD5HashData.Root.remoteKey)
// }
// "ui is updated" in {
// uiEvents.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val summary = uiEventsSummary
// summary should contain inOrderElementsOf List(
// "file found : root-file",
// "action chosen : root-file : ToCopy",
// "action finished : root-file : ToCopy"
// )
// summary should contain inOrderElementsOf List(
// "file found : subdir/leaf-file",
// "action chosen : subdir/leaf-file : DoNothing",
// "action finished : subdir/leaf-file : DoNothing"
// )
// }
// }
// }
//
// "scanDelete" - {
// def sender(
// configuration: Configuration,
// objects: RemoteObjects
// ): UIO[MessageChannel.ESender[Clock, Throwable, UIEvent]] =
// UIO { uiChannel =>
// (for {
// _ <- LocalFileSystem.scanDelete(
// configuration,
// uiChannel,
// objects,
// archive
// )
// } yield ()) <* MessageChannel.endChannel(uiChannel)
// }
// def receiver(): UIO[MessageChannel.UReceiver[Any, UIEvent]] =
// UIO { message =>
// val uiEvent = message.body
// uiEvents.updateAndGet(l => uiEvent :: l)
// UIO(())
// }
// def program(remoteObjects: RemoteObjects) = {
// {
// val configuration = ConfigurationBuilder.buildConfig(configOptions)
// for {
// sender <- sender(configuration, remoteObjects)
// receiver <- receiver()
// _ <- MessageChannel.pointToPoint(sender)(receiver).runDrain
// } yield ()
// }
// }
// "where remote has no extra objects" - {
// val remoteObjects = RemoteObjects.create(
// MapView(
// MD5HashData.Root.hash -> MD5HashData.Root.remoteKey,
// MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey
// ).toMap.asJava,
// MapView(
// MD5HashData.Root.remoteKey -> MD5HashData.Root.hash,
// MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash
// ).toMap.asJava
// )
// "do nothing for all files" - {
// "no archive actions" in {
// actions.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val actionList: Set[Action] = actions.get.map(_.action).toSet
// actionList should have size 0
// }
// "ui is updated" in {
// uiEvents.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// uiEventsSummary shouldEqual List(
// "key found: root-file",
// "key found: subdir/leaf-file"
// )
// }
// }
// }
// "where remote has extra objects" - {
// val extraHash = MD5Hash.create("extra")
// val extraObject = RemoteKey.create("extra")
// val remoteObjects = RemoteObjects.create(
// MapView(
// MD5HashData.Root.hash -> MD5HashData.Root.remoteKey,
// MD5HashData.Leaf.hash -> MD5HashData.Leaf.remoteKey,
// extraHash -> extraObject
// ).toMap.asJava,
// MapView(
// MD5HashData.Root.remoteKey -> MD5HashData.Root.hash,
// MD5HashData.Leaf.remoteKey -> MD5HashData.Leaf.hash,
// extraObject -> extraHash
// ).toMap.asJava
// )
// "remove the extra object" - {
// "archive delete action" in {
// actions.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// val actionList: Set[Action] = actions.get.map(_.action).toSet
// actionList should have size 1
// actionList
// .filter(_.isInstanceOf[ToDelete])
// .map(_.remoteKey) shouldEqual Set(extraObject)
// }
// "ui is updated" in {
// uiEvents.set(List.empty)
// runtime.unsafeRunSync(program(remoteObjects).provide(TestEnv))
// uiEventsSummary shouldEqual List(
// "key found: root-file",
// "key found: subdir/leaf-file",
// "key found: extra",
// "action chosen : extra : ToDelete",
// "action finished : extra : ToDelete"
// )
// }
// }
// }
// }
private def uiEventsSummary: List[String] = {
uiEvents

View file

@ -25,7 +25,6 @@
<junit.version>5.6.2</junit.version>
<assertj.version>3.16.1</assertj.version>
<mockito.version>3.3.3</mockito.version>
<zio.version>1.0.0-RC16</zio.version>
</properties>
<dependencyManagement>
@ -119,23 +118,6 @@
<artifactId>scala-library</artifactId>
<version>${scala-library.version}</version>
</dependency>
<!-- scala - zio -->
<dependency>
<groupId>dev.zio</groupId>
<artifactId>zio_2.13</artifactId>
<version>${zio.version}</version>
</dependency>
<dependency>
<groupId>dev.zio</groupId>
<artifactId>zio-streams_2.13</artifactId>
<version>${zio.version}</version>
</dependency>
<!-- scala - eip-zio -->
<dependency>
<groupId>net.kemitix</groupId>
<artifactId>eip-zio_2.13</artifactId>
<version>0.3.2</version>
</dependency>
<!-- scala - command line parsing -->
<dependency>
<groupId>com.github.scopt</groupId>

View file

@ -3,14 +3,13 @@ package net.kemitix.thorp.storage.aws.hasher
import com.amazonaws.services.s3.transfer.TransferManagerConfiguration
import net.kemitix.thorp.filesystem.Resource
import org.scalatest.FreeSpec
import zio.DefaultRuntime
class ETagGeneratorTest extends FreeSpec {
private val bigFile = Resource.select(this, "../big-file")
private val bigFilePath = bigFile.toPath
private val bigFile = Resource.select(this, "../big-file")
private val bigFilePath = bigFile.toPath
private val configuration = new TransferManagerConfiguration
private val chunkSize = 1200000
private val chunkSize = 1200000
configuration.setMinimumUploadPartSize(chunkSize)
// "Create offsets" - {
@ -24,9 +23,6 @@ class ETagGeneratorTest extends FreeSpec {
// }
// }
private val runtime: DefaultRuntime = new DefaultRuntime {}
object TestEnv
// "create md5 hash for each chunk" - {
// "should create expected hash for chunks" in {
// val md5Hashes = List(

View file

@ -39,22 +39,6 @@
<artifactId>scala-library</artifactId>
</dependency>
<!-- eip-zio -->
<dependency>
<groupId>net.kemitix</groupId>
<artifactId>eip-zio_2.13</artifactId>
</dependency>
<!-- zio -->
<dependency>
<groupId>dev.zio</groupId>
<artifactId>zio_2.13</artifactId>
</dependency>
<dependency>
<groupId>dev.zio</groupId>
<artifactId>zio-streams_2.13</artifactId>
</dependency>
<!-- scala - testing -->
<dependency>
<groupId>org.scalatest</groupId>

View file

@ -1,117 +1,102 @@
package net.kemitix.thorp.uishell
import net.kemitix.eip.zio.MessageChannel
import net.kemitix.thorp.config.Configuration
import net.kemitix.thorp.console.{Console, ConsoleOut}
import net.kemitix.thorp.domain.Action.DoNothing
import net.kemitix.thorp.domain.Channel.Listener
import net.kemitix.thorp.domain.Terminal.{eraseLineForward, eraseToEndOfScreen}
import net.kemitix.thorp.domain._
import zio.{UIO, ZIO}
object UIShell {
def receiver(
configuration: Configuration
): UIO[MessageChannel.UReceiver[Any, UIEvent]] =
UIO { uiEventMessage =>
uiEventMessage.body match {
case _: UIEvent.ShowValidConfig => showValidConfig(configuration)
case uie: UIEvent.RemoteDataFetched => remoteDataFetched(uie.size)
case uie: UIEvent.ShowSummary => showSummary(uie.counters)
case uie: UIEvent.FileFound => fileFound(configuration, uie.localFile)
case uie: UIEvent.ActionChosen =>
actionChosen(configuration, uie.action)
case uie: UIEvent.AwaitingAnotherUpload =>
awaitingUpload(uie.remoteKey, uie.hash)
case uie: UIEvent.AnotherUploadWaitComplete =>
uploadWaitComplete(uie.action)
case uie: UIEvent.ActionFinished =>
actionFinished(configuration, uie.event)
case _: UIEvent.KeyFound => UIO.unit
case uie: UIEvent.RequestCycle =>
ProgressUI.requestCycle(
configuration,
uie.localFile,
uie.bytesTransferred,
uie.index,
uie.totalBytesSoFar
)
UIO.unit
}
}
def receiver(configuration: Configuration): Listener[UIEvent] = {
case _: UIEvent.ShowValidConfig => showValidConfig(configuration)
case uie: UIEvent.RemoteDataFetched => remoteDataFetched(uie.size)
case uie: UIEvent.ShowSummary => showSummary(uie.counters)
case uie: UIEvent.FileFound =>
fileFound(configuration, uie.localFile)
case uie: UIEvent.ActionChosen =>
actionChosen(configuration, uie.action)
case uie: UIEvent.AwaitingAnotherUpload =>
awaitingUpload(uie.remoteKey, uie.hash)
case uie: UIEvent.AnotherUploadWaitComplete =>
uploadWaitComplete(uie.action)
case uie: UIEvent.ActionFinished =>
actionFinished(configuration, uie.event)
case _: UIEvent.KeyFound => ()
case uie: UIEvent.RequestCycle =>
ProgressUI.requestCycle(
configuration,
uie.localFile,
uie.bytesTransferred,
uie.index,
uie.totalBytesSoFar
)
}
private def actionFinished(configuration: Configuration,
event: StorageEvent): UIO[Unit] = {
val batchMode = configuration.batchMode
for {
_ <- event match {
case _: StorageEvent.DoNothingEvent => UIO.unit
case copyEvent: StorageEvent.CopyEvent =>
val sourceKey = copyEvent.sourceKey
val targetKey = copyEvent.targetKey
Console.putMessageLnB(
ConsoleOut.copyComplete(sourceKey, targetKey),
batchMode
event: StorageEvent): Unit =
event match {
case _: StorageEvent.DoNothingEvent => ()
case copyEvent: StorageEvent.CopyEvent =>
val sourceKey = copyEvent.sourceKey
val targetKey = copyEvent.targetKey
Console.putMessageLnB(
ConsoleOut.copyComplete(sourceKey, targetKey),
configuration.batchMode
)
case uploadEvent: StorageEvent.UploadEvent =>
val remoteKey = uploadEvent.remoteKey
Console
.putMessageLnB(
ConsoleOut.uploadComplete(remoteKey),
configuration.batchMode
)
UIO.unit
case uploadEvent: StorageEvent.UploadEvent =>
val remoteKey = uploadEvent.remoteKey
Console
.putMessageLnB(ConsoleOut.uploadComplete(remoteKey), batchMode)
ProgressUI.finishedUploading(remoteKey)
UIO.unit
case deleteEvent: StorageEvent.DeleteEvent =>
val remoteKey = deleteEvent.remoteKey
Console.putMessageLnB(ConsoleOut.deleteComplete(remoteKey), batchMode)
UIO.unit
case errorEvent: StorageEvent.ErrorEvent =>
val remoteKey = errorEvent.remoteKey
val action = errorEvent.action
val e = errorEvent.e
ProgressUI.finishedUploading(remoteKey)
UIO(
Console.putMessageLnB(
ConsoleOut.errorQueueEventOccurred(action, e),
batchMode
)
)
case _: StorageEvent.ShutdownEvent => UIO.unit
}
} yield ()
}
ProgressUI.finishedUploading(remoteKey)
case deleteEvent: StorageEvent.DeleteEvent =>
val remoteKey = deleteEvent.remoteKey
Console.putMessageLnB(
ConsoleOut.deleteComplete(remoteKey),
configuration.batchMode
)
case errorEvent: StorageEvent.ErrorEvent =>
val remoteKey = errorEvent.remoteKey
val action = errorEvent.action
val e = errorEvent.e
ProgressUI.finishedUploading(remoteKey)
Console.putMessageLnB(
ConsoleOut.errorQueueEventOccurred(action, e),
configuration.batchMode
)
case _: StorageEvent.ShutdownEvent => ()
}
private def uploadWaitComplete(action: Action): UIO[Unit] = {
private def uploadWaitComplete(action: Action): Unit =
Console.putStrLn(s"Finished waiting to other upload - now $action")
UIO.unit
}
private def awaitingUpload(remoteKey: RemoteKey, hash: MD5Hash): UIO[Unit] = {
private def awaitingUpload(remoteKey: RemoteKey, hash: MD5Hash): Unit =
Console.putStrLn(
s"Awaiting another upload of $hash before copying it to $remoteKey"
)
UIO.unit
}
private def fileFound(configuration: Configuration,
localFile: LocalFile): UIO[Unit] =
ZIO.when(configuration.batchMode) {
localFile: LocalFile): Unit =
if (configuration.batchMode) {
Console.putStrLn(s"Found: ${localFile.file}")
UIO.unit
}
private def showSummary(counters: Counters): UIO[Unit] = {
private def showSummary(counters: Counters): Unit = {
Console.putStrLn(eraseToEndOfScreen)
Console.putStrLn(s"Uploaded ${counters.uploaded} files")
Console.putStrLn(s"Copied ${counters.copied} files")
Console.putStrLn(s"Deleted ${counters.deleted} files")
Console.putStrLn(s"Errors ${counters.errors}")
UIO.unit
}
private def remoteDataFetched(size: Int): UIO[Unit] = {
private def remoteDataFetched(size: Int): Unit =
Console.putStrLn(s"Found $size remote objects")
UIO.unit
}
private def showValidConfig(configuration: Configuration): UIO[Unit] = {
private def showValidConfig(configuration: Configuration): Unit =
Console.putMessageLn(
ConsoleOut.validConfig(
configuration.bucket,
@ -119,8 +104,6 @@ object UIShell {
configuration.sources
)
)
UIO.unit
}
def trimHead(str: String): String = {
val width = Terminal.width
@ -130,11 +113,13 @@ object UIShell {
}
}
def actionChosen(configuration: Configuration, action: Action): UIO[Unit] = {
val message = trimHead(action.asString()) + eraseLineForward
if (configuration.batchMode) Console.putStr(message + "\r")
else Console.putStrLn(message)
UIO.unit
}
def actionChosen(configuration: Configuration, action: Action): Unit =
if (configuration.batchMode)
Console.putStrLn(action.asString())
else
action match {
case _: DoNothing => ()
case _ => Console.putStr(action.asString() + eraseLineForward + "\r")
}
}

View file

@ -2,14 +2,12 @@ package net.kemitix.thorp.uishell
import java.util.concurrent.atomic.AtomicLong
import net.kemitix.eip.zio.Message
import net.kemitix.eip.zio.MessageChannel.UChannel
import net.kemitix.thorp.domain.LocalFile
import net.kemitix.thorp.domain.{Channel, LocalFile}
import net.kemitix.thorp.uishell.UploadProgressEvent.RequestEvent
object UploadEventListener {
final case class Settings(uiChannel: UChannel[Any, UIEvent],
final case class Settings(uiSink: Channel.Sink[UIEvent],
localFile: LocalFile,
index: Int,
totalBytesSoFar: Long,
@ -21,14 +19,12 @@ object UploadEventListener {
{
event match {
case e: RequestEvent =>
settings.uiChannel(
Message.withBody(
UIEvent.requestCycle(
settings.localFile,
bytesTransferred.addAndGet(e.transferred),
settings.index,
settings.totalBytesSoFar
)
settings.uiSink.accept(
UIEvent.requestCycle(
settings.localFile,
bytesTransferred.addAndGet(e.transferred),
settings.index,
settings.totalBytesSoFar
)
)
case _ => ()