From 376ecb4a41b9facf58d3df03927fb042dfd9b2be Mon Sep 17 00:00:00 2001 From: Zlatin Balevsky Date: Sat, 10 Sep 2022 21:21:48 +0300 Subject: [PATCH] rehash stale files on loading of library --- .../muwire/core/files/FileHashedEvent.groovy | 3 +- .../com/muwire/core/files/FileManager.groovy | 5 ++-- .../muwire/core/files/HasherService.groovy | 12 ++++---- .../core/files/PersisterFolderService.groovy | 29 +++++++++++++++++-- .../com/muwire/gui/MainFrameModel.groovy | 4 +-- .../views/com/muwire/gui/MainFrameView.groovy | 11 ++++++- 6 files changed, 48 insertions(+), 16 deletions(-) diff --git a/core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy b/core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy index 6443f9b4..851ee271 100644 --- a/core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy +++ b/core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy @@ -11,9 +11,8 @@ class FileHashedEvent extends Event { /** * This will be non-null in case of a re-hash. - * Enriched by FileManager */ - SharedFile duplicate + SharedFile original @Override public String toString() { diff --git a/core/src/main/groovy/com/muwire/core/files/FileManager.groovy b/core/src/main/groovy/com/muwire/core/files/FileManager.groovy index dbc634dc..31572f20 100644 --- a/core/src/main/groovy/com/muwire/core/files/FileManager.groovy +++ b/core/src/main/groovy/com/muwire/core/files/FileManager.groovy @@ -83,9 +83,8 @@ class FileManager { if (sideCar.exists()) e.sharedFile.setComment(Base64.encode(DataUtil.encodei18nString(sideCar.text))) } - - e.duplicate = fileToSharedFile[f] - + if (e.original != null) + e.sharedFile.setComment(e.original.getComment()) addToIndex(e.sharedFile) } diff --git a/core/src/main/groovy/com/muwire/core/files/HasherService.groovy b/core/src/main/groovy/com/muwire/core/files/HasherService.groovy index d41e0f96..5634c84f 100644 --- a/core/src/main/groovy/com/muwire/core/files/HasherService.groovy +++ b/core/src/main/groovy/com/muwire/core/files/HasherService.groovy @@ -82,7 +82,7 @@ class HasherService { if (canonical.isDirectory()) executor.execute({processDirectory(evt.file)} as Runnable) else - throttlerExecutor.execute({ throttle(evt.file, canonical, evt.fromWatch) } as Runnable) + throttlerExecutor.execute({ throttle(evt.file, canonical, evt.fromWatch, null) } as Runnable) } } @@ -100,17 +100,17 @@ class HasherService { for (SharedFile sharedFile : event.sharedFiles) { File f = sharedFile.getFile() File canonical = f.getCanonicalFile() - throttlerExecutor.execute({ throttle(f, canonical, true) }) + throttlerExecutor.execute({ throttle(f, canonical, true, sharedFile) }) } } - private synchronized void throttle(File f, File canonical, boolean forceHash) { + private synchronized void throttle(File f, File canonical, boolean forceHash, SharedFile original) { while(currentHashes >= settings.hashingCores) wait(10) currentHashes++ if (++totalHashes % TARGET_Q_SIZE == 0) System.gc() - executor.execute({processFile(f, canonical, forceHash)} as Runnable) + executor.execute({processFile(f, canonical, forceHash, original)} as Runnable) } private void processDirectory(File f) { @@ -125,7 +125,7 @@ class HasherService { } } - private void processFile(File f, File canonical, boolean forceHash) { + private void processFile(File f, File canonical, boolean forceHash, SharedFile original) { try { if (f.length() == 0) { eventBus.publish new FileHashedEvent(error: "Not sharing empty file $f") @@ -144,7 +144,7 @@ class HasherService { log.fine("found an existing hash list for $f => $canonical") def sf = new SharedFile(f, hash.getRoot(), FileHasher.getPieceSize(f.length()), System.currentTimeMillis()) - eventBus.publish new FileHashedEvent(sharedFile: sf) + eventBus.publish new FileHashedEvent(sharedFile: sf, original: original) } } finally { synchronized (this) { diff --git a/core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy b/core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy index 00d14aac..2abe73e5 100644 --- a/core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy +++ b/core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy @@ -219,6 +219,7 @@ class PersisterFolderService extends BasePersisterService { }) if (core.muOptions.plugin || !core.muOptions.throttleLoadingFiles) stream = stream.parallel() + Set staleFiles = new HashSet<>() stream.forEach({ log.fine("processing path $it") def slurper = new JsonSlurper(type: JsonParserType.LAX) @@ -245,8 +246,11 @@ class PersisterFolderService extends BasePersisterService { } } else log.fine("loaded shared parent from json ${event.loadedFile.getPathToSharedParent()}") - - listener.publish event + + if (event.loadedFile.isStale()) + staleFiles.add(event.loadedFile) + else + listener.publish event if (!core.muOptions.plugin && core.muOptions.throttleLoadingFiles) { loaded++ @@ -258,9 +262,30 @@ class PersisterFolderService extends BasePersisterService { failed.incrementAndGet() } }) + waitForRehash(staleFiles) listener.publish(new AllFilesLoadedEvent(failed : failed.get())) } + private void waitForRehash(Set staleFiles) { + if (staleFiles.isEmpty()) + return + def waiter = new Object() { + synchronized void onFileHashedEvent(FileHashedEvent event) { + staleFiles.remove(event.original) + notify() + } + synchronized void await() { + while(!staleFiles.isEmpty()) + wait() + } + } + + listener.register(FileHashedEvent.class, waiter) + listener.publish(new FileModifiedEvent(sharedFiles: staleFiles.toArray(new SharedFile[0]))) + waiter.await() + listener.unregister(FileHashedEvent.class, waiter) + } + /** * Finds the path to the file to send over the network. The parts of the * path that are before the topmost shared parent are obfuscated, the diff --git a/gui/griffon-app/models/com/muwire/gui/MainFrameModel.groovy b/gui/griffon-app/models/com/muwire/gui/MainFrameModel.groovy index 5a09d151..badbeebe 100644 --- a/gui/griffon-app/models/com/muwire/gui/MainFrameModel.groovy +++ b/gui/griffon-app/models/com/muwire/gui/MainFrameModel.groovy @@ -498,8 +498,8 @@ class MainFrameModel { hashingFiles-- if (e.sharedFile.file == hashingFile) hashingFile = null - if (e.duplicate != null) - allSharedFiles.remove(e.duplicate) + if (e.original != null) + allSharedFiles.remove(e.original) allSharedFiles << e.sharedFile insertIntoTree(e.sharedFile, allFilesSharedTree, fileToNode) if (filter(e.sharedFile)) { diff --git a/gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy b/gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy index 67a93a88..1a9c72d3 100644 --- a/gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy +++ b/gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy @@ -361,7 +361,16 @@ class MainFrameView { panel (id: "library-title", constraints : BorderLayout.NORTH) { cardLayout() panel(constraints: "library-is-loading") { - label(text: trans("LIBRARY_IS_LOADING")) + label(text: bind { + if (model.hashingFile == null && model.hashingFiles == 0) { + trans("LIBRARY_IS_LOADING") + } else if (model.hashingFiles == 1 && model.hashingFile != null) { + trans("HASHING") + ": " + + model.hashingFile.getAbsolutePath() + " (" + formatSize(model.hashingFile.length(), "BYTES_SHORT") + ")" + } else { + trans("HASHING") + " " + model.hashingFiles + " " + trans("FILES") + } + }) } panel(constraints: "you-can-drag-and-drop") { label(text: bind {