From c3d0dce281ddbc14e32212aedc6f38aed8f53d2b Mon Sep 17 00:00:00 2001 From: Zlatin Balevsky Date: Tue, 10 Mar 2020 16:05:32 +0000 Subject: [PATCH] store last update attempt and do not retry active feeds --- .../muwire/core/filefeeds/FeedClient.groovy | 3 ++- .../muwire/core/filefeeds/FeedManager.groovy | 20 +++++++++++++------ .../java/com/muwire/core/filefeeds/Feed.java | 9 +++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/core/src/main/groovy/com/muwire/core/filefeeds/FeedClient.groovy b/core/src/main/groovy/com/muwire/core/filefeeds/FeedClient.groovy index 069fac97..a1978a57 100644 --- a/core/src/main/groovy/com/muwire/core/filefeeds/FeedClient.groovy +++ b/core/src/main/groovy/com/muwire/core/filefeeds/FeedClient.groovy @@ -1,6 +1,6 @@ package com.muwire.core.filefeeds -import java.lang.System.Logger.Level +import java.util.logging.Level import java.nio.charset.StandardCharsets import java.util.concurrent.Executor import java.util.concurrent.ExecutorService @@ -64,6 +64,7 @@ class FeedClient { Endpoint endpoint = null try { eventBus.publish(new FeedFetchEvent(host : feed.getPublisher(), status : FeedFetchStatus.CONNECTING)) + feed.setLastUpdateAttempt(System.currentTimeMillis()) endpoint = connector.connect(feed.getPublisher().getDestination()) OutputStream os = endpoint.getOutputStream() os.write("FEED\r\n".getBytes(StandardCharsets.US_ASCII)) diff --git a/core/src/main/groovy/com/muwire/core/filefeeds/FeedManager.groovy b/core/src/main/groovy/com/muwire/core/filefeeds/FeedManager.groovy index ebc47272..de465650 100644 --- a/core/src/main/groovy/com/muwire/core/filefeeds/FeedManager.groovy +++ b/core/src/main/groovy/com/muwire/core/filefeeds/FeedManager.groovy @@ -54,7 +54,8 @@ class FeedManager { public List getFeedsToUpdate() { long now = System.currentTimeMillis() feeds.values().stream(). - filter({Feed f -> f.getLastUpdated() + f.getUpdateInterval() <= now}) + filter({Feed f -> !f.getStatus().isActive()}). + filter({Feed f -> f.getLastUpdateAttempt() + f.getUpdateInterval() <= now}) .collect(Collectors.toList()) } @@ -78,8 +79,10 @@ class FeedManager { Feed feed = new Feed(publisher) feed.setUpdateInterval(parsed.updateInterval) feed.setLastUpdated(parsed.lastUpdated) + feed.setLastUpdateAttempt(parsed.lastUpdateAttempt) feed.setItemsToKeep(parsed.itemsToKeep) feed.setAutoDownload(parsed.autoDownload) + feed.setSequential(parsed.sequential) feed.setStatus(FeedFetchStatus.IDLE) @@ -122,18 +125,21 @@ class FeedManager { Feed feed = feeds.get(e.host) if (feed == null) { - log.severe("Finished fetching non-existent feed " + e.host.getHumanReadableName()) + log.severe("Fetching non-existent feed " + e.host.getHumanReadableName()) return } feed.setStatus(e.status) - if (e.status != FeedFetchStatus.FINISHED) + if (e.status.isActive()) return - feed.setStatus(FeedFetchStatus.IDLE) - feed.setLastUpdated(e.getTimestamp()) - // save feed items, then save feed + if (e.status == FeedFetchStatus.FINISHED) { + feed.setStatus(FeedFetchStatus.IDLE) + feed.setLastUpdated(e.getTimestamp()) + } + // save feed items, then save feed. This will save partial fetches too + // which is ok because the items are stored in a Set persister.submit({saveFeedItems(e.host)} as Runnable) persister.submit({saveFeedMetadata(feed)} as Runnable) } @@ -192,6 +198,8 @@ class FeedManager { json.lastUpdated = feed.getLastUpdated() json.updateInterval = feed.getUpdateInterval() json.autoDownload = feed.isAutoDownload() + json.sequential = feed.isSequential() + json.lastUpdateAttempt = feed.getLastUpdateAttempt() json = JsonOutput.toJson(json) writer.println(json) } diff --git a/core/src/main/java/com/muwire/core/filefeeds/Feed.java b/core/src/main/java/com/muwire/core/filefeeds/Feed.java index 36588d21..e42894f6 100644 --- a/core/src/main/java/com/muwire/core/filefeeds/Feed.java +++ b/core/src/main/java/com/muwire/core/filefeeds/Feed.java @@ -8,6 +8,7 @@ public class Feed { private int updateInterval; private long lastUpdated; + private volatile long lastUpdateAttempt; private int itemsToKeep; private boolean autoDownload; private boolean sequential; @@ -69,4 +70,12 @@ public class Feed { public boolean isSequential() { return sequential; } + + public void setLastUpdateAttempt(long lastUpdateAttempt) { + this.lastUpdateAttempt = lastUpdateAttempt; + } + + public long getLastUpdateAttempt() { + return lastUpdateAttempt; + } }