From 84c7da1fe09c246c78499f27ba1c714ccf89a93b Mon Sep 17 00:00:00 2001 From: Zlatin Balevsky Date: Sun, 26 Apr 2020 20:15:48 +0100 Subject: [PATCH] * More logging * Include leaseset in crawler pings * serialize hourly files in a directory, keep history --- .../main/groovy/com/muwire/hostcache/Crawler.groovy | 6 ++++-- .../src/main/groovy/com/muwire/hostcache/Host.groovy | 5 +++++ .../groovy/com/muwire/hostcache/HostCache.groovy | 6 ++++-- .../main/groovy/com/muwire/hostcache/HostPool.groovy | 11 ++++++++--- .../main/groovy/com/muwire/hostcache/Pinger.groovy | 12 +++++++++--- 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/host-cache/src/main/groovy/com/muwire/hostcache/Crawler.groovy b/host-cache/src/main/groovy/com/muwire/hostcache/Crawler.groovy index 7749be26..1cb248f8 100644 --- a/host-cache/src/main/groovy/com/muwire/hostcache/Crawler.groovy +++ b/host-cache/src/main/groovy/com/muwire/hostcache/Crawler.groovy @@ -40,12 +40,13 @@ class Crawler { try { uuid = UUID.fromString(pong.uuid) } catch (IllegalArgumentException bad) { + log.log(Level.WARNING,"couldn't parse uuid",bad) hostPool.fail(host) return } if (!uuid.equals(currentUUID)) { - log.info("uuid mismatch") + log.warn("uuid mismatch $uuid expected $currentUUID") hostPool.fail(host) return } @@ -75,11 +76,12 @@ class Crawler { } synchronized def startCrawl() { + currentUUID = UUID.randomUUID() + log.info("starting new crawl with uuid $currentUUID inFlight ${inFlight.size()}") if (!inFlight.isEmpty()) { inFlight.values().each { hostPool.fail(it) } inFlight.clear() } - currentUUID = UUID.randomUUID() hostPool.getUnverified(parallel).each { inFlight.put(it.destination, it) pinger.ping(it, currentUUID) diff --git a/host-cache/src/main/groovy/com/muwire/hostcache/Host.groovy b/host-cache/src/main/groovy/com/muwire/hostcache/Host.groovy index b04b3b4a..677bfacd 100644 --- a/host-cache/src/main/groovy/com/muwire/hostcache/Host.groovy +++ b/host-cache/src/main/groovy/com/muwire/hostcache/Host.groovy @@ -15,4 +15,9 @@ class Host { public boolean equals(other) { return destination.equals(other.destination) } + + @Override + public String toString() { + "Host[b32:${destination.toBase32()} verifyTime:$verifyTime verificationFailures:$verificationFailures]" + } } diff --git a/host-cache/src/main/groovy/com/muwire/hostcache/HostCache.groovy b/host-cache/src/main/groovy/com/muwire/hostcache/HostCache.groovy index d2fe7d3a..13e14d76 100644 --- a/host-cache/src/main/groovy/com/muwire/hostcache/HostCache.groovy +++ b/host-cache/src/main/groovy/com/muwire/hostcache/HostCache.groovy @@ -64,8 +64,10 @@ public class HostCache { Timer timer = new Timer("timer", true) timer.schedule({hostPool.age()} as TimerTask, 1000,1000) timer.schedule({crawler.startCrawl()} as TimerTask, 10000, 10000) - File verified = new File("verified.json") - File unverified = new File("unverified.json") + File verified = new File("verified") + File unverified = new File("unverified") + verified.mkdir() + unverified.mkdir() timer.schedule({hostPool.serialize(verified, unverified)} as TimerTask, 10000, 60 * 60 * 1000) session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler), diff --git a/host-cache/src/main/groovy/com/muwire/hostcache/HostPool.groovy b/host-cache/src/main/groovy/com/muwire/hostcache/HostPool.groovy index 754d87aa..a0478faa 100644 --- a/host-cache/src/main/groovy/com/muwire/hostcache/HostPool.groovy +++ b/host-cache/src/main/groovy/com/muwire/hostcache/HostPool.groovy @@ -1,11 +1,14 @@ package com.muwire.hostcache +import java.text.SimpleDateFormat import java.util.stream.Collectors import groovy.json.JsonOutput class HostPool { + private static final SimpleDateFormat SDF = new SimpleDateFormat("yyyyMMdd-HH") + final def maxFailures final def maxAge @@ -77,9 +80,11 @@ class HostPool { } } - synchronized void serialize(File verifiedFile, File unverifiedFile) { - write(verifiedFile, verified.values()) - write(unverifiedFile, unverified.values()) + synchronized void serialize(File verifiedPath, File unverifiedPath) { + def now = new Date() + now = SDF.format(now) + write(new File(verifiedPath, now), verified.values()) + write(new File(unverifiedPath, now), unverified.values()) } private void write(File target, Collection hosts) { diff --git a/host-cache/src/main/groovy/com/muwire/hostcache/Pinger.groovy b/host-cache/src/main/groovy/com/muwire/hostcache/Pinger.groovy index 8c03def1..ad7275b2 100644 --- a/host-cache/src/main/groovy/com/muwire/hostcache/Pinger.groovy +++ b/host-cache/src/main/groovy/com/muwire/hostcache/Pinger.groovy @@ -1,17 +1,21 @@ package com.muwire.hostcache import groovy.json.JsonOutput +import groovy.util.logging.Log import net.i2p.client.I2PSession +import net.i2p.client.SendMessageOptions import net.i2p.client.datagram.I2PDatagramMaker +@Log class Pinger { - final def session - Pinger(session) { + final I2PSession session + Pinger(I2PSession session) { this.session = session } def ping(host, uuid) { + log.info("pinging $host with uuid:$uuid") def maker = new I2PDatagramMaker(session) def payload = new HashMap() payload.type = "CrawlerPing" @@ -19,6 +23,8 @@ class Pinger { payload.uuid = uuid payload = JsonOutput.toJson(payload) payload = maker.makeI2PDatagram(payload.bytes) - session.sendMessage(host.destination, payload, I2PSession.PROTO_DATAGRAM, 0, 0) + def options = new SendMessageOptions() + options.setSendLeaseSet(true) + session.sendMessage(host.destination, payload, 0, payload.length, I2PSession.PROTO_DATAGRAM, 0, 0, options) } }