* More logging

* Include leaseset in crawler pings
* serialize hourly files in a directory, keep history
pull/53/head
Zlatin Balevsky 2020-04-26 20:15:48 +01:00
parent 3436af75bf
commit 84c7da1fe0
5 changed files with 30 additions and 10 deletions

View File

@ -40,12 +40,13 @@ class Crawler {
try { try {
uuid = UUID.fromString(pong.uuid) uuid = UUID.fromString(pong.uuid)
} catch (IllegalArgumentException bad) { } catch (IllegalArgumentException bad) {
log.log(Level.WARNING,"couldn't parse uuid",bad)
hostPool.fail(host) hostPool.fail(host)
return return
} }
if (!uuid.equals(currentUUID)) { if (!uuid.equals(currentUUID)) {
log.info("uuid mismatch") log.warn("uuid mismatch $uuid expected $currentUUID")
hostPool.fail(host) hostPool.fail(host)
return return
} }
@ -75,11 +76,12 @@ class Crawler {
} }
synchronized def startCrawl() { synchronized def startCrawl() {
currentUUID = UUID.randomUUID()
log.info("starting new crawl with uuid $currentUUID inFlight ${inFlight.size()}")
if (!inFlight.isEmpty()) { if (!inFlight.isEmpty()) {
inFlight.values().each { hostPool.fail(it) } inFlight.values().each { hostPool.fail(it) }
inFlight.clear() inFlight.clear()
} }
currentUUID = UUID.randomUUID()
hostPool.getUnverified(parallel).each { hostPool.getUnverified(parallel).each {
inFlight.put(it.destination, it) inFlight.put(it.destination, it)
pinger.ping(it, currentUUID) pinger.ping(it, currentUUID)

View File

@ -15,4 +15,9 @@ class Host {
public boolean equals(other) { public boolean equals(other) {
return destination.equals(other.destination) return destination.equals(other.destination)
} }
@Override
public String toString() {
"Host[b32:${destination.toBase32()} verifyTime:$verifyTime verificationFailures:$verificationFailures]"
}
} }

View File

@ -64,8 +64,10 @@ public class HostCache {
Timer timer = new Timer("timer", true) Timer timer = new Timer("timer", true)
timer.schedule({hostPool.age()} as TimerTask, 1000,1000) timer.schedule({hostPool.age()} as TimerTask, 1000,1000)
timer.schedule({crawler.startCrawl()} as TimerTask, 10000, 10000) timer.schedule({crawler.startCrawl()} as TimerTask, 10000, 10000)
File verified = new File("verified.json") File verified = new File("verified")
File unverified = new File("unverified.json") File unverified = new File("unverified")
verified.mkdir()
unverified.mkdir()
timer.schedule({hostPool.serialize(verified, unverified)} as TimerTask, 10000, 60 * 60 * 1000) timer.schedule({hostPool.serialize(verified, unverified)} as TimerTask, 10000, 60 * 60 * 1000)
session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler), session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler),

View File

@ -1,11 +1,14 @@
package com.muwire.hostcache package com.muwire.hostcache
import java.text.SimpleDateFormat
import java.util.stream.Collectors import java.util.stream.Collectors
import groovy.json.JsonOutput import groovy.json.JsonOutput
class HostPool { class HostPool {
private static final SimpleDateFormat SDF = new SimpleDateFormat("yyyyMMdd-HH")
final def maxFailures final def maxFailures
final def maxAge final def maxAge
@ -77,9 +80,11 @@ class HostPool {
} }
} }
synchronized void serialize(File verifiedFile, File unverifiedFile) { synchronized void serialize(File verifiedPath, File unverifiedPath) {
write(verifiedFile, verified.values()) def now = new Date()
write(unverifiedFile, unverified.values()) now = SDF.format(now)
write(new File(verifiedPath, now), verified.values())
write(new File(unverifiedPath, now), unverified.values())
} }
private void write(File target, Collection hosts) { private void write(File target, Collection hosts) {

View File

@ -1,17 +1,21 @@
package com.muwire.hostcache package com.muwire.hostcache
import groovy.json.JsonOutput import groovy.json.JsonOutput
import groovy.util.logging.Log
import net.i2p.client.I2PSession import net.i2p.client.I2PSession
import net.i2p.client.SendMessageOptions
import net.i2p.client.datagram.I2PDatagramMaker import net.i2p.client.datagram.I2PDatagramMaker
@Log
class Pinger { class Pinger {
final def session final I2PSession session
Pinger(session) { Pinger(I2PSession session) {
this.session = session this.session = session
} }
def ping(host, uuid) { def ping(host, uuid) {
log.info("pinging $host with uuid:$uuid")
def maker = new I2PDatagramMaker(session) def maker = new I2PDatagramMaker(session)
def payload = new HashMap() def payload = new HashMap()
payload.type = "CrawlerPing" payload.type = "CrawlerPing"
@ -19,6 +23,8 @@ class Pinger {
payload.uuid = uuid payload.uuid = uuid
payload = JsonOutput.toJson(payload) payload = JsonOutput.toJson(payload)
payload = maker.makeI2PDatagram(payload.bytes) payload = maker.makeI2PDatagram(payload.bytes)
session.sendMessage(host.destination, payload, I2PSession.PROTO_DATAGRAM, 0, 0) def options = new SendMessageOptions()
options.setSendLeaseSet(true)
session.sendMessage(host.destination, payload, 0, payload.length, I2PSession.PROTO_DATAGRAM, 0, 0, options)
} }
} }