mirror of https://github.com/zlatinb/muwire
* More logging
* Include leaseset in crawler pings * serialize hourly files in a directory, keep historypull/53/head
parent
3436af75bf
commit
84c7da1fe0
|
@ -40,12 +40,13 @@ class Crawler {
|
||||||
try {
|
try {
|
||||||
uuid = UUID.fromString(pong.uuid)
|
uuid = UUID.fromString(pong.uuid)
|
||||||
} catch (IllegalArgumentException bad) {
|
} catch (IllegalArgumentException bad) {
|
||||||
|
log.log(Level.WARNING,"couldn't parse uuid",bad)
|
||||||
hostPool.fail(host)
|
hostPool.fail(host)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!uuid.equals(currentUUID)) {
|
if (!uuid.equals(currentUUID)) {
|
||||||
log.info("uuid mismatch")
|
log.warn("uuid mismatch $uuid expected $currentUUID")
|
||||||
hostPool.fail(host)
|
hostPool.fail(host)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -75,11 +76,12 @@ class Crawler {
|
||||||
}
|
}
|
||||||
|
|
||||||
synchronized def startCrawl() {
|
synchronized def startCrawl() {
|
||||||
|
currentUUID = UUID.randomUUID()
|
||||||
|
log.info("starting new crawl with uuid $currentUUID inFlight ${inFlight.size()}")
|
||||||
if (!inFlight.isEmpty()) {
|
if (!inFlight.isEmpty()) {
|
||||||
inFlight.values().each { hostPool.fail(it) }
|
inFlight.values().each { hostPool.fail(it) }
|
||||||
inFlight.clear()
|
inFlight.clear()
|
||||||
}
|
}
|
||||||
currentUUID = UUID.randomUUID()
|
|
||||||
hostPool.getUnverified(parallel).each {
|
hostPool.getUnverified(parallel).each {
|
||||||
inFlight.put(it.destination, it)
|
inFlight.put(it.destination, it)
|
||||||
pinger.ping(it, currentUUID)
|
pinger.ping(it, currentUUID)
|
||||||
|
|
|
@ -15,4 +15,9 @@ class Host {
|
||||||
public boolean equals(other) {
|
public boolean equals(other) {
|
||||||
return destination.equals(other.destination)
|
return destination.equals(other.destination)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
"Host[b32:${destination.toBase32()} verifyTime:$verifyTime verificationFailures:$verificationFailures]"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -64,8 +64,10 @@ public class HostCache {
|
||||||
Timer timer = new Timer("timer", true)
|
Timer timer = new Timer("timer", true)
|
||||||
timer.schedule({hostPool.age()} as TimerTask, 1000,1000)
|
timer.schedule({hostPool.age()} as TimerTask, 1000,1000)
|
||||||
timer.schedule({crawler.startCrawl()} as TimerTask, 10000, 10000)
|
timer.schedule({crawler.startCrawl()} as TimerTask, 10000, 10000)
|
||||||
File verified = new File("verified.json")
|
File verified = new File("verified")
|
||||||
File unverified = new File("unverified.json")
|
File unverified = new File("unverified")
|
||||||
|
verified.mkdir()
|
||||||
|
unverified.mkdir()
|
||||||
timer.schedule({hostPool.serialize(verified, unverified)} as TimerTask, 10000, 60 * 60 * 1000)
|
timer.schedule({hostPool.serialize(verified, unverified)} as TimerTask, 10000, 60 * 60 * 1000)
|
||||||
|
|
||||||
session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler),
|
session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler),
|
||||||
|
|
|
@ -1,11 +1,14 @@
|
||||||
package com.muwire.hostcache
|
package com.muwire.hostcache
|
||||||
|
|
||||||
|
import java.text.SimpleDateFormat
|
||||||
import java.util.stream.Collectors
|
import java.util.stream.Collectors
|
||||||
|
|
||||||
import groovy.json.JsonOutput
|
import groovy.json.JsonOutput
|
||||||
|
|
||||||
class HostPool {
|
class HostPool {
|
||||||
|
|
||||||
|
private static final SimpleDateFormat SDF = new SimpleDateFormat("yyyyMMdd-HH")
|
||||||
|
|
||||||
final def maxFailures
|
final def maxFailures
|
||||||
final def maxAge
|
final def maxAge
|
||||||
|
|
||||||
|
@ -77,9 +80,11 @@ class HostPool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
synchronized void serialize(File verifiedFile, File unverifiedFile) {
|
synchronized void serialize(File verifiedPath, File unverifiedPath) {
|
||||||
write(verifiedFile, verified.values())
|
def now = new Date()
|
||||||
write(unverifiedFile, unverified.values())
|
now = SDF.format(now)
|
||||||
|
write(new File(verifiedPath, now), verified.values())
|
||||||
|
write(new File(unverifiedPath, now), unverified.values())
|
||||||
}
|
}
|
||||||
|
|
||||||
private void write(File target, Collection hosts) {
|
private void write(File target, Collection hosts) {
|
||||||
|
|
|
@ -1,17 +1,21 @@
|
||||||
package com.muwire.hostcache
|
package com.muwire.hostcache
|
||||||
|
|
||||||
import groovy.json.JsonOutput
|
import groovy.json.JsonOutput
|
||||||
|
import groovy.util.logging.Log
|
||||||
import net.i2p.client.I2PSession
|
import net.i2p.client.I2PSession
|
||||||
|
import net.i2p.client.SendMessageOptions
|
||||||
import net.i2p.client.datagram.I2PDatagramMaker
|
import net.i2p.client.datagram.I2PDatagramMaker
|
||||||
|
|
||||||
|
@Log
|
||||||
class Pinger {
|
class Pinger {
|
||||||
|
|
||||||
final def session
|
final I2PSession session
|
||||||
Pinger(session) {
|
Pinger(I2PSession session) {
|
||||||
this.session = session
|
this.session = session
|
||||||
}
|
}
|
||||||
|
|
||||||
def ping(host, uuid) {
|
def ping(host, uuid) {
|
||||||
|
log.info("pinging $host with uuid:$uuid")
|
||||||
def maker = new I2PDatagramMaker(session)
|
def maker = new I2PDatagramMaker(session)
|
||||||
def payload = new HashMap()
|
def payload = new HashMap()
|
||||||
payload.type = "CrawlerPing"
|
payload.type = "CrawlerPing"
|
||||||
|
@ -19,6 +23,8 @@ class Pinger {
|
||||||
payload.uuid = uuid
|
payload.uuid = uuid
|
||||||
payload = JsonOutput.toJson(payload)
|
payload = JsonOutput.toJson(payload)
|
||||||
payload = maker.makeI2PDatagram(payload.bytes)
|
payload = maker.makeI2PDatagram(payload.bytes)
|
||||||
session.sendMessage(host.destination, payload, I2PSession.PROTO_DATAGRAM, 0, 0)
|
def options = new SendMessageOptions()
|
||||||
|
options.setSendLeaseSet(true)
|
||||||
|
session.sendMessage(host.destination, payload, 0, payload.length, I2PSession.PROTO_DATAGRAM, 0, 0, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue