mirror of https://github.com/zlatinb/muwire
switch to jul, reduce aging interval
parent
dc2f675dd3
commit
c07d351c5d
|
@ -1,2 +1,3 @@
|
||||||
apply plugin : 'application'
|
apply plugin : 'application'
|
||||||
mainClassName = 'com.muwire.hostcache.HostCache'
|
mainClassName = 'com.muwire.hostcache.HostCache'
|
||||||
|
applicationDefaultJvmArgs = ['-Djava.util.logging.config.file=logging.properties']
|
||||||
|
|
|
@ -1,9 +1,12 @@
|
||||||
package com.muwire.hostcache
|
package com.muwire.hostcache
|
||||||
|
|
||||||
|
import java.util.logging.Level
|
||||||
import java.util.stream.Collectors
|
import java.util.stream.Collectors
|
||||||
|
|
||||||
|
import groovy.util.logging.Log
|
||||||
import net.i2p.data.Destination
|
import net.i2p.data.Destination
|
||||||
|
|
||||||
|
@Log
|
||||||
class Crawler {
|
class Crawler {
|
||||||
|
|
||||||
final def pinger
|
final def pinger
|
||||||
|
@ -22,12 +25,14 @@ class Crawler {
|
||||||
|
|
||||||
synchronized def handleCrawlerPong(pong, Destination source) {
|
synchronized def handleCrawlerPong(pong, Destination source) {
|
||||||
if (!inFlight.containsKey(source)) {
|
if (!inFlight.containsKey(source)) {
|
||||||
|
log.info("response from host that hasn't been crawled")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
Host host = inFlight.remove(source)
|
Host host = inFlight.remove(source)
|
||||||
|
|
||||||
if (pong.uuid == null || pong.leafSlots == null || pong.peerSlots == null || pong.peers == null) {
|
if (pong.uuid == null || pong.leafSlots == null || pong.peerSlots == null || pong.peers == null) {
|
||||||
hostPool.fail(host)
|
hostPool.fail(host)
|
||||||
|
log.info("invalid crawler pong")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -40,6 +45,7 @@ class Crawler {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!uuid.equals(currentUUID)) {
|
if (!uuid.equals(currentUUID)) {
|
||||||
|
log.info("uuid mismatch")
|
||||||
hostPool.fail(host)
|
hostPool.fail(host)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -50,7 +56,9 @@ class Crawler {
|
||||||
def peers
|
def peers
|
||||||
try {
|
try {
|
||||||
peers = pong.peers.stream().map({b64 -> new Destination(b64)}).collect(Collectors.toSet())
|
peers = pong.peers.stream().map({b64 -> new Destination(b64)}).collect(Collectors.toSet())
|
||||||
|
log.info("received ${peers.size()} peers")
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
log.log(Level.WARNING,"couldn't parse peers", e)
|
||||||
hostPool.fail(host)
|
hostPool.fail(host)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
package com.muwire.hostcache
|
package com.muwire.hostcache
|
||||||
|
|
||||||
|
import java.util.logging.Level
|
||||||
import java.util.stream.Collectors
|
import java.util.stream.Collectors
|
||||||
|
|
||||||
import groovy.json.JsonOutput
|
import groovy.json.JsonOutput
|
||||||
import groovy.json.JsonSlurper
|
import groovy.json.JsonSlurper
|
||||||
|
import groovy.util.logging.Log
|
||||||
import net.i2p.client.I2PClientFactory
|
import net.i2p.client.I2PClientFactory
|
||||||
import net.i2p.client.I2PSession
|
import net.i2p.client.I2PSession
|
||||||
import net.i2p.client.I2PSessionMuxedListener
|
import net.i2p.client.I2PSessionMuxedListener
|
||||||
|
@ -12,6 +14,7 @@ import net.i2p.client.datagram.I2PDatagramMaker
|
||||||
import net.i2p.util.SystemVersion
|
import net.i2p.util.SystemVersion
|
||||||
import net.i2p.data.*
|
import net.i2p.data.*
|
||||||
|
|
||||||
|
@Log
|
||||||
public class HostCache {
|
public class HostCache {
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
|
@ -53,7 +56,7 @@ public class HostCache {
|
||||||
myDest = session.getMyDestination()
|
myDest = session.getMyDestination()
|
||||||
|
|
||||||
// initialize hostpool and crawler
|
// initialize hostpool and crawler
|
||||||
HostPool hostPool = new HostPool(3, 60 * 1000 * 1000)
|
HostPool hostPool = new HostPool(3, 60 * 60 * 1000)
|
||||||
Pinger pinger = new Pinger(session)
|
Pinger pinger = new Pinger(session)
|
||||||
Crawler crawler = new Crawler(pinger, hostPool, 5)
|
Crawler crawler = new Crawler(pinger, hostPool, 5)
|
||||||
|
|
||||||
|
@ -64,7 +67,7 @@ public class HostCache {
|
||||||
session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler),
|
session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler),
|
||||||
I2PSession.PROTO_DATAGRAM, I2PSession.PORT_ANY)
|
I2PSession.PROTO_DATAGRAM, I2PSession.PORT_ANY)
|
||||||
session.connect()
|
session.connect()
|
||||||
println "INFO: connected, going to sleep"
|
log.info("connected, going to sleep")
|
||||||
Thread.sleep(Integer.MAX_VALUE)
|
Thread.sleep(Integer.MAX_VALUE)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -77,16 +80,16 @@ public class HostCache {
|
||||||
|
|
||||||
void reportAbuse(I2PSession sesison, int severity) {}
|
void reportAbuse(I2PSession sesison, int severity) {}
|
||||||
void disconnected(I2PSession session) {
|
void disconnected(I2PSession session) {
|
||||||
println "ERROR: session disconnected, exiting"
|
log.severe("session disconnected, exiting")
|
||||||
System.exit(1)
|
System.exit(1)
|
||||||
}
|
}
|
||||||
void errorOccurred(I2PSession session, String message, Throwable error) {
|
void errorOccurred(I2PSession session, String message, Throwable error) {
|
||||||
println "ERROR: ${message} ${error}"
|
log.warning("${message} ${error}")
|
||||||
}
|
}
|
||||||
void messageAvailable(I2PSession session, int msgId, long size, int proto,
|
void messageAvailable(I2PSession session, int msgId, long size, int proto,
|
||||||
int fromport, int toport) {
|
int fromport, int toport) {
|
||||||
if (proto != I2PSession.PROTO_DATAGRAM) {
|
if (proto != I2PSession.PROTO_DATAGRAM) {
|
||||||
println "WARN: received unexpected protocol ${proto}"
|
log.warning("received unexpected protocol ${proto}")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -95,19 +98,19 @@ public class HostCache {
|
||||||
try {
|
try {
|
||||||
dissector.loadI2PDatagram(payload)
|
dissector.loadI2PDatagram(payload)
|
||||||
def sender = dissector.getSender()
|
def sender = dissector.getSender()
|
||||||
println "INFO: Received something from ${sender.toBase32()}"
|
def b32 = sender.toBase32()
|
||||||
|
|
||||||
payload = dissector.getPayload()
|
payload = dissector.getPayload()
|
||||||
payload = json.parse(payload)
|
payload = json.parse(payload)
|
||||||
if (payload.type == null) {
|
if (payload.type == null) {
|
||||||
println "WARN: type field missing"
|
log.warning("type field missing from $b32")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
switch(payload.type) {
|
switch(payload.type) {
|
||||||
case "Ping" :
|
case "Ping" :
|
||||||
println "Ping"
|
log.info("ping from $b32")
|
||||||
if (payload.leaf == null) {
|
if (payload.leaf == null) {
|
||||||
println "WARN: ping didn't specify if leaf"
|
log.warning("ping didn't specify if leaf from $b32")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
payload.leaf = Boolean.parseBoolean(payload.leaf.toString())
|
payload.leaf = Boolean.parseBoolean(payload.leaf.toString())
|
||||||
|
@ -116,14 +119,14 @@ public class HostCache {
|
||||||
respond(session, sender, payload)
|
respond(session, sender, payload)
|
||||||
break
|
break
|
||||||
case "CrawlerPong":
|
case "CrawlerPong":
|
||||||
println "CrawlerPong"
|
log.info("CrawlerPong from $b32")
|
||||||
crawler.handleCrawlerPong(payload, sender)
|
crawler.handleCrawlerPong(payload, sender)
|
||||||
break
|
break
|
||||||
default:
|
default:
|
||||||
println "WARN: Unexpected message type ${payload.type}, dropping"
|
log.warning("Unexpected message type ${payload.type}, dropping from $b32")
|
||||||
}
|
}
|
||||||
} catch (Exception dfe) {
|
} catch (Exception dfe) {
|
||||||
println "WARN: invalid datagram ${dfe}"
|
log.log(Level.WARNING,"invalid datagram", dfe)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
void messageAvailable(I2PSession session, int msgId, long size) {
|
void messageAvailable(I2PSession session, int msgId, long size) {
|
||||||
|
|
Loading…
Reference in New Issue