* More logging

* Include leaseset in crawler pings
* serialize hourly files in a directory, keep history

host-cache/src/main/groovy/com/muwire/hostcache/Crawler.groovy

@@ -40,12 +40,13 @@ class Crawler {
         try {
             uuid = UUID.fromString(pong.uuid)
         } catch (IllegalArgumentException bad) {
+            log.log(Level.WARNING,"couldn't parse uuid",bad)
             hostPool.fail(host)
             return
         }
 
         if (!uuid.equals(currentUUID)) {
-            log.info("uuid mismatch")
+            log.warn("uuid mismatch $uuid expected $currentUUID")
             hostPool.fail(host)
             return
         }
@@ -75,11 +76,12 @@ class Crawler {
     }
 
     synchronized def startCrawl() {
+        currentUUID = UUID.randomUUID()
+        log.info("starting new crawl with uuid $currentUUID inFlight ${inFlight.size()}")
         if (!inFlight.isEmpty()) {
             inFlight.values().each { hostPool.fail(it) }
             inFlight.clear()
         }
-        currentUUID = UUID.randomUUID()
         hostPool.getUnverified(parallel).each {
             inFlight.put(it.destination, it)
             pinger.ping(it, currentUUID)

host-cache/src/main/groovy/com/muwire/hostcache/Host.groovy

@@ -15,4 +15,9 @@ class Host {
     public boolean equals(other) {
         return destination.equals(other.destination)
     }
+    
+    @Override
+    public String toString() {
+        "Host[b32:${destination.toBase32()} verifyTime:$verifyTime verificationFailures:$verificationFailures]"
+    }
 }

host-cache/src/main/groovy/com/muwire/hostcache/HostCache.groovy

@@ -64,8 +64,10 @@ public class HostCache {
         Timer timer = new Timer("timer", true)
         timer.schedule({hostPool.age()} as TimerTask, 1000,1000)
         timer.schedule({crawler.startCrawl()} as TimerTask, 10000, 10000)
-        File verified = new File("verified.json")
+        File verified = new File("verified")
-        File unverified = new File("unverified.json")
+        File unverified = new File("unverified")
+        verified.mkdir()
+        unverified.mkdir()
         timer.schedule({hostPool.serialize(verified, unverified)} as TimerTask, 10000, 60 * 60 * 1000)
 
         session.addMuxedSessionListener(new Listener(hostPool: hostPool, toReturn: 2, crawler: crawler),

host-cache/src/main/groovy/com/muwire/hostcache/HostPool.groovy

@@ -1,11 +1,14 @@
 package com.muwire.hostcache
 
+import java.text.SimpleDateFormat
 import java.util.stream.Collectors
 
 import groovy.json.JsonOutput
 
 class HostPool {
 
+    private static final SimpleDateFormat SDF = new SimpleDateFormat("yyyyMMdd-HH")
+    
     final def maxFailures
     final def maxAge
 
@@ -77,9 +80,11 @@ class HostPool {
         }
     }
 
-    synchronized void serialize(File verifiedFile, File unverifiedFile) {
+    synchronized void serialize(File verifiedPath, File unverifiedPath) {
-        write(verifiedFile, verified.values())
+        def now = new Date()
-        write(unverifiedFile, unverified.values())
+        now = SDF.format(now)
+        write(new File(verifiedPath, now), verified.values())
+        write(new File(unverifiedPath, now), unverified.values())
     }
 
     private void write(File target, Collection hosts) {

host-cache/src/main/groovy/com/muwire/hostcache/Pinger.groovy

@@ -1,17 +1,21 @@
 package com.muwire.hostcache
 
 import groovy.json.JsonOutput
+import groovy.util.logging.Log
 import net.i2p.client.I2PSession
+import net.i2p.client.SendMessageOptions
 import net.i2p.client.datagram.I2PDatagramMaker
 
+@Log
 class Pinger {
 
-    final def session
+    final I2PSession session
-    Pinger(session) {
+    Pinger(I2PSession session) {
         this.session = session
     }
 
     def ping(host, uuid) {
+        log.info("pinging $host with uuid:$uuid")
         def maker = new I2PDatagramMaker(session)
         def payload = new HashMap()
         payload.type = "CrawlerPing"
@@ -19,6 +23,8 @@ class Pinger {
         payload.uuid = uuid
         payload = JsonOutput.toJson(payload)
         payload = maker.makeI2PDatagram(payload.bytes)
-        session.sendMessage(host.destination, payload, I2PSession.PROTO_DATAGRAM, 0, 0)
+        def options = new SendMessageOptions()
+        options.setSendLeaseSet(true)
+        session.sendMessage(host.destination, payload, 0, payload.length, I2PSession.PROTO_DATAGRAM, 0, 0, options)
     }
 }