Merge cache, expire node individually

README.md

@@ -66,7 +66,7 @@ rebalance-lnd](https://github.com/accumulator/rebalance-lnd).
       --allow-unbalance-to    let the target channel go above 50% local liquidity, use if you want to refill a channel; you should also set --pto to >50
   -s, --stat=                 save successful rebalance information to the specified CSV file
       --node-cache=           save and load other nodes information to this file, improves cold start performance
-      --node-cache-lifetime=  the cache file will not be loaded if it's older than this time (in minutes) (default: 1440)
+      --node-cache-lifetime=  nodes with last update older than this time (in minutes) will be removed from cache after loading it (default: 1440)
 ```
 
 Look in `config.json.sample` or `config.toml.sample` for corresponding keys,

cache.go

@@ -2,59 +2,76 @@ package main
 
 import (
 	"encoding/gob"
+	"fmt"
 	"log"
 	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/gofrs/flock"
+	"github.com/lightningnetwork/lnd/lnrpc"
 )
 
 func lock() *flock.Flock {
 	return flock.New(filepath.Join(os.TempDir(), "regolancer.lock"))
 }
 
-func (r *regolancer) loadNodeCache(filename string, exp int) {
+func (r *regolancer) loadNodeCache(filename string, exp int, doLock bool) error {
 	if filename == "" {
-		return
+		return nil
+	}
+	if doLock {
+		log.Printf("Loading node cache from %s", filename)
+		l := lock()
+		l.RLock()
+		defer l.Unlock()
 	}
-	l := lock()
-	l.RLock()
-	defer l.Unlock()
 	f, err := os.Open(filename)
 	if err != nil {
 		if !os.IsNotExist(err) {
-			logErrorF("Error opening node cache file: %s", err)
+			return fmt.Errorf("error opening node cache file: %s", err)
 		}
-		return
+		return nil
 	}
 	defer f.Close()
-	fi, err := f.Stat()
-	if err != nil {
-		logErrorF("Error getting node cache information: %s", err)
-		return
-	}
-	if time.Since(fi.ModTime()) > time.Minute*time.Duration(exp) {
-		log.Print("Node cache expired, not loading")
-		return
-	}
-	log.Printf("Loading node cache from %s", filename)
 	gob.NewDecoder(f).Decode(&r.nodeCache)
+	for k, v := range r.nodeCache {
+		if time.Since(time.Unix(int64(v.Node.LastUpdate), 0)) >
+			time.Minute*time.Duration(exp) {
+			delete(r.nodeCache, k)
+		}
+	}
+	return nil
 }
 
-func (r *regolancer) saveNodeCache(filename string) {
+func (r *regolancer) saveNodeCache(filename string, exp int) error {
 	if filename == "" {
-		return
+		return nil
 	}
 	log.Printf("Saving node cache to %s", filename)
+
 	l := lock()
 	l.Lock()
 	defer l.Unlock()
+
+	old := regolancer{nodeCache: map[string]*lnrpc.NodeInfo{}}
+	err := old.loadNodeCache(filename, exp, false)
+
+	if err != nil {
+		logErrorF("Error merging cache, saving anew: %s", err)
+	}
+	for k, v := range old.nodeCache {
+		if n, ok := r.nodeCache[k]; !ok ||
+			n.Node.LastUpdate < v.Node.LastUpdate {
+			r.nodeCache[k] = v
+		}
+	}
+
 	f, err := os.Create(filename)
 	if err != nil {
-		logErrorF("Error creating node cache file %s: %s", filename, err)
-		return
+		return fmt.Errorf("error creating node cache file: %s", err)
 	}
 	defer f.Close()
 	gob.NewEncoder(f).Encode(r.nodeCache)
+	return nil
 }

main.go

@@ -47,7 +47,7 @@ type configParams struct {
 	AllowUnbalanceTo   bool     `long:"allow-unbalance-to" description:"let the target channel go above 50% local liquidity, use if you want to refill a channel; you should also set --pto to >50" json:"allow_unbalance_to" toml:"allow_unbalance_to"`
 	StatFilename       string   `short:"s" long:"stat" description:"save successful rebalance information to the specified CSV file" json:"stat" toml:"stat"`
 	NodeCacheFilename  string   `long:"node-cache" description:"save and load other nodes information to this file, improves cold start performance"  json:"node_cache_filename" toml:"node_cache_filename"`
-	NodeCacheLifetime  int      `long:"node-cache-lifetime" description:"the cache file will not be loaded if it's older than this time (in minutes)" json:"node_cache_lifetime" toml:"node_cache_lifetime" default:"1440"`
+	NodeCacheLifetime  int      `long:"node-cache-lifetime" description:"nodes with last update older than this time (in minutes) will be removed from cache after loading it" json:"node_cache_lifetime" toml:"node_cache_lifetime" default:"1440"`
 }
 
 var params, cfgParams configParams
@@ -289,13 +289,17 @@ func main() {
 	infoCtxCancel()
 	attempt := 1
 
-	r.loadNodeCache(params.NodeCacheFilename, params.NodeCacheLifetime)
-	defer r.saveNodeCache(params.NodeCacheFilename)
+	err = r.loadNodeCache(params.NodeCacheFilename, params.NodeCacheLifetime,
+		true)
+	if err != nil {
+		logErrorF("%s", err)
+	}
+	defer r.saveNodeCache(params.NodeCacheFilename, params.NodeCacheLifetime)
 	stopChan := make(chan os.Signal)
 	signal.Notify(stopChan, os.Interrupt)
 	go func() {
 		<-stopChan
-		r.saveNodeCache(params.NodeCacheFilename)
+		r.saveNodeCache(params.NodeCacheFilename, params.NodeCacheLifetime)
 		os.Exit(1)
 	}()