Add mission control helper with fail tolerance

README.md

@@ -86,6 +86,7 @@ in `~/go/bin/linux_arm`.
   -d, --exclude-node=            don't use this node for routing (can be specified multiple times)
       --to=                      try only this channel as target (should satisfy other constraints too; can be specified multiple times)
       --from=                    try only this channel as source (should satisfy other constraints too; can be specified multiple times)
+      --fail-tolerance=        if a channel failed before during this rebalance but chosen again by lnd, and the forward amount differs by less than this ppm, exclude the channel
       --allow-unbalance-from     let the source channel go below 50% local liquidity, use if you want to drain a channel; you should also set --pfrom to >50
       --allow-unbalance-to       let the target channel go above 50% local liquidity, use if you want to refill a channel; you should also set --pto to >50
   -s, --stat=                    save successful rebalance information to the specified CSV file

go.mod

@@ -5,6 +5,7 @@ go 1.18
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/fatih/color v1.13.0
+	github.com/gofrs/flock v0.8.1
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/lightninglabs/lndclient v0.15.1-0
 	github.com/lightningnetwork/lnd v0.15.1-beta.rc1
@@ -43,7 +44,6 @@ require (
 	github.com/fergusstrange/embedded-postgres v1.10.0 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
-	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect

go.sum

@@ -858,7 +858,6 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

main.go

@@ -46,6 +46,7 @@ type configParams struct {
 	ExcludeNodes        []string `short:"d" long:"exclude-node" description:"don't use this node for routing (can be specified multiple times)" json:"exclude_nodes" toml:"exclude_nodes"`
 	ToChannel           []string `long:"to" description:"try only this channel as target (should satisfy other constraints too; can be specified multiple times)" json:"to" toml:"to"`
 	FromChannel         []string `long:"from" description:"try only this channel as source (should satisfy other constraints too; can be specified multiple times)" json:"from" toml:"from"`
+	FailTolerance       int64    `long:"fail-tolerance" description:"if a channel failed before during this rebalance but chosen again by lnd, and the forward amount differs by less than this ppm, exclude the channel" json:"fail_tolerance" toml:"fail_tolerance"`
 	AllowUnbalanceFrom  bool     `long:"allow-unbalance-from" description:"let the source channel go below 50% local liquidity, use if you want to drain a channel; you should also set --pfrom to >50" json:"allow_unbalance_from" toml:"allow_unbalance_from"`
 	AllowUnbalanceTo    bool     `long:"allow-unbalance-to" description:"let the target channel go above 50% local liquidity, use if you want to refill a channel; you should also set --pto to >50" json:"allow_unbalance_to" toml:"allow_unbalance_to"`
 	StatFilename        string   `short:"s" long:"stat" description:"save successful rebalance information to the specified CSV file" json:"stat" toml:"stat"`
@@ -87,6 +88,8 @@ type regolancer struct {
 	statFilename  string
 	routeFound    bool
 	invoiceCache  map[int64]*lnrpc.AddInvoiceResponse
+	mcCache       map[string]int64
+	failedPairs   []*lnrpc.NodePair
 }
 
 func loadConfig() {
@@ -367,7 +370,9 @@ func preflightChecks(params *configParams) error {
 		(params.RelAmountFrom > 0 || params.RelAmountTo > 0) {
 		return fmt.Errorf("use either precise amount or relative amounts but not both")
 	}
-
+	if params.FailTolerance == 0 {
+		params.FailTolerance = 1000
+	}
 	if (params.RelAmountFrom > 0 || params.RelAmountTo > 0) && params.AllowRapidRebalance {
 		return fmt.Errorf("use either relative amounts or rapid rebalance but not both")
 
@@ -404,6 +409,7 @@ func main() {
 		chanCache:    map[uint64]*lnrpc.ChannelEdge{},
 		channelPairs: map[string][2]*lnrpc.Channel{},
 		failureCache: map[string]failedRoute{},
+		mcCache:      map[string]int64{},
 		statFilename: params.StatFilename,
 	}
 	r.lnClient = lnrpc.NewLightningClient(conn)

mission_ctl.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"encoding/hex"
+	"fmt"
+
+	"github.com/lightningnetwork/lnd/lnrpc"
+)
+
+func (r *regolancer) addFailedChan(fromStr string, toStr string, amount int64) {
+	r.mcCache[fromStr+toStr] = amount
+}
+
+func (r *regolancer) validateRoute(route *lnrpc.Route) error {
+	prevHopPK := r.myPK
+	for _, h := range route.Hops {
+		hopPK := h.PubKey
+		if fp, ok := r.mcCache[prevHopPK+hopPK]; ok && 1e6-h.AmtToForwardMsat*1e6/fp < params.FailTolerance {
+			from, err := hex.DecodeString(prevHopPK)
+			if err != nil {
+				return err
+			}
+			to, err := hex.DecodeString(hopPK)
+			if err != nil {
+				return err
+			}
+			r.failedPairs = append(r.failedPairs, &lnrpc.NodePair{From: from, To: to})
+			return fmt.Errorf("chan %d failed before with %d msat and will not be used anymore during this rebalance, payment attempt with %d msat cancelled", h.ChanId, fp, h.AmtToForwardMsat)
+		}
+		prevHopPK = hopPK
+	}
+	return nil
+}

payment.go

@@ -79,9 +79,11 @@ func (r *regolancer) pay(ctx context.Context, amount int64, minAmount int64,
 			return fmt.Errorf("error: %s @ %d", result.Failure.Code.String(),
 				result.Failure.FailureSourceIndex)
 		}
+		prevHop := route.Hops[result.Failure.FailureSourceIndex-1]
+		failedHop := route.Hops[result.Failure.FailureSourceIndex]
 		nodeCtx, cancel := context.WithTimeout(ctx, time.Minute)
 		defer cancel()
-		node1, err := r.getNodeInfo(nodeCtx, route.Hops[result.Failure.FailureSourceIndex-1].PubKey)
+		node1, err := r.getNodeInfo(nodeCtx, prevHop.PubKey)
 		node1name := ""
 		node2name := ""
 		if err != nil {
@@ -89,14 +91,17 @@ func (r *regolancer) pay(ctx context.Context, amount int64, minAmount int64,
 		} else {
 			node1name = node1.Node.Alias
 		}
-		node2, err := r.getNodeInfo(nodeCtx, route.Hops[result.Failure.FailureSourceIndex].PubKey)
+		node2, err := r.getNodeInfo(nodeCtx, failedHop.PubKey)
 		if err != nil {
 			node2name = fmt.Sprintf("node%d", result.Failure.FailureSourceIndex)
 		} else {
 			node2name = node2.Node.Alias
 		}
-		log.Printf("%s %s ⇒ %s", faintWhiteColor(result.Failure.Code.String()),
-			cyanColor(node1name), cyanColor(node2name))
+		log.Printf("%s %s ⇒ %s", faintWhiteColor(result.Failure.Code.String()), cyanColor(node1name), cyanColor(node2name))
+		if result.Failure.Code == lnrpc.Failure_TEMPORARY_CHANNEL_FAILURE {
+			r.addFailedChan(node1.Node.PubKey, node2.Node.PubKey, prevHop.
+				AmtToForwardMsat)
+		}
 		if probeSteps > 0 && int(result.Failure.FailureSourceIndex) == len(route.Hops)-2 &&
 			result.Failure.Code == lnrpc.Failure_TEMPORARY_CHANNEL_FAILURE {
 			fmt.Println("Probing route...")

routes.go

@@ -108,12 +108,24 @@ func (r *regolancer) getRoutes(ctx context.Context, from, to uint64, amtMsat int
 		UseMissionControl: true,
 		FeeLimit:          &lnrpc.FeeLimit{Limit: &lnrpc.FeeLimit_FixedMsat{FixedMsat: feeMsat}},
 		IgnoredNodes:      r.excludeNodes,
+		IgnoredPairs:      r.failedPairs,
 	})
 	if err != nil {
 		return nil, 0, err
 	}
+	result := []*lnrpc.Route{}
+	for i := range routes.Routes { // lnd always returns 1 route for now but just in case it changes
+		if err := r.validateRoute(routes.Routes[i]); err == nil {
+			result = append(result, routes.Routes[i])
+		} else {
+			log.Print(err)
+		}
+	}
+	if len(result) == 0 {
+		return r.getRoutes(ctx, from, to, amtMsat)
+	}
 	r.routeFound = true
-	return routes.Routes, feeMsat, nil
+	return result, feeMsat, nil
 }
 
 func (r *regolancer) getNodeInfo(ctx context.Context, pk string) (*lnrpc.NodeInfo, error) {