Decouple autossl code handling underlying bufs

And clean up redundant child callbacks

src/protoautossl.c

@@ -42,6 +42,26 @@ struct protoautossl_ctx {
 	unsigned int clienthello_found : 1;      /* 1 if conn upgrade to SSL */
 };
 
+#ifdef DEBUG_PROXY
+static void NONNULL(1,2,3)
+protoautossl_log_dbg_evbuf_info(pxy_conn_ctx_t *ctx, pxy_conn_desc_t *this, pxy_conn_desc_t *other)
+{
+	// This function is used by child conns too, they pass ctx->conn instead of ctx
+	if (OPTS_DEBUG(ctx->global)) {
+		prototcp_log_dbg_evbuf_info(ctx, &ctx->src, &ctx->dst);
+
+		struct bufferevent *ubev = bufferevent_get_underlying(this->bev);
+		struct bufferevent *ubev_other = other->closed ? NULL : bufferevent_get_underlying(other->bev);
+		if (ubev || ubev_other)
+			log_dbg_printf("underlying evbuffer size at EOF: i:%zu o:%zu i:%zu o:%zu\n",
+							ubev ? evbuffer_get_length(bufferevent_get_input(ubev)) : 0,
+							ubev ? evbuffer_get_length(bufferevent_get_output(ubev)) : 0,
+							ubev_other ? evbuffer_get_length(bufferevent_get_input(ubev_other)) : 0,
+							ubev_other ? evbuffer_get_length(bufferevent_get_output(ubev_other)) : 0);
+	}
+}
+#endif /* DEBUG_PROXY */
+
 /*
  * Free bufferevent and close underlying socket properly.
  * For OpenSSL bufferevents, this will shutdown the SSL connection.
@@ -285,19 +305,6 @@ protoautossl_peek_and_upgrade(pxy_conn_ctx_t *ctx)
 	return 0;
 }
 
-static void NONNULL(1)
-protoautossl_disable_srvdst(pxy_conn_ctx_t *ctx)
-{
-	log_finest("ENTER");
-
-	// Do not disable underlying bevs in autossl
-	bufferevent_setcb(ctx->srvdst.bev, NULL, NULL, NULL, NULL);
-	bufferevent_disable(ctx->srvdst.bev, EV_READ|EV_WRITE);
-
-	// Do not access srvdst.bev from this point on
-	ctx->srvdst.bev = NULL;
-}
-
 static int NONNULL(1) WUNRES
 protoautossl_conn_connect(pxy_conn_ctx_t *ctx)
 {
@@ -357,6 +364,38 @@ protoautossl_try_unset_watermark(struct bufferevent *bev, pxy_conn_ctx_t *ctx, p
 	}
 }
 
+static void NONNULL(1)
+protoautossl_try_discard_inbuf(struct bufferevent *bev)
+{
+	prototcp_try_discard_inbuf(bev);
+
+	struct bufferevent *ubev = bufferevent_get_underlying(bev);
+	if (ubev) {
+		struct evbuffer *ubev_inbuf = bufferevent_get_input(ubev);
+		size_t ubev_inbuf_size = evbuffer_get_length(ubev_inbuf);
+		if (ubev_inbuf_size) {
+			log_dbg_printf("Warning: Drained %zu bytes from inbuf underlying\n", ubev_inbuf_size);
+			evbuffer_drain(ubev_inbuf, ubev_inbuf_size);
+		}
+	}
+}
+
+static void NONNULL(1)
+protoautossl_try_discard_outbuf(struct bufferevent *bev)
+{
+	prototcp_try_discard_outbuf(bev);
+
+	struct bufferevent *ubev = bufferevent_get_underlying(bev);
+	if (ubev) {
+		struct evbuffer *ubev_outbuf = bufferevent_get_output(ubev);
+		size_t ubev_outbuf_size = evbuffer_get_length(ubev_outbuf);
+		if (ubev_outbuf_size) {
+			log_dbg_printf("Warning: Drained %zu bytes from outbuf underlying\n", ubev_outbuf_size);
+			evbuffer_drain(ubev_outbuf, ubev_outbuf_size);
+		}
+	}
+}
+
 static void NONNULL(1)
 protoautossl_bev_readcb_src(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 {
@@ -381,7 +420,7 @@ protoautossl_bev_readcb_src(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 	}
 
 	if (ctx->dst.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -412,7 +451,7 @@ protoautossl_bev_readcb_srvdst(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 	// as we do with the smtp protocol, @see protosmtp_bev_readcb_srvdst()
 
 	if (ctx->src.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -420,6 +459,23 @@ protoautossl_bev_readcb_srvdst(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 	ctx->protoctx->set_watermarkcb(bev, ctx, ctx->src.bev);
 }
 
+static int NONNULL(1) WUNRES
+protoautossl_outbuf_has_data(struct bufferevent *bev
+#ifdef DEBUG_PROXY
+	, char *reason, pxy_conn_ctx_t *ctx
+#endif /* DEBUG_PROXY */
+	)
+{
+	size_t outbuflen = evbuffer_get_length(bufferevent_get_output(bev));
+	struct bufferevent *ubev = bufferevent_get_underlying(bev);
+	if (outbuflen || (ubev && evbuffer_get_length(bufferevent_get_output(ubev)))) {
+		log_finest_va("Not closing %s, outbuflen=%zu, ubev outbuflen=%zu", reason,
+				outbuflen, ubev ? evbuffer_get_length(bufferevent_get_output(ubev)) : 0);
+		return 1;
+	}
+	return 0;
+}
+
 static void NONNULL(1,2)
 protoautossl_bev_eventcb_connected_src(UNUSED struct bufferevent *bev, UNUSED pxy_conn_ctx_t *ctx)
 {
@@ -719,8 +775,12 @@ protoautossl_setup(pxy_conn_ctx_t *ctx)
 
 	ctx->protoctx->set_watermarkcb = protoautossl_try_set_watermark;
 	ctx->protoctx->unset_watermarkcb = protoautossl_try_unset_watermark;
-
-	ctx->protoctx->disable_srvdstcb = protoautossl_disable_srvdst;
+	ctx->protoctx->discard_inbufcb = protoautossl_try_discard_inbuf;
+	ctx->protoctx->discard_outbufcb = protoautossl_try_discard_outbuf;
+	ctx->protoctx->outbuf_has_datacb = protoautossl_outbuf_has_data;
+#ifdef DEBUG_PROXY
+	ctx->protoctx->log_dbg_evbuf_infocb = protoautossl_log_dbg_evbuf_info;
+#endif /* DEBUG_PROXY */
 
 	ctx->protoctx->arg = malloc(sizeof(protoautossl_ctx_t));
 	if (!ctx->protoctx->arg) {
@@ -748,9 +808,6 @@ protoautossl_setup_child(pxy_conn_child_ctx_t *ctx)
 	ctx->protoctx->bev_writecb = prototcp_bev_writecb_child;
 	ctx->protoctx->bev_eventcb = protoautossl_bev_eventcb_child;
 
-	ctx->protoctx->set_watermarkcb = protoautossl_try_set_watermark;
-	ctx->protoctx->unset_watermarkcb = protoautossl_try_unset_watermark;
-
 	return PROTO_AUTOSSL;
 }
 

src/protohttp.c

@@ -252,10 +252,10 @@ protohttp_ocsp_deny(pxy_conn_ctx_t *ctx, protohttp_ctx_t *http_ctx)
 	return;
 
 deny:
-	pxy_try_discard_inbuf(ctx->src.bev);
+	ctx->protoctx->discard_inbufcb(ctx->src.bev);
 
 	// Do not send anything to the child conns
-	pxy_try_discard_outbuf(ctx->dst.bev);
+	ctx->protoctx->discard_outbufcb(ctx->dst.bev);
 
 	// Do not send duplicate OCSP denied responses
 	if (http_ctx->ocsp_denied)
@@ -792,7 +792,7 @@ protohttp_bev_readcb_src(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 	log_finest_va("ENTER, size=%zu", evbuffer_get_length(bufferevent_get_input(bev)));
 
 	if (ctx->dst.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -804,7 +804,7 @@ protohttp_bev_readcb_src(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 	if (ctx->conn_opts->user_auth && !ctx->user) {
 		log_finest("Redirecting conn");
 		char *url = protohttp_get_url(inbuf, ctx);
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		if (url) {
 			evbuffer_add_printf(bufferevent_get_output(bev), redirect_url, ctx->conn_opts->user_auth_url, url);
 			free(url);
@@ -843,8 +843,8 @@ protohttp_bev_readcb_src(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 		if (protohttp_validate(ctx) == -1) {
 			evbuffer_add(bufferevent_get_output(bev), proto_error, strlen(proto_error));
 			ctx->sent_protoerror_msg = 1;
-			pxy_try_discard_inbuf(bev);
-			pxy_try_discard_outbuf(ctx->dst.bev);
+			ctx->protoctx->discard_inbufcb(bev);
+			ctx->protoctx->discard_outbufcb(ctx->dst.bev);
 			return;
 		}
 	}
@@ -974,7 +974,7 @@ protohttp_bev_readcb_dst(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 	log_finest_va("ENTER, size=%zu", evbuffer_get_length(bufferevent_get_input(bev)));
 
 	if (ctx->src.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -1007,7 +1007,7 @@ protohttp_bev_readcb_src_child(struct bufferevent *bev, pxy_conn_child_ctx_t *ct
 	log_finest_va("ENTER, size=%zu", evbuffer_get_length(bufferevent_get_input(bev)));
 
 	if (ctx->dst.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->conn->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -1025,7 +1025,7 @@ protohttp_bev_readcb_src_child(struct bufferevent *bev, pxy_conn_child_ctx_t *ct
 		log_finest_va("HTTP Request Body, size=%zu", evbuffer_get_length(inbuf));
 		evbuffer_add_buffer(outbuf, inbuf);
 	}
-	ctx->protoctx->set_watermarkcb(bev, ctx->conn, ctx->dst.bev);
+	ctx->conn->protoctx->set_watermarkcb(bev, ctx->conn, ctx->dst.bev);
 }
 
 static void NONNULL(1)
@@ -1034,7 +1034,7 @@ protohttp_bev_readcb_dst_child(struct bufferevent *bev, pxy_conn_child_ctx_t *ct
 	log_finest_va("ENTER, size=%zu", evbuffer_get_length(bufferevent_get_input(bev)));
 		
 	if (ctx->src.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->conn->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -1053,7 +1053,7 @@ protohttp_bev_readcb_dst_child(struct bufferevent *bev, pxy_conn_child_ctx_t *ct
 		log_finest_va("HTTP Response Body, size=%zu", evbuffer_get_length(inbuf));
 		evbuffer_add_buffer(outbuf, inbuf);
 	}
-	ctx->protoctx->set_watermarkcb(bev, ctx->conn, ctx->src.bev);
+	ctx->conn->protoctx->set_watermarkcb(bev, ctx->conn, ctx->src.bev);
 }
 
 static void NONNULL(1)

src/protopassthrough.c

@@ -128,7 +128,7 @@ protopassthrough_bev_readcb_src(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 
 	// Passthrough packets are transferred between src and srvdst
 	if (ctx->srvdst.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -153,7 +153,7 @@ protopassthrough_bev_readcb_srvdst(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 
 	// Passthrough packets are transferred between src and srvdst
 	if (ctx->src.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -268,7 +268,7 @@ protopassthrough_bev_eventcb_eof_src(struct bufferevent *bev, pxy_conn_ctx_t *ct
 {
 #ifdef DEBUG_PROXY
 	log_finest("ENTER");
-	pxy_log_dbg_evbuf_info(ctx, &ctx->src, &ctx->srvdst);
+	ctx->protoctx->log_dbg_evbuf_infocb(ctx, &ctx->src, &ctx->srvdst);
 #endif /* DEBUG_PROXY */
 
 	if (!ctx->connected) {
@@ -290,7 +290,7 @@ protopassthrough_bev_eventcb_eof_srvdst(struct bufferevent *bev, pxy_conn_ctx_t
 {
 #ifdef DEBUG_PROXY
 	log_finest("ENTER");
-	pxy_log_dbg_evbuf_info(ctx, &ctx->srvdst, &ctx->src);
+	ctx->protoctx->log_dbg_evbuf_infocb(ctx, &ctx->srvdst, &ctx->src);
 #endif /* DEBUG_PROXY */
 
 	if (!ctx->connected) {

src/protosmtp.c

@@ -134,10 +134,10 @@ protosmtp_try_validate_response(struct bufferevent *bev, pxy_conn_ctx_t *ctx, st
 			ctx->sent_protoerror_msg = 1;
 
 			// Discard packets from the client: inbuf of src
-			pxy_try_discard_inbuf(ctx->src.bev);
+			ctx->protoctx->discard_inbufcb(ctx->src.bev);
 
 			// Discard packets to the server: outbuf of srvdst
-			pxy_try_discard_outbuf(bev);
+			ctx->protoctx->discard_outbufcb(bev);
 
 			free(packet);
 			return -1;
@@ -207,7 +207,7 @@ protosmtp_bev_readcb_srvdst(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 	}
 
 	if (ctx->src.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		return;
 	}
 

src/protossl.c

@@ -1520,7 +1520,7 @@ protossl_connect_child(pxy_conn_child_ctx_t *ctx)
 		ctx->dst = ctx->conn->srvdst;
 
 		// See the comments in prototcp_setup_dst()
-		ctx->conn->protoctx->disable_srvdstcb(ctx->conn);
+		prototcp_disable_srvdst(ctx->conn);
 
 		bufferevent_setcb(ctx->dst.bev, pxy_bev_readcb_child, pxy_bev_writecb_child, pxy_bev_eventcb_child, ctx);
 		ctx->protoctx->bev_eventcb(ctx->dst.bev, BEV_EVENT_CONNECTED, ctx);

src/prototcp.c

@@ -33,6 +33,21 @@
 #include <sys/param.h>
 #include <string.h>
 
+#ifdef DEBUG_PROXY
+void
+prototcp_log_dbg_evbuf_info(pxy_conn_ctx_t *ctx, pxy_conn_desc_t *this, pxy_conn_desc_t *other)
+{
+	// This function is used by child conns too, they pass ctx->conn instead of ctx
+	if (OPTS_DEBUG(ctx->global)) {
+		log_dbg_printf("evbuffer size at EOF: i:%zu o:%zu i:%zu o:%zu\n",
+						evbuffer_get_length(bufferevent_get_input(this->bev)),
+						evbuffer_get_length(bufferevent_get_output(this->bev)),
+						other->closed ? 0 : evbuffer_get_length(bufferevent_get_input(other->bev)),
+						other->closed ? 0 : evbuffer_get_length(bufferevent_get_output(other->bev)));
+	}
+}
+#endif /* DEBUG_PROXY */
+
 /*
  * Set up a bufferevent structure for either a dst or src connection,
  * optionally with or without SSL.  Sets all callbacks, enables read
@@ -111,19 +126,15 @@ prototcp_setup_src(pxy_conn_ctx_t *ctx)
 	return 0;
 }
 
-static void NONNULL(1)
+void
 prototcp_disable_srvdst(pxy_conn_ctx_t *ctx)
 {
 	log_finest("ENTER");
 
+	// Do not disable underlying bevs in autossl
 	bufferevent_setcb(ctx->srvdst.bev, NULL, NULL, NULL, NULL);
 	bufferevent_disable(ctx->srvdst.bev, EV_READ|EV_WRITE);
 
-	struct bufferevent *ubev = bufferevent_get_underlying(ctx->srvdst.bev);
-	if (ubev) {
-		bufferevent_setcb(ubev, NULL, NULL, NULL, NULL);
-		bufferevent_disable(ubev, EV_READ|EV_WRITE);
-	}
 	// Do not access srvdst.bev from this point on
 	ctx->srvdst.bev = NULL;
 }
@@ -153,7 +164,7 @@ prototcp_setup_dst(pxy_conn_ctx_t *ctx)
 		// This seems to be an issue with libevent.
 		// @todo Why does libevent raise the same event again for an already disabled and freed conn end?
 		// Note again that srvdst == dst or child_dst here.
-		ctx->protoctx->disable_srvdstcb(ctx);
+		prototcp_disable_srvdst(ctx);
 
 		bufferevent_setcb(ctx->dst.bev, pxy_bev_readcb, pxy_bev_writecb, pxy_bev_eventcb, ctx);
 		ctx->protoctx->bev_eventcb(ctx->dst.bev, BEV_EVENT_CONNECTED, ctx);
@@ -215,7 +226,7 @@ prototcp_connect_child(pxy_conn_child_ctx_t *ctx)
 		ctx->dst = ctx->conn->srvdst;
 
 		// See the comments in prototcp_setup_dst()
-		ctx->conn->protoctx->disable_srvdstcb(ctx->conn);
+		prototcp_disable_srvdst(ctx->conn);
 
 		bufferevent_setcb(ctx->dst.bev, pxy_bev_readcb_child, pxy_bev_writecb_child, pxy_bev_eventcb_child, ctx);
 		ctx->protoctx->bev_eventcb(ctx->dst.bev, BEV_EVENT_CONNECTED, ctx);
@@ -307,13 +318,35 @@ prototcp_try_unset_watermark(struct bufferevent *bev, pxy_conn_ctx_t *ctx, pxy_c
 	}
 }
 
+void
+prototcp_try_discard_inbuf(struct bufferevent *bev)
+{
+	struct evbuffer *inbuf = bufferevent_get_input(bev);
+	size_t inbuf_size = evbuffer_get_length(inbuf);
+	if (inbuf_size) {
+		log_dbg_printf("Warning: Drained %zu bytes from inbuf\n", inbuf_size);
+		evbuffer_drain(inbuf, inbuf_size);
+	}
+}
+
+void
+prototcp_try_discard_outbuf(struct bufferevent *bev)
+{
+	struct evbuffer *outbuf = bufferevent_get_output(bev);
+	size_t outbuf_size = evbuffer_get_length(outbuf);
+	if (outbuf_size) {
+		log_dbg_printf("Warning: Drained %zu bytes from outbuf\n", outbuf_size);
+		evbuffer_drain(outbuf, outbuf_size);
+	}
+}
+
 #ifndef WITHOUT_USERAUTH
 int
 prototcp_try_send_userauth_msg(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 {
 	if (ctx->conn_opts->user_auth && !ctx->user) {
 		log_finest("Sending userauth message");
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		evbuffer_add_printf(bufferevent_get_output(bev), USERAUTH_MSG, ctx->conn_opts->user_auth_url);
 		ctx->sent_userauth_msg = 1;
 		return 1;
@@ -341,10 +374,10 @@ prototcp_try_validate_proto(struct bufferevent *bev, pxy_conn_ctx_t *ctx, struct
 			ctx->sent_protoerror_msg = 1;
 
 			// Discard packets from the client: inbuf of src
-			pxy_try_discard_inbuf(bev);
+			ctx->protoctx->discard_inbufcb(bev);
 
 			// Discard packets to the server: outbuf of dst
-			pxy_try_discard_outbuf(other);
+			ctx->protoctx->discard_outbufcb(other);
 
 			free(packet);
 			return 1;
@@ -360,7 +393,7 @@ prototcp_bev_readcb_src(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 	log_finest_va("ENTER, size=%zu", evbuffer_get_length(bufferevent_get_input(bev)));
 
 	if (ctx->dst.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -393,7 +426,7 @@ prototcp_bev_readcb_dst(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 	log_finest_va("ENTER, size=%zu", evbuffer_get_length(bufferevent_get_input(bev)));
 
 	if (ctx->src.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -413,7 +446,7 @@ prototcp_bev_readcb_src_child(struct bufferevent *bev, pxy_conn_child_ctx_t *ctx
 	log_finest_va("ENTER, size=%zu", evbuffer_get_length(bufferevent_get_input(bev)));
 
 	if (ctx->dst.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->conn->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
@@ -437,7 +470,7 @@ prototcp_bev_readcb_src_child(struct bufferevent *bev, pxy_conn_child_ctx_t *ctx
 	} else {
 		evbuffer_add_buffer(outbuf, inbuf);
 	}
-	ctx->protoctx->set_watermarkcb(bev, ctx->conn, ctx->dst.bev);
+	ctx->conn->protoctx->set_watermarkcb(bev, ctx->conn, ctx->dst.bev);
 }
 
 static void NONNULL(1)
@@ -446,12 +479,27 @@ prototcp_bev_readcb_dst_child(struct bufferevent *bev, pxy_conn_child_ctx_t *ctx
 	log_finest_va("ENTER, size=%zu", evbuffer_get_length(bufferevent_get_input(bev)));
 
 	if (ctx->src.closed) {
-		pxy_try_discard_inbuf(bev);
+		ctx->conn->protoctx->discard_inbufcb(bev);
 		return;
 	}
 
 	evbuffer_add_buffer(bufferevent_get_output(ctx->src.bev), bufferevent_get_input(bev));
-	ctx->protoctx->set_watermarkcb(bev, ctx->conn, ctx->src.bev);
+	ctx->conn->protoctx->set_watermarkcb(bev, ctx->conn, ctx->src.bev);
+}
+
+static int NONNULL(1) WUNRES
+prototcp_outbuf_has_data(struct bufferevent *bev
+#ifdef DEBUG_PROXY
+	, char *reason, pxy_conn_ctx_t *ctx
+#endif /* DEBUG_PROXY */
+	)
+{
+	size_t outbuflen = evbuffer_get_length(bufferevent_get_output(bev));
+	if (outbuflen) {
+		log_finest_va("Not closing %s, outbuflen=%zu", reason, outbuflen);
+		return 1;
+	}
+	return 0;
 }
 
 #ifndef WITHOUT_USERAUTH
@@ -459,11 +507,12 @@ int
 prototcp_try_close_unauth_conn(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 {
 	if (ctx->conn_opts->user_auth && !ctx->user) {
-		size_t outbuflen = evbuffer_get_length(bufferevent_get_output(bev));
-		struct bufferevent *ubev = bufferevent_get_underlying(bev);
-		if (outbuflen || (ubev && evbuffer_get_length(bufferevent_get_output(ubev)))) {
-			log_finest_va("Not closing unauth conn, outbuflen=%zu, ubev outbuflen=%zu",
-					outbuflen, ubev ? evbuffer_get_length(bufferevent_get_output(ubev)) : 0);
+		if (ctx->protoctx->outbuf_has_datacb(bev
+#ifdef DEBUG_PROXY
+			, "unauth conn", ctx
+#endif /* DEBUG_PROXY */
+			)) {
+			// Nothing to do
 		} else if (ctx->sent_userauth_msg) {
 			log_finest("Closing unauth conn");
 			pxy_conn_term(ctx, 1);
@@ -480,11 +529,12 @@ int
 prototcp_try_close_protoerror_conn(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 {
 	if (ctx->conn_opts->validate_proto && ctx->sent_protoerror_msg) {
-		size_t outbuflen = evbuffer_get_length(bufferevent_get_output(bev));
-		struct bufferevent *ubev = bufferevent_get_underlying(bev);
-		if (outbuflen || (ubev && evbuffer_get_length(bufferevent_get_output(ubev)))) {
-			log_finest_va("Not closing protoerror conn, outbuflen=%zu, ubev outbuflen=%zu",
-					outbuflen, ubev ? evbuffer_get_length(bufferevent_get_output(ubev)) : 0);
+		if (ctx->protoctx->outbuf_has_datacb(bev
+#ifdef DEBUG_PROXY
+			, "protoerror conn", ctx
+#endif /* DEBUG_PROXY */
+			)) {
+			// Nothing to do
 		} else {
 			log_finest("Closing protoerror conn");
 			pxy_conn_term(ctx, 1);
@@ -546,7 +596,7 @@ prototcp_bev_writecb_src_child(struct bufferevent *bev, pxy_conn_child_ctx_t *ct
 		}
 		return;
 	}
-	ctx->protoctx->unset_watermarkcb(bev, ctx->conn, &ctx->dst);
+	ctx->conn->protoctx->unset_watermarkcb(bev, ctx->conn, &ctx->dst);
 }
 
 static void NONNULL(1)
@@ -561,7 +611,7 @@ prototcp_bev_writecb_dst_child(struct bufferevent *bev, pxy_conn_child_ctx_t *ct
 		}
 		return;
 	}
-	ctx->protoctx->unset_watermarkcb(bev, ctx->conn, &ctx->src);
+	ctx->conn->protoctx->unset_watermarkcb(bev, ctx->conn, &ctx->src);
 }
 
 int
@@ -636,7 +686,7 @@ prototcp_bev_eventcb_eof_src(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 {
 #ifdef DEBUG_PROXY
 	log_finest("ENTER");
-	pxy_log_dbg_evbuf_info(ctx, &ctx->src, &ctx->dst);
+	ctx->protoctx->log_dbg_evbuf_infocb(ctx, &ctx->src, &ctx->dst);
 #endif /* DEBUG_PROXY */
 
 	if (!ctx->connected) {
@@ -658,7 +708,7 @@ prototcp_bev_eventcb_eof_dst(struct bufferevent *bev, pxy_conn_ctx_t *ctx)
 {
 #ifdef DEBUG_PROXY
 	log_finest("ENTER");
-	pxy_log_dbg_evbuf_info(ctx, &ctx->dst, &ctx->src);
+	ctx->protoctx->log_dbg_evbuf_infocb(ctx, &ctx->dst, &ctx->src);
 #endif /* DEBUG_PROXY */
 
 	if (!ctx->connected) {
@@ -742,7 +792,7 @@ prototcp_bev_eventcb_eof_src_child(struct bufferevent *bev, pxy_conn_child_ctx_t
 {
 #ifdef DEBUG_PROXY
 	log_finest("ENTER");
-	pxy_log_dbg_evbuf_info(ctx->conn, &ctx->src, &ctx->dst);
+	ctx->conn->protoctx->log_dbg_evbuf_infocb(ctx->conn, &ctx->src, &ctx->dst);
 #endif /* DEBUG_PROXY */
 
 	// @todo How to handle the following case?
@@ -765,7 +815,7 @@ prototcp_bev_eventcb_eof_dst_child(struct bufferevent *bev, pxy_conn_child_ctx_t
 {
 #ifdef DEBUG_PROXY
 	log_finest("ENTER");
-	pxy_log_dbg_evbuf_info(ctx->conn, &ctx->dst, &ctx->src);
+	ctx->conn->protoctx->log_dbg_evbuf_infocb(ctx->conn, &ctx->dst, &ctx->src);
 #endif /* DEBUG_PROXY */
 
 	// @todo How to handle the following case?
@@ -989,8 +1039,13 @@ prototcp_setup(pxy_conn_ctx_t *ctx)
 
 	ctx->protoctx->set_watermarkcb = prototcp_try_set_watermark;
 	ctx->protoctx->unset_watermarkcb = prototcp_try_unset_watermark;
+	ctx->protoctx->discard_inbufcb = prototcp_try_discard_inbuf;
+	ctx->protoctx->discard_outbufcb = prototcp_try_discard_outbuf;
+	ctx->protoctx->outbuf_has_datacb = prototcp_outbuf_has_data;
+#ifdef DEBUG_PROXY
+	ctx->protoctx->log_dbg_evbuf_infocb = prototcp_log_dbg_evbuf_info;
+#endif /* DEBUG_PROXY */
 
-	ctx->protoctx->disable_srvdstcb = prototcp_disable_srvdst;
 	return PROTO_TCP;
 }
 
@@ -1004,9 +1059,6 @@ prototcp_setup_child(pxy_conn_child_ctx_t *ctx)
 	ctx->protoctx->bev_writecb = prototcp_bev_writecb_child;
 	ctx->protoctx->bev_eventcb = prototcp_bev_eventcb_child;
 
-	ctx->protoctx->set_watermarkcb = prototcp_try_set_watermark;
-	ctx->protoctx->unset_watermarkcb = prototcp_try_unset_watermark;
-
 	return PROTO_TCP;
 }
 

src/prototcp.h

@@ -38,6 +38,10 @@
  */
 #define OUTBUF_LIMIT	(128*1024)
 
+#ifdef DEBUG_PROXY
+void prototcp_log_dbg_evbuf_info(pxy_conn_ctx_t *, pxy_conn_desc_t *, pxy_conn_desc_t *) NONNULL(1,2,3);
+#endif /* DEBUG_PROXY */
+
 void prototcp_init_conn(evutil_socket_t, short, void *);
 
 #ifdef DEBUG_PROXY
@@ -46,6 +50,9 @@ char *prototcp_get_event_name(struct bufferevent *, pxy_conn_ctx_t *) NONNULL(2)
 void prototcp_try_set_watermark(struct bufferevent *, pxy_conn_ctx_t *, struct bufferevent *) NONNULL(1,2,3);
 void prototcp_try_unset_watermark(struct bufferevent *, pxy_conn_ctx_t *, pxy_conn_desc_t *) NONNULL(1,2,3);
 
+void prototcp_try_discard_inbuf(struct bufferevent *) NONNULL(1);
+void prototcp_try_discard_outbuf(struct bufferevent *) NONNULL(1);
+
 #ifndef WITHOUT_USERAUTH
 int prototcp_try_send_userauth_msg(struct bufferevent *, pxy_conn_ctx_t *) NONNULL(1,2);
 int prototcp_try_close_unauth_conn(struct bufferevent *, pxy_conn_ctx_t *) NONNULL(1,2);
@@ -82,6 +89,7 @@ int prototcp_enable_src(pxy_conn_ctx_t *) NONNULL(1);
 void prototcp_bev_eventcb_srvdst(struct bufferevent *, short, pxy_conn_ctx_t *) NONNULL(1);
 
 int prototcp_setup_src(pxy_conn_ctx_t *) NONNULL(1);
+void prototcp_disable_srvdst(pxy_conn_ctx_t *) NONNULL(1);
 int prototcp_setup_dst(pxy_conn_ctx_t *) NONNULL(1);
 int prototcp_setup_srvdst(pxy_conn_ctx_t *) NONNULL(1);
 

src/pxyconn.c

@@ -753,30 +753,6 @@ pxy_log_dbg_disconnect_child(pxy_conn_child_ctx_t *ctx)
 	}
 }
 
-#ifdef DEBUG_PROXY
-void
-pxy_log_dbg_evbuf_info(pxy_conn_ctx_t *ctx, pxy_conn_desc_t *this, pxy_conn_desc_t *other)
-{
-	// This function is used by child conns too, they pass ctx->conn instead of ctx
-	if (OPTS_DEBUG(ctx->global)) {
-		log_dbg_printf("evbuffer size at EOF: i:%zu o:%zu i:%zu o:%zu\n",
-						evbuffer_get_length(bufferevent_get_input(this->bev)),
-						evbuffer_get_length(bufferevent_get_output(this->bev)),
-						other->closed ? 0 : evbuffer_get_length(bufferevent_get_input(other->bev)),
-						other->closed ? 0 : evbuffer_get_length(bufferevent_get_output(other->bev)));
-
-		struct bufferevent *ubev = bufferevent_get_underlying(this->bev);
-		struct bufferevent *ubev_other = other->closed ? NULL : bufferevent_get_underlying(other->bev);
-		if (ubev || ubev_other)
-			log_dbg_printf("underlying evbuffer size at EOF: i:%zu o:%zu i:%zu o:%zu\n",
-							ubev ? evbuffer_get_length(bufferevent_get_input(ubev)) : 0,
-							ubev ? evbuffer_get_length(bufferevent_get_output(ubev)) : 0,
-							ubev_other ? evbuffer_get_length(bufferevent_get_input(ubev_other)) : 0,
-							ubev_other ? evbuffer_get_length(bufferevent_get_output(ubev_other)) : 0);
-	}
-}
-#endif /* DEBUG_PROXY */
-
 unsigned char *
 pxy_malloc_packet(size_t sz, pxy_conn_ctx_t *ctx)
 {
@@ -788,48 +764,6 @@ pxy_malloc_packet(size_t sz, pxy_conn_ctx_t *ctx)
 	return packet;
 }
 
-void
-pxy_try_discard_inbuf(struct bufferevent *bev)
-{
-	struct evbuffer *inbuf = bufferevent_get_input(bev);
-	size_t inbuf_size = evbuffer_get_length(inbuf);
-	if (inbuf_size) {
-		log_dbg_printf("Warning: Drained %zu bytes from inbuf\n", inbuf_size);
-		evbuffer_drain(inbuf, inbuf_size);
-	}
-
-	struct bufferevent *ubev = bufferevent_get_underlying(bev);
-	if (ubev) {
-		struct evbuffer *ubev_inbuf = bufferevent_get_input(ubev);
-		size_t ubev_inbuf_size = evbuffer_get_length(ubev_inbuf);
-		if (ubev_inbuf_size) {
-			log_dbg_printf("Warning: Drained %zu bytes from inbuf underlying\n", ubev_inbuf_size);
-			evbuffer_drain(ubev_inbuf, ubev_inbuf_size);
-		}
-	}
-}
-
-void
-pxy_try_discard_outbuf(struct bufferevent *bev)
-{
-	struct evbuffer *outbuf = bufferevent_get_output(bev);
-	size_t outbuf_size = evbuffer_get_length(outbuf);
-	if (outbuf_size) {
-		log_dbg_printf("Warning: Drained %zu bytes from outbuf\n", outbuf_size);
-		evbuffer_drain(outbuf, outbuf_size);
-	}
-
-	struct bufferevent *ubev = bufferevent_get_underlying(bev);
-	if (ubev) {
-		struct evbuffer *ubev_outbuf = bufferevent_get_output(ubev);
-		size_t ubev_outbuf_size = evbuffer_get_length(ubev_outbuf);
-		if (ubev_outbuf_size) {
-			log_dbg_printf("Warning: Drained %zu bytes from outbuf underlying\n", ubev_outbuf_size);
-			evbuffer_drain(ubev_outbuf, ubev_outbuf_size);
-		}
-	}
-}
-
 #ifdef DEBUG_PROXY
 static void
 pxy_insert_sslproxy_header(pxy_conn_ctx_t *ctx, unsigned char *packet, size_t *packet_size)
@@ -1270,10 +1204,12 @@ pxy_try_close_conn_end(pxy_conn_desc_t *conn_end, pxy_conn_ctx_t *ctx)
 	/* if the other end is still open and doesn't have data
 	 * to send, close it, otherwise its writecb will close
 	 * it after writing what's left in the output buffer */
-	struct bufferevent *ubev = bufferevent_get_underlying(conn_end->bev);
-	if (evbuffer_get_length(bufferevent_get_output(conn_end->bev)) == 0 &&
-			(!ubev || evbuffer_get_length(bufferevent_get_output(ubev)) == 0)) {
-		log_finest("evbuffer_get_length(outbuf) == 0, terminate conn");
+	if (!ctx->protoctx->outbuf_has_datacb(conn_end->bev
+#ifdef DEBUG_PROXY
+		, "conn", ctx
+#endif /* DEBUG_PROXY */
+		)) {
+		log_finest("outbuflen == 0, terminate conn");
 		conn_end->free(conn_end->bev, ctx);
 		conn_end->bev = NULL;
 		conn_end->closed = 1;

src/pxyconn.h

@@ -86,8 +86,16 @@ typedef void (*child_proto_free_func_t)(pxy_conn_child_ctx_t *);
 
 typedef void (*set_watermark_func_t)(struct bufferevent *, pxy_conn_ctx_t *, struct bufferevent *);
 typedef void (*unset_watermark_func_t)(struct bufferevent *, pxy_conn_ctx_t *, pxy_conn_desc_t *);
-
-typedef void (*disable_srvdstcb)(pxy_conn_ctx_t *);
+typedef void (*discard_inbuf_func_t)(struct bufferevent *) NONNULL(1);
+typedef void (*discard_outbuf_func_t)(struct bufferevent *) NONNULL(1);
+typedef int (*outbuf_has_data_func_t)(struct bufferevent *
+#ifdef DEBUG_PROXY
+	, char *, pxy_conn_ctx_t *
+#endif /* DEBUG_PROXY */
+	) NONNULL(1) WUNRES;
+#ifdef DEBUG_PROXY
+typedef void (*log_dbg_evbuf_info_func_t)(pxy_conn_ctx_t *, pxy_conn_desc_t *, pxy_conn_desc_t *) NONNULL(1,2,3);
+#endif /* DEBUG_PROXY */
 
 typedef filter_action_t * (*proto_filter_func_t)(pxy_conn_ctx_t *, filter_list_t *) NONNULL(1,2) WUNRES;
 
@@ -173,10 +181,15 @@ struct proto_ctx {
 	proto_classify_user_func_t classify_usercb;
 #endif /* !WITHOUT_USERAUTH */
 
+	// The following callback functions are for decoupling autossl code handling underlying bufs
 	set_watermark_func_t set_watermarkcb;
 	unset_watermark_func_t unset_watermarkcb;
-
-	disable_srvdstcb disable_srvdstcb;
+	discard_inbuf_func_t discard_inbufcb;
+	discard_outbuf_func_t discard_outbufcb;
+	outbuf_has_data_func_t outbuf_has_datacb;
+#ifdef DEBUG_PROXY
+	log_dbg_evbuf_info_func_t log_dbg_evbuf_infocb;
+#endif /* DEBUG_PROXY */
 
 	// For protocol specific fields, if any
 	void *arg;
@@ -193,9 +206,6 @@ struct proto_child_ctx {
 
 	child_proto_free_func_t proto_free;
 
-	set_watermark_func_t set_watermarkcb;
-	unset_watermark_func_t unset_watermarkcb;
-
 	// For protocol specific fields, if any
 	void *arg;
 };
@@ -412,9 +422,7 @@ int pxy_prepare_logging_local_procinfo(pxy_conn_ctx_t *) NONNULL(1);
 
 void pxy_log_connect_src(pxy_conn_ctx_t *) NONNULL(1);
 void pxy_log_connect_srvdst(pxy_conn_ctx_t *) NONNULL(1);
-
 void pxy_log_connect_nonhttp(pxy_conn_ctx_t *) NONNULL(1);
-void pxy_log_dbg_evbuf_info(pxy_conn_ctx_t *, pxy_conn_desc_t *, pxy_conn_desc_t *) NONNULL(1,2,3);
 
 unsigned char *pxy_malloc_packet(size_t, pxy_conn_ctx_t *) MALLOC NONNULL(2) WUNRES;
 
@@ -428,8 +436,6 @@ void pxy_try_disconnect_child(pxy_conn_child_ctx_t *, pxy_conn_desc_t *, pxy_con
 
 int pxy_try_consume_last_input(struct bufferevent *, pxy_conn_ctx_t *) NONNULL(1,2);
 int pxy_try_consume_last_input_child(struct bufferevent *, pxy_conn_child_ctx_t *) NONNULL(1,2);
-void pxy_try_discard_inbuf(struct bufferevent *) NONNULL(1);
-void pxy_try_discard_outbuf(struct bufferevent *) NONNULL(1);
 
 int pxy_conn_init(pxy_conn_ctx_t *) NONNULL(1);
 void pxy_conn_ctx_free(pxy_conn_ctx_t *, int) NONNULL(1);