From b74b0e00a0d61be18c21b129c417da421c524d3d Mon Sep 17 00:00:00 2001 From: Revertron Date: Sun, 3 Dec 2023 02:11:40 +0100 Subject: [PATCH] Added a debug system to catch some deadlock. --- Cargo.lock | 101 +++++++++++++++++++++++++++++++++++---------- Cargo.toml | 6 +-- src/p2p/network.rs | 28 ++++++++++++- 3 files changed, 110 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 803c143..9968cf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -775,9 +775,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.132" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "log" @@ -821,14 +821,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.5" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -1085,6 +1085,12 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + [[package]] name = "ryu" version = "1.0.5" @@ -1363,13 +1369,15 @@ dependencies = [ [[package]] name = "thread-priority" -version = "0.10.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978519ae4c6891352f964b88da4ab5a3a9b74a40247cda5baee145dae6cd3b71" +checksum = "0c56ce92f1285eaaa11fc1a3201e25de97898c50e87caa4c2aee836fe05288de" dependencies = [ + "bitflags", "cfg-if", "libc", "log", + "rustversion", "winapi", ] @@ -1802,26 +1810,20 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.42.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.2", + "windows-targets 0.42.2", ] [[package]] name = "windows-sys" -version = "0.45.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -1830,21 +1832,42 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm", + "windows_aarch64_gnullvm 0.42.2", "windows_aarch64_msvc 0.42.2", "windows_i686_gnu 0.42.2", "windows_i686_msvc 0.42.2", "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm", + "windows_x86_64_gnullvm 0.42.2", "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" @@ -1857,6 +1880,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_i686_gnu" version = "0.36.1" @@ -1869,6 +1898,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_msvc" version = "0.36.1" @@ -1881,6 +1916,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" @@ -1893,12 +1934,24 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" @@ -1911,6 +1964,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "winnow" version = "0.3.3" diff --git a/Cargo.toml b/Cargo.toml index da4d00f..ad5d942 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ rand = { package = "rand", version = "0.8.5" } rand-old = { package = "rand", version = "0.7.0" } # For ed25519-dalek sqlite = "0.30.4" uuid = { version = "1.3.0", features = ["serde", "v4"] } -mio = { version = "0.8.5", features = ["os-poll", "net"] } +mio = { version = "0.8.9", features = ["os-poll", "net"] } ureq = { version = "2.5", optional = true } lru = "0.9.0" derive_more = "0.99.17" @@ -53,10 +53,10 @@ open = { version = "3.0.3", optional = true } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["impl-default", "wincon", "shellscalingapi"] } windows-service = "0.6.0" -thread-priority = "0.10.0" +thread-priority = "0.13.1" [target.'cfg(any(target_os = "linux", target_os = "freebsd", target_os = "openbsd", target_os = "netbsd"))'.dependencies] -thread-priority = "0.10.0" +thread-priority = "0.13.1" [build-dependencies] winres = "0.1.12" diff --git a/src/p2p/network.rs b/src/p2p/network.rs index cc34111..5e90c82 100644 --- a/src/p2p/network.rs +++ b/src/p2p/network.rs @@ -6,7 +6,7 @@ use std::collections::{HashMap, HashSet}; use std::io::{Error, ErrorKind, Read, Write}; use std::net::{IpAddr, Shutdown, SocketAddr, SocketAddrV4, ToSocketAddrs}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, mpsc, Mutex}; use std::time::{Duration, Instant}; use std::{io, thread}; @@ -84,7 +84,27 @@ impl Network { let mut old_nodes = 0usize; let mut old_banned = 0usize; let mut seen_blocks = HashSet::new(); + + let (debug_send, debug_receive) = mpsc::channel(); + let _debug_thread = thread::spawn(move || { + let mut timer = Instant::now(); + let mut log = String::new(); + loop { + if let Ok(line) = debug_receive.try_recv() { + timer = Instant::now(); + log = line; + } else { + if timer.elapsed().as_secs() >= 60 { + timer = Instant::now(); + warn!("Stuck in '{log}'"); + } + thread::sleep(Duration::from_secs(1)); + } + } + }); + loop { + let _ = debug_send.send(String::from("Restart swarm")); if self.peers.get_peers_count() == 0 && bootstrap_timer.elapsed().as_secs() > 60 { warn!("Restarting swarm connections..."); wait_for_internet(WAIT_FOR_INTERNET); @@ -93,6 +113,7 @@ impl Network { bootstrap_timer = Instant::now(); last_events_time = Instant::now(); } + let _ = debug_send.send(String::from("Poll events")); // Poll Mio for events, blocking until we get an event. poll.poll(&mut events, POLL_TIMEOUT) .unwrap_or_else(|e| warn!("Error polling sockets: {}", e)); @@ -106,6 +127,7 @@ impl Network { // We can use the token we previously provided to `register` to determine for which socket the event is. match event.token() { SERVER => { + let _ = debug_send.send(String::from("Server accept")); //debug!("Event for server socket {} is {:?}", event.token().0, &event); // If this is an event for the server, it means a connection is ready to be accepted. while let Ok((mut stream, mut address)) = server.accept() { @@ -147,6 +169,7 @@ impl Network { } } token => { + let _ = debug_send.send(String::from("Handle connection event")); if !self.handle_connection_event(poll.registry(), event, &mut seen_blocks, &mut buffer) { let _ = self.peers.close_peer(poll.registry(), &token); let blocks = self.context.lock().unwrap().chain.get_height(); @@ -157,6 +180,7 @@ impl Network { } } } + let _ = debug_send.send(String::from("After events iter")); if last_events_time.elapsed().as_secs() > MAX_IDLE_SECONDS { if self.peers.get_peers_count() > 0 { warn!("Something is wrong with swarm connections, closing all."); @@ -169,6 +193,7 @@ impl Network { last_events_time = Instant::now(); } + let _ = debug_send.send(String::from("UI Timer")); if ui_timer.elapsed().as_millis() > UI_REFRESH_DELAY_MS { // Send pings to idle peers let (height, max_height, hash) = { @@ -210,6 +235,7 @@ impl Network { (blocks, max_height, context.chain.get_last_hash()) }; + let _ = debug_send.send(String::from("Peers update")); let have_blocks: HashSet = self.future_blocks.values().map(|block| block.index).collect(); self.peers.update(poll.registry(), hash, height, max_height, have_blocks); ui_timer = Instant::now();