Archives
/
distant
mirror of https://github.com/chipsenkbeil/distant
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix watch tests for linux & bump dependency versions (#104)

Browse Source 
* Fix #90
* Fix #103
* Update dependencies
pull/118/head
Chip Senkbeil 2 years ago committed by GitHub
parent 4180ae279a
commit baee1e2bfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 377 additions and 39 deletions
Show Stats Download Patch File Download Diff File

181
Cargo.lock generated
Unescape Escape View File

@ -63,6 +63,16 @@ dependencies = [
"tempfile",
]
[[package]]
name = "async-attributes"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "async-channel"
version = "1.6.1"
@ -112,6 +122,21 @@ dependencies = [
"futures-lite",
]
[[package]]
name = "async-global-executor"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd8b508d585e01084059b60f06ade4cb7415cd2e4084b71dd1cb44e7d3fb9880"
dependencies = [
"async-channel",
"async-executor",
"async-io",
"async-lock",
"blocking",
"futures-lite",
"once_cell",
]
[[package]]
name = "async-io"
version = "1.7.0"
@ -168,6 +193,34 @@ dependencies = [
"winapi",
]
[[package]]
name = "async-std"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52580991739c5cdb36cde8b2a516371c0a3b70dda36d916cc08b82372916808c"
dependencies = [
"async-attributes",
"async-channel",
"async-global-executor",
"async-io",
"async-lock",
"crossbeam-utils",
"futures-channel",
"futures-core",
"futures-io",
"futures-lite",
"gloo-timers",
"kv-log-macro",
"log",
"memchr",
"num_cpus",
"once_cell",
"pin-project-lite",
"pin-utils",
"slab",
"wasm-bindgen-futures",
]
[[package]]
name = "async-task"
version = "4.2.0"
@ -460,6 +513,16 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "ctor"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "derive_more"
version = "0.99.17"
@ -545,7 +608,7 @@ dependencies = [
"once_cell",
"predicates",
"rand 0.8.5",
"rstest",
"rstest 0.11.0",
"serde",
"serde_json",
"structopt",
@ -567,6 +630,7 @@ dependencies = [
"chacha20poly1305",
"ciborium",
"derive_more",
"flexi_logger 0.22.5",
"futures",
"hex",
"indoc",
@ -577,6 +641,7 @@ dependencies = [
"portable-pty",
"predicates",
"rand 0.8.5",
"rstest 0.13.0",
"serde",
"serde_json",
"structopt",
@ -602,7 +667,7 @@ dependencies = [
"predicates",
"rand 0.8.5",
"rpassword",
"rstest",
"rstest 0.11.0",
"serde",
"shell-words",
"smol",
@ -706,6 +771,23 @@ dependencies = [
"time 0.3.9",
]
[[package]]
name = "flexi_logger"
version = "0.22.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee9a6796ff68a1014f6665dac55341820f26e63ec706e58bfaee468cf0ac174f"
dependencies = [
"ansi_term",
"atty",
"glob",
"lazy_static",
"log",
"regex",
"rustversion",
"thiserror",
"time 0.3.9",
]
[[package]]
name = "float-cmp"
version = "0.9.0"
@ -825,6 +907,12 @@ version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a"
[[package]]
name = "futures-timer"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
[[package]]
name = "futures-util"
version = "0.3.21"
@ -905,6 +993,18 @@ dependencies = [
"walkdir",
]
[[package]]
name = "gloo-timers"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fb7d06c1c8cc2a29bee7ec961009a0b2caa0793ee4900c2ffb348734ba1c8f9"
dependencies = [
"futures-channel",
"futures-core",
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "half"
version = "1.8.2"
@ -1041,6 +1141,15 @@ dependencies = [
"libc",
]
[[package]]
name = "kv-log-macro"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f"
dependencies = [
"log",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
@ -1119,6 +1228,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if 1.0.0",
"value-bag",
]
[[package]]
@ -1719,6 +1829,32 @@ dependencies = [
"syn",
]
[[package]]
name = "rstest"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b939295f93cb1d12bc1a83cf9ee963199b133fb8a79832dd51b68bb9f59a04dc"
dependencies = [
"async-std",
"futures",
"futures-timer",
"rstest_macros",
"rustc_version",
]
[[package]]
name = "rstest_macros"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f78aba848123782ba59340928ec7d876ebe745aa0365d6af8a630f19a5c16116"
dependencies = [
"cfg-if 1.0.0",
"proc-macro2",
"quote",
"rustc_version",
"syn",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
@ -2027,9 +2163,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
version = "1.0.95"
version = "1.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf"
dependencies = [
"proc-macro2",
"quote",
@ -2201,13 +2337,20 @@ dependencies = [
"itoa",
"libc",
"num_threads",
"time-macros",
]
[[package]]
name = "time-macros"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
[[package]]
name = "tokio"
version = "1.18.2"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395"
checksum = "0f392c8f16bda3456c0b00c6de39cb100449b98de55ac41c6cdd2bfcf53a1245"
dependencies = [
"bytes",
"libc",
@ -2225,9 +2368,9 @@ dependencies = [
[[package]]
name = "tokio-macros"
version = "1.7.0"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
dependencies = [
"proc-macro2",
"quote",
@ -2294,6 +2437,16 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
[[package]]
name = "value-bag"
version = "1.0.0-alpha.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55"
dependencies = [
"ctor",
"version_check",
]
[[package]]
name = "vcpkg"
version = "0.2.15"
@ -2390,6 +2543,18 @@ dependencies = [
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2"
dependencies = [
"cfg-if 1.0.0",
"js-sys",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.80"

34
Cargo.toml
Unescape Escape View File

@ -25,35 +25,35 @@ libssh = ["distant-ssh2/libssh"]
ssh2 = ["distant-ssh2/ssh2"]
[dependencies]
derive_more = { version = "0.99.16", default-features = false, features = ["display", "from", "error", "is_variant"] }
derive_more = { version = "0.99.17", default-features = false, features = ["display", "from", "error", "is_variant"] }
distant-core = { version = "=0.16.4", path = "distant-core", features = ["structopt"] }
flexi_logger = "0.18.0"
indoc = "1.0.3"
log = "0.4.14"
once_cell = "1.8.0"
rand = { version = "0.8.4", features = ["getrandom"] }
serde = { version = "1.0.126", features = ["derive"] }
serde_json = "1.0.64"
structopt = "0.3.22"
flexi_logger = "0.18.1"
indoc = "1.0.6"
log = "0.4.17"
once_cell = "1.12.0"
rand = { version = "0.8.5", features = ["getrandom"] }
serde = { version = "1.0.137", features = ["derive"] }
serde_json = "1.0.81"
structopt = "0.3.26"
strum = { version = "0.21.0", features = ["derive"] }
sysinfo = "0.23.2"
tokio = { version = "1.12.0", features = ["full"] }
sysinfo = "0.23.13"
tokio = { version = "1.19.0", features = ["full"] }
terminal_size = "0.1.17"
termwiz = "0.15.0"
whoami = "1.1.2"
whoami = "1.2.1"
# Optional native SSH functionality
distant-ssh2 = { version = "=0.16.4", path = "distant-ssh2", default-features = false, features = ["serde"], optional = true }
[target.'cfg(unix)'.dependencies]
fork = "0.1.18"
fork = "0.1.19"
# [target.'cfg(windows)'.dependencies]
# sysinfo = "0.23.2"
[dev-dependencies]
assert_cmd = "2.0.0"
assert_fs = "1.0.4"
indoc = "1.0.3"
predicates = "2.0.2"
assert_cmd = "2.0.4"
assert_fs = "1.0.7"
indoc = "1.0.6"
predicates = "2.1.1"
rstest = "0.11.0"

4
distant-core/Cargo.toml
Unescape Escape View File

@ -20,7 +20,7 @@ derive_more = { version = "0.99.16", default-features = false, features = ["dere
futures = "0.3.16"
hex = "0.4.3"
log = "0.4.14"
notify = { version = "5.0.0-pre.14", features = ["serde"] }
notify = { version = "5.0.0-pre.15", features = ["serde"] }
normpath = "0.3.2"
once_cell = "1.8.0"
portable-pty = "0.7.0"
@ -37,5 +37,7 @@ structopt = { version = "0.3.22", optional = true }
[dev-dependencies]
assert_fs = "1.0.4"
flexi_logger = "0.22.3"
indoc = "1.0.3"
predicates = "2.0.2"
rstest = "0.13.0"

9
distant-core/src/constants.rs
Unescape Escape View File

@ -7,6 +7,12 @@ pub const CLIENT_PIPE_CAPACITY: usize = 10000;
/// Capacity associated with a client watcher receiving changes
pub const CLIENT_WATCHER_CAPACITY: usize = 100;
/// Capacity associated with the server's file watcher to pass events outbound
pub const SERVER_WATCHER_CAPACITY: usize = 10000;
/// Duration in milliseconds to sleep when reaching maximum watcher events in queue
pub const SERVER_WATCHER_PAUSE_MILLIS: u64 = 100;
/// Represents the maximum size (in bytes) that data will be read from pipes
/// per individual `read` call
///
@ -17,6 +23,9 @@ pub const MAX_PIPE_CHUNK_SIZE: usize = 16384;
/// to avoid sending many small messages to clients
pub const READ_PAUSE_MILLIS: u64 = 50;
/// Maximum message capacity per connection for the distant server
pub const MAX_MSG_CAPACITY: usize = 10000;
/// Test-only constants
#[cfg(test)]
pub mod test {

40
distant-core/src/server/distant/handler.rs
Unescape Escape View File

@ -1,4 +1,5 @@
use crate::{
constants::{SERVER_WATCHER_CAPACITY, SERVER_WATCHER_PAUSE_MILLIS},
data::{
self, Change, ChangeKind, ChangeKindSet, DirEntry, FileType, Metadata, PtySize, Request,
RequestData, Response, ResponseData, RunningProcess, SystemInfo,
@ -18,11 +19,14 @@ use std::{
path::{Path, PathBuf},
pin::Pin,
sync::Arc,
time::SystemTime,
time::{Duration, SystemTime},
};
use tokio::{
io::{self, AsyncWriteExt},
sync::{mpsc, Mutex},
sync::{
mpsc::{self, error::TrySendError},
Mutex,
},
};
use walkdir::WalkDir;
@ -398,15 +402,38 @@ where
// our state, we can be confident that no one else is modifying the watcher option
// concurrently; so, we do a naive check for option being populated
if state.watcher.is_none() {
let (tx, mut rx) = mpsc::channel(1);
// NOTE: Cannot be something small like 1 as this seems to cause a deadlock sometimes
// with a large volume of watch requests
let (tx, mut rx) = mpsc::channel(SERVER_WATCHER_CAPACITY);
let mut watcher = notify::recommended_watcher(move |res| {
let _ = tx.blocking_send(res);
let mut res = res;
// Attempt to send our result, breaking out of the loop
// if we succeed or it is impossible, otherwise trying
// again after a brief sleep
loop {
match tx.try_send(res) {
Ok(_) => break,
Err(TrySendError::Full(x)) => {
warn!(
"Reached watcher capacity of {}! Trying again after {}ms",
SERVER_WATCHER_CAPACITY, SERVER_WATCHER_PAUSE_MILLIS
);
res = x;
std::thread::sleep(Duration::from_millis(SERVER_WATCHER_PAUSE_MILLIS));
}
Err(TrySendError::Closed(_)) => {
warn!("Skipping watch event because watcher channel closed");
break;
}
}
}
})?;
// Attempt to configure watcher, but don't fail if these configurations fail
match watcher.configure(WatcherConfig::PreciseEvents(true)) {
Ok(true) => debug!("<Conn @ {}> Watcher configured for precise events", conn_id,),
Ok(true) => debug!("<Conn @ {}> Watcher configured for precise events", conn_id),
Ok(false) => debug!(
"<Conn @ {}> Watcher not configured for precise events",
conn_id,
@ -548,12 +575,13 @@ where
RecursiveMode::NonRecursive
},
)?;
debug!("<Conn @ {}> Now watching {:?}", conn_id, wp.path());
state.watcher_paths.insert(wp, Box::new(reply));
Ok(Outgoing::from(ResponseData::Ok))
}
None => Err(ServerError::Io(io::Error::new(
io::ErrorKind::BrokenPipe,
format!("<Conn @ {}> Unable to initialize watcher", conn_id,),
format!("<Conn @ {}> Unable to initialize watcher", conn_id),
))),
}
}

3
distant-core/src/server/distant/mod.rs
Unescape Escape View File

@ -6,6 +6,7 @@ pub(crate) use process::{InputChannel, ProcessKiller, ProcessPty};
use state::State;
use crate::{
constants::MAX_MSG_CAPACITY,
data::{Request, Response},
net::{Codec, DataStream, Transport, TransportListener, TransportReadHalf, TransportWriteHalf},
server::{
@ -39,7 +40,7 @@ impl Default for DistantServerOptions {
fn default() -> Self {
Self {
shutdown_after: None,
max_msg_capacity: 1,
max_msg_capacity: MAX_MSG_CAPACITY,
}
}
}

1
distant-core/tests/stress/distant/mod.rs
Unescape Escape View File

@ -0,0 +1 @@
mod watch;

53
distant-core/tests/stress/distant/watch.rs
Unescape Escape View File

@ -0,0 +1,53 @@
use crate::stress::fixtures::*;
use assert_fs::prelude::*;
use distant_core::{ChangeKindSet, SessionChannelExt};
use rstest::*;
const MAX_FILES: usize = 500;
#[rstest]
#[tokio::test]
#[ignore]
async fn should_handle_large_volume_of_file_watching(#[future] ctx: DistantSessionCtx) {
let ctx = ctx.await;
let mut channel = ctx.session.clone_channel();
let tenant = "watch-stress-test";
let root = assert_fs::TempDir::new().unwrap();
let mut files_and_watchers = Vec::new();
for n in 1..=MAX_FILES {
let file = root.child(format!("test-file-{}", n));
eprintln!("Generating {:?}", file.path());
file.touch().unwrap();
eprintln!("Watching {:?}", file.path());
let watcher = channel
.watch(
tenant,
file.path(),
false,
ChangeKindSet::modify_set(),
ChangeKindSet::empty(),
)
.await
.unwrap();
eprintln!("Now watching file {}", n);
files_and_watchers.push((file, watcher));
}
for (file, _watcher) in files_and_watchers.iter() {
eprintln!("Updating {:?}", file.path());
file.write_str("updated text").unwrap();
}
for (file, watcher) in files_and_watchers.iter_mut() {
eprintln!("Checking {:?} for changes", file.path());
match watcher.next().await {
Some(_) => {}
None => panic!("File {:?} did not have a change detected", file.path()),
}
}
}

58
distant-core/tests/stress/fixtures.rs
Unescape Escape View File

@ -0,0 +1,58 @@
use crate::stress::utils;
use distant_core::{DistantServer, SecretKey, SecretKey32, Session, XChaCha20Poly1305Codec};
use rstest::*;
use std::time::Duration;
use tokio::sync::mpsc;
const LOG_PATH: &str = "/tmp/test.distant.server.log";
pub struct DistantSessionCtx {
pub session: Session,
_done_tx: mpsc::Sender<()>,
}
impl DistantSessionCtx {
pub async fn initialize() -> Self {
let ip_addr = "127.0.0.1".parse().unwrap();
let (done_tx, mut done_rx) = mpsc::channel::<()>(1);
let (started_tx, mut started_rx) = mpsc::channel::<(u16, SecretKey32)>(1);
tokio::spawn(async move {
let logger = utils::init_logging(LOG_PATH);
let key = SecretKey::default();
let codec = XChaCha20Poly1305Codec::from(key.clone());
let (_server, port) =
DistantServer::bind(ip_addr, "0".parse().unwrap(), codec, Default::default())
.await
.unwrap();
started_tx.send((port, key)).await.unwrap();
let _ = done_rx.recv().await;
logger.flush();
logger.shutdown();
});
// Extract our server startup data if we succeeded
let (port, key) = started_rx.recv().await.unwrap();
// Now initialize our session
let session = Session::tcp_connect_timeout(
format!("{}:{}", ip_addr, port).parse().unwrap(),
XChaCha20Poly1305Codec::from(key),
Duration::from_secs(1),
)
.await
.unwrap();
DistantSessionCtx {
session,
_done_tx: done_tx,
}
}
}
#[fixture]
pub async fn ctx() -> DistantSessionCtx {
DistantSessionCtx::initialize().await
}

3
distant-core/tests/stress/mod.rs
Unescape Escape View File

@ -0,0 +1,3 @@
mod distant;
mod fixtures;
mod utils;

23
distant-core/tests/stress/utils.rs
Unescape Escape View File

@ -0,0 +1,23 @@
use std::path::PathBuf;
/// Initializes logging (should only call once)
pub fn init_logging(path: impl Into<PathBuf>) -> flexi_logger::LoggerHandle {
use flexi_logger::{FileSpec, LevelFilter, LogSpecification, Logger};
let modules = &["distant", "distant_core", "distant_ssh2"];
// Disable logging for everything but our binary, which is based on verbosity
let mut builder = LogSpecification::builder();
builder.default(LevelFilter::Off);
// For each module, configure logging
for module in modules {
builder.module(module, LevelFilter::Trace);
}
// Create our logger, but don't initialize yet
let logger = Logger::with(builder.build())
.format_for_files(flexi_logger::opt_format)
.log_to_file(FileSpec::try_from(path).expect("Failed to create log file spec"));
logger.start().expect("Failed to initialize logger")
}

1
distant-core/tests/stress_tests.rs
Unescape Escape View File

@ -0,0 +1 @@
mod stress;

6
tests/cli/action/watch.rs
Unescape Escape View File

@ -181,10 +181,7 @@ where
res
}
// TODO: For some reason, this always fails on linux, so we're skipping the test
// for that platform right now.
#[rstest]
#[cfg_attr(linux, ignore)]
fn should_support_watching_a_single_file(mut action_std_cmd: Command) {
let temp = assert_fs::TempDir::new().unwrap();
let file = temp.child("file");
@ -234,10 +231,7 @@ fn should_support_watching_a_single_file(mut action_std_cmd: Command) {
assert_eq!(stderr_data, "");
}
// TODO: For some reason, this always fails on linux, so we're skipping the test
// for that platform right now.
#[rstest]
#[cfg_attr(linux, ignore)]
fn should_support_watching_a_directory_recursively(mut action_std_cmd: Command) {
let temp = assert_fs::TempDir::new().unwrap();

Write Preview
Loading…
Cancel
Save