|
|
|
@ -23,6 +23,7 @@ static INSTANCES_RECHECK_DELAY: Duration = Duration::from_secs(5);
|
|
|
|
|
#[cfg(not(debug_assertions))]
|
|
|
|
|
static INSTANCES_RECHECK_DELAY: Duration = Duration::from_secs(60);
|
|
|
|
|
|
|
|
|
|
#[derive(Clone)]
|
|
|
|
|
pub struct Opts {
|
|
|
|
|
/// how many processes you are starting in total
|
|
|
|
|
pub process_count: i32,
|
|
|
|
@ -36,7 +37,7 @@ async fn start_stop_federation_workers(
|
|
|
|
|
federation_config: FederationConfig<LemmyContext>,
|
|
|
|
|
cancel: CancellationToken,
|
|
|
|
|
) -> anyhow::Result<()> {
|
|
|
|
|
let mut workers = HashMap::<InstanceId, CancellableTask<_>>::new();
|
|
|
|
|
let mut workers = HashMap::<InstanceId, CancellableTask>::new();
|
|
|
|
|
|
|
|
|
|
let (stats_sender, stats_receiver) = unbounded_channel();
|
|
|
|
|
let exit_print = tokio::spawn(receive_print_stats(pool.clone(), stats_receiver));
|
|
|
|
@ -66,40 +67,30 @@ async fn start_stop_federation_workers(
|
|
|
|
|
let should_federate = allowed && !is_dead;
|
|
|
|
|
if should_federate {
|
|
|
|
|
if workers.contains_key(&instance.id) {
|
|
|
|
|
if workers
|
|
|
|
|
.get(&instance.id)
|
|
|
|
|
.map(util::CancellableTask::has_ended)
|
|
|
|
|
.unwrap_or(false)
|
|
|
|
|
{
|
|
|
|
|
// task must have errored out, remove and recreated it
|
|
|
|
|
let worker = workers
|
|
|
|
|
.remove(&instance.id)
|
|
|
|
|
.expect("just checked contains_key");
|
|
|
|
|
tracing::error!(
|
|
|
|
|
"worker for {} has stopped, recreating: {:?}",
|
|
|
|
|
instance.domain,
|
|
|
|
|
worker.cancel().await
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
// worker already running
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
// create new worker
|
|
|
|
|
let config = federation_config.clone();
|
|
|
|
|
let stats_sender = stats_sender.clone();
|
|
|
|
|
let context = federation_config.to_request_data();
|
|
|
|
|
let pool = pool.clone();
|
|
|
|
|
workers.insert(
|
|
|
|
|
instance.id,
|
|
|
|
|
CancellableTask::spawn(WORKER_EXIT_TIMEOUT, |stop| async move {
|
|
|
|
|
InstanceWorker::init_and_loop(
|
|
|
|
|
instance,
|
|
|
|
|
context,
|
|
|
|
|
&mut DbPool::Pool(&pool),
|
|
|
|
|
stop,
|
|
|
|
|
stats_sender,
|
|
|
|
|
)
|
|
|
|
|
.await?;
|
|
|
|
|
Ok(())
|
|
|
|
|
CancellableTask::spawn(WORKER_EXIT_TIMEOUT, move |stop| {
|
|
|
|
|
let instance = instance.clone();
|
|
|
|
|
let req_data = config.clone().to_request_data();
|
|
|
|
|
let stats_sender = stats_sender.clone();
|
|
|
|
|
let pool = pool.clone();
|
|
|
|
|
async move {
|
|
|
|
|
InstanceWorker::init_and_loop(
|
|
|
|
|
instance,
|
|
|
|
|
req_data,
|
|
|
|
|
&mut DbPool::Pool(&pool),
|
|
|
|
|
stop,
|
|
|
|
|
stats_sender,
|
|
|
|
|
)
|
|
|
|
|
.await
|
|
|
|
|
}
|
|
|
|
|
}),
|
|
|
|
|
);
|
|
|
|
|
} else if !should_federate {
|
|
|
|
@ -135,9 +126,12 @@ pub fn start_stop_federation_workers_cancellable(
|
|
|
|
|
opts: Opts,
|
|
|
|
|
pool: ActualDbPool,
|
|
|
|
|
config: FederationConfig<LemmyContext>,
|
|
|
|
|
) -> CancellableTask<()> {
|
|
|
|
|
CancellableTask::spawn(WORKER_EXIT_TIMEOUT, move |c| {
|
|
|
|
|
start_stop_federation_workers(opts, pool, config, c)
|
|
|
|
|
) -> CancellableTask {
|
|
|
|
|
CancellableTask::spawn(WORKER_EXIT_TIMEOUT, move |stop| {
|
|
|
|
|
let opts = opts.clone();
|
|
|
|
|
let pool = pool.clone();
|
|
|
|
|
let config = config.clone();
|
|
|
|
|
async move { start_stop_federation_workers(opts, pool, config, stop).await }
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|