You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
209 lines
7.8 KiB
Rust
209 lines
7.8 KiB
Rust
use crate::{adapters::FileAdapter, preproc::ActiveAdapters};
|
|
use anyhow::{Context, Result};
|
|
use log::warn;
|
|
use path_clean::PathClean;
|
|
use rusqlite::{named_params, OptionalExtension};
|
|
use std::{path::Path, time::UNIX_EPOCH};
|
|
use tokio_rusqlite::Connection;
|
|
|
|
static SCHEMA_VERSION: i32 = 3;
|
|
#[derive(Clone)]
|
|
pub struct CacheKey {
|
|
config_hash: String,
|
|
adapter: String,
|
|
adapter_version: i32,
|
|
active_adapters: String,
|
|
file_path: String,
|
|
file_mtime_unix_ms: i64,
|
|
}
|
|
impl CacheKey {
|
|
pub fn new(
|
|
postprocess: bool,
|
|
filepath_hint: &Path,
|
|
adapter: &dyn FileAdapter,
|
|
active_adapters: &ActiveAdapters,
|
|
) -> Result<CacheKey> {
|
|
let meta = std::fs::metadata(filepath_hint)
|
|
.with_context(|| format!("reading metadata for {}", filepath_hint.to_string_lossy()))?;
|
|
let modified = meta.modified().expect("weird OS that can't into mtime");
|
|
let file_mtime_unix_ms = modified.duration_since(UNIX_EPOCH)?.as_millis() as i64;
|
|
let active_adapters = if adapter.metadata().recurses {
|
|
serde_json::to_string(
|
|
&active_adapters
|
|
.iter()
|
|
.map(|a| format!("{}.v{}", a.metadata().name, a.metadata().version))
|
|
.collect::<Vec<_>>(),
|
|
)?
|
|
} else {
|
|
"null".to_string()
|
|
};
|
|
Ok(CacheKey {
|
|
config_hash: if postprocess {
|
|
"a41e2e9".to_string()
|
|
} else {
|
|
"f1502a3".to_string()
|
|
}, // todo: when we add more config options that affect caching, create a struct and actually hash it
|
|
adapter: adapter.metadata().name.clone(),
|
|
adapter_version: adapter.metadata().version,
|
|
file_path: filepath_hint.clean().to_string_lossy().to_string(),
|
|
file_mtime_unix_ms,
|
|
active_adapters,
|
|
})
|
|
}
|
|
}
|
|
|
|
#[async_trait::async_trait]
|
|
pub trait PreprocCache {
|
|
async fn get(&self, key: &CacheKey) -> Result<Option<Vec<u8>>>;
|
|
async fn set(&mut self, key: &CacheKey, value: Vec<u8>) -> Result<()>;
|
|
}
|
|
|
|
async fn connect_pragmas(db: &Connection) -> Result<()> {
|
|
// https://phiresky.github.io/blog/2020/sqlite-performance-tuning/
|
|
//let want_page_size = 32768;
|
|
//db.execute(&format!("pragma page_size = {};", want_page_size))
|
|
// .context("setup pragma 1")?;
|
|
db.call(|db| {
|
|
// db.busy_timeout(Duration::from_secs(10))?;
|
|
db.pragma_update(None, "journal_mode", "wal")?;
|
|
db.pragma_update(None, "foreign_keys", "on")?;
|
|
db.pragma_update(None, "temp_store", "memory")?;
|
|
db.pragma_update(None, "synchronous", "off")?; // integrity isn't very important here
|
|
db.pragma_update(None, "mmap_size", "2000000000")?;
|
|
db.execute("
|
|
create table if not exists preproc_cache (
|
|
config_hash text not null,
|
|
adapter text not null,
|
|
adapter_version integer not null,
|
|
created_unix_ms integer not null default (unixepoch() * 1000),
|
|
active_adapters text not null, -- 'null' if adapter cannot recurse
|
|
file_path text not null,
|
|
file_mtime_unix_ms integer not null,
|
|
text_content_zstd blob not null
|
|
) strict", []
|
|
)?;
|
|
|
|
db.execute("create unique index if not exists preproc_cache_idx on preproc_cache (config_hash, adapter, adapter_version, file_path, active_adapters)", [])?;
|
|
|
|
Ok(())
|
|
})
|
|
.await.context("connect_pragmas")?;
|
|
let jm: i64 = db
|
|
.call(|db| Ok(db.pragma_query_value(None, "application_id", |r| r.get(0))?))
|
|
.await?;
|
|
if jm != 924716026 {
|
|
// (probably) newly created db
|
|
db.call(|db| Ok(db.pragma_update(None, "application_id", "924716026")?))
|
|
.await?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
struct SqliteCache {
|
|
db: Connection,
|
|
}
|
|
impl SqliteCache {
|
|
async fn new(path: &Path) -> Result<SqliteCache> {
|
|
let db = Connection::open(path.join("cache.sqlite3")).await?;
|
|
db.call(|db| {
|
|
let schema_version: i32 = db.pragma_query_value(None, "user_version", |r| r.get(0))?;
|
|
if schema_version != SCHEMA_VERSION {
|
|
warn!("Cache schema version mismatch, clearing cache");
|
|
db.execute("drop table if exists preproc_cache", [])?;
|
|
db.pragma_update(None, "user_version", format!("{SCHEMA_VERSION}"))?;
|
|
}
|
|
Ok(())
|
|
})
|
|
.await?;
|
|
|
|
connect_pragmas(&db).await?;
|
|
|
|
Ok(SqliteCache { db })
|
|
}
|
|
}
|
|
|
|
#[async_trait::async_trait]
|
|
impl PreprocCache for SqliteCache {
|
|
async fn get(&self, key: &CacheKey) -> Result<Option<Vec<u8>>> {
|
|
let key = (*key).clone(); // todo: without cloning
|
|
Ok(self
|
|
.db
|
|
.call(move |db| {
|
|
Ok(db
|
|
.query_row(
|
|
"select text_content_zstd from preproc_cache where
|
|
adapter = :adapter
|
|
and config_hash = :config_hash
|
|
and adapter_version = :adapter_version
|
|
and active_adapters = :active_adapters
|
|
and file_path = :file_path
|
|
and file_mtime_unix_ms = :file_mtime_unix_ms
|
|
",
|
|
named_params! {
|
|
":config_hash": &key.config_hash,
|
|
":adapter": &key.adapter,
|
|
":adapter_version": &key.adapter_version,
|
|
":active_adapters": &key.active_adapters,
|
|
":file_path": &key.file_path,
|
|
":file_mtime_unix_ms": &key.file_mtime_unix_ms
|
|
},
|
|
|r| r.get::<_, Vec<u8>>(0),
|
|
)
|
|
.optional()?)
|
|
})
|
|
.await
|
|
.context("reading from cache")?)
|
|
}
|
|
|
|
async fn set(&mut self, key: &CacheKey, value: Vec<u8>) -> Result<()> {
|
|
let key = (*key).clone(); // todo: without cloning
|
|
log::trace!(
|
|
"Writing to cache: {}, {}, {} byte",
|
|
key.adapter,
|
|
key.file_path,
|
|
value.len()
|
|
);
|
|
Ok(self
|
|
.db
|
|
.call(move |db| {
|
|
db.execute(
|
|
"insert into preproc_cache (config_hash, adapter, adapter_version, active_adapters, file_path, file_mtime_unix_ms, text_content_zstd) values
|
|
(:config_hash, :adapter, :adapter_version, :active_adapters, :file_path, :file_mtime_unix_ms, :text_content_zstd)
|
|
on conflict (config_hash, adapter, adapter_version, active_adapters, file_path) do update set
|
|
file_mtime_unix_ms = :file_mtime_unix_ms,
|
|
created_unix_ms = unixepoch() * 1000,
|
|
text_content_zstd = :text_content_zstd",
|
|
named_params! {
|
|
":config_hash": &key.config_hash,
|
|
":adapter": &key.adapter,
|
|
":adapter_version": &key.adapter_version,
|
|
":active_adapters": &key.active_adapters,
|
|
":file_path": &key.file_path,
|
|
":file_mtime_unix_ms": &key.file_mtime_unix_ms,
|
|
":text_content_zstd": value
|
|
})?;
|
|
Ok(())
|
|
})
|
|
.await?)
|
|
}
|
|
}
|
|
/// opens a default cache
|
|
pub async fn open_cache_db(path: &Path) -> Result<impl PreprocCache> {
|
|
std::fs::create_dir_all(path)?;
|
|
SqliteCache::new(path).await
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod test {
|
|
|
|
use crate::preproc_cache::*;
|
|
|
|
#[tokio::test]
|
|
async fn test_read_write() -> anyhow::Result<()> {
|
|
let path = tempfile::tempdir()?;
|
|
let _db = open_cache_db(&path.path().join("foo.sqlite3")).await?;
|
|
// db.set();
|
|
Ok(())
|
|
}
|
|
}
|