From 9f233f50eaae8ce6ce25631316ab3855980341bd Mon Sep 17 00:00:00 2001 From: Moonchild Date: Wed, 10 Feb 2021 20:22:31 +0100 Subject: [PATCH 1/3] Add rudimentary 5-minute cache to get_storage_total() --- weave_storage.php | 51 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/weave_storage.php b/weave_storage.php index 6e71fb0..95c107e 100644 --- a/weave_storage.php +++ b/weave_storage.php @@ -629,21 +629,62 @@ class WeaveStorage function get_storage_total() { + $username = $this->_username; + $time = time(); + try { - $select_stmt = 'select round(sum(length(payload))/1024) from wbo where username = :username'; + $select_stmt = 'select usage, usage_time from users where username = :username'; $sth = $this->_dbh->prepare($select_stmt); - $username = $this->_username; $sth->bindParam(':username', $username); $sth->execute(); } catch( PDOException $exception ) { - error_log("get_storage_total: " . $exception->getMessage()); + error_log("get_storage_total (user field): " . $exception->getMessage()); throw new Exception("Database unavailable", 503); } - - return (int)$sth->fetchColumn(); + $result = $sth->fetch(PDO::FETCH_ASSOC); + if ($result['usage'] != NULL && + $result['usage_time'] != NULL && + ((int)$result['usage'] != 0) && + ($time - (int)$result['usage_time'] < 300)) { + # We have a usage size and it's recent enough; use cached value + return (int)$result['usage']; + } + else + { + # We don't have a current cached value. Retrieve and store. + try + { + $select_stmt = 'select round(sum(length(payload))/1024) from wbo where username = :username'; + $sth = $this->_dbh->prepare($select_stmt); + $sth->bindParam(':username', $username); + $sth->execute(); + } + catch( PDOException $exception ) + { + error_log("get_storage_total: " . $exception->getMessage()); + throw new Exception("Database unavailable", 503); + } + + $usage = (int)$sth->fetchColumn(); + try + { + $update_stmt = 'update users set usage = :usage, usage_time = :usage_time where username = :username'; + $sth = $this->_dbh->prepare($update_stmt); + $sth->bindParam(':username', $username); + $sth->bindParam(':usage', $usage); + $sth->bindParam(':usage_time', $time); + $sth->execute(); + } + catch( PDOException $exception ) + { + error_log("get_storage_total (store): " . $exception->getMessage()); + throw new Exception("Database unavailable", 503); + } + return $usage; + } } function get_collection_storage_totals() From e092b46d7916607b76982112e6e3c722331ce7b1 Mon Sep 17 00:00:00 2001 From: Moonchild Date: Thu, 11 Feb 2021 04:15:04 +0100 Subject: [PATCH 2/3] Fix cache and increase TTL to 1 hour, make configurable. Apparently 'usage' is a reserved word which prevented this from working. Fixed whitespace/indenting too (really need to fix all whitespace someday) --- setup.php | 6 +++- weave_storage.php | 74 +++++++++++++++++++++++++---------------------- weave_utils.php | 10 +++---- 3 files changed, 49 insertions(+), 41 deletions(-) diff --git a/setup.php b/setup.php index 1f18d41..e587bc8 100644 --- a/setup.php +++ b/setup.php @@ -165,6 +165,9 @@ function write_config_file($dbt, $dbh, $dbn, $dbu, $dbp, $fsRoot) { $cfg_content .= " // set MinQuota and MaxQuota\n"; $cfg_content .= " define(\"MINQUOTA\", 30000);\n"; $cfg_content .= " define(\"MAXQUOTA\", 35000);\n"; + $cfg_content .= " // The setting below determines the time to live for quota totals\n"; + $cfg_content .= " // before recalculating how much database space has been used.\n"; + $cfg_content .= " define(\"QUOTA_TTL\", 3600);\n"; $cfg_content .= "\n?>\n"; @@ -357,7 +360,8 @@ if ( $action == "step2" ) { $create_statement = " create table wbo ( username varchar(100), id varchar(65), collection varchar(100), parentid varchar(65), predecessorid int, modified real, sortindex int, payload text, payload_size int, ttl int, primary key (username,collection,id))"; - $create_statement2 = " create table users ( username varchar(255), md5 varchar(124), login int, + $create_statement2 = " create table users ( username varchar(255), md5 varchar(124), login int, + quota_usage int, usage_time int, primary key (username)) "; $index1 = 'create index parentindex on wbo (username, parentid)'; $index2 = 'create index predecessorindex on wbo (username, predecessorid)'; diff --git a/weave_storage.php b/weave_storage.php index 95c107e..d9daf05 100644 --- a/weave_storage.php +++ b/weave_storage.php @@ -634,7 +634,7 @@ class WeaveStorage try { - $select_stmt = 'select usage, usage_time from users where username = :username'; + $select_stmt = 'select quota_usage, usage_time from users where username = :username'; $sth = $this->_dbh->prepare($select_stmt); $sth->bindParam(':username', $username); $sth->execute(); @@ -645,45 +645,48 @@ class WeaveStorage throw new Exception("Database unavailable", 503); } $result = $sth->fetch(PDO::FETCH_ASSOC); - if ($result['usage'] != NULL && + if ($result['quota_usage'] != NULL && $result['usage_time'] != NULL && - ((int)$result['usage'] != 0) && - ($time - (int)$result['usage_time'] < 300)) { + ((int)$result['quota_usage'] != 0) && + ($time - (int)$result['usage_time'] < QUOTA_TTL)) { # We have a usage size and it's recent enough; use cached value - return (int)$result['usage']; + return (int)$result['quota_usage']; } else { # We don't have a current cached value. Retrieve and store. - try - { - $select_stmt = 'select round(sum(length(payload))/1024) from wbo where username = :username'; - $sth = $this->_dbh->prepare($select_stmt); - $sth->bindParam(':username', $username); - $sth->execute(); - } - catch( PDOException $exception ) - { - error_log("get_storage_total: " . $exception->getMessage()); - throw new Exception("Database unavailable", 503); - } - - $usage = (int)$sth->fetchColumn(); - try - { - $update_stmt = 'update users set usage = :usage, usage_time = :usage_time where username = :username'; - $sth = $this->_dbh->prepare($update_stmt); - $sth->bindParam(':username', $username); - $sth->bindParam(':usage', $usage); - $sth->bindParam(':usage_time', $time); - $sth->execute(); - } - catch( PDOException $exception ) - { - error_log("get_storage_total (store): " . $exception->getMessage()); - throw new Exception("Database unavailable", 503); - } - return $usage; + try + { + $select_stmt = 'select round(sum(length(payload))/1024) from wbo where username = :username'; + $sth = $this->_dbh->prepare($select_stmt); + $sth->bindParam(':username', $username); + $sth->execute(); + } + catch( PDOException $exception ) + { + error_log("get_storage_total: " . $exception->getMessage()); + throw new Exception("Database unavailable", 503); + } + + $usage = (int)$sth->fetchColumn(); + + try + { + $update_stmt = 'update users set quota_usage = :usage, usage_time = :usage_time where username = :username'; + $sth = $this->_dbh->prepare($update_stmt); + $sth->bindParam(':username', $username); + $sth->bindParam(':usage', $usage); + $sth->bindParam(':usage_time', $time); + // error_log("Store query: update users set quota_usage = ".$usage.", usage_time = ".$time." where username = ".$username); + $sth->execute(); + } + catch( PDOException $exception ) + { + error_log("get_storage_total (store): " . $exception->getMessage()); + throw new Exception("Database unavailable", 503); + } + + return $usage; } } @@ -799,7 +802,8 @@ class WeaveStorage try { - $create_statement = "insert into users (username, md5, login) values (:username, :md5, null)"; + $create_statement = "insert into users (username, md5, login, quota_usage, usage_time) + values (:username, :md5, null, 0, 0)"; $sth = $this->_dbh->prepare($create_statement); $hash = WeaveHashFactory::factory(); diff --git a/weave_utils.php b/weave_utils.php index 586f38f..3d72f92 100644 --- a/weave_utils.php +++ b/weave_utils.php @@ -55,15 +55,15 @@ define ('LOG_THE_ERROR', 0); - define ('LOG_QUOTAS', 1); + define ('LOG_QUOTAS', 0); function log_quota($msg) { if ( LOG_QUOTAS == 1 ) { $datei = fopen("/tmp/FSyncMS-quota.log","a"); - $fmsg = sprintf("$msg\n"); - fputs($datei,$fmsg); + // $fmsg = sprintf("$msg\n"); + fputs($datei,"$msg\n"); // fputs($datei,"Server ".print_r( $_SERVER, true)); fclose($datei); } @@ -74,8 +74,8 @@ if ( LOG_THE_ERROR == 1 ) { $datei = fopen("/tmp/FSyncMS-error.txt","a"); - $fmsg = sprintf("$msg\n"); - fputs($datei,$fmsg); + // $fmsg = sprintf("$msg\n"); + fputs($datei,"$msg\n"); // fputs($datei,"Server ".print_r( $_SERVER, true)); fclose($datei); } From 5f1f5f914b790086b9b9634117657dc9a84db734 Mon Sep 17 00:00:00 2001 From: Moonchild Date: Thu, 11 Feb 2021 04:16:14 +0100 Subject: [PATCH 3/3] Clear quota usage immediately when data is deleted, triggering a recalc. --- index.php | 3 +++ weave_storage.php | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/index.php b/index.php index 0530074..3868491 100644 --- a/index.php +++ b/index.php @@ -280,6 +280,7 @@ if ($id) { $db->delete_object($collection, $id); + $db->clear_quota_usage($username); } else if ($collection) { @@ -293,12 +294,14 @@ $params['ids'], $params['index_above'], $params['index_below'] ); + $db->clear_quota_usage($username); } else if($function == 'storage') // ich vermute mal storage reinigen { if (!array_key_exists('HTTP_X_CONFIRM_DELETE', $_SERVER)) report_problem(WEAVE_ERROR_NO_OVERWRITE, 412); $db->delete_storage($username); + $db->clear_quota_usage($username); } else { diff --git a/weave_storage.php b/weave_storage.php index d9daf05..7eb7426 100644 --- a/weave_storage.php +++ b/weave_storage.php @@ -794,8 +794,25 @@ class WeaveStorage return 0; } return 1; - } + } + function clear_quota_usage($username) + { + try + { + $update_statement = "update users set quota_usage = 0 where username = :username"; + $sth = $this->_dbh->prepare($update_statement); + $sth->bindParam(':username', $username); + $sth->execute(); + } + catch( PDOException $exception ) + { + log_error("clear quota usage:" . $exception->getMessage()); + return 0; + } + return 1; + } + function create_user($username, $password) { log_error("Create User - Username: ".$username."|".$password);