From 8500fdd5194e257bc99d3ecc0e62f85e9931724f Mon Sep 17 00:00:00 2001 From: weijiuqiao <59040746+weijiuqiao@users.noreply.github.com> Date: Fri, 11 Nov 2022 22:53:06 +0800 Subject: [PATCH] Cloud-based sync for 2 plugins: reading statistics and vocabulary builder (#9709) This commit adds cross-device sync ability for two plugins: reading statistics and vocabulary builder. It relies on user setting up a Cloud server (DropBox and WebDAV but not FTP though) and designating a path. Behind the curtains sqlite databases are being passed around and updated. UI-wise, for the statistics plugin, two new menu items Synchronize now and Cloud sync to set it up (might not be the best wording) are added. As for vocabulary builder, a similar Cloud sync button is added to the menu and a shortcut icon button to Synchronize now is pinned at the bottom corner. CloudStorage new features: WebDAV creating folders and uploading files. And a new widget-like sync server chooser. In the end I decided not to add automatic sync, as the SQL commands part seem a bit much. --- frontend/apps/cloudstorage/cloudstorage.lua | 16 +- frontend/apps/cloudstorage/dropboxapi.lua | 9 +- frontend/apps/cloudstorage/syncservice.lua | 175 ++++++++++++++ frontend/apps/cloudstorage/webdav.lua | 35 ++- frontend/apps/cloudstorage/webdavapi.lua | 45 +++- plugins/statistics.koplugin/main.lua | 240 +++++++++++++++++++- plugins/vocabbuilder.koplugin/db.lua | 104 ++++++++- plugins/vocabbuilder.koplugin/main.lua | 122 +++++++++- 8 files changed, 727 insertions(+), 19 deletions(-) create mode 100644 frontend/apps/cloudstorage/syncservice.lua diff --git a/frontend/apps/cloudstorage/cloudstorage.lua b/frontend/apps/cloudstorage/cloudstorage.lua index 11cc576c7..8df5f5a4f 100644 --- a/frontend/apps/cloudstorage/cloudstorage.lua +++ b/frontend/apps/cloudstorage/cloudstorage.lua @@ -173,11 +173,11 @@ function CloudStorage:openCloudServer(url) if NetworkMgr:willRerunWhenConnected(function() self:openCloudServer(url) end) then return end - tbl, e = WebDav:run(self.address, self.username, self.password, url) + tbl, e = WebDav:run(self.address, self.username, self.password, url, self.choose_folder_mode) end if tbl then self:switchItemTable(url, tbl) - if self.type == "dropbox" then + if self.type == "dropbox" or self.type == "webdav" then self.onLeftButtonTap = function() self:showPlusMenu(url) end @@ -648,7 +648,11 @@ function CloudStorage:uploadFile(url) end) local url_base = url ~= "/" and url or "" UIManager:tickAfterNext(function() - DropBox:uploadFile(url_base, self.password, file_path, callback_close) + if self.type == "dropbox" then + DropBox:uploadFile(url_base, self.password, file_path, callback_close) + elseif self.type == "webdav" then + WebDav:uploadFile(url_base, self.address, self.username, self.password, file_path, callback_close) + end end) end end @@ -686,7 +690,11 @@ function CloudStorage:createFolder(url) end self:openCloudServer(url) end - DropBox:createFolder(url_base, self.password, folder_name, callback_close) + if self.type == "dropbox" then + DropBox:createFolder(url_base, self.password, folder_name, callback_close) + elseif self.type == "webdav" then + WebDav:createFolder(url_base, self.address, self.username, self.password, folder_name, callback_close) + end end, }, } diff --git a/frontend/apps/cloudstorage/dropboxapi.lua b/frontend/apps/cloudstorage/dropboxapi.lua index 745d1b135..23c08fa82 100644 --- a/frontend/apps/cloudstorage/dropboxapi.lua +++ b/frontend/apps/cloudstorage/dropboxapi.lua @@ -117,7 +117,7 @@ end function DropBoxApi:downloadFile(path, token, local_path) local data1 = "{\"path\": \"" .. path .. "\"}" socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT) - local code, _, status = socket.skip(1, http.request{ + local code, headers, status = socket.skip(1, http.request{ url = API_DOWNLOAD_FILE, method = "GET", headers = { @@ -130,12 +130,12 @@ function DropBoxApi:downloadFile(path, token, local_path) if code ~= 200 then logger.warn("DropBoxApi: Download failure:", status or code or "network unreachable") end - return code + return code, (headers or {}).etag end -function DropBoxApi:uploadFile(path, token, file_path) +function DropBoxApi:uploadFile(path, token, file_path, etag, overwrite) local data = "{\"path\": \"" .. path .. "/" .. BaseUtil.basename(file_path) .. - "\",\"mode\": \"add\",\"autorename\": true,\"mute\": false,\"strict_conflict\": false}" + "\",\"mode\":" .. (overwrite and "\"overwrite\"" or "\"add\"") .. ",\"autorename\": " .. (overwrite and "false" or "true") .. ",\"mute\": false,\"strict_conflict\": false}" socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT) local code, _, status = socket.skip(1, http.request{ url = API_UPLOAD_FILE, @@ -145,6 +145,7 @@ function DropBoxApi:uploadFile(path, token, file_path) ["Dropbox-API-Arg"] = data, ["Content-Type"] = "application/octet-stream", ["Content-Length"] = lfs.attributes(file_path, "size"), + ["If-Match"] = etag, }, source = ltn12.source.file(io.open(file_path, "r")), }) diff --git a/frontend/apps/cloudstorage/syncservice.lua b/frontend/apps/cloudstorage/syncservice.lua new file mode 100644 index 000000000..e82466c5a --- /dev/null +++ b/frontend/apps/cloudstorage/syncservice.lua @@ -0,0 +1,175 @@ +local DataStorage = require("datastorage") +local Font = require("ui/font") +local InfoMessage = require("ui/widget/infomessage") +local LuaSettings = require("luasettings") +local Menu = require("ui/widget/menu") +local Notification = require("ui/widget/notification") +local Screen = require("device").screen +local UIManager = require("ui/uimanager") +local ffiutil = require("ffi/util") +local util = require("util") + +local _ = require("gettext") + +local server_types = { + dropbox = _("Dropbox"), + webdav = _("WebDAV"), +} +local indent = "" + +local SyncService = Menu:extend{ + no_title = false, + show_parent = nil, + is_popout = false, + is_borderless = true, + title = _("Cloud sync settings"), + title_face = Font:getFace("smallinfofontbold"), +} + +function SyncService:init() + self.cs_settings = LuaSettings:open(DataStorage:getSettingsDir().."/cloudstorage.lua") + self.item_table = self:generateItemTable() + self.width = Screen:getWidth() + self.height = Screen:getHeight() + Menu.init(self) +end + +function SyncService:generateItemTable() + local item_table = {} + -- select and/or add server + local added_servers = self.cs_settings:readSetting("cs_servers") or {} + for _, server in ipairs(added_servers) do + if server.type == "dropbox" or server.type == "webdav" then + local item = { + text = indent .. server.name, + address = server.address, + username = server.username, + password = server.password, + type = server.type, + url = server.url, + mandatory = server_types[server.type], + } + item.callback = function() + require("ui/downloadmgr"):new{ + item = item, + onConfirm = function(path) + server.url = path + self.onConfirm(server) + self:onClose() + end, + }:chooseCloudDir() + end + table.insert(item_table, item) + end + end + if #item_table > 0 then + table.insert(item_table, 1, { + text = _("Choose cloud service:"), + bold = true, + }) + end + table.insert(item_table, { + text = _("Add service"), + bold = true, + callback = function() + local cloud_storage = require("apps/cloudstorage/cloudstorage"):new{} + UIManager:show(cloud_storage) + end + }) + return item_table +end + +function SyncService.getReadablePath(server) + local url = util.stringStartsWith(server.url, "/") and server.url:sub(2) or server.url + url = util.urlDecode(url) or url + url = util.stringEndsWith(url, "/") and url or url .. "/" + url = (server.address:sub(-1) == "/" and server.address or server.address .. "/") .. url + if url:sub(-2) == "//" then url = url:sub(1, -2) end + return url +end + +-- Prepares three files for sync_cb to call to do the actual syncing: +-- * local_file (one that is being used) +-- * income_file (one that has just been downloaded from Cloud to be merged, then to be deleted) +-- * cached_file (the one that was uploaded in the previous round of syncing) +-- +-- How it works: +-- +-- If we simply merge the local file with the income file (ignore duplicates), then items that have been deleted locally +-- but not remotely (on other devices) will re-emerge in the result file. The same goes for items deleted remotely but +-- not locally. To avoid this, we first need to delete them from both the income file and local file. +-- +-- The problem is how to identify them, and that is when the cached file comes into play. +-- The cached file represents what local and remote agreed on previously (was identical to local and remote after being uploaded +-- the previous round), by comparing it with local file, items no longer in local file are ones being recently deleted. +-- The same applies to income file. Then we can delete them from both local and income files to be ready for merging. (The actual +-- deletion and merging procedures happen in sync_cb as users of this service will have different file specifications) +-- +-- After merging, the income file is no longer needed and is deleted. The local file is uploaded and then a copy of it is saved +-- and renamed to replace the old cached file (thus the naming). The cached file stays (in the same folder) till being replaced +-- in the next round. +function SyncService.sync(server, file_path, sync_cb, is_silent) + local file_name = ffiutil.basename(file_path) + local income_file_path = file_path .. ".temp" -- file downloaded from server + local cached_file_path = file_path .. ".sync" -- file uploaded to server last time + + local fail_msg = _("Something went wrong when syncing, please check your network connection and try again later.") + local show_msg = function(msg) + if is_silent then return end + UIManager:show(InfoMessage:new{ + text = msg or fail_msg, + timeout = 3, + }) + end + if server.type ~= "dropbox" and server.type ~= "webdav" then + show_msg(_("Wrong server type.")) + return + end + local code_response = 412 -- If-Match header failed + local etag + local api = server.type == "dropbox" and require("apps/cloudstorage/dropboxapi") or require("apps/cloudstorage/webdavapi") + while code_response == 412 do + os.remove(income_file_path) + if server.type == "dropbox" then + local url_base = server.url:sub(-1) == "/" and server.url or server.url.."/" + code_response, etag = api:downloadFile(url_base..file_name, server.password, income_file_path) + elseif server.type == "webdav" then + local path = api:getJoinedPath(server.address, server.url) + path = api:getJoinedPath(path, file_name) + code_response, etag = api:downloadFile(path, server.username, server.password, income_file_path) + end + if code_response ~= 200 and code_response ~= 404 + and not (server.type == "dropbox" and code_response == 409) then + show_msg() + return + end + local ok, cb_return = pcall(sync_cb, file_path, cached_file_path, income_file_path) + if not ok or not cb_return then + show_msg() + if not ok then require("logger").err("sync service callback failed:", cb_return) end + return + end + if server.type == "dropbox" then + local url_base = server.url == "/" and "" or server.url + code_response = api:uploadFile(url_base, server.password, file_path, etag, true) + elseif server.type == "webdav" then + local path = api:getJoinedPath(server.address, server.url) + path = api:getJoinedPath(path, file_name) + code_response = api:uploadFile(path, server.username, server.password, file_path, etag) + end + end + os.remove(income_file_path) + if type(code_response) == "number" and code_response >= 200 and code_response < 300 then + os.remove(cached_file_path) + ffiutil.copyFile(file_path, cached_file_path) + UIManager:show(Notification:new{ + text = _("Successfully synchronized."), + timeout = 2, + }) + else + show_msg() + end +end + + +return SyncService diff --git a/frontend/apps/cloudstorage/webdav.lua b/frontend/apps/cloudstorage/webdav.lua index a613e649a..3beb1238b 100644 --- a/frontend/apps/cloudstorage/webdav.lua +++ b/frontend/apps/cloudstorage/webdav.lua @@ -7,13 +7,14 @@ local UIManager = require("ui/uimanager") local ReaderUI = require("apps/reader/readerui") local WebDavApi = require("apps/cloudstorage/webdavapi") local util = require("util") +local ffiutil = require("ffi/util") local _ = require("gettext") local T = require("ffi/util").template local WebDav = {} -function WebDav:run(address, user, pass, path) - return WebDavApi:listFolder(address, user, pass, path) +function WebDav:run(address, user, pass, path, folder_mode) + return WebDavApi:listFolder(address, user, pass, path, folder_mode) end function WebDav:downloadFile(item, address, username, password, local_path, callback_close) @@ -48,6 +49,36 @@ function WebDav:downloadFile(item, address, username, password, local_path, call end end +function WebDav:uploadFile(url, address, username, password, local_path, callback_close) + local path = WebDavApi:getJoinedPath(address, url) + path = WebDavApi:getJoinedPath(path, ffiutil.basename(local_path)) + local code_response = WebDavApi:uploadFile(path, username, password, local_path) + if code_response >= 200 and code_response < 300 then + UIManager:show(InfoMessage:new{ + text = T(_("File uploaded:\n%1"), BD.filepath(address)), + }) + if callback_close then callback_close() end + else + UIManager:show(InfoMessage:new{ + text = T(_("Could not upload file:\n%1"), BD.filepath(address)), + timeout = 3, + }) + end +end + +function WebDav:createFolder(url, address, username, password, folder_name, callback_close) + local code_response = WebDavApi:createFolder(address .. WebDavApi:urlEncode(url .. "/" .. folder_name), username, password, folder_name) + if code_response == 201 then + if callback_close then + callback_close() + end + else + UIManager:show(InfoMessage:new{ + text = T(_("Could not create folder:\n%1"), folder_name), + }) + end +end + function WebDav:config(item, callback) local text_info = _([[Server address must be of the form http(s)://domain.name/path This can point to a sub-directory of the WebDAV server. diff --git a/frontend/apps/cloudstorage/webdavapi.lua b/frontend/apps/cloudstorage/webdavapi.lua index 1d7e3f13a..9a73fd961 100644 --- a/frontend/apps/cloudstorage/webdavapi.lua +++ b/frontend/apps/cloudstorage/webdavapi.lua @@ -59,7 +59,7 @@ function WebDavApi:urlEncode(url_data) return url_data end -function WebDavApi:listFolder(address, user, pass, folder_path) +function WebDavApi:listFolder(address, user, pass, folder_path, folder_mode) local path = self:urlEncode( folder_path ) local webdav_list = {} local webdav_file = {} @@ -169,6 +169,14 @@ function WebDavApi:listFolder(address, user, pass, folder_path) type = files.type, }) end + if folder_mode then + table.insert(webdav_list, 1, { + text = _("Long-press to choose current folder"), + url = folder_path, + type = "folder_long_press", + bold = true + }) + end return webdav_list end @@ -187,7 +195,42 @@ function WebDavApi:downloadFile(file_url, user, pass, local_path) logger.warn("WebDavApi: Download failure:", status or code or "network unreachable") logger.dbg("WebDavApi: Response headers:", headers) end + return code, (headers or {}).etag +end + +function WebDavApi:uploadFile(file_url, user, pass, local_path, etag) + socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT) + local code, _, status = socket.skip(1, http.request{ + url = file_url, + method = "PUT", + source = ltn12.source.file(io.open(local_path, "r")), + user = user, + password = pass, + headers = { + ["If-Match"] = etag + } + }) + socketutil:reset_timeout() + if code < 200 or code > 299 then + logger.warn("WebDavApi: upload failure:", status or code or "network unreachable") + end return code end +function WebDavApi:createFolder(folder_url, user, pass, folder_name) + socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT) + local code, _, status = socket.skip(1, http.request{ + url = folder_url, + method = "MKCOL", + user = user, + password = pass, + }) + socketutil:reset_timeout() + if code ~= 201 then + logger.warn("WebDavApi: create folder failure:", status or code or "network unreachable") + end + return code +end + + return WebDavApi diff --git a/plugins/statistics.koplugin/main.lua b/plugins/statistics.koplugin/main.lua index ccdb30971..348c1c625 100644 --- a/plugins/statistics.koplugin/main.lua +++ b/plugins/statistics.koplugin/main.lua @@ -14,6 +14,7 @@ local ReaderProgress = require("readerprogress") local ReadHistory = require("readhistory") local Screensaver = require("ui/screensaver") local SQ3 = require("lua-ljsqlite3/init") +local SyncService = require("frontend/apps/cloudstorage/syncservice") local UIManager = require("ui/uimanager") local Widget = require("ui/widget/widget") local lfs = require("libs/libkoreader-lfs") @@ -33,7 +34,7 @@ local DEFAULT_CALENDAR_START_DAY_OF_WEEK = 2 -- Monday local DEFAULT_CALENDAR_NB_BOOK_SPANS = 3 -- Current DB schema version -local DB_SCHEMA_VERSION = 20201022 +local DB_SCHEMA_VERSION = 20221111 -- This is the query used to compute the total time spent reading distinct pages of the book, -- capped at self.settings.max_sec per distinct page. @@ -381,6 +382,10 @@ Do you want to create an empty database? self:upgradeDBto20201022(conn) end + if db_version < 20221111 then + self:upgradeDBto20221111(conn) + end + -- Get back the space taken by the deleted page_stat table conn:exec("PRAGMA temp_store = 2;") -- use memory for temp files local ok, errmsg = pcall(conn.exec, conn, "VACUUM;") -- this may take some time @@ -541,7 +546,7 @@ function ReaderStatistics:createDB(conn) conn:exec(sql_stmt) -- Index sql_stmt = [[ - CREATE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5); + CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5); ]] conn:exec(sql_stmt) @@ -598,6 +603,41 @@ function ReaderStatistics:upgradeDBto20201022(conn) conn:exec("PRAGMA user_version=20201022;") end +function ReaderStatistics:upgradeDBto20221111(conn) + conn:exec([[ + -- We make the index on book's (title, author, md5) unique in order to sync dbs + -- First we fill null authors with '' + UPDATE book SET authors = '' WHERE authors IS NULL; + -- Secondly, we unify the id_book in page_stat_data entries for duplicate books + -- to the smallest of each, so as to delete the others. + UPDATE page_stat_data SET id_book = ( + SELECT map.min_id FROM ( + SELECT id, ( + SELECT min(id) FROM book b2 + WHERE (book.title, book.authors, book.md5) = (b2.title, b2.authors, b2.md5) + ) as min_id + FROM book WHERE book.id >= min_id + ) as map WHERE page_stat_data.id_book = map.id + ); + -- Delete duplicate books and keep the one with smallest id. + DELETE FROM book WHERE id > ( + SELECT MIN(id) FROM book b2 + WHERE (book.title, book.authors, book.md5) = (b2.title, b2.authors, b2.md5) + ); + -- Then we recompute the book statistics based on merged books + UPDATE book SET (total_read_pages, total_read_time) = + (SELECT count(DISTINCT page), + sum(duration) + FROM page_stat + WHERE id_book = book.id); + -- Finally we update the index to be unique + DROP INDEX IF EXISTS book_title_authors_md5; + CREATE UNIQUE INDEX book_title_authors_md5 ON book(title, authors, md5);]]) + + -- Update DB schema version + conn:exec("PRAGMA user_version=20221111;") +end + function ReaderStatistics:addBookStatToDB(book_stats, conn) local id_book local last_open_book = 0 @@ -1046,6 +1086,71 @@ The max value ensures a page you stay on for a long time (because you fell aslee callback = function() self.settings.calendar_browse_future_months = not self.settings.calendar_browse_future_months end, + separator = true, + }, + { + text = _("Cloud sync"), + callback = function(touchmenu_instance) + local server = self.settings.sync_server + local edit_cb = function() + local sync_settings = SyncService:new{} + sync_settings.onClose = function(this) + UIManager:close(this) + end + sync_settings.onConfirm = function(sv) + self.settings.sync_server = sv + touchmenu_instance:updateItems() + end + UIManager:show(sync_settings) + end + if not server then + edit_cb() + return + end + local dialogue + local delete_button = { + text = _("Delete"), + callback = function() + UIManager:close(dialogue) + UIManager:show(ConfirmBox:new{ + text = _("Delete server info?"), + cancel_text = _("Cancel"), + cancel_callback = function() + return + end, + ok_text = _("Delete"), + ok_callback = function() + self.settings.sync_server = nil + touchmenu_instance:updateItems() + end, + }) + end, + } + local edit_button = { + text = _("Edit"), + callback = function() + UIManager:close(dialogue) + edit_cb() + end + } + local close_button = { + text = _("Close"), + callback = function() + UIManager:close(dialogue) + end + } + local type = server.type == "dropbox" and " (DropBox)" or " (WebDAV)" + dialogue = require("ui/widget/buttondialogtitle"):new{ + title = T(_("Cloud storage:\n%1\n\nFolder path:\n%2\n\nSet up the same cloud folder on each device to sync across your devices."), + server.name.." "..type, SyncService.getReadablePath(server)), + buttons = { + {delete_button, edit_button, close_button} + }, + } + UIManager:show(dialogue) + end, + enabled_func = function() return self.settings.is_enabled end, + keep_menu_open = true, }, }, }, @@ -1054,6 +1159,17 @@ The max value ensures a page you stay on for a long time (because you fell aslee sub_item_table = self:genResetBookSubItemTable(), separator = true, }, + { + text = _("Synchronize now"), + callback = function() + SyncService.sync(self.settings.sync_server, db_location, self.onSync ) + end, + enabled_func = function() + return self.settings.sync_server ~= nil and self.settings.is_enabled and require("ui/network/manager"):isWifiOn() + end, + keep_menu_open = true, + separator = true, + }, { text = _("Current book"), keep_menu_open = true, @@ -2636,4 +2752,124 @@ function ReaderStatistics:getCurrentBookReadPages() return read_pages end +function ReaderStatistics.onSync(local_path, cached_path, income_path) + local conn_income = SQ3.open(income_path) + local ok1, v1 = pcall(conn_income.rowexec, conn_income, "PRAGMA schema_version") + if not ok1 or tonumber(v1) == 0 then + -- no income db or wrong db, first time sync + logger.warn("statistics open income DB failed", v1) + return true + end + + local sql = "attach '" .. income_path:gsub("'", "''") .."' as income_db;" + -- then we try to open cached db + local conn_cached = SQ3.open(cached_path) + local ok2, v2 = pcall(conn_cached.rowexec, conn_cached, "PRAGMA schema_version") + local attached_cache + if not ok2 or tonumber(v2) == 0 then + -- no cached or error, no item to delete + logger.warn("statistics open cached DB failed", v2) + else + attached_cache = true + sql = sql .. "attach '" .. cached_path:gsub("'", "''") ..[[' as cached_db; + -- first we delete from income_db books that exist in cached_db but not in local_db, + -- namely the ones that were deleted since last sync + DELETE FROM income_db.page_stat_data WHERE id_book IN ( + SELECT id FROM income_db.book WHERE (title, authors, md5) IN ( + SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN ( + SELECT title, authors, md5 FROM book + ) + ) + ); + DELETE FROM income_db.book WHERE (title, authors, md5) IN ( + SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN ( + SELECT title, authors, md5 FROM book + ) + ); + + -- then we delete books from local db that were present in last sync but + -- not any more (ie. deleted in other devices) + DELETE FROM page_stat_data WHERE id_book IN ( + SELECT id FROM book WHERE (title, authors, md5) IN ( + SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN ( + SELECT title, authors, md5 FROM income_db.book + ) + ) + ); + DELETE FROM book WHERE (title, authors, md5) IN ( + SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN ( + SELECT title, authors, md5 FROM income_db.book + ) + ); + ]] + end + + conn_cached:close() + conn_income:close() + local conn = SQ3.open(local_path) + local ok3, v3 = pcall(conn.exec, conn, "PRAGMA schema_version") + if not ok3 or tonumber(v3) == 0 then + -- no local db, this is an error + logger.err("statistics open local DB", v3) + return false + end + + sql = sql .. [[ + -- We merge the local db with income db to form the synced db. + -- Do the books + INSERT INTO book ( + title, authors, notes, last_open, highlights, pages, series, language, md5, total_read_time, total_read_pages + ) SELECT + title, authors, notes, last_open, highlights, pages, series, language, md5, total_read_time, total_read_pages + FROM income_db.book WHERE true ON CONFLICT (title, authors, md5) DO NOTHING; + + -- We create a book_id mapping temp table (view not possible due to attached db) + CREATE TEMP TABLE book_id_map AS + SELECT m.id as mid, ifnull(i.id, m.id) as iid FROM book m --main + LEFT JOIN income_db.book i + ON (m.title, m.authors, m.md5) = (i.title, i.authors, i.md5); + ]] + if attached_cache then + -- more deletion needed + sql = sql .. [[ + -- DELETE stat_data items + DELETE FROM income_db.page_stat_data WHERE (id_book, page, start_time) IN ( + SELECT map.iid, page, start_time FROM cached_db.page_stat_data + LEFT JOIN book_id_map AS map ON id_book = map.mid + WHERE (id_book, page, start_time) NOT IN ( + SELECT id_book, page, start_time FROM page_stat_data + ) + ); + DELETE FROM page_stat_data WHERE (id_book, page, start_time) IN ( + SELECT id_book, page, start_time FROM cached_db.page_stat_data WHERE (id_book, page, start_time) NOT IN ( + SELECT map.mid, page, start_time FROM income_db.page_stat_data + LEFT JOIN book_id_map AS map on id_book = map.iid + ) + );]] + end + sql = sql .. [[ + -- Then we merge the income_db's contents into the local db + INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) + SELECT map.mid, page, start_time, duration, total_pages + FROM income_db.page_stat_data + LEFT JOIN book_id_map as map + ON id_book = map.iid + WHERE true + ON CONFLICT(id_book, page, start_time) DO UPDATE SET + duration = MAX(duration, excluded.duration); + + -- finally we update the total numbers of book + UPDATE book SET (total_read_pages, total_read_time) = + (SELECT count(DISTINCT page), + sum(duration) + FROM page_stat + WHERE id_book = book.id); + ]] + conn:exec(sql) + pcall(conn.exec, conn, "COMMIT;") + conn:exec("DETACH income_db;"..(attached_cache and "DETACH cached_db;" or "")) + conn:close() + return true +end + return ReaderStatistics diff --git a/plugins/vocabbuilder.koplugin/db.lua b/plugins/vocabbuilder.koplugin/db.lua index 19163407f..13288db2f 100644 --- a/plugins/vocabbuilder.koplugin/db.lua +++ b/plugins/vocabbuilder.koplugin/db.lua @@ -2,6 +2,7 @@ local DataStorage = require("datastorage") local Device = require("device") local SQ3 = require("lua-ljsqlite3/init") local LuaData = require("luadata") +local logger = require("logger") local db_location = DataStorage:getSettingsDir() .. "/vocabulary_builder.sqlite3" @@ -30,7 +31,9 @@ local VOCABULARY_DB_SCHEMA = [[ CREATE INDEX IF NOT EXISTS title_name_index ON title(name); ]] -local VocabularyBuilder = {} +local VocabularyBuilder = { + path = db_location +} function VocabularyBuilder:init() VocabularyBuilder:createDB() @@ -248,6 +251,7 @@ function VocabularyBuilder:batchUpdateItems(items) stmt:bind(item.review_count, item.streak_count, item.review_time, item.due_time, item.word) stmt:step() stmt:clearbind():reset() + item.review_time = nil end end @@ -338,6 +342,104 @@ function VocabularyBuilder:purge() conn:close() end + +-- Synchronization +function VocabularyBuilder.onSync(local_path, cached_path, income_path) + -- we try to open income db + local conn_income = SQ3.open(income_path) + local ok1, v1 = pcall(conn_income.rowexec, conn_income, "PRAGMA schema_version") + if not ok1 or tonumber(v1) == 0 then + -- no income db or wrong db, first time sync + logger.dbg("vocabbuilder open income DB failed", v1) + return true + end + + local sql = "attach '" .. income_path:gsub("'", "''") .."' as income_db;" + -- then we try to open cached db + local conn_cached = SQ3.open(cached_path) + local ok2, v2 = pcall(conn_cached.rowexec, conn_cached, "PRAGMA schema_version") + local attached_cache + if not ok2 or tonumber(v2) == 0 then + -- no cached or error, no item to delete + logger.dbg("vocabbuilder open cached DB failed", v2) + else + attached_cache = true + sql = sql .. "attach '" .. cached_path:gsub("'", "''") ..[[' as cached_db; + -- first we delete from income_db words that exist in cached_db but not in local_db, + -- namely the ones that were deleted since last sync + DELETE FROM income_db.vocabulary WHERE word IN ( + SELECT word FROM cached_db.vocabulary WHERE word NOT IN ( + SELECT word FROM vocabulary + ) + ); + -- We need to delete words that were delete in income_db since last sync + DELETE FROM vocabulary WHERE word IN ( + SELECT word FROM cached_db.vocabulary WHERE word NOT IN ( + SELECT word FROM income_db.vocabulary + ) + ); + ]] + end + + conn_cached:close() + conn_income:close() + local conn = SQ3.open(local_path) + local ok3, v3 = pcall(conn.exec, conn, "PRAGMA schema_version") + if not ok3 or tonumber(v3) == 0 then + -- no local db, this is an error + logger.err("vocabbuilder open local DB", v3) + return false + end + + sql = sql .. [[ + -- We merge the local db with income db to form the synced db. + -- First we do the books + INSERT OR IGNORE INTO title (name) SELECT name FROM income_db.title; + + -- Then update income db's book title id references + UPDATE income_db.vocabulary SET title_id = ifnull( + (SELECT mid FROM ( + SELECT m.id as mid, title_id as i_tid FROM title as m -- main db + INNER JOIN income_db.title as i -- income db + ON m.name = i.name + LEFT JOIN income_db.vocabulary + on title_id = i.id + ) WHERE income_db.vocabulary.title_id = i_tid + ) , title_id); + + -- Then we merge the income_db's contents into the local db + INSERT INTO vocabulary + (word, create_time, review_time, due_time, review_count, prev_context, next_context, title_id, streak_count) + SELECT word, create_time, review_time, due_time, review_count, prev_context, next_context, title_id, streak_count + FROM income_db.vocabulary WHERE true + ON CONFLICT(word) DO UPDATE SET + due_time = MAX(due_time, excluded.due_time), + review_count = CASE + WHEN create_time = excluded.create_time THEN MAX(review_count, excluded.review_count) + ELSE review_count + excluded.review_count + END, + prev_context = ifnull(excluded.prev_context, prev_context), + next_context = ifnull(excluded.next_context, next_context), + streak_count = CASE + WHEN review_time > excluded.review_time THEN streak_count + ELSE excluded.streak_count + END, + review_time = MAX(review_time, excluded.review_time), + create_time = excluded.create_time, -- we always use the remote value to eliminate duplicate review_count sum + title_id = excluded.title_id -- use remote in case re-assignable book id be supported + ]] + conn:exec(sql) + pcall(conn.exec, conn, "COMMIT;") + conn:exec("DETACH income_db;"..(attached_cache and "DETACH cached_db;" or "")) + conn:exec("PRAGMA temp_store = 2;") -- use memory for temp files + local ok, errmsg = pcall(conn.exec, conn, "VACUUM;") -- we upload a compact file + if not ok then + logger.warn("Failed compacting vocab database:", errmsg) + end + conn:close() + return true +end + VocabularyBuilder:init() return VocabularyBuilder diff --git a/plugins/vocabbuilder.koplugin/main.lua b/plugins/vocabbuilder.koplugin/main.lua index e4a48fa60..5c5b4a2f6 100644 --- a/plugins/vocabbuilder.koplugin/main.lua +++ b/plugins/vocabbuilder.koplugin/main.lua @@ -9,6 +9,7 @@ local Blitbuffer = require("ffi/blitbuffer") local BottomContainer = require("ui/widget/container/bottomcontainer") local DB = require("db") local Button = require("ui/widget/button") +local ButtonDialogTitle = require("ui/widget/buttondialogtitle") local ButtonTable = require("ui/widget/buttontable") local CenterContainer = require("ui/widget/container/centercontainer") local ConfirmBox = require("ui/widget/confirmbox") @@ -35,6 +36,7 @@ local OverlapGroup = require("ui/widget/overlapgroup") local Screen = Device.screen local Size = require("ui/size") local SortWidget = require("ui/widget/sortwidget") +local SyncService = require("frontend/apps/cloudstorage/syncservice") local TextWidget = require("ui/widget/textwidget") local TextBoxWidget = require("ui/widget/textboxwidget") local TitleBar = require("ui/widget/titlebar") @@ -171,7 +173,7 @@ function MenuDialog:init() end local size = Screen:getSize() - local width = math.floor(size.w * 0.8) + local width = math.floor(size.w * 0.9) -- Switch text translations could be long local temp_text_widget = TextWidget:new{ @@ -277,14 +279,85 @@ function MenuDialog:init() end, } + local show_sync_settings = function() + if not settings.server then + local sync_settings = SyncService:new{} + sync_settings.onClose = function(this) + UIManager:close(this) + end + sync_settings.onConfirm = function(server) + settings.server = server + saveSettings() + DB:batchUpdateItems(self.show_parent.item_table) + SyncService.sync(server, DB.path, DB.onSync, false) + self.show_parent:reloadItems() + end + UIManager:close(self.sync_dialogue) + UIManager:close(self) + UIManager:show(sync_settings) + return + end + local server = settings.server + local buttons = { + { + { + text = _("Delete"), + callback = function() + settings.server = nil + UIManager:close(self.sync_dialogue) + end + }, + { + text = _("Edit"), + callback = function() + UIManager:close(self.sync_dialogue) + UIManager:close(self) + local sync_settings = SyncService:new{} + sync_settings.onClose = function(this) + UIManager:close(this) + end + + sync_settings.onConfirm = function(chosen_server) + settings.server = chosen_server + end + UIManager:show(sync_settings) + end + }, + { + text = _("Synchronize now"), + callback = function() + UIManager:close(self.sync_dialogue) + UIManager:close(self) + DB:batchUpdateItems(self.show_parent.item_table) + SyncService.sync(server, DB.path, DB.onSync, false) + self.show_parent:reloadItems() + end + } + } + } + local type = server.type == "dropbox" and " (DropBox)" or " (WebDAV)" + self.sync_dialogue = ButtonDialogTitle:new{ + title = T(_("Cloud storage:\n%1\n\nFolder path:\n%2\n\nSet up the same cloud folder on each device to sync across your devices"), + server.name.." "..type, SyncService.getReadablePath(server)), + info_face = Font:getFace("smallinfofont"), + buttons = buttons, + } + UIManager:show(self.sync_dialogue) + end + local sync_button = { + text = _("Cloud sync"), + callback = function() + show_sync_settings() + end + } + local buttons = ButtonTable:new{ width = width, buttons = { - {filter_button}, {reverse_button}, - {edit_button}, - {reset_button}, - {clean_button} + {sync_button}, + {filter_button, edit_button}, + {reset_button, clean_button}, }, show_parent = self } @@ -1045,6 +1118,8 @@ function VocabularyBuilderWidget:init() self.item_width = self.dimen.w - 2 * padding self.footer_center_width = math.floor(self.width_widget * (32/100)) self.footer_button_width = math.floor(self.width_widget * (12/100)) + self.footer_left_corner_width = math.floor(self.width_widget * (8/100)) + self.footer_right_corner_width = math.floor(self.width_widget * (12/100)) -- group for footer local chevron_left = "chevron.left" local chevron_right = "chevron.right" @@ -1091,6 +1166,40 @@ function VocabularyBuilderWidget:init() show_parent = self, } + self.footer_sync = Button:new{ + text = "⇅", + width = self.footer_left_corner_width, + text_font_size = 18, + bordersize = 0, + radius = 0, + padding = Size.padding.large, + show_parent = self, + callback = function() + if not settings.server then + local sync_settings = SyncService:new{} + sync_settings.onClose = function(this) + UIManager:close(this) + end + sync_settings.onConfirm = function(server) + settings.server = server + saveSettings() + DB:batchUpdateItems(self.item_table) + SyncService.sync(server, DB.path, DB.onSync, false) + self:reloadItems() + end + UIManager:show(sync_settings) + else + -- manual sync + DB:batchUpdateItems(self.item_table) + UIManager:nextTick(function() + SyncService.sync(settings.server, DB.path, DB.onSync, false) + self:reloadItems() + end) + end + end + } + self.footer_sync.label_widget.fgcolor = Blitbuffer.COLOR_GRAY_3 + self.footer_page = Button:new{ text = "", hold_input = { @@ -1117,11 +1226,13 @@ function VocabularyBuilderWidget:init() show_parent = self, } self.page_info = HorizontalGroup:new{ + self.footer_sync, self.footer_first_up, self.footer_left, self.footer_page, self.footer_right, self.footer_last_down, + HorizontalSpan:new{ width = self.footer_right_corner_width } } local bottom_line = LineWidget:new{ @@ -1270,6 +1381,7 @@ function VocabularyBuilderWidget:_populateItems() item ) end + table.insert(self.layout, #self.layout, {self.footer_sync}) if #self.main_content == 0 then table.insert(self.main_content, HorizontalSpan:new{width = self.item_width}) end