Archives/koreader
2
0
Fork 0
mirror of https://github.com/koreader/koreader synced 2024-11-08 07:10:27 +00:00
Code Issues Projects Releases Wiki Activity
8500fdd519
koreader/plugins/statistics.koplugin/main.lua
weijiuqiao 8500fdd519
Cloud-based sync for 2 plugins: reading statistics and vocabulary builder (#9709) 
This commit adds cross-device sync ability for two plugins: reading statistics and vocabulary builder. It relies on user setting up a Cloud server (DropBox and WebDAV but not FTP though) and designating a path. Behind the curtains sqlite databases are being passed around and updated.

UI-wise, for the statistics plugin, two new menu items Synchronize now and Cloud sync to set it up (might not be the best wording) are added. As for vocabulary builder, a similar Cloud sync button is added to the menu and a shortcut icon button to Synchronize now is pinned at the bottom corner.

CloudStorage new features: WebDAV creating folders and uploading files. And a new widget-like sync server chooser. In the end I decided not to add automatic sync, as the SQL commands part seem a bit much.
2022-11-11 15:53:06 +01:00

2876 lines
114 KiB
Lua
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

local BD = require("ui/bidi")
local BookStatusWidget = require("ui/widget/bookstatuswidget")
local ConfirmBox = require("ui/widget/confirmbox")
local DataStorage = require("datastorage")
local Device = require("device")
local Dispatcher = require("dispatcher")
local DocSettings = require("docsettings")
local FFIUtil = require("ffi/util")
local InfoMessage = require("ui/widget/infomessage")
local KeyValuePage = require("ui/widget/keyvaluepage")
local Math = require("optmath")
local ReaderFooter = require("apps/reader/modules/readerfooter")
local ReaderProgress = require("readerprogress")
local ReadHistory = require("readhistory")
local Screensaver = require("ui/screensaver")
local SQ3 = require("lua-ljsqlite3/init")
local SyncService = require("frontend/apps/cloudstorage/syncservice")
local UIManager = require("ui/uimanager")
local Widget = require("ui/widget/widget")
local lfs = require("libs/libkoreader-lfs")
local logger = require("logger")
local util = require("util")
local _ = require("gettext")
local C_ = _.pgettext
local N_ = _.ngettext
local T = FFIUtil.template
local statistics_dir = DataStorage:getDataDir() .. "/statistics/"
local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3"
local MAX_PAGETURNS_BEFORE_FLUSH = 50
local DEFAULT_MIN_READ_SEC = 5
local DEFAULT_MAX_READ_SEC = 120
local DEFAULT_CALENDAR_START_DAY_OF_WEEK = 2 -- Monday
local DEFAULT_CALENDAR_NB_BOOK_SPANS = 3
-- Current DB schema version
local DB_SCHEMA_VERSION = 20221111
-- This is the query used to compute the total time spent reading distinct pages of the book,
-- capped at self.settings.max_sec per distinct page.
-- c.f., comments in insertDB
local STATISTICS_SQL_BOOK_CAPPED_TOTALS_QUERY = [[
SELECT count(*),
sum(durations)
FROM (
SELECT min(sum(duration), %d) AS durations
FROM page_stat
WHERE id_book = %d
GROUP BY page
);
]]
-- As opposed to the uncapped version
local STATISTICS_SQL_BOOK_TOTALS_QUERY = [[
SELECT count(DISTINCT page),
sum(duration)
FROM page_stat
WHERE id_book = %d;
]]
local ReaderStatistics = Widget:extend{
name = "statistics",
start_current_period = 0,
curr_page = 0,
id_curr_book = nil,
is_enabled = nil,
convert_to_db = nil, -- true when migration to DB has been done
pageturn_count = 0,
mem_read_time = 0,
mem_read_pages = 0,
book_read_pages = 0,
book_read_time = 0,
avg_time = nil,
page_stat = nil, -- Dictionary, indexed by page (hash), contains a list (array) of { timestamp, duration } tuples.
data = nil, -- table
}
local weekDays = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" } -- in Lua wday order
local shortDayOfWeekTranslation = {
["Mon"] = _("Mon"),
["Tue"] = _("Tue"),
["Wed"] = _("Wed"),
["Thu"] = _("Thu"),
["Fri"] = _("Fri"),
["Sat"] = _("Sat"),
["Sun"] = _("Sun"),
}
local longDayOfWeekTranslation = {
["Mon"] = _("Monday"),
["Tue"] = _("Tuesday"),
["Wed"] = _("Wednesday"),
["Thu"] = _("Thursday"),
["Fri"] = _("Friday"),
["Sat"] = _("Saturday"),
["Sun"] = _("Sunday"),
}
local monthTranslation = {
["January"] = _("January"),
["February"] = _("February"),
["March"] = _("March"),
["April"] = _("April"),
["May"] = _("May"),
["June"] = _("June"),
["July"] = _("July"),
["August"] = _("August"),
["September"] = _("September"),
["October"] = _("October"),
["November"] = _("November"),
["December"] = _("December"),
}
function ReaderStatistics:isDocless()
return self.ui == nil or self.ui.document == nil or self.ui.document.is_pic == true
end
-- NOTE: This is used in a migration script by ui/data/onetime_migration,
-- which is why it's public.
ReaderStatistics.default_settings = {
min_sec = DEFAULT_MIN_READ_SEC,
max_sec = DEFAULT_MAX_READ_SEC,
is_enabled = true,
convert_to_db = nil,
calendar_start_day_of_week = DEFAULT_CALENDAR_START_DAY_OF_WEEK,
calendar_nb_book_spans = DEFAULT_CALENDAR_NB_BOOK_SPANS,
calendar_show_histogram = true,
calendar_browse_future_months = false,
}
function ReaderStatistics:onDispatcherRegisterActions()
Dispatcher:registerAction("stats_calendar_view", {category="none", event="ShowCalendarView", title=_("Statistics calendar view"), general=true, separator=true})
Dispatcher:registerAction("book_statistics", {category="none", event="ShowBookStats", title=_("Book statistics"), reader=true, separator=true})
end
function ReaderStatistics:init()
-- Disable in PIC documents (but not the FM, as we want to be registered to the FM's menu).
if self.ui and self.ui.document and self.ui.document.is_pic then
return
end
-- Placeholder until onReaderReady
self.data = {
title = "",
authors = "N/A",
language = "N/A",
series = "N/A",
performance_in_pages = {},
total_time_in_sec = 0,
highlights = 0,
notes = 0,
pages = 0,
md5 = nil,
}
self.start_current_period = os.time()
self:resetVolatileStats()
self.settings = G_reader_settings:readSetting("statistics", self.default_settings)
self.ui.menu:registerToMainMenu(self)
self:onDispatcherRegisterActions()
self:checkInitDatabase()
BookStatusWidget.getStats = function()
return self:getStatsBookStatus(self.id_curr_book, self.settings.is_enabled)
end
ReaderFooter.getAvgTimePerPage = function()
if self.settings.is_enabled then
return self.avg_time
end
end
Screensaver.getAvgTimePerPage = function()
if self.settings.is_enabled then
return self.avg_time
end
end
Screensaver.getReaderProgress = function()
self:insertDB()
local current_duration, current_pages = self:getCurrentBookStats()
local today_duration, today_pages = self:getTodayBookStats()
local dates_stats = self:getReadingProgressStats(7)
local readingprogress
if dates_stats then
readingprogress = ReaderProgress:new{
dates = dates_stats,
current_duration = current_duration,
current_pages = current_pages,
today_duration = today_duration,
today_pages = today_pages,
readonly = true,
}
end
return readingprogress
end
end
function ReaderStatistics:initData()
if self:isDocless() or not self.settings.is_enabled then
return
end
-- first execution
if not self.data then
self.data = { performance_in_pages= {} }
end
local book_properties = self:getBookProperties()
self.data.title = book_properties.title
if self.data.title == nil or self.data.title == "" then
self.data.title = self.document.file:match("^.+/(.+)$")
end
self.data.authors = book_properties.authors
if self.data.authors == nil or self.data.authors == "" then
self.data.authors = "N/A"
end
self.data.language = book_properties.language
if self.data.language == nil or self.data.language == "" then
self.data.language = "N/A"
end
self.data.series = book_properties.series
if self.data.series == nil or self.data.series == "" then
self.data.series = "N/A"
end
self.data.pages = self.view.document:getPageCount()
if not self.data.md5 then
self.data.md5 = self:partialMd5(self.document.file)
end
-- Update these numbers to what's actually stored in the settings
-- (not that "notes" is invalid and does not represent edited highlights)
self.data.highlights, self.data.notes = self.ui.bookmark:getNumberOfHighlightsAndNotes()
self.id_curr_book = self:getIdBookDB()
self.book_read_pages, self.book_read_time = self:getPageTimeTotalStats(self.id_curr_book)
if self.book_read_pages > 0 then
self.avg_time = self.book_read_time / self.book_read_pages
else
-- NOTE: Possibly less weird-looking than initializing this to 0?
self.avg_time = math.floor(0.50 * self.settings.max_sec)
logger.dbg("ReaderStatistics: Initializing average time per page at 50% of the max value, i.e.,", self.avg_time)
end
end
function ReaderStatistics:isEnabled()
return not self:isDocless() and self.settings.is_enabled
end
-- Reset the (volatile) stats on page count changes (e.g., after a font size update)
function ReaderStatistics:onDocumentRerendered()
-- Note: this is called *after* onPageUpdate(new current page in new page count), which
-- has updated the duration for (previous current page in old page count) and created
-- a tuple for (new current page) with a 0-duration.
-- The insertDB() call below will save the previous page stat correctly with the old
-- page count, and will drop the new current page stat.
-- Only after this insertDB(), self.data.pages is updated with the new page count.
--
-- To make this clearer, here's what happens with an example:
-- - We were reading page 127/200 with latest self.page_stat[127]={..., {now-35s, 0}}
-- - Increasing font size, re-rendering... going to page 153/254
-- - OnPageUpdate(153) is called:
-- - it updates duration in self.page_stat[127]={..., {now-35s, 35}}
-- - it adds/creates self.page_stat[153]={..., {now, 0}}
-- - it sets self.curr_page=153
-- - (at this point, we don't know the new page count is 254)
-- - OnDocumentRerendered() is called:
-- - insertDB() is called, which will still use the previous self.data.pages=200 as the
-- page count, and will go at inserting or not in the DB:
-- - (127, now-35s, 35, 200) inserted
-- - (153, now, 0, 200) not inserted as 0-duration (and using 200 for its associated
-- page count would be erroneous)
-- and will restore self.page_stat[153]={{now, 0}}
-- - we only then update self.data.pages=254 as the new page count
-- - 5 minutes later, on the next insertDB(), (153, now-5mn, 42, 254) will be inserted in DB
local new_pagecount = self.view.document:getPageCount()
if new_pagecount ~= self.data.pages then
logger.dbg("ReaderStatistics: Pagecount change, flushing volatile book statistics")
-- Flush volatile stats to DB for current book, and update pagecount and average time per page stats
self:insertDB(new_pagecount)
end
-- Update our copy of the page count
self.data.pages = new_pagecount
end
function ReaderStatistics:resetVolatileStats(now_ts)
-- Computed by onPageUpdate
self.pageturn_count = 0
self.mem_read_time = 0
self.mem_read_pages = 0
-- Volatile storage pending flush to db
self.page_stat = {}
-- Re-seed the volatile stats with minimal data about the current page.
-- If a timestamp is passed, it's the caller's responsibility to ensure that self.curr_page is accurate.
if now_ts then
self.page_stat[self.curr_page] = { { now_ts, 0 } }
end
end
function ReaderStatistics:getStatsBookStatus(id_curr_book, stat_enable)
if not stat_enable or id_curr_book == nil then
return {}
end
self:insertDB()
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT count(*)
FROM (
SELECT strftime('%%Y-%%m-%%d', start_time, 'unixepoch', 'localtime') AS dates
FROM page_stat
WHERE id_book = %d
GROUP BY dates
);
]]
local total_days = conn:rowexec(string.format(sql_stmt, id_curr_book))
local total_read_pages, total_time_book = conn:rowexec(string.format(STATISTICS_SQL_BOOK_TOTALS_QUERY, id_curr_book))
conn:close()
if total_time_book == nil then
total_time_book = 0
end
if total_read_pages == nil then
total_read_pages = 0
end
return {
days = tonumber(total_days),
time = tonumber(total_time_book),
pages = tonumber(total_read_pages),
}
end
function ReaderStatistics:checkInitDatabase()
local conn = SQ3.open(db_location)
if self.settings.convert_to_db then -- if conversion to sqlite DB has already been done
if not conn:exec("PRAGMA table_info('book');") then
UIManager:show(ConfirmBox:new{
text = T(_([[
Cannot open database in %1.
The database may have been moved or deleted.
Do you want to create an empty database?
]]),
BD.filepath(db_location)),
cancel_text = _("Close"),
cancel_callback = function()
return
end,
ok_text = _("Create"),
ok_callback = function()
local conn_new = SQ3.open(db_location)
self:createDB(conn_new)
conn_new:close()
UIManager:show(InfoMessage:new{text =_("A new empty database has been created."), timeout = 3 })
self:initData()
end,
})
end
-- Check if we need to migrate to a newer schema
local db_version = tonumber(conn:rowexec("PRAGMA user_version;"))
if db_version < DB_SCHEMA_VERSION then
logger.info("ReaderStatistics: Migrating DB from schema", db_version, "to schema", DB_SCHEMA_VERSION, "...")
-- Backup the existing DB first
conn:close()
local bkp_db_location = db_location .. ".bkp." .. db_version .. "-to-" .. DB_SCHEMA_VERSION
-- Don't overwrite an existing backup
if lfs.attributes(bkp_db_location, "mode") == "file" then
logger.warn("ReaderStatistics: A DB backup from schema", db_version, "to schema", DB_SCHEMA_VERSION, "already exists!")
else
FFIUtil.copyFile(db_location, bkp_db_location)
logger.info("ReaderStatistics: Old DB backed up as", bkp_db_location)
end
conn = SQ3.open(db_location)
if db_version < 20201010 then
self:upgradeDBto20201010(conn)
end
if db_version < 20201022 then
self:upgradeDBto20201022(conn)
end
if db_version < 20221111 then
self:upgradeDBto20221111(conn)
end
-- Get back the space taken by the deleted page_stat table
conn:exec("PRAGMA temp_store = 2;") -- use memory for temp files
local ok, errmsg = pcall(conn.exec, conn, "VACUUM;") -- this may take some time
if not ok then
logger.warn("Failed compacting statistics database:", errmsg)
end
logger.info("ReaderStatistics: DB migration complete")
UIManager:show(InfoMessage:new{text =_("Statistics database updated."), timeout = 3 })
elseif db_version > DB_SCHEMA_VERSION then
logger.warn("ReaderStatistics: You appear to be using a database with an unknown schema version:", db_version, "instead of", DB_SCHEMA_VERSION)
logger.warn("ReaderStatistics: Expect things to break in fun and interesting ways!")
-- We can't know what might happen, so, back the DB up...
conn:close()
local bkp_db_location = db_location .. ".bkp." .. db_version .. "-to-" .. DB_SCHEMA_VERSION
-- Don't overwrite an existing backup
if lfs.attributes(bkp_db_location, "mode") == "file" then
logger.warn("ReaderStatistics: A DB backup from schema", db_version, "to schema", DB_SCHEMA_VERSION, "already exists!")
else
FFIUtil.copyFile(db_location, bkp_db_location)
logger.info("ReaderStatistics: Old DB backed up as", bkp_db_location)
end
conn = SQ3.open(db_location)
end
else -- Migrate stats for books in history from metadata.lua to sqlite database
self.settings.convert_to_db = true
if not conn:exec("PRAGMA table_info('book');") then
local filename_first_history, quickstart_filename, __
if #ReadHistory.hist == 1 then
filename_first_history = ReadHistory.hist[1]["text"]
local quickstart_path = require("ui/quickstart").quickstart_filename
__, quickstart_filename = util.splitFilePathName(quickstart_path)
end
if #ReadHistory.hist > 1 or (#ReadHistory.hist == 1 and filename_first_history ~= quickstart_filename) then
local info = InfoMessage:new{
text = _([[
New version of statistics plugin detected.
Statistics data needs to be converted into the new database format.
This may take a few minutes.
Please wait…
]])}
UIManager:show(info)
UIManager:forceRePaint()
local nr_book = self:migrateToDB(conn)
UIManager:close(info)
UIManager:forceRePaint()
UIManager:show(InfoMessage:new{
text = T(N_("Conversion complete.\nImported one book to the database.\nTap to continue.", "Conversion complete.\nImported %1 books to the database.\nTap to continue."), nr_book) })
else
self:createDB(conn)
end
end
end
conn:close()
end
function ReaderStatistics:partialMd5(file)
if file == nil then
return nil
end
local bit = require("bit")
local md5 = require("ffi/sha2").md5
local lshift = bit.lshift
local step, size = 1024, 1024
local update = md5()
local file_handle = io.open(file, 'rb')
for i = -1, 10 do
file_handle:seek("set", lshift(step, 2*i))
local sample = file_handle:read(size)
if sample then
update(sample)
else
break
end
end
file_handle:close()
return update()
end
-- Mainly so we don't duplicate the schema twice between the creation/upgrade codepaths
local STATISTICS_DB_PAGE_STAT_DATA_SCHEMA = [[
CREATE TABLE IF NOT EXISTS page_stat_data
(
id_book integer,
page integer NOT NULL DEFAULT 0,
start_time integer NOT NULL DEFAULT 0,
duration integer NOT NULL DEFAULT 0,
total_pages integer NOT NULL DEFAULT 0,
UNIQUE (id_book, page, start_time),
FOREIGN KEY(id_book) REFERENCES book(id)
);
]]
local STATISTICS_DB_PAGE_STAT_DATA_INDEX = [[
CREATE INDEX IF NOT EXISTS page_stat_data_start_time ON page_stat_data(start_time);
]]
local STATISTICS_DB_PAGE_STAT_VIEW_SCHEMA = [[
-- Create the numbers table, used as a source of extra rows when scaling pages in the page_stat view
CREATE TABLE IF NOT EXISTS numbers
(
number INTEGER PRIMARY KEY
);
WITH RECURSIVE counter AS
(
SELECT 1 as N UNION ALL
SELECT N + 1 FROM counter WHERE N < 1000
)
INSERT INTO numbers SELECT N AS number FROM counter;
-- Create the page_stat view
-- This view rescales data from the page_stat_data table to the current number of book pages
-- c.f., https://github.com/koreader/koreader/pull/6761#issuecomment-705660154
CREATE VIEW IF NOT EXISTS page_stat AS
SELECT id_book, first_page + idx - 1 AS page, start_time, duration / (last_page - first_page + 1) AS duration
FROM (
SELECT id_book, page, total_pages, pages, start_time, duration,
-- First page_number for this page after rescaling single row
((page - 1) * pages) / total_pages + 1 AS first_page,
-- Last page_number for this page after rescaling single row
max(((page - 1) * pages) / total_pages + 1, (page * pages) / total_pages) AS last_page,
idx
FROM page_stat_data
JOIN book ON book.id = id_book
-- Duplicate rows for multiple pages as needed (as a result of rescaling)
JOIN (SELECT number as idx FROM numbers) AS N ON idx <= (last_page - first_page + 1)
);
]]
function ReaderStatistics:createDB(conn)
-- Make it WAL, if possible
if Device:canUseWAL() then
conn:exec("PRAGMA journal_mode=WAL;")
else
conn:exec("PRAGMA journal_mode=TRUNCATE;")
end
local sql_stmt = [[
-- book
CREATE TABLE IF NOT EXISTS book
(
id integer PRIMARY KEY autoincrement,
title text,
authors text,
notes integer,
last_open integer,
highlights integer,
pages integer,
series text,
language text,
md5 text,
total_read_time integer,
total_read_pages integer
);
]]
conn:exec(sql_stmt)
-- Index
sql_stmt = [[
CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5);
]]
conn:exec(sql_stmt)
-- page_stat_data
conn:exec(STATISTICS_DB_PAGE_STAT_DATA_SCHEMA)
conn:exec(STATISTICS_DB_PAGE_STAT_DATA_INDEX)
-- page_stat view
conn:exec(STATISTICS_DB_PAGE_STAT_VIEW_SCHEMA)
-- DB schema version
conn:exec(string.format("PRAGMA user_version=%d;", DB_SCHEMA_VERSION))
end
function ReaderStatistics:upgradeDBto20201010(conn)
local sql_stmt = [[
-- Start by updating the layout of the old page_stat table
ALTER TABLE page_stat RENAME COLUMN period TO duration;
-- We're now using the user_version PRAGMA to keep track of schema version
DROP TABLE IF EXISTS info;
]]
conn:exec(sql_stmt)
-- Migrate page_stat content to page_stat_data, which we'll have to create first ;).
conn:exec(STATISTICS_DB_PAGE_STAT_DATA_SCHEMA)
sql_stmt = [[
-- Migrate page_stat content to page_stat_data, and populate total_pages from book's pages while we're at it.
-- NOTE: While doing a per-book migration could ensure a potentially more accurate page count,
-- we need to populate total_pages *now*, or queries against unopened books would return completely bogus values...
-- We'll just have to hope the current value of the column pages in the book table is not too horribly out of date,
-- and not too horribly out of phase with the actual page count at the time the data was originally collected...
INSERT INTO page_stat_data
SELECT id_book, page, start_time, duration, pages as total_pages FROM page_stat
JOIN book on book.id = id_book;
-- Drop old page_stat table
DROP INDEX IF EXISTS page_stat_id_book;
DROP TABLE IF EXISTS page_stat;
]]
conn:exec(sql_stmt)
-- Create the new page_stat view stuff
conn:exec(STATISTICS_DB_PAGE_STAT_VIEW_SCHEMA)
-- Update DB schema version
conn:exec("PRAGMA user_version=20201010;")
end
function ReaderStatistics:upgradeDBto20201022(conn)
conn:exec(STATISTICS_DB_PAGE_STAT_DATA_INDEX)
-- Update DB schema version
conn:exec("PRAGMA user_version=20201022;")
end
function ReaderStatistics:upgradeDBto20221111(conn)
conn:exec([[
-- We make the index on book's (title, author, md5) unique in order to sync dbs
-- First we fill null authors with ''
UPDATE book SET authors = '' WHERE authors IS NULL;
-- Secondly, we unify the id_book in page_stat_data entries for duplicate books
-- to the smallest of each, so as to delete the others.
UPDATE page_stat_data SET id_book = (
SELECT map.min_id FROM (
SELECT id, (
SELECT min(id) FROM book b2
WHERE (book.title, book.authors, book.md5) = (b2.title, b2.authors, b2.md5)
) as min_id
FROM book WHERE book.id >= min_id
) as map WHERE page_stat_data.id_book = map.id
);
-- Delete duplicate books and keep the one with smallest id.
DELETE FROM book WHERE id > (
SELECT MIN(id) FROM book b2
WHERE (book.title, book.authors, book.md5) = (b2.title, b2.authors, b2.md5)
);
-- Then we recompute the book statistics based on merged books
UPDATE book SET (total_read_pages, total_read_time) =
(SELECT count(DISTINCT page),
sum(duration)
FROM page_stat
WHERE id_book = book.id);
-- Finally we update the index to be unique
DROP INDEX IF EXISTS book_title_authors_md5;
CREATE UNIQUE INDEX book_title_authors_md5 ON book(title, authors, md5);]])
-- Update DB schema version
conn:exec("PRAGMA user_version=20221111;")
end
function ReaderStatistics:addBookStatToDB(book_stats, conn)
local id_book
local last_open_book = 0
local total_read_pages = 0
local total_read_time = 0
local sql_stmt
if book_stats.total_time_in_sec and book_stats.total_time_in_sec > 0
and util.tableSize(book_stats.performance_in_pages) > 0 then
local read_pages = util.tableSize(book_stats.performance_in_pages)
logger.dbg("Insert to database: " .. book_stats.title)
sql_stmt = [[
SELECT count(id)
FROM book
WHERE title = ?
AND authors = ?
AND md5 = ?;
]]
local stmt = conn:prepare(sql_stmt)
local result = stmt:reset():bind(self.data.title, self.data.authors, self.data.md5):step()
local nr_id = tonumber(result[1])
if nr_id == 0 then
stmt = conn:prepare("INSERT INTO book VALUES(NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);")
stmt:reset():bind(book_stats.title, book_stats.authors, book_stats.notes,
last_open_book, book_stats.highlights, book_stats.pages,
book_stats.series, book_stats.language, self:partialMd5(book_stats.file), total_read_time, total_read_pages) :step()
sql_stmt = [[
SELECT last_insert_rowid() AS num;
]]
id_book = conn:rowexec(sql_stmt)
else
sql_stmt = [[
SELECT id
FROM book
WHERE title = ?
AND authors = ?
AND md5 = ?;
]]
stmt = conn:prepare(sql_stmt)
result = stmt:reset():bind(self.data.title, self.data.authors, self.data.md5):step()
id_book = result[1]
end
local sorted_performance = {}
for k, _ in pairs(book_stats.performance_in_pages) do
table.insert(sorted_performance, k)
end
table.sort(sorted_performance)
conn:exec('BEGIN;')
stmt = conn:prepare("INSERT OR IGNORE INTO page_stat VALUES(?, ?, ?, ?);")
local avg_time = math.ceil(book_stats.total_time_in_sec / read_pages)
if avg_time > self.settings.max_sec then
avg_time = self.settings.max_sec
end
local first_read_page = book_stats.performance_in_pages[sorted_performance[1]]
if first_read_page > 1 then
first_read_page = first_read_page - 1
end
local start_open_page = sorted_performance[1]
--first page
stmt:reset():bind(id_book, first_read_page, start_open_page - avg_time, avg_time):step()
for i=2, #sorted_performance do
start_open_page = sorted_performance[i-1]
local diff_time = sorted_performance[i] - sorted_performance[i-1]
if diff_time <= self.settings.max_sec then
stmt:reset():bind(id_book, book_stats.performance_in_pages[sorted_performance[i-1]],
start_open_page, diff_time):step()
elseif diff_time > self.settings.max_sec then --and diff_time <= 2 * avg_time then
stmt:reset():bind(id_book, book_stats.performance_in_pages[sorted_performance[i-1]],
start_open_page, avg_time):step()
end
end
--last page
stmt:reset():bind(id_book, book_stats.performance_in_pages[sorted_performance[#sorted_performance]],
sorted_performance[#sorted_performance], avg_time):step()
--last open book
last_open_book = sorted_performance[#sorted_performance] + avg_time
conn:exec('COMMIT;')
total_read_pages, total_read_time = conn:rowexec(string.format(STATISTICS_SQL_BOOK_TOTALS_QUERY, tonumber(id_book)))
sql_stmt = [[
UPDATE book
SET last_open = ?,
total_read_time = ?,
total_read_pages = ?
WHERE id = ?;
]]
stmt = conn:prepare(sql_stmt)
stmt:reset():bind(last_open_book, total_read_time, total_read_pages, id_book):step()
stmt:close()
return true
end
end
function ReaderStatistics:migrateToDB(conn)
self:createDB(conn)
local nr_of_conv_books = 0
local exclude_titles = {}
for _, v in pairs(ReadHistory.hist) do
local book_stats = DocSettings:open(v.file):readSetting("stats")
if book_stats and book_stats.title == "" then
book_stats.title = v.file:match("^.+/(.+)$")
end
if book_stats then
book_stats.file = v.file
if self:addBookStatToDB(book_stats, conn) then
nr_of_conv_books = nr_of_conv_books + 1
exclude_titles[book_stats.title] = true
else
logger.dbg("Book not converted: " .. book_stats.title)
end
else
logger.dbg("Empty stats for file: ", v.file)
end
end
-- import from stats files (for backward compatibility)
if lfs.attributes(statistics_dir, "mode") == "directory" then
for curr_file in lfs.dir(statistics_dir) do
local path = statistics_dir .. curr_file
if lfs.attributes(path, "mode") == "file" then
local old_data = self:importFromFile(statistics_dir, curr_file)
if old_data and old_data.total_time > 0 and not exclude_titles[old_data.title] then
local book_stats = { performance_in_pages= {} }
for _, v in pairs(old_data.details) do
book_stats.performance_in_pages[v.time] = v.page
end
book_stats.title = old_data.title
book_stats.authors = old_data.authors
book_stats.notes = old_data.notes
book_stats.highlights = old_data.highlights
book_stats.pages = old_data.pages
book_stats.series = old_data.series
book_stats.language = old_data.language
book_stats.total_time_in_sec = old_data.total_time
book_stats.file = nil
if self:addBookStatToDB(book_stats, conn) then
nr_of_conv_books = nr_of_conv_books + 1
else
logger.dbg("Book not converted (old stats): " .. book_stats.title)
end
end
end
end
end
return nr_of_conv_books
end
function ReaderStatistics:getIdBookDB()
local conn = SQ3.open(db_location)
local id_book
local sql_stmt = [[
SELECT count(id)
FROM book
WHERE title = ?
AND authors = ?
AND md5 = ?;
]]
local stmt = conn:prepare(sql_stmt)
local result = stmt:reset():bind(self.data.title, self.data.authors, self.data.md5):step()
local nr_id = tonumber(result[1])
if nr_id == 0 then
-- Not in the DB yet, initialize it
stmt = conn:prepare("INSERT INTO book VALUES(NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);")
stmt:reset():bind(self.data.title, self.data.authors, self.data.notes,
os.time(), self.data.highlights, self.data.pages,
self.data.series, self.data.language, self.data.md5, 0, 0):step()
sql_stmt = [[
SELECT last_insert_rowid() AS num;
]]
id_book = conn:rowexec(sql_stmt)
else
sql_stmt = [[
SELECT id
FROM book
WHERE title = ?
AND authors = ?
AND md5 = ?;
]]
stmt = conn:prepare(sql_stmt)
result = stmt:reset():bind(self.data.title, self.data.authors, self.data.md5):step()
id_book = result[1]
end
stmt:close()
conn:close()
return tonumber(id_book)
end
function ReaderStatistics:insertDB(updated_pagecount)
if not self.id_curr_book then
return
end
local id_book = self.id_curr_book
local now_ts = os.time()
-- The current page stat, having yet no duration, will be ignored
-- in the insertion, and its start ts would be lost. We'll give it
-- to resetVolatileStats() so it can restore it
local cur_page_start_ts = now_ts
local cur_page_data = self.page_stat[self.curr_page]
local cur_page_data_tuple = cur_page_data and cur_page_data[#cur_page_data]
if cur_page_data_tuple and cur_page_data_tuple[2] == 0 then -- should always be true
cur_page_start_ts = cur_page_data_tuple[1]
end
local conn = SQ3.open(db_location)
conn:exec('BEGIN;')
local stmt = conn:prepare("INSERT OR IGNORE INTO page_stat_data VALUES(?, ?, ?, ?, ?);")
for page, data_list in pairs(self.page_stat) do
for _, data_tuple in ipairs(data_list) do
-- See self.page_stat declaration above about the tuple's layout
local ts = data_tuple[1]
local duration = data_tuple[2]
-- Skip placeholder durations
if duration > 0 then
-- NOTE: The fact that we update self.data.pages *after* this call on layout changes
-- should ensure that it matches the layout in which said data was collected.
-- Said data is used to re-scale page numbers, regardless of the document layout,
-- at query time, via a fancy SQL view.
-- This allows the progress tracking to be accurate even in the face of wild
-- document layout changes (e.g., after font size changes).
stmt:reset():bind(id_book, page, ts, duration, self.data.pages):step()
end
end
end
conn:exec('COMMIT;')
-- Update the new pagecount now, so that subsequent queries against the view are accurate
local sql_stmt = [[
UPDATE book
SET pages = ?
WHERE id = ?;
]]
stmt = conn:prepare(sql_stmt)
stmt:reset():bind(updated_pagecount and updated_pagecount or self.data.pages, id_book):step()
-- NOTE: See the tail end of the discussions in #6761 for more context on the choice of this heuristic.
-- Basically, we're counting distinct pages,
-- while making sure the sum of durations per distinct page is clamped to self.settings.max_sec
-- This is expressly tailored to a fairer computation of self.avg_time ;).
local book_read_pages, book_read_time = conn:rowexec(string.format(STATISTICS_SQL_BOOK_CAPPED_TOTALS_QUERY, self.settings.max_sec, id_book))
-- NOTE: What we cache in the book table is the plain uncapped sum (mainly for deleteBooksByTotalDuration's benefit)...
local total_read_pages, total_read_time = conn:rowexec(string.format(STATISTICS_SQL_BOOK_TOTALS_QUERY, id_book))
-- And now update the rest of the book table...
sql_stmt = [[
UPDATE book
SET last_open = ?,
notes = ?,
highlights = ?,
total_read_time = ?,
total_read_pages = ?
WHERE id = ?;
]]
stmt = conn:prepare(sql_stmt)
stmt:reset():bind(now_ts, self.data.notes, self.data.highlights, total_read_time, total_read_pages, id_book):step()
stmt:close()
conn:close()
-- NOTE: On the other hand, this is used for the average time estimate, so we use the capped variants here!
if book_read_pages then
self.book_read_pages = tonumber(book_read_pages)
else
self.book_read_pages = 0
end
if book_read_time then
self.book_read_time = tonumber(book_read_time)
else
self.book_read_time = 0
end
self.avg_time = self.book_read_time / self.book_read_pages
self:resetVolatileStats(cur_page_start_ts)
end
function ReaderStatistics:getPageTimeTotalStats(id_book)
if id_book == nil then
return
end
local conn = SQ3.open(db_location)
-- NOTE: Similarly, this one is used for time-based estimates and averages, so, use the capped variant
local total_pages, total_time = conn:rowexec(string.format(STATISTICS_SQL_BOOK_CAPPED_TOTALS_QUERY, self.settings.max_sec, id_book))
conn:close()
if total_pages then
total_pages = tonumber(total_pages)
else
total_pages = 0
end
if total_time then
total_time = tonumber(total_time)
else
total_time = 0
end
return total_pages, total_time
end
function ReaderStatistics:getBookProperties()
local props = self.view.document:getProps()
if props.title == "No document" or props.title == "" then
--- @fixme Sometimes crengine returns "No document", try one more time.
props = self.view.document:getProps()
end
return props
end
function ReaderStatistics:getStatisticEnabledMenuItem()
return {
text = _("Enabled"),
checked_func = function() return self.settings.is_enabled end,
callback = function()
-- if was enabled, have to save data to file
if self.settings.is_enabled and not self:isDocless() then
self:insertDB()
self.ui.doc_settings:saveSetting("stats", self.data)
end
self.settings.is_enabled = not self.settings.is_enabled
-- if was disabled have to get data from db
if self.settings.is_enabled and not self:isDocless() then
self:initData()
self.start_current_period = os.time()
self.curr_page = self.ui:getCurrentPage()
self:resetVolatileStats(self.start_current_period)
end
if not self:isDocless() then
self.view.footer:onUpdateFooter()
end
end,
}
end
function ReaderStatistics:addToMainMenu(menu_items)
menu_items.statistics = {
text = _("Reading statistics"),
sub_item_table = {
self:getStatisticEnabledMenuItem(),
{
text = _("Settings"),
sub_item_table = {
{
text_func = function()
return T(_("Read page duration limits: %1 s – %2 s"),
self.settings.min_sec, self.settings.max_sec)
end,
callback = function(touchmenu_instance)
local DoubleSpinWidget = require("/ui/widget/doublespinwidget")
local durations_widget
durations_widget = DoubleSpinWidget:new{
left_text = C_("Extrema", "Min"),
left_value = self.settings.min_sec,
left_default = DEFAULT_MIN_READ_SEC,
left_min = 0,
left_max = 120,
left_step = 1,
left_hold_step = 10,
right_text = C_("Extrema", "Max"),
right_value = self.settings.max_sec,
right_default = DEFAULT_MAX_READ_SEC,
right_min = 10,
right_max = 7200,
right_step = 10,
right_hold_step = 60,
is_range = true,
-- @translators This is the time unit for seconds.
unit = C_("Time", "s"),
title_text = _("Read page duration limits"),
info_text = _([[
Set min and max time spent (in seconds) on a page for it to be counted as read in statistics.
The min value ensures pages you quickly browse and skip are not included.
The max value ensures a page you stay on for a long time (because you fell asleep or went away) will be included, but with a duration capped to this specified max value.]]),
callback = function(min, max)
self.settings.min_sec = min
self.settings.max_sec = max
touchmenu_instance:updateItems()
end,
}
UIManager:show(durations_widget)
end,
keep_menu_open = true,
separator = true,
},
{
text_func = function()
return T(_("Calendar weeks start on %1"),
longDayOfWeekTranslation[weekDays[self.settings.calendar_start_day_of_week]])
end,
sub_item_table = {
{ -- Friday (Bangladesh and Maldives)
text = longDayOfWeekTranslation[weekDays[6]],
checked_func = function() return self.settings.calendar_start_day_of_week == 6 end,
callback = function() self.settings.calendar_start_day_of_week = 6 end
},
{ -- Saturday (some Middle East countries)
text = longDayOfWeekTranslation[weekDays[7]],
checked_func = function() return self.settings.calendar_start_day_of_week == 7 end,
callback = function() self.settings.calendar_start_day_of_week = 7 end
},
{ -- Sunday
text = longDayOfWeekTranslation[weekDays[1]],
checked_func = function() return self.settings.calendar_start_day_of_week == 1 end,
callback = function() self.settings.calendar_start_day_of_week = 1 end
},
{ -- Monday
text = longDayOfWeekTranslation[weekDays[2]],
checked_func = function() return self.settings.calendar_start_day_of_week == 2 end,
callback = function() self.settings.calendar_start_day_of_week = 2 end
},
},
},
{
text_func = function()
return T(_("Books per calendar day: %1"), self.settings.calendar_nb_book_spans)
end,
callback = function(touchmenu_instance)
local SpinWidget = require("ui/widget/spinwidget")
UIManager:show(SpinWidget:new{
value = self.settings.calendar_nb_book_spans,
value_min = 1,
value_max = 5,
ok_text = _("Set"),
title_text = _("Books per calendar day"),
info_text = _("Set the max number of book spans to show for a day"),
callback = function(spin)
self.settings.calendar_nb_book_spans = spin.value
touchmenu_instance:updateItems()
end,
extra_text = _("Use default"),
extra_callback = function()
self.settings.calendar_nb_book_spans = DEFAULT_CALENDAR_NB_BOOK_SPANS
touchmenu_instance:updateItems()
end
})
end,
keep_menu_open = true,
},
{
text = _("Show hourly histogram in calendar days"),
checked_func = function() return self.settings.calendar_show_histogram end,
callback = function()
self.settings.calendar_show_histogram = not self.settings.calendar_show_histogram
end,
},
{
text = _("Allow browsing coming months"),
checked_func = function() return self.settings.calendar_browse_future_months end,
callback = function()
self.settings.calendar_browse_future_months = not self.settings.calendar_browse_future_months
end,
separator = true,
},
{
text = _("Cloud sync"),
callback = function(touchmenu_instance)
local server = self.settings.sync_server
local edit_cb = function()
local sync_settings = SyncService:new{}
sync_settings.onClose = function(this)
UIManager:close(this)
end
sync_settings.onConfirm = function(sv)
self.settings.sync_server = sv
touchmenu_instance:updateItems()
end
UIManager:show(sync_settings)
end
if not server then
edit_cb()
return
end
local dialogue
local delete_button = {
text = _("Delete"),
callback = function()
UIManager:close(dialogue)
UIManager:show(ConfirmBox:new{
text = _("Delete server info?"),
cancel_text = _("Cancel"),
cancel_callback = function()
return
end,
ok_text = _("Delete"),
ok_callback = function()
self.settings.sync_server = nil
touchmenu_instance:updateItems()
end,
})
end,
}
local edit_button = {
text = _("Edit"),
callback = function()
UIManager:close(dialogue)
edit_cb()
end
}
local close_button = {
text = _("Close"),
callback = function()
UIManager:close(dialogue)
end
}
local type = server.type == "dropbox" and " (DropBox)" or " (WebDAV)"
dialogue = require("ui/widget/buttondialogtitle"):new{
title = T(_("Cloud storage:\n%1\n\nFolder path:\n%2\n\nSet up the same cloud folder on each device to sync across your devices."),
server.name.." "..type, SyncService.getReadablePath(server)),
buttons = {
{delete_button, edit_button, close_button}
},
}
UIManager:show(dialogue)
end,
enabled_func = function() return self.settings.is_enabled end,
keep_menu_open = true,
},
},
},
{
text = _("Reset statistics"),
sub_item_table = self:genResetBookSubItemTable(),
separator = true,
},
{
text = _("Synchronize now"),
callback = function()
SyncService.sync(self.settings.sync_server, db_location, self.onSync )
end,
enabled_func = function()
return self.settings.sync_server ~= nil and self.settings.is_enabled and require("ui/network/manager"):isWifiOn()
end,
keep_menu_open = true,
separator = true,
},
{
text = _("Current book"),
keep_menu_open = true,
callback = function()
self.kv = KeyValuePage:new{
title = _("Current statistics"),
kv_pairs = self:getCurrentStat()
}
UIManager:show(self.kv)
end,
enabled_func = function() return not self:isDocless() and self.settings.is_enabled end,
},
{
text = _("Reading progress"),
keep_menu_open = true,
callback = function()
self:insertDB()
local current_duration, current_pages = self:getCurrentBookStats()
local today_duration, today_pages = self:getTodayBookStats()
local dates_stats = self:getReadingProgressStats(7)
if dates_stats then
UIManager:show(ReaderProgress:new{
dates = dates_stats,
current_duration = current_duration,
current_pages = current_pages,
today_duration = today_duration,
today_pages = today_pages,
})
else
UIManager:show(InfoMessage:new{
text = _("Reading progress is not available.\nThere is no data for the last week."),
})
end
end
},
{
text = _("Time range"),
keep_menu_open = true,
callback = function()
self:statMenu()
end
},
{
text = _("Calendar view"),
keep_menu_open = true,
callback = function()
self:onShowCalendarView()
end,
},
},
}
end
function ReaderStatistics:statMenu()
self.kv = KeyValuePage:new{
title = _("Time range statistics"),
return_button = true,
kv_pairs = {
{ _("All books"),"",
callback = function()
local kv = self.kv
UIManager:close(self.kv)
local total_msg, kv_pairs = self:getTotalStats()
self.kv = KeyValuePage:new{
title = total_msg,
value_align = "right",
kv_pairs = kv_pairs,
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end, -- clean stack
}
UIManager:show(self.kv)
end,
},
{ _("Books by week"),"",
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = _("Books by week"),
value_overflow_align = "right",
kv_pairs = self:getDatesFromAll(0, "weekly", true),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
},
{ _("Books by month"),"",
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = _("Books by month"),
value_overflow_align = "right",
kv_pairs = self:getDatesFromAll(0, "monthly", true),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
separator = true,
},
{ _("Last week"),"",
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = _("Last week"),
value_overflow_align = "right",
kv_pairs = self:getDatesFromAll(7, "daily_weekday"),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
},
{ _("Last month by day"),"",
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = _("Last month by day"),
value_overflow_align = "right",
kv_pairs = self:getDatesFromAll(30, "daily_weekday"),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
},
{ _("Last year by day"),"",
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = _("Last year by day"),
value_overflow_align = "right",
kv_pairs = self:getDatesFromAll(365, "daily"),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
},
{ _("Last year by week"),"",
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = _("Last year by week"),
value_overflow_align = "right",
kv_pairs = self:getDatesFromAll(365, "weekly"),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
},
{ _("All stats by month"),"",
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = _("All stats by month"),
value_overflow_align = "right",
kv_pairs = self:getDatesFromAll(0, "monthly"),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
},
}
}
UIManager:show(self.kv)
end
function ReaderStatistics:getTodayBookStats()
local now_stamp = os.time()
local now_t = os.date("*t")
local from_begin_day = now_t.hour * 3600 + now_t.min * 60 + now_t.sec
local start_today_time = now_stamp - from_begin_day
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT count(*),
sum(sum_duration)
FROM (
SELECT sum(duration) AS sum_duration
FROM page_stat
WHERE start_time >= %d
GROUP BY id_book, page
);
]]
local today_pages, today_duration = conn:rowexec(string.format(sql_stmt, start_today_time))
conn:close()
if today_pages == nil then
today_pages = 0
end
if today_duration == nil then
today_duration = 0
end
today_duration = tonumber(today_duration)
today_pages = tonumber(today_pages)
return today_duration, today_pages
end
function ReaderStatistics:getCurrentBookStats()
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT count(*),
sum(sum_duration)
FROM (
SELECT sum(duration) AS sum_duration
FROM page_stat
WHERE start_time >= %d
GROUP BY id_book, page
);
]]
local current_pages, current_duration = conn:rowexec(string.format(sql_stmt, self.start_current_period))
conn:close()
if current_pages == nil then
current_pages = 0
end
if current_duration == nil then
current_duration = 0
end
current_duration = tonumber(current_duration)
current_pages = tonumber(current_pages)
return current_duration, current_pages
end
function ReaderStatistics:getCurrentStat()
self:insertDB()
local id_book = self.id_curr_book
local today_duration, today_pages = self:getTodayBookStats()
local current_duration, current_pages = self:getCurrentBookStats()
local conn = SQ3.open(db_location)
local highlights, notes = conn:rowexec(string.format("SELECT highlights, notes FROM book WHERE id = %d;", id_book)) -- luacheck: no unused
local sql_stmt = [[
SELECT count(*)
FROM (
SELECT strftime('%%Y-%%m-%%d', start_time, 'unixepoch', 'localtime') AS dates
FROM page_stat
WHERE id_book = %d
GROUP BY dates
);
]]
local total_days = conn:rowexec(string.format(sql_stmt, id_book))
-- NOTE: Here, we generally want to account for the *full* amount of time spent reading this book.
sql_stmt = [[
SELECT sum(duration),
count(DISTINCT page),
min(start_time)
FROM page_stat
WHERE id_book = %d;
]]
local total_time_book, total_read_pages, first_open = conn:rowexec(string.format(sql_stmt, id_book))
conn:close()
-- NOTE: But, as the "Average time per page" entry is already re-using self.avg_time,
-- which is computed slightly differently (c.f., insertDB), we'll be using this tweaked book read time
-- to compute the other time-based statistics...
local __, book_read_time = self:getPageTimeTotalStats(id_book)
local now_ts = os.time()
if total_time_book == nil then
total_time_book = 0
end
if total_read_pages == nil then
total_read_pages = 0
end
if first_open == nil then
first_open = now_ts
end
self.data.pages = self.view.document:getPageCount()
total_time_book = tonumber(total_time_book)
total_read_pages = tonumber(total_read_pages)
local time_to_read = self.view.state.page and ((self.data.pages - self.view.state.page) * self.avg_time) or 0
local estimate_days_to_read = math.ceil(time_to_read/(book_read_time/tonumber(total_days)))
local estimate_end_of_read_date = os.date("%Y-%m-%d", tonumber(now_ts + estimate_days_to_read * 86400))
local estimates_valid = time_to_read > 0 -- above values could be 'nan' and 'nil'
local user_duration_format = G_reader_settings:readSetting("duration_format", "classic")
return {
-- Global statistics (may consider other books than current book)
-- since last resume
{ _("Time spent reading this session"), util.secondsToClockDuration(user_duration_format, current_duration, false) },
{ _("Pages read this session"), tonumber(current_pages) },
-- today
{ _("Time spent reading today"), util.secondsToClockDuration(user_duration_format, today_duration, false) },
{ _("Pages read today"), tonumber(today_pages), separator = true },
-- Current book statistics
-- Includes re-reads
{ _("Total time spent on this book"), util.secondsToClockDuration(user_duration_format, total_time_book, false) },
-- Capped to self.settings.max_sec per distinct page
{ _("Time spent reading this book"), util.secondsToClockDuration(user_duration_format, book_read_time, false) },
-- per days
{ _("Reading started"), os.date("%Y-%m-%d (%H:%M)", tonumber(first_open))},
{ _("Days reading this book"), tonumber(total_days) },
{ _("Average time per day"), util.secondsToClockDuration(user_duration_format, book_read_time/tonumber(total_days), false) },
-- per page (% read)
{ _("Average time per page"), util.secondsToClockDuration(user_duration_format, self.avg_time, false) },
{ _("Pages read"), string.format("%d (%d%%)", total_read_pages, Math.round(100*total_read_pages/self.data.pages)) },
-- current page (% completed)
{ _("Current page/Total pages"), string.format("%d/%d (%d%%)", self.curr_page, self.data.pages, Math.round(100*self.curr_page/self.data.pages)) },
-- estimation, from current page to end of book
{ _("Estimated time to read"), estimates_valid and util.secondsToClockDuration(user_duration_format, time_to_read, false) or _("N/A") },
{ _("Estimated reading finished"), estimates_valid and
T(N_("%1 (1 day)", "%1 (%2 days)", estimate_days_to_read), estimate_end_of_read_date, estimate_days_to_read)
or _("N/A") },
-- highlights
{ _("Highlights"), tonumber(highlights), separator = true },
-- { _("Total notes"), tonumber(notes) }, -- not accurate, don't show it
{ _("Show days"), _("Tap to display"),
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = T(_("Days reading %1"), self.data.title),
value_overflow_align = "right",
kv_pairs = self:getDatesForBook(id_book),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
}
}
end
function ReaderStatistics:getBookStat(id_book)
if id_book == nil then
return
end
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT title, authors, pages, last_open, highlights, notes
FROM book
WHERE id = %d;
]]
local title, authors, pages, last_open, highlights, notes = conn:rowexec(string.format(sql_stmt, id_book))
-- Due to some bug, some books opened around April 2020 might
-- have notes and highlight NULL in the DB.
-- See: https://github.com/koreader/koreader/issues/6190#issuecomment-633693940
-- (We made these last in the SQL so NULL/nil doesn't prevent
-- fetching the other fields.)
-- Show "?" when these values are not known (they will be
-- fixed next time this book is opened).
highlights = highlights and tonumber(highlights) or "?"
notes = notes and tonumber(notes) or "?" -- luacheck: no unused
sql_stmt = [[
SELECT count(*)
FROM (
SELECT strftime('%%Y-%%m-%%d', start_time, 'unixepoch', 'localtime') AS dates
FROM page_stat
WHERE id_book = %d
GROUP BY dates
);
]]
local total_days = conn:rowexec(string.format(sql_stmt, id_book))
-- NOTE: Same general principle as getCurrentStat
sql_stmt = [[
SELECT sum(duration),
count(DISTINCT page),
min(start_time),
(select max(ps2.page) from page_stat as ps2 where ps2.start_time = max(page_stat.start_time))
FROM page_stat
WHERE id_book = %d;
]]
local total_time_book, total_read_pages, first_open, last_page = conn:rowexec(string.format(sql_stmt, id_book))
conn:close()
local book_read_pages, book_read_time = self:getPageTimeTotalStats(id_book)
if total_time_book == nil then
total_time_book = 0
end
if total_read_pages == nil then
total_read_pages = 0
end
if first_open == nil then
first_open = os.time()
end
total_time_book = tonumber(total_time_book)
total_read_pages = tonumber(total_read_pages)
last_page = tonumber(last_page)
if last_page == nil then
last_page = 0
end
pages = tonumber(pages)
if pages == nil or pages == 0 then
pages = 1
end
local avg_time_per_page = book_read_time / book_read_pages
local user_duration_format = G_reader_settings:readSetting("duration_format")
return {
{ _("Title"), title},
{ _("Authors"), authors},
{ _("Reading started"), os.date("%Y-%m-%d (%H:%M)", tonumber(first_open))},
{ _("Last read"), os.date("%Y-%m-%d (%H:%M)", tonumber(last_open))},
{ _("Days reading this book"), tonumber(total_days) },
{ _("Total time spent on this book"), util.secondsToClockDuration(user_duration_format, total_time_book, false) },
{ _("Time spent reading this book"), util.secondsToClockDuration(user_duration_format, book_read_time, false) },
{ _("Average time per day"), util.secondsToClockDuration(user_duration_format, book_read_time/tonumber(total_days), false) },
{ _("Average time per page"), util.secondsToClockDuration(user_duration_format, avg_time_per_page, false) },
{ _("Pages read"), string.format("%d (%d%%)", total_read_pages, Math.round(100*total_read_pages/pages)) },
{ _("Last read page/Total pages"), string.format("%d/%d (%d%%)", last_page, pages, Math.round(100*last_page/pages)) },
{ _("Highlights"), highlights, separator = true },
-- { _("Total notes"), notes }, -- not accurate, don't show it
{ _("Show days"), _("Tap to display"),
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = T(_("Days reading %1"), title),
value_overflow_align = "right",
kv_pairs = self:getDatesForBook(id_book),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
}
}
end
local function sqlDaily()
return
[[
SELECT dates,
count(*) AS pages,
sum(sum_duration) AS durations,
start_time
FROM (
SELECT strftime('%%Y-%%m-%%d', start_time, 'unixepoch', 'localtime') AS dates,
sum(duration) AS sum_duration,
start_time
FROM page_stat
WHERE start_time >= %d
GROUP BY id_book, page, dates
)
GROUP BY dates
ORDER BY dates DESC;
]]
end
local function sqlWeekly()
return
[[
SELECT dates,
count(*) AS pages,
sum(sum_duration) AS durations,
start_time
FROM (
SELECT strftime('%%Y-%%W', start_time, 'unixepoch', 'localtime') AS dates,
sum(duration) AS sum_duration,
start_time
FROM page_stat
WHERE start_time >= %d
GROUP BY id_book, page, dates
)
GROUP BY dates
ORDER BY dates DESC;
]]
end
local function sqlMonthly()
return
[[
SELECT dates,
count(*) AS pages,
sum(sum_duration) AS durations,
start_time
FROM (
SELECT strftime('%%Y-%%m', start_time, 'unixepoch', 'localtime') AS dates,
sum(duration) AS sum_duration,
start_time
FROM page_stat
WHERE start_time >= %d
GROUP BY id_book, page, dates
)
GROUP BY dates
ORDER BY dates DESC;
]]
end
function ReaderStatistics:callbackMonthly(begin, finish, date_text, book_mode)
local kv = self.kv
UIManager:close(kv)
if book_mode then
self.kv = KeyValuePage:new{
title = T(_("Books read in %1"), date_text),
value_align = "right",
kv_pairs = self:getBooksFromPeriod(begin, finish),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
else
self.kv = KeyValuePage:new{
title = date_text,
value_align = "right",
kv_pairs = self:getDaysFromPeriod(begin, finish),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
end
UIManager:show(self.kv)
end
function ReaderStatistics:callbackWeekly(begin, finish, date_text, book_mode)
local kv = self.kv
UIManager:close(kv)
if book_mode then
self.kv = KeyValuePage:new{
title = T(_("Books read in %1"), date_text),
value_align = "right",
kv_pairs = self:getBooksFromPeriod(begin, finish),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
else
self.kv = KeyValuePage:new{
title = date_text,
value_align = "right",
kv_pairs = self:getDaysFromPeriod(begin, finish),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
end
UIManager:show(self.kv)
end
function ReaderStatistics:callbackDaily(begin, finish, date_text)
local kv = self.kv
UIManager:close(kv)
self.kv = KeyValuePage:new{
title = date_text,
value_align = "right",
kv_pairs = self:getBooksFromPeriod(begin, finish),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end
-- sdays -> number of days to show
-- ptype -> daily - show daily without weekday name
-- daily_weekday - show daily with weekday name
-- weekly - show weekly
-- monthly - show monthly
-- book_mode = if true than show book in this period
function ReaderStatistics:getDatesFromAll(sdays, ptype, book_mode)
local results = {}
local now_t = os.date("*t")
local from_begin_day = now_t.hour *3600 + now_t.min*60 + now_t.sec
local now_stamp = os.time()
local one_day = 86400 -- one day in seconds
local period_begin = 0
local user_duration_format = G_reader_settings:readSetting("duration_format")
if sdays > 0 then
period_begin = now_stamp - ((sdays-1) * one_day) - from_begin_day
end
local sql_stmt_res_book
if ptype == "daily" or ptype == "daily_weekday" then
sql_stmt_res_book = sqlDaily()
elseif ptype == "weekly" then
sql_stmt_res_book = sqlWeekly()
elseif ptype == "monthly" then
sql_stmt_res_book = sqlMonthly()
end
self:insertDB()
local conn = SQ3.open(db_location)
local result_book = conn:exec(string.format(sql_stmt_res_book, period_begin))
conn:close()
if result_book == nil then
return {}
end
for i=1, #result_book.dates do
local timestamp = tonumber(result_book[4][i])
local date_text
if ptype == "daily_weekday" then
date_text = string.format("%s (%s)",
os.date("%Y-%m-%d", timestamp),
shortDayOfWeekTranslation[os.date("%a", timestamp)])
elseif ptype == "daily" then
date_text = result_book[1][i]
elseif ptype == "weekly" then
date_text = T(_("%1 Week %2"), os.date("%Y", timestamp), os.date(" %W", timestamp))
elseif ptype == "monthly" then
date_text = monthTranslation[os.date("%B", timestamp)] .. os.date(" %Y", timestamp)
else
date_text = result_book[1][i]
end
if ptype == "monthly" then
local year_begin = tonumber(os.date("%Y", timestamp))
local year_end
local month_begin = tonumber(os.date("%m", timestamp))
local month_end
if month_begin == 12 then
year_end = year_begin + 1
month_end = 1
else
year_end = year_begin
month_end = month_begin + 1
end
local start_month = os.time{year=year_begin, month=month_begin, day=1, hour=0, min=0 }
local stop_month = os.time{year=year_end, month=month_end, day=1, hour=0, min=0 }
table.insert(results, {
date_text,
T(_("Pages: (%1) Time: %2"), tonumber(result_book[2][i]), util.secondsToClockDuration(user_duration_format, tonumber(result_book[3][i]), false)),
callback = function()
self:callbackMonthly(start_month, stop_month, date_text, book_mode)
end,
})
elseif ptype == "weekly" then
local time_book = os.date("%H%M%S%w", timestamp)
local begin_week = tonumber(result_book[4][i]) - 3600 * tonumber(string.sub(time_book,1,2))
- 60 * tonumber(string.sub(time_book,3,4)) - tonumber(string.sub(time_book,5,6))
local weekday = tonumber(string.sub(time_book,7,8))
if weekday == 0 then weekday = 6 else weekday = weekday - 1 end
begin_week = begin_week - weekday * 86400
table.insert(results, {
date_text,
T(_("Pages: (%1) Time: %2"), tonumber(result_book[2][i]), util.secondsToClockDuration(user_duration_format, tonumber(result_book[3][i]), false)),
callback = function()
self:callbackWeekly(begin_week, begin_week + 7 * 86400, date_text, book_mode)
end,
})
else
local time_book = os.date("%H%M%S", timestamp)
local begin_day = tonumber(result_book[4][i]) - 3600 * tonumber(string.sub(time_book,1,2))
- 60 * tonumber(string.sub(time_book,3,4)) - tonumber(string.sub(time_book,5,6))
table.insert(results, {
date_text,
T(_("Pages: (%1) Time: %2"), tonumber(result_book[2][i]), util.secondsToClockDuration(user_duration_format, tonumber(result_book[3][i]), false)),
callback = function()
self:callbackDaily(begin_day, begin_day + 86400, date_text)
end,
})
end
end
return results
end
function ReaderStatistics:getDaysFromPeriod(period_begin, period_end)
local results = {}
local sql_stmt_res_book = [[
SELECT dates,
count(*) AS pages,
sum(sum_duration) AS durations,
start_time
FROM (
SELECT strftime('%%Y-%%m-%%d', start_time, 'unixepoch', 'localtime') AS dates,
sum(duration) AS sum_duration,
start_time
FROM page_stat
WHERE start_time BETWEEN %d AND %d
GROUP BY id_book, page, dates
)
GROUP BY dates
ORDER BY dates DESC;
]]
local conn = SQ3.open(db_location)
local result_book = conn:exec(string.format(sql_stmt_res_book, period_begin, period_end - 1))
conn:close()
if result_book == nil then
return {}
end
local user_duration_format = G_reader_settings:readSetting("duration_format")
for i=1, #result_book.dates do
local time_begin = os.time{year=string.sub(result_book[1][i],1,4), month=string.sub(result_book[1][i],6,7),
day=string.sub(result_book[1][i],9,10), hour=0, min=0, sec=0 }
table.insert(results, {
result_book[1][i],
T(_("Pages: (%1) Time: %2"), tonumber(result_book[2][i]), util.secondsToClockDuration(user_duration_format, tonumber(result_book[3][i]), false)),
callback = function()
local kv = self.kv
UIManager:close(kv)
self.kv = KeyValuePage:new{
title = T(_("Books read %1"), result_book[1][i]),
value_overflow_align = "right",
kv_pairs = self:getBooksFromPeriod(time_begin, time_begin + 86400),
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
})
end
return results
end
function ReaderStatistics:getBooksFromPeriod(period_begin, period_end, callback_shows_days)
local results = {}
local sql_stmt_res_book = [[
SELECT book_tbl.title AS title,
sum(page_stat_tbl.duration),
count(distinct page_stat_tbl.page),
book_tbl.id
FROM page_stat AS page_stat_tbl, book AS book_tbl
WHERE page_stat_tbl.id_book=book_tbl.id AND page_stat_tbl.start_time BETWEEN %d AND %d
GROUP BY book_tbl.id
ORDER BY book_tbl.last_open DESC;
]]
local conn = SQ3.open(db_location)
local result_book = conn:exec(string.format(sql_stmt_res_book, period_begin + 1, period_end))
conn:close()
if result_book == nil then
return {}
end
local user_duration_format = G_reader_settings:readSetting("duration_format")
for i=1, #result_book.title do
table.insert(results, {
result_book[1][i],
T(_("%1 (%2)"), util.secondsToClockDuration(user_duration_format, tonumber(result_book[2][i]), false), tonumber(result_book[3][i])),
callback = function()
local kv = self.kv
UIManager:close(self.kv)
if callback_shows_days then -- not used currently by any code
self.kv = KeyValuePage:new{
title = T(_("Days reading %1"), result_book[1][i]),
kv_pairs = self:getDatesForBook(tonumber(result_book[4][i])),
value_overflow_align = "right",
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
else
self.kv = KeyValuePage:new{
title = result_book[1][i],
kv_pairs = self:getBookStat(tonumber(result_book[4][i])),
value_overflow_align = "right",
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
end
UIManager:show(self.kv)
end,
hold_callback = function(kv_page, kv_item)
self:resetStatsForBookForPeriod(result_book[4][i], period_begin, period_end, false, function()
kv_page:removeKeyValueItem(kv_item) -- Reset, refresh what's displayed
end)
end,
})
end
return results
end
function ReaderStatistics:getReadingProgressStats(sdays)
local results = {}
local now_t = os.date("*t")
local from_begin_day = now_t.hour *3600 + now_t.min*60 + now_t.sec
local now_stamp = os.time()
local one_day = 86400 -- one day in seconds
local period_begin = now_stamp - ((sdays-1) * one_day) - from_begin_day
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT dates,
count(*) AS pages,
sum(sum_duration) AS durations,
start_time
FROM (
SELECT strftime('%%Y-%%m-%%d', start_time, 'unixepoch', 'localtime') AS dates,
sum(duration) AS sum_duration,
start_time
FROM page_stat
WHERE start_time >= %d
GROUP BY id_book, page, dates
)
GROUP BY dates
ORDER BY dates DESC;
]]
local result_book = conn:exec(string.format(sql_stmt, period_begin))
conn:close()
if not result_book then return end
for i = 1, sdays do
local pages = tonumber(result_book[2][i])
local duration = tonumber(result_book[3][i])
local date_read = result_book[1][i]
if pages == nil then pages = 0 end
if duration == nil then duration = 0 end
table.insert(results, {
pages,
duration,
date_read
})
end
return results
end
function ReaderStatistics:getDatesForBook(id_book)
local results = {}
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT date(start_time, 'unixepoch', 'localtime') AS dates,
count(DISTINCT page) AS pages,
sum(duration) AS durations,
min(start_time) AS min_start_time,
max(start_time) AS max_start_time
FROM page_stat
WHERE id_book = %d
GROUP BY Date(start_time, 'unixepoch', 'localtime')
ORDER BY dates DESC;
]]
local result_book = conn:exec(string.format(sql_stmt, id_book))
conn:close()
if result_book == nil then
return {}
end
local user_duration_format = G_reader_settings:readSetting("duration_format")
for i=1, #result_book.dates do
table.insert(results, {
result_book[1][i],
T(_("Pages: (%1) Time: %2"), tonumber(result_book[2][i]), util.secondsToClockDuration(user_duration_format, tonumber(result_book[3][i]), false)),
hold_callback = function(kv_page, kv_item)
self:resetStatsForBookForPeriod(id_book, result_book[4][i], result_book[5][i], result_book[1][i], function()
kv_page:removeKeyValueItem(kv_item) -- Reset, refresh what's displayed
end)
end,
})
end
return results
end
function ReaderStatistics:resetStatsForBookForPeriod(id_book, min_start_time, max_start_time, day_str, on_reset_confirmed_callback)
local confirm_text
if day_str then
-- From getDatesForBook(): we are showing a list of days, with book title at top title:
-- show the day string to confirm the long-press was on the right day
confirm_text = T(_("Do you want to reset statistics for day %1 for this book?"), day_str)
else
-- From getBooksFromPeriod(): we are showing a list of books, with the period as top title:
-- show the book title to confirm the long-press was on the right book
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT title
FROM book
WHERE id = %d;
]]
local book_title = conn:rowexec(string.format(sql_stmt, id_book))
conn:close()
confirm_text = T(_("Do you want to reset statistics for this period for book:\n%1"), book_title)
end
UIManager:show(ConfirmBox:new{
text = confirm_text,
cancel_text = _("Cancel"),
cancel_callback = function()
return
end,
ok_text = _("Reset"),
ok_callback = function()
local conn = SQ3.open(db_location)
local sql_stmt = [[
DELETE FROM page_stat_data
WHERE id_book = ?
AND start_time between ? and ?
]]
local stmt = conn:prepare(sql_stmt)
stmt:reset():bind(id_book, min_start_time, max_start_time):step()
stmt:close()
conn:close()
if on_reset_confirmed_callback then
on_reset_confirmed_callback()
end
end,
})
end
function ReaderStatistics:getTotalStats()
self:insertDB()
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT sum(duration)
FROM page_stat;
]]
local total_books_time = conn:rowexec(sql_stmt)
if total_books_time == nil then
total_books_time = 0
end
local total_stats = {}
sql_stmt = [[
SELECT id
FROM book
ORDER BY last_open DESC;
]]
local id_book_tbl = conn:exec(sql_stmt)
local nr_books
if id_book_tbl ~= nil then
nr_books = #id_book_tbl.id
else
nr_books = 0
end
local user_duration_format = G_reader_settings:readSetting("duration_format")
for i=1, nr_books do
local id_book = tonumber(id_book_tbl[1][i])
sql_stmt = [[
SELECT title
FROM book
WHERE id = %d;
]]
local book_title = conn:rowexec(string.format(sql_stmt, id_book))
sql_stmt = [[
SELECT sum(duration)
FROM page_stat
WHERE id_book = %d;
]]
local total_time_book = conn:rowexec(string.format(sql_stmt,id_book))
if total_time_book == nil then
total_time_book = 0
end
table.insert(total_stats, {
book_title,
util.secondsToClockDuration(user_duration_format, total_time_book, false),
callback = function()
local kv = self.kv
UIManager:close(self.kv)
self.kv = KeyValuePage:new{
title = book_title,
kv_pairs = self:getBookStat(id_book),
value_overflow_align = "right",
callback_return = function()
UIManager:show(kv)
self.kv = kv
end,
close_callback = function() self.kv = nil end,
}
UIManager:show(self.kv)
end,
})
end
conn:close()
return T(_("Total time spent reading: %1"), util.secondsToClockDuration(user_duration_format, total_books_time, false)), total_stats
end
function ReaderStatistics:genResetBookSubItemTable()
local sub_item_table = {}
table.insert(sub_item_table, {
text = _("Reset statistics for the current book"),
keep_menu_open = true,
callback = function()
self:resetCurrentBook()
end,
enabled_func = function() return not self:isDocless() and self.settings.is_enabled and self.id_curr_book end,
separator = true,
})
table.insert(sub_item_table, {
text = _("Reset statistics per book"),
keep_menu_open = true,
callback = function()
self:resetPerBook()
end,
separator = true,
})
local reset_minutes = { 1, 5, 15, 30, 60 }
for _, minutes in ipairs(reset_minutes) do
local text = T(N_("Reset stats for books read for < 1 m",
"Reset stats for books read for < %1 m",
minutes), minutes)
table.insert(sub_item_table, {
text = text,
keep_menu_open = true,
callback = function()
self:deleteBooksByTotalDuration(minutes)
end,
})
end
return sub_item_table
end
function ReaderStatistics:resetPerBook()
local total_stats = {}
self:insertDB()
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT id
FROM book
ORDER BY last_open DESC;
]]
local id_book_tbl = conn:exec(sql_stmt)
local nr_books
if id_book_tbl ~= nil then
nr_books = #id_book_tbl.id
else
nr_books = 0
end
local user_duration_format = G_reader_settings:readSetting("duration_format")
local total_time_book
local kv_reset_book
for i=1, nr_books do
local id_book = tonumber(id_book_tbl[1][i])
sql_stmt = [[
SELECT title
FROM book
WHERE id = %d;
]]
local book_title = conn:rowexec(string.format(sql_stmt, id_book))
sql_stmt = [[
SELECT sum(duration)
FROM page_stat
WHERE id_book = %d;
]]
total_time_book = conn:rowexec(string.format(sql_stmt,id_book))
if total_time_book == nil then
total_time_book = 0
end
if id_book ~= self.id_curr_book then
table.insert(total_stats, {
book_title,
util.secondsToClockDuration(user_duration_format, total_time_book, false),
id_book,
callback = function(kv_page, kv_item)
UIManager:show(ConfirmBox:new{
text = T(_("Do you want to reset statistics for book:\n%1"), book_title),
cancel_text = _("Cancel"),
cancel_callback = function()
return
end,
ok_text = _("Reset"),
ok_callback = function()
self:deleteBook(id_book)
kv_page:removeKeyValueItem(kv_item) -- Reset, refresh what's displayed
end,
})
end,
})
end
end
conn:close()
kv_reset_book = KeyValuePage:new{
title = _("Reset book statistics"),
value_align = "right",
kv_pairs = total_stats,
}
UIManager:show(kv_reset_book)
end
function ReaderStatistics:resetCurrentBook()
-- Flush to db first, so we get a resetVolatileStats
self:insertDB()
local conn = SQ3.open(db_location)
local sql_stmt = [[
SELECT title
FROM book
WHERE id = %d;
]]
local book_title = conn:rowexec(string.format(sql_stmt, self.id_curr_book))
conn:close()
UIManager:show(ConfirmBox:new{
text = T(_("Do you want to reset statistics for book:\n%1"), book_title),
cancel_text = _("Cancel"),
cancel_callback = function()
return
end,
ok_text = _("Reset"),
ok_callback = function()
self:deleteBook(self.id_curr_book)
-- We also need to reset the time/page/avg tracking
self.book_read_pages = 0
self.book_read_time = 0
self.avg_time = math.floor(0.50 * self.settings.max_sec)
logger.dbg("ReaderStatistics: Initializing average time per page at 50% of the max value, i.e.,", self.avg_time)
-- And the current volatile stats
self:resetVolatileStats(os.time())
-- And re-create the Book's data in the book table and get its new ID...
self.id_curr_book = self:getIdBookDB()
end,
})
end
function ReaderStatistics:deleteBook(id_book)
local conn = SQ3.open(db_location)
local sql_stmt = [[
DELETE FROM book
WHERE id = ?;
]]
local stmt = conn:prepare(sql_stmt)
stmt:reset():bind(id_book):step()
sql_stmt = [[
DELETE FROM page_stat_data
WHERE id_book = ?;
]]
stmt = conn:prepare(sql_stmt)
stmt:reset():bind(id_book):step()
stmt:close()
conn:close()
end
function ReaderStatistics:deleteBooksByTotalDuration(max_total_duration_mn)
local max_total_duration_sec = max_total_duration_mn * 60
UIManager:show(ConfirmBox:new{
text = T(N_("Permanently remove statistics for books read for less than 1 minute?",
"Permanently remove statistics for books read for less than %1 minutes?",
max_total_duration_mn), max_total_duration_mn),
ok_text = _("Remove"),
ok_callback = function()
-- Allow following SQL statements to work even when doc less by
-- using -1 as the book id, as real book ids are positive.
local id_curr_book = self.id_curr_book or -1
local conn = SQ3.open(db_location)
local sql_stmt = [[
DELETE FROM page_stat_data
WHERE id_book IN (
SELECT id FROM book WHERE id != ? AND (total_read_time IS NULL OR total_read_time < ?)
);
]]
local stmt = conn:prepare(sql_stmt)
stmt:reset():bind(id_curr_book, max_total_duration_sec):step()
sql_stmt = [[
DELETE FROM book
WHERE id != ? AND (total_read_time IS NULL OR total_read_time < ?);
]]
stmt = conn:prepare(sql_stmt)
stmt:reset():bind(id_curr_book, max_total_duration_sec):step()
stmt:close()
-- Get nb of deleted books
sql_stmt = [[
SELECT changes();
]]
local nb_deleted = conn:rowexec(sql_stmt)
nb_deleted = nb_deleted and tonumber(nb_deleted) or 0
if max_total_duration_mn >= 30 and nb_deleted >= 10 then
-- Do a VACUUM to reduce db size (but not worth doing if not much was removed)
conn:exec("PRAGMA temp_store = 2;") -- use memory for temp files
local ok, errmsg = pcall(conn.exec, conn, "VACUUM;") -- this may take some time
if not ok then
logger.warn("Failed compacting statistics database:", errmsg)
end
end
conn:close()
UIManager:show(InfoMessage:new{
text = nb_deleted > 0 and T(N_("Statistics for 1 book removed.",
"Statistics for %1 books removed.",
nb_deleted), nb_deleted)
or T(_("No statistics removed."))
})
end,
})
end
function ReaderStatistics:onPosUpdate(pos, pageno)
if self.curr_page ~= pageno then
self:onPageUpdate(pageno)
end
end
function ReaderStatistics:onPageUpdate(pageno)
if self:isDocless() or not self.settings.is_enabled then
return
end
-- We only care about *actual* page turns ;)
if self.curr_page == pageno then
return
end
local closing = false
if pageno == false then -- from onCloseDocument()
closing = true
pageno = self.curr_page -- avoid issues in following code
end
self.pageturn_count = self.pageturn_count + 1
local now_ts = os.time()
-- Get the previous page's last timestamp (if there is one)
local page_data = self.page_stat[self.curr_page]
-- This is a list of tuples, in insertion order, we want the last one
local data_tuple = page_data and page_data[#page_data]
-- Tuple layout is { timestamp, duration }
local then_ts = data_tuple and data_tuple[1]
-- If we don't have a previous timestamp to compare to, abort early
if not then_ts then
logger.dbg("ReaderStatistics: No timestamp for previous page", self.curr_page)
self.page_stat[pageno] = { { now_ts, 0 } }
self.curr_page = pageno
return
end
-- By now, we're sure that we actually have a tuple (and the rest of the code ensures they're sane, i.e., zero-initialized)
local curr_duration = data_tuple[2]
-- NOTE: If all goes well, given the earlier curr_page != pageno check, curr_duration should always be 0 here.
-- Compute the difference between now and the previous page's last timestamp
local diff_time = now_ts - then_ts
if diff_time >= self.settings.min_sec and diff_time <= self.settings.max_sec then
self.mem_read_time = self.mem_read_time + diff_time
-- If it's the first time we're computing a duration for this page, count it as read
if #page_data == 1 and curr_duration == 0 then
self.mem_read_pages = self.mem_read_pages + 1
end
-- Update the tuple with the computed duration
data_tuple[2] = curr_duration + diff_time
elseif diff_time > self.settings.max_sec then
self.mem_read_time = self.mem_read_time + self.settings.max_sec
if #page_data == 1 and curr_duration == 0 then
self.mem_read_pages = self.mem_read_pages + 1
end
-- Update the tuple with the computed duration
data_tuple[2] = curr_duration + self.settings.max_sec
end
if closing then
return -- current page data updated, nothing more needed
end
-- We want a flush to db every 50 page turns
if self.pageturn_count >= MAX_PAGETURNS_BEFORE_FLUSH then
-- I/O, delay until after the pageturn, but reset the count now, to avoid potentially scheduling multiple inserts...
self.pageturn_count = 0
UIManager:tickAfterNext(function()
self:insertDB()
-- insertDB will call resetVolatileStats for us ;)
end)
end
-- Update average time per page (if need be, insertDB will have updated the totals and cleared the volatiles)
-- NOTE: Until insertDB runs, while book_read_pages only counts *distinct* pages,
-- and while mem_read_pages does the same, there may actually be an overlap between the two!
-- (i.e., the same page may be counted as read both in total and in mem, inflating the pagecount).
-- Only insertDB will actually check that the count (and as such average time) is actually accurate.
if self.book_read_pages > 0 or self.mem_read_pages > 0 then
self.avg_time = (self.book_read_time + self.mem_read_time) / (self.book_read_pages + self.mem_read_pages)
end
-- We're done, update the current page tracker
self.curr_page = pageno
-- And, in the new page's list, append a new tuple with the current timestamp and a placeholder duration
-- (duration will be computed on next pageturn)
local new_page_data = self.page_stat[pageno]
if new_page_data then
table.insert(new_page_data, { now_ts, 0 })
else
self.page_stat[pageno] = { { now_ts, 0 } }
end
end
-- For backward compatibility
function ReaderStatistics:importFromFile(base_path, item)
item = util.trim(item)
if item ~= ".stat" then
local statistic_file = FFIUtil.joinPath(base_path, item)
if lfs.attributes(statistic_file, "mode") == "directory" then
return
end
local ok, stored = pcall(dofile, statistic_file)
if ok then
return stored
end
end
end
function ReaderStatistics:onCloseDocument()
if not self:isDocless() and self.settings.is_enabled then
self.ui.doc_settings:saveSetting("stats", self.data)
self:onPageUpdate(false) -- update current page duration
self:insertDB()
end
end
function ReaderStatistics:onAddHighlight()
if self.settings.is_enabled then
self.data.highlights = self.data.highlights + 1
end
end
function ReaderStatistics:onDelHighlight()
if self.settings.is_enabled then
if self.data.highlights > 0 then
self.data.highlights = self.data.highlights - 1
end
end
end
function ReaderStatistics:onAddNote()
if self.settings.is_enabled then
self.data.notes = self.data.notes + 1
end
end
-- Triggered by auto_save_settings_interval_minutes
function ReaderStatistics:onSaveSettings()
if not self:isDocless() then
self.ui.doc_settings:saveSetting("stats", self.data)
self:insertDB()
end
end
-- in case when screensaver starts
function ReaderStatistics:onSuspend()
if not self:isDocless() then
self.ui.doc_settings:saveSetting("stats", self.data)
self:insertDB()
self:onReadingPaused()
end
end
-- screensaver off
function ReaderStatistics:onResume()
self.start_current_period = os.time()
self:onReadingResumed()
end
function ReaderStatistics:onReadingPaused()
if self:isDocless() or not self.settings.is_enabled then
return
end
if not self._reading_paused_ts then
self._reading_paused_ts = os.time()
end
end
function ReaderStatistics:onReadingResumed()
if self:isDocless() or not self.settings.is_enabled then
self._reading_paused_ts = nil
return
end
if self._reading_paused_ts then
-- Just add the pause duration to the current page start_time
local pause_duration = os.time() - self._reading_paused_ts
local page_data = self.page_stat[self.curr_page]
local data_tuple = page_data and page_data[#page_data]
if data_tuple then
data_tuple[1] = data_tuple[1] + pause_duration
end
self._reading_paused_ts = nil
end
end
function ReaderStatistics:onReadSettings(config)
self.data = config.data.stats or {}
end
function ReaderStatistics:onReaderReady()
-- we have correct page count now, do the actual initialization work
self:initData()
self.view.footer:onUpdateFooter()
end
function ReaderStatistics:onShowCalendarView()
self:insertDB()
self.kv = nil -- clean left over stack link
local CalendarView = require("calendarview")
UIManager:show(CalendarView:new{
reader_statistics = self,
monthTranslation = monthTranslation,
shortDayOfWeekTranslation = shortDayOfWeekTranslation,
longDayOfWeekTranslation = longDayOfWeekTranslation,
start_day_of_week = self.settings.calendar_start_day_of_week,
nb_book_spans = self.settings.calendar_nb_book_spans,
show_hourly_histogram = self.settings.calendar_show_histogram,
browse_future_months = self.settings.calendar_browse_future_months,
})
end
-- Used by calendarview.lua CalendarView
function ReaderStatistics:getFirstTimestamp()
local sql_stmt = [[
SELECT min(start_time)
FROM page_stat;
]]
local conn = SQ3.open(db_location)
local first_ts = conn:rowexec(sql_stmt)
conn:close()
return first_ts and tonumber(first_ts) or nil
end
function ReaderStatistics:getReadingRatioPerHourByDay(month)
-- We used to have in the SQL statement (with ? = 'YYYY-MM'):
-- WHERE strftime('%Y-%m', start_time, 'unixepoch', 'localtime') = ?
-- but strftime()ing all start_time is slow.
-- Comverting the month into timestamp boundaries, and just comparing
-- integers, can be 5 times faster.
-- We let SQLite compute these timestamp boundaries from the provided
-- month; we need the start of the month to be a real date:
month = month.."-01"
local sql_stmt = [[
SELECT
strftime('%Y-%m-%d', start_time, 'unixepoch', 'localtime') day,
strftime('%H', start_time, 'unixepoch', 'localtime') hour,
sum(duration)/3600.0 ratio
FROM page_stat
WHERE start_time BETWEEN strftime('%s', ?, 'utc')
AND strftime('%s', ?, 'utc', '+33 days', 'start of month', '-1 second')
GROUP BY
strftime('%Y-%m-%d', start_time, 'unixepoch', 'localtime'),
strftime('%H', start_time, 'unixepoch', 'localtime')
ORDER BY day, hour;
]]
local conn = SQ3.open(db_location)
local stmt = conn:prepare(sql_stmt)
local res, nb = stmt:reset():bind(month, month):resultset("i")
stmt:close()
conn:close()
local per_day = {}
for i=1, nb do
local day, hour, ratio = res[1][i], res[2][i], res[3][i]
if not per_day[day] then
per_day[day] = {}
end
-- +1 as histogram starts counting at 1
per_day[day][tonumber(hour)+1] = ratio
end
return per_day
end
function ReaderStatistics:getReadBookByDay(month)
month = month.."-01"
local sql_stmt = [[
SELECT
strftime('%Y-%m-%d', start_time, 'unixepoch', 'localtime') day,
sum(duration) durations,
id_book book_id,
book.title book_title
FROM page_stat
JOIN book ON book.id = page_stat.id_book
WHERE start_time BETWEEN strftime('%s', ?, 'utc')
AND strftime('%s', ?, 'utc', '+33 days', 'start of month', '-1 second')
GROUP BY
strftime('%Y-%m-%d', start_time, 'unixepoch', 'localtime'),
id_book,
title
ORDER BY day, durations desc, book_id, book_title;
]]
local conn = SQ3.open(db_location)
local stmt = conn:prepare(sql_stmt)
local res, nb = stmt:reset():bind(month, month):resultset("i")
stmt:close()
conn:close()
local per_day = {}
for i=1, nb do
-- (We don't care about the duration, we just needed it
-- to have the books in decreasing duration order)
local day, duration, book_id, book_title = res[1][i], res[2][i], res[3][i], res[4][i] -- luacheck: no unused
if not per_day[day] then
per_day[day] = {}
end
table.insert(per_day[day], { id = tonumber(book_id), title = tostring(book_title) })
end
return per_day
end
function ReaderStatistics:onShowReaderProgress()
self:insertDB()
local current_duration, current_pages = self:getCurrentBookStats()
local today_duration, today_pages = self:getTodayBookStats()
local dates_stats = self:getReadingProgressStats(7)
local readingprogress
if dates_stats then
readingprogress = ReaderProgress:new{
dates = dates_stats,
current_duration = current_duration,
current_pages = current_pages,
today_duration = today_duration,
today_pages = today_pages,
--readonly = true,
}
end
UIManager:show(readingprogress)
end
function ReaderStatistics:onShowBookStats()
if self:isDocless() or not self.settings.is_enabled then return end
self.kv = KeyValuePage:new{
title = _("Current statistics"),
kv_pairs = self:getCurrentStat()
}
UIManager:show(self.kv)
end
function ReaderStatistics:getCurrentBookReadPages()
if self:isDocless() or not self.settings.is_enabled then return end
self:insertDB()
local sql_stmt = [[
SELECT
page,
min(sum(duration), ?) AS durations,
strftime("%s", "now") - max(start_time) AS delay
FROM page_stat
WHERE id_book = ?
GROUP BY page
ORDER BY page;
]]
local conn = SQ3.open(db_location)
local stmt = conn:prepare(sql_stmt)
local res, nb = stmt:reset():bind(self.settings.max_sec, self.id_curr_book):resultset("i")
stmt:close()
conn:close()
local read_pages = {}
local max_duration = 0
for i=1, nb do
local page, duration, delay = res[1][i], res[2][i], res[3][i]
page = tonumber(page)
duration = tonumber(duration)
delay = tonumber(delay)
read_pages[page] = {duration, delay}
if duration > max_duration then
max_duration = duration
end
end
for page, info in pairs(read_pages) do
-- Make the value a duration ratio (vs capped or max duration)
read_pages[page][1] = info[1] / max_duration
end
return read_pages
end
function ReaderStatistics.onSync(local_path, cached_path, income_path)
local conn_income = SQ3.open(income_path)
local ok1, v1 = pcall(conn_income.rowexec, conn_income, "PRAGMA schema_version")
if not ok1 or tonumber(v1) == 0 then
-- no income db or wrong db, first time sync
logger.warn("statistics open income DB failed", v1)
return true
end
local sql = "attach '" .. income_path:gsub("'", "''") .."' as income_db;"
-- then we try to open cached db
local conn_cached = SQ3.open(cached_path)
local ok2, v2 = pcall(conn_cached.rowexec, conn_cached, "PRAGMA schema_version")
local attached_cache
if not ok2 or tonumber(v2) == 0 then
-- no cached or error, no item to delete
logger.warn("statistics open cached DB failed", v2)
else
attached_cache = true
sql = sql .. "attach '" .. cached_path:gsub("'", "''") ..[[' as cached_db;
-- first we delete from income_db books that exist in cached_db but not in local_db,
-- namely the ones that were deleted since last sync
DELETE FROM income_db.page_stat_data WHERE id_book IN (
SELECT id FROM income_db.book WHERE (title, authors, md5) IN (
SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN (
SELECT title, authors, md5 FROM book
)
)
);
DELETE FROM income_db.book WHERE (title, authors, md5) IN (
SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN (
SELECT title, authors, md5 FROM book
)
);
-- then we delete books from local db that were present in last sync but
-- not any more (ie. deleted in other devices)
DELETE FROM page_stat_data WHERE id_book IN (
SELECT id FROM book WHERE (title, authors, md5) IN (
SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN (
SELECT title, authors, md5 FROM income_db.book
)
)
);
DELETE FROM book WHERE (title, authors, md5) IN (
SELECT title, authors, md5 FROM cached_db.book WHERE (title, authors, md5) NOT IN (
SELECT title, authors, md5 FROM income_db.book
)
);
]]
end
conn_cached:close()
conn_income:close()
local conn = SQ3.open(local_path)
local ok3, v3 = pcall(conn.exec, conn, "PRAGMA schema_version")
if not ok3 or tonumber(v3) == 0 then
-- no local db, this is an error
logger.err("statistics open local DB", v3)
return false
end
sql = sql .. [[
-- We merge the local db with income db to form the synced db.
-- Do the books
INSERT INTO book (
title, authors, notes, last_open, highlights, pages, series, language, md5, total_read_time, total_read_pages
) SELECT
title, authors, notes, last_open, highlights, pages, series, language, md5, total_read_time, total_read_pages
FROM income_db.book WHERE true ON CONFLICT (title, authors, md5) DO NOTHING;
-- We create a book_id mapping temp table (view not possible due to attached db)
CREATE TEMP TABLE book_id_map AS
SELECT m.id as mid, ifnull(i.id, m.id) as iid FROM book m --main
LEFT JOIN income_db.book i
ON (m.title, m.authors, m.md5) = (i.title, i.authors, i.md5);
]]
if attached_cache then
-- more deletion needed
sql = sql .. [[
-- DELETE stat_data items
DELETE FROM income_db.page_stat_data WHERE (id_book, page, start_time) IN (
SELECT map.iid, page, start_time FROM cached_db.page_stat_data
LEFT JOIN book_id_map AS map ON id_book = map.mid
WHERE (id_book, page, start_time) NOT IN (
SELECT id_book, page, start_time FROM page_stat_data
)
);
DELETE FROM page_stat_data WHERE (id_book, page, start_time) IN (
SELECT id_book, page, start_time FROM cached_db.page_stat_data WHERE (id_book, page, start_time) NOT IN (
SELECT map.mid, page, start_time FROM income_db.page_stat_data
LEFT JOIN book_id_map AS map on id_book = map.iid
)
);]]
end
sql = sql .. [[
-- Then we merge the income_db's contents into the local db
INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages)
SELECT map.mid, page, start_time, duration, total_pages
FROM income_db.page_stat_data
LEFT JOIN book_id_map as map
ON id_book = map.iid
WHERE true
ON CONFLICT(id_book, page, start_time) DO UPDATE SET
duration = MAX(duration, excluded.duration);
-- finally we update the total numbers of book
UPDATE book SET (total_read_pages, total_read_time) =
(SELECT count(DISTINCT page),
sum(duration)
FROM page_stat
WHERE id_book = book.id);
]]
conn:exec(sql)
pcall(conn.exec, conn, "COMMIT;")
conn:exec("DETACH income_db;"..(attached_cache and "DETACH cached_db;" or ""))
conn:close()
return true
end
return ReaderStatistics
Reference in New Issue View Git Blame Copy Permalink