SortWidget, DictQuickLookup: rename some functions (#11949)

addressing concerns found #11933

.ci/after_success.sh

@@ -15,7 +15,7 @@ if [ -z "${CIRCLE_PULL_REQUEST}" ] && [ "${CIRCLE_BRANCH}" = 'master' ]; then
 
     if [ "${CIRCLE_NODE_INDEX}" = 0 ]; then
         travis_retry make coverage
-        pushd koreader-*/koreader && {
+        pushd install/koreader && {
             # see https://github.com/codecov/example-lua
             bash <(curl -s https://codecov.io/bash)
         } && popd || exit

.ci/build.sh

@@ -4,4 +4,11 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${CI_DIR}/common.sh"
 
-make all
+# Build.
+cmd=(make all)
+if [[ -d base/build ]]; then
+    cmd+=(--assume-old=base)
+fi
+"${cmd[@]}"
+
+# vim: sw=4

.ci/common.sh

@@ -60,16 +60,3 @@ retry_cmd() {
     set -e
     return ${result}
 }
-
-# export CI_BUILD_DIR=${TRAVIS_BUILD_DIR}
-# use eval to get fully expanded path
-eval CI_BUILD_DIR="${CIRCLE_WORKING_DIRECTORY}"
-export CI_BUILD_DIR
-
-test -e "${HOME}/bin" || mkdir "${HOME}/bin"
-export PATH=${PWD}/bin:${HOME}/bin:${PATH}
-export PATH=${PATH}:${CI_BUILD_DIR}/install/bin
-if [ -f "${CI_BUILD_DIR}/install/bin/luarocks" ]; then
-    # add local rocks to $PATH
-    eval "$(luarocks path --bin)"
-fi

.ci/helper_luarocks.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=/dev/null
-source "${CI_DIR}/common.sh"
-
-rm -rf "${HOME}/.luarocks"
-mkdir "${HOME}/.luarocks"
-cp "${CI_BUILD_DIR}/install/etc/luarocks/config.lua" "${HOME}/.luarocks/config.lua"
-echo "wrap_bin_scripts = false" >>"${HOME}/.luarocks/config.lua"
-travis_retry luarocks --local install luafilesystem
-# for verbose_print module
-travis_retry luarocks --local install ansicolors
-travis_retry luarocks --local install busted 2.0.0-1
-#- mv -f $HOME/.luarocks/bin/busted_bootstrap $HOME/.luarocks/bin/busted
-
-travis_retry luarocks --local install luacheck 0.25.0-1
-travis_retry luarocks --local install lanes # for parallel luacheck
-
-# used only on master branch but added to cache for better speed
-travis_retry luarocks --local install ldoc
-travis_retry luarocks --local install luacov

.ci/install.sh

@@ -1,69 +0,0 @@
-#!/usr/bin/env bash
-
-CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=/dev/null
-source "${CI_DIR}/common.sh"
-
-# print some useful info
-echo "BUILD_DIR: ${CI_BUILD_DIR}"
-echo "pwd: $(pwd)"
-ls
-
-# toss submodules if there are any changes
-# if [ "$(git status --ignore-submodules=dirty --porcelain)" ]; then
-# "--ignore-submodules=dirty", removed temporarily, as it did not notice as
-# expected that base was updated and kept using old cached base
-if [ "$(git status --ignore-submodules=dirty --porcelain)" ]; then
-    # what changed?
-    git status
-    # purge and reinit submodules
-    git submodule deinit -f .
-    git submodule update --init
-else
-    echo -e "${ANSI_GREEN}Using cached submodules."
-fi
-
-# install our own updated luarocks
-echo "luarocks installation path: ${CI_BUILD_DIR}"
-if [ ! -f "${CI_BUILD_DIR}/install/bin/luarocks" ]; then
-    git clone https://github.com/torch/luajit-rocks.git
-    pushd luajit-rocks && {
-        git checkout 6529891
-        cmake . -DWITH_LUAJIT21=ON -DCMAKE_INSTALL_PREFIX="${CI_BUILD_DIR}/install"
-        make install
-    } && popd || exit
-else
-    echo -e "${ANSI_GREEN}Using cached luarocks."
-fi
-
-if [ ! -d "${HOME}/.luarocks" ] || [ ! -f "${HOME}/.luarocks/$(md5sum <"${CI_DIR}/helper_luarocks.sh")" ]; then
-    echo -e "${ANSI_GREEN}Grabbing new .luarocks."
-    sudo apt-get update
-    # install openssl devel for luasec
-    sudo apt-get -y install libssl-dev
-
-    "${CI_DIR}/helper_luarocks.sh"
-    touch "${HOME}/.luarocks/$(md5sum <"${CI_DIR}/helper_luarocks.sh")"
-else
-    echo -e "${ANSI_GREEN}Using cached .luarocks."
-fi
-
-#install our own updated shellcheck
-SHELLCHECK_VERSION="v0.8.0"
-SHELLCHECK_URL="https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION?}/shellcheck-${SHELLCHECK_VERSION?}.linux.x86_64.tar.xz"
-if ! command -v shellcheck; then
-    curl -sSL "${SHELLCHECK_URL}" | tar --exclude 'SHA256SUMS' --strip-components=1 -C "${HOME}/bin" -xJf -
-    chmod +x "${HOME}/bin/shellcheck"
-    shellcheck --version
-else
-    echo -e "${ANSI_GREEN}Using cached shellcheck."
-fi
-
-# install shfmt
-SHFMT_URL="https://github.com/mvdan/sh/releases/download/v3.2.0/shfmt_v3.2.0_linux_amd64"
-if [ "$(shfmt --version)" != "v3.2.0" ]; then
-    curl -sSL "${SHFMT_URL}" -o "${HOME}/bin/shfmt"
-    chmod +x "${HOME}/bin/shfmt"
-else
-    echo -e "${ANSI_GREEN}Using cached shfmt."
-fi

.ci/test.sh

@@ -4,9 +4,11 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${CI_DIR}/common.sh"
 
-pushd koreader-emulator-x86_64-linux-gnu/koreader && {
+pushd install/koreader && {
     # the circleci command spits out newlines; we want spaces instead
     BUSTED_SPEC_FILE="$(circleci tests glob "spec/front/unit/*_spec.lua" | circleci tests split --split-by=timings --timings-type=filename | tr '\n' ' ')"
 } && popd || exit
 
 make testfront BUSTED_SPEC_FILE="${BUSTED_SPEC_FILE}"
+
+# vim: sw=4

.circleci/config.yml

@@ -1,112 +1,165 @@
-version: 2
+version: "2.1"
 
-workflows:
-  version: 2
-  build:
-    jobs:
-      - build
-      - docs:
-          context: koreader-vars
-          filters:
-            branches:
-              only: master
-          requires:
-            - build
+# Parameters. {{{
 
-jobs:
-  build:
+parameters:
+
+    # Bump this to reset all caches.
+    cache_epoch:
+      type: integer
+      default: 0
+
+# }}}
+
+# Executors. {{{
+
+executors:
+
+  base:
     docker:
-      - image: koreader/kobase:0.3.0
+      - image: koreader/kobase:0.3.2-20.04
         auth:
           username: $DOCKER_USERNAME
           password: $DOCKER_PASSWORD
-        environment:
-          EMULATE_READER: 1
+
+# }}}
+
+# Jobs. {{{
+
+jobs:
+
+  # Build. {{{
+
+  build:
+    executor: base
+    resource_class: medium
+    environment:
+      BASH_ENV: "~/.bashrc"
+      CCACHE_MAXSIZE: "256M"
+      CLICOLOR_FORCE: "1"
+      EMULATE_READER: "1"
+      MAKEFLAGS: "OUTPUT_DIR=build INSTALL_DIR=install"
     parallelism: 2
     steps:
+      # Checkout / fetch. {{{
       - checkout
+      - run:
+          name: Fetch
+          command: .ci/fetch.sh
+      # }}}
+      # Check.
+      - run:
+          name: Check
+          command: .ci/check.sh
+      # Restore / setup caches. {{{
+      - run:
+          name: Generate cache key
+          command: make -C base TARGET= cache-key
       - restore_cache:
+          name: Restore build directory
           keys:
-            # binary dependencies require {{ arch }} because there are different CPUs in use on the servers
-            - deps-{{ arch }}-{{ checksum ".ci/install.sh" }}-{{ checksum ".ci/helper_luarocks.sh" }}
-      # need to init some stuff first or git will complain when sticking in base cache
-      - run: git submodule init base && git submodule update base && pushd base && git submodule init && git submodule update && popd
-      # we can't use command output directly for cache check so we write it to git-rev-base
-      - run: pushd base && git_rev_base=$(git describe HEAD) && popd && echo $git_rev_base && echo $git_rev_base >git-rev-base
+            - &CACHE_KEY_BUILD_DIR '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-build-{{ arch }}-{{ checksum "base/cache-key" }}'
       - restore_cache:
+          name: Restore build cache
           keys:
-            - build-{{ arch }}-{{ checksum "git-rev-base" }}
-      - run: echo 'export PATH=${HOME}/bin:${PATH}' >> $BASH_ENV
-
-      # installs and caches testing tools
-      - run:
-          name: install
-          command: .ci/install.sh
-      - save_cache:
-          key: deps-{{ arch }}-{{ checksum ".ci/install.sh" }}-{{ checksum ".ci/helper_luarocks.sh" }}
-          paths:
-            - "/home/ko/bin"
-            - "/home/ko/.luarocks"
-            # compiled luarocks binaries
-            - "install"
-
-      # installs everything and caches base
+            - &CACHE_KEY_BUILD_CACHE '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-ccache-{{ arch }}-{{ checksum "base/cache-key" }}'
+            - '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-ccache-{{ arch }}-'
       - run:
-          name: fetch
-          command: .ci/fetch.sh
+          name: Setup build cache
+          command: |
+            set -x
+            which ccache
+            ccache --version
+            ccache --zero-stats
+            ccache --show-config
+      # }}}
+      # Build.
       - run:
-          name: check
-          command: .ci/check.sh
-      - run:
-          name: build
+          name: Build
           command: .ci/build.sh
-      # we want to save cache prior to testing so we don't have to clean it up
+      # Clean / save caches. {{{
+      # We want to save cache prior to testing so we don't have to clean it up.
+      - run:
+          name: Clean caches
+          when: always
+          command: |
+            set -x
+            # Trim the build directory.
+            rm -rf base/build/{cmake,staging,thirdparty}
+            ccache --cleanup >/dev/null
+            ccache --show-stats
       - save_cache:
-          key: build-{{ arch }}-{{ checksum "git-rev-base" }}
+          name: Save build cache
+          key: *CACHE_KEY_BUILD_CACHE
           paths:
-            - "/home/ko/.ccache"
-            - "base"
-
-      # our lovely unit tests
+            - /home/ko/.ccache
+      - save_cache:
+          name: Save build directory
+          key: *CACHE_KEY_BUILD_DIR
+          paths:
+            - base/build
+      # }}}
+      # Tests / coverage. {{{
+      # Our lovely unit tests.
       - run:
-          name: test
+          name: Test
           command: .ci/test.sh
-
-      # docs, coverage, and test timing (can we use two outputs at once?); master branch only
+      # Docs, coverage, and test timing (can we use two outputs at once?); master branch only.
       - run:
-          name: coverage
+          name: Coverage
           command: .ci/after_success.sh
-      # by storing the test results CircleCI automatically distributes tests based on execution time
+      # By storing the test results CircleCI automatically distributes tests based on execution time.
       - store_test_results:
-          path: koreader-emulator-x86_64-linux-gnu/koreader
-      # CircleCI doesn't make the test results available as artifacts (October 2017)
+          path: &TESTS_XML install/koreader/junit-test-results.xml
+      # CircleCI doesn't make the test results available as artifacts (October 2017).
       - store_artifacts:
-          path: koreader-emulator-x86_64-linux-gnu/koreader/junit-test-results.xml
+          path: *TESTS_XML
+      # }}}
+
+  # }}}
+
+  # Docs. {{{
+
   docs:
-    docker:
-      - image: koreader/kobase:0.3.0
-        auth:
-          username: $DOCKER_USERNAME
-          password: $DOCKER_PASSWORD
-        environment:
-          EMULATE_READER: 1
+    executor: base
+    resource_class: small
+    environment:
+      BASH_ENV: "~/.bashrc"
     parallelism: 1
     steps:
       - checkout
-      - restore_cache:
-          keys:
-            # binary dependencies require {{ arch }} because there are different CPUs in use on the servers
-            - deps-{{ arch }}-{{ checksum ".ci/install.sh" }}-{{ checksum ".ci/helper_luarocks.sh" }}
-      # need to init some stuff first or git will complain when sticking in base cache
-      - run: git submodule init base && git submodule update base && pushd base && git submodule init && git submodule update && popd
-      # we can't use command output directly for cache check so we write it to git-rev-base
-      - run: pushd base && git_rev_base=$(git describe HEAD) && popd && echo $git_rev_base && echo $git_rev_base >git-rev-base
-
       - run:
-          name: init-submodules
-          command: git submodule init && git submodule sync && git submodule update
-
+          name: fetch
+          command: .ci/fetch.sh
       # docs, coverage, and test timing (can we use two outputs at once?); master branch only
       - run:
           name: docs-and-translation
           command: .ci/after_success_docs_translation.sh
+
+  # }}}
+
+# }}}
+
+# Workflows. {{{
+
+workflows:
+
+  version: 2
+
+  build:
+
+    jobs:
+
+      - build
+
+      - docs:
+          context: koreader-vars
+          filters:
+            branches:
+              only: master
+          requires:
+            - build
+
+# }}}
+
+# vim: foldmethod=marker foldlevel=0

.github/workflows/build.yml

@@ -1,12 +1,18 @@
-name: build
+name: macos
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 on: [push, pull_request]
 
-permissions:
-  contents: read
+defaults:
+  run:
+    shell: bash
 
 jobs:
-  macos_build:
+
+  macos:
 
     # macos-11, macos-12 & macos-13 are broken at this time being.
     # https://github.com/koreader/koreader/issues/8686,
@@ -17,31 +23,159 @@ jobs:
     # 10.15 is no longer supported so we are running 13 just to make sure the build does not break.
     runs-on: macos-13
 
+    env:
+      # Bump number to reset all caches.
+      CACHE_EPOCH: '0'
+      CLICOLOR_FORCE: '1'
+      MACOSX_DEPLOYMENT_TARGET: '10.15'
+      MAKEFLAGS: 'OUTPUT_DIR=build INSTALL_DIR=install TARGET=macos'
+
     steps:
       - name: XCode version
         run: xcode-select -p
 
-      - name: Check out Git repository
-        uses: actions/checkout@v3
+      # Checkout / fetch. {{{
+
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
+          clean: false
           fetch-depth: 0
+          filter: tree:0
+          show-progress: false
+
+      - name: Fetch
+        run: make fetchthirdparty
+
+      # }}}
+
+      # Restore / setup caches. {{{
+
+      - name: Generate cache key
+        run: make -C base TARGET= cache-key
+
+      - name: Restore build directory
+        id: build-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: base/build
+          key: ${{ env.CACHE_EPOCH }}-${{ runner.os }}-build-${{ hashFiles('base/cache-key') }}
 
-      - name: Homebrew install dependencies
-        # Compared to the README, adds ccache for faster compilation times
-        # Compared to the emulator, adds p7zip.
+      - name: Restore build cache
+        id: ccache-restore
+        if: steps.build-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/restore@v4
+        with:
+          path: /Users/runner/Library/Caches/ccache
+          key: ${{ env.CACHE_EPOCH }}-${{ runner.os }}-ccache-${{ hashFiles('base/cache-key') }}
+          restore-keys: ${{ env.CACHE_EPOCH }}-${{ runner.os }}-ccache-
+
+      - name: Install ccache
+        if: steps.build-restore.outputs.cache-hit != 'true'
+        run: |
+          wget --progress=dot:mega https://github.com/ccache/ccache/releases/download/v4.9.1/ccache-4.9.1-darwin.tar.gz
+          tar xf ccache-4.9.1-darwin.tar.gz
+          printf '%s\n' "$PWD/ccache-4.9.1-darwin" >>"${GITHUB_PATH}"
+
+      - name: Setup build cache
+        if: steps.build-restore.outputs.cache-hit != 'true'
+        run: |
+          set -x
+          which ccache
+          ccache --version
+          ccache --zero-stats
+          ccache --max-size=256M
+          ccache --show-config
+
+      # }}}
+
+      # Install dependencies. {{{
+
+      - name: Setup Python
+        if: steps.build-restore.outputs.cache-hit != 'true'
+        uses: actions/setup-python@v5
+        with:
+          # Note: Python 3.12 removal of `distutils` breaks GLib's build.
+          python-version: '3.11'
+
+      - name: Install homebrew dependencies
+        # Compared to the README, adds p7zip.
+        run: |
+          packages=(
+            nasm binutils coreutils libtool autoconf automake cmake make
+            sdl2 lua@5.1 luarocks gettext pkg-config wget
+            gnu-getopt grep p7zip ninja
+          )
+          # Lua 5.1 is disabled, so we need to work around that:
+          # - fetch all packages
+          brew fetch "${packages[@]}"
+          # - disable auto-updates
+          export HOMEBREW_NO_AUTO_UPDATE=1
+          # - install lua@5.1 from cache
+          brew install "$(brew --cache lua@5.1)"
+          # - and install the rest
+          brew install "${packages[@]}"
+
+      - name: Update PATH
         run: >
-          brew install -q nasm ragel binutils coreutils libtool autoconf automake cmake makedepend
-          sdl2 lua@5.1 luarocks gettext pkg-config wget gnu-getopt grep bison
-          ccache p7zip
+          printf '%s\n'
+          "$(brew --prefix)/opt/gettext/bin"
+          "$(brew --prefix)/opt/gnu-getopt/bin"
+          "$(brew --prefix)/opt/grep/libexec/gnubin"
+          "$(brew --prefix)/opt/make/libexec/gnubin"
+          | tee "${GITHUB_PATH}"
+
+      # }}}
+
+      # Build. {{{
+
+      - name: Build
+        if: steps.build-restore.outputs.cache-hit != 'true'
+        run: make base
+
+      - name: Dump binaries runtime path & dependencies
+        run: make bindeps
+
+      # }}}
+
+      # Clean / save caches. {{{
 
-      - name: Building in progress…
+      - name: Clean caches
+        if: steps.build-restore.outputs.cache-hit != 'true' && always()
         run: |
-          export MACOSX_DEPLOYMENT_TARGET=10.15;
-          export PATH="$(brew --prefix)/opt/gettext/bin:$(brew --prefix)/opt/gnu-getopt/bin:$(brew --prefix)/opt/bison/bin:$(brew --prefix)/opt/grep/libexec/gnubin:${PATH}";
-          ./kodev release macos
+          set -x
+          # Trim the build directory.
+          rm -rf base/build/{cmake,staging,thirdparty}
+          ccache --cleanup >/dev/null
+          ccache --show-stats --verbose
 
-      - name: Uploading artifacts
-        uses: actions/upload-artifact@v3
+      - name: Save build cache
+        uses: actions/cache/save@v4
+        if: steps.build-restore.outputs.cache-hit != 'true' && steps.ccache-restore.outputs.cache-hit != 'true'
+        with:
+          path: /Users/runner/Library/Caches/ccache
+          key: ${{ steps.ccache-restore.outputs.cache-primary-key }}
+
+      - name: Save build directory
+        uses: actions/cache/save@v4
+        if: steps.build-restore.outputs.cache-hit != 'true'
+        with:
+          path: base/build
+          key: ${{ steps.build-restore.outputs.cache-primary-key }}
+
+      # }}}
+
+      # Generate / upload artifact. {{{
+
+      - name: Generate artifact
+        run: make update --assume-old=base
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
         with:
           name: koreader-macos
           path: '*.7z'
+
+      # }}}
+
+# vim: foldmethod=marker foldlevel=0

.gitmodules

@@ -18,3 +18,4 @@
 [submodule "l10n"]
 	path = l10n
 	url = https://github.com/koreader/koreader-translations.git
+	shallow = true

.travis.yml

@@ -1,48 +0,0 @@
-language: c
-
-# sudo: false
-sudo: true
-dist: trusty
-
-compiler:
-  - gcc
-
-env:
-  global:
-    - "PATH=${HOME}/bin:${PATH}"
-  matrix:
-    - EMULATE_READER=1
-
-cache:
-  apt: true
-  directories:
-    - "${HOME}/bin"
-    # compiled luarocks binaries
-    - "${TRAVIS_BUILD_DIR}/install"
-    # base build
-    - "${TRAVIS_BUILD_DIR}/base"
-    - "${HOME}/.ccache"
-    - "${HOME}/.luarocks"
-before_cache:
-  # don't quote like you normally would or it won't expand
-  - rm -frv ${TRAVIS_BUILD_DIR}/base/build/*/cache/*
-  # don't cache unit tests
-  - rm -frv ${TRAVIS_BUILD_DIR}/base/build/*/spec
-
-addons:
-  apt:
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - g++-4.8
-      - libsdl1.2-dev
-      # luasec dependencies
-      - libssl1.0.0
-      - nasm
-      # OpenSSL likes this (package contains makedepend)
-      - xutils-dev
-
-before_install: .ci/before_install.sh
-install: .ci/install.sh
-script: .ci/script.sh
-after_success: .ci/after_success.sh

Makefile

@@ -1,20 +1,21 @@
+PHONY = all android-ndk android-sdk base clean coverage doc fetchthirdparty po pot static-check test testfront
+
 # koreader-base directory
 KOR_BASE?=base
 
-# the repository might not have been checked out yet, so make this
-# able to fail:
--include $(KOR_BASE)/Makefile.defs
+include $(KOR_BASE)/Makefile.defs
 
+RELEASE_DATE := $(shell git show -s --format=format:"%cd" --date=short HEAD)
 # We want VERSION to carry the version of the KOReader main repo, not that of koreader-base
-VERSION:=$(shell git describe HEAD)
+VERSION := $(shell git describe HEAD)
 # Only append date if we're not on a whole version, like v2018.11
 ifneq (,$(findstring -,$(VERSION)))
-	VERSION:=$(VERSION)_$(shell git describe HEAD | xargs git show -s --format=format:"%cd" --date=short)
+	VERSION := $(VERSION)_$(RELEASE_DATE)
 endif
 
 # releases do not contain tests and misc data
 IS_RELEASE := $(if $(or $(EMULATE_READER),$(WIN32)),,1)
-IS_RELEASE := $(if $(or $(IS_RELEASE),$(APPIMAGE),$(DEBIAN),$(MACOS)),1,)
+IS_RELEASE := $(if $(or $(IS_RELEASE),$(APPIMAGE),$(LINUX),$(MACOS)),1,)
 
 ifeq ($(ANDROID_ARCH), arm64)
 	ANDROID_ABI?=arm64-v8a
@@ -31,7 +32,18 @@ endif
 ANDROID_VERSION?=$(shell git rev-list --count HEAD)
 ANDROID_NAME?=$(VERSION)
 
-MACHINE=$(shell $(CC) -dumpmachine 2>/dev/null)
+LINUX_ARCH?=native
+ifeq ($(LINUX_ARCH), native)
+	LINUX_ARCH_NAME:=$(shell uname -m)
+else ifeq ($(LINUX_ARCH), arm64)
+	LINUX_ARCH_NAME:=aarch64
+else ifeq ($(LINUX_ARCH), arm)
+	LINUX_ARCH_NAME:=armv7l
+endif
+LINUX_ARCH_NAME?=$(LINUX_ARCH)
+
+
+MACHINE=$(TARGET_MACHINE)
 ifdef KODEBUG
 	MACHINE:=$(MACHINE)-debug
 	KODEDUG_SUFFIX:=-debug
@@ -43,72 +55,51 @@ else
 	DIST:=emulator
 endif
 
-INSTALL_DIR=koreader-$(DIST)-$(MACHINE)
+INSTALL_DIR ?= koreader-$(DIST)-$(MACHINE)
 
 # platform directories
 PLATFORM_DIR=platform
 COMMON_DIR=$(PLATFORM_DIR)/common
-ANDROID_DIR=$(PLATFORM_DIR)/android
-ANDROID_LAUNCHER_DIR:=$(ANDROID_DIR)/luajit-launcher
-ANDROID_ASSETS:=$(ANDROID_LAUNCHER_DIR)/assets/module
-ANDROID_LIBS_ROOT:=$(ANDROID_LAUNCHER_DIR)/libs
-ANDROID_LIBS_ABI:=$(ANDROID_LIBS_ROOT)/$(ANDROID_ABI)
-APPIMAGE_DIR=$(PLATFORM_DIR)/appimage
-CERVANTES_DIR=$(PLATFORM_DIR)/cervantes
-DEBIAN_DIR=$(PLATFORM_DIR)/debian
-KINDLE_DIR=$(PLATFORM_DIR)/kindle
-KOBO_DIR=$(PLATFORM_DIR)/kobo
-MACOS_DIR=$(PLATFORM_DIR)/mac
-POCKETBOOK_DIR=$(PLATFORM_DIR)/pocketbook
-REMARKABLE_DIR=$(PLATFORM_DIR)/remarkable
-SONY_PRSTUX_DIR=$(PLATFORM_DIR)/sony-prstux
-UBUNTUTOUCH_DIR=$(PLATFORM_DIR)/ubuntu-touch
-UBUNTUTOUCH_SDL_DIR:=$(UBUNTUTOUCH_DIR)/ubuntu-touch-sdl
 WIN32_DIR=$(PLATFORM_DIR)/win32
 
-# appimage setup
-APPIMAGETOOL=appimagetool-x86_64.AppImage
-APPIMAGETOOL_URL=https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage
-
-# set to 1 if in Docker
-DOCKER:=$(shell grep -q docker /proc/1/cgroup 2>/dev/null && echo 1)
-
 # files to link from main directory
 INSTALL_FILES=reader.lua setupkoenv.lua frontend resources defaults.lua datastorage.lua \
 		l10n tools README.md COPYING
 
-all: $(if $(ANDROID),,$(KOR_BASE)/$(OUTPUT_DIR)/luajit)
-	$(MAKE) -C $(KOR_BASE)
+ifeq ($(abspath $(OUTPUT_DIR)),$(OUTPUT_DIR))
+  ABSOLUTE_OUTPUT_DIR = $(OUTPUT_DIR)
+else
+  ABSOLUTE_OUTPUT_DIR = $(KOR_BASE)/$(OUTPUT_DIR)
+endif
+OUTPUT_DIR_ARTIFACTS = $(ABSOLUTE_OUTPUT_DIR)/!(cache|cmake|history|staging|thirdparty)
+
+all: base
 	install -d $(INSTALL_DIR)/koreader
 	rm -f $(INSTALL_DIR)/koreader/git-rev; echo "$(VERSION)" > $(INSTALL_DIR)/koreader/git-rev
 ifdef ANDROID
 	rm -f android-fdroid-version; echo -e "$(ANDROID_NAME)\n$(ANDROID_VERSION)" > koreader-android-fdroid-latest
 endif
 ifeq ($(IS_RELEASE),1)
-	$(RCP) -fL $(KOR_BASE)/$(OUTPUT_DIR)/. $(INSTALL_DIR)/koreader/.
+	bash -O extglob -c '$(RCP) -fL $(OUTPUT_DIR_ARTIFACTS) $(INSTALL_DIR)/koreader/'
 else
 	cp -f $(KOR_BASE)/ev_replay.py $(INSTALL_DIR)/koreader/
 	@echo "[*] create symlink instead of copying files in development mode"
-	cd $(INSTALL_DIR)/koreader && \
-		bash -O extglob -c "ln -sf ../../$(KOR_BASE)/$(OUTPUT_DIR)/!(cache|history) ."
+	bash -O extglob -c '$(SYMLINK) $(OUTPUT_DIR_ARTIFACTS) $(INSTALL_DIR)/koreader/'
+  ifneq (,$(EMULATE_READER))
 	@echo "[*] install front spec only for the emulator"
-	cd $(INSTALL_DIR)/koreader/spec && test -e front || \
-		ln -sf ../../../../spec ./front
-	cd $(INSTALL_DIR)/koreader/spec/front/unit && test -e data || \
-		ln -sf ../../test ./data
+	$(SYMLINK) $(abspath spec) $(INSTALL_DIR)/koreader/spec/front
+	$(SYMLINK) $(abspath test) $(INSTALL_DIR)/koreader/spec/front/unit/data
+  endif
 endif
-	for f in $(INSTALL_FILES); do \
-		ln -sf ../../$$f $(INSTALL_DIR)/koreader/; \
-	done
+	$(SYMLINK) $(abspath $(INSTALL_FILES)) $(INSTALL_DIR)/koreader/
 ifdef ANDROID
-	cd $(INSTALL_DIR)/koreader && \
-		ln -sf ../../$(ANDROID_DIR)/*.lua .
+	$(SYMLINK) $(abspath $(ANDROID_DIR)/*.lua) $(INSTALL_DIR)/koreader/
 endif
 	@echo "[*] Install update once marker"
 	@echo "# This file indicates that update once patches have not been applied yet." > $(INSTALL_DIR)/koreader/update_once.marker
 ifdef WIN32
 	@echo "[*] Install runtime libraries for win32..."
-	cd $(INSTALL_DIR)/koreader && cp ../../$(WIN32_DIR)/*.dll .
+	$(SYMLINK) $(abspath $(WIN32_DIR)/*.dll) $(INSTALL_DIR)/koreader/
 endif
 ifdef SHIP_SHARED_STL
 	@echo "[*] Install C++ runtime..."
@@ -117,36 +108,28 @@ ifdef SHIP_SHARED_STL
 	$(STRIP) --strip-unneeded $(INSTALL_DIR)/koreader/libs/$(notdir $(SHARED_STL_LIB))
 endif
 	@echo "[*] Install plugins"
-	@# TODO: link istead of cp?
-	$(RCP) plugins/. $(INSTALL_DIR)/koreader/plugins/.
-	@# purge deleted plugins
-	for d in $$(ls $(INSTALL_DIR)/koreader/plugins); do \
-		test -d plugins/$$d || rm -rf $(INSTALL_DIR)/koreader/plugins/$$d ; done
+	$(SYMLINK) $(abspath plugins) $(INSTALL_DIR)/koreader/
 	@echo "[*] Install resources"
-	$(RCP) -pL resources/fonts/. $(INSTALL_DIR)/koreader/fonts/.
+	$(SYMLINK) $(abspath resources/fonts/*) $(INSTALL_DIR)/koreader/fonts/
 	install -d $(INSTALL_DIR)/koreader/{screenshots,data/{dict,tessdata},fonts/host,ota}
 ifeq ($(IS_RELEASE),1)
 	@echo "[*] Clean up, remove unused files for releases"
 	rm -rf $(INSTALL_DIR)/koreader/data/{cr3.ini,cr3skin-format.txt,desktop,devices,manual}
 endif
 
-$(KOR_BASE)/$(OUTPUT_DIR)/luajit:
+base:
 	$(MAKE) -C $(KOR_BASE)
 
 $(INSTALL_DIR)/koreader/.busted: .busted
-	ln -sf ../../.busted $(INSTALL_DIR)/koreader
+	$(SYMLINK) $(abspath .busted) $@
 
 $(INSTALL_DIR)/koreader/.luacov:
-	test -e $(INSTALL_DIR)/koreader/.luacov || \
-		ln -sf ../../.luacov $(INSTALL_DIR)/koreader
+	$(SYMLINK) $(abspath .luacov) $@
 
 testfront: $(INSTALL_DIR)/koreader/.busted
 	# sdr files may have unexpected impact on unit testing
 	-rm -rf spec/unit/data/*.sdr
-	cd $(INSTALL_DIR)/koreader && ./luajit $(shell which busted) \
-		--sort-files \
-		--output=gtest \
-		--exclude-tags=notest $(BUSTED_OVERRIDES) $(BUSTED_SPEC_FILE)
+	cd $(INSTALL_DIR)/koreader && $(BUSTED_LUAJIT) $(BUSTED_OVERRIDES) $(BUSTED_SPEC_FILE)
 
 test: $(INSTALL_DIR)/koreader/.busted
 	$(MAKE) -C $(KOR_BASE) test
@@ -163,10 +146,21 @@ coverage: $(INSTALL_DIR)/koreader/.luacov
 		+$$(($$(grep -nm1 -e "^Summary$$" luacov.report.out|cut -d: -f1)-1)) \
 		luacov.report.out
 
-fetchthirdparty:
+$(KOR_BASE)/Makefile.defs fetchthirdparty:
 	git submodule init
 	git submodule sync
-	git submodule update
+ifneq (,$(CI))
+	git submodule update --depth 1 --jobs 3
+else
+	# Force shallow clones of submodules configured as such.
+	git submodule update --jobs 3 --depth 1 $(shell \
+		git config --file=.gitmodules --name-only --get-regexp '^submodule\.[^.]+\.shallow$$' true \
+		| sed 's/\.shallow$$/.path/' \
+		| xargs -n1 git config --file=.gitmodules \
+		)
+	# Update the rest.
+	git submodule update --jobs 3
+endif
 	$(MAKE) -C $(KOR_BASE) fetchthirdparty
 
 VERBOSE ?= @
@@ -184,413 +178,10 @@ dist-clean: clean
 	$(MAKE) -C $(KOR_BASE) dist-clean
 	$(MAKE) -C doc clean
 
-KINDLE_PACKAGE:=koreader-$(DIST)$(KODEDUG_SUFFIX)-$(VERSION).zip
-KINDLE_PACKAGE_OTA:=koreader-$(DIST)$(KODEDUG_SUFFIX)-$(VERSION).targz
-ZIP_EXCLUDE=-x "*.swp" -x "*.swo" -x "*.orig" -x "*.un~"
-# Don't bundle launchpad on touch devices..
-ifeq ($(TARGET), kindle-legacy)
-KINDLE_LEGACY_LAUNCHER:=launchpad
+# Include target specific rules.
+ifneq (,$(wildcard make/$(TARGET).mk))
+  include make/$(TARGET).mk
 endif
-kindleupdate: all
-	# ensure that the binaries were built for ARM
-	file $(INSTALL_DIR)/koreader/luajit | grep ARM || exit 1
-	# remove old package if any
-	rm -f $(KINDLE_PACKAGE)
-	# Kindle launching scripts
-	ln -sf ../$(KINDLE_DIR)/extensions $(INSTALL_DIR)/
-	ln -sf ../$(KINDLE_DIR)/launchpad $(INSTALL_DIR)/
-	ln -sf ../../$(KINDLE_DIR)/koreader.sh $(INSTALL_DIR)/koreader
-	ln -sf ../../$(KINDLE_DIR)/libkohelper.sh $(INSTALL_DIR)/koreader
-	ln -sf ../../../../../$(KINDLE_DIR)/libkohelper.sh $(INSTALL_DIR)/extensions/koreader/bin
-	ln -sf ../../$(COMMON_DIR)/spinning_zsync $(INSTALL_DIR)/koreader
-	ln -sf ../../$(KINDLE_DIR)/wmctrl $(INSTALL_DIR)/koreader
-	# create new package
-	cd $(INSTALL_DIR) && pwd && \
-		zip -9 -r \
-			../$(KINDLE_PACKAGE) \
-			extensions koreader $(KINDLE_LEGACY_LAUNCHER) \
-			-x "koreader/resources/fonts/*" "koreader/ota/*" \
-			"koreader/resources/icons/src/*" "koreader/spec/*" \
-			$(ZIP_EXCLUDE)
-	# generate kindleupdate package index file
-	zipinfo -1 $(KINDLE_PACKAGE) > \
-		$(INSTALL_DIR)/koreader/ota/package.index
-	echo "koreader/ota/package.index" >> $(INSTALL_DIR)/koreader/ota/package.index
-	# update index file in zip package
-	cd $(INSTALL_DIR) && zip -u ../$(KINDLE_PACKAGE) \
-		koreader/ota/package.index
-	# make gzip kindleupdate for zsync OTA update
-	# note that the targz file extension is intended to keep ISP from caching
-	# the file, see koreader#1644.
-	cd $(INSTALL_DIR) && \
-		tar --hard-dereference -I"gzip --rsyncable" -cah --no-recursion -f ../$(KINDLE_PACKAGE_OTA) \
-		-T koreader/ota/package.index
-
-KOBO_PACKAGE:=koreader-kobo$(KODEDUG_SUFFIX)-$(VERSION).zip
-KOBO_PACKAGE_OTA:=koreader-kobo$(KODEDUG_SUFFIX)-$(VERSION).targz
-koboupdate: all
-	# ensure that the binaries were built for ARM
-	file $(INSTALL_DIR)/koreader/luajit | grep ARM || exit 1
-	# remove old package if any
-	rm -f $(KOBO_PACKAGE)
-	# Kobo launching scripts
-	cp $(KOBO_DIR)/koreader.png $(INSTALL_DIR)/koreader.png
-	cp $(KOBO_DIR)/*.sh $(INSTALL_DIR)/koreader
-	cp $(COMMON_DIR)/spinning_zsync $(INSTALL_DIR)/koreader
-	# create new package
-	cd $(INSTALL_DIR) && \
-		zip -9 -r \
-			../$(KOBO_PACKAGE) \
-			koreader -x "koreader/resources/fonts/*" \
-			"koreader/resources/icons/src/*" "koreader/spec/*" \
-			$(ZIP_EXCLUDE)
-	# generate koboupdate package index file
-	zipinfo -1 $(KOBO_PACKAGE) > \
-		$(INSTALL_DIR)/koreader/ota/package.index
-	echo "koreader/ota/package.index" >> $(INSTALL_DIR)/koreader/ota/package.index
-	# update index file in zip package
-	cd $(INSTALL_DIR) && zip -u ../$(KOBO_PACKAGE) \
-		koreader/ota/package.index koreader.png README_kobo.txt
-	# make gzip koboupdate for zsync OTA update
-	cd $(INSTALL_DIR) && \
-		tar --hard-dereference -I"gzip --rsyncable" -cah --no-recursion -f ../$(KOBO_PACKAGE_OTA) \
-		-T koreader/ota/package.index
-
-PB_PACKAGE:=koreader-pocketbook$(KODEDUG_SUFFIX)-$(VERSION).zip
-PB_PACKAGE_OTA:=koreader-pocketbook$(KODEDUG_SUFFIX)-$(VERSION).targz
-pbupdate: all
-	# ensure that the binaries were built for ARM
-	file $(INSTALL_DIR)/koreader/luajit | grep ARM || exit 1
-	# remove old package if any
-	rm -f $(PB_PACKAGE)
-	# Pocketbook launching scripts
-	mkdir -p $(INSTALL_DIR)/applications
-	mkdir -p $(INSTALL_DIR)/system/bin
-	cp $(POCKETBOOK_DIR)/koreader.app $(INSTALL_DIR)/applications
-	cp $(POCKETBOOK_DIR)/system_koreader.app $(INSTALL_DIR)/system/bin/koreader.app
-	cp $(COMMON_DIR)/spinning_zsync $(INSTALL_DIR)/koreader
-	cp -rfL $(INSTALL_DIR)/koreader $(INSTALL_DIR)/applications
-	find $(INSTALL_DIR)/applications/koreader \
-		-type f \( -name "*.gif" -o -name "*.html" -o -name "*.md" -o -name "*.txt" \) \
-		-exec rm -vf {} \;
-	# create new package
-	cd $(INSTALL_DIR) && \
-		zip -9 -r \
-			../$(PB_PACKAGE) \
-			applications -x "applications/koreader/resources/fonts/*" \
-			"applications/koreader/resources/icons/src/*" "applications/koreader/spec/*" \
-			$(ZIP_EXCLUDE)
-	# generate koboupdate package index file
-	zipinfo -1 $(PB_PACKAGE) > \
-		$(INSTALL_DIR)/applications/koreader/ota/package.index
-	echo "applications/koreader/ota/package.index" >> \
-		$(INSTALL_DIR)/applications/koreader/ota/package.index
-	# hack file path when running tar in parent directory of koreader
-	sed -i -e 's/^/..\//' \
-		$(INSTALL_DIR)/applications/koreader/ota/package.index
-	# update index file in zip package
-	cd $(INSTALL_DIR) && zip -ru ../$(PB_PACKAGE) \
-		applications/koreader/ota/package.index system
-	# make gzip pbupdate for zsync OTA update
-	cd $(INSTALL_DIR)/applications && \
-		tar --hard-dereference -I"gzip --rsyncable" -cah --no-recursion -f ../../$(PB_PACKAGE_OTA) \
-		-T koreader/ota/package.index
-
-utupdate: all
-	# ensure that the binaries were built for ARM
-	file $(INSTALL_DIR)/koreader/luajit | grep ARM || exit 1
-	# remove old package if any
-	rm -f koreader-ubuntu-touch-$(MACHINE)-$(VERSION).click
-
-	ln -sf ../../$(UBUNTUTOUCH_DIR)/koreader.sh $(INSTALL_DIR)/koreader
-	ln -sf ../../$(UBUNTUTOUCH_DIR)/manifest.json $(INSTALL_DIR)/koreader
-	ln -sf ../../$(UBUNTUTOUCH_DIR)/koreader.apparmor $(INSTALL_DIR)/koreader
-	ln -sf ../../$(UBUNTUTOUCH_DIR)/koreader.apparmor.openstore $(INSTALL_DIR)/koreader
-	ln -sf ../../$(UBUNTUTOUCH_DIR)/koreader.desktop $(INSTALL_DIR)/koreader
-	ln -sf ../../$(UBUNTUTOUCH_DIR)/koreader.png $(INSTALL_DIR)/koreader
-	ln -sf ../../../$(UBUNTUTOUCH_DIR)/libSDL2.so $(INSTALL_DIR)/koreader/libs
-
-	# create new package
-	cd $(INSTALL_DIR) && pwd && \
-		zip -9 -r \
-			../koreader-$(DIST)-$(MACHINE)-$(VERSION).zip \
-			koreader -x "koreader/resources/fonts/*" "koreader/ota/*" \
-			"koreader/resources/icons/src/*" "koreader/spec/*" \
-			$(ZIP_EXCLUDE)
-
-	# generate ubuntu touch click package
-	rm -rf $(INSTALL_DIR)/tmp && mkdir -p $(INSTALL_DIR)/tmp
-	cd $(INSTALL_DIR)/tmp && \
-		unzip ../../koreader-$(DIST)-$(MACHINE)-$(VERSION).zip && \
-		click build koreader && \
-		mv *.click ../../koreader-$(DIST)-$(MACHINE)-$(VERSION).click
-
-appimageupdate: all
-	# remove old package if any
-	rm -f koreader-appimage-$(MACHINE)-$(VERSION).appimage
-
-	ln -sf ../../$(APPIMAGE_DIR)/AppRun $(INSTALL_DIR)/koreader
-	ln -sf ../../$(APPIMAGE_DIR)/koreader.appdata.xml $(INSTALL_DIR)/koreader
-	ln -sf ../../$(APPIMAGE_DIR)/koreader.desktop $(INSTALL_DIR)/koreader
-	ln -sf ../../resources/koreader.png $(INSTALL_DIR)/koreader
-	# TODO at best this is DebUbuntu specific
-	ln -sf /usr/lib/x86_64-linux-gnu/libSDL2-2.0.so.0 $(INSTALL_DIR)/koreader/libs/libSDL2.so
-	# required for our stock Ubuntu SDL even though we don't use sound
-	# the readlink is a half-hearted attempt at being generic; the echo libsndio.so.6.1 is specific to the nightly builds
-	ln -sf /usr/lib/x86_64-linux-gnu/$(shell readlink /usr/lib/x86_64-linux-gnu/libsndio.so || echo libsndio.so.6.1) $(INSTALL_DIR)/koreader/libs/
-	# also copy libbsd.so.0, cf. https://github.com/koreader/koreader/issues/4627
-	ln -sf /lib/x86_64-linux-gnu/libbsd.so.0 $(INSTALL_DIR)/koreader/libs/
-ifeq ("$(wildcard $(APPIMAGETOOL))","")
-	# download appimagetool
-	wget "$(APPIMAGETOOL_URL)"
-	chmod a+x "$(APPIMAGETOOL)"
-endif
-ifeq ($(DOCKER), 1)
-	# remove previously extracted appimagetool, if any
-	rm -rf squashfs-root
-	./$(APPIMAGETOOL) --appimage-extract
-endif
-	cd $(INSTALL_DIR) && pwd && \
-		rm -rf tmp && mkdir -p tmp && \
-		cp -Lr koreader tmp && \
-		rm -rf tmp/koreader/ota && \
-		rm -rf tmp/koreader/resources/icons/src && \
-		rm -rf tmp/koreader/spec
-
-	# generate AppImage
-	cd $(INSTALL_DIR)/tmp && \
-		ARCH=x86_64 ../../$(if $(DOCKER),squashfs-root/AppRun,$(APPIMAGETOOL)) koreader && \
-		mv *.AppImage ../../koreader-$(DIST)-$(MACHINE)-$(VERSION).AppImage
-
-androidupdate: all
-	# Note: do not remove the module directory so there's no need
-	# for `mk7z.sh` to always recreate `assets.7z` from scratch.
-	rm -rfv $(ANDROID_LIBS_ROOT)
-	mkdir -p $(ANDROID_ASSETS) $(ANDROID_LIBS_ABI)
-
-	# APK version
-	echo $(VERSION) > $(ANDROID_ASSETS)/version.txt
-
-	# shared libraries are stored as raw assets
-	cp -pR $(INSTALL_DIR)/koreader/libs $(ANDROID_LAUNCHER_DIR)/assets
-	# in runtime luajit-launcher's libluajit.so will be loaded
-	rm -vf $(ANDROID_LAUNCHER_DIR)/assets/libs/libluajit.so
-
-	# binaries are stored as shared libraries to prevent W^X exception on Android 10+
-	# https://developer.android.com/about/versions/10/behavior-changes-10#execute-permission
-	cp -pR $(INSTALL_DIR)/koreader/sdcv $(ANDROID_LIBS_ABI)/libsdcv.so
-	echo "sdcv libsdcv.so" > $(ANDROID_ASSETS)/map.txt
-
-	# assets are compressed manually and stored inside the APK.
-	cd $(INSTALL_DIR)/koreader && \
-		./tools/mk7z.sh \
-		../../$(ANDROID_ASSETS)/koreader.7z \
-		"$$(git show -s --format='%ci')" \
-		-m0=lzma2 -mx=9 \
-		-- . \
-		'-x!cache' \
-		'-x!clipboard' \
-		'-x!data/dict' \
-		'-x!data/tessdata' \
-		'-x!history' \
-		'-x!l10n/templates' \
-		'-x!libs' \
-		'-x!ota' \
-		'-x!resources/fonts*' \
-		'-x!resources/icons/src*' \
-		'-x!rocks/bin' \
-		'-x!rocks/lib/luarocks' \
-		'-x!screenshots' \
-		'-x!sdcv' \
-		'-x!spec' \
-		'-x!tools' \
-		'-xr!.*' \
-		'-xr!COPYING' \
-		'-xr!NOTES.txt' \
-		'-xr!NOTICE' \
-		'-xr!README.md' \
-		;
-
-	# make the android APK
-	# Note: filter out the `--debug=…` make flag
-	# so the old crummy version provided by the
-	# NDK does not blow a gasket.
-	MAKEFLAGS='$(filter-out --debug=%,$(MAKEFLAGS))' \
-		$(MAKE) -C $(ANDROID_LAUNCHER_DIR) $(if $(KODEBUG), debug, release) \
-		ANDROID_APPNAME=KOReader \
-		ANDROID_VERSION=$(ANDROID_VERSION) \
-		ANDROID_NAME=$(ANDROID_NAME) \
-		ANDROID_FLAVOR=$(ANDROID_FLAVOR)
-
-	cp $(ANDROID_LAUNCHER_DIR)/bin/NativeActivity.apk \
-		koreader-android-$(ANDROID_ARCH)$(KODEDUG_SUFFIX)-$(VERSION).apk
-
-debianupdate: all
-	mkdir -pv \
-		$(INSTALL_DIR)/debian/usr/bin \
-		$(INSTALL_DIR)/debian/usr/lib \
-		$(INSTALL_DIR)/debian/usr/share/pixmaps \
-		$(INSTALL_DIR)/debian/usr/share/applications \
-		$(INSTALL_DIR)/debian/usr/share/doc/koreader \
-		$(INSTALL_DIR)/debian/usr/share/man/man1
-
-	cp -pv resources/koreader.png $(INSTALL_DIR)/debian/usr/share/pixmaps
-	cp -pv $(DEBIAN_DIR)/koreader.desktop $(INSTALL_DIR)/debian/usr/share/applications
-	cp -pv $(DEBIAN_DIR)/copyright COPYING $(INSTALL_DIR)/debian/usr/share/doc/koreader
-	cp -pv $(DEBIAN_DIR)/koreader.sh $(INSTALL_DIR)/debian/usr/bin/koreader
-	cp -Lr $(INSTALL_DIR)/koreader $(INSTALL_DIR)/debian/usr/lib
-
-	gzip -cn9 $(DEBIAN_DIR)/changelog > $(INSTALL_DIR)/debian/usr/share/doc/koreader/changelog.Debian.gz
-	gzip -cn9 $(DEBIAN_DIR)/koreader.1 > $(INSTALL_DIR)/debian/usr/share/man/man1/koreader.1.gz
-
-	chmod 644 \
-		$(INSTALL_DIR)/debian/usr/share/doc/koreader/changelog.Debian.gz \
-		$(INSTALL_DIR)/debian/usr/share/doc/koreader/copyright \
-		$(INSTALL_DIR)/debian/usr/share/man/man1/koreader.1.gz
-
-	rm -rf \
-		$(INSTALL_DIR)/debian/usr/lib/koreader/{ota,cache,clipboard,screenshots,spec,tools,resources/fonts,resources/icons/src}
-
-macosupdate: all
-	mkdir -p \
-		$(INSTALL_DIR)/bundle/Contents/MacOS \
-		$(INSTALL_DIR)/bundle/Contents/Resources
-
-	cp -pv $(MACOS_DIR)/koreader.icns $(INSTALL_DIR)/bundle/Contents/Resources/icon.icns
-	cp -LR $(INSTALL_DIR)/koreader $(INSTALL_DIR)/bundle/Contents
-	cp -pRv $(MACOS_DIR)/menu.xml $(INSTALL_DIR)/bundle/Contents/MainMenu.xib
-	ibtool --compile "$(INSTALL_DIR)/bundle/Contents/Resources/Base.lproj/MainMenu.nib" "$(INSTALL_DIR)/bundle/Contents/MainMenu.xib"
-	rm -rfv "$(INSTALL_DIR)/bundle/Contents/MainMenu.xib"
-
-REMARKABLE_PACKAGE:=koreader-remarkable$(KODEDUG_SUFFIX)-$(VERSION).zip
-REMARKABLE_PACKAGE_OTA:=koreader-remarkable$(KODEDUG_SUFFIX)-$(VERSION).targz
-remarkableupdate: all
-	# ensure that the binaries were built for ARM
-	file $(INSTALL_DIR)/koreader/luajit | grep ARM || exit 1
-	# remove old package if any
-	rm -f $(REMARKABLE_PACKAGE)
-	# Remarkable scripts
-	cp $(REMARKABLE_DIR)/* $(INSTALL_DIR)/koreader
-	cp $(COMMON_DIR)/spinning_zsync $(INSTALL_DIR)/koreader
-	# create new package
-	cd $(INSTALL_DIR) && \
-	        zip -9 -r \
-	                ../$(REMARKABLE_PACKAGE) \
-	                koreader -x "koreader/resources/fonts/*" \
-	                "koreader/resources/icons/src/*" "koreader/spec/*" \
-	                $(ZIP_EXCLUDE)
-	# generate update package index file
-	zipinfo -1 $(REMARKABLE_PACKAGE) > \
-	        $(INSTALL_DIR)/koreader/ota/package.index
-	echo "koreader/ota/package.index" >> $(INSTALL_DIR)/koreader/ota/package.index
-	# update index file in zip package
-	cd $(INSTALL_DIR) && zip -u ../$(REMARKABLE_PACKAGE) \
-	        koreader/ota/package.index
-	# make gzip remarkable update for zsync OTA update
-	cd $(INSTALL_DIR) && \
-	        tar -I"gzip --rsyncable" -cah --no-recursion -f ../$(REMARKABLE_PACKAGE_OTA) \
-	        -T koreader/ota/package.index
-
-SONY_PRSTUX_PACKAGE:=koreader-sony-prstux$(KODEDUG_SUFFIX)-$(VERSION).zip
-SONY_PRSTUX_PACKAGE_OTA:=koreader-sony-prstux$(KODEDUG_SUFFIX)-$(VERSION).targz
-sony-prstuxupdate: all
-	# ensure that the binaries were built for ARM
-	file $(INSTALL_DIR)/koreader/luajit | grep ARM || exit 1
-	# remove old package if any
-	rm -f $(SONY_PRSTUX_PACKAGE)
-	# Sony PRSTUX launching scripts
-	cp $(SONY_PRSTUX_DIR)/*.sh $(INSTALL_DIR)/koreader
-	# create new package
-	cd $(INSTALL_DIR) && \
-	        zip -9 -r \
-	                ../$(SONY_PRSTUX_PACKAGE) \
-	                koreader -x "koreader/resources/fonts/*" \
-	                "koreader/resources/icons/src/*" "koreader/spec/*" \
-	                $(ZIP_EXCLUDE)
-	# generate update package index file
-	zipinfo -1 $(SONY_PRSTUX_PACKAGE) > \
-	        $(INSTALL_DIR)/koreader/ota/package.index
-	echo "koreader/ota/package.index" >> $(INSTALL_DIR)/koreader/ota/package.index
-	# update index file in zip package
-	cd $(INSTALL_DIR) && zip -u ../$(SONY_PRSTUX_PACKAGE) \
-	        koreader/ota/package.index
-	# make gzip sonyprstux update for zsync OTA update
-	cd $(INSTALL_DIR) && \
-	        tar --hard-dereference -I"gzip --rsyncable" -cah --no-recursion -f ../$(SONY_PRSTUX_PACKAGE_OTA) \
-	        -T koreader/ota/package.index
-
-CERVANTES_PACKAGE:=koreader-cervantes$(KODEDUG_SUFFIX)-$(VERSION).zip
-CERVANTES_PACKAGE_OTA:=koreader-cervantes$(KODEDUG_SUFFIX)-$(VERSION).targz
-cervantesupdate: all
-	# ensure that the binaries were built for ARM
-	file $(INSTALL_DIR)/koreader/luajit | grep ARM || exit 1
-	# remove old package if any
-	rm -f $(CERVANTES_PACKAGE)
-	# Cervantes launching scripts
-	cp $(COMMON_DIR)/spinning_zsync $(INSTALL_DIR)/koreader/spinning_zsync.sh
-	cp $(CERVANTES_DIR)/*.sh $(INSTALL_DIR)/koreader
-	cp $(CERVANTES_DIR)/spinning_zsync $(INSTALL_DIR)/koreader
-	# create new package
-	cd $(INSTALL_DIR) && \
-		zip -9 -r \
-			../$(CERVANTES_PACKAGE) \
-			koreader -x "koreader/resources/fonts/*" \
-			"koreader/resources/icons/src/*" "koreader/spec/*" \
-			$(ZIP_EXCLUDE)
-	# generate update package index file
-	zipinfo -1 $(CERVANTES_PACKAGE) > \
-		$(INSTALL_DIR)/koreader/ota/package.index
-	echo "koreader/ota/package.index" >> $(INSTALL_DIR)/koreader/ota/package.index
-	# update index file in zip package
-	cd $(INSTALL_DIR) && zip -u ../$(CERVANTES_PACKAGE) \
-	koreader/ota/package.index
-	# make gzip cervantes update for zsync OTA update
-	cd $(INSTALL_DIR) && \
-	tar --hard-dereference -I"gzip --rsyncable" -cah --no-recursion -f ../$(CERVANTES_PACKAGE_OTA) \
-	-T koreader/ota/package.index
-
-update:
-ifeq ($(TARGET), android)
-	make androidupdate
-else ifeq ($(TARGET), appimage)
-	make appimageupdate
-else ifeq ($(TARGET), cervantes)
-	make cervantesupdate
-else ifeq ($(TARGET), kindle)
-	make kindleupdate
-else ifeq ($(TARGET), kindle-legacy)
-	make kindleupdate
-else ifeq ($(TARGET), kindlepw2)
-	make kindleupdate
-else ifeq ($(TARGET), kobo)
-	make koboupdate
-else ifeq ($(TARGET), pocketbook)
-	make pbupdate
-else ifeq ($(TARGET), sony-prstux)
-	make sony-prstuxupdate
-else ifeq ($(TARGET), remarkable)
-	make remarkableupdate
-else ifeq ($(TARGET), ubuntu-touch)
-	make utupdate
-else ifeq ($(TARGET), debian)
-	make debianupdate
-	$(CURDIR)/platform/debian/do_debian_package.sh $(INSTALL_DIR)
-else ifeq ($(TARGET), debian-armel)
-	make debianupdate
-	$(CURDIR)/platform/debian/do_debian_package.sh $(INSTALL_DIR) armel
-else ifeq ($(TARGET), debian-armhf)
-	make debianupdate
-	$(CURDIR)/platform/debian/do_debian_package.sh $(INSTALL_DIR) armhf
-else ifeq ($(TARGET), debian-arm64)
-	make debianupdate
-	$(CURDIR)/platform/debian/do_debian_package.sh $(INSTALL_DIR) arm64
-else ifeq ($(TARGET), macos)
-	make macosupdate
-	$(CURDIR)/platform/mac/do_mac_bundle.sh $(INSTALL_DIR)
-endif
-
-androiddev: androidupdate
-	$(MAKE) -C $(ANDROID_LAUNCHER_DIR) dev
 
 android-ndk:
 	$(MAKE) -C $(KOR_BASE)/toolchain $(ANDROID_NDK_HOME)
@@ -598,7 +189,6 @@ android-ndk:
 android-sdk:
 	$(MAKE) -C $(KOR_BASE)/toolchain $(ANDROID_HOME)
 
-
 # for gettext
 DOMAIN=koreader
 TEMPLATE_DIR=l10n/templates
@@ -629,4 +219,10 @@ static-check:
 doc:
 	make -C doc
 
-.PHONY: all clean doc test update
+.NOTPARALLEL:
+.PHONY: $(PHONY)
+
+LEFTOVERS = $(filter-out $(PHONY) $(INSTALL_DIR)/%,$(MAKECMDGOALS))
+.PHONY: $(LEFTOVERS)
+$(LEFTOVERS):
+	$(MAKE) -C $(KOR_BASE) $@

README.md

@@ -11,7 +11,7 @@
 [![Weblate Status][badge-weblate]][link-weblate]
 
 [Download](https://github.com/koreader/koreader/releases) •
-[User guide](http://koreader.rocks/koreader-user-guide.pdf) •
+[User guide](http://koreader.rocks/user_guide/) •
 [Wiki](https://github.com/koreader/koreader/wiki) •
 [Developer docs](http://koreader.rocks/doc/)
 
@@ -33,7 +33,7 @@
 
 * **and much more**: look up words with StarDict dictionaries / Wikipedia, add your own online OPDS catalogs and RSS feeds, over-the-air software updates, an FTP client, an SSH server, …
 
-Please check the [user guide](http://koreader.rocks/koreader-user-guide.pdf) and the [wiki][link-wiki] to discover more features and to help us document them.
+Please check the [user guide](http://koreader.rocks/user_guide/) and the [wiki][link-wiki] to discover more features and to help us document them.
 
 ## Screenshots
 
@@ -71,7 +71,7 @@ KOReader is developed and supported by volunteers all around the world. There ar
 - document lesser-known features on the [wiki][link-wiki]
 - help others with your knowledge on the [forum][link-forum]
 
-Right now we only support [liberapay](https://liberapay.com/KOReader) donations, but you can also create a [bounty][link-bountysource] to motivate others to work on a specific bug or feature request.
+Right now we only support [liberapay](https://liberapay.com/KOReader) donations.
 
 ## Contributors
 

base

@@ -1 +1 @@
-Subproject commit bab6f040e73a5118335a777849c2f6e896f40638
+Subproject commit a6e112a5ee06e6939203accbd0736134af8939cd

datastorage.lua

@@ -17,7 +17,7 @@ function DataStorage:getDataDir()
         local package_name = app_id:match("^(.-)_")
         -- confined ubuntu app has write access to this dir
         data_dir = string.format("%s/%s", os.getenv("XDG_DATA_HOME"), package_name)
-    elseif os.getenv("APPIMAGE") or os.getenv("KO_MULTIUSER") then
+    elseif os.getenv("APPIMAGE") or os.getenv("FLATPAK") or os.getenv("KO_MULTIUSER") then
         if os.getenv("XDG_CONFIG_HOME") then
             data_dir = string.format("%s/%s", os.getenv("XDG_CONFIG_HOME"), "koreader")
             if lfs.attributes(os.getenv("XDG_CONFIG_HOME"), "mode") ~= "directory" then
@@ -53,6 +53,10 @@ function DataStorage:getDocSettingsDir()
     return self:getDataDir() .. "/docsettings"
 end
 
+function DataStorage:getDocSettingsHashDir()
+    return self:getDataDir() .. "/hashdocsettings"
+end
+
 function DataStorage:getFullDataDir()
     if full_data_dir then return full_data_dir end
 
@@ -73,6 +77,7 @@ local function initDataDir()
         "data/dict",
         "data/tessdata",
         -- "docsettings", -- created when needed
+        -- "hashdocsettings", -- created when needed
         -- "history", -- legacy/obsolete sidecar files
         "ota",
         -- "patches", -- must be created manually by the interested user

defaults.lua

@@ -36,14 +36,6 @@ DCREREADER_VIEW_MODE = "page",
 -- default to false
 DSHOWOVERLAP = false,
 
--- show hidden files in filemanager
--- default to false
-DSHOWHIDDENFILES = false,
-
--- landscape clockwise rotation
--- default to true, set to false for counterclockwise rotation
-DLANDSCAPE_CLOCKWISE_ROTATION = true,
-
 -- default minimum screen height for reading with 2 pages in landscape mode
 DCREREADER_TWO_PAGE_THRESHOLD = 7,
 
@@ -73,6 +65,10 @@ DTAP_ZONE_BOTTOM_LEFT = {x = 0, y = 7/8, w = 1/8, h = 1/8},
 DTAP_ZONE_BOTTOM_RIGHT = {x = 7/8, y = 7/8, w = 1/8, h = 1/8},
 DDOUBLE_TAP_ZONE_NEXT_CHAPTER = {x = 1/4, y = 0, w = 3/4, h = 1},
 DDOUBLE_TAP_ZONE_PREV_CHAPTER = {x = 0, y = 0, w = 1/4, h = 1},
+DSWIPE_ZONE_LEFT_EDGE = { x = 0, y = 0, w = 1/8, h = 1},
+DSWIPE_ZONE_RIGHT_EDGE = { x = 7/8, y = 0, w = 1/8, h = 1},
+DSWIPE_ZONE_TOP_EDGE = { x = 0, y = 0, w = 1, h = 1/8},
+DSWIPE_ZONE_BOTTOM_EDGE = { x = 0, y = 7/8, w = 1, h = 1/8},
 
 -- koptreader config defaults
 DKOPTREADER_CONFIG_FONT_SIZE = 1.0,        -- range from 0.1 to 3.0

doc/Building.md

@@ -13,7 +13,7 @@ You can skip most of the following instructions if desired, and use our premade
 
 To get and compile the source you must have `patch`, `wget`, `unzip`, `git`,
 `cmake` and `luarocks` installed, as well as a version of `autoconf`
-greater than 2.64. You also need `nasm`, `ragel`, and of course a compiler like `gcc`
+greater than 2.64. You also need `nasm`, and of course a compiler like `gcc`
 or `clang`.
 
 ### Debian/Ubuntu and derivates
@@ -22,8 +22,8 @@ Install the prerequisites using APT:
 
 ```
 sudo apt-get install build-essential git patch wget unzip \
-gettext autoconf automake cmake libtool nasm ragel luarocks lua5.1 libsdl2-dev \
-libssl-dev libffi-dev libsdl2-dev libc6-dev-i386 xutils-dev linux-libc-dev:i386 zlib1g:i386
+gettext autoconf automake cmake libtool libtool-bin nasm luarocks lua5.1 libsdl2-dev \
+libssl-dev libffi-dev libc6-dev-i386 xutils-dev linux-libc-dev:i386 zlib1g:i386
 ```
 
 ### Fedora/Red Hat
@@ -31,7 +31,7 @@ libssl-dev libffi-dev libsdl2-dev libc6-dev-i386 xutils-dev linux-libc-dev:i386
 Install the prerequisites using DNF:
 
 ```
-sudo dnf install libstdc++-static SDL SDL-devel patch wget unzip git cmake luarocks autoconf nasm ragel gcc
+sudo dnf install libstdc++-static SDL SDL-devel patch wget unzip git cmake luarocks autoconf nasm gcc
 ```
 
 ### macOS
@@ -39,7 +39,7 @@ sudo dnf install libstdc++-static SDL SDL-devel patch wget unzip git cmake luaro
 Install the prerequisites using [Homebrew](https://brew.sh/):
 
 ```
-brew install nasm ragel binutils coreutils libtool autoconf automake cmake makedepend \
+brew install nasm binutils coreutils libtool autoconf automake cmake makedepend \
 sdl2 lua@5.1 luarocks gettext pkg-config wget gnu-getopt grep bison
 ```
 

doc/Building_targets.md

@@ -17,28 +17,15 @@ Each target has its own architecture and you'll need to setup a proper cross-com
 A compatible version of the Android NDK and SDK will be downloaded automatically by `./kodev release android` if no NDK or SDK is provided in environment variables. For that purpose you can use:
 
 ```
-NDK=/ndk/location SDK=/sdk/location ./kodev release android
+ANDROID_NDK_HOME=/ndk/location ANDROID_HOME=/sdk/location ./kodev release android
 ```
 
-If you want to use your own installed tools please make sure that you have the **NDKr15c** and the SDK for Android 9 (**API level 28**) already installed.
+If you want to use your own installed tools please make sure that you have the **NDKr23c** and the SDK for Android 9 (**API level 28**) already installed.
 
 #### for embedded linux devices
 
 Cross compile toolchains are available for Ubuntu users through these commands:
 
-##### Ubuntu Touch
-
-```
-sudo apt-get install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf
-```
-
-**NOTE 1:** The packages `pkg-config-arm-linux-gnueabihf` and `pkg-config-arm-linux-gnueabi` may
-block you from building. Remove them if you get the following ld error
-
-```
-/usr/lib/gcc-cross/arm-linux-gnueabihf/4.8/../../../../arm-linux-gnueabihf/bin/ld: cannot find -lglib-2.0
-```
-
 ##### e-Ink devices (e.g., Kindle, Kobo, Cervantes, reMarkable, PocketBook)
 
 **NOTE:** While, for some targets (specifically, Cervantes, Kindle & Kobo), we make *some* effort to support Linaro/Ubuntu TCs,
@@ -69,20 +56,6 @@ sudo apt-get install openjdk-8-jdk p7zip-full
 
 Building a debian package requires the `dpkg-deb` tool. It should be already installed if you're on a Debian/Ubuntu based distribution.
 
-#### for Ubuntu Touch
-
-Building for Ubuntu Touch requires the `click` package management tool.
-
-Ubuntu users can install it with:
-
-```
-sudo apt-get install click
-```
-
-**NOTE**: The Ubuntu Touch build won't start anymore, and none of the currently active developers have any physical devices. Please visit [#4960](https://github.com/koreader/koreader/issues/4960) if you want to help.
-
-The Ubuntu Touch builds are therefore no longer published under releases on GitHub, but they are still available from [the nightly build server](http://build.koreader.rocks/download/nightly/).
-
 ## Building 
 
 You can check out our [nightlybuild script][nb-script] to see how to build a package from scratch.
@@ -167,14 +140,6 @@ See [Building](https://github.com/koreader/koreader/blob/master/doc/Building.md)
 ./kodev release remarkable
 ```
 
-### Embedded Linux devices
-
-#### Ubuntu Touch
-
-```
-./kodev release ubuntu-touch
-```
-
 ## Porting to a new target.
 
 See [Porting.md](Porting.md)

frontend/apps/cloudstorage/webdav.lua

@@ -84,21 +84,9 @@ function WebDav:config(item, callback)
 This can point to a sub-directory of the WebDAV server.
 The start folder is appended to the server path.]])
 
-    local hint_name = _("Server display name")
-    local text_name = ""
-    local hint_address = _("WebDAV address, for example https://example.com/dav")
-    local text_address = ""
-    local hint_username = _("Username")
-    local text_username = ""
-    local hint_password = _("Password")
-    local text_password = ""
-    local hint_folder = _("Start folder")
-    local text_folder = ""
-    local title
-    local text_button_ok = _("Add")
+    local title, text_name, text_address, text_username, text_password, text_folder
     if item then
         title = _("Edit WebDAV account")
-        text_button_ok = _("Apply")
         text_name = item.text
         text_address = item.address
         text_username = item.username
@@ -112,29 +100,24 @@ The start folder is appended to the server path.]])
         fields = {
             {
                 text = text_name,
-                input_type = "string",
-                hint = hint_name ,
+                hint = _("Server display name"),
             },
             {
                 text = text_address,
-                input_type = "string",
-                hint = hint_address ,
+                hint = _("WebDAV address, for example https://example.com/dav"),
             },
             {
                 text = text_username,
-                input_type = "string",
-                hint = hint_username,
+                hint = _("Username"),
             },
             {
                 text = text_password,
-                input_type = "string",
                 text_type = "password",
-                hint = hint_password,
+                hint = _("Password"),
             },
             {
                 text = text_folder,
-                input_type = "string",
-                hint = hint_folder,
+                hint = _("Start folder, for example /books"),
             },
         },
         buttons = {
@@ -154,18 +137,17 @@ The start folder is appended to the server path.]])
                     end
                 },
                 {
-                    text = text_button_ok,
+                    text = _("Save"),
                     callback = function()
                         local fields = self.settings_dialog:getFields()
-
-                        -- make sure the URL is a valid path
-                        if fields[5] ~= "" then
-                            if string.sub(fields[5], 1, 1) ~= '/' then
-                                fields[5] = '/' .. fields[5]
-                            end
-                        end
-
                         if fields[1] ~= "" and fields[2] ~= "" then
+                            -- make sure the URL is a valid path
+                            if fields[5] ~= "" then
+                                if not fields[5]:match('^/') then
+                                    fields[5] = '/' .. fields[5]
+                                end
+                                fields[5] = fields[5]:gsub("/$", "")
+                            end
                             if item then
                                 -- edit
                                 callback(item, fields)
@@ -184,11 +166,9 @@ The start folder is appended to the server path.]])
                 },
             },
         },
-        input_type = "text",
     }
     UIManager:show(self.settings_dialog)
     self.settings_dialog:onShowKeyboard()
-
 end
 
 function WebDav:info(item)

frontend/apps/filemanager/filemanagerbookinfo.lua

@@ -4,38 +4,67 @@ This module provides a way to display book information (filename and book metada
 
 local BD = require("ui/bidi")
 local ButtonDialog = require("ui/widget/buttondialog")
+local ConfirmBox = require("ui/widget/confirmbox")
+local Device = require("device")
 local DocSettings = require("docsettings")
+local Document = require("document/document")
 local DocumentRegistry = require("document/documentregistry")
+local Event = require("ui/event")
 local InfoMessage = require("ui/widget/infomessage")
+local InputDialog = require("ui/widget/inputdialog")
+local TextViewer = require("ui/widget/textviewer")
 local UIManager = require("ui/uimanager")
 local WidgetContainer = require("ui/widget/container/widgetcontainer")
-local ffiutil = require("ffi/util")
+local Utf8Proc = require("ffi/utf8proc")
 local filemanagerutil = require("apps/filemanager/filemanagerutil")
 local lfs = require("libs/libkoreader-lfs")
 local util = require("util")
 local _ = require("gettext")
-local Screen = require("device").screen
+local N_ = _.ngettext
+local T = require("ffi/util").template
 
 local BookInfo = WidgetContainer:extend{
+    title = _("Book information"),
+    props = {
+        "title",
+        "authors",
+        "series",
+        "series_index",
+        "language",
+        "keywords",
+        "description",
+    },
+    prop_text = {
+        cover        = _("Cover image:"),
+        title        = _("Title:"),
+        authors      = _("Authors:"),
+        series       = _("Series:"),
+        series_index = _("Series index:"),
+        language     = _("Language:"),
+        keywords     = _("Keywords:"),
+        description  = _("Description:"),
+        pages        = _("Pages:"),
+    },
 }
 
 function BookInfo:init()
-    if self.ui then -- only for Reader menu
+    if self.document then -- only for Reader menu
         self.ui.menu:registerToMainMenu(self)
     end
 end
 
 function BookInfo:addToMainMenu(menu_items)
     menu_items.book_info = {
-        text = _("Book information"),
+        text = self.title,
         callback = function()
             self:onShowBookInfo()
         end,
     }
 end
 
-function BookInfo:show(file, book_props, metadata_updated_caller_callback)
-    self.updated = nil
+-- Shows book information.
+function BookInfo:show(file, book_props)
+    self.prop_updated = nil
     local kv_pairs = {}
 
     -- File section
@@ -55,20 +84,32 @@ function BookInfo:show(file, book_props, metadata_updated_caller_callback)
     -- book_props may be provided if caller already has them available
     -- but it may lack "pages", that we may get from sidecar file
     if not book_props or not book_props.pages then
-        book_props = self:getBookProps(file, book_props)
+        book_props = BookInfo.getDocProps(file, book_props)
     end
-    local values_lang
-    local props = {
-        { _("Title:"), "title" },
-        { _("Authors:"), "authors" },
-        { _("Series:"), "series" },
-        { _("Pages:"), "pages" },
-        { _("Language:"), "language" },
-        { _("Keywords:"), "keywords" },
-        { _("Description:"), "description" },
-    }
-    for _i, v in ipairs(props) do
-        local prop_text, prop_key = unpack(v)
+    -- cover image
+    self.custom_book_cover = DocSettings:findCustomCoverFile(file)
+    local key_text = self.prop_text["cover"]
+    if self.custom_book_cover then
+        key_text = "\u{F040} " .. key_text
+    end
+    table.insert(kv_pairs, { key_text, _("Tap to display"),
+        callback = function()
+            self:onShowBookCover(file)
+        end,
+        hold_callback = function()
+            self:showCustomDialog(file, book_props)
+        end,
+        separator = true,
+    })
+    -- metadata
+    local custom_props
+    local custom_metadata_file = DocSettings:findCustomMetadataFile(file)
+    if custom_metadata_file then
+        self.custom_doc_settings = DocSettings.openSettingsFile(custom_metadata_file)
+        custom_props = self.custom_doc_settings:readSetting("custom_props")
+    end
+    local values_lang, callback
+    for _i, prop_key in ipairs(self.props) do
         local prop = book_props[prop_key]
         if prop == nil or prop == "" then
             prop = _("N/A")
@@ -84,16 +125,6 @@ function BookInfo:show(file, book_props, metadata_updated_caller_callback)
             else
                 prop = BD.auto(prop)
             end
-        elseif prop_key == "series" then
-            -- If we were fed a BookInfo book_props (e.g., covermenu), series index is in a separate field
-            if book_props.series_index then
-                -- Here, we're assured that series_index is a Lua number, so round integers are automatically
-                -- displayed without decimals
-                prop = prop .. " #" .. book_props.series_index
-            else
-                -- But here, if we have a plain doc_props series with an index, drop empty decimals from round integers.
-                prop = prop:gsub("(#%d+)%.0+$", "%1")
-            end
         elseif prop_key == "language" then
             -- Get a chance to have title, authors... rendered with alternate
             -- glyphs for the book language (e.g. japanese book in chinese UI)
@@ -101,31 +132,28 @@ function BookInfo:show(file, book_props, metadata_updated_caller_callback)
         elseif prop_key == "description" then
             -- Description may (often in EPUB, but not always) or may not (rarely in PDF) be HTML
             prop = util.htmlToPlainTextIfHtml(prop)
+            callback = function() -- proper text_type in TextViewer
+                self:showBookProp("description", prop)
+            end
         end
-        table.insert(kv_pairs, { prop_text, prop })
-    end
-    -- cover image
-    local is_doc = self.document and true or false
-    self.custom_book_cover = DocSettings:findCoverFile(file)
-    table.insert(kv_pairs, {
-        _("Cover image:"),
-        _("Tap to display"),
-        callback = function() self:onShowBookCover(file, true) end,
-        separator = is_doc and not self.custom_book_cover,
-    })
-    -- custom cover image
-    if self.custom_book_cover then
-        table.insert(kv_pairs, {
-            _("Custom cover image:"),
-            _("Tap to display"),
-            callback = function() self:onShowBookCover(file) end,
-            separator = is_doc,
+        key_text = self.prop_text[prop_key]
+        if custom_props and custom_props[prop_key] then -- customized
+            key_text = "\u{F040} " .. key_text
+        end
+        table.insert(kv_pairs, { key_text, prop,
+            callback = callback,
+            hold_callback = function()
+                self:showCustomDialog(file, book_props, prop_key)
+            end,
         })
     end
+    -- pages
+    local is_doc = self.document and true or false
+    table.insert(kv_pairs, { self.prop_text["pages"], book_props["pages"] or _("N/A"), separator = is_doc })
 
     -- Page section
     if is_doc then
-        local lines_nb, words_nb = self:getCurrentPageLineWordCounts()
+        local lines_nb, words_nb = self.ui.view:getCurrentPageLineWordCounts()
         if lines_nb == 0 then
             lines_nb = _("N/A")
             words_nb = _("N/A")
@@ -136,40 +164,48 @@ function BookInfo:show(file, book_props, metadata_updated_caller_callback)
 
     local KeyValuePage = require("ui/widget/keyvaluepage")
     self.kvp_widget = KeyValuePage:new{
-        title = _("Book information"),
+        title = self.title,
         value_overflow_align = "right",
         kv_pairs = kv_pairs,
         values_lang = values_lang,
         close_callback = function()
+            self.custom_doc_settings = nil
             self.custom_book_cover = nil
-            if self.updated then
-                local FileManager = require("apps/filemanager/filemanager")
-                local fm_ui = FileManager.instance
-                local ui = self.ui or fm_ui
-                if not ui then
-                    local ReaderUI = require("apps/reader/readerui")
-                    ui = ReaderUI.instance
-                end
-                if ui and ui.coverbrowser then
-                    ui.coverbrowser:deleteBookInfo(file)
-                end
-                if fm_ui then
-                    fm_ui:onRefresh()
-                end
-                if metadata_updated_caller_callback then
-                    metadata_updated_caller_callback()
-                end
+            if self.prop_updated then
+                UIManager:broadcastEvent(Event:new("InvalidateMetadataCache", file))
+                UIManager:broadcastEvent(Event:new("BookMetadataChanged", self.prop_updated))
             end
         end,
-        title_bar_left_icon = "appbar.menu",
-        title_bar_left_icon_tap_callback = function()
-            self:showCustomMenu(file, book_props, metadata_updated_caller_callback)
-        end,
     }
     UIManager:show(self.kvp_widget)
 end
 
-function BookInfo:getBookProps(file, book_props, no_open_document)
+function BookInfo.getCustomProp(prop_key, filepath)
+    local custom_metadata_file = DocSettings:findCustomMetadataFile(filepath)
+    return custom_metadata_file
+        and DocSettings.openSettingsFile(custom_metadata_file):readSetting("custom_props")[prop_key]
+end
+
+-- Returns extended and customized metadata.
+function BookInfo.extendProps(original_props, filepath)
+    -- do not customize if filepath is not passed (eg from covermenu)
+    local custom_metadata_file = filepath and DocSettings:findCustomMetadataFile(filepath)
+    local custom_props = custom_metadata_file
+        and DocSettings.openSettingsFile(custom_metadata_file):readSetting("custom_props") or {}
+    original_props = original_props or {}
+
+    local props = {}
+    for _, prop_key in ipairs(BookInfo.props) do
+        props[prop_key] = custom_props[prop_key] or original_props[prop_key]
+    end
+    props.pages = original_props.pages
+    -- if original title is empty, generate it as filename without extension
+    props.display_title = props.title or filemanagerutil.splitFileNameType(filepath)
+    return props
+end
+
+-- Returns customized document metadata, including number of pages.
+function BookInfo.getDocProps(file, book_props, no_open_document)
     if DocSettings:hasSidecarFile(file) then
         local doc_settings = DocSettings:open(file)
         if not book_props then
@@ -184,9 +220,8 @@ function BookInfo:getBookProps(file, book_props, no_open_document)
             -- the fact that stats.page = 0)
             local stats = doc_settings:readSetting("stats")
             if stats and stats.pages ~= 0 then
-                -- Let's use them as is (which was what was done before), even if
-                -- incomplete, to avoid expensive book opening
-                book_props = stats
+                -- title, authors, series, series_index, language
+                book_props = Document:getProps(stats)
             end
         end
         -- Files opened after 20170701 have an accurate "doc_pages" setting.
@@ -196,7 +231,16 @@ function BookInfo:getBookProps(file, book_props, no_open_document)
         end
     end
 
-    -- If still no book_props (book never opened or empty "stats"), open the document to get them
+    -- If still no book_props (book never opened or empty "stats"),
+    -- but custom metadata exists, it has a copy of original doc_props
+    if not book_props then
+        local custom_metadata_file = DocSettings:findCustomMetadataFile(file)
+        if custom_metadata_file then
+            book_props = DocSettings.openSettingsFile(custom_metadata_file):readSetting("doc_props")
+        end
+    end
+
+    -- If still no book_props, open the document to get them
     if not book_props and not no_open_document then
         local document = DocumentRegistry:openDocument(file)
         if document then
@@ -225,41 +269,54 @@ function BookInfo:getBookProps(file, book_props, no_open_document)
         end
     end
 
-    -- If still no book_props, fall back to empty ones
-    return book_props or {}
+    return BookInfo.extendProps(book_props, file)
+end
+
+function BookInfo:findInProps(book_props, search_string, case_sensitive)
+    for _, key in ipairs(self.props) do
+        local prop = book_props[key]
+        if prop then
+            if key == "series_index" then
+                prop = tostring(prop)
+            elseif key == "description" then
+                prop = util.htmlToPlainTextIfHtml(prop)
+            end
+            if not case_sensitive then
+                prop = Utf8Proc.lowercase(util.fixUtf8(prop, "?"))
+            end
+            if prop:find(search_string) then
+                return true
+            end
+        end
+    end
 end
 
+-- Shows book information for currently opened document.
 function BookInfo:onShowBookInfo()
-    if not self.document then return end
-    -- Get them directly from ReaderUI's doc_settings
-    local doc_props = self.ui.doc_settings:readSetting("doc_props")
-    -- Make a copy, so we don't add "pages" to the original doc_props
-    -- that will be saved at some point by ReaderUI.
-    local book_props = { pages = self.ui.doc_settings:readSetting("doc_pages") }
-    for k, v in pairs(doc_props) do
-        book_props[k] = v
+    if self.document then
+        self.ui.doc_props.pages = self.ui.doc_settings:readSetting("doc_pages")
+        self:show(self.document.file, self.ui.doc_props)
     end
-    self:show(self.document.file, book_props)
+end
+
+function BookInfo:showBookProp(prop_key, prop_text)
+    UIManager:show(TextViewer:new{
+        title = self.prop_text[prop_key],
+        text = prop_text,
+        text_type = prop_key == "description" and "book_info" or nil,
+    })
 end
 
 function BookInfo:onShowBookDescription(description, file)
     if not description then
         if file then
-            description = self:getBookProps(file).description
-        elseif self.document then
-            description = self.ui.doc_settings:readSetting("doc_props").description
-                       or self.document:getProps().description
+            description = BookInfo.getDocProps(file).description
+        elseif self.document then -- currently opened document
+            description = self.ui.doc_props.description
         end
     end
-    if description and description ~= "" then
-        -- Description may (often in EPUB, but not always) or may not (rarely
-        -- in PDF) be HTML.
-        description = util.htmlToPlainTextIfHtml(description)
-        local TextViewer = require("ui/widget/textviewer")
-        UIManager:show(TextViewer:new{
-            title = _("Description:"),
-            text = description,
-        })
+    if description then
+        self:showBookProp("description", util.htmlToPlainTextIfHtml(description))
     else
         UIManager:show(InfoMessage:new{
             text = _("No book description available."),
@@ -288,7 +345,7 @@ function BookInfo:getCoverImage(doc, file, force_orig)
     local cover_bb
     -- check for a custom cover (orig cover is forcibly requested in "Book information" only)
     if not force_orig then
-        local custom_cover = DocSettings:findCoverFile(file or (doc and doc.file))
+        local custom_cover = DocSettings:findCustomCoverFile(file or (doc and doc.file))
         if custom_cover then
             local cover_doc = DocumentRegistry:openDocument(custom_cover)
             if cover_doc then
@@ -315,28 +372,26 @@ function BookInfo:getCoverImage(doc, file, force_orig)
     return cover_bb
 end
 
-function BookInfo:setCustomBookCover(file, book_props, metadata_updated_caller_callback)
-    local function kvp_update()
-        if self.ui then
-            self.ui.doc_settings:getCoverFile(true) -- reset cover file cache
-        end
-        self.updated = true
-        self.kvp_widget:onClose()
-        self:show(file, book_props, metadata_updated_caller_callback)
+function BookInfo:updateBookInfo(file, book_props, prop_updated, prop_value_old)
+    if self.document and prop_updated == "cover" then
+        self.ui.doc_settings:getCustomCoverFile(true) -- reset cover file cache
     end
+    self.prop_updated = {
+        filepath = file,
+        doc_props = book_props,
+        metadata_key_updated = prop_updated,
+        metadata_value_old = prop_value_old,
+    }
+    self.kvp_widget:onClose()
+    self:show(file, book_props)
+end
+
+function BookInfo:setCustomCover(file, book_props)
     if self.custom_book_cover then -- reset custom cover
-        local ConfirmBox = require("ui/widget/confirmbox")
-        local confirm_box = ConfirmBox:new{
-            text = _("Reset custom cover?\nImage file will be deleted."),
-            ok_text = _("Reset"),
-            ok_callback = function()
-                if os.remove(self.custom_book_cover) then
-                    DocSettings:removeSidecarDir(file, util.splitFilePathName(self.custom_book_cover))
-                    kvp_update()
-                end
-            end,
-        }
-        UIManager:show(confirm_box)
+        if os.remove(self.custom_book_cover) then
+            DocSettings.removeSidecarDir(util.splitFilePathName(self.custom_book_cover))
+            self:updateBookInfo(file, book_props, "cover")
+        end
     else -- choose an image and set custom cover
         local PathChooser = require("ui/widget/pathchooser")
         local path_chooser = PathChooser:new{
@@ -345,22 +400,8 @@ function BookInfo:setCustomBookCover(file, book_props, metadata_updated_caller_c
                 return DocumentRegistry:isImageFile(filename)
             end,
             onConfirm = function(image_file)
-                local sidecar_dir
-                local sidecar_file = DocSettings:findCoverFile(file) -- existing cover file
-                if sidecar_file then
-                    os.remove(sidecar_file)
-                else -- no existing cover, get metadata file path
-                    sidecar_file = DocSettings:hasSidecarFile(file, true) -- new sdr locations only
-                end
-                if sidecar_file then
-                    sidecar_dir = util.splitFilePathName(sidecar_file)
-                else -- no sdr folder, create new
-                    sidecar_dir = DocSettings:getSidecarDir(file) .. "/"
-                    util.makePath(sidecar_dir)
-                end
-                local new_cover_file = sidecar_dir .. "cover." .. util.getFileNameSuffix(image_file):lower()
-                if ffiutil.copyFile(image_file, new_cover_file) == nil then
-                    kvp_update()
+                if DocSettings:flushCustomCover(file, image_file) then
+                    self:updateBookInfo(file, book_props, "cover")
                 end
             end,
         }
@@ -368,63 +409,275 @@ function BookInfo:setCustomBookCover(file, book_props, metadata_updated_caller_c
     end
 end
 
-function BookInfo:getCurrentPageLineWordCounts()
-    local lines_nb, words_nb = 0, 0
-    if self.ui.rolling then
-        local res = self.ui.document:getTextFromPositions({x = 0, y = 0},
-            {x = Screen:getWidth(), y = Screen:getHeight()}, true) -- do not highlight
-        if res then
-            lines_nb = #self.ui.document:getScreenBoxesFromPositions(res.pos0, res.pos1, true)
-            for word in util.gsplit(res.text, "[%s%p]+", false) do
-                if util.hasCJKChar(word) then
-                    for char in util.gsplit(word, "[\192-\255][\128-\191]+", true) do
-                        words_nb = words_nb + 1
-                    end
-                else
-                    words_nb = words_nb + 1
-                end
-            end
-        end
+function BookInfo:setCustomCoverFromImage(file, image_file)
+    local custom_book_cover = DocSettings:findCustomCoverFile(file)
+    if custom_book_cover then
+        os.remove(custom_book_cover)
+    end
+    DocSettings:flushCustomCover(file, image_file)
+    if self.ui.doc_settings then
+        self.ui.doc_settings:getCustomCoverFile(true) -- reset cover file cache
+    end
+    UIManager:broadcastEvent(Event:new("InvalidateMetadataCache", file))
+    UIManager:broadcastEvent(Event:new("BookMetadataChanged"))
+end
+
+function BookInfo:setCustomMetadata(file, book_props, prop_key, prop_value)
+    -- in file
+    local custom_doc_settings, custom_props, display_title, no_custom_metadata
+    if self.custom_doc_settings then
+        custom_doc_settings = self.custom_doc_settings
+    else -- no custom metadata file, create new
+        custom_doc_settings = DocSettings.openSettingsFile()
+        display_title = book_props.display_title -- backup
+        book_props.display_title = nil
+        custom_doc_settings:saveSetting("doc_props", book_props) -- save a copy of original props
+    end
+    custom_props = custom_doc_settings:readSetting("custom_props", {})
+    local prop_value_old = custom_props[prop_key] or book_props[prop_key]
+    custom_props[prop_key] = prop_value -- nil when resetting a custom prop
+    if next(custom_props) == nil then -- no more custom metadata
+        os.remove(custom_doc_settings.sidecar_file)
+        DocSettings.removeSidecarDir(util.splitFilePathName(custom_doc_settings.sidecar_file))
+        no_custom_metadata = true
     else
-        local page_boxes = self.ui.document:getTextBoxes(self.ui:getCurrentPage())
-        if page_boxes and page_boxes[1][1].word then
-            lines_nb = #page_boxes
-            for _, line in ipairs(page_boxes) do
-                if #line == 1 and line[1].word == "" then -- empty line
-                    lines_nb = lines_nb - 1
-                else
-                    words_nb = words_nb + #line
-                    local last_word = line[#line].word
-                    if last_word:sub(-1) == "-" and last_word ~= "-" then -- hyphenated
-                        words_nb = words_nb - 1
-                    end
-                end
-            end
+        if book_props.pages then -- keep a copy of original 'pages' up to date
+            local original_props = custom_doc_settings:readSetting("doc_props")
+            original_props.pages = book_props.pages
         end
+        custom_doc_settings:flushCustomMetadata(file)
     end
-    return lines_nb, words_nb
+    book_props.display_title = book_props.display_title or display_title -- restore
+    -- in memory
+    prop_value = prop_value or custom_doc_settings:readSetting("doc_props")[prop_key] -- set custom or restore original
+    book_props[prop_key] = prop_value
+    if prop_key == "title" then -- generate when resetting the customized title and original is empty
+        book_props.display_title = book_props.title or filemanagerutil.splitFileNameType(file)
+    end
+    if self.document and self.document.file == file then -- currently opened document
+        self.ui.doc_props[prop_key] = prop_value
+        if prop_key == "title" then
+            self.ui.doc_props.display_title = book_props.display_title
+        end
+        if no_custom_metadata then
+            self.ui.doc_settings:getCustomMetadataFile(true) -- reset metadata file cache
+        end
+    end
+    self:updateBookInfo(file, book_props, prop_key, prop_value_old)
 end
 
-function BookInfo:showCustomMenu(file, book_props, metadata_updated_caller_callback)
+function BookInfo:showCustomEditDialog(file, book_props, prop_key)
+    local prop = book_props[prop_key]
+    if prop and prop_key == "description" then
+        prop = util.htmlToPlainTextIfHtml(prop)
+    end
+    local input_dialog
+    input_dialog = InputDialog:new{
+        title = _("Edit book metadata:") .. " " .. self.prop_text[prop_key]:gsub(":", ""),
+        input = prop,
+        input_type = prop_key == "series_index" and "number",
+        allow_newline = prop_key == "authors" or prop_key == "keywords" or prop_key == "description",
+        buttons = {
+            {
+                {
+                    text = _("Cancel"),
+                    id = "close",
+                    callback = function()
+                        UIManager:close(input_dialog)
+                    end,
+                },
+                {
+                    text = _("Save"),
+                    callback = function()
+                        local prop_value = input_dialog:getInputValue()
+                        if prop_value and prop_value ~= "" then
+                            UIManager:close(input_dialog)
+                            self:setCustomMetadata(file, book_props, prop_key, prop_value)
+                        end
+                    end,
+                },
+            },
+        },
+    }
+    UIManager:show(input_dialog)
+    input_dialog:onShowKeyboard()
+end
+
+function BookInfo:showCustomDialog(file, book_props, prop_key)
+    local original_prop, custom_prop, prop_is_cover
+    if prop_key then -- metadata
+        if self.custom_doc_settings then
+            original_prop = self.custom_doc_settings:readSetting("doc_props")[prop_key]
+            custom_prop = self.custom_doc_settings:readSetting("custom_props")[prop_key]
+        else
+            original_prop = book_props[prop_key]
+        end
+        if original_prop and prop_key == "description" then
+            original_prop = util.htmlToPlainTextIfHtml(original_prop)
+        end
+        prop_is_cover = false
+    else -- cover
+        prop_key = "cover"
+        prop_is_cover = true
+    end
+
     local button_dialog
-    local buttons = {{
+    local buttons = {
         {
-            text = self.custom_book_cover and _("Reset cover image") or _("Set cover image"),
-            align = "left",
-            callback = function()
-                UIManager:close(button_dialog)
-                self:setCustomBookCover(file, book_props, metadata_updated_caller_callback)
-            end,
+            {
+                text = _("Copy original"),
+                enabled = original_prop ~= nil and Device:hasClipboard(),
+                callback = function()
+                    UIManager:close(button_dialog)
+                    Device.input.setClipboardText(original_prop)
+                end,
+            },
+            {
+                text = _("View original"),
+                enabled = original_prop ~= nil or prop_is_cover,
+                callback = function()
+                    if prop_is_cover then
+                        self:onShowBookCover(file, true)
+                    else
+                        self:showBookProp(prop_key, original_prop)
+                    end
+                end,
+            },
+        },
+        {
+            {
+                text = _("Reset custom"),
+                enabled = custom_prop ~= nil or (prop_is_cover and self.custom_book_cover ~= nil),
+                callback = function()
+                    local confirm_box = ConfirmBox:new{
+                        text = prop_is_cover and _("Reset custom cover?\nImage file will be deleted.")
+                                              or _("Reset custom book metadata field?"),
+                        ok_text = _("Reset"),
+                        ok_callback = function()
+                            UIManager:close(button_dialog)
+                            if prop_is_cover then
+                                self:setCustomCover(file, book_props)
+                            else
+                                self:setCustomMetadata(file, book_props, prop_key)
+                            end
+                        end,
+                    }
+                    UIManager:show(confirm_box)
+                end,
+            },
+            {
+                text = _("Set custom"),
+                enabled = not prop_is_cover or (prop_is_cover and self.custom_book_cover == nil),
+                callback = function()
+                    UIManager:close(button_dialog)
+                    if prop_is_cover then
+                        self:setCustomCover(file, book_props)
+                    else
+                        self:showCustomEditDialog(file, book_props, prop_key)
+                    end
+                end,
+            },
         },
-    }}
+    }
     button_dialog = ButtonDialog:new{
-        shrink_unneeded_width = true,
+        title = _("Book metadata:") .. " " .. self.prop_text[prop_key]:gsub(":", ""),
+        title_align = "center",
         buttons = buttons,
-        anchor = function()
-            return self.kvp_widget.title_bar.left_button.image.dimen
-        end,
     }
     UIManager:show(button_dialog)
 end
 
+function BookInfo:moveBookMetadata()
+    -- called by filemanagermenu only
+    local file_chooser = self.ui.file_chooser
+    local function scanPath()
+        local sys_folders = { -- do not scan sys_folders
+            ["/dev"]  = true,
+            ["/proc"] = true,
+            ["/sys"]  = true,
+        }
+        local books_to_move = {}
+        local dirs = { file_chooser.path }
+        while #dirs ~= 0 do
+            local new_dirs = {}
+            for _, d in ipairs(dirs) do
+                local ok, iter, dir_obj = pcall(lfs.dir, d)
+                if ok then
+                    for f in iter, dir_obj do
+                        local fullpath = "/" .. f
+                        if d ~= "/" then
+                            fullpath = d .. fullpath
+                        end
+                        local attributes = lfs.attributes(fullpath) or {}
+                        if attributes.mode == "directory" and f ~= "." and f ~= ".."
+                                and file_chooser:show_dir(f) and not sys_folders[fullpath] then
+                            table.insert(new_dirs, fullpath)
+                        elseif attributes.mode == "file" and not util.stringStartsWith(f, "._")
+                                and file_chooser:show_file(f)
+                                and DocSettings.isSidecarFileNotInPreferredLocation(fullpath) then
+                            table.insert(books_to_move, fullpath)
+                        end
+                    end
+                end
+            end
+            dirs = new_dirs
+        end
+        return books_to_move
+    end
+    UIManager:show(ConfirmBox:new{
+        text = _("Scan books in current folder and subfolders for their metadata location?"),
+        ok_text = _("Scan"),
+        ok_callback = function()
+            local books_to_move = scanPath()
+            local books_to_move_nb = #books_to_move
+            if books_to_move_nb == 0 then
+                UIManager:show(InfoMessage:new{
+                    text = _("No books with metadata not in your preferred location found."),
+                })
+            else
+                UIManager:show(ConfirmBox:new{
+                    text = T(N_("1 book with metadata not in your preferred location found.",
+                              "%1 books with metadata not in your preferred location found.",
+                              books_to_move_nb), books_to_move_nb) .. "\n" ..
+                              _("Move book metadata to your preferred location?"),
+                    ok_text = _("Move"),
+                    ok_callback = function()
+                        UIManager:close(self.menu_container)
+                        for _, book in ipairs(books_to_move) do
+                            DocSettings.updateLocation(book, book)
+                        end
+                        file_chooser:refreshPath()
+                    end,
+                })
+            end
+        end,
+    })
+end
+
+function BookInfo.showBooksWithHashBasedMetadata()
+    local header = T(_("Hash-based metadata has been saved in %1 for the following documents. Hash-based storage may slow down file browser navigation in large directories. Thus, if not using hash-based metadata storage, it is recommended to open the associated documents in KOReader to automatically migrate their metadata to the preferred storage location, or to delete %1, which will speed up file browser navigation."),
+        DocSettings.getSidecarStorage("hash"))
+    local file_info = { header .. "\n" }
+    local sdrs = DocSettings.findSidecarFilesInHashLocation()
+    for i, sdr in ipairs(sdrs) do
+        local sidecar_file, custom_metadata_file = unpack(sdr)
+        local doc_settings = DocSettings.openSettingsFile(sidecar_file)
+        local doc_props = doc_settings:readSetting("doc_props")
+        local custom_props = custom_metadata_file
+            and DocSettings.openSettingsFile(custom_metadata_file):readSetting("custom_props") or {}
+        local doc_path = doc_settings:readSetting("doc_path")
+        local title = custom_props.title or doc_props.title or filemanagerutil.splitFileNameType(doc_path)
+        local author = custom_props.authors or doc_props.authors or _("N/A")
+        doc_path = lfs.attributes(doc_path, "mode") == "file" and doc_path or _("N/A")
+        local text = T(_("%1. Title: %2; Author: %3\nDocument: %4"), i, title, author, doc_path)
+        table.insert(file_info, text)
+    end
+    local doc_nb = #file_info - 1
+    UIManager:show(TextViewer:new{
+        title = T(N_("1 document with hash-based metadata", "%1 documents with hash-based metadata", doc_nb), doc_nb),
+        title_multilines = true,
+        text = table.concat(file_info, "\n"),
+    })
+end
+
 return BookInfo

frontend/apps/filemanager/filemanagercollection.lua

@@ -1,16 +1,26 @@
+local BD = require("ui/bidi")
 local ButtonDialog = require("ui/widget/buttondialog")
+local ConfirmBox = require("ui/widget/confirmbox")
 local Device = require("device")
-local DocumentRegistry = require("document/documentregistry")
+local DocSettings = require("docsettings")
+local FileManagerBookInfo = require("apps/filemanager/filemanagerbookinfo")
+local InfoMessage = require("ui/widget/infomessage")
+local InputDialog = require("ui/widget/inputdialog")
 local Menu = require("ui/widget/menu")
 local ReadCollection = require("readcollection")
+local SortWidget = require("ui/widget/sortwidget")
 local UIManager = require("ui/uimanager")
 local WidgetContainer = require("ui/widget/container/widgetcontainer")
 local Screen = require("device").screen
 local filemanagerutil = require("apps/filemanager/filemanagerutil")
 local _ = require("gettext")
+local T = require("ffi/util").template
+local util = require("util")
 
 local FileManagerCollection = WidgetContainer:extend{
-    coll_menu_title = _("Favorites"),
+    title = _("Collections"),
+    default_collection_title = _("Favorites"),
+    checkmark = "\u{2713}",
 }
 
 function FileManagerCollection:init()
@@ -18,30 +28,90 @@ function FileManagerCollection:init()
 end
 
 function FileManagerCollection:addToMainMenu(menu_items)
+    menu_items.favorites = {
+        text = self.default_collection_title,
+        callback = function()
+            self:onShowColl()
+        end,
+    }
     menu_items.collections = {
-        text = self.coll_menu_title,
+        text = self.title,
         callback = function()
-            self:onShowColl("favorites")
+            self:onShowCollList()
         end,
     }
 end
 
-function FileManagerCollection:updateItemTable()
-    -- Try to stay on current page.
-    local select_number = nil
-    if self.coll_menu.page and self.coll_menu.perpage then
-        select_number = (self.coll_menu.page - 1) * self.coll_menu.perpage + 1
+-- collection
+
+function FileManagerCollection:getCollectionTitle(collection_name)
+    return collection_name == ReadCollection.default_collection_name
+        and self.default_collection_title -- favorites
+         or collection_name
+end
+
+function FileManagerCollection:onShowColl(collection_name)
+    collection_name = collection_name or ReadCollection.default_collection_name
+    self.coll_menu = Menu:new{
+        ui = self.ui,
+        covers_fullscreen = true, -- hint for UIManager:_repaint()
+        is_borderless = true,
+        is_popout = false,
+        -- item and book cover thumbnail dimensions in Mosaic and Detailed list display modes
+        -- must be equal in File manager, History and Collection windows to avoid image scaling
+        title_bar_fm_style = true,
+        title_bar_left_icon = "appbar.menu",
+        onLeftButtonTap = function() self:showCollDialog() end,
+        onMenuChoice = self.onMenuChoice,
+        onMenuHold = self.onMenuHold,
+        onSetRotationMode = self.MenuSetRotationModeHandler,
+        _manager = self,
+        collection_name = collection_name,
+    }
+    self.coll_menu.close_callback = function()
+        if self.files_updated then
+            if self.ui.file_chooser then
+                self.ui.file_chooser:refreshPath()
+            end
+            self.files_updated = nil
+        end
+        UIManager:close(self.coll_menu)
+        self.coll_menu = nil
+    end
+    self:updateItemTable()
+    UIManager:show(self.coll_menu)
+    return true
+end
+
+function FileManagerCollection:updateItemTable(show_last_item)
+    local item_table = {}
+    for _, item in pairs(ReadCollection.coll[self.coll_menu.collection_name]) do
+        table.insert(item_table, item)
+    end
+    if #item_table > 1 then
+        table.sort(item_table, function(v1, v2) return v1.order < v2.order end)
     end
-    self.coll_menu:switchItemTable(self.coll_menu_title,
-        ReadCollection:prepareList(self.coll_menu.collection), select_number)
+    local title = self:getCollectionTitle(self.coll_menu.collection_name)
+    title = T("%1 (%2)", title, #item_table)
+    local item_number = show_last_item and #item_table or -1
+    self.coll_menu:switchItemTable(title, item_table, item_number)
 end
 
 function FileManagerCollection:onMenuChoice(item)
-    require("apps/reader/readerui"):showReader(item.file)
+    if self.ui.document then
+        if self.ui.document.file ~= item.file then
+            self.ui:switchDocument(item.file)
+        end
+    else
+        self.ui:openFile(item.file)
+    end
 end
 
 function FileManagerCollection:onMenuHold(item)
+    local file = item.file
     self.collfile_dialog = nil
+    self.book_props = self.ui.coverbrowser and self.ui.coverbrowser:getBookInfo(file)
+
     local function close_dialog_callback()
         UIManager:close(self.collfile_dialog)
     end
@@ -49,45 +119,63 @@ function FileManagerCollection:onMenuHold(item)
         UIManager:close(self.collfile_dialog)
         self._manager.coll_menu.close_callback()
     end
-    local function status_button_callback()
+    local function close_dialog_update_callback()
         UIManager:close(self.collfile_dialog)
         self._manager:updateItemTable()
+        self._manager.files_updated = true
     end
-    local is_currently_opened = item.file == (self.ui.document and self.ui.document.file)
+    local is_currently_opened = file == (self.ui.document and self.ui.document.file)
 
     local buttons = {}
-    if not (item.dim or is_currently_opened) then
-        table.insert(buttons, filemanagerutil.genStatusButtonsRow(item.file, status_button_callback))
-        table.insert(buttons, {}) -- separator
+    local doc_settings_or_file
+    if is_currently_opened then
+        doc_settings_or_file = self.ui.doc_settings
+        if not self.book_props then
+            self.book_props = self.ui.doc_props
+            self.book_props.has_cover = true
+        end
+    else
+        if DocSettings:hasSidecarFile(file) then
+            doc_settings_or_file = DocSettings:open(file)
+            if not self.book_props then
+                local props = doc_settings_or_file:readSetting("doc_props")
+                self.book_props = FileManagerBookInfo.extendProps(props, file)
+                self.book_props.has_cover = true
+            end
+        else
+            doc_settings_or_file = file
+        end
     end
+    table.insert(buttons, filemanagerutil.genStatusButtonsRow(doc_settings_or_file, close_dialog_update_callback))
+    table.insert(buttons, {}) -- separator
     table.insert(buttons, {
-        filemanagerutil.genResetSettingsButton(item.file, status_button_callback, is_currently_opened),
+        filemanagerutil.genResetSettingsButton(doc_settings_or_file, close_dialog_update_callback, is_currently_opened),
         {
-            text = _("Remove from favorites"),
+            text = _("Remove from collection"),
             callback = function()
                 UIManager:close(self.collfile_dialog)
-                ReadCollection:removeItem(item.file, self._manager.coll_menu.collection)
+                ReadCollection:removeItem(file, self.collection_name)
                 self._manager:updateItemTable()
             end,
         },
     })
     table.insert(buttons, {
-        filemanagerutil.genShowFolderButton(item.file, close_dialog_menu_callback, item.dim),
-        filemanagerutil.genBookInformationButton(item.file, close_dialog_callback, item.dim),
+        filemanagerutil.genShowFolderButton(file, close_dialog_menu_callback),
+        filemanagerutil.genBookInformationButton(file, self.book_props, close_dialog_callback),
     })
     table.insert(buttons, {
-        filemanagerutil.genBookCoverButton(item.file, close_dialog_callback, item.dim),
-        filemanagerutil.genBookDescriptionButton(item.file, close_dialog_callback, item.dim),
+        filemanagerutil.genBookCoverButton(file, self.book_props, close_dialog_callback),
+        filemanagerutil.genBookDescriptionButton(file, self.book_props, close_dialog_callback),
     })
 
-    if Device:canExecuteScript(item.file) then
+    if Device:canExecuteScript(file) then
         table.insert(buttons, {
-            filemanagerutil.genExecuteScriptButton(item.file, close_dialog_menu_callback)
+            filemanagerutil.genExecuteScriptButton(file, close_dialog_menu_callback)
         })
     end
 
     self.collfile_dialog = ButtonDialog:new{
-        title = item.text:match("([^/]+)$"),
+        title = BD.filename(item.text),
         title_align = "center",
         buttons = buttons,
     }
@@ -95,91 +183,62 @@ function FileManagerCollection:onMenuHold(item)
     return true
 end
 
-function FileManagerCollection:MenuSetRotationModeHandler(rotation)
-    if rotation ~= nil and rotation ~= Screen:getRotationMode() then
-        UIManager:close(self._manager.coll_menu)
-        if self._manager.ui.view and self._manager.ui.view.onSetRotationMode then
-            self._manager.ui.view:onSetRotationMode(rotation)
-        elseif self._manager.ui.onSetRotationMode then
-            self._manager.ui:onSetRotationMode(rotation)
-        else
-            Screen:setRotationMode(rotation)
-        end
-        self._manager:onShowColl()
-    end
-    return true
-end
-
-function FileManagerCollection:onShowColl(collection)
-    self.coll_menu = Menu:new{
-        ui = self.ui,
-        covers_fullscreen = true, -- hint for UIManager:_repaint()
-        is_borderless = true,
-        is_popout = false,
-        title_bar_left_icon = "appbar.menu",
-        onLeftButtonTap = function() self:showCollDialog() end,
-        onMenuChoice = self.onMenuChoice,
-        onMenuHold = self.onMenuHold,
-        onSetRotationMode = self.MenuSetRotationModeHandler,
-        _manager = self,
-        collection = collection,
-    }
-    self:updateItemTable()
-    self.coll_menu.close_callback = function()
-        UIManager:close(self.coll_menu)
-    end
-    UIManager:show(self.coll_menu)
-    return true
-end
-
 function FileManagerCollection:showCollDialog()
     local coll_dialog
-    local is_added = self.ui.document and ReadCollection:checkItemExist(self.ui.document.file)
     local buttons = {
         {{
-            text_func = function()
-                return is_added and _("Remove current book from favorites") or _("Add current book to favorites")
+            text = _("Collections"),
+            callback = function()
+                UIManager:close(coll_dialog)
+                self.coll_menu.close_callback()
+                self:onShowCollList()
             end,
-            enabled = self.ui.document and true or false,
+        }},
+        {}, -- separator
+        {{
+            text = _("Arrange books in collection"),
             callback = function()
                 UIManager:close(coll_dialog)
-                if is_added then
-                    ReadCollection:removeItem(self.ui.document.file)
-                else
-                    ReadCollection:addItem(self.ui.document.file)
-                end
-                self:updateItemTable()
+                self:sortCollection()
             end,
         }},
         {{
-            text = _("Add a book to favorites"),
+            text = _("Add a book to collection"),
             callback = function()
                 UIManager:close(coll_dialog)
                 local PathChooser = require("ui/widget/pathchooser")
                 local path_chooser = PathChooser:new{
                     path = G_reader_settings:readSetting("home_dir"),
                     select_directory = false,
-                    file_filter = function(file)
-                        return DocumentRegistry:getProviders(file) ~= nil
-                    end,
                     onConfirm = function(file)
-                        if not ReadCollection:checkItemExist(file) then
-                            ReadCollection:addItem(file)
-                            self:updateItemTable()
+                        if not ReadCollection:isFileInCollection(file, self.coll_menu.collection_name) then
+                            ReadCollection:addItem(file, self.coll_menu.collection_name)
+                            self:updateItemTable(true) -- show added item
                         end
                     end,
                 }
                 UIManager:show(path_chooser)
             end,
         }},
-        {{
-            text = _("Sort favorites"),
+    }
+    if self.ui.document then
+        local file = self.ui.document.file
+        local is_in_collection = ReadCollection:isFileInCollection(file, self.coll_menu.collection_name)
+        table.insert(buttons, {{
+            text_func = function()
+                return is_in_collection and _("Remove current book from collection") or _("Add current book to collection")
+            end,
             callback = function()
                 UIManager:close(coll_dialog)
-                self:sortCollection()
+                if is_in_collection then
+                    ReadCollection:removeItem(file, self.coll_menu.collection_name)
+                else
+                    ReadCollection:addItem(file, self.coll_menu.collection_name)
+                end
+                self:updateItemTable(not is_in_collection)
             end,
-        }},
-    }
+        }})
+    end
     coll_dialog = ButtonDialog:new{
         buttons = buttons,
     }
@@ -187,25 +246,353 @@ function FileManagerCollection:showCollDialog()
 end
 
 function FileManagerCollection:sortCollection()
-    local item_table = {}
-    for _, v in ipairs(self.coll_menu.item_table) do
-        table.insert(item_table, { text = v.text, label = v.file })
-    end
-    local SortWidget = require("ui/widget/sortwidget")
     local sort_widget
     sort_widget = SortWidget:new{
-        title = _("Sort favorites"),
-        item_table = item_table,
+        title = _("Arrange books in collection"),
+        item_table = ReadCollection:getOrderedCollection(self.coll_menu.collection_name),
         callback = function()
-            local new_order_table = {}
-            for i, v in ipairs(sort_widget.item_table) do
-                table.insert(new_order_table, { file = v.label, order = i })
-            end
-            ReadCollection:writeCollection(new_order_table, self.coll_menu.collection)
+            ReadCollection:updateCollectionOrder(self.coll_menu.collection_name, sort_widget.item_table)
             self:updateItemTable()
         end
     }
     UIManager:show(sort_widget)
 end
 
+function FileManagerCollection:MenuSetRotationModeHandler(rotation)
+    if rotation ~= nil and rotation ~= Screen:getRotationMode() then
+        UIManager:close(self._manager.coll_menu)
+        if self._manager.ui.view and self._manager.ui.view.onSetRotationMode then
+            self._manager.ui.view:onSetRotationMode(rotation)
+        elseif self._manager.ui.onSetRotationMode then
+            self._manager.ui:onSetRotationMode(rotation)
+        else
+            Screen:setRotationMode(rotation)
+        end
+        self._manager:onShowColl()
+    end
+    return true
+end
+
+function FileManagerCollection:onBookMetadataChanged()
+    if self.coll_menu then
+        self.coll_menu:updateItems()
+    end
+end
+
+-- collection list
+
+function FileManagerCollection:onShowCollList(file_or_files, caller_callback, no_dialog)
+    self.selected_colections = nil
+    if file_or_files then -- select mode
+        if type(file_or_files) == "string" then -- checkmark collections containing the file
+            self.selected_colections = ReadCollection:getCollectionsWithFile(file_or_files)
+        else -- do not checkmark any
+            self.selected_colections = {}
+        end
+    end
+    self.coll_list = Menu:new{
+        subtitle = "",
+        covers_fullscreen = true,
+        is_borderless = true,
+        is_popout = false,
+        title_bar_fm_style = true,
+        title_bar_left_icon = file_or_files and "check" or "appbar.menu",
+        onLeftButtonTap = function() self:showCollListDialog(caller_callback, no_dialog) end,
+        onMenuChoice = self.onCollListChoice,
+        onMenuHold = self.onCollListHold,
+        onSetRotationMode = self.MenuSetRotationModeHandler,
+        _manager = self,
+    }
+    self.coll_list.close_callback = function(force_close)
+        if force_close or self.selected_colections == nil then
+            UIManager:close(self.coll_list)
+            self.coll_list = nil
+        end
+    end
+    self:updateCollListItemTable(true) -- init
+    UIManager:show(self.coll_list)
+    return true
+end
+
+function FileManagerCollection:updateCollListItemTable(do_init, item_number)
+    local item_table
+    if do_init then
+        item_table = {}
+        for name, coll in pairs(ReadCollection.coll) do
+            local mandatory
+            if self.selected_colections then
+                mandatory = self.selected_colections[name] and self.checkmark or "  "
+                self.coll_list.items_mandatory_font_size = self.coll_list.font_size
+            else
+                mandatory = util.tableSize(coll)
+            end
+            table.insert(item_table, {
+                text      = self:getCollectionTitle(name),
+                mandatory = mandatory,
+                name      = name,
+                order     = ReadCollection.coll_order[name],
+            })
+        end
+        if #item_table > 1 then
+            table.sort(item_table, function(v1, v2) return v1.order < v2.order end)
+        end
+    else
+        item_table = self.coll_list.item_table
+    end
+    local title = T(_("Collections (%1)"), #item_table)
+    local subtitle
+    if self.selected_colections then
+        local selected_nb = util.tableSize(self.selected_colections)
+        subtitle = self.selected_colections and T(_("Selected collections: %1"), selected_nb)
+        if do_init and selected_nb > 0 then -- show first collection containing the long-pressed book
+            for i, item in ipairs(item_table) do
+                if self.selected_colections[item.name] then
+                    item_number = i
+                    break
+                end
+            end
+        end
+    end
+    self.coll_list:switchItemTable(title, item_table, item_number or -1, nil, subtitle)
+end
+
+function FileManagerCollection:onCollListChoice(item)
+    if self._manager.selected_colections then
+        if item.mandatory == self._manager.checkmark then
+            self.item_table[item.idx].mandatory = "  "
+            self._manager.selected_colections[item.name] = nil
+        else
+            self.item_table[item.idx].mandatory = self._manager.checkmark
+            self._manager.selected_colections[item.name] = true
+        end
+        self._manager:updateCollListItemTable()
+    else
+        self._manager:onShowColl(item.name)
+    end
+end
+
+function FileManagerCollection:onCollListHold(item)
+    if item.name == ReadCollection.default_collection_name -- Favorites non-editable
+            or self._manager.selected_colections then -- select mode
+        return
+    end
+
+    local button_dialog
+    local buttons = {
+        {
+            {
+                text = _("Remove collection"),
+                callback = function()
+                    UIManager:close(button_dialog)
+                    self._manager:removeCollection(item)
+                end
+            },
+            {
+                text = _("Rename collection"),
+                callback = function()
+                    UIManager:close(button_dialog)
+                    self._manager:renameCollection(item)
+                end
+            },
+        },
+    }
+    button_dialog = ButtonDialog:new{
+        title = item.text,
+        title_align = "center",
+        buttons = buttons,
+    }
+    UIManager:show(button_dialog)
+    return true
+end
+
+function FileManagerCollection:showCollListDialog(caller_callback, no_dialog)
+    if no_dialog then
+        caller_callback()
+        self.coll_list.close_callback(true)
+        return
+    end
+
+    local button_dialog, buttons
+    local new_collection_button = {
+        {
+            text = _("New collection"),
+            callback = function()
+                UIManager:close(button_dialog)
+                self:addCollection()
+            end,
+        },
+    }
+    if self.selected_colections then -- select mode
+        buttons = {
+            new_collection_button,
+            {}, -- separator
+            {
+                {
+                    text = _("Deselect all"),
+                    callback = function()
+                        UIManager:close(button_dialog)
+                        for name in pairs(self.selected_colections) do
+                            self.selected_colections[name] = nil
+                        end
+                        self:updateCollListItemTable(true)
+                    end,
+                },
+                {
+                    text = _("Select all"),
+                    callback = function()
+                        UIManager:close(button_dialog)
+                        for name in pairs(ReadCollection.coll) do
+                            self.selected_colections[name] = true
+                        end
+                        self:updateCollListItemTable(true)
+                    end,
+                },
+            },
+            {
+                {
+                    text = _("Apply selection"),
+                    callback = function()
+                        UIManager:close(button_dialog)
+                        caller_callback()
+                        self.coll_list.close_callback(true)
+                    end,
+                },
+            },
+        }
+    else
+        buttons = {
+            new_collection_button,
+            {
+                {
+                    text = _("Arrange collections"),
+                    callback = function()
+                        UIManager:close(button_dialog)
+                        self:sortCollections()
+                    end,
+                },
+            },
+        }
+    end
+    button_dialog = ButtonDialog:new{
+        buttons = buttons,
+    }
+    UIManager:show(button_dialog)
+end
+
+function FileManagerCollection:editCollectionName(editCallback, old_name)
+    local input_dialog
+    input_dialog = InputDialog:new{
+        title =  _("Enter collection name"),
+        input = old_name,
+        input_hint = old_name,
+        buttons = {{
+            {
+                text = _("Cancel"),
+                id = "close",
+                callback = function()
+                    UIManager:close(input_dialog)
+                end,
+            },
+            {
+                text = _("Save"),
+                callback = function()
+                    local new_name = input_dialog:getInputText()
+                    if new_name == "" or new_name == old_name then return end
+                    if ReadCollection.coll[new_name] then
+                        UIManager:show(InfoMessage:new{
+                            text = T(_("Collection already exists: %1"), new_name),
+                        })
+                    else
+                        UIManager:close(input_dialog)
+                        editCallback(new_name)
+                    end
+                end,
+            },
+        }},
+    }
+    UIManager:show(input_dialog)
+    input_dialog:onShowKeyboard()
+end
+
+function FileManagerCollection:addCollection()
+    local editCallback = function(name)
+        ReadCollection:addCollection(name)
+        local mandatory
+        if self.selected_colections then
+            self.selected_colections[name] = true
+            mandatory = self.checkmark
+        else
+            mandatory = 0
+        end
+        table.insert(self.coll_list.item_table, {
+            text      = name,
+            mandatory = mandatory,
+            name      = name,
+            order     = ReadCollection.coll_order[name],
+        })
+        self:updateCollListItemTable(false, #self.coll_list.item_table) -- show added item
+    end
+    self:editCollectionName(editCallback)
+end
+
+function FileManagerCollection:renameCollection(item)
+    local editCallback = function(name)
+        ReadCollection:renameCollection(item.name, name)
+        self.coll_list.item_table[item.idx].text = name
+        self.coll_list.item_table[item.idx].name = name
+        self:updateCollListItemTable()
+    end
+    self:editCollectionName(editCallback, item.name)
+end
+
+function FileManagerCollection:removeCollection(item)
+    UIManager:show(ConfirmBox:new{
+        text = _("Remove collection?") .. "\n\n" .. item.text,
+        ok_text = _("Remove"),
+        ok_callback = function()
+            ReadCollection:removeCollection(item.name)
+            table.remove(self.coll_list.item_table, item.idx)
+            self:updateCollListItemTable()
+        end,
+    })
+end
+
+function FileManagerCollection:sortCollections()
+    local sort_widget
+    sort_widget = SortWidget:new{
+        title = _("Arrange collections"),
+        item_table = util.tableDeepCopy(self.coll_list.item_table),
+        callback = function()
+            ReadCollection:updateCollectionListOrder(sort_widget.item_table)
+            self:updateCollListItemTable(true) -- init
+        end,
+    }
+    UIManager:show(sort_widget)
+end
+
+-- external
+
+function FileManagerCollection:genAddToCollectionButton(file_or_files, caller_pre_callback, caller_post_callback, button_disabled)
+    return {
+        text = _("Add to collection"),
+        enabled = not button_disabled,
+        callback = function()
+            if caller_pre_callback then
+                caller_pre_callback()
+            end
+            local caller_callback = function()
+                if type(file_or_files) == "string" then
+                    ReadCollection:addRemoveItemMultiple(file_or_files, self.selected_colections)
+                else -- selected files
+                    ReadCollection:addItemsMultiple(file_or_files, self.selected_colections)
+                end
+                if caller_post_callback then
+                    caller_post_callback()
+                end
+            end
+            self:onShowCollList(file_or_files, caller_callback)
+        end,
+    }
+end
+
 return FileManagerCollection

frontend/apps/filemanager/filemanagerfilesearcher.lua

@@ -1,21 +1,21 @@
-local ButtonDialogTitle = require("ui/widget/buttondialogtitle")
+local ButtonDialog = require("ui/widget/buttondialog")
 local CheckButton = require("ui/widget/checkbutton")
-local CenterContainer = require("ui/widget/container/centercontainer")
+local ConfirmBox = require("ui/widget/confirmbox")
+local DocSettings = require("docsettings")
 local DocumentRegistry = require("document/documentregistry")
 local FileChooser = require("ui/widget/filechooser")
-local FileManagerBookInfo = require("apps/filemanager/filemanagerbookinfo")
+local InfoMessage = require("ui/widget/infomessage")
 local InputDialog = require("ui/widget/inputdialog")
 local Menu = require("ui/widget/menu")
 local UIManager = require("ui/uimanager")
 local WidgetContainer = require("ui/widget/container/widgetcontainer")
-local BaseUtil = require("ffi/util")
 local Utf8Proc = require("ffi/utf8proc")
+local filemanagerutil = require("apps/filemanager/filemanagerutil")
 local lfs = require("libs/libkoreader-lfs")
 local util = require("util")
 local _ = require("gettext")
 local N_ = _.ngettext
-local Screen = require("device").screen
-local T = BaseUtil.template
+local T = require("ffi/util").template
 
 local FileSearcher = WidgetContainer:extend{
     case_sensitive = false,
@@ -23,15 +23,12 @@ local FileSearcher = WidgetContainer:extend{
     include_metadata = false,
 }
 
-function FileSearcher:init()
-end
-
 function FileSearcher:onShowFileSearch(search_string)
     local search_dialog
     local check_button_case, check_button_subfolders, check_button_metadata
     search_dialog = InputDialog:new{
         title = _("Enter text to search for in filename"),
-        input = search_string or self.search_value,
+        input = search_string or self.search_string,
         buttons = {
             {
                 {
@@ -45,8 +42,8 @@ function FileSearcher:onShowFileSearch(search_string)
                     text = _("Home folder"),
                     enabled = G_reader_settings:has("home_dir"),
                     callback = function()
-                        self.search_value = search_dialog:getInputText()
-                        if self.search_value == "" then return end
+                        self.search_string = search_dialog:getInputText()
+                        if self.search_string == "" then return end
                         UIManager:close(search_dialog)
                         self.path = G_reader_settings:readSetting("home_dir")
                         self:doSearch()
@@ -56,8 +53,8 @@ function FileSearcher:onShowFileSearch(search_string)
                     text = self.ui.file_chooser and _("Current folder") or _("Book folder"),
                     is_enter_default = true,
                     callback = function()
-                        self.search_value = search_dialog:getInputText()
-                        if self.search_value == "" then return end
+                        self.search_string = search_dialog:getInputText()
+                        if self.search_string == "" then return end
                         UIManager:close(search_dialog)
                         self.path = self.ui.file_chooser and self.ui.file_chooser.path or self.ui:getLastDirFile()
                         self:doSearch()
@@ -66,7 +63,6 @@ function FileSearcher:onShowFileSearch(search_string)
             },
         },
     }
-
     check_button_case = CheckButton:new{
         text = _("Case sensitive"),
         checked = self.case_sensitive,
@@ -96,7 +92,6 @@ function FileSearcher:onShowFileSearch(search_string)
         }
         search_dialog:addWidget(check_button_metadata)
     end
-
     UIManager:show(search_dialog)
     search_dialog:onShowKeyboard()
 end
@@ -124,20 +119,18 @@ function FileSearcher:getList()
         ["/proc"] = true,
         ["/sys"] = true,
     }
-    local show_hidden = G_reader_settings:isTrue("show_hidden")
-    local show_unsupported = G_reader_settings:isTrue("show_unsupported")
-    local collate = G_reader_settings:readSetting("collate")
-    local keywords = self.search_value
-    if keywords ~= "*" then -- one * to show all files
+    local collate = FileChooser:getCollate()
+    local search_string = self.search_string
+    if search_string ~= "*" then -- one * to show all files
         if not self.case_sensitive then
-            keywords = Utf8Proc.lowercase(util.fixUtf8(keywords, "?"))
+            search_string = Utf8Proc.lowercase(util.fixUtf8(search_string, "?"))
         end
         -- replace '.' with '%.'
-        keywords = keywords:gsub("%.","%%%.")
+        search_string = search_string:gsub("%.","%%%.")
         -- replace '*' with '.*'
-        keywords = keywords:gsub("%*","%.%*")
+        search_string = search_string:gsub("%*","%.%*")
         -- replace '?' with '.'
-        keywords = keywords:gsub("%?","%.")
+        search_string = search_string:gsub("%?","%.")
     end
 
     local dirs, files = {}, {}
@@ -157,20 +150,20 @@ function FileSearcher:getList()
                     local attributes = lfs.attributes(fullpath) or {}
                     -- Don't traverse hidden folders if we're not showing them
                     if attributes.mode == "directory" and f ~= "." and f ~= ".."
-                            and (show_hidden or not util.stringStartsWith(f, "."))
+                            and (FileChooser.show_hidden or not util.stringStartsWith(f, "."))
                             and FileChooser:show_dir(f) then
                         if self.include_subfolders and not sys_folders[fullpath] then
                             table.insert(new_dirs, fullpath)
                         end
-                        if self:isFileMatch(f, fullpath, keywords) then
-                            table.insert(dirs, FileChooser:getListItem(f, fullpath, attributes))
+                        if self:isFileMatch(f, fullpath, search_string) then
+                            table.insert(dirs, FileChooser:getListItem(nil, f, fullpath, attributes, collate))
                         end
                     -- Always ignore macOS resource forks, too.
                     elseif attributes.mode == "file" and not util.stringStartsWith(f, "._")
-                            and (show_unsupported or DocumentRegistry:hasProvider(fullpath))
+                            and (FileChooser.show_unsupported or DocumentRegistry:hasProvider(fullpath))
                             and FileChooser:show_file(f) then
-                        if self:isFileMatch(f, fullpath, keywords, true) then
-                            table.insert(files, FileChooser:getListItem(f, fullpath, attributes, collate))
+                        if self:isFileMatch(f, fullpath, search_string, true) then
+                            table.insert(files, FileChooser:getListItem(nil, f, fullpath, attributes, collate))
                         end
                     end
                 end
@@ -181,41 +174,22 @@ function FileSearcher:getList()
     return dirs, files
 end
 
-function FileSearcher:isFileMatch(filename, fullpath, keywords, is_file)
-    local metadata_keys = {
-        "authors",
-        "title",
-        "series",
-        "description",
-        "keywords",
-        "language",
-    }
-    if keywords == "*" then
+function FileSearcher:isFileMatch(filename, fullpath, search_string, is_file)
+    if search_string == "*" then
         return true
     end
     if not self.case_sensitive then
         filename = Utf8Proc.lowercase(util.fixUtf8(filename, "?"))
     end
-    if string.find(filename, keywords) then
+    if string.find(filename, search_string) then
         return true
     end
     if self.include_metadata and is_file and DocumentRegistry:hasProvider(fullpath) then
         local book_props = self.ui.coverbrowser:getBookInfo(fullpath) or
-                           FileManagerBookInfo:getBookProps(fullpath, nil, true)
+                           self.ui.bookinfo.getDocProps(fullpath, nil, true) -- do not open the document
         if next(book_props) ~= nil then
-            for _, key in ipairs(metadata_keys) do
-                local prop = book_props[key]
-                if prop and prop ~= "" then
-                    if not self.case_sensitive then
-                        prop = Utf8Proc.lowercase(util.fixUtf8(prop, "?"))
-                    end
-                    if key == "description" then
-                        prop = util.htmlToPlainTextIfHtml(prop)
-                    end
-                    if string.find(prop, keywords) then
-                        return true
-                    end
-                end
+            if self.ui.bookinfo:findInProps(book_props, search_string, self.case_sensitive) then
+                return true
             end
         else
             self.no_metadata_count = self.no_metadata_count + 1
@@ -224,16 +198,14 @@ function FileSearcher:isFileMatch(filename, fullpath, keywords, is_file)
 end
 
 function FileSearcher:showSearchResultsMessage(no_results)
-    local text = no_results and T(_("No results for '%1'."), self.search_value)
+    local text = no_results and T(_("No results for '%1'."), self.search_string)
     if self.no_metadata_count == 0 then
-        local InfoMessage = require("ui/widget/infomessage")
         UIManager:show(InfoMessage:new{ text = text })
     else
         local txt = T(N_("1 book has been skipped.", "%1 books have been skipped.",
             self.no_metadata_count), self.no_metadata_count) .. "\n" ..
             _("Not all books metadata extracted yet.\nExtract metadata now?")
         text = no_results and text .. "\n\n" .. txt or txt
-        local ConfirmBox = require("ui/widget/confirmbox")
         UIManager:show(ConfirmBox:new{
             text = text,
             ok_text = _("Extract"),
@@ -248,72 +220,99 @@ function FileSearcher:showSearchResultsMessage(no_results)
 end
 
 function FileSearcher:showSearchResults(results)
-    local menu_container = CenterContainer:new{
-        dimen = Screen:getSize(),
-    }
     self.search_menu = Menu:new{
-        width = Screen:getWidth(),
-        height = Screen:getHeight(),
+        title = T(_("Search results (%1)"), #results),
+        subtitle = T(_("Query: %1"), self.search_string),
+        item_table = results,
+        ui = self.ui,
+        covers_fullscreen = true, -- hint for UIManager:_repaint()
         is_borderless = true,
         is_popout = false,
-        show_parent = menu_container,
+        title_bar_fm_style = true,
         onMenuSelect = self.onMenuSelect,
         onMenuHold = self.onMenuHold,
         handle_hold_on_hold_release = true,
-        _manager = self,
     }
-    table.insert(menu_container, self.search_menu)
     self.search_menu.close_callback = function()
-        UIManager:close(menu_container)
+        UIManager:close(self.search_menu)
+        if self.ui.file_chooser then
+            self.ui.file_chooser:refreshPath()
+        end
     end
-    self.search_menu:switchItemTable(T(_("Search results (%1)"), #results), results)
-    UIManager:show(menu_container)
+    UIManager:show(self.search_menu)
     if self.no_metadata_count ~= 0 then
         self:showSearchResultsMessage()
     end
 end
 
 function FileSearcher:onMenuSelect(item)
-    local dialog
+    local file = item.path
+    local bookinfo, dialog
+    local function close_dialog_callback()
+        UIManager:close(dialog)
+    end
+    local function close_dialog_menu_callback()
+        UIManager:close(dialog)
+        self.close_callback()
+    end
     local buttons = {}
     if item.is_file then
+        local is_currently_opened = self.ui.document and self.ui.document.file == file
+        if DocumentRegistry:hasProvider(file) or DocSettings:hasSidecarFile(file) then
+            bookinfo = self.ui.coverbrowser and self.ui.coverbrowser:getBookInfo(file)
+            local doc_settings_or_file = is_currently_opened and self.ui.doc_settings or file
+            table.insert(buttons, filemanagerutil.genStatusButtonsRow(doc_settings_or_file, close_dialog_callback))
+            table.insert(buttons, {}) -- separator
+            table.insert(buttons, {
+                filemanagerutil.genResetSettingsButton(file, close_dialog_callback, is_currently_opened),
+                self.ui.collections:genAddToCollectionButton(file, close_dialog_callback),
+            })
+        end
         table.insert(buttons, {
             {
-                text = _("Book information"),
+                text = _("Delete"),
+                enabled = not is_currently_opened,
                 callback = function()
-                    UIManager:close(dialog)
-                    FileManagerBookInfo:show(item.path)
-                end,
-            },
-            {
-                text = _("Open"),
-                enabled = DocumentRegistry:hasProvider(item.path),
-                callback = function()
-                    UIManager:close(dialog)
-                    self.close_callback()
-                    require("apps/reader/readerui"):showReader(item.path)
+                    local function post_delete_callback()
+                        UIManager:close(dialog)
+                        for i, menu_item in ipairs(self.item_table) do
+                            if menu_item.path == file then
+                                table.remove(self.item_table, i)
+                                break
+                            end
+                            self:switchItemTable(T(_("Search results (%1)"), #self.item_table), self.item_table)
+                        end
+                    end
+                    local FileManager = require("apps/filemanager/filemanager")
+                    FileManager:showDeleteFileDialog(file, post_delete_callback)
                 end,
             },
+            filemanagerutil.genBookInformationButton(file, bookinfo, close_dialog_callback),
         })
     end
     table.insert(buttons, {
+        filemanagerutil.genShowFolderButton(file, close_dialog_menu_callback),
         {
-            text = _("Cancel"),
-            callback = function()
-                UIManager:close(dialog)
-            end,
-        },
-        {
-            text = _("Show folder"),
+            text = _("Open"),
+            enabled = DocumentRegistry:hasProvider(file, nil, true), -- allow auxiliary providers
             callback = function()
-                UIManager:close(dialog)
-                self.close_callback()
-                self._manager:showFolder(item.path)
+                close_dialog_callback()
+                local FileManager = require("apps/filemanager/filemanager")
+                FileManager.openFile(self.ui, file, nil, self.close_callback)
             end,
         },
     })
-    dialog = ButtonDialogTitle:new{
-        title = item.path,
+    local title = file
+    if bookinfo then
+        if bookinfo.title then
+            title = title .. "\n\n" .. T(_("Title: %1"), bookinfo.title)
+        end
+        if bookinfo.authors then
+            title = title .. "\n" .. T(_("Authors: %1"), bookinfo.authors:gsub("[\n\t]", "|"))
+        end
+    end
+    dialog = ButtonDialog:new{
+        title = title .. "\n",
         buttons = buttons,
     }
     UIManager:show(dialog)
@@ -321,25 +320,21 @@ end
 
 function FileSearcher:onMenuHold(item)
     if item.is_file then
-        if DocumentRegistry:hasProvider(item.path) then
-            self.close_callback()
-            require("apps/reader/readerui"):showReader(item.path)
+        if DocumentRegistry:hasProvider(item.path, nil, true) then
+            local FileManager = require("apps/filemanager/filemanager")
+            FileManager.openFile(self.ui, item.path, nil, self.close_callback)
         end
     else
         self.close_callback()
-        self._manager:showFolder(item.path)
+        if self.ui.file_chooser then
+            local pathname = util.splitFilePathName(item.path)
+            self.ui.file_chooser:changeToPath(pathname, item.path)
+        else -- called from Reader
+            self.ui:onClose()
+            self.ui:showFileManager(item.path)
+        end
     end
     return true
 end
 
-function FileSearcher:showFolder(path)
-    if self.ui.file_chooser then
-        local pathname = util.splitFilePathName(path)
-        self.ui.file_chooser:changeToPath(pathname, path)
-    else -- called from Reader
-        self.ui:onClose()
-        self.ui:showFileManager(path)
-    end
-end
-
 return FileSearcher

frontend/apps/filemanager/filemanagerhistory.lua

@@ -1,11 +1,18 @@
 local BD = require("ui/bidi")
 local ButtonDialog = require("ui/widget/buttondialog")
+local CheckButton = require("ui/widget/checkbutton")
 local ConfirmBox = require("ui/widget/confirmbox")
+local DocSettings = require("docsettings")
+local FileManagerBookInfo = require("apps/filemanager/filemanagerbookinfo")
+local InputDialog = require("ui/widget/inputdialog")
 local Menu = require("ui/widget/menu")
+local ReadCollection = require("readcollection")
 local UIManager = require("ui/uimanager")
 local WidgetContainer = require("ui/widget/container/widgetcontainer")
 local Screen = require("device").screen
+local Utf8Proc = require("ffi/utf8proc")
 local filemanagerutil = require("apps/filemanager/filemanagerutil")
+local util = require("util")
 local _ = require("gettext")
 local C_ = _.pgettext
 local T = require("ffi/util").template
@@ -15,12 +22,12 @@ local FileManagerHistory = WidgetContainer:extend{
 }
 
 local filter_text = {
-    all = C_("Book status filter", "All"),
-    reading = C_("Book status filter", "Reading"),
+    all       = C_("Book status filter", "All"),
+    reading   = C_("Book status filter", "Reading"),
     abandoned = C_("Book status filter", "On hold"),
-    complete = C_("Book status filter", "Finished"),
-    deleted = C_("Book status filter", "Deleted"),
-    new = C_("Book status filter", "New"),
+    complete  = C_("Book status filter", "Finished"),
+    deleted   = C_("Book status filter", "Deleted"),
+    new       = C_("Book status filter", "New"),
 }
 
 function FileManagerHistory:init()
@@ -38,39 +45,79 @@ end
 
 function FileManagerHistory:fetchStatuses(count)
     for _, v in ipairs(require("readhistory").hist) do
-        v.status = v.dim and "deleted" or filemanagerutil.getStatus(v.file)
-        if v.status == "new" and v.file == (self.ui.document and self.ui.document.file) then
-            v.status = "reading" -- file currently opened for the first time
+        local status
+        if v.dim then -- deleted file
+            status = "deleted"
+        elseif v.file == (self.ui.document and self.ui.document.file) then -- currently opened file
+            status = self.ui.doc_settings:readSetting("summary").status
+        else
+            status = filemanagerutil.getStatus(v.file)
+        end
+        if not filter_text[status] then
+            status = "reading"
         end
         if count then
-            self.count[v.status] = self.count[v.status] + 1
+            self.count[status] = self.count[status] + 1
         end
+        v.status = status
     end
     self.statuses_fetched = true
 end
 
 function FileManagerHistory:updateItemTable()
-    -- try to stay on current page
-    local select_number = nil
-    if self.hist_menu.page and self.hist_menu.perpage and self.hist_menu.page > 0 then
-        select_number = (self.hist_menu.page - 1) * self.hist_menu.perpage + 1
-    end
     self.count = { all = #require("readhistory").hist,
         reading = 0, abandoned = 0, complete = 0, deleted = 0, new = 0, }
     local item_table = {}
     for _, v in ipairs(require("readhistory").hist) do
-        if self.filter == "all" or v.status == self.filter then
-            table.insert(item_table, v)
+        if self:isItemMatch(v) then
+            local item = util.tableDeepCopy(v)
+            if item.select_enabled and ReadCollection:isFileInCollections(item.file) then
+                item.mandatory = "☆ " .. item.mandatory
+            end
+            if self.is_frozen and item.status == "complete" then
+                item.mandatory_dim = true
+            end
+            table.insert(item_table, item)
         end
         if self.statuses_fetched then
             self.count[v.status] = self.count[v.status] + 1
         end
     end
-    local title = self.hist_menu_title
-    if self.filter ~= "all" then
-        title = title .. " (" .. filter_text[self.filter] .. ": " .. self.count[self.filter] .. ")"
+    local subtitle = ""
+    if self.search_string then
+        subtitle = T(_("Search results (%1)"), #item_table)
+    elseif self.selected_colections then
+        subtitle = T(_("Filtered by collections (%1)"), #item_table)
+    elseif self.filter ~= "all" then
+        subtitle = T(_("Status: %1 (%2)"), filter_text[self.filter]:lower(), #item_table)
+    end
+    self.hist_menu:switchItemTable(nil, item_table, -1, nil, subtitle)
+end
+
+function FileManagerHistory:isItemMatch(item)
+    if self.search_string then
+        local filename = self.case_sensitive and item.text or Utf8Proc.lowercase(util.fixUtf8(item.text, "?"))
+        if not filename:find(self.search_string) then
+            local book_props
+            if self.ui.coverbrowser then
+                book_props = self.ui.coverbrowser:getBookInfo(item.file)
+            end
+            if not book_props then
+                book_props = self.ui.bookinfo.getDocProps(item.file, nil, true) -- do not open the document
+            end
+            if not self.ui.bookinfo:findInProps(book_props, self.search_string, self.case_sensitive) then
+                return false
+            end
+        end
+    end
+    if self.selected_colections then
+        for name in pairs(self.selected_colections) do
+            if not ReadCollection:isFileInCollection(item.file, name) then
+                return false
+            end
+        end
     end
-    self.hist_menu:switchItemTable(title, item_table, select_number)
+    return self.filter == "all" or item.status == self.filter
 end
 
 function FileManagerHistory:onSetDimensions(dimen)
@@ -78,11 +125,20 @@ function FileManagerHistory:onSetDimensions(dimen)
 end
 
 function FileManagerHistory:onMenuChoice(item)
-    require("apps/reader/readerui"):showReader(item.file)
+    if self.ui.document then
+        if self.ui.document.file ~= item.file then
+            self.ui:switchDocument(item.file)
+        end
+    else
+        self.ui:openFile(item.file)
+    end
 end
 
 function FileManagerHistory:onMenuHold(item)
+    local file = item.file
     self.histfile_dialog = nil
+    self.book_props = self.ui.coverbrowser and self.ui.coverbrowser:getBookInfo(file)
+
     local function close_dialog_callback()
         UIManager:close(self.histfile_dialog)
     end
@@ -90,9 +146,9 @@ function FileManagerHistory:onMenuHold(item)
         UIManager:close(self.histfile_dialog)
         self._manager.hist_menu.close_callback()
     end
-    local function status_button_callback()
+    local function close_dialog_update_callback()
         UIManager:close(self.histfile_dialog)
-        if self._manager.filter ~= "all" then
+        if self._manager.filter ~= "all" or self._manager.is_frozen then
             self._manager:fetchStatuses(false)
         else
             self._manager.statuses_fetched = false
@@ -100,16 +156,38 @@ function FileManagerHistory:onMenuHold(item)
         self._manager:updateItemTable()
         self._manager.files_updated = true -- sidecar folder may be created/deleted
     end
-    local is_currently_opened = item.file == (self.ui.document and self.ui.document.file)
+    local function update_callback()
+        self._manager:updateItemTable()
+    end
+    local is_currently_opened = file == (self.ui.document and self.ui.document.file)
 
     local buttons = {}
-    if not (item.dim or is_currently_opened) then
-        table.insert(buttons, filemanagerutil.genStatusButtonsRow(item.file, status_button_callback))
+    local doc_settings_or_file
+    if is_currently_opened then
+        doc_settings_or_file = self.ui.doc_settings
+        if not self.book_props then
+            self.book_props = self.ui.doc_props
+            self.book_props.has_cover = true
+        end
+    else
+        if DocSettings:hasSidecarFile(file) then
+            doc_settings_or_file = DocSettings:open(file)
+            if not self.book_props then
+                local props = doc_settings_or_file:readSetting("doc_props")
+                self.book_props = FileManagerBookInfo.extendProps(props, file)
+                self.book_props.has_cover = true
+            end
+        else
+            doc_settings_or_file = file
+        end
+    end
+    if not item.dim then
+        table.insert(buttons, filemanagerutil.genStatusButtonsRow(doc_settings_or_file, close_dialog_update_callback))
         table.insert(buttons, {}) -- separator
     end
     table.insert(buttons, {
-        filemanagerutil.genResetSettingsButton(item.file, status_button_callback, is_currently_opened),
-        filemanagerutil.genAddRemoveFavoritesButton(item.file, close_dialog_callback, item.dim),
+        filemanagerutil.genResetSettingsButton(doc_settings_or_file, close_dialog_update_callback, is_currently_opened),
+        self._manager.ui.collections:genAddToCollectionButton(file, close_dialog_callback, update_callback, item.dim),
     })
     table.insert(buttons, {
         {
@@ -122,7 +200,7 @@ function FileManagerHistory:onMenuHold(item)
                     self._manager.files_updated = true
                 end
                 local FileManager = require("apps/filemanager/filemanager")
-                FileManager:showDeleteFileDialog(item.file, post_delete_callback)
+                FileManager:showDeleteFileDialog(file, post_delete_callback)
             end,
         },
         {
@@ -135,16 +213,16 @@ function FileManagerHistory:onMenuHold(item)
         },
     })
     table.insert(buttons, {
-        filemanagerutil.genShowFolderButton(item.file, close_dialog_menu_callback, item.dim),
-        filemanagerutil.genBookInformationButton(item.file, close_dialog_callback, item.dim),
+        filemanagerutil.genShowFolderButton(file, close_dialog_menu_callback, item.dim),
+        filemanagerutil.genBookInformationButton(file, self.book_props, close_dialog_callback, item.dim),
     })
     table.insert(buttons, {
-        filemanagerutil.genBookCoverButton(item.file, close_dialog_callback, item.dim),
-        filemanagerutil.genBookDescriptionButton(item.file, close_dialog_callback, item.dim),
+        filemanagerutil.genBookCoverButton(file, self.book_props, close_dialog_callback, item.dim),
+        filemanagerutil.genBookDescriptionButton(file, self.book_props, close_dialog_callback, item.dim),
     })
 
     self.histfile_dialog = ButtonDialog:new{
-        title = BD.filename(item.text:match("([^/]+)$")),
+        title = BD.filename(item.text),
         title_align = "center",
         buttons = buttons,
     }
@@ -169,12 +247,16 @@ function FileManagerHistory:MenuSetRotationModeHandler(rotation)
     return true
 end
 
-function FileManagerHistory:onShowHist()
+function FileManagerHistory:onShowHist(search_info)
     self.hist_menu = Menu:new{
         ui = self.ui,
         covers_fullscreen = true, -- hint for UIManager:_repaint()
         is_borderless = true,
         is_popout = false,
+        title = self.hist_menu_title,
+        -- item and book cover thumbnail dimensions in Mosaic and Detailed list display modes
+        -- must be equal in File manager, History and Collection windows to avoid image scaling
+        title_bar_fm_style = true,
         title_bar_left_icon = "appbar.menu",
         onLeftButtonTap = function() self:showHistDialog() end,
         onMenuChoice = self.onMenuChoice,
@@ -183,24 +265,32 @@ function FileManagerHistory:onShowHist()
         _manager = self,
     }
 
+    if search_info then
+        self.search_string = search_info.search_string
+        self.case_sensitive = search_info.case_sensitive
+    else
+        self.search_string = nil
+        self.selected_colections = nil
+    end
     self.filter = G_reader_settings:readSetting("history_filter", "all")
-    if self.filter ~= "all" then
+    self.is_frozen = G_reader_settings:isTrue("history_freeze_finished_books")
+    if self.filter ~= "all" or self.is_frozen then
         self:fetchStatuses(false)
     end
     self:updateItemTable()
     self.hist_menu.close_callback = function()
         if self.files_updated then -- refresh Filemanager list of files
-            local FileManager = require("apps/filemanager/filemanager")
-            if FileManager.instance then
-                FileManager.instance:onRefresh()
+            if self.ui.file_chooser then
+                self.ui.file_chooser:refreshPath()
             end
             self.files_updated = nil
         end
         self.statuses_fetched = nil
         UIManager:close(self.hist_menu)
+        self.hist_menu = nil
         G_reader_settings:saveSetting("history_filter", self.filter)
     end
-    UIManager:show(self.hist_menu)
+    UIManager:show(self.hist_menu, "flashui")
     return true
 end
 
@@ -217,6 +307,10 @@ function FileManagerHistory:showHistDialog()
             callback = function()
                 UIManager:close(hist_dialog)
                 self.filter = filter
+                if filter == "all" then -- reset all filters
+                    self.search_string = nil
+                    self.selected_colections = nil
+                end
                 self:updateItemTable()
             end,
         }
@@ -231,6 +325,28 @@ function FileManagerHistory:showHistDialog()
         genFilterButton("abandoned"),
         genFilterButton("complete"),
     })
+    table.insert(buttons, {
+        {
+            text = _("Filter by collections"),
+            callback = function()
+                UIManager:close(hist_dialog)
+                local caller_callback = function()
+                    self.selected_colections = self.ui.collections.selected_colections
+                    self:updateItemTable()
+                end
+                self.ui.collections:onShowCollList({}, caller_callback, true) -- do not select any, no dialog to apply
+            end,
+        },
+    })
+    table.insert(buttons, {
+        {
+            text = _("Search in filename and book metadata"),
+            callback = function()
+                UIManager:close(hist_dialog)
+                self:onSearchHistory()
+            end,
+        },
+    })
     if self.count.deleted > 0 then
         table.insert(buttons, {}) -- separator
         table.insert(buttons, {
@@ -259,4 +375,61 @@ function FileManagerHistory:showHistDialog()
     UIManager:show(hist_dialog)
 end
 
+function FileManagerHistory:onSearchHistory()
+    local search_dialog, check_button_case
+    search_dialog = InputDialog:new{
+        title = _("Enter text to search history for"),
+        input = self.search_string,
+        buttons = {
+            {
+                {
+                    text = _("Cancel"),
+                    id = "close",
+                    callback = function()
+                        UIManager:close(search_dialog)
+                    end,
+                },
+                {
+                    text = _("Search"),
+                    is_enter_default = true,
+                    callback = function()
+                        local search_string = search_dialog:getInputText()
+                        if search_string ~= "" then
+                            UIManager:close(search_dialog)
+                            self.search_string = self.case_sensitive and search_string or search_string:lower()
+                            if self.hist_menu then -- called from History
+                                self:updateItemTable()
+                            else -- called by Dispatcher
+                                local search_info = {
+                                    search_string = self.search_string,
+                                    case_sensitive = self.case_sensitive,
+                                }
+                                self:onShowHist(search_info)
+                            end
+                        end
+                    end,
+                },
+            },
+        },
+    }
+    check_button_case = CheckButton:new{
+        text = _("Case sensitive"),
+        checked = self.case_sensitive,
+        parent = search_dialog,
+        callback = function()
+            self.case_sensitive = check_button_case.checked
+        end,
+    }
+    search_dialog:addWidget(check_button_case)
+    UIManager:show(search_dialog)
+    search_dialog:onShowKeyboard()
+    return true
+end
+
+function FileManagerHistory:onBookMetadataChanged()
+    if self.hist_menu then
+        self.hist_menu:updateItems()
+    end
+end
+
 return FileManagerHistory

frontend/apps/filemanager/filemanagermenu.lua

@@ -2,20 +2,21 @@ local BD = require("ui/bidi")
 local CenterContainer = require("ui/widget/container/centercontainer")
 local ConfirmBox = require("ui/widget/confirmbox")
 local Device = require("device")
-local Event = require("ui/event")
 local FFIUtil = require("ffi/util")
 local InputContainer = require("ui/widget/container/inputcontainer")
+local KeyValuePage = require("ui/widget/keyvaluepage")
 local PluginLoader = require("pluginloader")
 local SetDefaults = require("apps/filemanager/filemanagersetdefaults")
 local Size = require("ui/size")
+local SpinWidget = require("ui/widget/spinwidget")
 local UIManager = require("ui/uimanager")
 local Screen = Device.screen
+local filemanagerutil = require("apps/filemanager/filemanagerutil")
 local dbg = require("dbg")
 local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
 local util  = require("util")
 local _ = require("gettext")
-local N_ = _.ngettext
 local T = FFIUtil.template
 
 local FileManagerMenu = InputContainer:extend{
@@ -60,11 +61,15 @@ end
 function FileManagerMenu:registerKeyEvents()
     if Device:hasKeys() then
         self.key_events.ShowMenu = { { "Menu" } }
+        if Device:hasScreenKB() then
+            self.key_events.OpenLastDoc = { { "ScreenKB", "Back" } }
+        end
     end
 end
 
 FileManagerMenu.onPhysicalKeyboardConnected = FileManagerMenu.registerKeyEvents
 
+-- NOTE: FileManager emits a SetDimensions on init, it's our only caller
 function FileManagerMenu:initGesListener()
     if not Device:isTouchDevice() then return end
 
@@ -143,69 +148,77 @@ function FileManagerMenu:onOpenLastDoc()
 end
 
 function FileManagerMenu:setUpdateItemTable()
+    local FileChooser = self.ui.file_chooser
 
     -- setting tab
     self.menu_items.filebrowser_settings = {
         text = _("Settings"),
         sub_item_table = {
+            {
+                text = _("Show finished books"),
+                checked_func = function() return FileChooser.show_finished end,
+                callback = function() FileChooser:toggleShowFilesMode("show_finished") end,
+            },
             {
                 text = _("Show hidden files"),
-                checked_func = function() return self.ui.file_chooser.show_hidden end,
-                callback = function() self.ui:toggleHiddenFiles() end,
+                checked_func = function() return FileChooser.show_hidden end,
+                callback = function() FileChooser:toggleShowFilesMode("show_hidden") end,
             },
             {
                 text = _("Show unsupported files"),
-                checked_func = function() return self.ui.file_chooser.show_unsupported end,
-                callback = function() self.ui:toggleUnsupportedFiles() end,
+                checked_func = function() return FileChooser.show_unsupported end,
+                callback = function() FileChooser:toggleShowFilesMode("show_unsupported") end,
                 separator = true,
             },
             {
                 text = _("Classic mode settings"),
                 sub_item_table = {
                     {
-                        text = _("Items per page"),
+                        text_func = function()
+                            return T(_("Items per page: %1"),
+                                G_reader_settings:readSetting("items_per_page") or FileChooser.items_per_page_default)
+                        end,
                         help_text = _([[This sets the number of items per page in:
 - File browser, history and favorites in 'classic' display mode
 - Search results and folder shortcuts
 - File and folder selection
 - Calibre and OPDS browsers/search results]]),
-                        callback = function()
-                            local SpinWidget = require("ui/widget/spinwidget")
-                            local Menu = require("ui/widget/menu")
-                            local default_perpage = Menu.items_per_page_default
-                            local curr_perpage = G_reader_settings:readSetting("items_per_page") or default_perpage
-                            local items = SpinWidget:new{
-                                value = curr_perpage,
-                                value_min = 6,
-                                value_max = 24,
-                                default_value = default_perpage,
+                        callback = function(touchmenu_instance)
+                            local default_value = FileChooser.items_per_page_default
+                            local current_value = G_reader_settings:readSetting("items_per_page") or default_value
+                            local widget = SpinWidget:new{
                                 title_text =  _("Items per page"),
+                                value = current_value,
+                                value_min = 6,
+                                value_max = 30,
+                                default_value = default_value,
                                 keep_shown_on_apply = true,
                                 callback = function(spin)
                                     G_reader_settings:saveSetting("items_per_page", spin.value)
-                                    self.ui:onRefresh()
-                                end
+                                    FileChooser:refreshPath()
+                                    touchmenu_instance:updateItems()
+                                end,
                             }
-                            UIManager:show(items)
+                            UIManager:show(widget)
                         end,
                     },
                     {
-                        text = _("Item font size"),
-                        callback = function()
-                            local SpinWidget = require("ui/widget/spinwidget")
-                            local Menu = require("ui/widget/menu")
-                            local curr_perpage = G_reader_settings:readSetting("items_per_page") or Menu.items_per_page_default
-                            local default_font_size = Menu.getItemFontSize(curr_perpage)
-                            local curr_font_size = G_reader_settings:readSetting("items_font_size") or default_font_size
-                            local items_font = SpinWidget:new{
-                                value = curr_font_size,
+                        text_func = function()
+                            return T(_("Item font size: %1"), FileChooser.font_size)
+                        end,
+                        callback = function(touchmenu_instance)
+                            local current_value = FileChooser.font_size
+                            local default_value = FileChooser.getItemFontSize(G_reader_settings:readSetting("items_per_page")
+                                or FileChooser.items_per_page_default)
+                            local widget = SpinWidget:new{
+                                title_text =  _("Item font size"),
+                                value = current_value,
                                 value_min = 10,
                                 value_max = 72,
-                                default_value = default_font_size,
+                                default_value = default_value,
                                 keep_shown_on_apply = true,
-                                title_text =  _("Item font size"),
                                 callback = function(spin)
-                                    if spin.value == default_font_size then
+                                    if spin.value == default_value then
                                         -- We can't know if the user has set a size or hit "Use default", but
                                         -- assume that if it is the default font size, he will prefer to have
                                         -- our default font size if he later updates per-page
@@ -213,10 +226,11 @@ function FileManagerMenu:setUpdateItemTable()
                                     else
                                         G_reader_settings:saveSetting("items_font_size", spin.value)
                                     end
-                                    self.ui:onRefresh()
-                                end
+                                    FileChooser:refreshPath()
+                                    touchmenu_instance:updateItems()
+                                end,
                             }
-                            UIManager:show(items_font)
+                            UIManager:show(widget)
                         end,
                     },
                     {
@@ -270,7 +284,16 @@ function FileManagerMenu:setUpdateItemTable()
                         end,
                         callback = function()
                             G_reader_settings:flipNilOrFalse("history_datetime_short")
-                            require("readhistory"):reload(true)
+                            require("readhistory"):updateDateTimeString()
+                        end,
+                    },
+                    {
+                        text = _("Freeze last read date of finished books"),
+                        checked_func = function()
+                            return G_reader_settings:isTrue("history_freeze_finished_books")
+                        end,
+                        callback = function()
+                            G_reader_settings:flipNilOrFalse("history_freeze_finished_books")
                         end,
                         separator = true,
                     },
@@ -313,29 +336,14 @@ function FileManagerMenu:setUpdateItemTable()
                     {
                         text = _("Set home folder"),
                         callback = function()
-                            local text
-                            local home_dir = G_reader_settings:readSetting("home_dir")
-                            if home_dir then
-                                text = T(_("Home folder is set to:\n%1"), home_dir)
-                            else
-                                text = _("Home folder is not set.")
-                                home_dir = Device.home_dir
+                            local title_header = _("Current home folder:")
+                            local current_path = G_reader_settings:readSetting("home_dir")
+                            local default_path = filemanagerutil.getDefaultDir()
+                            local caller_callback = function(path)
+                                G_reader_settings:saveSetting("home_dir", path)
+                                self.ui:updateTitleBarPath()
                             end
-                            UIManager:show(ConfirmBox:new{
-                                text = text .. "\n" .. _("Choose new folder to set as home?"),
-                                ok_text = _("Choose folder"),
-                                ok_callback = function()
-                                    local path_chooser = require("ui/widget/pathchooser"):new{
-                                        select_file = false,
-                                        show_files = false,
-                                        path = home_dir,
-                                        onConfirm = function(new_path)
-                                            G_reader_settings:saveSetting("home_dir", new_path)
-                                        end
-                                    }
-                                    UIManager:show(path_chooser)
-                                end,
-                            })
+                            filemanagerutil.showChooseDialog(title_header, caller_callback, current_path, default_path)
                         end,
                     },
                     {
@@ -345,8 +353,7 @@ function FileManagerMenu:setUpdateItemTable()
                         end,
                         callback = function()
                             G_reader_settings:flipNilOrTrue("shorten_home_dir")
-                            local FileManager = require("apps/filemanager/filemanager")
-                            if FileManager.instance then FileManager.instance:reinit() end
+                            self.ui:updateTitleBarPath()
                         end,
                         help_text = _([[
 "Shorten home folder" will display the home folder itself as "Home" instead of its full path.
@@ -375,26 +382,28 @@ To:
                 separator = true,
             },
             {
-                text = _("Info lists items per page"),
+                text_func = function()
+                    local default_value = KeyValuePage.getDefaultItemsPerPage()
+                    local current_value = G_reader_settings:readSetting("keyvalues_per_page") or default_value
+                    return T(_("Info lists items per page: %1"), current_value)
+                end,
                 help_text = _([[This sets the number of items per page in:
 - Book information
 - Dictionary and Wikipedia lookup history
 - Reading statistics details
 - A few other plugins]]),
                 keep_menu_open = true,
-                callback = function()
-                    local SpinWidget = require("ui/widget/spinwidget")
-                    local KeyValuePage = require("ui/widget/keyvaluepage")
-                    local default_perpage = KeyValuePage:getDefaultKeyValuesPerPage()
-                    local curr_perpage = G_reader_settings:readSetting("keyvalues_per_page") or default_perpage
-                    local items = SpinWidget:new{
-                        value = curr_perpage,
+                callback = function(touchmenu_instance)
+                    local default_value = KeyValuePage.getDefaultItemsPerPage()
+                    local current_value = G_reader_settings:readSetting("keyvalues_per_page") or default_value
+                    local widget = SpinWidget:new{
+                        value = current_value,
                         value_min = 10,
-                        value_max = 24,
-                        default_value = default_perpage,
+                        value_max = 30,
+                        default_value = default_value,
                         title_text =  _("Info lists items per page"),
                         callback = function(spin)
-                            if spin.value == default_perpage then
+                            if spin.value == default_value then
                                 -- We can't know if the user has set a value or hit "Use default", but
                                 -- assume that if it is the default, he will prefer to stay with our
                                 -- default if he later changes screen DPI
@@ -402,9 +411,10 @@ To:
                             else
                                 G_reader_settings:saveSetting("keyvalues_per_page", spin.value)
                             end
+                            touchmenu_instance:updateItems()
                         end
                     }
-                    UIManager:show(items)
+                    UIManager:show(widget)
                 end,
             },
         },
@@ -425,36 +435,29 @@ To:
         end,
         callback = function()
             G_reader_settings:flipNilOrFalse("reverse_collate")
-            self.ui.file_chooser:refreshPath()
+            FileChooser:refreshPath()
         end,
     }
     self.menu_items.sort_mixed = {
         text = _("Folders and files mixed"),
         enabled_func = function()
-            local collate = G_reader_settings:readSetting("collate")
-            return collate ~= "size" and
-                   collate ~= "type" and
-                   collate ~= "percent_unopened_first" and
-                   collate ~= "percent_unopened_last"
+            local collate = FileChooser:getCollate()
+            return collate.can_collate_mixed
         end,
         checked_func = function()
-            local collate = G_reader_settings:readSetting("collate")
-            return G_reader_settings:isTrue("collate_mixed") and
-                   collate ~= "size" and
-                   collate ~= "type" and
-                   collate ~= "percent_unopened_first" and
-                   collate ~= "percent_unopened_last"
+            local collate = FileChooser:getCollate()
+            return collate.can_collate_mixed and G_reader_settings:isTrue("collate_mixed")
         end,
         callback = function()
             G_reader_settings:flipNilOrFalse("collate_mixed")
-            self.ui.file_chooser:refreshPath()
+            FileChooser:refreshPath()
         end,
     }
     self.menu_items.start_with = self:getStartWithMenuTable()
 
     if Device:supportsScreensaver() then
         self.menu_items.screensaver = {
-            text = _("Screensaver"),
+            text = _("Sleep screen"),
             sub_item_table = require("ui/elements/screensaver_menu"),
         }
     end
@@ -474,7 +477,7 @@ To:
         text = _("Move book metadata"),
         keep_menu_open = true,
         callback = function()
-            self:moveBookMetadata()
+            self.ui.bookinfo:moveBookMetadata()
         end,
     }
 
@@ -626,6 +629,30 @@ To:
             end,
         })
     end
+    if Device:isKobo() and Device:hasColorScreen() then
+        table.insert(self.menu_items.developer_options.sub_item_table, {
+            -- We default to a flag (G2) that slightly boosts saturation,
+            -- but it *is* a destructive process, so we want to allow disabling it.
+            -- @translators CFA is a technical term for the technology behind eInk's color panels. It stands for Color Film/Filter Array, leave the abbreviation alone ;).
+            text = _("Disable CFA post-processing"),
+            checked_func = function()
+                return G_reader_settings:isTrue("no_cfa_post_processing")
+            end,
+            callback = function()
+                G_reader_settings:flipNilOrFalse("no_cfa_post_processing")
+                UIManager:askForRestart()
+            end,
+        })
+    end
+    table.insert(self.menu_items.developer_options.sub_item_table, {
+        text = _("Anti-alias rounded corners"),
+        checked_func = function()
+            return G_reader_settings:nilOrTrue("anti_alias_ui")
+        end,
+        callback = function()
+            G_reader_settings:flipNilOrTrue("anti_alias_ui")
+        end,
+    })
     --- @note: Currently, only Kobo implements this quirk
     if Device:hasEinkScreen() and Device:isKobo() then
         table.insert(self.menu_items.developer_options.sub_item_table, {
@@ -779,7 +806,7 @@ The sorting order is the same as in filemanager.
 
 Tap a book in the search results to open it.]]),
         callback = function()
-            self.ui:handleEvent(Event:new("ShowFileSearch"))
+            self.ui.filesearcher:onShowFileSearch()
         end
     }
 
@@ -846,37 +873,27 @@ dbg:guard(FileManagerMenu, 'setUpdateItemTable',
     end)
 
 function FileManagerMenu:getSortingMenuTable()
-    local collates = {
-        { _("name"), "strcoll" },
-        { _("name (natural sorting)"), "natural" },
-        { _("last read date"), "access" },
-        { _("date modified"), "date" },
-        { _("size"), "size" },
-        { _("type"), "type" },
-        { _("percent – unopened first"), "percent_unopened_first" },
-        { _("percent – unopened last"), "percent_unopened_last" },
-    }
     local sub_item_table = {}
-    for i, v in ipairs(collates) do
+    for k, v in pairs(self.ui.file_chooser.collates) do
         table.insert(sub_item_table, {
-            text = v[1],
+            text = v.text,
+            menu_order = v.menu_order,
             checked_func = function()
-                return v[2] == G_reader_settings:readSetting("collate", "strcoll")
+                local _, id = self.ui.file_chooser:getCollate()
+                return k == id
             end,
             callback = function()
-                G_reader_settings:saveSetting("collate", v[2])
+                G_reader_settings:saveSetting("collate", k)
+                self.ui.file_chooser:clearSortingCache()
                 self.ui.file_chooser:refreshPath()
             end,
         })
     end
+    table.sort(sub_item_table, function(a, b) return a.menu_order < b.menu_order end)
     return {
         text_func = function()
-            local collate = G_reader_settings:readSetting("collate")
-            for i, v in ipairs(collates) do
-                if v[2] == collate then
-                    return T(_("Sort by: %1"), v[1])
-                end
-            end
+            local collate = self.ui.file_chooser:getCollate()
+            return T(_("Sort by: %1"), collate.text)
         end,
         sub_item_table = sub_item_table,
     }
@@ -915,74 +932,6 @@ function FileManagerMenu:getStartWithMenuTable()
     }
 end
 
-function FileManagerMenu:moveBookMetadata()
-    local DocSettings = require("docsettings")
-    local FileChooser = self.ui.file_chooser
-    local function scanPath()
-        local sys_folders = { -- do not scan sys_folders
-            ["/dev"]  = true,
-            ["/proc"] = true,
-            ["/sys"]  = true,
-        }
-        local books_to_move = {}
-        local dirs = {FileChooser.path}
-        while #dirs ~= 0 do
-            local new_dirs = {}
-            for _, d in ipairs(dirs) do
-                local ok, iter, dir_obj = pcall(lfs.dir, d)
-                if ok then
-                    for f in iter, dir_obj do
-                        local fullpath = "/" .. f
-                        if d ~= "/" then
-                            fullpath = d .. fullpath
-                        end
-                        local attributes = lfs.attributes(fullpath) or {}
-                        if attributes.mode == "directory" and f ~= "." and f ~= ".."
-                                and FileChooser:show_dir(f) and not sys_folders[fullpath] then
-                            table.insert(new_dirs, fullpath)
-                        elseif attributes.mode == "file" and not util.stringStartsWith(f, "._")
-                                and FileChooser:show_file(f) and DocSettings:hasSidecarFile(fullpath)
-                                and lfs.attributes(DocSettings:getSidecarFile(fullpath), "mode") ~= "file" then
-                            table.insert(books_to_move, fullpath)
-                        end
-                    end
-                end
-            end
-            dirs = new_dirs
-        end
-        return books_to_move
-    end
-    UIManager:show(ConfirmBox:new{
-        text = _("Scan books in current folder and subfolders for their metadata location?"),
-        ok_text = _("Scan"),
-        ok_callback = function()
-            local books_to_move = scanPath()
-            local books_to_move_nb = #books_to_move
-            if books_to_move_nb == 0 then
-                local InfoMessage = require("ui/widget/infomessage")
-                UIManager:show(InfoMessage:new{
-                    text = _("No books with metadata not in your preferred location found."),
-                })
-            else
-                UIManager:show(ConfirmBox:new{
-                    text = T(N_("1 book with metadata not in your preferred location found.",
-                              "%1 books with metadata not in your preferred location found.",
-                              books_to_move_nb), books_to_move_nb) .. "\n" ..
-                              _("Move book metadata to your preferred location?"),
-                    ok_text = _("Move"),
-                    ok_callback = function()
-                        UIManager:close(self.menu_container)
-                        for _, book in ipairs(books_to_move) do
-                            DocSettings:updateLocation(book, book)
-                        end
-                        FileChooser:refreshPath()
-                    end,
-                })
-            end
-        end,
-    })
-end
-
 function FileManagerMenu:exitOrRestart(callback, force)
     UIManager:close(self.menu_container)
 
@@ -1049,10 +998,11 @@ function FileManagerMenu:onShowMenu(tab_index)
 end
 
 function FileManagerMenu:onCloseFileManagerMenu()
-    if not self.menu_container then return end
+    if not self.menu_container then return true end
     local last_tab_index = self.menu_container[1].last_index
     G_reader_settings:saveSetting("filemanagermenu_tab_index", last_tab_index)
     UIManager:close(self.menu_container)
+    self.menu_container = nil
     return true
 end
 
@@ -1090,11 +1040,14 @@ function FileManagerMenu:onSwipeShowMenu(ges)
 end
 
 function FileManagerMenu:onSetDimensions(dimen)
-    self:onCloseFileManagerMenu()
-    -- update listening according to new screen dimen
-    if Device:isTouchDevice() then
-        self:initGesListener()
+    -- This widget doesn't support in-place layout updates, so, close & reopen
+    if self.menu_container then
+        self:onCloseFileManagerMenu()
+        self:onShowMenu()
     end
+
+    -- update gesture zones according to new screen dimen
+    self:initGesListener()
 end
 
 function FileManagerMenu:onMenuSearch()

frontend/apps/filemanager/filemanagershortcuts.lua

@@ -7,199 +7,184 @@ local Screen = require("device").screen
 local UIManager = require("ui/uimanager")
 local WidgetContainer = require("ui/widget/container/widgetcontainer")
 local lfs = require("libs/libkoreader-lfs")
-local util = require("ffi/util")
 local _ = require("gettext")
-local T = util.template
 
 local FileManagerShortcuts = WidgetContainer:extend{
+    title = _("Folder shortcuts"),
     folder_shortcuts = G_reader_settings:readSetting("folder_shortcuts", {}),
 }
 
-function FileManagerShortcuts:updateItemTable(select_callback)
+function FileManagerShortcuts:updateItemTable()
     local item_table = {}
-    for _, item in ipairs(self.folder_shortcuts) do
+    for folder, item in pairs(self.folder_shortcuts) do
         table.insert(item_table, {
-            text = string.format("%s (%s)", item.text, item.folder),
-            folder = item.folder,
-            friendly_name = item.text,
-            deletable = true,
-            editable = true,
-            callback = function()
-                UIManager:close(self.fm_bookmark)
-
-                local folder = item.folder
-                if folder ~= nil and lfs.attributes(folder, "mode") == "directory" then
-                    if select_callback then
-                        select_callback(folder)
-                    else
-                        if self.ui.file_chooser then
-                            self.ui.file_chooser:changeToPath(folder)
-                        else -- called from Reader
-                            self.ui:onClose()
-                            self.ui:showFileManager(folder .. "/")
-                        end
-                    end
-                end
-            end,
+            text = string.format("%s (%s)", item.text, folder),
+            folder = folder,
+            name = item.text,
         })
     end
-
     table.sort(item_table, function(l, r)
         return l.text < r.text
     end)
+    self.shortcuts_menu:switchItemTable(nil, item_table, -1)
+end
 
-    -- try to stay on current page
-    local select_number
+function FileManagerShortcuts:hasFolderShortcut(folder)
+    return self.folder_shortcuts[folder] and true or false
+end
 
-    if self.fm_bookmark.page and self.fm_bookmark.perpage and self.fm_bookmark.page > 0 then
-        select_number = (self.fm_bookmark.page - 1) * self.fm_bookmark.perpage + 1
+function FileManagerShortcuts:onMenuChoice(item)
+    local folder = item.folder
+    if lfs.attributes(folder, "mode") ~= "directory" then return end
+    if self.select_callback then
+        self.select_callback(folder)
+    else
+        if self._manager.ui.file_chooser then
+            self._manager.ui.file_chooser:changeToPath(folder)
+        else -- called from Reader
+            self._manager.ui:onClose()
+            self._manager.ui:showFileManager(folder .. "/")
+        end
     end
-
-    self.fm_bookmark:switchItemTable(nil,
-                                     item_table, select_number)
 end
 
-function FileManagerShortcuts:addNewFolder()
-    local PathChooser = require("ui/widget/pathchooser")
-    local path_chooser = PathChooser:new{
-        select_directory = true,
-        select_file = false,
-        path = self.fm_bookmark.curr_path,
-        onConfirm = function(path)
-            local add_folder_input
-            local friendly_name = util.basename(path) or _("my folder")
-            add_folder_input = InputDialog:new{
-                title = _("Enter friendly name"),
-                input = friendly_name,
-                description = T(_("Title for selected folder:\n%1"), BD.dirpath(path)),
-                buttons = {
-                    {
-                        {
-                            text = _("Cancel"),
-                            id = "close",
-                            callback = function()
-                                UIManager:close(add_folder_input)
-                            end,
-                        },
-                        {
-                            text = _("Add"),
-                            is_enter_default = true,
-                            callback = function()
-                                self:addFolderFromInput(add_folder_input:getInputValue(), path)
-                                UIManager:close(add_folder_input)
-                            end,
-                        },
-                    }
-                },
-            }
-            UIManager:show(add_folder_input)
-            add_folder_input:onShowKeyboard()
-        end
+function FileManagerShortcuts:onMenuHold(item)
+    local dialog
+    local buttons = {
+        {
+            {
+                text = _("Remove shortcut"),
+                callback = function()
+                    UIManager:close(dialog)
+                    self._manager:removeShortcut(item.folder)
+                end
+            },
+            {
+                text = _("Rename shortcut"),
+                callback = function()
+                    UIManager:close(dialog)
+                    self._manager:editShortcut(item.folder)
+                end
+            },
+        },
+        self._manager.ui.file_chooser and self._manager.ui.clipboard and {
+            {
+                text = _("Paste to folder"),
+                callback = function()
+                    UIManager:close(dialog)
+                    self._manager.ui:pasteFileFromClipboard(item.folder)
+                end
+            },
+        },
     }
-    UIManager:show(path_chooser)
+    dialog = ButtonDialog:new{
+        title = item.name .. "\n" .. BD.dirpath(item.folder),
+        title_align = "center",
+        buttons = buttons,
+    }
+    UIManager:show(dialog)
+    return true
 end
 
-function FileManagerShortcuts:addFolderFromInput(friendly_name, folder)
-    for __, item in ipairs(self.folder_shortcuts) do
-        if item.text == friendly_name and item.folder == folder then
-            UIManager:show(InfoMessage:new{
-                text = _("A shortcut to this folder already exists."),
-            })
-            return
-        end
+function FileManagerShortcuts:removeShortcut(folder)
+    self.folder_shortcuts[folder] = nil
+    if self.shortcuts_menu then
+        self.fm_updated = true
+        self:updateItemTable()
     end
-    table.insert(self.folder_shortcuts, {
-        text = friendly_name,
-        folder = folder,
-    })
-    self:updateItemTable()
 end
 
-function FileManagerShortcuts:onMenuHold(item)
-    if item.deletable or item.editable then
-        local folder_shortcuts_dialog
-        folder_shortcuts_dialog = ButtonDialog:new{
-            buttons = {
-                {
-                    {
-                        text = _("Paste file"),
-                        enabled = (self._manager.ui.file_chooser and self._manager.ui.clipboard) and true or false,
-                        callback = function()
-                            UIManager:close(folder_shortcuts_dialog)
-                            self._manager.ui:pasteHere(item.folder)
-                        end
-                    },
-                    {
-                        text = _("Edit"),
-                        enabled = item.editable,
-                        callback = function()
-                            UIManager:close(folder_shortcuts_dialog)
-                            self._manager:editFolderShortcut(item)
-                        end
-                    },
-                    {
-                        text = _("Delete"),
-                        enabled = item.deletable,
-                        callback = function()
-                            UIManager:close(folder_shortcuts_dialog)
-                            self._manager:deleteFolderShortcut(item)
+function FileManagerShortcuts:editShortcut(folder, post_callback)
+    local item = self.folder_shortcuts[folder]
+    local name = item and item.text -- rename
+    local input_dialog
+    input_dialog = InputDialog:new {
+        title = _("Enter folder shortcut name"),
+        input = name,
+        description = BD.dirpath(folder),
+        buttons = {{
+            {
+                text = _("Cancel"),
+                id = "close",
+                callback = function()
+                    UIManager:close(input_dialog)
+                end,
+            },
+            {
+                text = _("Save"),
+                is_enter_default = true,
+                callback = function()
+                    local new_name = input_dialog:getInputText()
+                    if new_name == "" or new_name == name then return end
+                    UIManager:close(input_dialog)
+                    if item then
+                        item.text = new_name
+                    else
+                        self.folder_shortcuts[folder] = { text = new_name, time = os.time() }
+                        if post_callback then
+                            post_callback()
                         end
-                    },
-                },
-            }
-        }
-        UIManager:show(folder_shortcuts_dialog)
-        return true
-    end
+                    end
+                    if self.shortcuts_menu then
+                        self.fm_updated = true
+                        self:updateItemTable()
+                    end
+                end,
+            },
+        }},
+    }
+    UIManager:show(input_dialog)
+    input_dialog:onShowKeyboard()
 end
 
-function FileManagerShortcuts:editFolderShortcut(item)
-    local edit_folder_input
-    edit_folder_input = InputDialog:new {
-        title = _("Edit friendly name"),
-        input = item.friendly_name,
-        description = T(_("Rename title for selected folder:\n%1"), BD.dirpath(item.folder)),
-        buttons = {
-            {
-                {
-                    text = _("Cancel"),
-                    id = "close",
-                    callback = function()
-                        UIManager:close(edit_folder_input)
-                    end,
-                },
-                {
-                    text = _("Apply"),
-                    is_enter_default = true,
-                    callback = function()
-                        self:renameFolderShortcut(item, edit_folder_input:getInputText())
-                        UIManager:close(edit_folder_input)
-                    end,
-                },
-            }
-        },
+function FileManagerShortcuts:addShortcut()
+    local PathChooser = require("ui/widget/pathchooser")
+    local path_chooser = PathChooser:new{
+        select_directory = true,
+        select_file = false,
+        path = self.ui.file_chooser and self.ui.file_chooser.path or self.ui:getLastDirFile(),
+        onConfirm = function(path)
+            if self:hasFolderShortcut(path) then
+                UIManager:show(InfoMessage:new{
+                    text = _("Shortcut already exists."),
+                })
+            else
+                self:editShortcut(path)
+            end
+        end,
     }
-    UIManager:show(edit_folder_input)
-    edit_folder_input:onShowKeyboard()
+    UIManager:show(path_chooser)
 end
 
-function FileManagerShortcuts:renameFolderShortcut(item, new_name)
-    for _, element in ipairs(self.folder_shortcuts) do
-        if element.text == item.friendly_name and element.folder == item.folder then
-            element.text = new_name
-        end
-    end
-    self:updateItemTable()
+function FileManagerShortcuts:genShowFolderShortcutsButton(pre_callback)
+    return {
+        text = self.title,
+        callback = function()
+            pre_callback()
+            self:onShowFolderShortcutsDialog()
+        end,
+    }
 end
 
-function FileManagerShortcuts:deleteFolderShortcut(item)
-    for i = #self.folder_shortcuts, 1, -1 do
-        local element = self.folder_shortcuts[i]
-        if element.text == item.friendly_name and element.folder == item.folder then
-            table.remove(self.folder_shortcuts, i)
-        end
+function FileManagerShortcuts:genAddRemoveShortcutButton(folder, pre_callback, post_callback)
+    if self:hasFolderShortcut(folder) then
+        return {
+            text = _("Remove from folder shortcuts"),
+            callback = function()
+                pre_callback()
+                self:removeShortcut(folder)
+                post_callback()
+            end,
+        }
+    else
+        return {
+            text = _("Add to folder shortcuts"),
+            callback = function()
+                pre_callback()
+                self:editShortcut(folder, post_callback)
+            end,
+        }
     end
-    self:updateItemTable()
 end
 
 function FileManagerShortcuts:onSetDimensions(dimen)
@@ -208,7 +193,7 @@ end
 
 function FileManagerShortcuts:MenuSetRotationModeHandler(rotation)
     if rotation ~= nil and rotation ~= Screen:getRotationMode() then
-        UIManager:close(self._manager.fm_bookmark)
+        UIManager:close(self._manager.shortcuts_menu)
         if self._manager.ui.view and self._manager.ui.view.onSetRotationMode then
             self._manager.ui.view:onSetRotationMode(rotation)
         elseif self._manager.ui.onSetRotationMode then
@@ -222,23 +207,32 @@ function FileManagerShortcuts:MenuSetRotationModeHandler(rotation)
 end
 
 function FileManagerShortcuts:onShowFolderShortcutsDialog(select_callback)
-    self.fm_bookmark = Menu:new{
-        title = _("Folder shortcuts"),
-        show_parent = self.ui,
-        no_title = false,
-        parent = nil,
-        is_popout = false,
+    self.shortcuts_menu = Menu:new{
+        title = self.title,
+        covers_fullscreen = true,
         is_borderless = true,
-        curr_path = self.ui.file_chooser and self.ui.file_chooser.path or self.ui:getLastDirFile(),
+        is_popout = false,
+        select_callback = select_callback, -- called from PathChooser titlebar left button
+        title_bar_left_icon = not select_callback and "plus" or nil,
+        onLeftButtonTap = function() self:addShortcut() end,
+        onMenuChoice = self.onMenuChoice,
         onMenuHold = not select_callback and self.onMenuHold or nil,
         onSetRotationMode = self.MenuSetRotationModeHandler,
-        title_bar_left_icon = not select_callback and "plus" or nil,
-        onLeftButtonTap = function() self:addNewFolder() end,
         _manager = self,
     }
-
-    self:updateItemTable(select_callback)
-    UIManager:show(self.fm_bookmark)
+    self.shortcuts_menu.close_callback = function()
+        UIManager:close(self.shortcuts_menu)
+        if self.fm_updated then
+            if self.ui.file_chooser then
+                self.ui.file_chooser:refreshPath()
+                self.ui:updateTitleBarPath()
+            end
+            self.fm_updated = nil
+        end
+        self.shortcuts_menu = nil
+    end
+    self:updateItemTable()
+    UIManager:show(self.shortcuts_menu)
 end
 
 return FileManagerShortcuts

frontend/apps/filemanager/filemanagerutil.lua

@@ -5,8 +5,10 @@ This module contains miscellaneous helper functions for FileManager
 local BD = require("ui/bidi")
 local Device = require("device")
 local DocSettings = require("docsettings")
+local Event = require("ui/event")
 local UIManager = require("ui/uimanager")
 local ffiutil = require("ffi/util")
+local lfs = require("libs/libkoreader-lfs")
 local util = require("util")
 local _ = require("gettext")
 local T = ffiutil.template
@@ -33,7 +35,8 @@ function filemanagerutil.abbreviate(path)
     return path
 end
 
-function filemanagerutil.splitFileNameType(filename)
+function filemanagerutil.splitFileNameType(filepath)
+    local _, filename = util.splitFilePathName(filepath)
     local filename_without_suffix, filetype = util.splitFileNameSuffix(filename)
     filetype = filetype:lower()
     if filetype == "zip" then
@@ -47,22 +50,41 @@ function filemanagerutil.splitFileNameType(filename)
     return filename_without_suffix, filetype
 end
 
--- Purge doc settings in sidecar directory
-function filemanagerutil.purgeSettings(file)
-    local file_abs_path = ffiutil.realpath(file)
-    if file_abs_path then
-        return DocSettings:open(file_abs_path):purge()
+function filemanagerutil.getRandomFile(dir, match_func)
+    if not dir:match("/$") then
+        dir = dir .. "/"
+    end
+    local files = {}
+    local ok, iter, dir_obj = pcall(lfs.dir, dir)
+    if ok then
+        for entry in iter, dir_obj do
+            local file = dir .. entry
+            if lfs.attributes(file, "mode") == "file" and match_func(file) then
+                table.insert(files, entry)
+            end
+        end
+        if #files > 0 then
+            math.randomseed(os.time())
+            return dir .. files[math.random(#files)]
+        end
     end
 end
 
 -- Purge doc settings except kept
 function filemanagerutil.resetDocumentSettings(file)
     local settings_to_keep = {
+        annotations = true,
+        annotations_paging = true,
+        annotations_rolling = true,
         bookmarks = true,
+        bookmarks_paging = true,
+        bookmarks_rolling = true,
         bookmarks_sorted_20220106 = true,
         bookmarks_version = true,
         cre_dom_version = true,
         highlight = true,
+        highlight_paging = true,
+        highlight_rolling = true,
         highlights_imported = true,
         last_page = true,
         last_xpointer = true,
@@ -93,13 +115,17 @@ function filemanagerutil.getStatus(file)
 end
 
 -- Set a document status ("reading", "complete", or "abandoned")
-function filemanagerutil.setStatus(file, status)
+function filemanagerutil.setStatus(doc_settings_or_file, status)
     -- In case the book doesn't have a sidecar file, this'll create it
-    local doc_settings = DocSettings:open(file)
-    local summary = doc_settings:readSetting("summary") or {}
+    local doc_settings
+    if type(doc_settings_or_file) == "table" then
+        doc_settings = doc_settings_or_file
+    else
+        doc_settings = DocSettings:open(doc_settings_or_file)
+    end
+    local summary = doc_settings:readSetting("summary", {})
     summary.status = status
     summary.modified = os.date("%Y-%m-%d", os.time())
-    doc_settings:saveSetting("summary", summary)
     doc_settings:flush()
 end
 
@@ -115,15 +141,25 @@ function filemanagerutil.statusToString(status)
 end
 
 -- Generate all book status file dialog buttons in a row
-function filemanagerutil.genStatusButtonsRow(file, caller_callback, current_status)
-    local status = current_status or filemanagerutil.getStatus(file)
+function filemanagerutil.genStatusButtonsRow(doc_settings_or_file, caller_callback)
+    local file, summary, status
+    if type(doc_settings_or_file) == "table" then
+        file = doc_settings_or_file:readSetting("doc_path")
+        summary = doc_settings_or_file:readSetting("summary", {})
+        status = summary.status
+    else
+        file = doc_settings_or_file
+        summary = {}
+        status = filemanagerutil.getStatus(file)
+    end
     local function genStatusButton(to_status)
         return {
             text = filemanagerutil.statusToString(to_status) .. (status == to_status and "  ✓" or ""),
-            id = to_status, -- used by covermenu
             enabled = status ~= to_status,
             callback = function()
-                filemanagerutil.setStatus(file, to_status)
+                summary.status = to_status
+                filemanagerutil.setStatus(doc_settings_or_file, to_status)
+                UIManager:broadcastEvent(Event:new("DocSettingsItemsChanged", file, { summary = summary })) -- for CoverBrowser
                 caller_callback()
             end,
         }
@@ -136,59 +172,75 @@ function filemanagerutil.genStatusButtonsRow(file, caller_callback, current_stat
 end
 
 -- Generate "Reset" file dialog button
-function filemanagerutil.genResetSettingsButton(file, caller_callback, button_disabled)
+function filemanagerutil.genResetSettingsButton(doc_settings_or_file, caller_callback, button_disabled)
+    local doc_settings, file, has_sidecar_file
+    if type(doc_settings_or_file) == "table" then
+        doc_settings = doc_settings_or_file
+        file = doc_settings_or_file:readSetting("doc_path")
+        has_sidecar_file = true
+    else
+        file = ffiutil.realpath(doc_settings_or_file) or doc_settings_or_file
+        has_sidecar_file = DocSettings:hasSidecarFile(file)
+    end
+    local custom_cover_file = DocSettings:findCustomCoverFile(file)
+    local has_custom_cover_file = custom_cover_file and true or false
+    local custom_metadata_file = DocSettings:findCustomMetadataFile(file)
+    local has_custom_metadata_file = custom_metadata_file and true or false
     return {
         text = _("Reset"),
-        id = "reset", -- used by covermenu
-        enabled = (not button_disabled and DocSettings:hasSidecarFile(ffiutil.realpath(file))) and true or false,
+        enabled = not button_disabled and (has_sidecar_file or has_custom_metadata_file or has_custom_cover_file),
         callback = function()
+            local CheckButton = require("ui/widget/checkbutton")
             local ConfirmBox = require("ui/widget/confirmbox")
+            local check_button_settings, check_button_cover, check_button_metadata
             local confirmbox = ConfirmBox:new{
                 text = T(_("Reset this document?") .. "\n\n%1\n\n" ..
-                    _("Document progress, settings, bookmarks, highlights, notes and custom cover image will be permanently lost."),
+                         _("Information will be permanently lost."),
                     BD.filepath(file)),
                 ok_text = _("Reset"),
                 ok_callback = function()
-                    local custom_metadata_purged = filemanagerutil.purgeSettings(file)
-                    if custom_metadata_purged then -- refresh coverbrowser cached book info
-                        local FileManager = require("apps/filemanager/filemanager")
-                        local ui = FileManager.instance
-                        if not ui then
-                            local ReaderUI = require("apps/reader/readerui")
-                            ui = ReaderUI.instance
-                        end
-                        if ui and ui.coverbrowser then
-                            ui.coverbrowser:deleteBookInfo(file)
-                        end
+                    local data_to_purge = {
+                        doc_settings         = check_button_settings.checked,
+                        custom_cover_file    = check_button_cover.checked and custom_cover_file,
+                        custom_metadata_file = check_button_metadata.checked and custom_metadata_file,
+                    }
+                    (doc_settings or DocSettings:open(file)):purge(nil, data_to_purge)
+                    if data_to_purge.custom_cover_file or data_to_purge.custom_metadata_file then
+                        UIManager:broadcastEvent(Event:new("InvalidateMetadataCache", file))
+                    end
+                    if data_to_purge.doc_settings then
+                        UIManager:broadcastEvent(Event:new("DocSettingsItemsChanged", file)) -- for CoverBrowser
+                        require("readhistory"):fileSettingsPurged(file)
                     end
-                    require("readhistory"):fileSettingsPurged(file)
                     caller_callback()
                 end,
             }
+            check_button_settings = CheckButton:new{
+                text = _("document settings, progress, bookmarks, highlights, notes"),
+                checked = has_sidecar_file,
+                enabled = has_sidecar_file,
+                parent = confirmbox,
+            }
+            confirmbox:addWidget(check_button_settings)
+            check_button_cover = CheckButton:new{
+                text = _("custom cover image"),
+                checked = has_custom_cover_file,
+                enabled = has_custom_cover_file,
+                parent = confirmbox,
+            }
+            confirmbox:addWidget(check_button_cover)
+            check_button_metadata = CheckButton:new{
+                text = _("custom book metadata"),
+                checked = has_custom_metadata_file,
+                enabled = has_custom_metadata_file,
+                parent = confirmbox,
+            }
+            confirmbox:addWidget(check_button_metadata)
             UIManager:show(confirmbox)
         end,
     }
 end
 
-function filemanagerutil.genAddRemoveFavoritesButton(file, caller_callback, button_disabled)
-    local ReadCollection = require("readcollection")
-    local is_added = ReadCollection:checkItemExist(file)
-    return {
-        text_func = function()
-            return is_added and _("Remove from favorites") or _("Add to favorites")
-        end,
-        enabled = not button_disabled,
-        callback = function()
-            caller_callback()
-            if is_added then
-                ReadCollection:removeItem(file)
-            else
-                ReadCollection:addItem(file)
-            end
-        end,
-    }
-end
-
 function filemanagerutil.genShowFolderButton(file, caller_callback, button_disabled)
     return {
         text = _("Show folder"),
@@ -208,38 +260,41 @@ function filemanagerutil.genShowFolderButton(file, caller_callback, button_disab
     }
 end
 
-function filemanagerutil.genBookInformationButton(file, caller_callback, button_disabled)
+function filemanagerutil.genBookInformationButton(file, book_props, caller_callback, button_disabled)
     return {
         text = _("Book information"),
-        id = "book_information", -- used by covermenu
         enabled = not button_disabled,
         callback = function()
             caller_callback()
-            require("apps/filemanager/filemanagerbookinfo"):show(file)
+            local FileManagerBookInfo = require("apps/filemanager/filemanagerbookinfo")
+            FileManagerBookInfo:show(file, book_props and FileManagerBookInfo.extendProps(book_props))
         end,
     }
 end
 
-function filemanagerutil.genBookCoverButton(file, caller_callback, button_disabled)
+function filemanagerutil.genBookCoverButton(file, book_props, caller_callback, button_disabled)
+    local has_cover = book_props and book_props.has_cover
     return {
         text = _("Book cover"),
-        id = "book_cover", -- used by covermenu
-        enabled = not button_disabled,
+        enabled = (not button_disabled and (not book_props or has_cover)) and true or false,
         callback = function()
             caller_callback()
-            require("apps/filemanager/filemanagerbookinfo"):onShowBookCover(file)
+            local FileManagerBookInfo = require("apps/filemanager/filemanagerbookinfo")
+            FileManagerBookInfo:onShowBookCover(file)
         end,
     }
 end
 
-function filemanagerutil.genBookDescriptionButton(file, caller_callback, button_disabled)
+function filemanagerutil.genBookDescriptionButton(file, book_props, caller_callback, button_disabled)
+    local description = book_props and book_props.description
     return {
         text = _("Book description"),
-        id = "book_description", -- used by covermenu
-        enabled = not button_disabled,
+        -- enabled for deleted books if description is kept in CoverBrowser bookinfo cache
+        enabled = (not (button_disabled or book_props) or description) and true or false,
         callback = function()
             caller_callback()
-            require("apps/filemanager/filemanagerbookinfo"):onShowBookDescription(nil, file)
+            local FileManagerBookInfo = require("apps/filemanager/filemanagerbookinfo")
+            FileManagerBookInfo:onShowBookDescription(description, file)
         end,
     }
 end
@@ -283,4 +338,60 @@ function filemanagerutil.genExecuteScriptButton(file, caller_callback)
     }
 end
 
+function filemanagerutil.showChooseDialog(title_header, caller_callback, current_path, default_path, file_filter)
+    local is_file = file_filter and true or false
+    local path = current_path or default_path
+    local dialog
+    local buttons = {
+        {
+            {
+                text = is_file and _("Choose file") or _("Choose folder"),
+                callback = function()
+                    UIManager:close(dialog)
+                    if path then
+                        if is_file then
+                            path = path:match("(.*/)")
+                        end
+                        if lfs.attributes(path, "mode") ~= "directory" then
+                            path = G_reader_settings:readSetting("home_dir") or filemanagerutil.getDefaultDir()
+                        end
+                    end
+                    local PathChooser = require("ui/widget/pathchooser")
+                    local path_chooser = PathChooser:new{
+                        select_directory = not is_file,
+                        select_file = is_file,
+                        show_files = is_file,
+                        file_filter = file_filter,
+                        path = path,
+                        onConfirm = function(new_path)
+                            caller_callback(new_path)
+                        end,
+                    }
+                    UIManager:show(path_chooser)
+                end,
+            },
+        }
+    }
+    if default_path then
+        table.insert(buttons, {
+            {
+                text = _("Use default"),
+                enabled = path ~= default_path,
+                callback = function()
+                    UIManager:close(dialog)
+                    caller_callback(default_path)
+                end,
+            },
+        })
+    end
+    local title_value = path and (is_file and BD.filepath(path) or BD.dirpath(path))
+                              or _("not set")
+    local ButtonDialog = require("ui/widget/buttondialog")
+    dialog = ButtonDialog:new{
+        title = title_header .. "\n\n" .. title_value .. "\n",
+        buttons = buttons,
+    }
+    UIManager:show(dialog)
+end
+
 return filemanagerutil

frontend/apps/reader/modules/readerannotation.lua

@@ -0,0 +1,423 @@
+local WidgetContainer = require("ui/widget/container/widgetcontainer")
+local logger = require("logger")
+local _ = require("gettext")
+local T = require("ffi/util").template
+
+local ReaderAnnotation = WidgetContainer:extend{
+    annotations = nil, -- array sorted by annotation position order, ascending
+}
+
+-- build, read, save
+
+function ReaderAnnotation:buildAnnotation(bm, highlights, init)
+    -- bm: associated single bookmark ; highlights: tables with all highlights
+    local note = bm.text
+    if note == "" then
+        note = nil
+    end
+    local chapter = bm.chapter
+    local hl, pageno = self:getHighlightByDatetime(highlights, bm.datetime)
+    if init then
+        if note and self.ui.bookmark:isBookmarkAutoText(bm) then
+            note = nil
+        end
+        if chapter == nil then
+            chapter = self.ui.toc:getTocTitleByPage(bm.page)
+        end
+        pageno = self.ui.paging and bm.page or self.document:getPageFromXPointer(bm.page)
+    end
+    if self.ui.paging and bm.pos0 and not bm.pos0.page then
+        -- old single-page reflow highlights do not have page in position
+        bm.pos0.page = bm.page
+        bm.pos1.page = bm.page
+    end
+    if not hl then -- page bookmark or orphaned bookmark
+        hl = {}
+        if bm.highlighted then -- orphaned bookmark
+            hl.drawer = self.view.highlight.saved_drawer
+            hl.color = self.view.highlight.saved_color
+            if self.ui.paging then
+                if bm.pos0.page == bm.pos1.page then
+                    hl.pboxes = self.document:getPageBoxesFromPositions(bm.page, bm.pos0, bm.pos1)
+                else -- multi-page highlight, restore the first box only
+                    hl.pboxes = self.document:getPageBoxesFromPositions(bm.page, bm.pos0, bm.pos0)
+                end
+            end
+        end
+    end
+    return { -- annotation
+        datetime    = bm.datetime, -- creation time, not changeable
+        drawer      = hl.drawer,   -- highlight drawer
+        color       = hl.color,    -- highlight color
+        text        = bm.notes,    -- highlighted text, editable
+        text_edited = hl.edited,   -- true if highlighted text has been edited
+        note        = note,        -- user's note, editable
+        chapter     = chapter,     -- book chapter title
+        pageno      = pageno,      -- book page number
+        page        = bm.page,     -- highlight location, xPointer or number (pdf)
+        pos0        = bm.pos0,     -- highlight start position, xPointer (== page) or table (pdf)
+        pos1        = bm.pos1,     -- highlight end position, xPointer or table (pdf)
+        pboxes      = hl.pboxes,   -- pdf pboxes, used only and changeable by addMarkupAnnotation
+        ext         = hl.ext,      -- pdf multi-page highlight
+    }
+end
+
+function ReaderAnnotation:getHighlightByDatetime(highlights, datetime)
+    for pageno, page_highlights in pairs(highlights) do
+        for _, highlight in ipairs(page_highlights) do
+            if highlight.datetime == datetime then
+                return highlight, pageno
+            end
+        end
+    end
+end
+
+function ReaderAnnotation:getAnnotationsFromBookmarksHighlights(bookmarks, highlights, init)
+    local annotations = {}
+    for i = #bookmarks, 1, -1 do
+        table.insert(annotations, self:buildAnnotation(bookmarks[i], highlights, init))
+    end
+    if init then
+        self:sortItems(annotations)
+    end
+    return annotations
+end
+
+function ReaderAnnotation:onReadSettings(config)
+    local annotations = config:readSetting("annotations")
+    if annotations then
+        -- KOHighlights may set this key when it has merged annotations from different sources:
+        -- we want to make sure they are updated and sorted
+        local needs_update = config:isTrue("annotations_externally_modified")
+        local needs_sort -- if incompatible annotations were built of old highlights/bookmarks
+        -- Annotation formats in crengine and mupdf are incompatible.
+        local has_annotations = #annotations > 0
+        local annotations_type = has_annotations and type(annotations[1].page)
+        if self.ui.rolling and annotations_type ~= "string" then -- incompatible format loaded, or empty
+            if has_annotations then -- backup incompatible format if not empty
+                config:saveSetting("annotations_paging", annotations)
+            end
+             -- load compatible format
+            annotations = config:readSetting("annotations_rolling") or {}
+            config:delSetting("annotations_rolling")
+            needs_sort = true
+        elseif self.ui.paging and annotations_type ~= "number" then
+            if has_annotations then
+                config:saveSetting("annotations_rolling", annotations)
+            end
+            annotations = config:readSetting("annotations_paging") or {}
+            config:delSetting("annotations_paging")
+            needs_sort = true
+        end
+        self.annotations = annotations
+        if needs_update or needs_sort then
+            if self.ui.rolling then
+                self.ui:registerPostInitCallback(function()
+                    self:updatedAnnotations(needs_update, needs_sort)
+                end)
+            else
+                self:updatedAnnotations(needs_update, needs_sort)
+            end
+            config:delSetting("annotations_externally_modified")
+        end
+    else -- first run
+        if self.ui.rolling then
+            self.ui:registerPostInitCallback(function()
+                self:migrateToAnnotations(config)
+            end)
+        else
+            self:migrateToAnnotations(config)
+        end
+    end
+end
+
+function ReaderAnnotation:migrateToAnnotations(config)
+    local bookmarks = config:readSetting("bookmarks") or {}
+    local highlights = config:readSetting("highlight") or {}
+
+    if config:hasNot("highlights_imported") then
+        -- before 2014, saved highlights were not added to bookmarks when they were created.
+        for page, hls in pairs(highlights) do
+            for _, hl in ipairs(hls) do
+                local hl_page = self.ui.paging and page or hl.pos0
+                -- highlights saved by some old versions don't have pos0 field
+                -- we just ignore those highlights
+                if hl_page then
+                    local item = {
+                        datetime    = hl.datetime,
+                        highlighted = true,
+                        notes       = hl.text,
+                        page        = hl_page,
+                        pos0        = hl.pos0,
+                        pos1        = hl.pos1,
+                    }
+                    if self.ui.paging then
+                        item.pos0.page = page
+                        item.pos1.page = page
+                    end
+                    table.insert(bookmarks, item)
+                end
+            end
+        end
+    end
+
+    -- Bookmarks/highlights formats in crengine and mupdf are incompatible.
+    local has_bookmarks = #bookmarks > 0
+    local bookmarks_type = has_bookmarks and type(bookmarks[1].page)
+    if self.ui.rolling then
+        if bookmarks_type == "string" then -- compatible format loaded, check for incompatible old backup
+            if config:has("bookmarks_paging") then -- save incompatible old backup
+                local bookmarks_paging = config:readSetting("bookmarks_paging")
+                local highlights_paging = config:readSetting("highlight_paging")
+                local annotations = self:getAnnotationsFromBookmarksHighlights(bookmarks_paging, highlights_paging)
+                config:saveSetting("annotations_paging", annotations)
+                config:delSetting("bookmarks_paging")
+                config:delSetting("highlight_paging")
+            end
+        else -- incompatible format loaded, or empty
+            if has_bookmarks then -- save incompatible format if not empty
+                local annotations = self:getAnnotationsFromBookmarksHighlights(bookmarks, highlights)
+                config:saveSetting("annotations_paging", annotations)
+            end
+            -- load compatible format
+            bookmarks = config:readSetting("bookmarks_rolling") or {}
+            highlights = config:readSetting("highlight_rolling") or {}
+            config:delSetting("bookmarks_rolling")
+            config:delSetting("highlight_rolling")
+        end
+    else -- self.ui.paging
+        if bookmarks_type == "number" then
+            if config:has("bookmarks_rolling") then
+                local bookmarks_rolling = config:readSetting("bookmarks_rolling")
+                local highlights_rolling = config:readSetting("highlight_rolling")
+                local annotations = self:getAnnotationsFromBookmarksHighlights(bookmarks_rolling, highlights_rolling)
+                config:saveSetting("annotations_rolling", annotations)
+                config:delSetting("bookmarks_rolling")
+                config:delSetting("highlight_rolling")
+            end
+        else
+            if has_bookmarks then
+                local annotations = self:getAnnotationsFromBookmarksHighlights(bookmarks, highlights)
+                config:saveSetting("annotations_rolling", annotations)
+            end
+            bookmarks = config:readSetting("bookmarks_paging") or {}
+            highlights = config:readSetting("highlight_paging") or {}
+            config:delSetting("bookmarks_paging")
+            config:delSetting("highlight_paging")
+        end
+    end
+
+    self.annotations = self:getAnnotationsFromBookmarksHighlights(bookmarks, highlights, true)
+end
+
+function ReaderAnnotation:onDocumentRerendered()
+    self.needs_update = true
+end
+
+function ReaderAnnotation:onCloseDocument()
+    self:updatePageNumbers()
+end
+
+function ReaderAnnotation:onSaveSettings()
+    self:updatePageNumbers()
+    self.ui.doc_settings:saveSetting("annotations", self.annotations)
+end
+
+-- items handling
+
+function ReaderAnnotation:updatePageNumbers()
+    if self.needs_update and self.ui.rolling then -- triggered by ReaderRolling on document layout change
+        for _, item in ipairs(self.annotations) do
+            item.pageno = self.document:getPageFromXPointer(item.page)
+        end
+    end
+    self.needs_update = nil
+end
+
+function ReaderAnnotation:sortItems(items)
+    if #items > 1 then
+        local sort_func = self.ui.rolling and function(a, b) return self:isItemInPositionOrderRolling(a, b) end
+                                           or function(a, b) return self:isItemInPositionOrderPaging(a, b) end
+        table.sort(items, sort_func)
+    end
+end
+
+function ReaderAnnotation:updatedAnnotations(needs_update, needs_sort)
+    if needs_update then
+        self.needs_update = true
+        self:updatePageNumbers()
+        needs_sort = true
+    end
+    if needs_sort then
+        self:sortItems(self.annotations)
+    end
+end
+
+function ReaderAnnotation:updateItemByXPointer(item)
+    -- called by ReaderRolling:checkXPointersAndProposeDOMVersionUpgrade()
+    local chapter = self.ui.toc:getTocTitleByPage(item.page)
+    if chapter == "" then
+        chapter = nil
+    end
+    if not item.drawer then -- page bookmark
+        item.text = chapter and T(_("in %1"), chapter) or nil
+    end
+    item.chapter = chapter
+    item.pageno = self.document:getPageFromXPointer(item.page)
+end
+
+function ReaderAnnotation:isItemInPositionOrderRolling(a, b)
+    local a_page = self.document:getPageFromXPointer(a.page)
+    local b_page = self.document:getPageFromXPointer(b.page)
+    if a_page == b_page then -- both items in the same page
+        if a.drawer and b.drawer then -- both items are highlights, compare positions
+            local compare_xp = self.document:compareXPointers(a.page, b.page)
+            if compare_xp then
+                if compare_xp == 0 then -- both highlights with the same start, compare ends
+                    compare_xp = self.document:compareXPointers(a.pos1, b.pos1)
+                    if compare_xp then
+                        return compare_xp > 0
+                    end
+                    logger.warn("Invalid xpointer in highlight:", a.pos1, b.pos1)
+                    return true
+                end
+                return compare_xp > 0
+            end
+            -- if compare_xp is nil, some xpointer is invalid and "a" will be sorted first to page 1
+            logger.warn("Invalid xpointer in highlight:", a.page, b.page)
+            return true
+        end
+        return not a.drawer -- have page bookmarks before highlights
+    end
+    return a_page < b_page
+end
+
+function ReaderAnnotation:isItemInPositionOrderPaging(a, b)
+    if a.page == b.page then -- both items in the same page
+        if a.drawer and b.drawer then -- both items are highlights, compare positions
+            local is_reflow = self.document.configurable.text_wrap -- save reflow mode
+            self.document.configurable.text_wrap = 0 -- native positions
+            -- sort start and end positions of each highlight
+            local a_start, a_end, b_start, b_end, result
+            if self.document:comparePositions(a.pos0, a.pos1) > 0 then
+                a_start, a_end = a.pos0, a.pos1
+            else
+                a_start, a_end = a.pos1, a.pos0
+            end
+            if self.document:comparePositions(b.pos0, b.pos1) > 0 then
+                b_start, b_end = b.pos0, b.pos1
+            else
+                b_start, b_end = b.pos1, b.pos0
+            end
+            -- compare start positions
+            local compare_pos = self.document:comparePositions(a_start, b_start)
+            if compare_pos == 0 then -- both highlights with the same start, compare ends
+                result = self.document:comparePositions(a_end, b_end) > 0
+            else
+                result = compare_pos > 0
+            end
+            self.document.configurable.text_wrap = is_reflow -- restore reflow mode
+            return result
+        end
+        return not a.drawer -- have page bookmarks before highlights
+    end
+    return a.page < b.page
+end
+
+function ReaderAnnotation:getItemIndex(item, no_binary)
+    local doesMatch
+    if item.datetime then
+        doesMatch = function(a, b)
+            return a.datetime == b.datetime
+        end
+    else
+        if self.ui.rolling then
+            doesMatch = function(a, b)
+                if a.text ~= b.text or a.pos0 ~= b.pos0 or a.pos1 ~= b.pos1 then
+                    return false
+                end
+                return true
+            end
+        else
+            doesMatch = function(a, b)
+                if a.text ~= b.text or a.pos0.page ~= b.pos0.page
+                                    or a.pos0.x ~= b.pos0.x or a.pos1.x ~= b.pos1.x
+                                    or a.pos0.y ~= b.pos0.y or a.pos1.y ~= b.pos1.y then
+                    return false
+                end
+                return true
+            end
+        end
+    end
+
+    if not no_binary then
+        local isInOrder = self.ui.rolling and self.isItemInPositionOrderRolling or self.isItemInPositionOrderPaging
+        local _start, _end, _middle = 1, #self.annotations
+        while _start <= _end do
+            _middle = bit.rshift(_start + _end, 1)
+            local v = self.annotations[_middle]
+            if doesMatch(item, v) then
+                return _middle
+            elseif isInOrder(self, item, v) then
+                _end = _middle - 1
+            else
+                _start = _middle + 1
+            end
+        end
+    end
+
+    for i, v in ipairs(self.annotations) do
+        if doesMatch(item, v) then
+            return i
+        end
+    end
+end
+
+function ReaderAnnotation:getInsertionIndex(item)
+    local isInOrder = self.ui.rolling and self.isItemInPositionOrderRolling or self.isItemInPositionOrderPaging
+    local _start, _end, _middle, direction = 1, #self.annotations, 1, 0
+    while _start <= _end do
+        _middle = bit.rshift(_start + _end, 1)
+        if isInOrder(self, item, self.annotations[_middle]) then
+            _end, direction = _middle - 1, 0
+        else
+            _start, direction = _middle + 1, 1
+        end
+    end
+    return _middle + direction
+end
+
+function ReaderAnnotation:addItem(item)
+    item.datetime = os.date("%Y-%m-%d %H:%M:%S")
+    item.pageno = self.ui.paging and item.page or self.document:getPageFromXPointer(item.page)
+    local index = self:getInsertionIndex(item)
+    table.insert(self.annotations, index, item)
+    return index
+end
+
+-- info
+
+function ReaderAnnotation:hasAnnotations()
+    return #self.annotations > 0
+end
+
+function ReaderAnnotation:getNumberOfAnnotations()
+    return #self.annotations
+end
+
+function ReaderAnnotation:getNumberOfHighlightsAndNotes() -- for Statistics plugin
+    local highlights = 0
+    local notes = 0
+    for _, item in ipairs(self.annotations) do
+        if item.drawer then
+            if item.note then
+                notes = notes + 1
+            else
+                highlights = highlights + 1
+            end
+        end
+    end
+    return highlights, notes
+end
+
+return ReaderAnnotation

frontend/apps/reader/modules/readerconfig.lua

@@ -158,11 +158,11 @@ function ReaderConfig:onSwipeShowConfigMenu(ges)
     end
 end
 
+-- For some reason, things are fine and dandy without any of this for rotations, but we need it for actual resizes...
 function ReaderConfig:onSetDimensions(dimen)
-    -- since we cannot redraw config_dialog with new size, we close
-    -- the old one on screen size change
     if self.config_dialog then
-        self.config_dialog:closeDialog()
+        -- init basically calls update & initGesListener and nothing else, which is exactly what we want.
+        self.config_dialog:init()
     end
 end
 

frontend/apps/reader/modules/readercoptlistener.lua

@@ -13,14 +13,11 @@ local ReaderCoptListener = EventListener:extend{}
 local CRE_HEADER_DEFAULT_SIZE = 20
 
 function ReaderCoptListener:onReadSettings(config)
-    local view_mode = config:readSetting("copt_view_mode")
-                   or G_reader_settings:readSetting("copt_view_mode")
-                   or 0 -- default to "page" mode
-    local view_mode_name = view_mode == 0 and "page" or "scroll"
+    local view_mode_name = self.document.configurable.view_mode == 0 and "page" or "scroll"
     -- Let crengine know of the view mode before rendering, as it can
     -- cause a rendering change (2-pages would become 1-page in
     -- scroll mode).
-    self.ui.document:setViewMode(view_mode_name)
+    self.document:setViewMode(view_mode_name)
     -- ReaderView is the holder of the view_mode state
     self.view.view_mode = view_mode_name
 
@@ -35,20 +32,24 @@ function ReaderCoptListener:onReadSettings(config)
     self.battery_percent = G_reader_settings:readSetting("cre_header_battery_percent", 0)
     self.chapter_marks = G_reader_settings:readSetting("cre_header_chapter_marks", 1)
 
-    self.ui.document._document:setIntProperty("window.status.title", self.title)
-    self.ui.document._document:setIntProperty("window.status.clock", self.clock)
-    self.ui.document._document:setIntProperty("window.status.pos.page.number", self.page_number)
-    self.ui.document._document:setIntProperty("window.status.pos.page.count", self.page_count)
-    self.ui.document._document:setIntProperty("crengine.page.header.chapter.marks", self.chapter_marks)
-    self.ui.document._document:setIntProperty("window.status.battery", self.battery)
-    self.ui.document._document:setIntProperty("window.status.battery.percent", self.battery_percent)
-    self.ui.document._document:setIntProperty("window.status.pos.percent", self.reading_percent)
+    self.document._document:setIntProperty("window.status.title", self.title)
+    self.document._document:setIntProperty("window.status.clock", self.clock)
+    self.document._document:setIntProperty("window.status.pos.page.number", self.page_number)
+    self.document._document:setIntProperty("window.status.pos.page.count", self.page_count)
+    self.document._document:setIntProperty("crengine.page.header.chapter.marks", self.chapter_marks)
+    self.document._document:setIntProperty("window.status.battery", self.battery)
+    self.document._document:setIntProperty("window.status.battery.percent", self.battery_percent)
+    self.document._document:setIntProperty("window.status.pos.percent", self.reading_percent)
+
+    -- We will build the top status bar page info string ourselves,
+    -- if we have to display any chunk of it
+    self.page_info_override = self.page_number == 1 or self.page_count == 1 or self.reading_percent == 1
+    self.document:setPageInfoOverride("") -- an empty string lets crengine display its own page info
 
     self:onTimeFormatChanged()
 
     -- Enable or disable crengine header status line (note that for crengine, 0=header enabled, 1=header disabled)
-    local status_line = config:readSetting("copt_status_line") or G_reader_settings:readSetting("copt_status_line", 1)
-    self.ui:handleEvent(Event:new("SetStatusLine", status_line))
+    self.ui:handleEvent(Event:new("SetStatusLine", self.document.configurable.status_line))
 
     self.old_battery_level = self.ui.rolling:updateBatteryState()
 
@@ -68,6 +69,101 @@ function ReaderCoptListener:onReadSettings(config)
     self:rescheduleHeaderRefreshIfNeeded() -- schedule (or not) first refresh
 end
 
+function ReaderCoptListener:onReaderReady()
+    -- custom metadata support for alt status bar and cre synthetic cover
+    for prop_key in pairs(self.document.prop_to_cre_prop) do
+        local orig_prop_value = self.ui.doc_settings:readSetting(prop_key)
+        local custom_prop_key = prop_key == "title" and "display_title" or prop_key
+        local custom_prop_value = self.ui.doc_props[custom_prop_key]
+        if custom_prop_value ~= orig_prop_value then
+            self.document:setAltDocumentProp(prop_key, custom_prop_value)
+        end
+    end
+end
+
+function ReaderCoptListener:updatePageInfoOverride(pageno)
+    if not (self.document.configurable.status_line == 0 and self.view.view_mode == "page" and self.page_info_override) then
+        return
+    end
+    -- There are a few cases where we may not be updated on change, at least:
+    -- - when toggling ReaderPageMap's "Use reference page numbers"
+    -- - when changing footer's nb of digits after decimal point
+    -- but we will update on next page turn. Let's not bother.
+
+    local page_pre = ""
+    local page_number = pageno
+    local page_sep = " / "
+    local page_count = self.ui.document:getPageCount()
+    local page_post = ""
+    local percentage = page_number / page_count
+    local percentage_pre = ""
+    local percentage_post = ""
+    -- Let's use the same setting for nb of digits after decimal point as configured for the footer
+    local percentage_digits =  self.ui.view.footer.settings.progress_pct_format
+    local percentage_fmt = "%." .. percentage_digits .. "f%%"
+
+    -- We want the same output as with ReaderFooter's page_progress() and percentage()
+    -- but here each item (page number, page counte, percentage) is individually toggable,
+    -- so try to get something that make sense when not all are enabled
+    if self.ui.pagemap and self.ui.pagemap:wantsPageLabels() then
+        -- These become strings here
+        page_number = self.ui.pagemap:getCurrentPageLabel(true)
+        page_count = self.ui.pagemap:getLastPageLabel(true)
+    elseif self.ui.document:hasHiddenFlows() then
+        local flow = self.ui.document:getPageFlow(pageno)
+        page_number = tostring(self.ui.document:getPageNumberInFlow(pageno))
+        page_count = tostring(self.ui.document:getTotalPagesInFlow(flow))
+        percentage = page_number / page_count
+        if flow == 0 then
+            page_sep = " // "
+        else
+            page_pre = "["
+            page_post = "]"..tostring(flow)
+            percentage_pre = "["
+            percentage_post = "]"
+        end
+    end
+
+    local page_info = ""
+    if self.page_number or self.page_count then
+        page_info = page_info .. page_pre
+        if self.page_number then
+            page_info = page_info .. page_number
+            if self.page_count then
+                page_info = page_info .. page_sep
+            end
+        end
+        if self.page_count then
+            page_info = page_info .. page_count
+        end
+        page_info = page_info .. page_post
+        if self.reading_percent then
+            page_info = page_info .. "  " -- (double space as done by crengine's own drawing)
+        end
+    end
+    if self.reading_percent then
+        page_info = page_info .. percentage_pre .. percentage_fmt:format(percentage*100) .. percentage_post
+    end
+    self.document:setPageInfoOverride(page_info)
+end
+
+function ReaderCoptListener:onPageUpdate(pageno)
+    self:updatePageInfoOverride(pageno)
+end
+
+function ReaderCoptListener:onPosUpdate(pos, pageno)
+    self:updatePageInfoOverride(pageno)
+end
+
+function ReaderCoptListener:onBookMetadataChanged(prop_updated)
+    -- custom metadata support for alt status bar and cre synthetic cover
+    local prop_key = prop_updated and prop_updated.metadata_key_updated
+    if prop_key and self.document.prop_to_cre_prop[prop_key] then
+        self.document:setAltDocumentProp(prop_key, prop_updated.doc_props[prop_key])
+        self:updateHeader()
+    end
+end
+
 function ReaderCoptListener:onConfigChange(option_name, option_value)
     -- font_size and line_spacing are historically and sadly shared by both mupdf and cre reader modules,
     -- but fortunately they can be distinguished by their different ranges
@@ -77,17 +173,13 @@ function ReaderCoptListener:onConfigChange(option_name, option_value)
     return true
 end
 
-function ReaderCoptListener:onSetFontSize(font_size)
-    self.document.configurable.font_size = font_size
-end
-
 function ReaderCoptListener:onCharging()
     self:headerRefresh()
 end
 ReaderCoptListener.onNotCharging = ReaderCoptListener.onCharging
 
 function ReaderCoptListener:onTimeFormatChanged()
-    self.ui.document._document:setIntProperty("window.status.clock.12hours", G_reader_settings:isTrue("twelve_hour_clock") and 1 or 0)
+    self.document._document:setIntProperty("window.status.clock.12hours", G_reader_settings:isTrue("twelve_hour_clock") and 1 or 0)
 end
 
 function ReaderCoptListener:shouldHeaderBeRepainted()
@@ -107,14 +199,14 @@ end
 
 function ReaderCoptListener:updateHeader()
     -- Have crengine display accurate time and battery on its next drawing
-    self.ui.document:resetBufferCache() -- be sure next repaint is a redrawing
+    self.document:resetBufferCache() -- be sure next repaint is a redrawing
     -- Force a refresh if we're not hidden behind another widget
     if self:shouldHeaderBeRepainted() then
         UIManager:setDirty(self.view.dialog, "ui",
             Geom:new{
                 x = 0, y = 0,
                 w = Device.screen:getWidth(),
-                h = self.ui.document:getHeaderHeight(),
+                h = self.document:getHeaderHeight(),
             }
         )
     end
@@ -176,8 +268,10 @@ ReaderCoptListener.onCloseDocument = ReaderCoptListener.unscheduleHeaderRefresh
 ReaderCoptListener.onSuspend = ReaderCoptListener.unscheduleHeaderRefresh
 
 function ReaderCoptListener:setAndSave(setting, property, value)
-    self.ui.document._document:setIntProperty(property, value)
+    self.document._document:setIntProperty(property, value)
     G_reader_settings:saveSetting(setting, value)
+    self.page_info_override = self.page_number == 1 or self.page_count == 1 or self.reading_percent == 1
+    self.document:setPageInfoOverride("")
     -- Have crengine redraw it (even if hidden by the menu at this time)
     self.ui.rolling:updateBatteryState()
     self:updateHeader()
@@ -198,7 +292,7 @@ function ReaderCoptListener:getAltStatusBarMenu()
         separator = true,
         sub_item_table = {
             {
-                text = _("About alternate status bar"),
+                text = _("About alt status bar"),
                 keep_menu_open = true,
                 callback = function()
                     UIManager:show(InfoMessage:new{
@@ -281,15 +375,18 @@ function ReaderCoptListener:getAltStatusBarMenu()
             },
             {
                 text_func = function()
-                    local status = _("off")
+                    local status = _("Battery status")
                     if self.battery == 1 then
                         if self.battery_percent == 1 then
-                            status = _("percentage")
+                            status = _("Battery status: percentage")
                         else
-                            status = _("icon")
+                            status = _("Battery status: icon")
                         end
                     end
-                    return T(_("Battery status: %1"), status)
+                    return status
+                end,
+                checked_func = function()
+                    return self.battery == 1
                 end,
                 sub_item_table = {
                     {

frontend/apps/reader/modules/readercropping.lua

@@ -21,8 +21,6 @@ function ReaderCropping:onPageCrop(mode)
     -- backup original zoom mode as cropping use "page" zoom mode
     self.orig_zoom_mode = self.view.zoom_mode
     if mode == "auto" then
-        --- @fixme: This is weird. "auto" crop happens to be the default, yet the default zoom mode/genus is "page", not "content".
-        ---         This effectively yields different results whether auto is enabled by default, or toggled at runtime...
         if self.document.configurable.text_wrap ~= 1 then
             self:setCropZoomMode(true)
         end
@@ -57,8 +55,6 @@ function ReaderCropping:onPageCrop(mode)
     -- prepare bottom buttons so we know the size available for the page above it
     local button_table = ButtonTable:new{
         width = Screen:getWidth(),
-        button_font_face = "cfont",
-        button_font_size = 20,
         buttons = {{
             {
                 text = _("Cancel"),

frontend/apps/reader/modules/readerdictionary.lua

@@ -424,7 +424,7 @@ function ReaderDictionary:addToMainMenu(menu_items)
     end
 end
 
-function ReaderDictionary:onLookupWord(word, is_sane, boxes, highlight, link, tweak_buttons_func)
+function ReaderDictionary:onLookupWord(word, is_sane, boxes, highlight, link)
     logger.dbg("dict lookup word:", word, boxes)
     -- escape quotes and other funny characters in word
     word = self:cleanSelection(word, is_sane)
@@ -440,7 +440,7 @@ function ReaderDictionary:onLookupWord(word, is_sane, boxes, highlight, link, tw
 
     -- Wrapped through Trapper, as we may be using Trapper:dismissablePopen() in it
     Trapper:wrap(function()
-        self:stardictLookup(word, self.enabled_dict_names, not disable_fuzzy_search, boxes, link, tweak_buttons_func)
+        self:stardictLookup(word, self.enabled_dict_names, not disable_fuzzy_search, boxes, link)
     end)
     return true
 end
@@ -932,18 +932,12 @@ function ReaderDictionary:startSdcv(word, dict_names, fuzzy_search)
     return results
 end
 
-function ReaderDictionary:stardictLookup(word, dict_names, fuzzy_search, boxes, link, tweak_buttons_func)
+function ReaderDictionary:stardictLookup(word, dict_names, fuzzy_search, boxes, link)
     if word == "" then
         return
     end
 
-    local book_title = self.ui.doc_settings and self.ui.doc_settings:readSetting("doc_props").title or _("Dictionary lookup")
-    if book_title == "" then -- no or empty metadata title
-        if self.ui.document and self.ui.document.file then
-            local directory, filename = util.splitFilePathName(self.ui.document.file) -- luacheck: no unused
-            book_title = util.splitFileNameSuffix(filename)
-        end
-    end
+    local book_title = self.ui.doc_props and self.ui.doc_props.display_title or _("Dictionary lookup")
 
     -- Event for plugin to catch lookup with book title
     self.ui:handleEvent(Event:new("WordLookedUp", word, book_title))
@@ -998,16 +992,15 @@ function ReaderDictionary:stardictLookup(word, dict_names, fuzzy_search, boxes,
         return
     end
 
-    self:showDict(word, tidyMarkup(results), boxes, link, tweak_buttons_func)
+    self:showDict(word, tidyMarkup(results), boxes, link)
 end
 
-function ReaderDictionary:showDict(word, results, boxes, link, tweak_buttons_func)
+function ReaderDictionary:showDict(word, results, boxes, link)
     if results and results[1] then
         logger.dbg("showing quick lookup window", #DictQuickLookup.window_list+1, ":", word, results)
         self.dict_window = DictQuickLookup:new{
             ui = self.ui,
             highlight = self.highlight,
-            tweak_buttons_func = tweak_buttons_func,
             dialog = self.dialog,
             -- original lookup word
             word = word,
@@ -1120,15 +1113,24 @@ function ReaderDictionary:downloadDictionary(dict, download_location, continue)
         --logger.dbg(headers)
         file_size = headers and headers["content-length"]
 
-        UIManager:show(ConfirmBox:new{
-            text =  T(_("Dictionary filesize is %1 (%2 bytes). Continue with download?"), util.getFriendlySize(file_size), util.getFormattedSize(file_size)),
-            ok_text =  _("Download"),
-            ok_callback = function()
-                -- call ourselves with continue = true
-                self:downloadDictionary(dict, download_location, true)
-            end,
-        })
-        return
+        if file_size then
+            UIManager:show(ConfirmBox:new{
+                text =  T(_("Dictionary filesize is %1 (%2 bytes). Continue with download?"), util.getFriendlySize(file_size), util.getFormattedSize(file_size)),
+                ok_text =  _("Download"),
+                ok_callback = function()
+                    -- call ourselves with continue = true
+                    self:downloadDictionary(dict, download_location, true)
+                end,
+            })
+            return
+        else
+            logger.dbg("ReaderDictionary: Request failed; response headers:", headers)
+            UIManager:show(InfoMessage:new{
+                text = _("Failed to fetch dictionary. Are you online?"),
+                --timeout = 3,
+            })
+            return false
+        end
     else
         UIManager:nextTick(function()
             UIManager:show(InfoMessage:new{

frontend/apps/reader/modules/readerdogear.lua

@@ -56,16 +56,9 @@ end
 function ReaderDogear:onReadSettings(config)
     if self.ui.rolling then
         -- Adjust to CreDocument margins (as done in ReaderTypeset)
-        local h_margins = config:readSetting("copt_h_page_margins")
-                       or G_reader_settings:readSetting("copt_h_page_margins")
-                       or G_defaults:readSetting("DCREREADER_CONFIG_H_MARGIN_SIZES_MEDIUM")
-        local t_margin = config:readSetting("copt_t_page_margin")
-                      or G_reader_settings:readSetting("copt_t_page_margin")
-                      or G_defaults:readSetting("DCREREADER_CONFIG_T_MARGIN_SIZES_LARGE")
-        local b_margin = config:readSetting("copt_b_page_margin")
-                      or G_reader_settings:readSetting("copt_b_page_margin")
-                      or G_defaults:readSetting("DCREREADER_CONFIG_B_MARGIN_SIZES_LARGE")
-        local margins = { h_margins[1], t_margin, h_margins[2], b_margin }
+        local configurable = self.ui.document.configurable
+        local margins = { configurable.h_page_margins[1], configurable.t_page_margin,
+                          configurable.h_page_margins[2], configurable.b_page_margin }
         self:onSetPageMargins(margins)
     end
 end

frontend/apps/reader/modules/readerflipping.lua

@@ -26,6 +26,12 @@ function ReaderFlipping:init()
         width = icon_size,
         height = icon_size,
     }
+    self.long_hold_widget = IconWidget:new{
+        icon = "appbar.pokeball",
+        width = icon_size,
+        height = icon_size,
+        alpha = true,
+    }
     icon_size = Screen:scaleBySize(36)
     self.select_mode_widget = IconWidget:new{
         icon = "texture-box",
@@ -93,6 +99,8 @@ function ReaderFlipping:paintTo(bb, x, y)
     elseif self.ui.highlight.select_mode then
         -- highlight select mode
         widget = self.select_mode_widget
+    elseif self.ui.highlight.long_hold_reached then
+        widget = self.long_hold_widget
     elseif self.ui.rolling and self.ui.rolling.rendering_state then
         -- epub rerendering
         widget = self:getRollingRenderingStateIconWidget()

frontend/apps/reader/modules/readerfont.lua

@@ -1,5 +1,4 @@
 local BD = require("ui/bidi")
-local CenterContainer = require("ui/widget/container/centercontainer")
 local ConfirmBox = require("ui/widget/confirmbox")
 local Device = require("device")
 local Event = require("ui/event")
@@ -8,7 +7,6 @@ local FontList = require("fontlist")
 local InfoMessage = require("ui/widget/infomessage")
 local Input = Device.input
 local InputContainer = require("ui/widget/container/inputcontainer")
-local Menu = require("ui/widget/menu")
 local MultiConfirmBox = require("ui/widget/multiconfirmbox")
 local Notification = require("ui/widget/notification")
 local Screen = require("device").screen
@@ -23,12 +21,8 @@ local optionsutil = require("ui/data/optionsutil")
 
 local ReaderFont = InputContainer:extend{
     font_face = nil,
-    font_size = nil,
-    line_space_percent = nil,
     font_menu_title = _("Font"),
     face_table = nil,
-    -- default gamma from crengine's lvfntman.cpp
-    gamma_index = nil,
     steps = {0,1,1,1,1,1,2,2,2,3,3,3,4,4,5},
 }
 
@@ -146,20 +140,19 @@ function ReaderFont:onGesture() end
 
 function ReaderFont:registerKeyEvents()
     if Device:hasKeyboard() then
-        -- add shortcut for keyboard
-        self.key_events = {
-            ShowFontMenu = { { "F" } },
-            IncreaseSize = {
+        if not (Device:hasScreenKB() or Device:hasSymKey()) then
+            -- add shortcut for keyboard
+            self.key_events.IncreaseSize = {
                 { "Shift", Input.group.PgFwd },
                 event = "ChangeSize",
                 args = 0.5
-            },
-            DecreaseSize = {
+            }
+            self.key_events.DecreaseSize = {
                 { "Shift", Input.group.PgBack },
                 event = "ChangeSize",
                 args = -0.5
-            },
-        }
+            }
+        end
     end
 end
 
@@ -175,56 +168,18 @@ function ReaderFont:onReadSettings(config)
                   or self.ui.document.default_font
     self.ui.document:setFontFace(self.font_face)
 
-    self.header_font_face = config:readSetting("header_font_face")
-                         or G_reader_settings:readSetting("header_font")
-                         or self.ui.document.header_font
-    self.ui.document:setHeaderFont(self.header_font_face)
-
-    self.font_size = config:readSetting("font_size")
-                  or G_reader_settings:readSetting("copt_font_size")
-                  or G_defaults:readSetting("DCREREADER_CONFIG_DEFAULT_FONT_SIZE")
-                  or 22
-    self.ui.document:setFontSize(Screen:scaleBySize(self.font_size))
-
-    self.font_base_weight = config:readSetting("font_base_weight")
-                      or G_reader_settings:readSetting("copt_font_base_weight")
-                      or 0
-    self.ui.document:setFontBaseWeight(self.font_base_weight)
-
-    self.font_hinting = config:readSetting("font_hinting")
-                     or G_reader_settings:readSetting("copt_font_hinting")
-                     or 2 -- auto (default in cre.cpp)
-    self.ui.document:setFontHinting(self.font_hinting)
-
-    self.font_kerning = config:readSetting("font_kerning")
-                     or G_reader_settings:readSetting("copt_font_kerning")
-                     or 3 -- harfbuzz (slower, but needed for proper arabic)
-    self.ui.document:setFontKerning(self.font_kerning)
-
-    self.word_spacing = config:readSetting("word_spacing")
-                     or G_reader_settings:readSetting("copt_word_spacing")
-                     or {95, 75}
-    self.ui.document:setWordSpacing(self.word_spacing)
-
-    self.word_expansion = config:readSetting("word_expansion")
-                       or G_reader_settings:readSetting("copt_word_expansion")
-                       or 0
-    self.ui.document:setWordExpansion(self.word_expansion)
-
-    self.cjk_width_scaling = config:readSetting("cjk_width_scaling")
-                       or G_reader_settings:readSetting("copt_cjk_width_scaling")
-                       or 100
-    self.ui.document:setCJKWidthScaling(self.cjk_width_scaling)
-
-    self.line_space_percent = config:readSetting("line_space_percent")
-                           or G_reader_settings:readSetting("copt_line_spacing")
-                           or G_defaults:readSetting("DCREREADER_CONFIG_LINE_SPACE_PERCENT_MEDIUM")
-    self.ui.document:setInterlineSpacePercent(self.line_space_percent)
-
-    self.gamma_index = config:readSetting("gamma_index")
-                    or G_reader_settings:readSetting("copt_font_gamma")
-                    or 15 -- gamma = 1.0
-    self.ui.document:setGammaIndex(self.gamma_index)
+    local header_font = G_reader_settings:readSetting("header_font") or self.ui.document.header_font
+    self.ui.document:setHeaderFont(header_font)
+
+    self.ui.document:setFontSize(Screen:scaleBySize(self.configurable.font_size))
+    self.ui.document:setFontBaseWeight(self.configurable.font_base_weight)
+    self.ui.document:setFontHinting(self.configurable.font_hinting)
+    self.ui.document:setFontKerning(self.configurable.font_kerning)
+    self.ui.document:setWordSpacing(self.configurable.word_spacing)
+    self.ui.document:setWordExpansion(self.configurable.word_expansion)
+    self.ui.document:setCJKWidthScaling(self.configurable.cjk_width_scaling)
+    self.ui.document:setInterlineSpacePercent(self.configurable.line_spacing)
+    self.ui.document:setGammaIndex(self.configurable.font_gamma)
 
     self.font_family_fonts = config:readSetting("font_family_fonts") or {}
     self:updateFontFamilyFonts()
@@ -238,64 +193,34 @@ function ReaderFont:onReadSettings(config)
     end)
 end
 
-function ReaderFont:onShowFontMenu()
-    -- build menu widget
-    local main_menu = Menu:new{
-        title = self.font_menu_title,
-        item_table = self.face_table,
-        width = Screen:getWidth() - 100,
-        height = math.floor(Screen:getHeight() * 0.5),
-        single_line = true,
-        items_per_page = 8,
-        items_font_size = Menu.getItemFontSize(8),
-    }
-    -- build container
-    local menu_container = CenterContainer:new{
-        dimen = Screen:getSize(),
-        main_menu,
-    }
-    main_menu.close_callback = function()
-        UIManager:close(menu_container)
-    end
-    -- show menu
-
-    main_menu.show_parent = menu_container
-
-    UIManager:show(menu_container)
-
-    return true
-end
-
 --[[
     UpdatePos event is used to tell ReaderRolling to update pos.
 --]]
 function ReaderFont:onChangeSize(delta)
-    self.font_size = self.font_size + delta
-    self.ui:handleEvent(Event:new("SetFontSize", self.font_size))
+    self:onSetFontSize(self.configurable.font_size + delta)
     return true
 end
 
-function ReaderFont:onSetFontSize(new_size)
-    if new_size > 255 then new_size = 255 end
-    if new_size < 12 then new_size = 12 end
-
-    self.font_size = new_size
-    self.ui.document:setFontSize(Screen:scaleBySize(new_size))
+function ReaderFont:onSetFontSize(size)
+    size = math.max(12, math.min(size, 255))
+    self.configurable.font_size = size
+    self.ui.document:setFontSize(Screen:scaleBySize(size))
     self.ui:handleEvent(Event:new("UpdatePos"))
-    Notification:notify(T(_("Font size set to: %1."), self.font_size))
+    Notification:notify(T(_("Font size set to: %1."), size))
     return true
 end
 
 function ReaderFont:onSetLineSpace(space)
-    self.line_space_percent = math.min(200, math.max(50, space))
-    self.ui.document:setInterlineSpacePercent(self.line_space_percent)
+    space = math.max(50, math.min(space, 200))
+    self.configurable.line_spacing = space
+    self.ui.document:setInterlineSpacePercent(space)
     self.ui:handleEvent(Event:new("UpdatePos"))
-    Notification:notify(T(_("Line spacing set to: %1%."), self.line_space_percent))
+    Notification:notify(T(_("Line spacing set to: %1%."), space))
     return true
 end
 
 function ReaderFont:onSetFontBaseWeight(weight)
-    self.font_base_weight = weight
+    self.configurable.font_base_weight = weight
     self.ui.document:setFontBaseWeight(weight)
     self.ui:handleEvent(Event:new("UpdatePos"))
     Notification:notify(T(_("Font weight set to: %1."), optionsutil:getOptionText("SetFontBaseWeight", weight)))
@@ -303,7 +228,7 @@ function ReaderFont:onSetFontBaseWeight(weight)
 end
 
 function ReaderFont:onSetFontHinting(mode)
-    self.font_hinting = mode
+    self.configurable.font_hinting = mode
     self.ui.document:setFontHinting(mode)
     self.ui:handleEvent(Event:new("UpdatePos"))
     Notification:notify(T(_("Font hinting set to: %1"), optionsutil:getOptionText("SetFontHinting", mode)))
@@ -311,7 +236,7 @@ function ReaderFont:onSetFontHinting(mode)
 end
 
 function ReaderFont:onSetFontKerning(mode)
-    self.font_kerning = mode
+    self.configurable.font_kerning = mode
     self.ui.document:setFontKerning(mode)
     self.ui:handleEvent(Event:new("UpdatePos"))
     Notification:notify(T(_("Font kerning set to: %1"), optionsutil:getOptionText("SetFontKerning", mode)))
@@ -319,7 +244,7 @@ function ReaderFont:onSetFontKerning(mode)
 end
 
 function ReaderFont:onSetWordSpacing(values)
-    self.word_spacing = values
+    self.configurable.word_spacing = values
     self.ui.document:setWordSpacing(values)
     self.ui:handleEvent(Event:new("UpdatePos"))
     Notification:notify(T(_("Word spacing set to: %1%, %2%"), values[1], values[2]))
@@ -327,7 +252,7 @@ function ReaderFont:onSetWordSpacing(values)
 end
 
 function ReaderFont:onSetWordExpansion(value)
-    self.word_expansion = value
+    self.configurable.word_expansion = value
     self.ui.document:setWordExpansion(value)
     self.ui:handleEvent(Event:new("UpdatePos"))
     Notification:notify(T(_("Word expansion set to: %1%."), value))
@@ -335,7 +260,7 @@ function ReaderFont:onSetWordExpansion(value)
 end
 
 function ReaderFont:onSetCJKWidthScaling(value)
-    self.cjk_width_scaling = value
+    self.configurable.cjk_width_scaling = value
     self.ui.document:setCJKWidthScaling(value)
     self.ui:handleEvent(Event:new("UpdatePos"))
     Notification:notify(T(_("CJK width scaling set to: %1%."), value))
@@ -343,8 +268,8 @@ function ReaderFont:onSetCJKWidthScaling(value)
 end
 
 function ReaderFont:onSetFontGamma(gamma)
-    self.gamma_index = gamma
-    self.ui.document:setGammaIndex(self.gamma_index)
+    self.configurable.font_gamma = gamma
+    self.ui.document:setGammaIndex(gamma)
     local gamma_level = self.ui.document:getGammaLevel()
     self.ui:handleEvent(Event:new("RedrawCurrentView"))
     Notification:notify(T(_("Font gamma set to: %1."), gamma_level))
@@ -353,16 +278,6 @@ end
 
 function ReaderFont:onSaveSettings()
     self.ui.doc_settings:saveSetting("font_face", self.font_face)
-    self.ui.doc_settings:saveSetting("header_font_face", self.header_font_face)
-    self.ui.doc_settings:saveSetting("font_size", self.font_size)
-    self.ui.doc_settings:saveSetting("font_base_weight", self.font_base_weight)
-    self.ui.doc_settings:saveSetting("font_hinting", self.font_hinting)
-    self.ui.doc_settings:saveSetting("font_kerning", self.font_kerning)
-    self.ui.doc_settings:saveSetting("word_spacing", self.word_spacing)
-    self.ui.doc_settings:saveSetting("word_expansion", self.word_expansion)
-    self.ui.doc_settings:saveSetting("cjk_width_scaling", self.cjk_width_scaling)
-    self.ui.doc_settings:saveSetting("line_space_percent", self.line_space_percent)
-    self.ui.doc_settings:saveSetting("gamma_index", self.gamma_index)
     self.ui.doc_settings:saveSetting("font_family_fonts", self.font_family_fonts)
 end
 
@@ -382,7 +297,7 @@ function ReaderFont:makeDefault(face, is_monospace, touchmenu_instance)
             -- to be set as a fallback font, and allow it to be set as the
             -- default monospace font.
             UIManager:show(MultiConfirmBox:new{
-                text = T(_("Would you like %1 to be used as the default font (★), or the monospace font (\u{1F13C})?"), face),
+                text = T(_("Would you like %1 to be used as the default font (★), or the monospace font (🄼)?"), face), -- [M] is U+1F13C
                 choice1_text = _("Default"),
                 choice1_callback = function()
                     G_reader_settings:saveSetting("cre_font", face)
@@ -487,8 +402,8 @@ local FONT_FAMILIES = {
     -- On 2nd page
     { "cursive", _("Cursive") },
     { "fantasy", _("Fantasy") },
-    { "emoji", _("Emoji \u{1F60A}") },
-    { "fangsong", _("Fang Song \u{4EFF}\u{5B8B}") },
+    { "emoji", _("Emoji") .. " 😊" }, -- U+1F60A
+    { "fangsong", _("Fang Song") .. " 仿宋" }, -- U+4EFF U+5B8B
     { "math", _("Math") },
 }
 

frontend/apps/reader/modules/readergoto.lua

@@ -28,12 +28,7 @@ function ReaderGoto:addToMainMenu(menu_items)
 end
 
 function ReaderGoto:onShowGotoDialog()
-    local curr_page
-    if self.document.info.has_pages then
-        curr_page = self.ui.paging.current_page
-    else
-        curr_page = self.document:getCurrentPage()
-    end
+    local curr_page = self.ui:getCurrentPage()
     local input_hint
     if self.ui.pagemap and self.ui.pagemap:wantsPageLabels() then
         input_hint = T("@%1 (%2 - %3)", self.ui.pagemap:getCurrentPageLabel(true),
@@ -58,16 +53,7 @@ x for an absolute page number
                     text = _("Skim"),
                     callback = function()
                         self:close()
-                        self.skimto = SkimToWidget:new{
-                            document = self.document,
-                            ui = self.ui,
-                            callback_switch_to_goto = function()
-                                UIManager:close(self.skimto)
-                                self:onShowGotoDialog()
-                            end,
-                        }
-                        UIManager:show(self.skimto)
-
+                        self:onShowSkimtoDialog()
                     end,
                 },
                 {
@@ -195,4 +181,28 @@ function ReaderGoto:onGoToEnd()
     return true
 end
 
+function ReaderGoto:onGoToRandomPage()
+    local page_count = self.document:getPageCount()
+    if page_count == 1 then return true end
+    local current_page = self.ui:getCurrentPage()
+    if self.pages_pool == nil then
+        self.pages_pool = {}
+    end
+    if #self.pages_pool == 0 or (#self.pages_pool == 1 and self.pages_pool[1] == current_page) then
+        for i = 1, page_count do
+            self.pages_pool[i] = i
+        end
+    end
+    while true do
+        local random_page_idx = math.random(1, #self.pages_pool)
+        local random_page = self.pages_pool[random_page_idx]
+        if random_page ~= current_page then
+            table.remove(self.pages_pool, random_page_idx)
+            self.ui.link:addCurrentLocationToStack()
+            self.ui:handleEvent(Event:new("GotoPage", random_page))
+            return true
+        end
+    end
+end
+
 return ReaderGoto

frontend/apps/reader/modules/readerhandmade.lua

@@ -0,0 +1,719 @@
+local ConfirmBox = require("ui/widget/confirmbox")
+local Device = require("device")
+local Event = require("ui/event")
+local InfoMessage = require("ui/widget/infomessage")
+local InputDialog = require("ui/widget/inputdialog")
+local UIManager = require("ui/uimanager")
+local WidgetContainer = require("ui/widget/container/widgetcontainer")
+local util = require("util")
+local T = require("ffi/util").template
+local _ = require("gettext")
+
+local ReaderHandMade = WidgetContainer:extend{
+    custom_toc_symbol = "\u{EAEC}", -- used in a few places
+}
+
+function ReaderHandMade:init()
+    self.ui.menu:registerToMainMenu(self)
+end
+
+function ReaderHandMade:onReadSettings(config)
+    self.toc_enabled = config:isTrue("handmade_toc_enabled")
+    self.toc_edit_enabled = config:nilOrTrue("handmade_toc_edit_enabled")
+    self.toc = config:readSetting("handmade_toc") or {}
+    self.flows_enabled = config:isTrue("handmade_flows_enabled")
+    self.flows_edit_enabled = config:nilOrTrue("handmade_flows_edit_enabled")
+    self.flow_points = config:readSetting("handmade_flow_points") or {}
+    self.inactive_flow_points = {}
+
+    -- Don't mess toc and flow_points made on that document if saved when
+    -- we were using a different engine - backup them if that's the case.
+    if #self.toc > 0 then
+        local has_xpointers = self.toc[1].xpointer ~= nil
+        if self.ui.rolling and not has_xpointers then
+            config:saveSetting("handmade_toc_paging", self.toc)
+            self.toc = config:readSetting("handmade_toc_rolling") or {}
+            config:delSetting("handmade_toc_rolling")
+        elseif self.ui.paging and has_xpointers then
+            config:saveSetting("handmade_toc_rolling", self.toc)
+            self.toc = config:readSetting("handmade_toc_paging") or {}
+            config:delSetting("handmade_toc_paging")
+        end
+    else
+        if self.ui.rolling and config:has("handmade_toc_rolling") then
+            self.toc = config:readSetting("handmade_toc_rolling")
+            config:delSetting("handmade_toc_rolling")
+        elseif self.ui.paging and config:has("handmade_toc_paging") then
+            self.toc = config:readSetting("handmade_toc_paging")
+            config:delSetting("handmade_toc_paging")
+        end
+    end
+    if #self.flow_points > 0 then
+        local has_xpointers = self.flow_points[1].xpointer ~= nil
+        if self.ui.rolling and not has_xpointers then
+            config:saveSetting("handmade_flow_points_paging", self.flow_points)
+            self.flow_points = config:readSetting("handmade_flow_points_rolling") or {}
+            config:delSetting("handmade_flow_points_rolling")
+        elseif self.ui.paging and has_xpointers then
+            config:saveSetting("handmade_flow_points_rolling", self.flow_points)
+            self.flow_points = config:readSetting("handmade_flow_points_paging") or {}
+            config:delSetting("handmade_flow_points_paging")
+        end
+    else
+        if self.ui.rolling and config:has("handmade_flow_points_rolling") then
+            self.flow_points = config:readSetting("handmade_flow_points_rolling")
+            config:delSetting("handmade_flow_points_rolling")
+        elseif self.ui.paging and config:has("handmade_flow_points_paging") then
+            self.flow_points = config:readSetting("handmade_flow_points_paging")
+            config:delSetting("handmade_flow_points_paging")
+        end
+    end
+end
+
+function ReaderHandMade:onSaveSettings()
+    self.ui.doc_settings:saveSetting("handmade_toc_enabled", self.toc_enabled)
+    self.ui.doc_settings:saveSetting("handmade_toc_edit_enabled", self.toc_edit_enabled)
+    if #self.toc > 0 then
+        self.ui.doc_settings:saveSetting("handmade_toc", self.toc)
+    else
+        self.ui.doc_settings:delSetting("handmade_toc")
+    end
+    self.ui.doc_settings:saveSetting("handmade_flows_enabled", self.flows_enabled)
+    self.ui.doc_settings:saveSetting("handmade_flows_edit_enabled", self.flows_edit_enabled)
+    if #self.flow_points > 0 then
+        self.ui.doc_settings:saveSetting("handmade_flow_points", self.flow_points)
+    else
+        self.ui.doc_settings:delSetting("handmade_flow_points")
+    end
+end
+
+function ReaderHandMade:isHandmadeTocEnabled()
+    return self.toc_enabled
+end
+
+function ReaderHandMade:isHandmadeTocEditEnabled()
+    return self.toc_edit_enabled
+end
+
+function ReaderHandMade:isHandmadeHiddenFlowsEnabled()
+    -- Even if currently empty, we return true, which allows showing '//' in
+    -- the footer and let know hidden flows are enabled.
+    return self.flows_enabled
+end
+
+function ReaderHandMade:isHandmadeHiddenFlowsEditEnabled()
+    return self.flows_edit_enabled
+end
+
+function ReaderHandMade:onToggleHandmadeToc()
+    self.toc_enabled = not self.toc_enabled
+    self:setupToc()
+    -- Have footer updated, so we may see this took effect
+    self.view.footer:onUpdateFooter(self.view.footer_visible)
+end
+
+function ReaderHandMade:onToggleHandmadeFlows()
+    self.flows_enabled = not self.flows_enabled
+    self:setupFlows()
+    -- Have footer updated, so we may see this took effect
+    self.view.footer:onUpdateFooter(self.view.footer_visible)
+end
+
+function ReaderHandMade:addToMainMenu(menu_items)
+    -- As it's currently impossible to create custom hidden flows on non-touch, and really impractical to create a custom toc, it's better hide these features completely for now.
+    if not Device:isTouchDevice() then
+        return
+    end
+    menu_items.handmade_toc = {
+        text = _("Custom table of contents") .. " " .. self.custom_toc_symbol,
+        checked_func = function() return self.toc_enabled end,
+        callback = function()
+            self:onToggleHandmadeToc()
+        end,
+    }
+    menu_items.handmade_hidden_flows = {
+        text = _("Custom hidden flows"),
+        checked_func = function() return self.flows_enabled end,
+        callback = function()
+            self:onToggleHandmadeFlows()
+        end,
+    }
+    --[[ Not yet implemented
+    menu_items.handmade_page_numbers = {
+        text = _("Custom page numbers"),
+        checked_func = function() return false end,
+        callback = function()
+        end,
+    }
+    ]]--
+    menu_items.handmade_settings = {
+        text = _("Custom layout features"),
+        sub_item_table_func = function()
+            return {
+                {
+                    text = _("About custom table of contents") .. " " .. self.custom_toc_symbol,
+                    callback = function()
+                        UIManager:show(InfoMessage:new{
+                            text = _([[
+If the book has no table of contents or you would like to substitute it with your own, you can create a custom TOC. The original TOC (if available) will not be altered.
+
+You can create, edit and remove chapters:
+- in Page browser, by long-pressing on a thumbnail;
+- on a book page, by selecting some text to be used as the chapter title.
+(Once you're done building it and don't want to see the buttons anymore, you can disable Edit mode.)
+
+This custom table of contents is currently limited to a single level and can't have sub-chapters.]])
+                        })
+                    end,
+                    keep_menu_open = true,
+                },
+                {
+                    text = _("Edit mode"),
+                    enabled_func = function()
+                        return self:isHandmadeTocEnabled()
+                    end,
+                    checked_func = function()
+                        return self:isHandmadeTocEditEnabled()
+                    end,
+                    callback = function()
+                        self.toc_edit_enabled = not self.toc_edit_enabled
+                        self:updateHighlightDialog()
+                    end,
+                },
+                --[[ Not yet implemented
+                {
+                    text = _("Add multiple chapter start page numbers"),
+                },
+                ]]--
+                {
+                    text = _("Clear custom table of contents"),
+                    enabled_func = function()
+                        return #self.toc > 0
+                    end,
+                    callback = function(touchmenu_instance)
+                        UIManager:show(ConfirmBox:new{
+                            text = _("Are you sure you want to clear your custom table of contents?"),
+                            ok_callback = function()
+                                self.toc = {}
+                                self.ui:handleEvent(Event:new("UpdateToc"))
+                                -- The footer may be visible, so have it update its chapter related items
+                                self.view.footer:onUpdateFooter(self.view.footer_visible)
+                                if touchmenu_instance then
+                                    touchmenu_instance:updateItems()
+                                end
+                            end,
+                        })
+                    end,
+                    keep_menu_open = true,
+                    separator = true,
+                },
+                {
+                    text = _("About custom hidden flows"),
+                    callback = function()
+                        UIManager:show(InfoMessage:new{
+                            text = _([[
+Custom hidden flows can be created to exclude sections of the book from your normal reading flow:
+- hidden flows will automatically be skipped when turning pages within the regular flow;
+- pages part of hidden flows are assigned distinct page numbers and won't be considered in the various book & chapter progress and time to read features;
+- following direct links to pages in hidden flows will still work, including from the TOC or Book map.
+
+This can be useful to exclude long footnotes or bibliography sections.
+It can also be handy when interested in reading only a subset of a book.
+
+In Page browser, you can long-press on a thumbnail to start a hidden flow or restart the regular flow on this page.
+(Once you're done building it and don't want to see the button anymore, you can disable Edit mode.)
+
+Hidden flows are shown with gray or hatched background in Book map and Page browser.]])
+                        })
+                    end,
+                    keep_menu_open = true,
+                },
+                {
+                    text = _("Edit mode"),
+                    enabled_func = function()
+                        return self:isHandmadeHiddenFlowsEnabled()
+                    end,
+                    checked_func = function()
+                        return self:isHandmadeHiddenFlowsEditEnabled()
+                    end,
+                    callback = function()
+                        self.flows_edit_enabled = not self.flows_edit_enabled
+                    end,
+                },
+                {
+                    text_func = function()
+                        return T(_("Clear inactive marked pages (%1)"), #self.inactive_flow_points)
+                    end,
+                    enabled_func = function()
+                        return #self.inactive_flow_points > 0
+                    end,
+                    callback = function(touchmenu_instance)
+                        UIManager:show(ConfirmBox:new{
+                            text = _("Inactive marked pages are pages that you tagged as start hidden flow or restart regular flow, but that other marked pages made them have no effect.\nAre you sure you want to clear them?"),
+                            ok_callback = function()
+                                for i=#self.inactive_flow_points, 1, -1 do
+                                    table.remove(self.flow_points, self.inactive_flow_points[i])
+                                end
+                                self:updateDocFlows()
+                                self.ui:handleEvent(Event:new("UpdateToc"))
+                                self.ui:handleEvent(Event:new("InitScrollPageStates"))
+                                -- The footer may be visible, so have it update its dependant items
+                                self.view.footer:onUpdateFooter(self.view.footer_visible)
+                                if touchmenu_instance then
+                                    touchmenu_instance:updateItems()
+                                end
+                            end,
+                        })
+                    end,
+                    keep_menu_open = true,
+                },
+                {
+                    text = _("Clear all marked pages"),
+                    enabled_func = function()
+                        return #self.flow_points > 0
+                    end,
+                    callback = function(touchmenu_instance)
+                        UIManager:show(ConfirmBox:new{
+                            text = _("Are you sure you want to clear all your custom hidden flows?"),
+                            ok_callback = function()
+                                self.flow_points = {}
+                                self:updateDocFlows()
+                                self.ui:handleEvent(Event:new("UpdateToc"))
+                                self.ui:handleEvent(Event:new("InitScrollPageStates"))
+                                -- The footer may be visible, so have it update its dependant items
+                                self.view.footer:onUpdateFooter(self.view.footer_visible)
+                                if touchmenu_instance then
+                                    touchmenu_instance:updateItems()
+                                end
+                            end,
+                        })
+                    end,
+                    keep_menu_open = true,
+                    separator = true,
+                },
+                --[[ Not yet implemented
+                {
+                    text = _("About custom page numbers"),
+                },
+                {
+                    text = _("Clear custom page numbers"),
+                },
+                ]]--
+            }
+        end,
+    }
+end
+
+function ReaderHandMade:updateHandmagePages()
+    if not self.ui.rolling then
+        return
+    end
+    for _, item in ipairs(self.toc) do
+        item.page = self.document:getPageFromXPointer(item.xpointer)
+    end
+    for _, item in ipairs(self.flow_points) do
+        item.page = self.document:getPageFromXPointer(item.xpointer)
+    end
+end
+
+function ReaderHandMade:onReaderReady()
+    -- Called on load, and with a CRE document when reloading after partial rerendering.
+    -- Notes:
+    -- - ReaderFooter (from ReaderView) will have its onReaderReady() called before ours,
+    --   and it may fillToc(). So, it may happen that the expensive validateAndFixToc()
+    --   is called twice (first with the original ToC, then with ours).
+    -- - ReaderRolling will have its onReaderReady() called after ours, and if we
+    --   have set up hidden flows, we'll have overriden some documents methods so
+    --   its cacheFlows() is a no-op.
+    self:updateHandmagePages()
+    -- Don't have each of these send their own events: we'll send them once afterwards
+    self:setupFlows(true)
+    self:setupToc(true)
+    -- Now send the events
+    if self.toc_enabled or self.flows_enabled then
+        self.ui:handleEvent(Event:new("UpdateToc"))
+    end
+    if self.flows_enabled then
+        -- Needed to skip hidden flows if PDF in scroll mode
+        self.ui:handleEvent(Event:new("InitScrollPageStates"))
+    end
+end
+
+function ReaderHandMade:onDocumentRerendered()
+    -- Called with CRE document when partial rerendering not enabled
+    self:updateHandmagePages()
+    -- Don't have these send events their own events
+    self:setupFlows(true)
+    self:setupToc(true)
+    -- ReaderToc will process this event just after us, and will
+    -- call its onUpdateToc: we don't need to send it.
+    -- (Also, no need for InitScrollPageStates with CRE.)
+end
+
+function ReaderHandMade:setupToc(no_event)
+    if self.toc_enabled then
+        -- If enabled, plug one method into the document object,
+        -- so it is used instead of the method from its class.
+        self.document.getToc = function(this)
+            -- ReaderToc may add fieds to ToC items: return a copy,
+            -- so the one we will save doesn't get polluted.
+            return util.tableDeepCopy(self.toc)
+        end
+    else
+        -- If disabled, remove our plug so the method from the
+        -- class gets used again.
+        self.document.getToc = nil
+    end
+    self:updateHighlightDialog()
+    if not no_event then
+        self.ui:handleEvent(Event:new("UpdateToc"))
+    end
+end
+
+function ReaderHandMade:updateHighlightDialog()
+    if self.toc_enabled and self.toc_edit_enabled then
+        -- We don't want this button to be the last wide one, and rather
+        -- keep having the Search button being that one: so plug this one
+        -- just before 12_search.
+        self.ui.highlight:addToHighlightDialog("12_0_make_handmade_toc_item", function(this)
+            return {
+                text_func = function()
+                    local selected_text = this.selected_text
+                    local pageno, xpointer
+                    if self.ui.rolling then
+                        xpointer = selected_text.pos0
+                    else
+                        pageno = selected_text.pos0.page
+                    end
+                    local text
+                    if self:hasPageTocItem(pageno, xpointer) then
+                        text = _("Edit TOC chapter")
+                    else
+                        text = _("Start TOC chapter")
+                    end
+                    text = text .. " " .. self.custom_toc_symbol
+                    return text
+                end,
+                callback = function()
+                    local selected_text = this.selected_text
+                    this:onClose()
+                    self:addOrEditPageTocItem(nil, nil, selected_text)
+                end,
+            }
+        end)
+    else
+        self.ui.highlight:removeFromHighlightDialog("12_0_make_handmade_toc_item")
+    end
+end
+
+function ReaderHandMade:_getItemIndex(tab, pageno, xpointer)
+    if not pageno and xpointer then
+        pageno = self.document:getPageFromXPointer(xpointer)
+    end
+    -- (No need to use a binary search, our user made tables should
+    -- not be too large)
+    local matching_idx
+    local insertion_idx = #tab + 1
+    for i, item in ipairs(tab) do
+        if item.page >= pageno then
+            if item.page > pageno then
+                insertion_idx = i
+                break
+            end
+            -- Same page numbers.
+            -- (We can trust page numbers, and only compare xpointers when both
+            -- resolve to the same page.)
+            if xpointer and item.xpointer then
+                local order = self.document:compareXPointers(xpointer, item.xpointer)
+                if order > 0 then -- item.xpointer after xpointer
+                    insertion_idx = i
+                    break
+                elseif order == 0 then
+                    matching_idx = i
+                    break
+                end
+            else
+                matching_idx = i
+                break
+            end
+        end
+    end
+    -- We always return an index, and a boolean stating if this index is a match or not
+    -- (if not, the index is the insertion index if we ever want to insert an item with
+    -- the asked pageno/xpointer)
+    return matching_idx or insertion_idx, matching_idx and true or false
+end
+
+function ReaderHandMade:hasPageTocItem(pageno, xpointer)
+    local _, is_match = self:_getItemIndex(self.toc, pageno, xpointer)
+    return is_match
+end
+
+function ReaderHandMade:addOrEditPageTocItem(pageno, when_updated_callback, selected_text)
+    local xpointer, title
+    if selected_text then
+        -- If we get selected_text, it's from the highlight dialog after text selection
+        title = selected_text.text
+        if self.ui.rolling then
+            xpointer = selected_text.pos0
+            pageno = self.document:getPageFromXPointer(xpointer)
+        else
+            pageno = selected_text.pos0.page
+        end
+    end
+    local idx, item_found = self:_getItemIndex(self.toc, pageno, xpointer)
+    local item
+    if item_found then
+        -- Chapter found: it's an update (edit text or remove item)
+        item = self.toc[idx]
+    else
+        -- No chapter starting on this page or at this xpointer:
+        -- we'll add a new item
+        if not xpointer and self.ui.rolling and type(pageno) == "number" then
+            xpointer = self.document:getPageXPointer(pageno)
+        end
+        item = {
+            title = title or "",
+            page = pageno,
+            xpointer = xpointer,
+            depth = 1, -- we only support 1-level chapters to keep the UX simple
+        }
+    end
+    local dialog
+    dialog = InputDialog:new{
+        title = item_found and _("Edit custom TOC chapter") or _("Create new custom ToC chapter"),
+        input = item.title,
+        input_hint = _("TOC chapter title"),
+        description = T(_([[On page %1.]]), pageno),
+        buttons = {
+            {
+                {
+                    text = _("Cancel"),
+                    id = "close",
+                    callback = function()
+                        UIManager:close(dialog)
+                    end,
+                },
+                {
+                    text = item_found and _("Save") or _("Create"),
+                    is_enter_default = true,
+                    callback = function()
+                        item.title = dialog:getInputText()
+                        UIManager:close(dialog)
+                        if not item_found then
+                            table.insert(self.toc, idx, item)
+                        end
+                        self.ui:handleEvent(Event:new("UpdateToc"))
+                        if when_updated_callback then
+                            when_updated_callback()
+                        end
+                    end,
+                },
+            },
+            item_found and {
+                {
+                    text = _("Remove"),
+                    callback = function()
+                        UIManager:close(dialog)
+                        table.remove(self.toc, idx)
+                        self.ui:handleEvent(Event:new("UpdateToc"))
+                        if when_updated_callback then
+                            when_updated_callback()
+                        end
+                    end,
+                },
+                selected_text and
+                    {
+                        text = _("Use selected text"),
+                        callback = function()
+                            -- Just replace the text without saving, to allow editing/fixing it
+                            dialog:setInputText(selected_text.text, nil, false)
+                        end,
+                    } or nil,
+            } or nil,
+        },
+    }
+    UIManager:show(dialog)
+    dialog:onShowKeyboard()
+    return true
+end
+
+function ReaderHandMade:isInHiddenFlow(pageno)
+    local idx, is_match = self:_getItemIndex(self.flow_points, pageno)
+    if is_match then
+        return self.flow_points[idx].hidden
+    else
+        if idx > 1 then
+            return self.flow_points[idx-1].hidden
+        end
+    end
+    -- Before any first flow_point: not hidden
+    return false
+end
+
+function ReaderHandMade:toggleHiddenFlow(pageno)
+    local idx, is_match = self:_getItemIndex(self.flow_points, pageno)
+    if is_match then
+        -- Just remove the item (it feels we can, and that we don't
+        -- have to just toggle its hidden value)
+        table.remove(self.flow_points, idx)
+        self:updateDocFlows()
+        return
+    end
+    local hidden
+    if idx > 1 then
+        local previous_item = self.flow_points[idx-1]
+        hidden = not previous_item.hidden
+    else
+        -- First item, can only start an hidden flow
+        hidden = true
+    end
+    local xpointer
+    if self.ui.rolling and type(pageno) == "number" then
+        xpointer = self.document:getPageXPointer(pageno)
+    end
+    local item = {
+        hidden = hidden,
+        page = pageno,
+        xpointer = xpointer,
+    }
+    table.insert(self.flow_points, idx, item)
+    -- We could remove any followup item(s) with the same hidden state, but by keeping them,
+    -- we allow users to adjust the start of a flow without killing its end. One can clean
+    -- all the unnefective ones via the "Clear inactive marked pages" menu item.
+    self:updateDocFlows()
+end
+
+function ReaderHandMade:updateDocFlows()
+    local flows = {}
+    local inactive_flow_points = {}
+    -- (getPageCount(), needing the document to be fully loaded, is not available
+    -- until ReaderReady, so be sure this is called only after ReaderReady.)
+    local nb_pages = self.document:getPageCount()
+    local nb_hidden_pages = 0
+    local cur_hidden_flow
+    for i, point in ipairs(self.flow_points) do
+        if point.hidden and not cur_hidden_flow then
+            cur_hidden_flow = {point.page, 0}
+        elseif not point.hidden and cur_hidden_flow then
+            local cur_hidden_pages = point.page - cur_hidden_flow[1]
+            if cur_hidden_pages > 0 then
+                cur_hidden_flow[2] = cur_hidden_pages
+                nb_hidden_pages = nb_hidden_pages + cur_hidden_pages
+                table.insert(flows, cur_hidden_flow)
+            end
+            cur_hidden_flow = nil
+        else
+            table.insert(inactive_flow_points, i)
+        end
+    end
+    if cur_hidden_flow then
+        local cur_hidden_pages = nb_pages + 1 - cur_hidden_flow[1]
+        if cur_hidden_pages > 0 then
+            cur_hidden_flow[2] = cur_hidden_pages
+            nb_hidden_pages = nb_hidden_pages + cur_hidden_pages
+            table.insert(flows, cur_hidden_flow)
+        end
+    end
+    local first_linear_page
+    local last_linear_page
+    local prev_flow
+    for i, flow in ipairs(flows) do
+        if not prev_flow or prev_flow[1] + prev_flow[2] < flow[1] then
+            if not first_linear_page and flow[1] > 1 then
+                first_linear_page = prev_flow and prev_flow[1] + prev_flow[2] or 1
+            end
+            last_linear_page = flow[1] - 1
+        end
+        prev_flow = flow
+    end
+    if not prev_flow or prev_flow[1] + prev_flow[2] < nb_pages then
+        last_linear_page = nb_pages
+    end
+    if not first_linear_page then -- no flow met
+        first_linear_page = 1
+    end
+    -- CreDocument adds and item with key [0] with info about the main flow
+    flows[0] = {first_linear_page, nb_pages - nb_hidden_pages}
+    self.last_linear_page = last_linear_page
+    self.flows = flows
+    self.inactive_flow_points = inactive_flow_points
+    -- We plug our flows table into the document, as some code peeks into it
+    self.document.flows = self.flows
+end
+
+function ReaderHandMade:setupFlows(no_event)
+    if self.flows_enabled then
+        self:updateDocFlows()
+        -- If enabled, plug some methods into the document object,
+        -- so they are used instead of the methods from its class.
+        self.document.hasHiddenFlows = function(this)
+            return true
+        end
+        self.document.cacheFlows = function(this)
+            return
+        end
+        self.document.getPageFlow = function(this, page)
+            for i, flow in ipairs(self.flows) do
+                if page < flow[1] then
+                    return 0 -- page is not in a hidden flow
+                end
+                if page < flow[1] + flow[2] then
+                    return i
+                end
+            end
+            return 0
+        end
+        self.document.getFirstPageInFlow = function(this, flow)
+            return self.flows[flow][1]
+        end
+        self.document.getTotalPagesInFlow = function(this, flow)
+            return self.flows[flow][2]
+        end
+        self.document.getPageNumberInFlow = function(this, page)
+            local nb_hidden_pages = 0
+            for i, flow in ipairs(self.flows) do
+                if page < flow[1] then
+                    break -- page is not in a hidden flow
+                end
+                if page < flow[1] + flow[2] then
+                    return page - flow[1] + 1
+                end
+                nb_hidden_pages = nb_hidden_pages + flow[2]
+            end
+            return page - nb_hidden_pages
+        end
+        self.document.getLastLinearPage = function(this)
+            return self.last_linear_page
+        end
+        -- We can reuse as-is these ones from CreDocument, which uses the ones defined above.
+        -- Note: these could probably be rewritten and simplified.
+        local CreDocument = require("document/credocument")
+        self.document.getTotalPagesLeft = CreDocument.getTotalPagesLeft
+        self.document.getNextPage = CreDocument.getNextPage
+        self.document.getPrevPage = CreDocument.getPrevPage
+    else
+        -- Remove all our overrides, so the class methods can be used again
+        self.document.hasHiddenFlows = nil
+        self.document.cacheFlows = nil
+        self.document.getPageFlow = nil
+        self.document.getFirstPageInFlow = nil
+        self.document.getTotalPagesInFlow = nil
+        self.document.getPageNumberInFlow = nil
+        self.document.getLastLinearPage = nil
+        self.document.getTotalPagesLeft = nil
+        self.document.getNextPage = nil
+        self.document.getPrevPage = nil
+        self.document.flows = nil
+        if self.document.cacheFlows then
+            self.document:cacheFlows()
+        end
+    end
+    if not no_event then
+        self.ui:handleEvent(Event:new("UpdateToc"))
+        -- Needed to skip hidden flows if PDF in scroll mode
+        self.ui:handleEvent(Event:new("InitScrollPageStates"))
+    end
+end
+
+return ReaderHandMade

frontend/apps/reader/modules/readerkoptlistener.lua

@@ -21,10 +21,7 @@ function ReaderKoptListener:onReadSettings(config)
     normal_zoom_mode = ReaderZooming.zoom_mode_label[normal_zoom_mode] and normal_zoom_mode or ReaderZooming.DEFAULT_ZOOM_MODE
     self.normal_zoom_mode = normal_zoom_mode
     self:setZoomMode(normal_zoom_mode)
-    self.document.configurable.contrast = config:readSetting("kopt_contrast")
-                                       or G_reader_settings:readSetting("kopt_contrast")
-                                       or 1.0
-    self.ui:handleEvent(Event:new("GammaUpdate", 1/self.document.configurable.contrast))
+    self.ui:handleEvent(Event:new("GammaUpdate", self.document.configurable.contrast))
     -- since K2pdfopt v2.21 negative value of word spacing is also used, for config
     -- compatability we should manually change previous -1 to a more reasonable -0.2
     if self.document.configurable.word_spacing == -1 then

frontend/apps/reader/modules/readerlink.lua

@@ -3,9 +3,10 @@ ReaderLink is an abstraction for document-specific link interfaces.
 ]]
 
 local BD = require("ui/bidi")
-local ButtonDialogTitle = require("ui/widget/buttondialogtitle")
+local ButtonDialog = require("ui/widget/buttondialog")
 local ConfirmBox = require("ui/widget/confirmbox")
 local Device = require("device")
+local DocumentRegistry = require("document/documentregistry")
 local Event = require("ui/event")
 local InfoMessage = require("ui/widget/infomessage")
 local InputContainer = require("ui/widget/container/inputcontainer")
@@ -66,6 +67,7 @@ local ReaderLink = InputContainer:extend{
     location_stack = nil, -- table, per-instance
     forward_location_stack = nil, -- table, per-instance
     _external_link_buttons = nil,
+    supported_external_schemes = {"http", "https"},
 }
 
 function ReaderLink:init()
@@ -117,12 +119,12 @@ function ReaderLink:init()
     end)
     if G_reader_settings:isTrue("opening_page_location_stack") then
         -- Add location at book opening to stack
-        self.ui:registerPostReadyCallback(function()
+        self.ui:registerPostReaderReadyCallback(function()
             self:addCurrentLocationToStack()
         end)
     end
     -- For relative local file links
-    local directory, filename = util.splitFilePathName(self.ui.document.file) -- luacheck: no unused
+    local directory, filename = util.splitFilePathName(self.document.file) -- luacheck: no unused
     self.document_dir = directory
     -- Migrate these old settings to the new common one
     if G_reader_settings:isTrue("tap_link_footnote_popup")
@@ -141,6 +143,7 @@ function ReaderLink:init()
         return {
             text = _("Copy"),
             callback = function()
+                Device.input.setClipboardText(link_url)
                 UIManager:close(this.external_link_dialog)
             end,
         }
@@ -223,10 +226,25 @@ function ReaderLink:init()
     end
 end
 
+function ReaderLink:registerScheme(scheme)
+    table.insert(self.supported_external_schemes, scheme)
+end
+
 function ReaderLink:onGesture() end
 
 function ReaderLink:registerKeyEvents()
-    if Device:hasKeys() then
+    if Device:hasScreenKB() or Device:hasSymKey() then
+        self.key_events.GotoSelectedPageLink = { { "Press" }, event = "GotoSelectedPageLink" }
+        if Device:hasKeyboard() then
+            self.key_events.AddCurrentLocationToStackNonTouch = { { "Shift", "Press" } }
+            self.key_events.SelectNextPageLink = { { "Shift", "LPgFwd" }, event = "SelectNextPageLink" }
+            self.key_events.SelectPrevPageLink = { { "Shift", "LPgBack" }, event = "SelectPrevPageLink" }
+        else
+            self.key_events.AddCurrentLocationToStackNonTouch = { { "ScreenKB", "Press" } }
+            self.key_events.SelectNextPageLink = { { "ScreenKB", "LPgFwd" }, event = "SelectNextPageLink" }
+            self.key_events.SelectPrevPageLink = { { "ScreenKB", "LPgBack" }, event = "SelectPrevPageLink" }
+        end
+    elseif Device:hasKeys() then
         self.key_events = {
             SelectNextPageLink = {
                 { "Tab" },
@@ -234,7 +252,6 @@ function ReaderLink:registerKeyEvents()
             },
             SelectPrevPageLink = {
                 { "Shift", "Tab" },
-                { "Sym", "Tab" }, -- Shift or Sym + Tab
                 event = "SelectPrevPageLink",
             },
             GotoSelectedPageLink = {
@@ -401,7 +418,7 @@ If any of the other Swipe to follow link options is enabled, this will work only
     -- less visual feedback on PDF document of what is a link, or that we just
     -- followed a link, than on EPUB, it's safer to not use them on PDF documents
     -- even if the user enabled these features for EPUB documents).
-    if not self.ui.document.info.has_pages then
+    if self.ui.rolling then
         -- Tap section
         table.insert(menu_items.follow_links.sub_item_table, 2, {
             text = _("Allow larger tap area around links"),
@@ -481,7 +498,7 @@ From the footnote popup, you can jump to the footnote location in the book by sw
                         spin_widget = SpinWidget:new{
                             width = math.floor(Screen:getWidth() * 0.75),
                             value = G_reader_settings:readSetting("footnote_popup_absolute_font_size")
-                                            or Screen:scaleBySize(self.ui.font.font_size),
+                                            or Screen:scaleBySize(self.document.configurable.font_size),
                             value_min = 12,
                             value_max = 255,
                             precision = "%d",
@@ -542,16 +559,16 @@ end
 --- Check if a xpointer to <a> node really points to itself
 function ReaderLink:isXpointerCoherent(a_xpointer)
     -- Get screen coordinates of xpointer
-    local screen_y, screen_x = self.ui.document:getScreenPositionFromXPointer(a_xpointer)
+    local screen_y, screen_x = self.document:getScreenPositionFromXPointer(a_xpointer)
     -- Get again link and a_xpointer from this position
-    local re_link_xpointer, re_a_xpointer = self.ui.document:getLinkFromPosition({x = screen_x, y = screen_y}) -- luacheck: no unused
+    local re_link_xpointer, re_a_xpointer = self.document:getLinkFromPosition({x = screen_x, y = screen_y}) -- luacheck: no unused
     -- We should get the same a_xpointer. If not, crengine has messed up
     -- and we should not trust this xpointer to get back to this link.
     if re_a_xpointer ~= a_xpointer then
         -- Try it again with screen_x+1 (in the rare cases where screen_x
         -- fails, screen_x+1 usually works - probably something in crengine,
         -- but easier to workaround here that way)
-        re_link_xpointer, re_a_xpointer = self.ui.document:getLinkFromPosition({x = screen_x+1, y = screen_y}) -- luacheck: no unused
+        re_link_xpointer, re_a_xpointer = self.document:getLinkFromPosition({x = screen_x+1, y = screen_y}) -- luacheck: no unused
         if re_a_xpointer ~= a_xpointer then
             logger.info("noncoherent a_xpointer:", a_xpointer)
             return false
@@ -564,11 +581,11 @@ end
 -- `Document:getLinkFromPosition()` behaves differently depending on
 -- document type, so this function provides a wrapper.
 function ReaderLink:getLinkFromGes(ges)
-    if self.ui.document.info.has_pages then
+    if self.ui.paging then
         local pos = self.view:screenToPageTransform(ges.pos)
         if pos then
             -- link box in native page
-            local link, lbox = self.ui.document:getLinkFromPosition(pos.page, pos)
+            local link, lbox = self.document:getLinkFromPosition(pos.page, pos)
             if link and lbox then
                 return {
                     link = link,
@@ -578,7 +595,7 @@ function ReaderLink:getLinkFromGes(ges)
             end
         end
     else
-        local link_xpointer, a_xpointer = self.ui.document:getLinkFromPosition(ges.pos)
+        local link_xpointer, a_xpointer = self.document:getLinkFromPosition(ges.pos)
         logger.dbg("ReaderLink:getLinkFromPosition @", ges.pos.x, ges.pos.y, "from a_xpointer:", a_xpointer, "to link_xpointer:", link_xpointer)
 
         -- On some documents, crengine may sometimes give a wrong a_xpointer
@@ -616,7 +633,7 @@ function ReaderLink:showLinkBox(link, allow_footnote_popup)
     if link and link.lbox then -- pdfdocument
         -- screen box that holds the link
         local sbox = self.view:pageToScreenTransform(link.pos.page,
-            self.ui.document:nativeToPageRectTransform(link.pos.page, link.lbox))
+            self.document:nativeToPageRectTransform(link.pos.page, link.lbox))
         if sbox then
             UIManager:show(LinkBox:new{
                 box = sbox,
@@ -634,9 +651,8 @@ end
 
 function ReaderLink:onTap(_, ges)
     if not isTapToFollowLinksOn() then return end
-    if self.ui.document.info.has_pages then
-        -- (footnote popup and larger tap area are for not
-        -- not supported with non-CreDocuments)
+    if self.ui.paging then
+        -- (footnote popup and larger tap area are not supported with non-CreDocuments)
         local link = self:getLinkFromGes(ges)
         if link then
             if link.link and link.link.uri and isTapIgnoreExternalLinksEnabled() then
@@ -679,13 +695,8 @@ function ReaderLink:onTap(_, ges)
 end
 
 function ReaderLink:getCurrentLocation()
-    local location
-    if self.ui.document.info.has_pages then
-        location = self.ui.paging:getBookLocation()
-    else
-        location = {xpointer = self.ui.rolling:getBookLocation(),}
-    end
-    return location
+    return self.ui.paging and self.ui.paging:getBookLocation()
+                           or {xpointer = self.ui.rolling:getBookLocation()}
 end
 
 -- Returns true, current_location if the current location is the same as the
@@ -707,6 +718,13 @@ function ReaderLink:onAddCurrentLocationToStack(show_notification)
     if show_notification then
         Notification:notify(_("Current location added to history."))
     end
+    return true
+end
+
+function ReaderLink:onAddCurrentLocationToStackNonTouch()
+    self:addCurrentLocationToStack()
+    Notification:notify(_("Current location added to history."), Notification.SOURCE_ALWAYS_SHOW)
+    return true
 end
 
 -- Remember current location so we can go back to it
@@ -716,6 +734,10 @@ function ReaderLink:addCurrentLocationToStack(loc)
     table.insert(self.location_stack, location)
 end
 
+function ReaderLink:popFromLocationStack()
+    return table.remove(self.location_stack)
+end
+
 function ReaderLink:onClearLocationStack(show_notification)
     self.location_stack = {}
     self:onClearForwardLocationStack()
@@ -737,7 +759,7 @@ function ReaderLink:getPreviousLocationPages()
     if #self.location_stack > 0 then
         for num, location in ipairs(self.location_stack) do
             if self.ui.rolling and location.xpointer then
-                previous_locations[self.ui.document:getPageFromXPointer(location.xpointer)] = num
+                previous_locations[self.document:getPageFromXPointer(location.xpointer)] = num
             end
             if self.ui.paging and location[1] and location[1].page then
                 previous_locations[location[1].page] = num
@@ -752,7 +774,7 @@ end
 -- they should not provide allow_footnote_popup=true)
 function ReaderLink:onGotoLink(link, neglect_current_location, allow_footnote_popup)
     local link_url
-    if self.ui.document.info.has_pages then
+    if self.ui.paging then
         -- internal pdf links have a "page" attribute, while external ones have an "uri" attribute
         if link.page then -- Internal link
             logger.dbg("ReaderLink:onGotoLink: Internal link:", link)
@@ -770,7 +792,7 @@ function ReaderLink:onGotoLink(link, neglect_current_location, allow_footnote_po
         -- If the XPointer does not exist (or is a full url), we will jump to page 1
         -- Best to check that this link exists in document with the following,
         -- which accepts both of the above legitimate xpointer as input.
-        if self.ui.document:isXPointerInDocument(link.xpointer) then
+        if self.document:isXPointerInDocument(link.xpointer) then
             logger.dbg("ReaderLink:onGotoLink: Internal link:", link)
             if allow_footnote_popup then
                 if self:showAsFootnotePopup(link, neglect_current_location) then
@@ -812,8 +834,9 @@ function ReaderLink:onGotoLink(link, neglect_current_location, allow_footnote_po
     end
     logger.dbg("ReaderLink:onGotoLink: External link:", link_url)
 
-    local is_http_link = link_url:find("^https?://") ~= nil
-    if is_http_link and self:onGoToExternalLink(link_url) then
+    local scheme = link_url:match("^(%w[%w+%-.]*):")
+    local is_supported_external_link = scheme and util.arrayContains(self.supported_external_schemes, scheme:lower())
+    if is_supported_external_link and self:onGoToExternalLink(link_url) then
         return true
     end
 
@@ -828,9 +851,7 @@ function ReaderLink:onGotoLink(link, neglect_current_location, allow_footnote_po
     linked_filename  = ffiutil.joinPath(self.document_dir, linked_filename) -- get full path
     linked_filename = ffiutil.realpath(linked_filename) -- clean full path from ./ or ../
     if linked_filename and lfs.attributes(linked_filename, "mode") == "file" then
-        local DocumentRegistry = require("document/documentregistry")
-        local provider = DocumentRegistry:getProvider(linked_filename)
-        if provider then
+        if DocumentRegistry:hasProvider(linked_filename) then
             -- Display filename with anchor or query string, so the user gets
             -- this information and can manually go to the appropriate place
             local display_filename = linked_filename
@@ -864,7 +885,7 @@ end
 
 function ReaderLink:onGoToExternalLink(link_url)
     local buttons, title = self:getButtonsForExternalLinkDialog(link_url)
-    self.external_link_dialog = ButtonDialogTitle:new{
+    self.external_link_dialog = ButtonDialog:new{
         title = title,
         buttons = buttons,
     }
@@ -964,12 +985,12 @@ function ReaderLink:onGoToPageLink(ges, internal_links_only, max_distance)
     local selected_link, selected_distance2
     -- We use squared distances throughout the computations,
     -- no need to math.sqrt() anything for comparisons.
-    if self.ui.document.info.has_pages then
+    if self.ui.paging then
         local pos = self.view:screenToPageTransform(ges.pos)
         if not pos then
             return
         end
-        local links = self.ui.document:getPageLinks(pos.page)
+        local links = self.document:getPageLinks(pos.page)
         if not links or #links == 0 then
             return
         end
@@ -1011,7 +1032,7 @@ function ReaderLink:onGoToPageLink(ges, internal_links_only, max_distance)
         -- getPageLinks goes through the CRe call cache, so at least repeat calls are cheaper.
         -- If we only care about internal links, we only request those.
         -- That expensive segments work is always skipped on external links.
-        local links = self.ui.document:getPageLinks(internal_links_only)
+        local links = self.document:getPageLinks(internal_links_only)
         if not links or #links == 0 then
             return
         end
@@ -1065,7 +1086,9 @@ function ReaderLink:onGoToPageLink(ges, internal_links_only, max_distance)
         for _, link in ipairs(links) do
             -- link.uri may be an empty string with some invalid links: ignore them
             if link.section or (link.uri and link.uri ~= "") then
-                if link.segments then
+                -- Note: we may get segments empty in some conditions (in which
+                -- case we'll fallback to the 'else' branch and using x/y)
+                if link.segments and #link.segments > 0 then
                     -- With segments, each is a horizontal segment, with start_x < end_x,
                     -- and we should compute the distance from gesture position to
                     -- each segment.
@@ -1169,14 +1192,14 @@ function ReaderLink:onSelectPrevPageLink()
 end
 
 function ReaderLink:selectRelPageLink(rel)
-    if self.ui.document.info.has_pages then
+    if self.ui.paging then
         -- not implemented for now (see at doing like in showLinkBox()
         -- to highlight the link before jumping to it)
         return
     end
     -- Follow swipe_ignore_external_links setting to allow
     -- skipping external links when using keys
-    local links = self.ui.document:getPageLinks(isSwipeIgnoreExternalLinksEnabled())
+    local links = self.document:getPageLinks(isSwipeIgnoreExternalLinksEnabled())
     if not links or #links == 0 then
         return
     end
@@ -1198,7 +1221,7 @@ function ReaderLink:selectRelPageLink(rel)
     end
     if not self.cur_selected_page_link_num then
         self.cur_selected_link = nil
-        self.ui.document:highlightXPointer()
+        self.document:highlightXPointer()
         UIManager:setDirty(self.dialog, "ui")
         return
     end
@@ -1227,8 +1250,8 @@ function ReaderLink:selectRelPageLink(rel)
         -- a bit more time if it was hidden by the footnote popup
         link_y = link_y,
     }
-    self.ui.document:highlightXPointer() -- clear any previous one
-    self.ui.document:highlightXPointer(self.cur_selected_link.from_xpointer)
+    self.document:highlightXPointer() -- clear any previous one
+    self.document:highlightXPointer(self.cur_selected_link.from_xpointer)
     UIManager:setDirty(self.dialog, "ui")
     return true
 end
@@ -1241,7 +1264,7 @@ end
 
 function ReaderLink:onPageUpdate()
     if self.cur_selected_link then
-        self.ui.document:highlightXPointer()
+        self.document:highlightXPointer()
         self.cur_selected_page_link_num = nil
         self.cur_selected_link = nil
     end
@@ -1249,7 +1272,7 @@ end
 
 function ReaderLink:onPosUpdate()
     if self.cur_selected_link then
-        self.ui.document:highlightXPointer()
+        self.document:highlightXPointer()
         self.cur_selected_page_link_num = nil
         self.cur_selected_link = nil
     end
@@ -1258,7 +1281,7 @@ end
 function ReaderLink:onGoToLatestBookmark(ges)
     local latest_bookmark = self.ui.bookmark:getLatestBookmark()
     if latest_bookmark then
-        if self.ui.document.info.has_pages then
+        if self.ui.paging then
             -- self:onGotoLink() needs something with a page attribute.
             -- we need to substract 1 to bookmark page, as links start from 0
             -- and onGotoLink will add 1 - we need a fake_link (with a single
@@ -1285,7 +1308,7 @@ function ReaderLink:onGoToLatestBookmark(ges)
 end
 
 function ReaderLink:showAsFootnotePopup(link, neglect_current_location)
-    if self.ui.document.info.has_pages then
+    if self.ui.paging then
         return false -- not supported
     end
 
@@ -1376,7 +1399,7 @@ function ReaderLink:showAsFootnotePopup(link, neglect_current_location)
 
     logger.dbg("Checking if link is to a footnote:", flags, source_xpointer, target_xpointer)
     local is_footnote, reason, extStopReason, extStartXP, extEndXP =
-            self.ui.document:isLinkToFootnote(source_xpointer, target_xpointer, flags, max_text_size)
+            self.document:isLinkToFootnote(source_xpointer, target_xpointer, flags, max_text_size)
     if not is_footnote then
         logger.dbg("not a footnote:", reason)
         return false
@@ -1404,9 +1427,9 @@ function ReaderLink:showAsFootnotePopup(link, neglect_current_location)
     -- from parent nodes
     local html
     if extStartXP and extEndXP then
-        html = self.ui.document:getHTMLFromXPointers(extStartXP, extEndXP, 0x1001)
+        html = self.document:getHTMLFromXPointers(extStartXP, extEndXP, 0x1001)
     else
-        html = self.ui.document:getHTMLFromXPointer(target_xpointer, 0x1001, true)
+        html = self.document:getHTMLFromXPointer(target_xpointer, 0x1001, true)
         -- from_final_parent = true to get a possibly more complete footnote
     end
     if not html then
@@ -1423,20 +1446,20 @@ function ReaderLink:showAsFootnotePopup(link, neglect_current_location)
     -- (which might not be seen when covered by FootnoteWidget)
     local close_callback = nil
     if link.from_xpointer then -- coherent xpointer
-        self.ui.document:highlightXPointer() -- clear any previous one
-        self.ui.document:highlightXPointer(link.from_xpointer)
+        self.document:highlightXPointer() -- clear any previous one
+        self.document:highlightXPointer(link.from_xpointer)
         -- Don't let a previous footnote popup clear our highlight
         self._footnote_popup_discard_previous_close_callback = true
         UIManager:setDirty(self.dialog, "ui")
         close_callback = function(footnote_height)
             -- remove this highlight (actually all) on close
-            local highlight_page = self.ui.document:getCurrentPage()
+            local highlight_page = self.document:getCurrentPage()
             local clear_highlight = function()
-                self.ui.document:highlightXPointer()
+                self.document:highlightXPointer()
                 -- Only refresh if we stayed on the same page, otherwise
                 -- this could remove too early a marker on the target page
                 -- after this footnote is followed
-                if self.ui.document:getCurrentPage() == highlight_page then
+                if self.document:getCurrentPage() == highlight_page then
                     UIManager:setDirty(self.dialog, "ui")
                 end
             end
@@ -1465,8 +1488,8 @@ function ReaderLink:showAsFootnotePopup(link, neglect_current_location)
     popup = FootnoteWidget:new{
         html = html,
         doc_font_name = self.ui.font.font_face,
-        doc_font_size = Screen:scaleBySize(self.ui.font.font_size),
-        doc_margins = self.ui.document:getPageMargins(),
+        doc_font_size = Screen:scaleBySize(self.document.configurable.font_size),
+        doc_margins = self.document:getPageMargins(),
         close_callback = close_callback,
         follow_callback = function() -- follow the link on swipe west
             UIManager:close(popup)

frontend/apps/reader/modules/readermenu.lua

@@ -95,7 +95,7 @@ function ReaderMenu:getPreviousFile()
     return require("readhistory"):getPreviousFile(self.ui.document.file)
 end
 
-function ReaderMenu:onReaderReady()
+function ReaderMenu:initGesListener()
     if not Device:isTouchDevice() then return end
 
     local DTAP_ZONE_MENU = G_defaults:readSetting("DTAP_ZONE_MENU")
@@ -179,6 +179,8 @@ function ReaderMenu:onReaderReady()
     })
 end
 
+ReaderMenu.onReaderReady = ReaderMenu.initGesListener
+
 function ReaderMenu:setUpdateItemTable()
     for _, widget in pairs(self.registered_widgets) do
         local ok, err = pcall(widget.addToMainMenu, widget, self.menu_items)
@@ -252,7 +254,7 @@ function ReaderMenu:setUpdateItemTable()
 
     if Device:supportsScreensaver() then
         local ss_book_settings = {
-            text = _("Exclude this book's content and cover from screensaver"),
+            text = _("Do not show this book cover on sleep screen"),
             enabled_func = function()
                 if self.ui and self.ui.document then
                     local screensaverType = G_reader_settings:readSetting("screensaver_type")
@@ -283,7 +285,7 @@ function ReaderMenu:setUpdateItemTable()
         end
         table.insert(screensaver_sub_item_table, ss_book_settings)
         self.menu_items.screensaver = {
-            text = _("Screensaver"),
+            text = _("Sleep screen"),
             sub_item_table = screensaver_sub_item_table,
         }
     end
@@ -456,16 +458,24 @@ function ReaderMenu:onShowMenu(tab_index)
 end
 
 function ReaderMenu:onCloseReaderMenu()
-    if self.menu_container then
-        self.last_tab_index = self.menu_container[1].last_index
-        self:onSaveSettings()
-        UIManager:close(self.menu_container)
-    end
+    if not self.menu_container then return true end
+    self.last_tab_index = self.menu_container[1].last_index
+    self:onSaveSettings()
+    UIManager:close(self.menu_container)
+    self.menu_container = nil
     return true
 end
 
 function ReaderMenu:onSetDimensions(dimen)
-    self:onCloseReaderMenu()
+    -- This widget doesn't support in-place layout updates, so, close & reopen
+    if self.menu_container then
+        self:onCloseReaderMenu()
+        self:onShowMenu()
+    end
+
+    -- update gesture zones according to new screen dimen
+    -- (On CRe, this will get called a second time by ReaderReady once the document is reloaded).
+    self:initGesListener()
 end
 
 function ReaderMenu:onCloseDocument()

frontend/apps/reader/modules/readerpagemap.lua

@@ -87,9 +87,7 @@ function ReaderPageMap:resetLayout()
 end
 
 function ReaderPageMap:onReadSettings(config)
-    local h_margins = config:readSetting("copt_h_page_margins")
-                   or G_reader_settings:readSetting("copt_h_page_margins")
-                   or G_defaults:readSetting("DCREREADER_CONFIG_H_MARGIN_SIZES_MEDIUM")
+    local h_margins = self.ui.document.configurable.h_page_margins
     self.max_left_label_width = Screen:scaleBySize(h_margins[1])
     self.max_right_label_width = Screen:scaleBySize(h_margins[2])
 
@@ -282,8 +280,11 @@ function ReaderPageMap:getCurrentPageLabel(clean_label)
     -- For consistency, getPageMapCurrentPageLabel() returns the last page
     -- label shown in the view if there are more than one (or the previous
     -- one if there is none).
-    local label = self.ui.document:getPageMapCurrentPageLabel()
-    return clean_label and self:cleanPageLabel(label) or label
+    local label, idx, count = self.ui.document:getPageMapCurrentPageLabel()
+    if clean_label then
+        label = self:cleanPageLabel(label)
+    end
+    return label, idx, count
 end
 
 function ReaderPageMap:getFirstPageLabel(clean_label)

frontend/apps/reader/modules/readerpaging.lua

@@ -51,7 +51,28 @@ end
 function ReaderPaging:onGesture() end
 
 function ReaderPaging:registerKeyEvents()
-    if Device:hasKeys() then
+    if Device:hasDPad() and Device:useDPadAsActionKeys() then
+        self.key_events.GotoNextPos = {
+            { { "RPgFwd", "LPgFwd" } },
+            event = "GotoPosRel",
+            args = 1,
+        }
+        self.key_events.GotoPrevPos = {
+            { { "RPgBack", "LPgBack" } },
+            event = "GotoPosRel",
+            args = -1,
+        }
+        self.key_events.GotoNextChapter = {
+            { "Right" },
+            event = "GotoNextChapter",
+            args = 1,
+        }
+        self.key_events.GotoPrevChapter = {
+            { "Left" },
+            event = "GotoPrevChapter",
+            args = -1,
+        }
+    elseif Device:hasKeys() then
         self.key_events.GotoNextPage = {
             { { "RPgFwd", "LPgFwd", not Device:hasFewKeys() and "Right" } },
             event = "GotoViewRel",
@@ -673,10 +694,10 @@ function ReaderPaging:onInitScrollPageStates(orig_mode)
     if self.view.page_scroll and self.view.state.page then
         self.orig_page = self.current_page
         self.view.page_states = {}
-        local blank_area = Geom:new{}
+        local blank_area = Geom:new()
         blank_area:setSizeTo(self.view.visible_area)
         while blank_area.h > 0 do
-            local offset = Geom:new{}
+            local offset = Geom:new()
             -- caculate position in current page
             if self.current_page == self.orig_page then
                 local page_area = self.view:getPageArea(
@@ -691,7 +712,9 @@ function ReaderPaging:onInitScrollPageStates(orig_mode)
                 blank_area.h = blank_area.h - self.view.page_gap.height
             end
             if blank_area.h > 0 then
-                self:_gotoPage(self.current_page + 1, "scrolling")
+                local next_page = self.ui.document:getNextPage(self.current_page)
+                if next_page == 0 then break end -- end of document reached
+                self:_gotoPage(next_page, "scrolling")
             end
         end
         self:_gotoPage(self.orig_page, "scrolling")
@@ -773,7 +796,7 @@ function ReaderPaging:updateTopPageState(state, blank_area, offset)
         w = blank_area.w,
         h = blank_area.h,
     }
-    if state.page == self.number_of_pages then
+    if self.ui.document:getNextPage(state.page) == 0 then -- last page
         visible_area:offsetWithin(state.page_area, offset.x, offset.y)
     else
         visible_area = visible_area:shrinkInside(state.page_area, offset.x, offset.y)
@@ -790,7 +813,7 @@ function ReaderPaging:updateBottomPageState(state, blank_area, offset)
         w = blank_area.w,
         h = blank_area.h,
     }
-    if state.page == 1 then
+    if self.ui.document:getPrevPage(state.page) == 0 then -- first page
         visible_area:offsetWithin(state.page_area, offset.x, offset.y)
     else
         visible_area = visible_area:shrinkInside(state.page_area, offset.x, offset.y)
@@ -818,10 +841,10 @@ function ReaderPaging:genPageStatesFromTop(top_page_state, blank_area, offset)
     while blank_area.h > 0 do
         blank_area.h = blank_area.h - self.view.page_gap.height
         if blank_area.h > 0 then
-            if current_page == self.number_of_pages then break end
-            self:_gotoPage(current_page + 1, "scrolling")
-            current_page = current_page + 1
-            state = self:getNextPageState(blank_area, Geom:new{})
+            current_page = self.ui.document:getNextPage(current_page)
+            if current_page == 0 then break end -- end of document reached
+            self:_gotoPage(current_page, "scrolling")
+            state = self:getNextPageState(blank_area, Geom:new())
             table.insert(page_states, state)
         end
     end
@@ -843,13 +866,23 @@ function ReaderPaging:genPageStatesFromBottom(bottom_page_state, blank_area, off
     while blank_area.h > 0 do
         blank_area.h = blank_area.h - self.view.page_gap.height
         if blank_area.h > 0 then
-            if current_page == 1 then break end
-            self:_gotoPage(current_page - 1, "scrolling")
-            current_page = current_page - 1
-            state = self:getPrevPageState(blank_area, Geom:new{})
+            current_page = self.ui.document:getPrevPage(current_page)
+            if current_page == 0 then break end -- start of document reached
+            self:_gotoPage(current_page, "scrolling")
+            state = self:getPrevPageState(blank_area, Geom:new())
             table.insert(page_states, 1, state)
         end
     end
+    if current_page == 0 then
+        -- We reached the start of document: we may have truncated too much
+        -- of the bottom page while scrolling up.
+        -- Re-generate everything with first page starting at top
+        offset = Geom:new{x = 0, y = 0}
+        blank_area:setSizeTo(self.view.visible_area)
+        local first_page_state = page_states[1]
+        first_page_state.visible_area.y = 0 -- anchor first page at top
+        return self:genPageStatesFromTop(first_page_state, blank_area, offset)
+    end
     return page_states
 end
 
@@ -857,7 +890,7 @@ function ReaderPaging:onScrollPanRel(diff)
     if diff == 0 then return true end
     logger.dbg("pan relative height:", diff)
     local offset = Geom:new{x = 0, y = diff}
-    local blank_area = Geom:new{}
+    local blank_area = Geom:new()
     blank_area:setSizeTo(self.view.visible_area)
     local new_page_states
     if diff > 0 then
@@ -894,14 +927,14 @@ function ReaderPaging:onScrollPageRel(page_diff)
         -- page down, last page should be moved to top
         local last_page_state = table.remove(self.view.page_states)
         local last_visible_area = last_page_state.visible_area
-        if last_page_state.page == self.number_of_pages and
+        if self.ui.document:getNextPage(last_page_state.page) == 0 and
                 last_visible_area.y + last_visible_area.h >= last_page_state.page_area.h then
             table.insert(self.view.page_states, last_page_state)
             self.ui:handleEvent(Event:new("EndOfBook"))
             return true
         end
 
-        local blank_area = Geom:new{}
+        local blank_area = Geom:new()
         blank_area:setSizeTo(self.view.visible_area)
         local overlap = self.overlap
         local offset = Geom:new{
@@ -911,7 +944,7 @@ function ReaderPaging:onScrollPageRel(page_diff)
         self.view.page_states = self:genPageStatesFromTop(last_page_state, blank_area, offset)
     elseif page_diff < 0 then
         -- page up, first page should be moved to bottom
-        local blank_area = Geom:new{}
+        local blank_area = Geom:new()
         blank_area:setSizeTo(self.view.visible_area)
         local overlap = self.overlap
         local first_page_state = table.remove(self.view.page_states, 1)
@@ -991,14 +1024,41 @@ function ReaderPaging:onGotoPageRel(diff)
         goto_end(x, -x_diff)
     end
     local function goto_next_page()
-        local new_page = self.current_page + diff
+        local new_page
+        if self.ui.document:hasHiddenFlows() then
+            local forward = diff > 0
+            local pdiff = forward and math.ceil(diff) or math.ceil(-diff)
+            new_page = self.current_page
+            for i=1, pdiff do
+                local test_page = forward and self.ui.document:getNextPage(new_page)
+                                           or self.ui.document:getPrevPage(new_page)
+                if test_page == 0 then -- start or end of document reached
+                    if forward then
+                        new_page = self.number_of_pages + 1 -- to trigger EndOfBook below
+                    else
+                        new_page = 0
+                    end
+                    break
+                end
+                new_page = test_page
+            end
+        else
+            new_page = self.current_page + diff
+        end
         if new_page > self.number_of_pages then
             self.ui:handleEvent(Event:new("EndOfBook"))
             goto_end(y)
             goto_end(x)
         elseif new_page > 0 then
+            -- Be sure that the new and old view areas are reset so that no value is carried over to next page.
+            -- Without this, we would have panned_y = new_va.y - old_va.y > 0, and panned_y will be added to the next page's y direction.
+            -- This occurs when the current page has a y > 0 position (for example, a cropped page) and can fit the whole page height,
+            -- while the next page needs scrolling in the height.
             self:_gotoPage(new_page)
+            new_va = self.visible_area:copy()
+            old_va = self.visible_area
             goto_end(y, -y_diff)
+            goto_end(x, -x_diff)
         else
             goto_end(x)
         end
@@ -1107,7 +1167,18 @@ function ReaderPaging:onGotoPage(number, pos)
 end
 
 function ReaderPaging:onGotoRelativePage(number)
-    self:_gotoPage(self.current_page + number)
+    local new_page = self.current_page
+    local test_page = new_page
+    local forward = number > 0
+    for i=1, math.abs(number) do
+        test_page = forward and self.ui.document:getNextPage(test_page)
+                             or self.ui.document:getPrevPage(test_page)
+        if test_page == 0 then -- start or end of document reached
+            break
+        end
+        new_page = test_page
+    end
+    self:_gotoPage(new_page)
     return true
 end
 

frontend/apps/reader/modules/readerrolling.lua

@@ -61,7 +61,6 @@ local ReaderRolling = InputContainer:extend{
     xpointer = nil,
     panning_steps = ReaderPanning.panning_steps,
     cre_top_bar_enabled = false,
-    visible_pages = 1,
     -- With visible_pages=2, in 2-pages mode, ensure the first
     -- page is always odd or even (odd is logical to avoid a
     -- same page when turning first 2-pages set of document)
@@ -78,7 +77,11 @@ local ReaderRolling = InputContainer:extend{
         FULL_RENDERING_READY = 3,
         RELOADING_DOCUMENT = 4,
         DO_RELOAD_DOCUMENT = 5,
-    }
+    },
+
+    mark_func = nil,
+    unmark_func = nil,
+    _stepRerenderingAutomation = nil,
 }
 
 function ReaderRolling:init()
@@ -97,7 +100,7 @@ function ReaderRolling:init()
             self.valid_cache_rendering_hash = self.ui.document:getDocumentRenderingHash(false)
         end
     end)
-    table.insert(self.ui.postReaderCallback, function()
+    table.insert(self.ui.postReaderReadyCallback, function()
         self:updatePos()
         -- Disable crengine internal history, with required redraw
         self.ui.document:enableInternalHistory(false)
@@ -113,7 +116,30 @@ end
 function ReaderRolling:onGesture() end
 
 function ReaderRolling:registerKeyEvents()
-    if Device:hasKeys() then
+    if Device:hasScreenKB() or Device:hasSymKey() then
+        self.key_events.GotoNextView = {
+            { { "RPgFwd", "LPgFwd" } },
+            event = "GotoViewRel",
+            args = 1,
+        }
+        self.key_events.GotoPrevView = {
+            { { "RPgBack", "LPgBack" } },
+            event = "GotoViewRel",
+            args = -1,
+        }
+        if Device:hasKeyboard() then
+            self.key_events.MoveUp = {
+                { "Shift", "RPgBack" },
+                event = "Panning",
+                args = {0, -1},
+            }
+            self.key_events.MoveDown = {
+                { "Shift", "RPgFwd" },
+                event = "Panning",
+                args = {0,  1},
+            }
+        end
+    elseif Device:hasKeys() then
         self.key_events.GotoNextView = {
             { { "RPgFwd", "LPgFwd", "Right" } },
             event = "GotoViewRel",
@@ -125,7 +151,18 @@ function ReaderRolling:registerKeyEvents()
             args = -1,
         }
     end
-    if Device:hasDPad() then
+    if Device:hasDPad() and Device:useDPadAsActionKeys() then
+        self.key_events.GotoNextChapter = {
+            { "Right" },
+            event = "GotoNextChapter",
+            args = 1,
+        }
+        self.key_events.GotoPrevChapter = {
+            { "Left" },
+            event = "GotoPrevChapter",
+            args = -1,
+        }
+    elseif Device:hasDPad() then
         self.key_events.MoveUp = {
             { "Up" },
             event = "Panning",
@@ -137,6 +174,18 @@ function ReaderRolling:registerKeyEvents()
             args = {0,  1},
         }
     end
+    if Device:hasScreenKB() then
+        self.key_events.MoveUp = {
+            { "ScreenKB", "RPgBack" },
+            event = "Panning",
+            args = {0, -1},
+        }
+        self.key_events.MoveDown = {
+            { "ScreenKB", "RPgFwd" },
+            event = "Panning",
+            args = {0,  1},
+        }
+    end
     if Device:hasKeyboard() then
         self.key_events.GotoFirst = {
             { "1" },
@@ -231,7 +280,7 @@ function ReaderRolling:onReadSettings(config)
         -- And check if we can migrate to a newest DOM version after
         -- the book is loaded (unless the user told us not to).
         if config:nilOrFalse("cre_keep_old_dom_version") then
-            self.ui:registerPostReadyCallback(function()
+            self.ui:registerPostReaderReadyCallback(function()
                 self:checkXPointersAndProposeDOMVersionUpgrade()
             end)
         end
@@ -274,15 +323,13 @@ function ReaderRolling:onReadSettings(config)
         end
     end
 
-    -- This self.visible_pages may not be the current nb of visible pages
+    -- self.configurable.visible_pages may not be the current nb of visible pages
     -- as crengine may decide to not ensure that in some conditions.
     -- It's the one we got from settings, the one the user has decided on
     -- with config toggle, and the one that we will save for next load.
     -- Use self.ui.document:getVisiblePageCount() to get the current
     -- crengine used value.
-    self.visible_pages = config:readSetting("visible_pages") or
-        G_reader_settings:readSetting("copt_visible_pages") or 1
-    self.ui.document:setVisiblePageCount(self.visible_pages)
+    self.ui.document:setVisiblePageCount(self.configurable.visible_pages)
 
     if config:has("hide_nonlinear_flows") then
         self.hide_nonlinear_flows = config:isTrue("hide_nonlinear_flows")
@@ -312,11 +359,20 @@ function ReaderRolling:onReadSettings(config)
     end)
 end
 
--- in scroll mode percent_finished must be save before close document
--- we cannot do it in onSaveSettings() because getLastPercent() uses self.ui.document
 function ReaderRolling:onCloseDocument()
     self:tearDownRerenderingAutomation()
+    -- Unschedule anything that might still somehow be...
+    if self.mark_func then
+        UIManager:unschedule(self.mark_func)
+    end
+    if self.unmark_func then
+        UIManager:unschedule(self.unmark_func)
+    end
+    UIManager:unschedule(self.onCheckDomStyleCoherence)
+    UIManager:unschedule(self.onUpdatePos)
+
     self.current_header_height = nil -- show unload progress bar at top
+    -- we cannot do it in onSaveSettings() because getLastPercent() uses self.ui.document
     self.ui.doc_settings:saveSetting("percent_finished", self:getLastPercent())
 
     local cache_file_path = self.ui.document:getCacheFilePath() -- nil if no cache file
@@ -352,7 +408,7 @@ function ReaderRolling:onCheckDomStyleCoherence()
             ok_callback = function()
                 -- Allow for ConfirmBox to be closed before showing
                 -- "Opening file" InfoMessage
-                UIManager:scheduleIn(0.5, function ()
+                UIManager:scheduleIn(0.5, function()
                     -- And check we haven't quit reader in these 0.5s
                     if self.ui.document then
                         self.ui:reloadDocument()
@@ -364,16 +420,8 @@ function ReaderRolling:onCheckDomStyleCoherence()
 end
 
 function ReaderRolling:onSaveSettings()
-    -- remove last_percent config since its deprecated
-    self.ui.doc_settings:delSetting("last_percent")
+    self.ui.doc_settings:delSetting("last_percent") -- deprecated
     self.ui.doc_settings:saveSetting("last_xpointer", self.xpointer)
-    -- in scrolling mode, the document may already be closed,
-    -- so we have to check the condition to avoid crash function self:getLastPercent()
-    -- that uses self.ui.document
-    if self.ui.document then
-        self.ui.doc_settings:saveSetting("percent_finished", self:getLastPercent())
-    end
-    self.ui.doc_settings:saveSetting("visible_pages", self.visible_pages)
     self.ui.doc_settings:saveSetting("hide_nonlinear_flows", self.hide_nonlinear_flows)
     self.ui.doc_settings:saveSetting("partial_rerendering", self.partial_rerendering)
 end
@@ -456,7 +504,9 @@ function ReaderRolling:addToMainMenu(menu_items)
         menu_items.hide_nonlinear_flows = {
             text = _("Hide non-linear fragments"),
             enabled_func = function()
+                -- Custom hidden flows have precedence over publisher hidden non-linear fragments
                 return self.view.view_mode == "page" and self.ui.document:getVisiblePageCount() == 1
+                                                     and not self.ui.handmade:isHandmadeHiddenFlowsEnabled()
             end,
             checked_func = function() return self.hide_nonlinear_flows end,
             callback = function()
@@ -805,7 +855,7 @@ end
 
 function ReaderRolling:onGotoXPointer(xp, marker_xp)
     if self.mark_func then
-        -- unschedule previous marker as it's no more accurate
+        -- Unschedule previous marker as it's no longer accurate.
         UIManager:unschedule(self.mark_func)
         self.mark_func = nil
     end
@@ -835,7 +885,7 @@ function ReaderRolling:onGotoXPointer(xp, marker_xp)
         -- where xpointer target is (and remove if after 1s)
         local screen_y, screen_x = self.ui.document:getScreenPositionFromXPointer(marker_xp)
         local doc_margins = self.ui.document:getPageMargins()
-        local marker_h = Screen:scaleBySize(self.ui.font.font_size * 1.1 * self.ui.font.line_space_percent * (1/100))
+        local marker_h = Screen:scaleBySize(self.configurable.font_size * 1.1 * self.configurable.line_spacing * (1/100))
         -- Make it 4/5 of left margin wide (and bigger when huge margin)
         local marker_w = math.floor(math.max(doc_margins["left"] - Screen:scaleBySize(5), doc_margins["left"] * 4/5))
 
@@ -933,7 +983,7 @@ function ReaderRolling:onGotoViewRel(diff)
         local pan_diff = diff * page_visible_height
         if self.view.page_overlap_enable then
             local overlap_lines = G_reader_settings:readSetting("copt_overlap_lines") or 1
-            local overlap_h = Screen:scaleBySize(self.ui.font.font_size * 1.1 * self.ui.font.line_space_percent * (1/100)) * overlap_lines
+            local overlap_h = Screen:scaleBySize(self.configurable.font_size * 1.1 * self.configurable.line_spacing * (1/100)) * overlap_lines
             if pan_diff > overlap_h then
                 pan_diff = pan_diff - overlap_h
             elseif pan_diff < -overlap_h then
@@ -1009,9 +1059,7 @@ function ReaderRolling:onBatchedUpdateDone()
         self.batched_update_count = 0
         -- Be sure any Notification gets a chance to be painted before
         -- a blocking rerendering
-        UIManager:nextTick(function()
-            self:onUpdatePos()
-        end)
+        UIManager:nextTick(self.onUpdatePos, self)
     end
 end
 
@@ -1027,9 +1075,9 @@ function ReaderRolling:onUpdatePos(force)
     if self.batched_update_count > 0 then
         return
     end
-    if self.ui.postReaderCallback ~= nil then -- ReaderUI:init() not yet done
+    if self.ui.postReaderReadyCallback ~= nil then -- ReaderUI:init() not yet done
         -- Don't schedule any updatePos as long as ReaderUI:init() is
-        -- not finished (one will be called in the ui.postReaderCallback
+        -- not finished (one will be called in the ui.postReaderReadyCallback
         -- we have set above) to avoid multiple refreshes.
         return true
     end
@@ -1040,7 +1088,10 @@ function ReaderRolling:onUpdatePos(force)
     -- Calling this now ensures the re-rendering is done by crengine
     -- so updatePos() has good info and can reposition
     -- the previous xpointer accurately:
-    self.ui.document:getCurrentPos()
+    if self.ui.document then
+        -- This can be racy with CloseDocument, as it's scheduled by onBatchedUpdateDone, guard it
+        self.ui.document:getCurrentPos()
+    end
 
     -- Otherwise, _readMetadata() would do that, but the positioning
     -- would not work as expected, for some reason (it worked
@@ -1091,16 +1142,14 @@ function ReaderRolling:updatePos(force)
     -- Allow for the new rendering to be shown before possibly showing
     -- the "Styles have changed..." ConfirmBox so the user can decide
     -- if it is really needed
-    UIManager:scheduleIn(0.1, function ()
-        self:onCheckDomStyleCoherence()
-    end)
+    UIManager:scheduleIn(0.1, self.onCheckDomStyleCoherence, self)
 end
 
 function ReaderRolling:onChangeViewMode()
     self.current_header_height = self.view.view_mode == "page" and self.ui.document:getHeaderHeight() or 0
     -- Restore current position when switching page/scroll mode
     if self.xpointer then
-        if self.visible_pages == 2 then
+        if self.configurable.visible_pages == 2 then
             -- Switching from 2-pages page mode to scroll mode has crengine switch to 1-page,
             -- and we need to notice this re-rendering and keep things sane
             self:onUpdatePos()
@@ -1126,7 +1175,7 @@ function ReaderRolling:onRedrawCurrentView()
 end
 
 function ReaderRolling:onSetDimensions(dimen)
-    if self.ui.postReaderCallback ~= nil then
+    if self.ui.postReaderReadyCallback ~= nil then
         -- ReaderUI:init() not yet done: just set document dimensions
         self.ui.document:setViewDimen(Screen:getSize())
         -- (what's done in the following else is done elsewhere by
@@ -1258,7 +1307,7 @@ function ReaderRolling:onSetVisiblePages(visible_pages)
     -- We nevertheless update the setting (that will be saved) with what
     -- the user has requested - and not what crengine has enforced, and
     -- always query crengine for if it ends up ensuring it or not.
-    self.visible_pages = visible_pages
+    self.configurable.visible_pages = visible_pages
     local prev_visible_pages = self.ui.document:getVisiblePageCount()
     self.ui.document:setVisiblePageCount(visible_pages)
     local cur_visible_pages = self.ui.document:getVisiblePageCount()
@@ -1442,25 +1491,12 @@ function ReaderRolling:checkXPointersAndProposeDOMVersionUpgrade()
     local applyFuncToXPointersSlots = function(func)
         -- Last position
         func(self, "xpointer", "last position in book")
-        -- Bookmarks
-        if self.ui.bookmark and self.ui.bookmark.bookmarks and #self.ui.bookmark.bookmarks > 0 then
+        -- Annotations
+        if self.ui.annotation and self.ui.annotation.annotations and #self.ui.annotation.annotations > 0 then
             local slots = { "page", "pos0", "pos1" }
-            for _, bookmark in ipairs(self.ui.bookmark.bookmarks) do
+            for _, item in ipairs(self.ui.annotation.annotations) do
                 for _, slot in ipairs(slots) do
-                    func(bookmark, slot, bookmark.notes or "bookmark")
-                end
-            end
-        end
-        -- Highlights
-        if self.view.highlight and self.view.highlight.saved then
-            local slots = { "pos0", "pos1" }
-            for page, items in pairs(self.view.highlight.saved) do
-                if items and #items > 0 then
-                    for _, highlight in ipairs(items) do
-                        for _, slot in ipairs(slots) do
-                            func(highlight, slot, highlight.text or "highlight")
-                        end
-                    end
+                    func(item, slot, item.text or "annotation")
                 end
             end
         end
@@ -1506,6 +1542,9 @@ function ReaderRolling:checkXPointersAndProposeDOMVersionUpgrade()
         local new_xp = normalized_xpointers[xp]
         if new_xp then
             obj[slot] = new_xp
+            if slot == "page" then
+                self.ui.annotation:updateItemByXPointer(obj)
+            end
         else
             -- Let lost/not-found XPointer be. There is a small chance that
             -- it will be found (it it was made before the boxing code moved
@@ -1560,10 +1599,9 @@ function ReaderRolling:checkXPointersAndProposeDOMVersionUpgrade()
             g_block_rendering_mode = 3 -- default in ReaderTypeset:onReadSettings()
         end
         if g_block_rendering_mode ~= 0 then -- default is not "legacy"
-            -- This setting is actually saved by self.ui.document.configurable
-            local block_rendering_mode = self.ui.document.configurable.block_rendering_mode
+            local block_rendering_mode = self.configurable.block_rendering_mode
             if block_rendering_mode == 0 then
-                self.ui.document.configurable.block_rendering_mode = g_block_rendering_mode
+                self.configurable.block_rendering_mode = g_block_rendering_mode
                 logger.info("  block_rendering_mode switched to", g_block_rendering_mode)
             end
         end
@@ -1624,7 +1662,7 @@ Note that %1 (out of %2) xpaths from your bookmarks and highlights have been nor
         ok_text = _("Upgrade now"),
         ok_callback = function()
             -- Allow for ConfirmBox to be closed before migrating
-            UIManager:scheduleIn(0.5, function ()
+            UIManager:scheduleIn(0.5, function()
                 -- And check we haven't quit reader in these 0.5s
                 if self.ui.document then
                     -- We'd rather not have any painting between the upgrade

frontend/apps/reader/modules/readersearch.lua

@@ -4,12 +4,16 @@ local CheckButton = require("ui/widget/checkbutton")
 local Device = require("device")
 local InfoMessage = require("ui/widget/infomessage")
 local InputDialog = require("ui/widget/inputdialog")
+local Menu = require("ui/widget/menu")
 local Notification = require("ui/widget/notification")
+local SpinWidget = require("ui/widget/spinwidget")
+local TextBoxWidget = require("ui/widget/textboxwidget")
 local UIManager = require("ui/uimanager")
 local Utf8Proc = require("ffi/utf8proc")
 local WidgetContainer = require("ui/widget/container/widgetcontainer")
 local logger = require("logger")
 local _ = require("gettext")
+local C_ = _.pgettext
 local Screen = Device.screen
 local T = require("ffi/util").template
 
@@ -27,7 +31,11 @@ local ReaderSearch = WidgetContainer:extend{
     -- The speed of the search depends on the regexs. Complex ones might need some time, easy ones
     -- go with the speed of light.
     -- Setting max_hits higher, does not mean to require more memory. More hits means smaller single hits.
-    max_hits = 2048, -- maximum hits for search; timinges tested on a Tolino
+    max_hits = 2048, -- maximum hits for findText search; timinges tested on a Tolino
+    findall_max_hits = 5000, -- maximum hits for findAllText search
+     -- number of words before and after the search string in All search results
+    findall_nb_context_words = G_reader_settings:readSetting("fulltext_search_nb_context_words") or 3,
+    findall_results_per_page = G_reader_settings:readSetting("fulltext_search_results_per_page") or 10,
 
     -- internal: whether we expect results on previous pages
     -- (can be different from self.direction, if, from a page in the
@@ -72,26 +80,108 @@ SRELL_ERROR_CODES[111] = _("Expression too complex, some hits will not be shown.
 SRELL_ERROR_CODES[666] = _("Expression may lead to an extremely long search time.")
 
 function ReaderSearch:addToMainMenu(menu_items)
+    menu_items.fulltext_search_settings = {
+        text = _("Fulltext search settings"),
+        sub_item_table = {
+            {
+                text = _("Show all results on text selection"),
+                help_text = _("When invoked after text selection, show a list with all results instead of highlighting matches in book pages."),
+                checked_func = function()
+                    return G_reader_settings:isTrue("fulltext_search_find_all")
+                end,
+                callback = function()
+                    G_reader_settings:flipNilOrFalse("fulltext_search_find_all")
+                end,
+            },
+            {
+                text_func = function()
+                    return T(_("Words in context: %1"), self.findall_nb_context_words)
+                end,
+                keep_menu_open = true,
+                callback = function(touchmenu_instance)
+                    local widget = SpinWidget:new{
+                        title_text =  _("Words in context"),
+                        value = self.findall_nb_context_words,
+                        value_min = 1,
+                        value_max = 20,
+                        default_value = 3,
+                        callback = function(spin)
+                            self.last_search_hash = nil
+                            self.findall_nb_context_words = spin.value
+                            G_reader_settings:saveSetting("fulltext_search_nb_context_words", spin.value)
+                            touchmenu_instance:updateItems()
+                        end,
+                    }
+                    UIManager:show(widget)
+                end,
+            },
+            {
+                text_func = function()
+                    return T(_("Results per page: %1"), self.findall_results_per_page)
+                end,
+                keep_menu_open = true,
+                callback = function(touchmenu_instance)
+                    local widget = SpinWidget:new{
+                        title_text =  _("Results per page"),
+                        value = self.findall_results_per_page,
+                        value_min = 6,
+                        value_max = 24,
+                        default_value = 10,
+                        callback = function(spin)
+                            self.findall_results_per_page = spin.value
+                            G_reader_settings:saveSetting("fulltext_search_results_per_page", spin.value)
+                            touchmenu_instance:updateItems()
+                        end,
+                    }
+                    UIManager:show(widget)
+                end,
+            },
+        },
+    }
     menu_items.fulltext_search = {
         text = _("Fulltext search"),
         callback = function()
             self:onShowFulltextSearchInput()
         end,
     }
+    menu_items.fulltext_search_findall_results = {
+        text = _("Last fulltext search results"),
+        callback = function()
+            self:onShowFindAllResults()
+        end,
+    }
+end
+
+function ReaderSearch:searchText(text) -- from highlight dialog
+    if G_reader_settings:isTrue("fulltext_search_find_all") then
+        self.ui.highlight:clear()
+        self:searchCallback(nil, text)
+    else
+        self:searchCallback(0, text) -- forward
+    end
 end
 
--- if reverse ~= 0 search backwards
-function ReaderSearch:searchCallback(reverse)
-    local search_text = self.input_dialog:getInputText()
-    if search_text == "" then return end
-    -- search_text comes from our keyboard, and may contain multiple diacritics ordered
-    -- in any order: we'd rather have them normalized, and expect the book content to
-    -- be proper and normalized text.
-    self.last_search_text = search_text -- if shown again, show it as it has been inputted
-    search_text = Utf8Proc.normalize_NFC(search_text)
-    self.use_regex = self.check_button_regex.checked
-    self.case_insensitive = not self.check_button_case.checked
-    local regex_error = self.use_regex and self.ui.document:checkRegex(search_text)
+-- if reverse == 1 search backwards
+function ReaderSearch:searchCallback(reverse, text)
+    local search_text = text or self.input_dialog:getInputText()
+    if search_text == nil or search_text == "" then return end
+    self.ui.doc_settings:saveSetting("fulltext_search_last_search_text", search_text)
+    self.last_search_text = search_text
+
+    local regex_error
+    if text then -- from highlight dialog
+        self.use_regex = false
+        self.case_insensitive = true
+    else -- from input dialog
+        -- search_text comes from our keyboard, and may contain multiple diacritics ordered
+        -- in any order: we'd rather have them normalized, and expect the book content to
+        -- be proper and normalized text.
+        search_text = Utf8Proc.normalize_NFC(search_text)
+        self.use_regex = self.check_button_regex.checked
+        self.case_insensitive = not self.check_button_case.checked
+        regex_error = self.use_regex and self.ui.document:checkRegex(search_text)
+    end
+
     if self.use_regex and regex_error ~= 0 then
         logger.dbg("ReaderSearch: regex error", regex_error, SRELL_ERROR_CODES[regex_error])
         local error_message
@@ -103,7 +193,15 @@ function ReaderSearch:searchCallback(reverse)
         UIManager:show(InfoMessage:new{ text = error_message })
     else
         UIManager:close(self.input_dialog)
-        self:onShowSearchDialog(search_text, reverse, self.use_regex, self.case_insensitive)
+        if reverse then
+            self.last_search_hash = nil
+            self:onShowSearchDialog(search_text, reverse, self.use_regex, self.case_insensitive)
+        else
+            local Trapper = require("ui/trapper")
+            Trapper:wrap(function()
+                self:findAllText(search_text)
+            end)
+        end
     end
 end
 
@@ -116,7 +214,7 @@ function ReaderSearch:onShowFulltextSearchInput()
     self.input_dialog = InputDialog:new{
         title = _("Enter text to search for"),
         width = math.floor(math.min(Screen:getWidth(), Screen:getHeight()) * 0.9),
-        input = self.last_search_text,
+        input = self.last_search_text or self.ui.doc_settings:readSetting("fulltext_search_last_search_text"),
         buttons = {
             {
                 {
@@ -126,6 +224,13 @@ function ReaderSearch:onShowFulltextSearchInput()
                         UIManager:close(self.input_dialog)
                     end,
                 },
+                {
+                    -- @translators Find all results in entire document, button displayed on the search bar, should be short.
+                    text = C_("Search text", "All"),
+                    callback = function()
+                        self:searchCallback()
+                    end,
+                },
                 {
                     text = backward_text,
                     callback = function()
@@ -183,7 +288,10 @@ function ReaderSearch:onShowSearchDialog(text, direction, regex, case_insensitiv
             local no_results = true -- for notification
             local res = search_func(self, search_term, param, regex, case_insensitive)
             if res then
-                if self.ui.document.info.has_pages then
+                if self.ui.paging then
+                    if not current_page then -- initial search
+                        current_page = self.ui.paging.current_page
+                    end
                     no_results = false
                     self.ui.link:onGotoLink({page = res.page - 1}, neglect_current_location)
                     self.view.highlight.temp[res.page] = res
@@ -252,8 +360,24 @@ function ReaderSearch:onShowSearchDialog(text, direction, regex, case_insensitiv
                         self.ui.link:onGotoLink({xpointer=valid_link}, neglect_current_location)
                     end
                 end
-                -- Don't add result pages to location ("Go back") stack
-                neglect_current_location = true
+                if not neglect_current_location then
+                    -- Initial search: onGotoLink() has added the current page to the location stack,
+                    -- and we don't want this to be done when showing further pages with results.
+                    -- But if this initial search is showing results on the current page, we don't want
+                    -- the original page added: we will do it when we jump to a different page.
+                    -- For now, only do this with CreDocument. With PDF, whether in single page mode or
+                    -- in scroll mode, the view can scroll a bit when showing results, and we want to
+                    -- allow "go back" to restore the original viewport.
+                    if self.ui.rolling and self.view.view_mode == "page" then
+                        if current_page == (self.ui.rolling and self.ui.document:getCurrentPage() or self.ui.paging.current_page) then
+                            self.ui.link:popFromLocationStack()
+                            neglect_current_location = false
+                        else
+                            -- We won't add further result pages to the location stack ("Go back").
+                            neglect_current_location = true
+                        end
+                    end
+                end
             end
             if no_results then
                 local notification_text
@@ -287,12 +411,12 @@ function ReaderSearch:onShowSearchDialog(text, direction, regex, case_insensitiv
                 self.wait_button.movable:setMovedOffset(self.search_dialog.movable:getMovedOffset())
                 UIManager:show(self.wait_button)
                 UIManager:tickAfterNext(function()
-                    do_search(func, pattern, param, regex, case_insensitive)()
+                    do_search(func, pattern, param)()
                     UIManager:close(self.wait_button)
                 end)
             end
         else
-            return do_search(func, pattern, param, regex, case_insensitive)
+            return do_search(func, pattern, param)
         end
     end
     self.search_dialog = ButtonDialog:new{
@@ -315,7 +439,6 @@ function ReaderSearch:onShowSearchDialog(text, direction, regex, case_insensitiv
                     icon_height = Screen:scaleBySize(DGENERIC_ICON_SIZE * 0.8),
                     callback = function()
                         self.search_dialog:onClose()
-                        self.last_search_text = text
                         self:onShowFulltextSearchInput()
                     end,
                 },
@@ -342,14 +465,14 @@ function ReaderSearch:onShowSearchDialog(text, direction, regex, case_insensitiv
         -- initial position: center of the screen
         UIManager:show(self.wait_button)
         UIManager:tickAfterNext(function()
-            do_search(self.searchFromCurrent, text, direction, regex, case_insensitive)()
+            do_search(self.searchFromCurrent, text, direction)()
             UIManager:close(self.wait_button)
             UIManager:show(self.search_dialog)
             --- @todo regional
             UIManager:setDirty(self.dialog, "partial")
         end)
     else
-        do_search(self.searchFromCurrent, text, direction, regex, case_insensitive)()
+        do_search(self.searchFromCurrent, text, direction)()
         UIManager:show(self.search_dialog)
         --- @todo regional
         UIManager:setDirty(self.dialog, "partial")
@@ -370,6 +493,13 @@ function ReaderSearch:search(pattern, origin, regex, case_insensitive)
     Device:setIgnoreInput(true)
     local retval, words_found = self.ui.document:findText(pattern, origin, direction, case_insensitive, page, regex, self.max_hits)
     Device:setIgnoreInput(false)
+    self:showErrorNotification(words_found, regex, self.max_hits)
+    return retval
+end
+
+function ReaderSearch:showErrorNotification(words_found, regex, max_hits)
+    regex = regex or self.use_regex
+    max_hits = max_hits or self.findall_max_hits
     local regex_retval = regex and self.ui.document:getAndClearRegexSearchError()
     if regex and regex_retval ~= 0 then
         local error_message
@@ -382,13 +512,12 @@ function ReaderSearch:search(pattern, origin, regex, case_insensitive)
             text = error_message,
             timeout = false,
         })
-    elseif words_found and words_found > self.max_hits then
+    elseif words_found and words_found >= max_hits then
         UIManager:show(Notification:new{
             text =_("Too many hits"),
             timeout = 4,
          })
     end
-    return retval
 end
 
 function ReaderSearch:searchFromStart(pattern, _, regex, case_insensitive)
@@ -416,4 +545,102 @@ function ReaderSearch:searchNext(pattern, direction, regex, case_insensitive)
     return self:search(pattern, 1, regex, case_insensitive)
 end
 
+function ReaderSearch:findAllText(search_text)
+    local last_search_hash = (self.last_search_text or "") .. tostring(self.case_insensitive) .. tostring(self.use_regex)
+    local not_cached = self.last_search_hash ~= last_search_hash
+    if not_cached then
+        local Trapper = require("ui/trapper")
+        local info = InfoMessage:new{ text = _("Searching… (tap to cancel)") }
+        UIManager:show(info)
+        UIManager:forceRePaint()
+        local completed, res = Trapper:dismissableRunInSubprocess(function()
+            return self.ui.document:findAllText(search_text,
+                self.case_insensitive, self.findall_nb_context_words, self.findall_max_hits, self.use_regex)
+        end, info)
+        if not completed then return end
+        UIManager:close(info)
+        self.last_search_hash = last_search_hash
+        self.findall_results = res
+        self.findall_results_item_index = nil
+    end
+    if self.findall_results then
+        self:onShowFindAllResults(not_cached)
+    else
+        UIManager:show(InfoMessage:new{ text = _("No results in the document") })
+    end
+end
+
+function ReaderSearch:onShowFindAllResults(not_cached)
+    if not self.last_search_hash or (not not_cached and self.findall_results == nil) then
+        -- no cached results, show input dialog
+        self:onShowFulltextSearchInput()
+        return
+    end
+
+    if self.ui.rolling and not_cached then -- for ui.paging: items are built in KoptInterface:findAllText()
+        for _, item in ipairs(self.findall_results) do
+            -- PDF/Kopt shows full words when only some part matches; let's do the same with CRE
+            local word = item.matched_text or ""
+            if item.matched_word_prefix then
+                word = item.matched_word_prefix .. word
+            end
+            if item.matched_word_suffix then
+                word = word .. item.matched_word_suffix
+            end
+            -- Make this word bolder, using Poor Text Formatting provided by TextBoxWidget
+            -- (we know this text ends up in a TextBoxWidget).
+            local text = TextBoxWidget.PTF_BOLD_START .. word .. TextBoxWidget.PTF_BOLD_END
+            -- append context before and after the word
+            if item.prev_text then
+                if not item.prev_text:find("%s$") then
+                    text = " " .. text
+                end
+                text = item.prev_text .. text
+            end
+            if item.next_text then
+                if not item.next_text:find("^[%s%p]") then
+                    text = text .. " "
+                end
+                text = text .. item.next_text
+            end
+            text = TextBoxWidget.PTF_HEADER .. text -- enable handling of our bold tags
+            item.text = text
+            item.mandatory = self.ui.bookmark:getBookmarkPageString(item.start)
+        end
+    end
+
+    local menu
+    menu = Menu:new{
+        title = T(_("Search results (%1)"), #self.findall_results),
+        subtitle = T(_("Query: %1"), self.last_search_text),
+        items_per_page = self.findall_results_per_page,
+        covers_fullscreen = true,
+        is_borderless = true,
+        is_popout = false,
+        title_bar_fm_style = true,
+        onMenuChoice = function(_, item)
+            if self.ui.rolling then
+                self.ui.link:addCurrentLocationToStack()
+                self.ui.rolling:onGotoXPointer(item.start, item.start) -- show target line marker
+                self.ui.document:getTextFromXPointers(item.start, item["end"], true) -- highlight
+            else
+                local page = item.mandatory
+                local boxes = {}
+                for i, box in ipairs(item.boxes) do
+                    boxes[i] = self.ui.document:nativeToPageRectTransform(page, box)
+                end
+                self.ui.link:onGotoLink({ page = page - 1 })
+                self.view.highlight.temp[page] = boxes
+            end
+        end,
+        close_callback = function()
+            self.findall_results_item_index = menu.page * menu.perpage -- save page number to reopen
+            UIManager:close(menu)
+        end,
+    }
+    menu:switchItemTable(nil, self.findall_results, self.findall_results_item_index)
+    UIManager:show(menu)
+    self:showErrorNotification(#self.findall_results)
+end
+
 return ReaderSearch

frontend/apps/reader/modules/readerstatus.lua

@@ -38,7 +38,10 @@ function ReaderStatus:onEndOfBook()
     local QuickStart = require("ui/quickstart")
     local last_file = G_reader_settings:readSetting("lastfile")
     if last_file == QuickStart.quickstart_filename then
-        self:openFileBrowser()
+        -- Like onOpenNextDocumentInFolder, delay this so as not to break instance lifecycle
+        UIManager:nextTick(function()
+            self:openFileBrowser()
+        end)
         return
     end
 
@@ -59,15 +62,15 @@ function ReaderStatus:onEndOfBook()
                         return self.summary.status == "complete" and _("Mark as reading") or _("Mark as finished")
                     end,
                     callback = function()
-                        self:onMarkBook()
                         UIManager:close(button_dialog)
+                        self:onMarkBook()
                     end,
                 },
                 {
                     text = _("Book status"),
                     callback = function()
-                        self:onShowBookStatus()
                         UIManager:close(button_dialog)
+                        self:onShowBookStatus()
                     end,
                 },
 
@@ -76,16 +79,16 @@ function ReaderStatus:onEndOfBook()
                 {
                     text = _("Go to beginning"),
                     callback = function()
-                        self.ui:handleEvent(Event:new("GoToBeginning"))
                         UIManager:close(button_dialog)
+                        self.ui:handleEvent(Event:new("GoToBeginning"))
                     end,
                 },
                 {
                     text = _("Open next file"),
                     enabled = next_file_enabled,
                     callback = function()
-                        self:onOpenNextDocumentInFolder()
                         UIManager:close(button_dialog)
+                        self:onOpenNextDocumentInFolder()
                     end,
                 },
             },
@@ -93,15 +96,18 @@ function ReaderStatus:onEndOfBook()
                 {
                     text = _("Delete file"),
                     callback = function()
-                        self:deleteFile()
                         UIManager:close(button_dialog)
+                        self:deleteFile()
                     end,
                 },
                 {
                     text = _("File browser"),
                     callback = function()
-                        self:openFileBrowser()
                         UIManager:close(button_dialog)
+                        -- Ditto
+                        UIManager:nextTick(function()
+                            self:openFileBrowser()
+                        end)
                     end,
                 },
             },
@@ -123,11 +129,7 @@ function ReaderStatus:onEndOfBook()
             UIManager:show(info)
             UIManager:forceRePaint()
             UIManager:close(info)
-            -- Delay until the next tick, as this will destroy the Document instance,
-            -- but we may not be the final Event caught by said Document...
-            UIManager:nextTick(function()
-                self:onOpenNextDocumentInFolder()
-            end)
+            self:onOpenNextDocumentInFolder()
         else
             UIManager:show(InfoMessage:new{
                 text = _("Could not open next file. Sort by last read date does not support this feature."),
@@ -169,14 +171,14 @@ function ReaderStatus:openFileBrowser()
 end
 
 function ReaderStatus:onOpenNextDocumentInFolder()
-    local FileManager = require("apps/filemanager/filemanager")
-    if not FileManager.instance then
-        self.ui:showFileManager()
-    end
-    local next_file = FileManager.instance.file_chooser:getNextFile(self.document.file)
-    FileManager.instance:onClose()
+    local FileChooser = require("ui/widget/filechooser")
+    local next_file = FileChooser:getNextFile(self.document.file)
     if next_file then
-        self.ui:switchDocument(next_file)
+        -- Delay until the next tick, as this will destroy the Document instance,
+        -- but we may not be the final Event caught by said Document...
+        UIManager:nextTick(function()
+            self.ui:switchDocument(next_file)
+        end)
     else
         UIManager:show(InfoMessage:new{
             text = _("This is the last file in the current folder. No next file to open."),
@@ -200,7 +202,7 @@ end
 function ReaderStatus:onShowBookStatus(before_show_callback)
     local status_page = BookStatusWidget:new {
         thumbnail = FileManagerBookInfo:getCoverImage(self.document),
-        props = self.document:getProps(),
+        props = self.ui.doc_props,
         document = self.document,
         settings = self.settings,
         ui = self.ui,
@@ -217,6 +219,7 @@ end
 -- Otherwise we change status from reading/abandoned to complete or from complete to reading.
 function ReaderStatus:onMarkBook(mark_read)
     self.summary.status = (not mark_read and self.summary.status == "complete") and "reading" or "complete"
+    self.summary.modified = os.date("%Y-%m-%d", os.time())
     -- If History is called over Reader, it will read the file to get the book status, so save and flush
     self.settings:saveSetting("summary", self.summary)
     self.settings:flush()

frontend/apps/reader/modules/readerstyletweak.lua

@@ -1,5 +1,6 @@
 local BD = require("ui/bidi")
 local Blitbuffer = require("ffi/blitbuffer")
+local ButtonDialog = require("ui/widget/buttondialog")
 local ButtonTable = require("ui/widget/buttontable")
 local CenterContainer = require("ui/widget/container/centercontainer")
 local CssTweaks = require("ui/data/css_tweaks")
@@ -152,8 +153,6 @@ function TweakInfoWidget:init()
 
     local button_table = ButtonTable:new{
         width = content:getSize().w,
-        button_font_face = "cfont",
-        button_font_size = 20,
         buttons = buttons,
         zero_sep = true,
         show_parent = self,
@@ -568,10 +567,8 @@ You can enable individual tweaks on this book with a tap, or view more details a
                             if self.tweaks_in_dispatcher[item.id] then
                                 self.tweaks_in_dispatcher[item.id] = nil
                                 dispatcherUnregisterStyleTweak(item.id)
-                                local Profiles = self.ui.profiles
-                                if Profiles then
-                                    Profiles:updateGestures(self.dispatcher_prefix..item.id)
-                                    Profiles:updateProfiles(self.dispatcher_prefix..item.id)
+                                if self.ui.profiles then
+                                    self.ui.profiles:updateProfiles(self.dispatcher_prefix..item.id)
                                 end
                             else
                                 self.tweaks_in_dispatcher[item.id] = item.title
@@ -787,13 +784,123 @@ local BOOK_TWEAK_INPUT_HINT = T([[
 %2]], _("You can add CSS snippets which will be applied only to this book."), BOOK_TWEAK_SAMPLE_CSS)
 
 local CSS_SUGGESTIONS = {
-    { "-cr-hint: footnote-inpage;", _("When set on a block element containing the target id of a href, this block element will be shown as an in-page footnote.")},
-    { "-cr-hint: non-linear-combining;", _("Can be set on some specific DocFragments (ie. DocFragment[id*=16]) to ignore them in the linear pages flow.")},
-    { "-cr-hint: toc-level1;", _("When set on an element, its text can be used to build the alternative table of contents.")},
-    { "display: run-in !important,", _("When set on a block element, this element content will be inlined with the next block element.")},
-    { "font-size: 1rem !important;", _("1rem will enforce your main font size")},
-    { "hyphens: none !important", _("Disables hyphenation inside the targeted elements.")},
-    { "text-indent: 1.2em !important;", _("1.2em is our default text indentation.")},
+    { _("Long-press for info ⓘ"), _([[
+This menu provides a non-exhaustive CSS syntax and properties list. It also shows some KOReader-specific, non-standard CSS features that can be useful with e-books.
+
+Most of these bits are already used by our categorized 'Style tweaks' (found in the top menu). Long-press on any style-tweak option to see its code and its expected results. Should these not be enough to achieve your desired look, you may need to adjust them slightly: tap once on the CSS code-box to copy the code to the clipboard, paste it here and edit it.
+
+Long-press on any item in this popup to get more information on what it does and what it can help solving.
+
+Tap on the item to insert it: you can then edit it and combine it with others.]]), true },
+
+    { _("Matching elements"), {
+        { "p.className", _([[
+p.className matches a <p> with class='className'.
+
+*.className matches any element with class='className'.
+
+p:not([class]) matches a <p> without any class= attribute.]])},
+        { "aside > p", _([[
+aside > p matches a <p> children of an <aside> element.
+
+aside p (without any intermediate symbol) matches a <p> descendant of an <aside> element.]])},
+        { "p + img", _([[
+p + img matches a <img> if its immediate previous sibling is a <p>.
+
+p ~ img matches a <img> if any of its previous siblings is a <p>.]])},
+
+        { "p[name='what']", _([[
+[name="what"] matches if the element has the attribute 'name' and its value is exactly 'what'.
+
+[name] matches if the attribute 'name' is present.
+
+[name~="what"] matches if the value of the attribute 'name' contains 'what' as a word (among other words separated by spaces).]])},
+
+        { "p[name*='what' i]", _([[
+[name*="what" i] matches any element having the attribute 'name' with a value that contains 'what', case insensitive.
+
+[name^="what"] matches if the attribute value starts with 'what'.
+
+[name$="what"] matches if the attribute value ends with 'what'.]])},
+
+        { "p[_='what']", _([[
+Similar in syntax to attribute matching, but matches the inner text of an element.
+
+p[_="what"] matches any <p> whose text is exactly 'what'.
+
+p[_] matches any non-empty <p>.
+
+p:not([_]) matches any empty <p>.
+
+p[_~="what"] matches any <p> that contains the word 'what'.]])},
+
+        { "p[_*='what' i]", _([[
+Similar in syntax to attribute matching, but matches the inner text of an element.
+
+p[_*="what" i] matches any <p> that contains 'what', case insensitive.
+
+p[_^="what"] matches any <p> whose text starts with 'what'.
+(This can be used to match "Act" or "Scene", or character names in plays, and make them stand out.)
+
+p[_$="what"] matches any <p> whose text ends with 'what'.]])},
+
+        { "p:first-child", _([[
+p:first-child matches a <p> that is the first child of its parent.
+
+p:last-child matches a <p> that is the last child of its parent.
+
+p:nth-child(odd) matches any other <p> in a series of sibling <p>.]])},
+
+        { "Tip: use View HTML ⓘ", _([[
+On a book page, select some text spanning around (before and after) the element you are interested in, and use 'View HTML'.
+In the HTML viewer, long press on tags or text to get a list of selectors matching the element: tap on one of them to copy it to the clipboard.
+You can then paste it here with long-press in the text box.]]), true},
+
+    }},
+
+    { _("Common classic properties"), {
+        { "font-size: 1rem !important;", _("1rem will enforce your main font size.")},
+        { "font-weight: normal !important;", _("Remove bold. Use 'bold' to get bold.")},
+        { "hyphens: none !important;", _("Disables hyphenation inside the targeted elements.")},
+        { "text-indent: 1.2em !important;", _("1.2em is our default text indentation.")},
+        { "break-before: always !important;", _("Start a new page with this element. Use 'avoid' to avoid a new page.")},
+        { "color: black !important;", _("Force text to be black.")},
+        { "background: transparent !important;", _("Remove any background color.")},
+        { "max-width: 50vw !important;", _("Limit an element width to 50% of your screen width (use 'max-height: 50vh' for 50% of the screen height). Can be useful with <img> to limit their size.")},
+    }},
+
+    { _("Private CSS properties"), {
+        { "-cr-hint: footnote-inpage;", _("When set on a block element containing the target id of a href, this block element will be shown as an in-page footnote.")},
+        { "-cr-hint: non-linear;", _("Can be set on some specific DocFragments (e.g. DocFragment[id$=_16]) to ignore them in the linear pages flow.")},
+        { "-cr-hint: non-linear-combining;", _("Can be set on contiguous footnote blocks to ignore them in the linear pages flow.")},
+        { "-cr-hint: toc-level1;", _("When set on an element, its text can be used to build the alternative table of contents. toc-level2 to toc-level6 can be used for nested chapters.")},
+        { "-cr-hint: toc-ignore;", _("When set on an element, it will be ignored when building the alternative table of contents.")},
+        { "-cr-hint: footnote;", _("Can be set on target of links (<div id='..'>) to have their link trigger as footnote popup, in case KOReader wrongly detect this target is not a footnote.")},
+        { "-cr-hint: noteref;", _("Can be set on links (<a href='#..'>) to have them trigger as footnote popups, in case KOReader wrongly detect the links is not to a footnote.")},
+        { "-cr-hint: noteref-ignore;", _([[
+Can be set on links (<a href='#..'>) to have them NOT trigger footnote popups and in-page footnotes.
+If some DocFragment presents an index of names with cross references, resulting in in-page footnotes taking half of these pages, you can avoid this with:
+DocFragment[id$=_16] a { -cr-hint: noteref-ignore }]])},
+    }},
+
+    { _("Useful 'content:' values"), {
+        { _("Caution ⚠"), _([[
+Be careful with these: stick them to a proper discriminating selector, like:
+
+span.specificClassName
+
+p[_*="keyword" i]
+
+If used as-is, they will act on ALL elements!]]), true},
+        { "::before {content: ' '}", _("Insert a visible space before an element.")},
+        { "::before {content: '\\A0 '}", _("Insert a visible non-breakable space before an element, so it sticks to what's before.")},
+        { "::before {content: '\\2060'}", _("U+2060 WORD JOINER may act as a glue (like an invisible non-breakable space) before an element, so it sticks to what's before.")},
+        { "::before {content: '\\200B'}", _("U+200B ZERO WIDTH SPACE may allow a linebreak before an element, in case the absence of any space prevents that.")},
+        { "::before {content: attr(title)}", _("Insert the value of the attribute 'title' at start of an element content.")},
+        { "::before {content: '▶ '}", _("Prepend a visible marker.")},
+        { "::before {content: '● '}", _("Prepend a visible marker.")},
+        { "::before {content: '█ '}", _("Prepend a visible marker.")},
+    }},
 }
 
 function ReaderStyleTweak:editBookTweak(touchmenu_instance)
@@ -873,19 +980,89 @@ function ReaderStyleTweak:editBookTweak(touchmenu_instance)
                     local suggestions_popup_widget
                     local buttons = {}
                     for _, suggestion in ipairs(CSS_SUGGESTIONS) do
+                        local title = suggestion[1]
+                        local is_submenu, submenu_items, description
+                        if type(suggestion[2]) == "table" then
+                            is_submenu = true
+                            submenu_items = suggestion[2]
+                        else
+                            description = suggestion[2]
+                        end
+                        local is_info_only = suggestion[3]
+                        local text
+                        if is_submenu then -- add the same arrow we use for top menu submenus
+                            text = require("ui/widget/menu").getMenuText({text=title, sub_item_table=true})
+                        elseif is_info_only then
+                            text = title
+                        else
+                            text = BD.ltr(title) -- CSS code, keep it LTR
+                        end
                         table.insert(buttons, {{
-                            text = suggestion[1],
+                            text = text,
+                            id = title,
                             align = "left",
                             callback = function()
-                                UIManager:close(suggestions_popup_widget)
-                                editor._input_widget:addChars(suggestion[1])
+                                if is_info_only then
+                                    -- No CSS bit to insert, show description also on tap
+                                    UIManager:show(InfoMessage:new{ text = description })
+                                    return
+                                end
+                                if not is_submenu then -- insert as-is on tap
+                                    UIManager:close(suggestions_popup_widget)
+                                    editor:addTextToInput(title)
+                                else
+                                    local sub_suggestions_popup_widget
+                                    local sub_buttons = {}
+                                    for _, sub_suggestion in ipairs(submenu_items) do
+                                        -- (No 2nd level submenu needed for now)
+                                        local sub_title = sub_suggestion[1]
+                                        local sub_description = sub_suggestion[2]
+                                        local sub_is_info_only = sub_suggestion[3]
+                                        local sub_text = sub_is_info_only and sub_title or BD.ltr(sub_title)
+                                        table.insert(sub_buttons, {{
+                                            text = sub_text,
+                                            align = "left",
+                                            callback = function()
+                                                if sub_is_info_only then
+                                                    UIManager:show(InfoMessage:new{ text = sub_description })
+                                                    return
+                                                end
+                                                UIManager:close(sub_suggestions_popup_widget)
+                                                UIManager:close(suggestions_popup_widget)
+                                                editor:addTextToInput(sub_title)
+                                            end,
+                                            hold_callback = sub_description and function()
+                                                UIManager:show(InfoMessage:new{ text = sub_description })
+                                            end,
+                                        }})
+                                    end
+                                    local anchor_func = function()
+                                        local d = suggestions_popup_widget:getButtonById(title).dimen:copy()
+                                        if BD.mirroredUILayout() then
+                                            d.x = d.x - d.w + Size.padding.default
+                                        else
+                                            d.x = d.x + d.w - Size.padding.default
+                                        end
+                                        -- As we don't know if we will pop up or down, anchor it on the middle of the item
+                                        d.y = d.y + math.floor(d.h / 2)
+                                        d.h = 1
+                                        return d, true
+                                    end
+                                    sub_suggestions_popup_widget = ButtonDialog:new{
+                                        modal = true, -- needed when keyboard is shown
+                                        width = math.floor(Screen:getWidth() * 0.9), -- max width, will get smaller
+                                        shrink_unneeded_width = true,
+                                        buttons = sub_buttons,
+                                        anchor = anchor_func,
+                                    }
+                                    UIManager:show(sub_suggestions_popup_widget)
+                                end
                             end,
-                            hold_callback = suggestion[2] and function()
-                                UIManager:show(InfoMessage:new{ text = suggestion[2] })
+                            hold_callback = description and function()
+                                UIManager:show(InfoMessage:new{ text = description })
                             end or nil
                         }})
                     end
-                    local ButtonDialog = require("ui/widget/buttondialog")
                     suggestions_popup_widget = ButtonDialog:new{
                         modal = true, -- needed when keyboard is shown
                         width = math.floor(Screen:getWidth() * 0.9), -- max width, will get smaller
@@ -920,10 +1097,10 @@ function ReaderStyleTweak:editBookTweak(touchmenu_instance)
                 end
             end
         end,
-        -- Set/save view and cursor position callback
+        -- Store/retrieve view and cursor position callback
         view_pos_callback = function(top_line_num, charpos)
-            -- This same callback is called with no argument to get initial position,
-            -- and with arguments to give back final position when closed.
+            -- This same callback is called with no arguments on init to retrieve the stored initial position,
+            -- and with arguments to store the final position on close.
             if top_line_num and charpos then
                 self.book_style_tweak_last_edit_pos = {top_line_num, charpos}
             else

frontend/apps/reader/modules/readerthumbnail.lua

@@ -186,11 +186,11 @@ function ReaderThumbnail:removeFromCache(hash_subs, remove_only_non_matching)
     return nb_removed, size_removed
 end
 
-function ReaderThumbnail:resetCachedPagesForBookmarks(...)
+function ReaderThumbnail:resetCachedPagesForBookmarks(annotations)
     -- Multiple bookmarks may be provided
     local start_page, end_page
-    for i = 1, select("#", ...) do
-        local bm = select(i, ...)
+    for i = 1, #annotations do
+        local bm = annotations[i]
         if self.ui.rolling then
             -- Look at all properties that may be xpointers
             for _, k in ipairs({"page", "pos0", "pos1"}) do
@@ -250,6 +250,10 @@ end
 function ReaderThumbnail:getPageThumbnail(page, width, height, batch_id, when_generated_callback)
     self:setupCache()
     self.current_target_size_tag = string.format("w%d_h%d", width, height)
+    if self.ui.rolling and Screen.night_mode and self.ui.document.configurable.nightmode_images == 1 then
+        -- We'll get a different bb in this case: it needs its own cache hash
+        self.current_target_size_tag = self.current_target_size_tag .. "_nm"
+    end
     local hash = string.format("p%d-%s", page, self.current_target_size_tag)
     local tile = self.tile_cache and self.tile_cache:check(hash)
     if tile then
@@ -274,6 +278,10 @@ function ReaderThumbnail:getPageThumbnail(page, width, height, batch_id, when_ge
 end
 
 function ReaderThumbnail:ensureTileGeneration()
+    if not self._standby_prevented then
+        self._standby_prevented = true
+        UIManager:preventStandby()
+    end
     local has_pids_still_to_collect = self:collectPids()
 
     local still_in_progress = false
@@ -314,6 +322,11 @@ function ReaderThumbnail:ensureTileGeneration()
     end
     if self.req_in_progress or has_pids_still_to_collect or next(self.thumbnails_requests) then
         self._ensureTileGeneration_action()
+    else
+        if self._standby_prevented then
+            self._standby_prevented = false
+            UIManager:allowStandby()
+        end
     end
 end
 
@@ -420,12 +433,18 @@ function ReaderThumbnail:_getPageImage(page)
         -- CRE documents: pages all have the aspect ratio of our screen (alt top status bar
         -- will be croped out after drawing), we will show them just as rendered.
         self.ui.rolling.rendering_state = nil -- Remove any partial rerendering icon
-        self.ui.view:onSetViewMode("page") -- Get out of scroll mode
-        if self.ui.font.gamma_index < 30 then  -- Increase font gamma (if not already increased),
+        if self.ui.view.view_mode == "scroll" then
+            -- Get out of scroll mode, and be sure we'll be in one-page mode as that
+            -- is what is shown in scroll mode (needs to do the following in that
+            -- order to avoid rendering hash change)
+            self.ui.rolling:onSetVisiblePages(1)
+            self.ui.view:onSetViewMode("page")
+        end
+        if self.ui.document.configurable.font_gamma < 30 then  -- Increase font gamma (if not already increased),
             self.ui.document:setGammaIndex(30) -- as downscaling will make text grayer
         end
         self.ui.document:setImageScaling(false) -- No need for smooth scaling as all will be downscaled
-        self.ui.document:setNightmodeImages(false) -- We don't invert page images even if nightmode set: keep images as-is
+        -- (We keep "nighmode_images" as it was set: we may get and cache a different bb whether nightmode is on or off)
         self.ui.view.state.page = page -- Be on requested page
         self.ui.document:gotoPage(page) -- Current xpointer needs to be updated for some of what follows
         self.ui.bookmark:onPageUpdate(page) -- Update dogear state for this page
@@ -503,6 +522,10 @@ function ReaderThumbnail:onCloseDocument()
         self.tile_cache:clear()
         self.tile_cache = nil
     end
+    if self._standby_prevented then
+        self._standby_prevented = false
+        UIManager:allowStandby()
+    end
 end
 
 function ReaderThumbnail:onColorRenderingUpdate()
@@ -514,9 +537,6 @@ end
 ReaderThumbnail.onDocumentRerendered = ReaderThumbnail.resetCache
 ReaderThumbnail.onDocumentPartiallyRerendered = ReaderThumbnail.resetCache
 -- Emitted When adding/removing/updating bookmarks and highlights
-ReaderThumbnail.onBookmarkAdded = ReaderThumbnail.resetCachedPagesForBookmarks
-ReaderThumbnail.onBookmarkRemoved = ReaderThumbnail.resetCachedPagesForBookmarks
-ReaderThumbnail.onBookmarkUpdated = ReaderThumbnail.resetCachedPagesForBookmarks
-ReaderThumbnail.onBookmarkEdited = ReaderThumbnail.resetCachedPagesForBookmarks
+ReaderThumbnail.onAnnotationsModified = ReaderThumbnail.resetCachedPagesForBookmarks
 
 return ReaderThumbnail

frontend/apps/reader/modules/readertoc.lua

@@ -29,9 +29,8 @@ local ReaderToc = InputContainer:extend{
     collapsed_toc = nil, -- table
     collapse_depth = 2,
     expanded_nodes = nil, -- table
-    toc_menu_title = _("Table of contents"),
-    alt_toc_menu_title = _("Table of contents *"),
     toc_items_per_page_default = 14,
+    alt_toc_symbol = "\u{E298}" -- two-folders in circle (custom toc uses a pen in square symbol)
 }
 
 function ReaderToc:init()
@@ -61,11 +60,24 @@ end
 function ReaderToc:onGesture() end
 
 function ReaderToc:registerKeyEvents()
-    if Device:hasKeyboard() then
+    if Device:hasScreenKB() then
+        self.key_events.ShowToc = { { "ScreenKB", "Up" } }
+    elseif Device:hasKeyboard() then
         self.key_events.ShowToc = { { "T" } }
     end
 end
 
+function ReaderToc:getTitle()
+    local title = _("Table of contents")
+    -- Handmade ToC has precedence over alternative ToC
+    if self.ui.handmade:isHandmadeTocEnabled() then
+        title = title .. " " .. self.ui.handmade.custom_toc_symbol
+    elseif self.ui.document:isTocAlternativeToc() then
+        title = title .. " " .. self.alt_toc_symbol
+    end
+    return title
+end
+
 ReaderToc.onPhysicalKeyboardConnected = ReaderToc.registerKeyEvents
 
 function ReaderToc:onReadSettings(config)
@@ -94,6 +106,7 @@ end
 
 function ReaderToc:resetToc()
     self.toc = nil
+    self.toc_menu_items_built = false
     self.toc_depth = nil
     self.ticks = nil
     self.ticks_flattened = nil
@@ -180,8 +193,10 @@ function ReaderToc:validateAndFixToc()
     local has_bogus
     local cur_page = 0
     local max_depth = 0
+    local cur_seq_by_level = {}
     for i = first, last do
-        local page = toc[i].page
+        local item = toc[i]
+        local page = item.page
         if page < cur_page then
             has_bogus = true
             break
@@ -189,9 +204,15 @@ function ReaderToc:validateAndFixToc()
         cur_page = page
         -- Use this loop to compute max_depth here (if has_bogus,
         -- we will recompute it in the loop below)
-        if toc[i].depth > max_depth then
-            max_depth = toc[i].depth
+        local depth = item.depth
+        if depth > max_depth then
+            max_depth = depth
         end
+        -- Also use this loop to compute seq_in_level for each
+        -- item, needed by BookMap with alternative theme
+        local seq = (cur_seq_by_level[depth] or 0) + 1
+        item.seq_in_level = seq
+        cur_seq_by_level[depth] = seq
     end
     if not has_bogus then -- no TOC items, or all are valid
         logger.dbg("validateAndFixToc(): TOC is fine")
@@ -203,6 +224,7 @@ function ReaderToc:validateAndFixToc()
     -- Bad ordering previously noticed: try to fix the wrong items' page
     -- by setting it to the previous or next good item page.
     max_depth = 0 -- recompute this
+    cur_seq_by_level = {}
     local nb_bogus = 0
     local nb_fixed_pages = 0
     -- We fix only one bogus item per loop, taking the option that
@@ -215,10 +237,17 @@ function ReaderToc:validateAndFixToc()
     -- (These cases are met in the following code with cur_page=57 and page=6)
     cur_page = 0
     for i = first, last do
-        if toc[i].depth > max_depth then
-            max_depth = toc[i].depth
+        local item = toc[i]
+        -- Recompute max_depth and item's seq_in_level
+        local depth = item.depth
+        if depth > max_depth then
+            max_depth = depth
         end
-        local page = toc[i].fixed_page or toc[i].page
+        local seq = (cur_seq_by_level[depth] or 0) + 1
+        item.seq_in_level = seq
+        cur_seq_by_level[depth] = seq
+        -- Look for bogus page
+        local page = item.fixed_page or item.page
         if page >= cur_page then
             cur_page = page
         else
@@ -272,7 +301,7 @@ function ReaderToc:validateAndFixToc()
                     logger.dbg("  fix next", j, toc[j].page, "=>", fixed_page)
                 end
             end
-            cur_page = toc[i].fixed_page or toc[i].page
+            cur_page = item.fixed_page or item.page
         end
     end
     if nb_bogus > 0 then
@@ -288,6 +317,37 @@ function ReaderToc:validateAndFixToc()
     self.toc_depth = max_depth
 end
 
+function ReaderToc:completeTocWithChapterLengths()
+    local toc = self.toc
+    local first = 1
+    local last = #toc
+    if last == 0 then
+        return
+    end
+    local prev_item_by_level = {}
+    for i = first, last do
+        local item = toc[i]
+        local page = item.page
+        local depth = item.depth
+        for j=#prev_item_by_level, depth, -1 do
+            local prev_item = prev_item_by_level[j]
+            if prev_item then
+                prev_item.chapter_length = page - prev_item.page
+            end
+            prev_item_by_level[j] = nil
+        end
+        prev_item_by_level[depth] = item
+    end
+    -- Set the length of the last ones
+    local page = self.ui.document:getPageCount()
+    for j=#prev_item_by_level, 0, -1 do
+        local prev_item = prev_item_by_level[j]
+        if prev_item then
+            prev_item.chapter_length = page - prev_item.page
+        end
+    end
+end
+
 function ReaderToc:getTocIndexByPage(pn_or_xp, skip_ignored_ticks)
     self:fillToc()
     if #self.toc == 0 then return end
@@ -295,7 +355,7 @@ function ReaderToc:getTocIndexByPage(pn_or_xp, skip_ignored_ticks)
     if type(pn_or_xp) == "string" then
         return self:getAccurateTocIndexByXPointer(pn_or_xp, skip_ignored_ticks)
     end
-    local prev_index = 1
+    local prev_index = 0
     for _k,_v in ipairs(self.toc) do
         if not skip_ignored_ticks or not self.toc_ticks_ignored_levels[_v.depth] then
             if _v.page == pageno then
@@ -310,7 +370,7 @@ function ReaderToc:getTocIndexByPage(pn_or_xp, skip_ignored_ticks)
             prev_index = _k
         end
     end
-    return prev_index
+    return prev_index > 0 and prev_index or nil
 end
 
 function ReaderToc:getAccurateTocIndexByXPointer(xptr, skip_ignored_ticks)
@@ -543,78 +603,54 @@ function ReaderToc:isChapterEnd(cur_pageno)
 end
 
 function ReaderToc:getChapterPageCount(pageno)
-    if self.ui.document:hasHiddenFlows() then
-        -- Count pages until new chapter, starting by going backwards to the beginning of the current chapter if necessary
-        local page_count = 1
-        if not self:isChapterStart(pageno) then
-            local test_page = self.ui.document:getPrevPage(pageno)
-            while test_page > 0 do
-                page_count = page_count + 1
-                if self:isChapterStart(test_page) then
-                    break
-                end
-                test_page = self.ui.document:getPrevPage(test_page)
-            end
-        end
-
-        -- Then forward
-        local test_page = self.ui.document:getNextPage(pageno)
-        while test_page > 0 do
-            page_count = page_count + 1
-            if self:isChapterStart(test_page) then
-                return page_count - 1
+    local next_chapter = self:getNextChapter(pageno) or self.ui.document:getPageCount() + 1
+    local previous_chapter = self:isChapterStart(pageno) and pageno or self:getPreviousChapter(pageno) or 1
+    local page_count = next_chapter - previous_chapter
+    if self.ui.document:hasHiddenFlows() and self.ui.document:getPageFlow(pageno) == 0 then
+        -- If current page in a hidden flow, return the full amount of pages in this chapter.
+        -- Otherwise, count only pages in the main flow
+        for page = previous_chapter, next_chapter - 1 do
+            if self.ui.document:getPageFlow(page) ~= 0 then
+                page_count = page_count - 1
             end
-            test_page = self.ui.document:getNextPage(test_page)
         end
-    else
-        local next_chapter = self:getNextChapter(pageno) or self.ui.document:getPageCount() + 1
-        local previous_chapter = self:isChapterStart(pageno) and pageno or self:getPreviousChapter(pageno) or 1
-        local page_count = next_chapter - previous_chapter
-        return page_count
     end
+    return page_count
 end
 
 function ReaderToc:getChapterPagesLeft(pageno)
-    if self.ui.document:hasHiddenFlows() then
-        -- Count pages until new chapter
-        local pages_left = 0
-        local test_page = self.ui.document:getNextPage(pageno)
-        while test_page > 0 do
-            pages_left = pages_left + 1
-            if self:isChapterStart(test_page) then
-                return pages_left - 1
+    local next_chapter = self:getNextChapter(pageno)
+    if not next_chapter then
+        -- (ReaderFooter deals itself with nil and pageno in last chapter)
+        return
+    end
+    local pages_left = next_chapter - pageno - 1
+    if self.ui.document:hasHiddenFlows() and self.ui.document:getPageFlow(pageno) == 0 then
+        for page = pageno, next_chapter - 1 do
+            if self.ui.document:getPageFlow(page) ~= 0 then
+                pages_left = pages_left - 1
             end
-            test_page = self.ui.document:getNextPage(test_page)
         end
-    else
-        local next_chapter = self:getNextChapter(pageno)
-        if next_chapter then
-            next_chapter = next_chapter - pageno - 1
-        end
-        return next_chapter
     end
+    return pages_left
 end
 
 function ReaderToc:getChapterPagesDone(pageno)
     if self:isChapterStart(pageno) then return 0 end
-    if self.ui.document:hasHiddenFlows() then
-        -- Count pages until chapter start
-        local pages_done = 0
-        local test_page = self.ui.document:getPrevPage(pageno)
-        while test_page > 0 do
-            pages_done = pages_done + 1
-            if self:isChapterStart(test_page) then
-                return pages_done
+    local previous_chapter = self:getPreviousChapter(pageno)
+    if not previous_chapter then
+        -- (ReaderFooter deals itself with nil and pageno not yet in first chapter)
+        return
+    end
+    local pages_done = pageno - previous_chapter
+    if self.ui.document:hasHiddenFlows() and self.ui.document:getPageFlow(pageno) == 0 then
+        for page = previous_chapter, pageno - 1 do
+            if self.ui.document:getPageFlow(page) ~= 0 then
+                pages_done = pages_done - 1
             end
-            test_page = self.ui.document:getPrevPage(test_page)
-        end
-    else
-        local previous_chapter = self:getPreviousChapter(pageno)
-        if previous_chapter then
-            previous_chapter = pageno - previous_chapter
         end
-        return previous_chapter
     end
+    return pages_done
 end
 
 function ReaderToc:updateCurrentNode()
@@ -659,11 +695,16 @@ function ReaderToc:onShowToc()
 
     local items_per_page = G_reader_settings:readSetting("toc_items_per_page") or self.toc_items_per_page_default
     local items_font_size = G_reader_settings:readSetting("toc_items_font_size") or Menu.getItemFontSize(items_per_page)
+    local items_show_chapter_length = G_reader_settings:isTrue("toc_items_show_chapter_length")
     local items_with_dots = G_reader_settings:nilOrTrue("toc_items_with_dots")
 
     self:fillToc()
     -- build menu items
-    if #self.toc > 0 and not self.toc[1].text then
+    if #self.toc > 0 and not self.toc_menu_items_built then
+        self.toc_menu_items_built = true
+        if items_show_chapter_length then
+            self:completeTocWithChapterLengths()
+        end
         -- Have the width of 4 spaces be the unit of indentation
         local tmp = TextWidget:new{
             text = "    ",
@@ -677,6 +718,11 @@ function ReaderToc:onShowToc()
             v.index = k
             v.indent = toc_indent * (v.depth-1)
             v.text = self:cleanUpTocTitle(v.title, true)
+            if items_show_chapter_length then
+                v.post_text = T("(%1)", v.chapter_length)
+            else
+                v.post_text = nil
+            end
             v.bidi_wrap_func = BD.auto
             v.mandatory = v.page
             if has_hidden_flows then
@@ -764,7 +810,7 @@ function ReaderToc:onShowToc()
     --       This yields *slightly* better alignment between state & mandatory (in terms of effective margins).
     local button_size = self.expand_button:getSize()
     local toc_menu = Menu:new{
-        title = _("Table of Contents"),
+        title = self:getTitle(),
         item_table = self.collapsed_toc,
         state_w = can_collapse and button_size.w or 0,
         ui = self.ui,
@@ -946,7 +992,7 @@ function ReaderToc:addToMainMenu(menu_items)
     -- insert table to main reader menu
     menu_items.table_of_contents = {
         text_func = function()
-            return self.ui.document:isTocAlternativeToc() and self.alt_toc_menu_title or self.toc_menu_title
+            return self:getTitle()
         end,
         callback = function()
             self:onShowToc()
@@ -959,13 +1005,22 @@ function ReaderToc:addToMainMenu(menu_items)
     -- Alternative ToC (only available with CRE documents)
     if self.ui.document:canHaveAlternativeToc() then
         menu_items.toc_alt_toc = {
-            text = _("Alternative table of contents"),
-            help_text = _([[
+            text = _("Alternative table of contents") .. " " .. self.alt_toc_symbol,
+            help_text_func = function()
+                local help_text = _([[
 An alternative table of contents can be built from document headings <H1> to <H6>.
 If the document contains no headings, or all are ignored, the alternative ToC will be built from document fragments and will point to the start of each individual HTML file in the EPUB.
 
 Some of the headings can be ignored, and hints can be set to other non-heading elements in a user style tweak, so they can be used as ToC items.
-See Style tweaks → Miscellaneous → Alternative ToC hints.]]),
+See Style tweaks → Miscellaneous → Alternative ToC hints.]])
+                if self.ui.handmade:isHandmadeTocEnabled() then
+                    help_text = _([[To use the alternative ToC, disable your custom table of contents first.]]) .. "\n\n" .. help_text
+                end
+                return help_text
+            end,
+            enabled_func = function()
+                return not self.ui.handmade:isHandmadeTocEnabled()
+            end,
             checked_func = function()
                 return self.ui.document:isTocAlternativeToc()
             end,
@@ -1136,6 +1191,17 @@ Enabling this option will restrict display to the chapter titles of progress bar
             UIManager:show(items_font)
         end,
     }
+    menu_items.toc_items_show_chapter_length = {
+        text = _("Show chapter length"),
+        keep_menu_open = true,
+        checked_func = function()
+            return not G_reader_settings:nilOrFalse("toc_items_show_chapter_length")
+        end,
+        callback = function()
+            G_reader_settings:flipNilOrFalse("toc_items_show_chapter_length")
+            self.toc_menu_items_built = false
+        end
+    }
     menu_items.toc_items_with_dots = {
         text = _("With dots"),
         keep_menu_open = true,

frontend/apps/reader/modules/readertypeset.lua

@@ -17,7 +17,6 @@ local ReaderTypeset = WidgetContainer:extend{
     -- @translators This is style in the sense meant by CSS (cascading style sheets), relating to the layout and presentation of the document. See <https://en.wikipedia.org/wiki/CSS> for more information.
     css_menu_title = C_("CSS", "Style"),
     css = nil,
-    internal_css = true,
     unscaled_margins = nil,
 }
 
@@ -27,38 +26,24 @@ end
 
 function ReaderTypeset:onReadSettings(config)
     self.css = config:readSetting("css")
-            or G_reader_settings:readSetting("copt_css")
-            or self.ui.document.default_css
+    if not self.css then
+        if self.ui.document.is_fb2 then
+            self.css = G_reader_settings:readSetting("copt_fb2_css")
+        else
+            self.css = G_reader_settings:readSetting("copt_css")
+        end
+    end
+    if not self.css then
+        self.css = self.ui.document.default_css
+    end
     local tweaks_css = self.ui.styletweak:getCssText()
     self.ui.document:setStyleSheet(self.css, tweaks_css)
 
-    if config:has("embedded_fonts") then
-        self.embedded_fonts = config:isTrue("embedded_fonts")
-    else
-        -- default to enable embedded fonts
-        -- note that it's a bit confusing here:
-        -- global settins store 0/1, while document settings store false/true
-        -- we leave it that way for now to maintain backwards compatibility
-        local global = G_reader_settings:readSetting("copt_embedded_fonts")
-        self.embedded_fonts = (global == nil or global == 1) and true or false
-    end
-    -- As this is new, call it only when embedded_fonts are explicitely disabled
-    -- self.ui.document:setEmbeddedFonts(self.embedded_fonts and 1 or 0)
-    if not self.embedded_fonts then
-        self.ui.document:setEmbeddedFonts(0)
-    end
+    -- default to enable embedded fonts
+    self.ui.document:setEmbeddedFonts(self.configurable.embedded_fonts)
 
-    if config:has("embedded_css") then
-        self.embedded_css = config:isTrue("embedded_css")
-    else
-        -- default to enable embedded CSS
-        -- note that it's a bit confusing here:
-        -- global settings store 0/1, while document settings store false/true
-        -- we leave it that way for now to maintain backwards compatibility
-        local global = G_reader_settings:readSetting("copt_embedded_css")
-        self.embedded_css = (global == nil or global == 1) and true or false
-    end
-    self.ui.document:setEmbeddedStyleSheet(self.embedded_css and 1 or 0)
+    -- default to enable embedded CSS
+    self.ui.document:setEmbeddedStyleSheet(self.configurable.embedded_css)
 
     -- Block rendering mode: stay with legacy rendering for books
     -- previously opened so bookmarks and highlights stay valid.
@@ -74,104 +59,98 @@ function ReaderTypeset:onReadSettings(config)
                                      or 3 -- default to 'web' mode
         end
         -- Let ConfigDialog know so it can update it on screen and have it saved on quit
-        self.ui.document.configurable.block_rendering_mode = self.block_rendering_mode
+        self.configurable.block_rendering_mode = self.block_rendering_mode
     end
     self:setBlockRenderingMode(self.block_rendering_mode)
 
-    -- set render DPI
-    self.render_dpi = config:readSetting("render_dpi")
-                   or G_reader_settings:readSetting("copt_render_dpi")
-                   or 96
-    self:setRenderDPI(self.render_dpi)
+    -- default to 96 dpi
+    self.ui.document:setRenderDPI(self.configurable.render_dpi)
 
     -- uncomment if we want font size to follow DPI changes
     -- self.ui.document:setRenderScaleFontWithDPI(1)
 
     -- set page margins
-    local h_margins = config:readSetting("copt_h_page_margins")
-                   or G_reader_settings:readSetting("copt_h_page_margins")
-                   or G_defaults:readSetting("DCREREADER_CONFIG_H_MARGIN_SIZES_MEDIUM")
-    local t_margin = config:readSetting("copt_t_page_margin")
-                  or G_reader_settings:readSetting("copt_t_page_margin")
-                  or G_defaults:readSetting("DCREREADER_CONFIG_T_MARGIN_SIZES_LARGE")
-    local b_margin = config:readSetting("copt_b_page_margin")
-                  or G_reader_settings:readSetting("copt_b_page_margin")
-                  or G_defaults:readSetting("DCREREADER_CONFIG_B_MARGIN_SIZES_LARGE")
-    self.unscaled_margins = { h_margins[1], t_margin, h_margins[2], b_margin }
+    self.unscaled_margins = { self.configurable.h_page_margins[1], self.configurable.t_page_margin,
+                              self.configurable.h_page_margins[2], self.configurable.b_page_margin }
     self:onSetPageMargins(self.unscaled_margins)
-    self.sync_t_b_page_margins = config:readSetting("copt_sync_t_b_page_margins")
-                              or G_reader_settings:readSetting("copt_sync_t_b_page_margins")
-                              or 0
-    self.sync_t_b_page_margins = self.sync_t_b_page_margins == 1 and true or false
+    self.sync_t_b_page_margins = self.configurable.sync_t_b_page_margins == 1 and true or false
 
-    -- default to disable TXT formatting as it does more harm than good
+    -- default to disable TXT formatting as it does more harm than good (the setting is not in UI)
     self.txt_preformatted = config:readSetting("txt_preformatted")
                          or G_reader_settings:readSetting("txt_preformatted")
                          or 1
-    self:toggleTxtPreFormatted(self.txt_preformatted)
+    self.ui.document:setTxtPreFormatted(self.txt_preformatted)
 
-    -- default to disable smooth scaling for now.
-    if config:has("smooth_scaling") then
-        self.smooth_scaling = config:isTrue("smooth_scaling")
-    else
-        local global = G_reader_settings:readSetting("copt_smooth_scaling")
-        self.smooth_scaling = global == 1 and true or false
-    end
-    self:toggleImageScaling(self.smooth_scaling)
+    -- default to disable smooth scaling
+    self.ui.document:setImageScaling(self.configurable.smooth_scaling == 1)
 
     -- default to automagic nightmode-friendly handling of images
-    if config:has("nightmode_images") then
-        self.nightmode_images = config:isTrue("nightmode_images")
-    else
-        local global = G_reader_settings:readSetting("copt_nightmode_images")
-        self.nightmode_images = (global == nil or global == 1) and true or false
-    end
-    self:toggleNightmodeImages(self.nightmode_images)
+    self.ui.document:setNightmodeImages(self.configurable.nightmode_images == 1)
 end
 
 function ReaderTypeset:onSaveSettings()
     self.ui.doc_settings:saveSetting("css", self.css)
-    self.ui.doc_settings:saveSetting("embedded_css", self.embedded_css)
-    self.ui.doc_settings:saveSetting("embedded_fonts", self.embedded_fonts)
-    self.ui.doc_settings:saveSetting("render_dpi", self.render_dpi)
-    self.ui.doc_settings:saveSetting("smooth_scaling", self.smooth_scaling)
-    self.ui.doc_settings:saveSetting("nightmode_images", self.nightmode_images)
 end
 
 function ReaderTypeset:onToggleEmbeddedStyleSheet(toggle)
-    self:toggleEmbeddedStyleSheet(toggle)
+    local text
     if toggle then
-        Notification:notify(_("Enabled embedded styles."))
+        self.configurable.embedded_css = 1
+        text = _("Enabled embedded styles.")
     else
-        Notification:notify(_("Disabled embedded styles."))
+        self.configurable.embedded_css = 0
+        text = _("Disabled embedded styles.")
     end
+    self.ui.document:setEmbeddedStyleSheet(self.configurable.embedded_css)
+    self.ui:handleEvent(Event:new("UpdatePos"))
+    Notification:notify(text)
     return true
 end
 
 function ReaderTypeset:onToggleEmbeddedFonts(toggle)
-    self:toggleEmbeddedFonts(toggle)
+    local text
     if toggle then
-        Notification:notify(_("Enabled embedded fonts."))
+        self.configurable.embedded_fonts = 1
+        text = _("Enabled embedded fonts.")
     else
-        Notification:notify(_("Disabled embedded fonts."))
+        self.configurable.embedded_fonts = 0
+        text = _("Disabled embedded fonts.")
     end
+    self.ui.document:setEmbeddedFonts(self.configurable.embedded_fonts)
+    self.ui:handleEvent(Event:new("UpdatePos"))
+    Notification:notify(text)
     return true
 end
 
 function ReaderTypeset:onToggleImageScaling(toggle)
-    self:toggleImageScaling(toggle)
-    Notification:notify(T( _("Image scaling set to: %1"), optionsutil:getOptionText("ToggleImageScaling", toggle)))
+    self.configurable.smooth_scaling = toggle and 1 or 0
+    self.ui.document:setImageScaling(toggle)
+    self.ui:handleEvent(Event:new("UpdatePos"))
+    local text = T(_("Image scaling set to: %1"), optionsutil:getOptionText("ToggleImageScaling", toggle))
+    Notification:notify(text)
     return true
 end
 
 function ReaderTypeset:onToggleNightmodeImages(toggle)
-    self:toggleNightmodeImages(toggle)
+    self.configurable.nightmode_images = toggle and 1 or 0
+    self.ui.document:setNightmodeImages(toggle)
+    self.ui:handleEvent(Event:new("UpdatePos"))
     return true
 end
 
 function ReaderTypeset:onSetBlockRenderingMode(mode)
     self:setBlockRenderingMode(mode)
-    Notification:notify(T( _("Render mode set to: %1"), optionsutil:getOptionText("SetBlockRenderingMode", mode)))
+    local text = T(_("Render mode set to: %1"), optionsutil:getOptionText("SetBlockRenderingMode", mode))
+    Notification:notify(text)
+    return true
+end
+
+function ReaderTypeset:onSetRenderDPI(dpi)
+    self.configurable.render_dpi = dpi
+    self.ui.document:setRenderDPI(dpi)
+    self.ui:handleEvent(Event:new("UpdatePos"))
+    local text = T(_("Zoom set to: %1"), optionsutil:getOptionText("SetRenderDPI", dpi))
+    Notification:notify(text)
     return true
 end
 
@@ -191,23 +170,18 @@ local OBSOLETED_CSS = {
     "txt.css",
 }
 
-function ReaderTypeset:onSetRenderDPI(dpi)
-    self:setRenderDPI(dpi)
-    Notification:notify(T( _("Zoom set to: %1"), optionsutil:getOptionText("SetRenderDPI", dpi)))
-    return true
-end
-
 function ReaderTypeset:genStyleSheetMenu()
-    local getStyleMenuItem = function(text, css_file, separator)
+    local getStyleMenuItem = function(text, css_file, description, fb2_compatible, separator)
         return {
             text_func = function()
-                return text .. (css_file == G_reader_settings:readSetting("copt_css") and "   ★" or "")
+                local css_opt = self.ui.document.is_fb2 and "copt_fb2_css" or "copt_css"
+                return text .. (css_file == G_reader_settings:readSetting(css_opt) and "   ★" or "")
             end,
             callback = function()
                 self:setStyleSheet(css_file or self.ui.document.default_css)
             end,
             hold_callback = function(touchmenu_instance)
-                self:makeDefaultStyleSheet(css_file, text, touchmenu_instance)
+                self:makeDefaultStyleSheet(css_file, text, description, touchmenu_instance)
             end,
             checked_func = function()
                 if not css_file then -- "Auto"
@@ -215,6 +189,16 @@ function ReaderTypeset:genStyleSheetMenu()
                 end
                 return css_file == self.css
             end,
+            enabled_func = function()
+                if fb2_compatible == true and not self.ui.document.is_fb2 then
+                    return false
+                end
+                if fb2_compatible == false and self.ui.document.is_fb2 then
+                    return false
+                end
+                -- if fb2_compatible==nil, we don't know (user css file)
+                return true
+            end,
             separator = separator,
         }
     end
@@ -222,8 +206,18 @@ function ReaderTypeset:genStyleSheetMenu()
     local style_table = {}
     local obsoleted_table = {}
 
-    table.insert(style_table, getStyleMenuItem(_("None"), ""))
-    table.insert(style_table, getStyleMenuItem(_("Auto"), nil, true))
+    table.insert(style_table, getStyleMenuItem(
+        _("None"),
+        "",
+        _("This sets an empty User-Agent stylesheet, and expects the document stylesheet to style everything (which publishers probably don't).\nThis is mostly only interesting for testing.")
+    ))
+    table.insert(style_table, getStyleMenuItem(
+        _("Auto"),
+        nil,
+        _("This selects the default and preferred stylesheet for the document type."),
+        nil,
+        true -- separator
+    ))
 
     local css_files = {}
     for f in lfs.dir("./data") do
@@ -233,15 +227,39 @@ function ReaderTypeset:genStyleSheetMenu()
     end
     -- Add the 3 main styles
     if css_files["epub.css"] then
-        table.insert(style_table, getStyleMenuItem(_("HTML / EPUB (epub.css)"), css_files["epub.css"]))
+        table.insert(style_table, getStyleMenuItem(
+            _("Traditional book look (epub.css)"),
+            css_files["epub.css"],
+            _([[
+This is our book look-alike stylesheet: it extends the HTML standard stylesheet with styles aimed at making HTML content look more like a paper book (with justified text and indentation on paragraphs) than like a web page.
+It is perfect for unstyled books, and might make styled books more readable.
+It may cause some small issues on some books (miscentered titles, headings or separators, or unexpected text indentation), as publishers don't expect to have our added styles at play and need to reset them; try switching to html5.css when you notice such issues.]]),
+            false -- not fb2_compatible
+        ))
         css_files["epub.css"] = nil
     end
     if css_files["html5.css"] then
-        table.insert(style_table, getStyleMenuItem(_("HTML5 (html5.css)"), css_files["html5.css"]))
+        table.insert(style_table, getStyleMenuItem(
+            _("HTML Standard rendering (html5.css)"),
+            css_files["html5.css"],
+            _([[
+This stylesheet conforms to the HTML Standard rendering suggestions (with a few limitations), similar to what most web browsers use.
+As most publishers nowadays make and test their book with tools based on web browser engines, it is the stylesheet to use to see a book as these publishers intended.
+On unstyled books though, it may give them the look of a web page (left aligned paragraphs without indentation and with spacing between them); try switching to epub.css when that happens.]]),
+            false -- not fb2_compatible
+        ))
         css_files["html5.css"] = nil
     end
     if css_files["fb2.css"] then
-        table.insert(style_table, getStyleMenuItem(_("FictionBook (fb2.css)"), css_files["fb2.css"], true))
+        table.insert(style_table, getStyleMenuItem(
+            _("FictionBook (fb2.css)"),
+            css_files["fb2.css"],
+            _([[
+This stylesheet is to be used only with FB2 and FB3 documents, which are not classic HTML, and need some specific styling.
+(FictionBook 2 & 3 are open XML-based e-book formats which originated and gained popularity in Russia.)]]),
+            true, -- fb2_compatible
+            true -- separator
+        ))
         css_files["fb2.css"] = nil
     end
     -- Add the obsoleted ones to the Obsolete sub menu
@@ -249,7 +267,7 @@ function ReaderTypeset:genStyleSheetMenu()
     for __, css in ipairs(OBSOLETED_CSS) do
         obsoleted_css[css_files[css]] = css
         if css_files[css] then
-            table.insert(obsoleted_table, getStyleMenuItem(css, css_files[css]))
+            table.insert(obsoleted_table, getStyleMenuItem(css, css_files[css], _("This stylesheet is obsolete: don't use it. It is kept solely to be able to open documents last read years ago and to migrate their highlights.")))
             css_files[css] = nil
         end
     end
@@ -260,7 +278,7 @@ function ReaderTypeset:genStyleSheetMenu()
     end
     table.sort(user_files)
     for __, css in ipairs(user_files) do
-        table.insert(style_table, getStyleMenuItem(css, css_files[css]))
+        table.insert(style_table, getStyleMenuItem(css, css_files[css], _("This is a user added stylesheet.")))
     end
 
     style_table[#style_table].separator = true
@@ -299,6 +317,7 @@ function ReaderTypeset:setStyleSheet(new_css)
     end
 end
 
+-- Not used
 function ReaderTypeset:setEmbededStyleSheetOnly()
     if self.css ~= nil then
         -- clear applied css
@@ -309,30 +328,6 @@ function ReaderTypeset:setEmbededStyleSheetOnly()
     end
 end
 
-function ReaderTypeset:toggleEmbeddedStyleSheet(toggle)
-    if not toggle then
-        self.embedded_css = false
-        self:setStyleSheet(self.ui.document.default_css)
-        self.ui.document:setEmbeddedStyleSheet(0)
-    else
-        self.embedded_css = true
-        --self:setStyleSheet(self.ui.document.default_css)
-        self.ui.document:setEmbeddedStyleSheet(1)
-    end
-    self.ui:handleEvent(Event:new("UpdatePos"))
-end
-
-function ReaderTypeset:toggleEmbeddedFonts(toggle)
-    if not toggle then
-        self.embedded_fonts = false
-        self.ui.document:setEmbeddedFonts(0)
-    else
-        self.embedded_fonts = true
-        self.ui.document:setEmbeddedFonts(1)
-    end
-    self.ui:handleEvent(Event:new("UpdatePos"))
-end
-
 -- crengine enhanced block rendering feature/flags (see crengine/include/lvrend.h):
 --                                               legacy flat book web
 -- ENHANCED                           0x00000001          x    x   x
@@ -404,39 +399,6 @@ function ReaderTypeset:ensureSanerBlockRenderingFlags(mode)
     self:setBlockRenderingMode(self.block_rendering_mode)
 end
 
-function ReaderTypeset:toggleImageScaling(toggle)
-    if toggle and (toggle == true or toggle == 1) then
-        self.smooth_scaling = true
-        self.ui.document:setImageScaling(true)
-    else
-        self.smooth_scaling = false
-        self.ui.document:setImageScaling(false)
-    end
-    self.ui:handleEvent(Event:new("UpdatePos"))
-end
-
-function ReaderTypeset:toggleNightmodeImages(toggle)
-    if toggle and (toggle == true or toggle == 1) then
-        self.nightmode_images = true
-        self.ui.document:setNightmodeImages(true)
-    else
-        self.nightmode_images = false
-        self.ui.document:setNightmodeImages(false)
-    end
-    self.ui:handleEvent(Event:new("UpdatePos"))
-end
-
-function ReaderTypeset:toggleTxtPreFormatted(toggle)
-    self.ui.document:setTxtPreFormatted(toggle)
-    self.ui:handleEvent(Event:new("UpdatePos"))
-end
-
-function ReaderTypeset:setRenderDPI(dpi)
-    self.render_dpi = dpi
-    self.ui.document:setRenderDPI(dpi)
-    self.ui:handleEvent(Event:new("UpdatePos"))
-end
-
 function ReaderTypeset:addToMainMenu(menu_items)
     -- insert table to main reader menu
     menu_items.set_render_style = {
@@ -445,11 +407,20 @@ function ReaderTypeset:addToMainMenu(menu_items)
     }
 end
 
-function ReaderTypeset:makeDefaultStyleSheet(css, text, touchmenu_instance)
+function ReaderTypeset:makeDefaultStyleSheet(css, name, description, touchmenu_instance)
+    local text = self.ui.document.is_fb2 and T(_("Set default style for FB2 documents to %1?"), BD.filename(name))
+                                          or T(_("Set default style to %1?"), BD.filename(name))
+    if description then
+        text = text .. "\n\n" .. description
+    end
     UIManager:show(ConfirmBox:new{
-        text = T( _("Set default style to %1?"), BD.filename(text)),
+        text = text,
         ok_callback = function()
-            G_reader_settings:saveSetting("copt_css", css)
+            if self.ui.document.is_fb2 then
+                G_reader_settings:saveSetting("copt_fb2_css", css)
+            else
+                G_reader_settings:saveSetting("copt_css", css)
+            end
             if touchmenu_instance then touchmenu_instance:updateItems() end
         end,
     })
@@ -465,7 +436,7 @@ function ReaderTypeset:onSetPageTopMargin(t_margin, when_applied_callback)
     if self.sync_t_b_page_margins then
         self.unscaled_margins[4] = t_margin
         -- Let ConfigDialog know so it can update it on screen and have it saved on quit
-        self.ui.document.configurable.b_page_margin = t_margin
+        self.configurable.b_page_margin = t_margin
     end
     self.ui:handleEvent(Event:new("SetPageMargins", self.unscaled_margins, when_applied_callback))
 end
@@ -475,7 +446,7 @@ function ReaderTypeset:onSetPageBottomMargin(b_margin, when_applied_callback)
     if self.sync_t_b_page_margins then
         self.unscaled_margins[2] = b_margin
         -- Let ConfigDialog know so it can update it on screen and have it saved on quit
-        self.ui.document.configurable.t_page_margin = b_margin
+        self.configurable.t_page_margin = b_margin
     end
     self.ui:handleEvent(Event:new("SetPageMargins", self.unscaled_margins, when_applied_callback))
 end
@@ -486,7 +457,7 @@ function ReaderTypeset:onSetPageTopAndBottomMargin(t_b_margins, when_applied_cal
     if t_margin ~= b_margin then
         -- Set Sync T/B Margins toggle to off, as user explicitly made them differ
         self.sync_t_b_page_margins = false
-        self.ui.document.configurable.sync_t_b_page_margins = 0
+        self.configurable.sync_t_b_page_margins = 0
     end
     self.ui:handleEvent(Event:new("SetPageMargins", self.unscaled_margins, when_applied_callback))
 end
@@ -503,8 +474,8 @@ function ReaderTypeset:onSyncPageTopBottomMargins(toggle, when_applied_callback)
             -- and later scaled, the end result could still be different.
             -- So just take the mean and make them equal.
             local mean_margin = Math.round((self.unscaled_margins[2] + self.unscaled_margins[4]) / 2)
-            self.ui.document.configurable.t_page_margin = mean_margin
-            self.ui.document.configurable.b_page_margin = mean_margin
+            self.configurable.t_page_margin = mean_margin
+            self.configurable.b_page_margin = mean_margin
             self.unscaled_margins = { self.unscaled_margins[1], mean_margin, self.unscaled_margins[3], mean_margin }
             self.ui:handleEvent(Event:new("SetPageMargins", self.unscaled_margins, when_applied_callback))
             when_applied_callback = nil

frontend/apps/reader/modules/readertypography.lua

@@ -1,6 +1,7 @@
 local BD = require("ui/bidi")
 local Device = require("device")
 local Event = require("ui/event")
+local FileManagerBookInfo = require("apps/filemanager/filemanagerbookinfo")
 local InfoMessage = require("ui/widget/infomessage")
 local MultiConfirmBox = require("ui/widget/multiconfirmbox")
 local UIManager = require("ui/uimanager")
@@ -83,9 +84,9 @@ local LANGUAGES = {
     { "pt-BR",                 {},   "HB  ",   _("Portuguese (BR)"),        "Portuguese_BR.pattern" },
     { "rm",               {"roh"},   "H   ",   _("Romansh"),                "Romansh.pattern" },
     { "ro",               {"ron"},   "H   ",   _("Romanian"),               "Romanian.pattern" },
-    { "ru",               {"rus"},   "Hb  ",   _("Russian"),                "Russian.pattern" },
-    { "ru-GB",                 {},   "Hb  ",   _("Russian + English (UK)"), "Russian_EnGB.pattern" },
-    { "ru-US",                 {},   "Hb  ",   _("Russian + English (US)"), "Russian_EnUS.pattern" },
+    { "ru",               {"rus"},   "HB  ",   _("Russian"),                "Russian.pattern" },
+    { "ru-GB",                 {},   "HB  ",   _("Russian + English (UK)"), "Russian_EnGB.pattern" },
+    { "ru-US",                 {},   "HB  ",   _("Russian + English (US)"), "Russian_EnUS.pattern" },
     { "sr",               {"srp"},   "HB  ",   _("Serbian"),                "Serbian.pattern" },
     { "sk",               {"slk"},   "HB  ",   _("Slovak"),                 "Slovak.pattern" },
     { "sl",               {"slv"},   "H   ",   _("Slovenian"),              "Slovenian.pattern" },
@@ -196,11 +197,10 @@ When the book's language tag is not among our presets, no specific features will
             -- Text might be too long for InfoMessage
             local status_text = table.concat(lang_infos, "\n")
             local TextViewer = require("ui/widget/textviewer")
-            local Font = require("ui/font")
             UIManager:show(TextViewer:new{
                 title = _("Language tags (and hyphenation dictionaries) used since start up"),
                 text = status_text,
-                text_face = Font:getFace("smallinfont"),
+                text_type = "code",
                 height = math.floor(Screen:getHeight() * 0.8),
             })
         end,
@@ -776,7 +776,8 @@ end
 function ReaderTypography:onPreRenderDocument(config)
     -- This is called after the document has been loaded,
     -- when we know and can access the document language.
-    local doc_language = self.ui.document:getProps().language
+    local doc_language = FileManagerBookInfo.getCustomProp("language", self.ui.document.file)
+                      or self.ui.document:getProps().language
     self.book_lang_tag = self:fixLangTag(doc_language)
 
     local is_known_lang_tag = self.book_lang_tag and LANG_TAG_TO_LANG_NAME[self.book_lang_tag] ~= nil

frontend/apps/reader/modules/readerview.lua

@@ -22,6 +22,7 @@ local logger = require("logger")
 local optionsutil = require("ui/data/optionsutil")
 local Size = require("ui/size")
 local time = require("ui/time")
+local util = require("util")
 local _ = require("gettext")
 local Screen = Device.screen
 local T = require("ffi/util").template
@@ -47,7 +48,7 @@ local ReaderView = OverlapGroup:extend{
     -- properties of the gap drawn between each page in scroll mode:
     page_gap = nil, -- table
     -- DjVu page rendering mode (used in djvu.c:drawPage())
-    render_mode = G_defaults:readSetting("DRENDER_MODE"), -- default to COLOR
+    render_mode = nil, -- default to COLOR, will be set in onReadSettings()
     -- Crengine view mode
     view_mode = G_defaults:readSetting("DCREREADER_VIEW_MODE"), -- default to page mode
     hinting = true,
@@ -70,6 +71,10 @@ local ReaderView = OverlapGroup:extend{
     -- might be directly updated by readerpaging/readerrolling when
     -- they handle some panning/scrolling, to request "fast" refreshes
     currently_scrolling = false,
+
+    -- image content stats of the current page, if supported by the Document engine
+    img_count = nil,
+    img_coverage = nil,
 }
 
 function ReaderView:init()
@@ -90,7 +95,6 @@ function ReaderView:init()
         temp_drawer = "invert",
         temp = {},
         saved_drawer = "lighten",
-        saved = {},
         indicator = nil, -- geom: non-touch highlight position indicator: {x = 50, y=50}
     }
     self.page_states = {}
@@ -102,6 +106,10 @@ function ReaderView:init()
     self.page_area = Geom:new{x = 0, y = 0, w = 0, h = 0}
     self.dim_area = Geom:new{x = 0, y = 0, w = 0, h = 0}
 
+    -- Zero-init for sanity
+    self.img_count = 0
+    self.img_coverage = 0
+
     self:addWidgets()
     self.emitHintPageEvent = function()
         self.ui:handleEvent(Event:new("HintPage", self.hinting))
@@ -201,22 +209,26 @@ function ReaderView:paintTo(bb, x, y)
         end
     end
 
-    -- dim last read area
+    -- mark last read area of overlapped pages
     if not self.dim_area:isEmpty() and self:isOverlapAllowed() then
         if self.page_overlap_style == "dim" then
-            bb:dimRect(
-                self.dim_area.x, self.dim_area.y,
-                self.dim_area.w, self.dim_area.h
-            )
-        elseif self.page_overlap_style == "arrow" then
-            local center_offset = bit.rshift(self.arrow.height, 1)
-            -- Paint at the proper y origin depending on wheter we paged forward (dim_area.y == 0) or backward
-            self.arrow:paintTo(bb, 0, self.dim_area.y == 0 and self.dim_area.h - center_offset or self.dim_area.y - center_offset)
-        elseif self.page_overlap_style == "line" then
-            bb:paintRect(0, self.dim_area.y == 0 and self.dim_area.h or self.dim_area.y,
-                self.dim_area.w, Size.line.medium, Blitbuffer.COLOR_BLACK)
+            bb:dimRect(self.dim_area.x, self.dim_area.y, self.dim_area.w, self.dim_area.h)
+        else
+            -- Paint at the proper y origin depending on whether we paged forward (dim_area.y == 0) or backward
+            local paint_y = self.dim_area.y == 0 and self.dim_area.h or self.dim_area.y
+            if self.page_overlap_style == "arrow" then
+                local center_offset = bit.rshift(self.arrow.height, 1)
+                self.arrow:paintTo(bb, 0, paint_y - center_offset)
+            elseif self.page_overlap_style == "line" then
+                bb:paintRect(0, paint_y, self.dim_area.w, Size.line.medium, Blitbuffer.COLOR_DARK_GRAY)
+            elseif self.page_overlap_style == "dashed_line" then
+                for i = 0, self.dim_area.w - 20, 20 do
+                    bb:paintRect(i, paint_y, 14, Size.line.medium, Blitbuffer.COLOR_DARK_GRAY)
+                end
+            end
         end
     end
+
     -- draw saved highlight
     if self.highlight_visible then
         self:drawSavedHighlight(bb, x, y)
@@ -252,18 +264,29 @@ function ReaderView:paintTo(bb, x, y)
     if self.ui.paging then
         if self.document.hw_dithering then
             self.dialog.dithered = true
+            -- Assume we're going to be showing colorful stuff on kaleido panels...
+            if Device:hasKaleidoWfm() then
+                UIManager:setDirty(nil, "color")
+            end
         end
     else
         -- Whereas for CRe,
         -- If we're attempting to show a large enough amount of image data, request dithering (without triggering another repaint ;)).
         local img_count, img_coverage = self.document:getDrawnImagesStatistics()
-        -- With some nil guards because this may not be implemented in every engine ;).
-        if img_count and img_count > 0 and img_coverage and img_coverage >= 0.075 then
+        -- We also want to deal with paging *away* from image content, which would have adverse effect on ghosting.
+        local coverage_diff = math.abs(img_coverage - self.img_coverage)
+        -- Which is why we remember the stats of the *previous* page.
+        self.img_count, self.img_coverage = img_count, img_coverage
+        if img_coverage >= 0.075 or coverage_diff >= 0.075 then
             self.dialog.dithered = true
             -- Request a flashing update while we're at it, but only if it's the first time we're painting it
             if self.state.drawn == false and G_reader_settings:nilOrTrue("refresh_on_pages_with_images") then
                 UIManager:setDirty(nil, "full")
             end
+            -- On Kaleido panels, we'll want to use GCC16 on the actual image, always...
+            if Device:hasKaleidoWfm() and img_coverage >= 0.075 then
+                UIManager:setDirty(nil, "color")
+            end
         end
         self.state.drawn = true
     end
@@ -406,7 +429,7 @@ function ReaderView:getScrollPagePosition(pos)
 end
 
 function ReaderView:getScrollPageRect(page, rect_p)
-    local rect_s = Geom:new{}
+    local rect_s = Geom:new()
     for _, state in ipairs(self.page_states) do
         local trans_p = Geom:new(rect_p):copy()
         trans_p:transformByScale(state.zoom, state.zoom)
@@ -451,7 +474,7 @@ function ReaderView:getSinglePagePosition(pos)
 end
 
 function ReaderView:getSinglePageRect(rect_p)
-    local rect_s = Geom:new{}
+    local rect_s = Geom:new()
     local trans_p = Geom:new(rect_p):copy()
     trans_p:transformByScale(self.state.zoom, self.state.zoom)
     if self.visible_area:intersectWith(trans_p) then
@@ -510,6 +533,7 @@ function ReaderView:drawTempHighlight(bb, x, y)
 end
 
 function ReaderView:drawSavedHighlight(bb, x, y)
+    if #self.ui.annotation.annotations == 0 then return end
     if self.ui.paging then
         self:drawPageSavedHighlight(bb, x, y)
     else
@@ -517,45 +541,18 @@ function ReaderView:drawSavedHighlight(bb, x, y)
     end
 end
 
--- Returns the list of highlights in page.
--- The list includes full single-page highlights and parts of multi-page highlights.
-function ReaderView:getPageSavedHighlights(page)
-    local highlights = {}
-    local is_reflow = self.document.configurable.text_wrap
-    self.document.configurable.text_wrap = 0
-    for page_num, page_highlights in pairs(self.highlight.saved) do
-        for i, highlight in ipairs(page_highlights) do
-            -- old single-page reflow highlights do not have page in position
-            local pos0_page = highlight.pos0.page or page_num
-            local pos1_page = highlight.pos1.page or page_num
-            if pos0_page <= page and page <= pos1_page then
-                if pos0_page == pos1_page then -- single-page highlight
-                    table.insert(highlights, highlight)
-                else -- multi-page highlight
-                    local item = self.ui.highlight:getSavedExtendedHighlightPage(highlight, page, i)
-                    table.insert(highlights, item)
-                end
-            end
-        end
-    end
-    self.document.configurable.text_wrap = is_reflow
-    return highlights
-end
-
 function ReaderView:drawPageSavedHighlight(bb, x, y)
     local pages = self:getCurrentPageList()
     for _, page in ipairs(pages) do
-        local items = self:getPageSavedHighlights(page)
+        local items = self.ui.highlight:getPageSavedHighlights(page)
         for _, item in ipairs(items) do
             local boxes = self.document:getPageBoxesFromPositions(page, item.pos0, item.pos1)
             if boxes then
-                local drawer = item.drawer or self.highlight.saved_drawer
-                local draw_note_mark = self.highlight.note_mark and
-                    self.ui.bookmark:getBookmarkNote({datetime = item.datetime})
+                local draw_note_mark = item.note and self.highlight.note_mark
                 for _, box in ipairs(boxes) do
                     local rect = self:pageToScreenTransform(page, box)
                     if rect then
-                        self:drawHighlightRect(bb, x, y, rect, drawer, draw_note_mark)
+                        self:drawHighlightRect(bb, x, y, rect, item.drawer, draw_note_mark)
                         if draw_note_mark and self.highlight.note_mark == "sidemark" then
                             draw_note_mark = false -- side mark in the first line only
                         end
@@ -571,48 +568,38 @@ function ReaderView:drawXPointerSavedHighlight(bb, x, y)
     -- showing menu...). We might want to cache these boxes per page (and
     -- clear that cache when page layout change or highlights are added
     -- or removed).
-    local cur_view_top, cur_view_bottom
-    for _, items in pairs(self.highlight.saved) do
-        if items then
-            for j = 1, #items do
-                local item = items[j]
-                local pos0, pos1 = item.pos0, item.pos1
-                -- document:getScreenBoxesFromPositions() is expensive, so we
-                -- first check this item is on current page
-                if not cur_view_top then
-                    -- Even in page mode, it's safer to use pos and ui.dimen.h
-                    -- than pages' xpointers pos, even if ui.dimen.h is a bit
-                    -- larger than pages' heights
-                    cur_view_top = self.document:getCurrentPos()
-                    if self.view_mode == "page" and self.document:getVisiblePageCount() > 1 then
-                        cur_view_bottom = cur_view_top + 2 * self.ui.dimen.h
-                    else
-                        cur_view_bottom = cur_view_top + self.ui.dimen.h
-                    end
-                end
-                local spos0 = self.document:getPosFromXPointer(pos0)
-                local spos1 = self.document:getPosFromXPointer(pos1)
-                local start_pos = math.min(spos0, spos1)
-                local end_pos = math.max(spos0, spos1)
-                if start_pos <= cur_view_bottom and end_pos >= cur_view_top then
-                    local boxes = self.document:getScreenBoxesFromPositions(pos0, pos1, true) -- get_segments=true
-                    if boxes then
-                        local drawer = item.drawer or self.highlight.saved_drawer
-                        local draw_note_mark = self.highlight.note_mark and
-                            self.ui.bookmark:getBookmarkNote({datetime = item.datetime})
-                        for _, box in ipairs(boxes) do
-                            if box.h ~= 0 then
-                                self:drawHighlightRect(bb, x, y, box, drawer, draw_note_mark)
-                                if draw_note_mark and self.highlight.note_mark == "sidemark" then
-                                    draw_note_mark = false -- side mark in the first line only
-                                end
+    -- Even in page mode, it's safer to use pos and ui.dimen.h
+    -- than pages' xpointers pos, even if ui.dimen.h is a bit
+    -- larger than pages' heights
+    local cur_view_top = self.document:getCurrentPos()
+    local cur_view_bottom
+    if self.view_mode == "page" and self.document:getVisiblePageCount() > 1 then
+        cur_view_bottom = cur_view_top + 2 * self.ui.dimen.h
+    else
+        cur_view_bottom = cur_view_top + self.ui.dimen.h
+    end
+    for _, item in ipairs(self.ui.annotation.annotations) do
+        if item.drawer then
+            -- document:getScreenBoxesFromPositions() is expensive, so we
+            -- first check if this item is on current page
+            local start_pos = self.document:getPosFromXPointer(item.pos0)
+            local end_pos = self.document:getPosFromXPointer(item.pos1)
+            if start_pos <= cur_view_bottom and end_pos >= cur_view_top then
+                local boxes = self.document:getScreenBoxesFromPositions(item.pos0, item.pos1, true) -- get_segments=true
+                if boxes then
+                    local draw_note_mark = item.note and self.highlight.note_mark
+                    for _, box in ipairs(boxes) do
+                        if box.h ~= 0 then
+                            self:drawHighlightRect(bb, x, y, box, item.drawer, draw_note_mark)
+                            if draw_note_mark and self.highlight.note_mark == "sidemark" then
+                                draw_note_mark = false -- side mark in the first line only
                             end
-                        end -- end for each box
-                    end -- end if boxes
+                        end
+                    end
                 end
-            end -- end for each highlight
+            end
         end
-    end -- end for all saved highlight
+    end
 end
 
 function ReaderView:drawHighlightRect(bb, _x, _y, rect, drawer, draw_note_mark)
@@ -620,7 +607,7 @@ function ReaderView:drawHighlightRect(bb, _x, _y, rect, drawer, draw_note_mark)
     if drawer == "lighten" then
         bb:lightenRect(x, y, w, h, self.highlight.lighten_factor)
     elseif drawer == "underscore" then
-        bb:paintRect(x, y + h - 1, w, Size.line.medium, Blitbuffer.COLOR_GRAY)
+        bb:paintRect(x, y + h - 1, w, Size.line.thick, Blitbuffer.COLOR_GRAY_4)
     elseif drawer == "strikeout" then
         local line_y = y + math.floor(h / 2) + 1
         if self.ui.paging then
@@ -636,7 +623,7 @@ function ReaderView:drawHighlightRect(bb, _x, _y, rect, drawer, draw_note_mark)
         else
             local note_mark_pos_x
             if self.ui.paging or
-                    (self.ui.document:getVisiblePageCount() == 1) or -- one-page mode
+                    (self.document:getVisiblePageCount() == 1) or -- one-page mode
                     (x < Screen:getWidth() / 2) then -- page 1 in two-page mode
                 note_mark_pos_x = self.note_mark_pos_x1
             else
@@ -683,7 +670,8 @@ function ReaderView:recalculate()
             -- start from right of page_area
             self.visible_area.x = self.page_area.x + self.page_area.w - self.visible_area.w
         end
-        if self.ui.zooming.zoom_bottom_to_top then
+        -- Check if we are in zoom_bottom_to_top
+        if self.document.configurable.zoom_direction and self.document.configurable.zoom_direction >= 2 and self.document.configurable.zoom_direction <= 5 then
             -- starts from bottom of page_area
             self.visible_area.y = self.page_area.y + self.page_area.h - self.visible_area.h
         else
@@ -803,18 +791,38 @@ end
 
 function ReaderView:onSetRotationMode(rotation)
     if rotation ~= nil then
-        if rotation == Screen:getRotationMode() then
-            return true
+        local old_rotation = Screen:getRotationMode()
+        if rotation == old_rotation then
+            return
+        end
+
+        -- NOTE: We cannot rely on getScreenMode, as it actually checks the screen dimensions, instead of the rotation mode.
+        --       (i.e., it returns how the screen *looks* like, not how it's oriented relative to its native layout).
+        --       This would horribly break if you started in Portrait (both rotation and visually),
+        --       then resized your window to a Landscape layout *without* changing the rotation.
+        --       If you then attempted to switch to a Landscape *rotation*, it would mistakenly think the layout hadn't changed!
+        --       So, instead, as we're concerned with *rotation* layouts, just compare the two.
+        --       We use LinuxFB-style constants, so, Portraits are even, Landscapes are odds, making this trivial.
+        local matching_orientation = bit.band(rotation, 1) == bit.band(old_rotation, 1)
+
+        if rotation ~= old_rotation and matching_orientation then
+            -- No layout change, just rotate & repaint with a flash
+            Screen:setRotationMode(rotation)
+            UIManager:setDirty(self.dialog, "full")
+            Notification:notify(T(_("Rotation mode set to: %1"), optionsutil:getOptionText("SetRotationMode", rotation)))
+            return
         end
+
         Screen:setRotationMode(rotation)
     end
-    UIManager:setDirty(self.dialog, "full")
+
+    UIManager:setDirty(nil, "full") -- SetDimensions will only request a partial, we want a flash
     local new_screen_size = Screen:getSize()
     self.ui:handleEvent(Event:new("SetDimensions", new_screen_size))
     self.ui:onScreenResize(new_screen_size)
     self.ui:handleEvent(Event:new("InitScrollPageStates"))
     Notification:notify(T(_("Rotation mode set to: %1"), optionsutil:getOptionText("SetRotationMode", rotation)))
-    return true
+    return
 end
 
 function ReaderView:onSetDimensions(dimensions)
@@ -860,23 +868,18 @@ end
 function ReaderView:onReadSettings(config)
     if self.ui.paging then
         self.document:setTileCacheValidity(config:readSetting("tile_cache_validity_ts"))
-        self.render_mode = config:readSetting("render_mode") or 0
-        self.state.gamma = config:readSetting("gamma") or 1.0
-    end
-    local rotation_mode = nil
-    local locked = G_reader_settings:isTrue("lock_rotation")
-    -- Keep current rotation by doing nothing when sticky rota is enabled.
-    if not locked then
-        -- Honor docsettings's rotation
-        if config:has("rotation_mode") then
-            rotation_mode = config:readSetting("rotation_mode") -- Doc's
-        else
-            -- No doc specific rotation, pickup global defaults for the doc type
-            local setting_name = self.ui.paging and "kopt_rotation_mode" or "copt_rotation_mode"
-            rotation_mode = G_reader_settings:readSetting(setting_name) or Screen.DEVICE_ROTATED_UPRIGHT
+        self.render_mode = config:readSetting("render_mode") or G_defaults:readSetting("DRENDER_MODE")
+        if config:has("gamma") then -- old doc contrast setting
+            config:saveSetting("kopt_contrast", config:readSetting("gamma"))
+            config:delSetting("gamma")
         end
     end
-    if rotation_mode then
+    if G_reader_settings:nilOrFalse("lock_rotation") then
+        local setting_name = self.ui.paging and "kopt_rotation_mode" or "copt_rotation_mode"
+        -- document.configurable.rotation_mode is not ready yet
+        local rotation_mode = config:readSetting(setting_name)
+                           or G_reader_settings:readSetting(setting_name)
+                           or Screen.DEVICE_ROTATED_UPRIGHT
         self:onSetRotationMode(rotation_mode)
     end
     local full_screen = config:readSetting("kopt_full_screen") or self.document.configurable.full_screen
@@ -886,40 +889,6 @@ function ReaderView:onReadSettings(config)
     self:resetLayout()
     local page_scroll = config:readSetting("kopt_page_scroll") or self.document.configurable.page_scroll
     self.page_scroll = page_scroll == 1 and true or false
-    self.highlight.saved = config:readSetting("highlight", {})
-    -- Highlight formats in crengine and mupdf are incompatible.
-    -- Backup highlights when the document is opened with incompatible engine.
-    local page, page_highlights
-    while true do -- remove empty tables for pages without highlights and get the first page with highlights
-        page, page_highlights = next(self.highlight.saved)
-        if not page or #page_highlights > 0 then
-            break -- we're done (there is none, or there is some usable)
-        else
-            self.highlight.saved[page] = nil -- clean it up while we're at it, and find another one
-        end
-    end
-    if page_highlights then
-        local highlight_type = type(page_highlights[1].pos0)
-        if self.ui.rolling and highlight_type == "table" then
-            config:saveSetting("highlight_paging", self.highlight.saved)
-            self.highlight.saved = config:readSetting("highlight_rolling", {})
-            config:saveSetting("highlight", self.highlight.saved)
-            config:delSetting("highlight_rolling")
-        elseif self.ui.paging and highlight_type == "string" then
-            config:saveSetting("highlight_rolling", self.highlight.saved)
-            self.highlight.saved = config:readSetting("highlight_paging", {})
-            config:saveSetting("highlight", self.highlight.saved)
-            config:delSetting("highlight_paging")
-        end
-    else
-        if self.ui.rolling and config:has("highlight_rolling") then
-            self.highlight.saved = config:readSetting("highlight_rolling")
-            config:delSetting("highlight_rolling")
-        elseif self.ui.paging and config:has("highlight_paging") then
-            self.highlight.saved = config:readSetting("highlight_paging")
-            config:delSetting("highlight_paging")
-        end
-    end
     self.inverse_reading_order = config:isTrue("inverse_reading_order") or G_reader_settings:isTrue("inverse_reading_order")
     self.page_overlap_enable = config:isTrue("show_overlap_enable") or G_reader_settings:isTrue("page_overlap_enable") or G_defaults:readSetting("DSHOWOVERLAP")
     self.page_overlap_style = config:readSetting("page_overlap_style") or G_reader_settings:readSetting("page_overlap_style") or "dim"
@@ -995,7 +964,7 @@ function ReaderView:onGammaUpdate(gamma)
     if self.page_scroll then
         self.ui:handleEvent(Event:new("UpdateScrollPageGamma", gamma))
     end
-    Notification:notify(T(_("Font gamma set to: %1."), gamma))
+    Notification:notify(T(_("Contrast set to: %1."), gamma))
 end
 
 -- For ReaderKOptListener
@@ -1071,15 +1040,14 @@ function ReaderView:onSaveSettings()
             self.document:resetTileCacheValidity()
         end
         self.ui.doc_settings:saveSetting("tile_cache_validity_ts", self.document:getTileCacheValidity())
-        self.ui.doc_settings:saveSetting("render_mode", self.render_mode)
-        self.ui.doc_settings:saveSetting("gamma", self.state.gamma)
+        if self.document.is_djvu then
+            self.ui.doc_settings:saveSetting("render_mode", self.render_mode)
+        end
     end
     -- Don't etch the current rotation in stone when sticky rotation is enabled
-    local locked = G_reader_settings:isTrue("lock_rotation")
-    if not locked then
-        self.ui.doc_settings:saveSetting("rotation_mode", Screen:getRotationMode())
+    if G_reader_settings:nilOrFalse("lock_rotation") then
+        self.document.configurable.rotation_mode = Screen:getRotationMode() -- will be saved by ReaderConfig
     end
-    self.ui.doc_settings:saveSetting("highlight", self.highlight.saved)
     self.ui.doc_settings:saveSetting("inverse_reading_order", self.inverse_reading_order)
     self.ui.doc_settings:saveSetting("show_overlap_enable", self.page_overlap_enable)
     self.ui.doc_settings:saveSetting("page_overlap_style", self.page_overlap_style)
@@ -1247,17 +1215,17 @@ function ReaderView:setupNoteMarkPosition()
                 self.note_mark_pos_x1 = screen_w - sign_gap - sign_w
             end
         else
-            local doc_margins = self.ui.document:getPageMargins()
+            local doc_margins = self.document:getPageMargins()
             local pos_x_r = screen_w - doc_margins["right"] + sign_gap -- mark in the right margin
             local pos_x_l = doc_margins["left"] - sign_gap - sign_w -- mark in the left margin
-            if self.ui.document:getVisiblePageCount() == 1 then
+            if self.document:getVisiblePageCount() == 1 then
                 if BD.mirroredUILayout() then
                     self.note_mark_pos_x1 = pos_x_l
                 else
                     self.note_mark_pos_x1 = pos_x_r
                 end
             else -- two-page mode
-                local page2_x = self.ui.document:getPageOffsetX(self.ui.document:getCurrentPage(true)+1)
+                local page2_x = self.document:getPageOffsetX(self.document:getCurrentPage(true)+1)
                 if BD.mirroredUILayout() then
                     self.note_mark_pos_x1 = pos_x_l
                     self.note_mark_pos_x2 = pos_x_l + page2_x
@@ -1270,4 +1238,41 @@ function ReaderView:setupNoteMarkPosition()
     end
 end
 
+function ReaderView:getCurrentPageLineWordCounts()
+    local lines_nb, words_nb = 0, 0
+    if self.ui.rolling then
+        local res = self.document:getTextFromPositions({x = 0, y = 0},
+            {x = Screen:getWidth(), y = Screen:getHeight()}, true) -- do not highlight
+        if res then
+            lines_nb = #self.document:getScreenBoxesFromPositions(res.pos0, res.pos1, true)
+            for word in util.gsplit(res.text, "[%s%p]+", false) do
+                if util.hasCJKChar(word) then
+                    for char in util.gsplit(word, "[\192-\255][\128-\191]+", true) do
+                        words_nb = words_nb + 1
+                    end
+                else
+                    words_nb = words_nb + 1
+                end
+            end
+        end
+    else
+        local page_boxes = self.document:getTextBoxes(self.ui:getCurrentPage())
+        if page_boxes and page_boxes[1][1].word then
+            lines_nb = #page_boxes
+            for _, line in ipairs(page_boxes) do
+                if #line == 1 and line[1].word == "" then -- empty line
+                    lines_nb = lines_nb - 1
+                else
+                    words_nb = words_nb + #line
+                    local last_word = line[#line].word
+                    if last_word:sub(-1) == "-" and last_word ~= "-" then -- hyphenated
+                        words_nb = words_nb - 1
+                    end
+                end
+            end
+        end
+    end
+    return lines_nb, words_nb
+end
+
 return ReaderView

frontend/apps/reader/modules/readerwikipedia.lua

@@ -1,5 +1,3 @@
-local BD = require("ui/bidi")
-local ButtonDialogTitle = require("ui/widget/buttondialogtitle")
 local ConfirmBox = require("ui/widget/confirmbox")
 local DataStorage = require("datastorage")
 local DictQuickLookup = require("ui/widget/dictquicklookup")
@@ -13,6 +11,7 @@ local Trapper = require("ui/trapper")
 local Translator = require("ui/translator")
 local UIManager = require("ui/uimanager")
 local Wikipedia = require("ui/wikipedia")
+local filemanagerutil = require("apps/filemanager/filemanagerutil")
 local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
 local util  = require("util")
@@ -114,6 +113,17 @@ function ReaderWikipedia:addToMainMenu(menu_items)
             })
         end,
     }
+    local function genChoiceMenuEntry(title, setting, value, default)
+        return {
+            text = title,
+            checked_func = function()
+                return G_reader_settings:readSetting(setting, default) == value
+            end,
+            callback = function()
+                G_reader_settings:saveSetting(setting, value)
+            end,
+        }
+    end
     menu_items.wikipedia_settings = {
         text = _("Wikipedia settings"),
         sub_item_table = {
@@ -171,113 +181,26 @@ function ReaderWikipedia:addToMainMenu(menu_items)
                     UIManager:show(wikilang_input)
                     wikilang_input:onShowKeyboard()
                 end,
+                separator = true,
             },
             { -- setting used by dictquicklookup
                 text = _("Set Wikipedia 'Save as EPUB' folder"),
                 keep_menu_open = true,
-                callback = function()
-                    local choose_directory = function()
-                        -- Default directory as chosen by DictQuickLookup
-                        local default_dir = G_reader_settings:readSetting("wikipedia_save_dir")
-                                         or G_reader_settings:readSetting("home_dir")
-                                         or require("apps/filemanager/filemanagerutil").getDefaultDir()
-                        local dialog
-                        dialog = ButtonDialogTitle:new{
-                            title = T(_("Current Wikipedia 'Save as EPUB' folder:\n\n%1\n"), BD.dirpath(default_dir)),
-                            buttons = {
-                                {
-                                    {
-                                        text = _("Keep this folder"),
-                                        callback = function()
-                                            UIManager:close(dialog)
-                                        end,
-                                    },
-                                },
-                                {
-                                    {
-                                    text = _("Choose other folder"),
-                                    callback = function()
-                                        UIManager:close(dialog)
-                                        -- Use currently read book's directory as starting point,
-                                        -- so a user reading a wikipedia article can quickly select
-                                        -- it to save related new articles in the same directory
-                                        local dir = G_reader_settings:readSetting("wikipedia_save_dir")
-                                                 or G_reader_settings:readSetting("home_dir")
-                                                 or require("apps/filemanager/filemanagerutil").getDefaultDir()
-                                                 or "/"
-                                        -- If this directory has no subdirectory, we would be displaying
-                                        -- a single "..", so use parent directory in that case.
-                                        local has_subdirectory = false
-                                        for f in lfs.dir(dir) do
-                                            local attributes = lfs.attributes(dir.."/"..f)
-                                            if attributes and attributes.mode == "directory" then
-                                                if f ~= "." and f ~= ".." and f:sub(-4) ~= ".sdr"then
-                                                    has_subdirectory = true
-                                                    break
-                                                end
-                                            end
-                                        end
-                                        if not has_subdirectory then
-                                            dir = dir:match("(.*)/")
-                                        end
-                                        local PathChooser = require("ui/widget/pathchooser")
-                                        local path_chooser = PathChooser:new{
-                                            select_directory = true,
-                                            select_file = false,
-                                            path = dir,
-                                            onConfirm = function(path)
-                                                G_reader_settings:saveSetting("wikipedia_save_dir", path)
-                                                UIManager:show(InfoMessage:new{
-                                                    text = T(_("Wikipedia 'Save as EPUB' folder set to:\n%1"), BD.dirpath(path)),
-                                                })
-                                            end
-                                        }
-                                        UIManager:show(path_chooser)
-                                    end,
-                                    },
-                                },
-                            },
-                        }
-                        UIManager:show(dialog)
-                    end
-                    -- If wikipedia_save_dir has not yet been set, propose to use
-                    -- home_dir/Wikipedia/
-                    if not G_reader_settings:readSetting("wikipedia_save_dir") then
-                        local home_dir = G_reader_settings:readSetting("home_dir")
-                        if not home_dir or lfs.attributes(home_dir, "mode") ~= "directory" then
-                            home_dir = require("apps/filemanager/filemanagerutil").getDefaultDir()
-                        end
-                        home_dir = home_dir:gsub("^(.-)/*$", "%1") -- remove trailing slash
-                        if home_dir and lfs.attributes(home_dir, "mode") == "directory" then
-                            local wikipedia_dir = home_dir.."/Wikipedia"
-                            local text = _([[
+                help_text = _([[
 Wikipedia articles can be saved as an EPUB for more comfortable reading.
 
-You can choose an existing folder, or use a default folder named "Wikipedia" in your reader's home folder.
-
-Where do you want them saved?]])
-                            UIManager:show(ConfirmBox:new{
-                                text = text,
-                                ok_text = _("Use ~/Wikipedia/"),
-                                ok_callback = function()
-                                    if not util.pathExists(wikipedia_dir) then
-                                        lfs.mkdir(wikipedia_dir)
-                                    end
-                                    G_reader_settings:saveSetting("wikipedia_save_dir", wikipedia_dir)
-                                    UIManager:show(InfoMessage:new{
-                                        text = T(_("Wikipedia 'Save as EPUB' folder set to:\n%1"), BD.dirpath(wikipedia_dir)),
-                                    })
-                                end,
-                                cancel_text = _("Choose folder"),
-                                cancel_callback = function()
-                                    choose_directory()
-                                end,
-                            })
-                            return
+You can choose an existing folder, or use a default folder named "Wikipedia" in your reader's home folder.]]),
+                callback = function()
+                    local title_header = _("Current Wikipedia 'Save as EPUB' folder:")
+                    local current_path = G_reader_settings:readSetting("wikipedia_save_dir")
+                    local default_path = DictQuickLookup.getWikiSaveEpubDefaultDir()
+                    local caller_callback = function(path)
+                        G_reader_settings:saveSetting("wikipedia_save_dir", path)
+                        if not util.pathExists(path) then
+                            lfs.mkdir(path)
                         end
                     end
-                    -- If setting exists, or no home_dir found, let user choose directory
-                    choose_directory()
+                    filemanagerutil.showChooseDialog(title_header, caller_callback, current_path, default_path)
                 end,
             },
             { -- setting used by dictquicklookup
@@ -288,6 +211,41 @@ Where do you want them saved?]])
                 callback = function()
                     G_reader_settings:flipNilOrFalse("wikipedia_save_in_book_dir")
                 end,
+            },
+            { -- setting used in wikipedia.lua
+                text_func = function()
+                    local include_images = _("ask")
+                    if G_reader_settings:readSetting("wikipedia_epub_include_images") == true then
+                        include_images = _("always")
+                    elseif G_reader_settings:readSetting("wikipedia_epub_include_images") == false then
+                        include_images = _("never")
+                    end
+                    return T(_("Include images in EPUB: %1"), include_images)
+                end,
+                sub_item_table = {
+                    genChoiceMenuEntry(_("Ask"), "wikipedia_epub_include_images", nil),
+                    genChoiceMenuEntry(_("Include images"), "wikipedia_epub_include_images", true),
+                    genChoiceMenuEntry(_("Don't include images"), "wikipedia_epub_include_images", false),
+                },
+            },
+            { -- setting used in wikipedia.lua
+                text_func = function()
+                    local images_quality = _("ask")
+                    if G_reader_settings:readSetting("wikipedia_epub_highres_images") == true then
+                        images_quality = _("higher")
+                    elseif G_reader_settings:readSetting("wikipedia_epub_highres_images") == false then
+                        images_quality = _("standard")
+                    end
+                    return T(_("Images quality in EPUB: %1"), images_quality)
+                end,
+                enabled_func = function()
+                    return G_reader_settings:readSetting("wikipedia_epub_include_images") ~= false
+                end,
+                sub_item_table = {
+                    genChoiceMenuEntry(_("Ask"), "wikipedia_epub_highres_images", nil),
+                    genChoiceMenuEntry(_("Standard quality"), "wikipedia_epub_highres_images", false),
+                    genChoiceMenuEntry(_("Higher quality"), "wikipedia_epub_highres_images", true),
+                },
                 separator = true,
             },
             {
@@ -371,7 +329,7 @@ function ReaderWikipedia:initLanguages(word)
         end
         -- use book and UI languages
         if self.view then
-            addLanguage(self.view.document:getProps().language)
+            addLanguage(self.ui.doc_props.language)
         end
         addLanguage(G_reader_settings:readSetting("language"))
         if #self.wiki_languages == 0 and word then
@@ -427,13 +385,7 @@ function ReaderWikipedia:lookupWikipedia(word, is_sane, box, get_fullpage, force
     local display_word = word:gsub("_", " ")
 
     if not self.disable_history then
-        local book_title = self.ui.doc_settings and self.ui.doc_settings:readSetting("doc_props").title or _("Wikipedia lookup")
-        if book_title == "" then -- no or empty metadata title
-            if self.ui.document and self.ui.document.file then
-                local directory, filename = util.splitFilePathName(self.ui.document.file) -- luacheck: no unused
-                book_title = util.splitFileNameSuffix(filename)
-            end
-        end
+        local book_title = self.ui.doc_props and self.ui.doc_props.display_title or _("Wikipedia lookup")
         wikipedia_history:addTableItem("wikipedia_history", {
             book_title = book_title,
             time = os.time(),

frontend/apps/reader/modules/readerzooming.lua

@@ -230,11 +230,10 @@ function ReaderZooming:onReadSettings(config)
         -- Otherwise, build it from the split genus & type settings
         local zoom_mode_genus = config:readSetting("kopt_zoom_mode_genus")
                              or G_reader_settings:readSetting("kopt_zoom_mode_genus")
+                             or 3 -- autocrop is default then pagewidth will be the default as well
         local zoom_mode_type = config:readSetting("kopt_zoom_mode_type")
                             or G_reader_settings:readSetting("kopt_zoom_mode_type")
-        if zoom_mode_genus or zoom_mode_type then
-            zoom_mode = self:combo_to_mode(zoom_mode_genus, zoom_mode_type)
-        end
+        zoom_mode = self:combo_to_mode(zoom_mode_genus, zoom_mode_type)
 
         -- Validate it
         zoom_mode = self.zoom_mode_label[zoom_mode] and zoom_mode or self.DEFAULT_ZOOM_MODE

frontend/apps/reader/readerui.lua

@@ -21,9 +21,11 @@ local InfoMessage = require("ui/widget/infomessage")
 local InputContainer = require("ui/widget/container/inputcontainer")
 local InputDialog = require("ui/widget/inputdialog")
 local LanguageSupport = require("languagesupport")
+local NetworkListener = require("ui/network/networklistener")
 local Notification = require("ui/widget/notification")
 local PluginLoader = require("pluginloader")
 local ReaderActivityIndicator = require("apps/reader/modules/readeractivityindicator")
+local ReaderAnnotation = require("apps/reader/modules/readerannotation")
 local ReaderBack = require("apps/reader/modules/readerback")
 local ReaderBookmark = require("apps/reader/modules/readerbookmark")
 local ReaderConfig = require("apps/reader/modules/readerconfig")
@@ -33,6 +35,7 @@ local ReaderDeviceStatus = require("apps/reader/modules/readerdevicestatus")
 local ReaderDictionary = require("apps/reader/modules/readerdictionary")
 local ReaderFont = require("apps/reader/modules/readerfont")
 local ReaderGoto = require("apps/reader/modules/readergoto")
+local ReaderHandMade = require("apps/reader/modules/readerhandmade")
 local ReaderHinting = require("apps/reader/modules/readerhinting")
 local ReaderHighlight = require("apps/reader/modules/readerhighlight")
 local ReaderScrolling = require("apps/reader/modules/readerscrolling")
@@ -59,6 +62,7 @@ local Screenshoter = require("ui/widget/screenshoter")
 local SettingsMigration = require("ui/data/settings_migration")
 local UIManager = require("ui/uimanager")
 local ffiUtil  = require("ffi/util")
+local filemanagerutil = require("apps/filemanager/filemanagerutil")
 local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
 local time = require("ui/time")
@@ -82,7 +86,7 @@ local ReaderUI = InputContainer:extend{
     password = nil,
 
     postInitCallback = nil,
-    postReaderCallback = nil,
+    postReaderReadyCallback = nil,
 }
 
 function ReaderUI:registerModule(name, ui_module, always_active)
@@ -101,8 +105,8 @@ function ReaderUI:registerPostInitCallback(callback)
     table.insert(self.postInitCallback, callback)
 end
 
-function ReaderUI:registerPostReadyCallback(callback)
-    table.insert(self.postReaderCallback, callback)
+function ReaderUI:registerPostReaderReadyCallback(callback)
+    table.insert(self.postReaderReadyCallback, callback)
 end
 
 function ReaderUI:init()
@@ -115,7 +119,7 @@ function ReaderUI:init()
     Device:setIgnoreInput(true) -- Avoid ANRs on Android with unprocessed events.
 
     self.postInitCallback = {}
-    self.postReaderCallback = {}
+    self.postReaderReadyCallback = {}
     -- if we are not the top level dialog ourselves, it must be given in the table
     if not self.dialog then
         self.dialog = self
@@ -162,6 +166,13 @@ function ReaderUI:init()
         view = self.view,
         ui = self
     })
+    -- Handmade/custom ToC and hidden flows
+    self:registerModule("handmade", ReaderHandMade:new{
+        dialog = self.dialog,
+        view = self.view,
+        ui = self,
+        document = self.document,
+    })
     -- Table of content controller
     self:registerModule("toc", ReaderToc:new{
         dialog = self.dialog,
@@ -174,6 +185,12 @@ function ReaderUI:init()
         view = self.view,
         ui = self
     })
+    self:registerModule("annotation", ReaderAnnotation:new{
+        dialog = self.dialog,
+        view = self.view,
+        ui = self,
+        document = self.document,
+    })
     -- reader goto controller
     -- "goto" being a dirty keyword in Lua?
     self:registerModule("gotopage", ReaderGoto:new{
@@ -318,12 +335,14 @@ function ReaderUI:init()
         })
         -- typeset controller
         self:registerModule("typeset", ReaderTypeset:new{
+            configurable = self.document.configurable,
             dialog = self.dialog,
             view = self.view,
             ui = self
         })
         -- font menu
         self:registerModule("font", ReaderFont:new{
+            configurable = self.document.configurable,
             dialog = self.dialog,
             view = self.view,
             ui = self
@@ -342,6 +361,7 @@ function ReaderUI:init()
         })
         -- rolling controller
         self:registerModule("rolling", ReaderRolling:new{
+            configurable = self.document.configurable,
             pan_rate = pan_rate,
             dialog = self.dialog,
             view = self.view,
@@ -416,6 +436,12 @@ function ReaderUI:init()
         view = self.view,
         ui = self,
     })
+    self:registerModule("networklistener", NetworkListener:new {
+        document = self.document,
+        view = self.view,
+        ui = self,
+    })
+
     -- koreader plugins
     for _, plugin_module in ipairs(PluginLoader:loadPlugins()) do
         local ok, plugin_or_err = PluginLoader:createPluginInstance(
@@ -433,15 +459,6 @@ function ReaderUI:init()
         end
     end
 
-    if Device:hasWifiToggle() then
-        local NetworkListener = require("ui/network/networklistener")
-        self:registerModule("networklistener", NetworkListener:new {
-            document = self.document,
-            view = self.view,
-            ui = self,
-        })
-    end
-
     -- Allow others to change settings based on external factors
     -- Must be called after plugins are loaded & before setting are read.
     self:handleEvent(Event:new("DocSettingsLoad", self.doc_settings, self.document))
@@ -453,33 +470,37 @@ function ReaderUI:init()
     end
     self.postInitCallback = nil
 
-    -- Now that document is loaded, store book metadata in settings
-    -- (so that filemanager can use it from sideCar file to display
-    -- Book information).
-    self.doc_settings:saveSetting("doc_props", self.document:getProps())
+    -- Now that document is loaded, store book metadata in settings.
+    local props = self.document:getProps()
+    self.doc_settings:saveSetting("doc_props", props)
+    -- And have an extended and customized copy in memory for quick access.
+    self.doc_props = FileManagerBookInfo.extendProps(props, self.document.file)
 
-    -- Set "reading" status if there is no status.
-    local summary = self.doc_settings:readSetting("summary")
-    if not (summary and summary.status) then
-        if not summary then
-            summary = {}
-        end
+    local md5 = self.doc_settings:readSetting("partial_md5_checksum")
+    if md5 == nil then
+        md5 = util.partialMD5(self.document.file)
+        self.doc_settings:saveSetting("partial_md5_checksum", md5)
+    end
+
+    local summary = self.doc_settings:readSetting("summary", {})
+    if summary.status == nil then
         summary.status = "reading"
         summary.modified = os.date("%Y-%m-%d", os.time())
-        self.doc_settings:saveSetting("summary", summary)
     end
 
-    require("readhistory"):addItem(self.document.file) -- (will update "lastfile")
+    if summary.status ~= "complete" or not G_reader_settings:isTrue("history_freeze_finished_books") then
+        require("readhistory"):addItem(self.document.file) -- (will update "lastfile")
+    end
 
     -- After initialisation notify that document is loaded and rendered
     -- CREngine only reports correct page count after rendering is done
     -- Need the same event for PDF document
     self:handleEvent(Event:new("ReaderReady", self.doc_settings))
 
-    for _,v in ipairs(self.postReaderCallback) do
+    for _,v in ipairs(self.postReaderReadyCallback) do
         v()
     end
-    self.postReaderCallback = nil
+    self.postReaderReadyCallback = nil
 
     Device:setIgnoreInput(false) -- Allow processing of events (on Android).
     Input:inhibitInputUntil(0.2)
@@ -502,6 +523,18 @@ function ReaderUI:registerKeyEvents()
     if Device:hasKeys() then
         self.key_events.Home = { { "Home" } }
         self.key_events.Reload = { { "F5" } }
+        if Device:hasDPad() and Device:useDPadAsActionKeys() then
+            self.key_events.KeyContentSelection = { { { "Up", "Down" } }, event = "StartHighlightIndicator" }
+        end
+        if Device:hasScreenKB() or Device:hasSymKey() then
+            if Device:hasKeyboard() then
+                self.key_events.KeyToggleWifi = { { "Shift", "Home" }, event = "ToggleWifi" }
+                self.key_events.OpenLastDoc = { { "Shift", "Back" } }
+            else -- Currently exclusively targets Kindle 4.
+                self.key_events.KeyToggleWifi = { { "ScreenKB", "Home" }, event = "ToggleWifi" }
+                self.key_events.OpenLastDoc = { { "ScreenKB", "Back" } }
+            end
+        end
     end
 end
 
@@ -536,7 +569,6 @@ function ReaderUI:showFileManager(file)
     local last_dir, last_file
     if file then
         last_dir = util.splitFilePathName(file)
-        last_dir = last_dir:match("(.*)/")
         last_file = file
     else
         last_dir, last_file = self:getLastDirFile(true)
@@ -569,16 +601,17 @@ end
 function ReaderUI:showReader(file, provider, seamless)
     logger.dbg("show reader ui")
 
+    file = ffiUtil.realpath(file)
     if lfs.attributes(file, "mode") ~= "file" then
         UIManager:show(InfoMessage:new{
-             text = T(_("File '%1' does not exist."), BD.filepath(file))
+             text = T(_("File '%1' does not exist."), BD.filepath(filemanagerutil.abbreviate(file)))
         })
         return
     end
 
     if not DocumentRegistry:hasProvider(file) and provider == nil then
         UIManager:show(InfoMessage:new{
-            text = T(_("File '%1' is not supported."), BD.filepath(file))
+            text = T(_("File '%1' is not supported."), BD.filepath(filemanagerutil.abbreviate(file)))
         })
         self:showFileManager(file)
         return
@@ -594,7 +627,7 @@ end
 
 function ReaderUI:showReaderCoroutine(file, provider, seamless)
     UIManager:show(InfoMessage:new{
-        text = T(_("Opening file '%1'."), BD.filepath(file)),
+        text = T(_("Opening file '%1'."), BD.filepath(filemanagerutil.abbreviate(file))),
         timeout = 0.0,
         invisible = seamless,
     })
@@ -656,15 +689,8 @@ function ReaderUI:doShowReader(file, provider, seamless)
         document = document,
     }
 
-    local title = reader.doc_settings:readSetting("doc_props").title
-
-    if title ~= "" then
-        Screen:setWindowTitle(title)
-    else
-        local _, filename = util.splitFilePathName(file)
-        Screen:setWindowTitle(filename)
-    end
-    Device:notifyBookState(title, document)
+    Screen:setWindowTitle(reader.doc_props.display_title)
+    Device:notifyBookState(reader.doc_props.display_title, document)
 
     -- This is mostly for the few callers that bypass the coroutine shenanigans and call doShowReader directly,
     -- instead of showReader...

frontend/configurable.lua

@@ -38,9 +38,6 @@ function Configurable:loadDefaults(config_options)
             local settings_key = prefix..key
             local default = G_reader_settings:readSetting(settings_key)
             self[key] = default or options[j].default_value
-            if not self[key] then
-                self[key] = options[j].default_arg
-            end
         end
     end
 end

frontend/datetime.lua

@@ -307,4 +307,13 @@ function datetime.secondsToDateTime(seconds, twelve_hour_clock, use_locale)
     return message_text
 end
 
+--- Converts a date+time string to seconds
+---- @string "YYYY-MM-DD HH:MM:SS", time may be absent
+---- @treturn seconds
+function datetime.stringToSeconds(datetime_string)
+    local year, month, day = datetime_string:match("(%d+)-(%d+)-(%d+)")
+    local hour, min, sec   = datetime_string:match("(%d+):(%d+):(%d+)")
+    return os.time({ year = year, month = month, day = day, hour = hour or 0, min = min or 0, sec = sec or 0 })
+end
+
 return datetime

frontend/dbg.lua

@@ -86,6 +86,13 @@ function Dbg:v(...)
     end
 end
 
+--- Conditional logging with a stable ref.
+function Dbg.log(...)
+    if Dbg.is_on then
+        return LvDEBUG(...)
+    end
+end
+
 --- Simple traceback.
 function Dbg:traceback()
     return LvDEBUG(debug.traceback())

frontend/device/android/device.lua

@@ -74,6 +74,7 @@ local Device = Generic:extend{
     model = android.prop.product,
     hasKeys = yes,
     hasDPad = no,
+    hasSeamlessWifiToggle = no, -- Requires losing focus to the sytem's network settings and user interaction
     hasExitOptions = no,
     hasEinkScreen = function() return android.isEink() end,
     hasColorScreen = android.isColorScreen,
@@ -459,6 +460,8 @@ function Device:test()
 end
 
 function Device:exit()
+    Generic.exit(self)
+
     android.LOGI(string.format("Stopping %s main activity", android.prop.name))
     android.lib.ANativeActivity_finish(android.app.activity)
 end
@@ -515,7 +518,6 @@ function Device:_showLightDialog()
     elseif action == C.ALIGHTS_DIALOG_CANCEL then
         logger.dbg("Dialog Cancel, brightness: " .. self.powerd.fl_intensity)
         self.powerd:setIntensityHW(self.powerd.fl_intensity)
-        self.powerd:_decideFrontlightState()
         if android.isWarmthDevice() then
             logger.dbg("Dialog Cancel, warmth: " .. self.powerd.fl_warmth)
             self.powerd:setWarmth(self.powerd.fl_warmth)

frontend/device/android/powerd.lua

@@ -11,11 +11,12 @@ function AndroidPowerD:frontlightIntensityHW()
 end
 
 function AndroidPowerD:setIntensityHW(intensity)
-    -- if frontlight switch was toggled of, turn it on
+    -- If the frontlight switch was off, turn it on.
     android.enableFrontlightSwitch()
 
     self.fl_intensity = intensity
     android.setScreenBrightness(math.floor(intensity * self.bright_diff / self.fl_max))
+    self:_decideFrontlightState()
 end
 
 function AndroidPowerD:init()
@@ -57,9 +58,13 @@ function AndroidPowerD:turnOffFrontlightHW()
         return
     end
     android.setScreenBrightness(self.fl_min)
+
+    if android.hasStandaloneWarmth() then
+        android.setScreenWarmth(self.fl_warmth_min)
+    end
 end
 
-function AndroidPowerD:turnOnFrontlightHW()
+function AndroidPowerD:turnOnFrontlightHW(done_callback)
     if self:isFrontlightOn() and self:isFrontlightOnHW() then
         return
     end
@@ -67,6 +72,11 @@ function AndroidPowerD:turnOnFrontlightHW()
     android.enableFrontlightSwitch()
 
     android.setScreenBrightness(math.floor(self.fl_intensity * self.bright_diff / self.fl_max))
+
+    if android.hasStandaloneWarmth() then
+        android.setScreenWarmth(math.floor(self.fl_warmth / self.warm_diff))
+    end
+    return false
 end
 
 return AndroidPowerD

frontend/device/cervantes/powerd.lua

@@ -51,10 +51,13 @@ function CervantesPowerD:init()
         if self.device:hasNaturalLight() then
             local nl_config = G_reader_settings:readSetting("natural_light_config")
             if nl_config then
-                for key,val in pairs(nl_config) do
+                for key, val in pairs(nl_config) do
                     self.device.frontlight_settings[key] = val
                 end
             end
+            -- Does this device's NaturalLight use a custom scale?
+            self.fl_warmth_min = self.device.frontlight_settings.nl_min or self.fl_warmth_min
+            self.fl_warmth_max = self.device.frontlight_settings.nl_max or self.fl_warmth_max
             -- If this device has a mixer, we can use the ioctl for brightness control, as it's much lower latency.
             if self.device:hasNaturalLightMixer() then
                 local kobolight = require("ffi/kobolight")

frontend/device/devicelistener.lua

@@ -42,87 +42,68 @@ function DeviceListener:onShowIntensity()
     return true
 end
 
-function DeviceListener:onShowWarmth(value)
+function DeviceListener:onShowWarmth()
+    if not Device:hasNaturalLight() then return true end
+    -- Display it in the native scale, like FrontLightWidget
     local powerd = Device:getPowerDevice()
-    if powerd.fl_warmth ~= nil then
-        -- powerd.fl_warmth holds the warmth-value in the internal koreader scale [0,100]
-        -- powerd.fl_warmth_max is the maximum value the hardware accepts
-        Notification:notify(T(_("Warmth set to %1%."), math.floor(powerd.fl_warmth)))
-    end
+    Notification:notify(T(_("Warmth set to %1."), powerd:toNativeWarmth(powerd:frontlightWarmth())))
     return true
 end
 
 -- frontlight controller
 if Device:hasFrontlight() then
-
     local function calculateGestureDelta(ges, direction, min, max)
         local delta_int
         if type(ges) == "table" then
-            -- here we are using just two scales
-            -- big scale is for high dynamic ranges (e.g. brightness from 1..100)
-            --           original scale maybe tuned by hand
-            -- small scale is for lower dynamic ranges (e.g. warmth from 1..10)
-            --           scale entries are calculated by math.round(1*sqrt(2)^n)
-            local steps_fl_big_scale = { 0.1, 0.1, 0.2, 0.4, 0.7, 1.1, 1.6, 2.2, 2.9, 3.7, 4.6, 5.6, 6.7, 7.9, 9.2, 10.6, }
-            local steps_fl_small_scale = { 1.0, 1.0, 2.0, 3.0, 4.0, 6.0, 8.1, 11.3 }
-            local steps_fl = steps_fl_big_scale
-            if (max - min) < 50  then
-                steps_fl = steps_fl_small_scale
-            end
-            local gestureScale
-            local scale_multiplier
+            local gesture_multiplier
             if ges.ges == "two_finger_swipe" or ges.ges == "swipe" then
-                scale_multiplier = 0.8
+                gesture_multiplier = 0.8
             else
-                scale_multiplier = 1
+                gesture_multiplier = 1
             end
 
+            local gestureScale
             if ges.direction == "south" or ges.direction == "north" then
-                gestureScale = Screen:getHeight() * scale_multiplier
+                gestureScale = Screen:getHeight() * gesture_multiplier
             elseif ges.direction == "west" or ges.direction == "east" then
-                gestureScale = Screen:getWidth() * scale_multiplier
+                gestureScale = Screen:getWidth() * gesture_multiplier
             else
                 local width = Screen:getWidth()
                 local height = Screen:getHeight()
                 -- diagonal
-                gestureScale = math.sqrt(width * width + height * height) * scale_multiplier
-            end
-
-            local steps_tbl = {}
-            local scale = (max - min) / steps_fl[#steps_fl] / 2 -- full swipe gives half scale
-            for i = 1, #steps_fl, 1 do
-                steps_tbl[i] = math.ceil(steps_fl[i] * scale)
+                gestureScale = math.sqrt(width^2 + height^2) * gesture_multiplier
             end
 
+            -- In case we're passed a gesture that doesn't imply movement (e.g., tap or hold)
             if ges.distance == nil then
                 ges.distance = 1
             end
 
-            local step = math.ceil(#steps_tbl * ges.distance / gestureScale)
-            delta_int = steps_tbl[step] or steps_tbl[#steps_tbl]
+            -- delta_int is calculated by a function f(x) = coeff * x^2
+            -- *) f(x) has the boundary condition: f(1) = max/2;
+            -- *) x is roughly the swipe distance as a fraction of the screen geometry,
+            --    clamped between 0 and 1
+            local x = math.min(1, ges.distance / gestureScale)
+            delta_int = math.ceil(1/2 * max * x^2)
         else
-            -- received amount to change
+            -- The ges arg passed by our caller wasn't a gesture, but an absolute integer increment
             delta_int = ges
         end
         if direction ~= -1 and direction ~= 1 then
-            -- set default value (increase frontlight)
+            -- If the caller didn't specify, opt to *increase* by default
             direction = 1
         end
-        return direction, delta_int
+        return direction * delta_int
     end
 
     -- direction +1 - increase frontlight
     -- direction -1 - decrease frontlight
     function DeviceListener:onChangeFlIntensity(ges, direction)
         local powerd = Device:getPowerDevice()
-        local delta_int
-        --received gesture
-
-        direction, delta_int = calculateGestureDelta(ges, direction, powerd.fl_min, powerd.fl_max)
+        local delta = calculateGestureDelta(ges, direction, powerd.fl_min, powerd.fl_max)
 
-        local new_intensity = powerd.fl_intensity + direction * delta_int
-        if new_intensity == nil then return true end
-        -- when new_intensity <=0, toggle light off
+        local new_intensity = powerd:frontlightIntensity() + delta
+        -- when new_intensity <= 0, toggle light off
         self:onSetFlIntensity(new_intensity)
         self:onShowIntensity()
         return true
@@ -151,15 +132,17 @@ if Device:hasFrontlight() then
     -- direction +1 - increase frontlight warmth
     -- direction -1 - decrease frontlight warmth
     function DeviceListener:onChangeFlWarmth(ges, direction)
-        local powerd = Device:getPowerDevice()
-        if powerd.fl_warmth == nil then return false end
+        if not Device:hasNaturalLight() then return true end
 
-        local delta_int
-        --received gesture
+        local powerd = Device:getPowerDevice()
+        local delta = calculateGestureDelta(ges, direction, powerd.fl_warmth_min, powerd.fl_warmth_max)
 
-        direction, delta_int = calculateGestureDelta(ges, direction, powerd.fl_warmth_min, powerd.fl_warmth_max)
+        -- Given that the native warmth ranges are usually pretty restrictive (e.g., [0, 10] or [0, 24]),
+        -- do the computations in the native scale, to ensure we always actually *change* something,
+        -- in case both the old and new value would round to the same native step,
+        -- despite being different in the API scale, which is stupidly fixed at [0, 100]...
+        local warmth = powerd:fromNativeWarmth(powerd:toNativeWarmth(powerd:frontlightWarmth()) + delta)
 
-        local warmth = math.floor(powerd.fl_warmth + direction * delta_int * 100 / powerd.fl_warmth_max)
         self:onSetFlWarmth(warmth)
         self:onShowWarmth()
         return true
@@ -187,7 +170,7 @@ if Device:hasFrontlight() then
     function DeviceListener:onToggleFrontlight()
         local powerd = Device:getPowerDevice()
         local new_text
-        if powerd.is_fl_on then
+        if powerd:isFrontlightOn() then
             new_text = _("Frontlight disabled.")
         else
             new_text = _("Frontlight enabled.")
@@ -221,6 +204,19 @@ if Device:hasGSensor() then
         Notification:notify(new_text)
         return true
     end
+
+    function DeviceListener:onLockGSensor()
+        G_reader_settings:flipNilOrFalse("input_lock_gsensor")
+        Device:lockGSensor(G_reader_settings:isTrue("input_lock_gsensor"))
+        local new_text
+        if G_reader_settings:isTrue("input_lock_gsensor") then
+            new_text = _("Orientation locked.")
+        else
+            new_text = _("Orientation unlocked.")
+        end
+        Notification:notify(new_text)
+        return true
+    end
 end
 
 if not Device:isAlwaysFullscreen() then
@@ -328,6 +324,14 @@ function DeviceListener:onToggleKeyRepeat(toggle)
     Device:toggleKeyRepeat(G_reader_settings:nilOrFalse("input_no_key_repeat"))
 end
 
+function DeviceListener:onRequestUSBMS()
+    local MassStorage = require("ui/elements/mass_storage")
+    -- It already takes care of the canToggleMassStorage cap check for us
+    -- NOTE: Never request confirmation, it's sorted right next to exit, restart & friends in Dispatcher,
+    --       and they don't either...
+    MassStorage:start(false)
+end
+
 function DeviceListener:onRestart()
     self.ui.menu:exitOrRestart(function() UIManager:restartKOReader() end)
 end
@@ -355,4 +359,10 @@ function DeviceListener:onFullRefresh()
     UIManager:setDirty(nil, "full")
 end
 
+-- On resume, make sure we restore Gestures handling in InputContainer, to avoid confusion for scatter-brained users ;).
+-- It's also helpful when the IgnoreTouchInput event is emitted by Dispatcher through other means than Gestures.
+function DeviceListener:onResume()
+    UIManager:setIgnoreTouchInput(false)
+end
+
 return DeviceListener

frontend/device/generic/device.lua

@@ -41,11 +41,15 @@ local Device = {
     hasAuxBattery = no,
     hasKeyboard = no,
     hasKeys = no,
+    hasScreenKB = no, -- in practice only some Kindles
+    hasSymKey = no, -- in practice only some Kindles
     canKeyRepeat = no,
     hasDPad = no,
+    useDPadAsActionKeys = no,
     hasExitOptions = yes,
     hasFewKeys = no,
     hasWifiToggle = yes,
+    hasSeamlessWifiToggle = yes, -- Can toggle Wi-Fi without focus loss and extra user interaction (i.e., not Android)
     hasWifiManager = no,
     hasWifiRestore = no,
     isDefaultFullscreen = yes,
@@ -61,6 +65,7 @@ local Device = {
     hasExternalSD = no, -- or other storage volume that cannot be accessed using the File Manager
     canHWDither = no,
     canHWInvert = no,
+    hasKaleidoWfm = no,
     canDoSwipeAnimation = no,
     canModifyFBInfo = no, -- some NTX boards do wonky things with the rotate flag after a FBIOPUT_VSCREENINFO ioctl
     canUseCBB = yes, -- The C BB maintains a 1:1 feature parity with the Lua BB, except that is has NO support for BB4, and limited support for BBRGB24
@@ -72,6 +77,7 @@ local Device = {
     isGSensorLocked = no,
     canToggleMassStorage = no,
     canToggleChargingLED = no,
+    _updateChargingLED = nil,
     canUseWAL = yes, -- requires mmap'ed I/O on the target FS
     canRestart = yes,
     canSuspend = no,
@@ -283,6 +289,11 @@ function Device:init()
             self.screen:toggleSWDithering(true)
         end
     end
+
+    -- Can't be seamless if you can't do it at all ;)
+    if not self:hasWifiToggle() then
+        self.hasSeamlessWifiToggle = no
+    end
 end
 
 function Device:setScreenDPI(dpi_override)
@@ -319,7 +330,7 @@ function Device:onPowerEvent(ev)
                 self:resume()
                 local widget_was_closed = Screensaver:close()
                 if widget_was_closed and self:needsScreenRefreshAfterResume() then
-                    UIManager:scheduleIn(1, function() self.screen:refreshFull() end)
+                    UIManager:scheduleIn(1, function() self.screen:refreshFull(0, 0, self.screen:getWidth(), self.screen:getHeight()) end)
                 end
                 self.powerd:afterResume()
             end
@@ -364,7 +375,7 @@ function Device:onPowerEvent(ev)
         --       and on platforms where we defer to a system tool, it'd probably suspend too early!
         --       c.f., #6676
         if self:needsScreenRefreshAfterResume() then
-            self.screen:refreshFull()
+            self.screen:refreshFull(0, 0, self.screen:getWidth(), self.screen:getHeight())
         end
         -- NOTE: In the same vein as above, make sure we update the screen *now*, before dealing with Wi-Fi.
         UIManager:forceRePaint()
@@ -408,6 +419,15 @@ function Device:install()
             end
             UIManager:broadcastEvent(Event:new("Exit", save_quit))
         end,
+        cancel_text = _("Later"),
+        cancel_callback = function()
+            local InfoMessage = require("ui/widget/infomessage")
+            UIManager:show(InfoMessage:new{
+                text = _("The update will be applied the next time KOReader is started."),
+                unmovable = true,
+            })
+        end,
+        unmovable = true,
     })
 end
 
@@ -552,7 +572,7 @@ function Device:exit()
     G_reader_settings:close()
 
     -- I/O teardown
-    require("ffi/input"):closeAll()
+    self.input.teardown()
 end
 
 -- Lifted from busybox's libbb/inet_cksum.c
@@ -1002,6 +1022,9 @@ function Device:_UIManagerReady(uimgr)
     -- Setup PM event handlers
     -- NOTE: We keep forwarding the uimgr reference because some implementations don't actually have a module-local UIManager ref to update
     self:_setEventHandlers(uimgr)
+
+    -- Returns a self-debouncing scheduling call (~4s to give some leeway to the kernel, and debounce to deal with potential chattering)
+    self._updateChargingLED = UIManager:debounce(4, false, function() self:setupChargingLED() end)
 end
 -- In case implementations *also* need a reference to UIManager, *this* is the one to implement!
 function Device:UIManagerReady(uimgr) end
@@ -1015,7 +1038,6 @@ function Device:_setEventHandlers(uimgr)
                 text = message_text or _("Are you sure you want to reboot the device?"),
                 ok_text = _("Reboot"),
                 ok_callback = function()
-                    UIManager:broadcastEvent(Event:new("Reboot"))
                     UIManager:nextTick(UIManager.reboot_action)
                 end,
             })
@@ -1031,7 +1053,6 @@ function Device:_setEventHandlers(uimgr)
                 text = message_text or _("Are you sure you want to power off the device?"),
                 ok_text = _("Power off"),
                 ok_callback = function()
-                    UIManager:broadcastEvent(Event:new("PowerOff"))
                     UIManager:nextTick(UIManager.poweroff_action)
                 end,
             })
@@ -1093,8 +1114,10 @@ end
 -- The common operations that should be performed after resuming the device.
 function Device:_afterResume(inhibit)
     if inhibit ~= false then
-        -- Restore key repeat
-        self:restoreKeyRepeat()
+        -- Restore key repeat if it's not disabled
+        if G_reader_settings:nilOrFalse("input_no_key_repeat") then
+            self:restoreKeyRepeat()
+        end
 
         -- Restore full input handling
         self.input:inhibitInput(false)
@@ -1105,15 +1128,16 @@ end
 
 -- The common operations that should be performed when the device is plugged to a power source.
 function Device:_beforeCharging()
-    -- Leave the kernel some time to figure it out ;o).
-    UIManager:scheduleIn(1, function() self:setupChargingLED() end)
+    -- Invalidate the capacity cache to make sure we poll up-to-date values for the LED check
+    self.powerd:invalidateCapacityCache()
+    self:_updateChargingLED()
     UIManager:broadcastEvent(Event:new("Charging"))
 end
 
 -- The common operations that should be performed when the device is unplugged from a power source.
 function Device:_afterNotCharging()
-    -- Leave the kernel some time to figure it out ;o).
-    UIManager:scheduleIn(1, function() self:setupChargingLED() end)
+    self.powerd:invalidateCapacityCache()
+    self:_updateChargingLED()
     UIManager:broadcastEvent(Event:new("NotCharging"))
 end
 

frontend/device/generic/powerd.lua

@@ -43,7 +43,10 @@ function BasePowerD:new(o)
 end
 
 function BasePowerD:init() end
-function BasePowerD:setIntensityHW(intensity) end
+--- @note: This should *always* call self:_decideFrontlightState() in its coda (unless you have a custom isFrontlightOn implementation)!
+function BasePowerD:setIntensityHW(intensity)
+    self:_decideFrontlightState()
+end
 --- @note: Unlike the "public" setWarmth, this one takes a value in the *native* scale!
 function BasePowerD:setWarmthHW(warmth) end
 function BasePowerD:getCapacityHW() return 0 end
@@ -207,7 +210,6 @@ function BasePowerD:setIntensity(intensity)
     if not self.device:hasFrontlight() then return false end
     if intensity == self:frontlightIntensity() then return false end
     self.fl_intensity = self:normalizeIntensity(intensity)
-    self:_decideFrontlightState()
     logger.dbg("set light intensity", self.fl_intensity)
     self:setIntensityHW(self.fl_intensity)
     self:stateChanged()

frontend/device/gesturedetector.lua

@@ -118,6 +118,16 @@ function GestureDetector:init()
     self.MULTISWIPE_THRESHOLD = self.DOUBLE_TAP_DISTANCE
 end
 
+local function deepCopyEv(tev)
+    return {
+        x = tev.x,
+        y = tev.y,
+        id = tev.id,
+        slot = tev.slot,
+        timev = tev.timev, -- A ref is enough for this table, it's re-assigned to a new object on every SYN_REPORT
+    }
+end
+
 -- Contact object, it'll keep track of everything we need for a single contact across its lifetime
 -- i.e., from this contact's down to up (or its *effective* up for double-taps, e.g., when the tap or double_tap is emitted).
 -- We'll identify contacts by their slot numbers, and store 'em in GestureDetector's active_contacts table (hash).
@@ -139,7 +149,7 @@ function GestureDetector:newContact(slot)
         state = Contact.initialState, -- Current state function
         slot = slot, -- Current ABS_MT_SLOT value (also its key in the active_contacts hash)
         id = -1, -- Current ABS_MT_TRACKING_ID value
-        initial_tev = nil, -- Copy of the input event table at first contact (i.e., at contact down)
+        initial_tev = nil, -- Copy of the input event table at first contact (i.e., at contact down [iff the platform is sane, might be a copy of current_tev otherwise])
         current_tev = nil, -- Pointer to the current input event table, ref is *stable*, c.f., NOTE in feedEvent below
         down = false, -- Contact is down (as opposed to up, i.e., lifted). Only really happens for double-tap handling, in every other case the Contact object is destroyed on lift.
         pending_double_tap_timer = false, -- Contact is pending a double_tap timer
@@ -157,6 +167,12 @@ function GestureDetector:newContact(slot)
     -- If we have a buddy contact, point its own buddy ref to us
     if buddy_contact then
         buddy_contact.buddy_contact = self.active_contacts[slot]
+
+        -- And make sure it has an initial_tev recorded, for misbehaving platforms...
+        if not buddy_contact.initial_tev then
+            buddy_contact.initial_tev = deepCopyEv(buddy_contact.current_tev)
+            logger.warn("GestureDetector:newContact recorded an initial_tev out of order for buddy slot", buddy_contact.slot)
+        end
     end
 
     return self.active_contacts[slot]
@@ -230,16 +246,6 @@ function GestureDetector:feedEvent(tevs)
     return gestures
 end
 
-local function deepCopyEv(tev)
-    return {
-        x = tev.x,
-        y = tev.y,
-        id = tev.id,
-        slot = tev.slot,
-        timev = tev.timev, -- A ref is enough for this table, it's re-assigned to a new object on every SYN_REPORT
-    }
-end
-
 --[[
 tap2 is the later tap
 --]]
@@ -379,6 +385,18 @@ function Contact:switchState(state_func, func_arg)
     return state_func(self, func_arg)
 end
 
+-- Unlike switchState, we don't *call* the new state, and we ensure that initial_tev is set,
+-- in case initialState never ran on a contact down because the platform screwed up (e.g., PB with broken MT).
+-- The rest of the code, in particular the buddy system, assumes initial_tev is always set (and supposedly sane).
+function Contact:setState(state_func)
+    -- NOTE: Safety net for broken platforms that might screw up slot order...
+    if not self.initial_tev then
+        self.initial_tev = deepCopyEv(self.current_tev)
+        logger.warn("Contact:setState recorded an initial_tev out of order for slot", self.slot)
+    end
+    self.state = state_func
+end
+
 function Contact:initialState()
     local tev = self.current_tev
 
@@ -483,7 +501,7 @@ function Contact:tapState(new_tap)
                 -- Mark that slot
                 self.mt_gesture = "tap"
                 -- Neuter its buddy
-                buddy_contact.state = Contact.voidState
+                buddy_contact:setState(Contact.voidState)
                 buddy_contact.mt_gesture = "tap"
 
                 local pos0 = Geom:new{
@@ -513,7 +531,7 @@ function Contact:tapState(new_tap)
                 logger.dbg("Contact:tapState: Two-contact tap failed to pass the two_finger_tap constraints -> single tap @", tev.x, tev.y)
                 -- We blew the gesture position/time constraints,
                 -- neuter buddy and send a single tap on this slot.
-                buddy_contact.state = Contact.voidState
+                buddy_contact:setState(Contact.voidState)
                 gesture_detector:dropContact(self)
 
                 return {
@@ -714,7 +732,7 @@ function Contact:panState(keep_contact)
                 else
                     buddy_contact.mt_gesture = "swipe"
                 end
-                buddy_contact.state = Contact.voidState
+                buddy_contact:setState(Contact.voidState)
 
                 local ges_ev = self:handleTwoFingerPan(buddy_contact)
                 if ges_ev then
@@ -818,8 +836,8 @@ function Contact:voidState()
                     -- and if it lifts even later, we'd have to deal with spurious moves first, probably leading into a tap...
                     -- If the gesture *succeeds*, the buddy contact will be dropped whenever it's actually lifted,
                     -- thanks to the temporary tracking id switcheroo & voidState...
-                    buddy_contact.state = Contact.voidState
                     buddy_contact.current_tev.id = buddy_tid
+                    buddy_contact:setState(Contact.voidState)
                 end
                 -- Regardless of whether we detected a gesture, this is a contact lift, so it's curtains for us!
                 gesture_detector:dropContact(self)
@@ -942,7 +960,7 @@ function Contact:handlePan()
         else
             buddy_contact.mt_gesture = "pan"
         end
-        buddy_contact.state = Contact.voidState
+        buddy_contact:setState(Contact.voidState)
 
         return self:handleTwoFingerPan(buddy_contact)
     elseif self.down then
@@ -1123,7 +1141,7 @@ function Contact:handleTwoFingerPan(buddy_contact)
                 ges_ev._end_pos = nil
             end
             ges_ev.direction = gesture_detector.DIRECTION_TABLE[tpan_dir]
-            -- Use the the sum of both contacts' travel for the distance
+            -- Use the sum of both contacts' travel for the distance
             ges_ev.distance = tpan_dis + rpan_dis
             -- Some handlers might also want to know the distance between the two contacts on lift & down.
             ges_ev.span = end_distance
@@ -1166,7 +1184,7 @@ function Contact:handlePanRelease(keep_contact)
         -- Both main contacts are actives and we are down, mark that slot
         self.mt_gesture = "pan_release"
         -- Neuter its buddy
-        buddy_contact.state = Contact.voidState
+        buddy_contact:setState(Contact.voidState)
         buddy_contact.mt_gesture = "pan_release"
 
         logger.dbg("Contact:handlePanRelease: two_finger_pan_release detected")
@@ -1212,7 +1230,7 @@ function Contact:holdState(new_hold)
             -- Both main contacts are actives and we are down, mark that slot
             self.mt_gesture = "hold"
             -- Neuter its buddy
-            buddy_contact.state = Contact.voidState
+            buddy_contact:setState(Contact.voidState)
             buddy_contact.mt_gesture = "hold"
 
             local pos0 = Geom:new{
@@ -1269,7 +1287,7 @@ function Contact:holdState(new_hold)
                     end
                 end
                 -- Regardless of whether this panned out (pun intended), this is a lift, so we'll defer to voidState next.
-                buddy_contact.state = Contact.voidState
+                buddy_contact:setState(Contact.voidState)
                 gesture_detector:dropContact(self)
                 return ges_ev
             elseif self.mt_gesture == "hold_pan" or self.mt_gesture == "pan" then
@@ -1280,7 +1298,7 @@ function Contact:holdState(new_hold)
                 buddy_contact.mt_gesture = "hold_release"
             end
             -- Neuter its buddy
-            buddy_contact.state = Contact.voidState
+            buddy_contact:setState(Contact.voidState)
 
             -- Don't drop buddy, voidState will handle it
             gesture_detector:dropContact(self)

frontend/device/input.lua

@@ -139,23 +139,19 @@ local Input = {
             "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M",
             "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
             "Up", "Down", "Left", "Right", "Press", "Backspace", "End",
-            "Back", "Sym", "AA", "Menu", "Home", "Del",
+            "Back", "Sym", "AA", "Menu", "Home", "Del", "ScreenKB",
             "LPgBack", "RPgBack", "LPgFwd", "RPgFwd"
         },
     },
 
     fake_event_set = {
-        IntoSS = true, OutOfSS = true,
+        IntoSS = true, OutOfSS = true, ExitingSS = true,
         UsbPlugIn = true, UsbPlugOut = true,
         Charging = true, NotCharging = true,
         WakeupFromSuspend = true, ReadyToSuspend = true,
         UsbDevicePlugIn = true, UsbDevicePlugOut = true,
     },
-    -- Subset of fake_event_set for events that require passing a parameter along
-    complex_fake_event_set = {
-        UsbDevicePlugIn = true, UsbDevicePlugOut = true,
-    },
-    -- Crappy FIFO to forward parameters for those events to UIManager
+    -- Crappy FIFO to forward parameters to UIManager for the subset of fake_event_set that require passing a parameter along
     fake_event_args = {
         UsbDevicePlugIn = {},
         UsbDevicePlugOut = {},
@@ -175,6 +171,7 @@ local Input = {
         Shift = false,
         Sym = false,
         Meta = false,
+        ScreenKB = false,
     },
 
     -- repeat state:
@@ -182,6 +179,7 @@ local Input = {
 
     -- touch state:
     main_finger_slot = 0,
+    pen_slot = 4,
     cur_slot = 0,
     MTSlots = nil, -- table, object may be replaced at runtime
     active_slots = nil, -- ditto
@@ -198,6 +196,10 @@ local Input = {
     setClipboardText = function(text)
         _internal_clipboard_text = text or ""
     end,
+
+    -- open'ed input devices hashmap (key: path, value: fd number)
+    -- Must be a class member, both because Input is a singleton and that state is process-wide anyway.
+    opened_devices = {},
 }
 
 function Input:new(o)
@@ -223,6 +225,10 @@ function Input:init()
         },
     }
 
+    -- Always send pen data to a slot far enough away from our main finger slot that it can never be matched with a finger buddy in GestureDetector (i.e., +/- 1),
+    -- with an extra bit of leeway, since we don't even actually support three finger gestures ;).
+    self.pen_slot = self.main_finger_slot + 4
+
     self.gesture_detector = GestureDetector:new{
         screen = self.device.screen,
         input = self,
@@ -246,8 +252,9 @@ function Input:init()
     end
 
     -- set up fake event map
-    self.event_map[10000] = "IntoSS" -- go into screen saver
-    self.event_map[10001] = "OutOfSS" -- go out of screen saver
+    self.event_map[10000] = "IntoSS" -- Requested to go into screen saver
+    self.event_map[10001] = "OutOfSS" -- Requested to go out of screen saver
+    self.event_map[10002] = "ExitingSS" -- Specific to Kindle, SS *actually* closed
     self.event_map[10010] = "UsbPlugIn"
     self.event_map[10011] = "UsbPlugOut"
     self.event_map[10020] = "Charging"
@@ -293,14 +300,97 @@ function Input:disableRotationMap()
 end
 
 --[[--
-Wrapper for FFI input open.
+Wrapper for our Lua/C input module's open.
+
+Note that we adhere to the "." syntax here for compatibility.
+
+The `name` argument is optional, and used for logging purposes only.
+--]]
+function Input.open(path, name)
+    -- Make sure we don't open the same device twice.
+    if not Input.opened_devices[path] then
+        local fd = input.open(path)
+        if fd then
+            Input.opened_devices[path] = fd
+            if name then
+                logger.dbg("Opened fd", fd, "for input device", name, "@", path)
+            else
+                logger.dbg("Opened fd", fd, "for input device @", path)
+            end
+        end
+        -- No need to log failures, input will have raised an error already,
+        -- and we want to make those fatal, so we don't protect this call.
+        return fd
+    end
+end
+
+--[[--
+Wrapper for our Lua/C input module's fdopen.
+
+Note that we adhere to the "." syntax here for compatibility.
+
+The `name` argument is optional, and used for logging purposes only.
+`path` is mandatory, though!
+--]]
+function Input.fdopen(fd, path, name)
+    -- Make sure we don't open the same device twice.
+    if not Input.opened_devices[path] then
+        input.fdopen(fd)
+        -- As with input.open, it will throw on error (closing the fd first)
+        Input.opened_devices[path] = fd
+        if name then
+            logger.dbg("Kept fd", fd, "open for input device", name, "@", path)
+        else
+            logger.dbg("Kept fd", fd, "open for input device @", path)
+        end
+        return fd
+    end
+end
+
+--[[--
+Wrapper for our Lua/C input module's close.
 
 Note that we adhere to the "." syntax here for compatibility.
+--]]
+function Input.close(path)
+    -- Make sure we actually know about this device
+    local fd = Input.opened_devices[path]
+    if fd then
+        local ok, err = input.close(fd)
+        if ok or err == C.ENODEV then
+            -- Either the call succeeded,
+            -- or the backend had already caught an ENODEV in waitForInput and closed the fd internally.
+            -- (Because the EvdevInputRemove Event comes from an UsbDevicePlugOut uevent forwarded as an... *input* EV_KEY event ;)).
+            -- Regardless, that device is gone, so clear its spot in the hashmap.
+            Input.opened_devices[path] = nil
+        end
+    else
+        logger.warn("Tried to close an unknown input device @", path)
+    end
+end
+
+--[[--
+Wrapper for our Lua/C input module's closeAll.
 
-@todo Clean up separation FFI/this.
+Note that we adhere to the "." syntax here for compatibility.
 --]]
-function Input.open(device, is_emu_events)
-    return input.open(device, is_emu_events and 1 or 0)
+function Input.teardown()
+    input.closeAll()
+    Input.opened_devices = {}
+end
+
+-- Wrappers for the custom FFI implementations with no concept of paths or fd
+if input.is_ffi then
+    -- Pass args as-is. None of 'em actually *take* arguments, but some may be invoked as methods...
+    function Input.open(...)
+        return input.open(...)
+    end
+    function Input.close(...)
+        return input.close(...)
+    end
+    function Input.teardown(...)
+        return input.closeAll(...)
+    end
 end
 
 --[[--
@@ -532,15 +622,17 @@ function Input:resetState()
 end
 
 function Input:handleKeyBoardEv(ev)
-    -- Detect loss of contact for the "snow" protocol...
-    -- NOTE: Some ST devices may also behave similarly, but we handle those via ABS_PRESSURE
+    -- Detect loss of contact for the "snow" protocol, as we *never* get EV_ABS:ABS_MT_TRACKING_ID:-1 on those...
+    -- NOTE: The same logic *could* be used on *some* ST devices to detect contact states,
+    --       but we instead prefer using EV_ABS:ABS_PRESSURE on those,
+    --       as it appears to be more common than EV_KEY:BTN_TOUCH on the devices we care about...
     if self.snow_protocol then
         if ev.code == C.BTN_TOUCH then
             if ev.value == 0 then
                 -- Kernel sends it after loss of contact for *all* slots,
                 -- only once the final contact point has been lifted.
                 if #self.MTSlots == 0 then
-                    -- Likely, since this is usually in its own event stream,
+                    -- Likely, since this is usually in its own input frame,
                     -- meaning self.MTSlots has *just* been cleared by our last EV_SYN:SYN_REPORT handler...
                     -- So, poke at the actual data to find the slots that are currently active (i.e., in the down state),
                     -- and re-populate a minimal self.MTSlots array that simply switches them to the up state ;).
@@ -553,7 +645,7 @@ function Input:handleKeyBoardEv(ev)
                 else
                     -- Unlikely, given what we mentioned above...
                     -- Note that, funnily enough, its EV_KEY:BTN_TOUCH:1 counterpart
-                    -- *can* be in the same initial event stream as the EV_ABS batch...
+                    -- *can* be in the same initial input frame as the EV_ABS batch...
                     for _, MTSlot in ipairs(self.MTSlots) do
                         self:setMtSlot(MTSlot.slot, "id", -1)
                     end
@@ -564,24 +656,30 @@ function Input:handleKeyBoardEv(ev)
         end
     elseif self.wacom_protocol then
         if ev.code == C.BTN_TOOL_PEN then
-            -- Always send pen data to slot 2
-            self:setupSlotData(2)
+            -- Switch to the dedicated pen slot, and make sure it's active, as this can come in a dedicated input frame
+            self:setupSlotData(self.pen_slot)
             if ev.value == 1 then
                 self:setCurrentMtSlot("tool", TOOL_TYPE_PEN)
             else
                 self:setCurrentMtSlot("tool", TOOL_TYPE_FINGER)
+                -- Switch back to our main finger slot
+                self.cur_slot = self.main_finger_slot
             end
 
             return
         elseif ev.code == C.BTN_TOUCH then
             -- BTN_TOUCH is bracketed by BTN_TOOL_PEN, so we can limit this to pens, to avoid stomping on panel slots.
             if self:getCurrentMtSlotData("tool") == TOOL_TYPE_PEN then
+                -- Make sure the pen slot is active, as this can come in a dedicated input frame
+                -- (i.e., we need it to be referenced by self.MTSlots for the lift to be picked up in the EV_SYN:SYN_REPORT handler).
+                -- (Conversely, getCurrentMtSlotData pokes at the *persistent* slot data in self.ev_slots,
+                -- so it can keep track of data across input frames).
+                self:setupSlotData(self.pen_slot)
                 -- Much like on snow, use this to detect contact down & lift,
                 -- as ABS_PRESSURE may be entirely omitted from hover events,
                 -- and ABS_DISTANCE is not very clear cut...
-                self:setupSlotData(2)
                 if ev.value == 1 then
-                    self:setCurrentMtSlot("id", 2)
+                    self:setCurrentMtSlot("id", self.pen_slot)
                 else
                     self:setCurrentMtSlot("id", -1)
                 end
@@ -616,7 +714,7 @@ function Input:handleKeyBoardEv(ev)
         -- So, we simply store it somewhere our handler can find and call it a day.
         -- And we use an array as a FIFO because we cannot guarantee that insertions and removals will interleave nicely.
         -- (This is all in the name of avoiding complexifying the common codepaths for events that should be few and far between).
-        if self.complex_fake_event_set[keycode] then
+        if self.fake_event_args[keycode] then
             table.insert(self.fake_event_args[keycode], ev.value)
         end
         return keycode
@@ -706,6 +804,9 @@ function Input:handlePowerManagementOnlyEv(ev)
     end
 
     if self.fake_event_set[keycode] then
+        if self.fake_event_args[keycode] then
+            table.insert(self.fake_event_args[keycode], ev.value)
+        end
         return keycode
     end
 
@@ -719,6 +820,16 @@ function Input:handlePowerManagementOnlyEv(ev)
         end
     end
 
+    -- Make sure we don't leave modifiers in an inconsistent state
+    if self.modifiers[keycode] ~= nil then
+        if ev.value == KEY_PRESS then
+            self.modifiers[keycode] = true
+        elseif ev.value == KEY_RELEASE then
+            self.modifiers[keycode] = false
+        end
+        return
+    end
+
     -- Nothing to see, move along!
     return
 end
@@ -789,17 +900,25 @@ function Input:handleTouchEv(ev)
         elseif ev.code == C.ABS_MT_TRACKING_ID then
             if self.snow_protocol then
                 -- NOTE: We'll never get an ABS_MT_SLOT event, instead we have a slot-like ABS_MT_TRACKING_ID value...
-                --       This also means this may never be set to -1 on contact lift,
-                --       which is why we instead rely on EV_KEY:BTN_TOUCH:0 for that (c.f., handleKeyBoardEv).
-                self:setupSlotData(ev.value)
-            else
-                -- The Elan driver needlessly repeats unchanged ABS_MT_TRACKING_ID values,
-                -- which allows us to do this here instead of relying more aggressively on setCurrentMtSlotChecked.
-                if #self.MTSlots == 0 then
-                    self:addSlot(self.cur_slot)
+                --       This also means that, unlike on sane devices, this will *never* be set to -1 on contact lift,
+                --       which is why we instead have to rely on EV_KEY:BTN_TOUCH:0 for that (c.f., handleKeyBoardEv).
+                if ev.value == -1 then
+                    -- NOTE: While *actual* snow_protocol devices will *never* emit an EV_ABS:ABS_MT_TRACKING_ID:-1 event,
+                    --       we've seen brand new revisions of snow_protocol devices shipping with sane panels instead,
+                    --       so we'll need to disable the quirks at runtime to handle these properly...
+                    --       (c.f., https://www.mobileread.com/forums/showpost.php?p=4383629&postcount=997).
+                    -- NOTE: Simply skipping the slot storage setup for -1 would not be enough, as it would only fix ST handling.
+                    --       MT would be broken, because buddy contact detection in GestureDetector looks at slot +/- 1,
+                    --       whereas we'd be having the main contact point at a stupidly large slot number
+                    --       (because it would match ABS_MT_TRACKING_ID, given the lack of ABS_MT_SLOT, at least for the first input frame),
+                    --       while the second contact would be at slot 1, because it would immediately have required emitting a proper ABS_MT_SLOT event...
+                    logger.warn("Input: Disabled snow_protocol quirks because your device's hardware revision doesn't appear to need them!")
+                    self.snow_protocol = false
+                else
+                    self:setupSlotData(ev.value)
                 end
             end
-            self:setCurrentMtSlot("id", ev.value)
+            self:setCurrentMtSlotChecked("id", ev.value)
         elseif ev.code == C.ABS_MT_TOOL_TYPE then
             -- NOTE: On the Elipsa: Finger == 0; Pen == 1
             self:setCurrentMtSlot("tool", ev.value)
@@ -934,14 +1053,15 @@ end
 
 function Input:handleTouchEvLegacy(ev)
     -- Single Touch Protocol.
-    -- Some devices emit both singletouch and multitouch events,
-    -- on those devices, the 'handleTouchEv' function may not behave as expected. Use this one instead.
+    -- Some devices emit both singletouch and multitouch events.
+    -- On those devices, `handleTouchEv` may not behave as expected. Use this one instead.
     if ev.type == C.EV_ABS then
         if ev.code == C.ABS_X then
             self:setCurrentMtSlotChecked("x", ev.value)
         elseif ev.code == C.ABS_Y then
             self:setCurrentMtSlotChecked("y", ev.value)
         elseif ev.code == C.ABS_PRESSURE then
+            -- This is the least common denominator we can use to detect contact down & lift...
             if ev.value ~= 0 then
                 self:setCurrentMtSlotChecked("id", 1)
             else
@@ -984,39 +1104,37 @@ end
 --- (Translation should be done via registerEventAdjustHook in Device implementations).
 --- This needs to be called *via handleGyroEv* in a handleMiscEv implementation (c.f., Kobo, Kindle or PocketBook).
 function Input:handleMiscGyroEv(ev)
-    local rotation_mode, screen_mode
+    local rotation
     if ev.value == C.DEVICE_ROTATED_UPRIGHT then
         -- i.e., UR
-        rotation_mode = framebuffer.DEVICE_ROTATED_UPRIGHT
-        screen_mode = "portrait"
+        rotation = framebuffer.DEVICE_ROTATED_UPRIGHT
     elseif ev.value == C.DEVICE_ROTATED_CLOCKWISE then
         -- i.e., CW
-        rotation_mode = framebuffer.DEVICE_ROTATED_CLOCKWISE
-        screen_mode = "landscape"
+        rotation = framebuffer.DEVICE_ROTATED_CLOCKWISE
     elseif ev.value == C.DEVICE_ROTATED_UPSIDE_DOWN then
         -- i.e., UD
-        rotation_mode = framebuffer.DEVICE_ROTATED_UPSIDE_DOWN
-        screen_mode = "portrait"
+        rotation = framebuffer.DEVICE_ROTATED_UPSIDE_DOWN
     elseif ev.value == C.DEVICE_ROTATED_COUNTER_CLOCKWISE then
         -- i.e., CCW
-        rotation_mode = framebuffer.DEVICE_ROTATED_COUNTER_CLOCKWISE
-        screen_mode = "landscape"
+        rotation = framebuffer.DEVICE_ROTATED_COUNTER_CLOCKWISE
     else
         -- Discard FRONT/BACK
         return
     end
 
-    local old_rotation_mode = self.device.screen:getRotationMode()
+    local old_rotation = self.device.screen:getRotationMode()
     if self.device:isGSensorLocked() then
-        local old_screen_mode = self.device.screen:getScreenMode()
-        if rotation_mode and rotation_mode ~= old_rotation_mode and screen_mode == old_screen_mode then
+        local matching_orientation = bit.band(rotation, 1) == bit.band(old_rotation, 1)
+        if rotation and rotation ~= old_rotation and matching_orientation then
             -- Cheaper than a full SetRotationMode event, as we don't need to re-layout anything.
-            self.device.screen:setRotationMode(rotation_mode)
+            self.device.screen:setRotationMode(rotation)
             UIManager:onRotation()
         end
     else
-        if rotation_mode and rotation_mode ~= old_rotation_mode then
-            return Event:new("SetRotationMode", rotation_mode)
+        if rotation and rotation ~= old_rotation then
+            -- NOTE: We do *NOT* send a broadcast manually, and instead rely on the main loop's sendEvent:
+            --       this ensures that only widgets that actually know how to handle a rotation will do so ;).
+            return Event:new("SetRotationMode", rotation)
         end
     end
 end
@@ -1053,29 +1171,29 @@ function Input:initMtSlot(slot)
     end
 end
 
+function Input:getMtSlot(slot)
+    return self.ev_slots[slot]
+end
+
+function Input:getCurrentMtSlot()
+    return self.ev_slots[self.cur_slot]
+end
+
 function Input:setMtSlot(slot, key, val)
     self.ev_slots[slot][key] = val
 end
 
 function Input:setCurrentMtSlot(key, val)
-    self:setMtSlot(self.cur_slot, key, val)
+    self.ev_slots[self.cur_slot][key] = val
 end
 
 -- Same as above, but ensures the current slot actually has a live ref first
 function Input:setCurrentMtSlotChecked(key, val)
-    if #self.MTSlots == 0 then
+    if not self.active_slots[self.cur_slot] then
         self:addSlot(self.cur_slot)
     end
 
-    self:setMtSlot(self.cur_slot, key, val)
-end
-
-function Input:getMtSlot(slot)
-    return self.ev_slots[slot]
-end
-
-function Input:getCurrentMtSlot()
-    return self:getMtSlot(self.cur_slot)
+    self.ev_slots[self.cur_slot][key] = val
 end
 
 function Input:getCurrentMtSlotData(key)
@@ -1102,22 +1220,13 @@ function Input:addSlot(value)
     self.cur_slot = value
 end
 
-function Input:addSlotIfChanged(value)
-    if self.cur_slot ~= value then
-        -- We've already seen that slot in this frame, don't insert a duplicate reference!
-        if self.active_slots[value] then
-            self.cur_slot = value
-        else
-            self:addSlot(value)
-        end
-    end
-end
-
 function Input:setupSlotData(value)
-    if #self.MTSlots == 0 then
+    if not self.active_slots[value] then
         self:addSlot(value)
     else
-        self:addSlotIfChanged(value)
+        -- We've already seen that slot in this frame, don't insert a duplicate reference!
+        -- NOTE: May already be set to the correct value if the driver repeats ABS_MT_SLOT (e.g., our android/PB translation layers; or ABS_MT_TRACKING_ID for snow_protocol).
+        self.cur_slot = value
     end
 end
 

frontend/device/kindle/device.lua

@@ -10,6 +10,7 @@ local C = ffi.C
 require("ffi/linux_fb_h")
 require("ffi/linux_input_h")
 require("ffi/posix_h")
+require("ffi/fbink_input_h")
 
 local function yes() return true end
 local function no() return false end  -- luacheck: ignore
@@ -159,9 +160,11 @@ local Kindle = Generic:extend{
     isNightModeChallenged = no,
     -- NOTE: While this ought to behave on Zelda/Rex, turns out, nope, it really doesn't work on *any* of 'em :/ (c.f., ko#5884).
     canHWDither = no,
+    -- Device has an Ambient Light Sensor
+    hasLightSensor = no,
     -- The time the device went into suspend
     suspend_time = 0,
-    framework_lipc_handle = frameworkStopped()
+    framework_lipc_handle = frameworkStopped(),
 }
 
 function Kindle:initNetworkManager(NetworkMgr)
@@ -203,6 +206,58 @@ function Kindle:supportsScreensaver()
     end
 end
 
+function Kindle:openInputDevices()
+    -- Auto-detect input devices (via FBInk's fbink_input_scan)
+    local ok, FBInkInput = pcall(ffi.load, "fbink_input")
+    if not ok then
+        -- NOP fallback for the testsuite...
+        FBInkInput = { fbink_input_scan = function() end }
+    end
+    local dev_count = ffi.new("size_t[1]")
+    -- We care about: the touchscreen, a properly scaled stylus, pagination buttons, a home button and a fiveway.
+    local match_mask = bit.bor(C.INPUT_TOUCHSCREEN, C.INPUT_SCALED_TABLET, C.INPUT_PAGINATION_BUTTONS, C.INPUT_HOME_BUTTON, C.INPUT_DPAD)
+    local devices = FBInkInput.fbink_input_scan(match_mask, 0, 0, dev_count)
+    if devices ~= nil then
+        for i = 0, tonumber(dev_count[0]) - 1 do
+            local dev = devices[i]
+            if dev.matched then
+                self.input.fdopen(tonumber(dev.fd), ffi.string(dev.path), ffi.string(dev.name))
+            end
+        end
+        C.free(devices)
+    else
+        -- Auto-detection failed, warn and fall back to defaults
+        logger.warn("We failed to auto-detect the proper input devices, input handling may be inconsistent!")
+        if self.touch_dev then
+            -- We've got a preferred path specified for the touch panel
+            self.input.open(self.touch_dev)
+        else
+            -- That generally works out well enough on legacy devices...
+            self.input.open("/dev/input/event0")
+            self.input.open("/dev/input/event1")
+        end
+    end
+
+    -- Getting the device where rotation events end up without catching a bunch of false-positives is... trickier,
+    -- thanks to the inane event code being used...
+    if self:hasGSensor() then
+        -- i.e., we want something that reports EV_ABS:ABS_PRESSURE that isn't *also* a pen (because those are pretty much guaranteed to report pressure...).
+        --       And let's add that isn't also a touchscreen to the mix, because while not true at time of writing, that's an event touchscreens sure can support...
+        devices = FBInkInput.fbink_input_scan(C.INPUT_ROTATION_EVENT, bit.bor(C.INPUT_TABLET, C.INPUT_TOUCHSCREEN), C.NO_RECAP, dev_count)
+        if devices ~= nil then
+            for i = 0, tonumber(dev_count[0]) - 1 do
+                local dev = devices[i]
+                if dev.matched then
+                    self.input.fdopen(tonumber(dev.fd), ffi.string(dev.path), ffi.string(dev.name))
+                end
+            end
+            C.free(devices)
+        end
+    end
+
+    self.input.open("fake_events")
+end
+
 function Kindle:init()
     -- Check if the device supports deep sleep/quick boot
     if lfs.attributes("/sys/devices/platform/falconblk/uevent", "mode") == "file" then
@@ -231,6 +286,14 @@ function Kindle:init()
         self.canDeepSleep = false
     end
 
+    -- If the device-specific init hasn't done so already (devices without keys don't), instantiate Input.
+    if not self.input then
+        self.input = require("device/input"):new{ device = self }
+    end
+
+    -- Auto-detect & open input devices
+    self:openInputDevices()
+
     Generic.init(self)
 end
 
@@ -277,7 +340,18 @@ function Kindle:usbPlugIn()
     -- NOTE: If the device is put in USBNet mode before we even start, everything's peachy, though :).
 end
 
-function Kindle:intoScreenSaver()
+-- Hopefully, the event sources are fairly portable...
+-- c.f., https://github.com/koreader/koreader/pull/11174#issuecomment-1830064445
+-- NOTE: There's no distinction between real button presses and powerd_test -p or lipc-set-prop -i com.lab126.powerd powerButton 1
+local POWERD_EVENT_SOURCES = {
+    [1] = "BUTTON_WAKEUP",  -- outOfScreenSaver 1
+    [2] = "BUTTON_SUSPEND", -- goingToScreenSaver 2
+    [4] = "HALL_SUSPEND",   -- goingToScreenSaver 4
+    [6] = "HALL_WAKEUP",    -- outOfScreenSaver 6
+}
+
+function Kindle:intoScreenSaver(source)
+    logger.dbg("Kindle:intoScreenSaver via", POWERD_EVENT_SOURCES[source] or string.format("UNKNOWN_SUSPEND (%d)", source or -1))
     if not self.screen_saver_mode then
         if self:supportsScreensaver() then
             -- NOTE: Meaning this is not a SO device ;)
@@ -296,12 +370,13 @@ function Kindle:intoScreenSaver()
             -- so that we do the right thing on resume ;).
             self.screen_saver_mode = true
         end
-
-        self.powerd:beforeSuspend()
     end
+
+    self.powerd:beforeSuspend()
 end
 
-function Kindle:outofScreenSaver()
+function Kindle:outofScreenSaver(source)
+    logger.dbg("Kindle:outofScreenSaver via", POWERD_EVENT_SOURCES[source] or string.format("UNKNOWN_WAKEUP (%d)", source or -1))
     if self.screen_saver_mode then
         if self:supportsScreensaver() then
             local Screensaver = require("ui/screensaver")
@@ -349,23 +424,28 @@ function Kindle:outofScreenSaver()
             -- Flip the switch again
             self.screen_saver_mode = false
         end
-
-        self.powerd:afterResume()
     end
+
+    self.powerd:afterResume()
 end
 
+-- On stock, there's a distinction between OutOfSS (which *requests* closing the SS) and ExitingSS, which fires once they're *actually* closed...
+function Kindle:exitingScreenSaver() end
+
 function Kindle:usbPlugOut()
     -- NOTE: See usbPlugIn(), we don't have anything fancy to do here either.
 end
 
-function Kindle:wakeupFromSuspend()
-    self.powerd:wakeupFromSuspend()
+function Kindle:wakeupFromSuspend(ts)
+    logger.dbg("Kindle:wakeupFromSuspend", ts)
+    self.powerd:wakeupFromSuspend(ts)
     self.last_suspend_time = time.boottime_or_realtime_coarse() - self.suspend_time
     self.total_suspend_time = self.total_suspend_time + self.last_suspend_time
 end
 
-function Kindle:readyToSuspend()
-    self.powerd:readyToSuspend()
+function Kindle:readyToSuspend(delay)
+    logger.dbg("Kindle:readyToSuspend", delay)
+    self.powerd:readyToSuspend(delay)
     self.suspend_time = time.boottime_or_realtime_coarse()
 end
 
@@ -379,14 +459,26 @@ function Kindle:UIManagerReady(uimgr)
 end
 
 function Kindle:setEventHandlers(uimgr)
+    -- These custom fake events *will* pass an argument...
+    self.input.fake_event_args.IntoSS = {}
+    self.input.fake_event_args.OutOfSS = {}
+    self.input.fake_event_args.WakeupFromSuspend = {}
+    self.input.fake_event_args.ReadyToSuspend = {}
+
     UIManager.event_handlers.Suspend = function()
         self.powerd:toggleSuspend()
     end
-    UIManager.event_handlers.IntoSS = function()
-        self:intoScreenSaver()
+    UIManager.event_handlers.IntoSS = function(input_event)
+        -- Retrieve the argument set by Input:handleKeyBoardEv
+        local arg = table.remove(self.input.fake_event_args[input_event])
+        self:intoScreenSaver(arg)
     end
-    UIManager.event_handlers.OutOfSS = function()
-        self:outofScreenSaver()
+    UIManager.event_handlers.OutOfSS = function(input_event)
+        local arg = table.remove(self.input.fake_event_args[input_event])
+        self:outofScreenSaver(arg)
+    end
+    UIManager.event_handlers.ExitingSS = function()
+        self:exitingScreenSaver()
     end
     UIManager.event_handlers.Charging = function()
         self:_beforeCharging()
@@ -396,11 +488,13 @@ function Kindle:setEventHandlers(uimgr)
         self:usbPlugOut()
         self:_afterNotCharging()
     end
-    UIManager.event_handlers.WakeupFromSuspend = function()
-        self:wakeupFromSuspend()
+    UIManager.event_handlers.WakeupFromSuspend = function(input_event)
+        local arg = table.remove(self.input.fake_event_args[input_event])
+        self:wakeupFromSuspend(arg)
     end
-    UIManager.event_handlers.ReadyToSuspend = function()
-        self:readyToSuspend()
+    UIManager.event_handlers.ReadyToSuspend = function(input_event)
+        local arg = table.remove(self.input.fake_event_args[input_event])
+        self:readyToSuspend(arg)
     end
 end
 
@@ -424,7 +518,9 @@ local Kindle2 = Kindle:extend{
     isREAGL = no,
     hasKeyboard = yes,
     hasKeys = yes,
+    hasSymKey = yes,
     hasDPad = yes,
+    useDPadAsActionKeys = yes,
     canHWInvert = no,
     canModifyFBInfo = no,
     canUseCBB = no, -- 4bpp
@@ -437,7 +533,9 @@ local KindleDXG = Kindle:extend{
     isREAGL = no,
     hasKeyboard = yes,
     hasKeys = yes,
+    hasSymKey = yes,
     hasDPad = yes,
+    useDPadAsActionKeys = yes,
     canHWInvert = no,
     canModifyFBInfo = no,
     canUseCBB = no, -- 4bpp
@@ -450,7 +548,9 @@ local Kindle3 = Kindle:extend{
     isREAGL = no,
     hasKeyboard = yes,
     hasKeys = yes,
+    hasSymKey = yes,
     hasDPad = yes,
+    useDPadAsActionKeys = yes,
     canHWInvert = no,
     canModifyFBInfo = no,
     canUseCBB = no, -- 4bpp
@@ -461,7 +561,9 @@ local Kindle4 = Kindle:extend{
     model = "Kindle4",
     isREAGL = no,
     hasKeys = yes,
+    hasScreenKB = yes,
     hasDPad = yes,
+    useDPadAsActionKeys = yes,
     canHWInvert = no,
     canModifyFBInfo = no,
     -- NOTE: It could *technically* use the C BB, as it's running @ 8bpp, but it's expecting an inverted palette...
@@ -507,6 +609,7 @@ local KindleVoyage = Kindle:extend{
     isTouchDevice = yes,
     hasFrontlight = yes,
     canTurnFrontlightOff = no,
+    hasLightSensor = yes,
     hasKeys = yes,
     display_dpi = 300,
     touch_dev = "/dev/input/event1",
@@ -542,6 +645,7 @@ local KindleOasis2 = Kindle:extend{
     isZelda = yes,
     isTouchDevice = yes,
     hasFrontlight = yes,
+    hasLightSensor = yes,
     hasKeys = yes,
     hasGSensor = yes,
     display_dpi = 300,
@@ -555,6 +659,7 @@ local KindleOasis3 = Kindle:extend{
     hasFrontlight = yes,
     hasNaturalLight = yes,
     hasNaturalLightMixer = yes,
+    hasLightSensor = yes,
     hasKeys = yes,
     hasGSensor = yes,
     display_dpi = 300,
@@ -601,10 +706,10 @@ local KindlePaperWhite5 = Kindle:extend{
     --       and the widget is designed for the Kobo Aura One anyway, so, hahaha, nope.
     hasNaturalLightMixer = yes,
     display_dpi = 300,
-    touch_dev = "/dev/input/by-path/platform-1001e000.i2c-event",
     -- NOTE: While hardware dithering (via MDP) should be a thing, it doesn't appear to do anything right now :/.
     canHWDither = no,
     canDoSwipeAnimation = yes,
+    -- NOTE: Input device path is variable, see findInputDevices
 }
 
 local KindleBasic4 = Kindle:extend{
@@ -613,10 +718,27 @@ local KindleBasic4 = Kindle:extend{
     isTouchDevice = yes,
     hasFrontlight = yes,
     display_dpi = 300,
-    -- TBD
-    touch_dev = "/dev/input/by-path/platform-1001e000.i2c-event",
     canHWDither = no,
     canDoSwipeAnimation = yes,
+    -- NOTE: Like the PW5, input device path is variable, see findInputDevices
+}
+
+local KindleScribe = Kindle:extend{
+    model = "KindleScribe",
+    isMTK = yes,
+    isTouchDevice = yes,
+    hasFrontlight = yes,
+    hasNaturalLight = yes,
+    -- NOTE: We *can* technically control both LEDs independently,
+    --       but the mix is device-specific, we don't have access to the LUT for the mix powerd is using,
+    --       and the widget is designed for the Kobo Aura One anyway, so, hahaha, nope.
+    hasNaturalLightMixer = yes,
+    hasLightSensor = yes,
+    hasGSensor = yes,
+    display_dpi = 300,
+    touch_dev = "/dev/input/touch",
+    canHWDither = yes,
+    canDoSwipeAnimation = yes,
 }
 
 function Kindle2:init()
@@ -629,9 +751,6 @@ function Kindle2:init()
         device = self,
         event_map = require("device/kindle/event_map_keyboard"),
     }
-    self.input.open("/dev/input/event0")
-    self.input.open("/dev/input/event1")
-    self.input.open("fake_events")
     Kindle.init(self)
 end
 
@@ -646,9 +765,6 @@ function KindleDXG:init()
         event_map = require("device/kindle/event_map_keyboard"),
     }
     self.keyboard_layout = require("device/kindle/keyboard_layout")
-    self.input.open("/dev/input/event0")
-    self.input.open("/dev/input/event1")
-    self.input.open("fake_events")
     Kindle.init(self)
 end
 
@@ -664,9 +780,6 @@ function Kindle3:init()
         event_map = require("device/kindle/event_map_kindle4"),
     }
     self.keyboard_layout = require("device/kindle/keyboard_layout")
-    self.input.open("/dev/input/event0")
-    self.input.open("/dev/input/event1")
-    self.input.open("fake_events")
     Kindle.init(self)
 end
 
@@ -681,9 +794,6 @@ function Kindle4:init()
         device = self,
         event_map = require("device/kindle/event_map_kindle4"),
     }
-    self.input.open("/dev/input/event0")
-    self.input.open("/dev/input/event1")
-    self.input.open("fake_events")
     Kindle.init(self)
 end
 
@@ -703,11 +813,6 @@ function KindleTouch:init()
     -- Kindle Touch needs event modification for proper coordinates
     self.input:registerEventAdjustHook(self.input.adjustTouchScale, {x=600/4095, y=800/4095})
 
-    -- event0 in KindleTouch is "WM8962 Beep Generator" (useless)
-    -- event1 in KindleTouch is "imx-yoshi Headset" (useless)
-    self.input.open("/dev/input/event2") -- Home button
-    self.input.open(self.touch_dev) -- touchscreen
-    self.input.open("fake_events")
     Kindle.init(self)
 end
 
@@ -721,9 +826,6 @@ function KindlePaperWhite:init()
     }
 
     Kindle.init(self)
-
-    self.input.open(self.touch_dev)
-    self.input.open("fake_events")
 end
 
 function KindlePaperWhite2:init()
@@ -737,9 +839,6 @@ function KindlePaperWhite2:init()
     }
 
     Kindle.init(self)
-
-    self.input.open(self.touch_dev)
-    self.input.open("fake_events")
 end
 
 function KindleBasic:init()
@@ -752,9 +851,6 @@ function KindleBasic:init()
     }
 
     Kindle.init(self)
-
-    self.input.open(self.touch_dev)
-    self.input.open("fake_events")
 end
 
 function KindleVoyage:init()
@@ -805,11 +901,7 @@ function KindleVoyage:init()
 
     Kindle.init(self)
 
-    self.input.open(self.touch_dev)
-    self.input.open("/dev/input/event2") -- WhisperTouch
-    self.input.open("fake_events")
-
-    -- reenable WhisperTouch keys when started without framework
+    -- Re-enable WhisperTouch keys when started without framework
     if self.framework_lipc_handle then
         self.framework_lipc_handle:set_int_property("com.lab126.deviced", "fsrkeypadEnable", 1)
         self.framework_lipc_handle:set_int_property("com.lab126.deviced", "fsrkeypadPrevEnable", 1)
@@ -828,9 +920,6 @@ function KindlePaperWhite3:init()
     }
 
     Kindle.init(self)
-
-    self.input.open(self.touch_dev)
-    self.input.open("fake_events")
 end
 
 -- HAL for gyro orientation switches (EV_ABS:ABS_PRESSURE (?!) w/ custom values to EV_MSC:MSC_GYRO w/ our own custom values)
@@ -874,6 +963,11 @@ local function OasisGyroTranslation(this, ev)
 end
 
 function KindleOasis:init()
+    -- temporarily wake up awesome
+    if os.getenv("AWESOME_STOPPED") == "yes" then
+        os.execute("killall -CONT awesome")
+    end
+
     self.screen = require("ffi/framebuffer_mxcfb"):new{device = self, debug = logger.dbg}
     self.powerd = require("device/kindle/powerd"):new{
         device = self,
@@ -893,12 +987,16 @@ function KindleOasis:init()
         }
     }
 
+    Kindle.init(self)
+
+    --- @note See comments in KindleOasis2:init() for details.
     local haslipc, lipc = pcall(require, "liblipclua")
     if haslipc and lipc then
         local lipc_handle = lipc.init("com.github.koreader.screen")
         if lipc_handle then
             local orientation_code = lipc_handle:get_string_property(
                 "com.lab126.winmgr", "accelerometer")
+            logger.dbg("orientation_code =", orientation_code)
             local rotation_mode = 0
             if orientation_code then
                 if orientation_code == "U" then
@@ -911,17 +1009,17 @@ function KindleOasis:init()
                     rotation_mode = self.screen.DEVICE_ROTATED_COUNTER_CLOCKWISE
                 end
             end
-
             if rotation_mode > 0 then
                 self.screen.native_rotation_mode = rotation_mode
-                self.screen.cur_rotation_mode = rotation_mode
             end
-
+            self.screen:setRotationMode(rotation_mode)
             lipc_handle:close()
         end
     end
-
-    Kindle.init(self)
+    -- put awesome back to sleep
+    if os.getenv("AWESOME_STOPPED") == "yes" then
+        os.execute("killall -STOP awesome")
+    end
 
     self.input:registerEventAdjustHook(OasisGyroTranslation)
     self.input.handleMiscEv = function(this, ev)
@@ -929,26 +1027,13 @@ function KindleOasis:init()
             return this:handleGyroEv(ev)
         end
     end
-
-    self.input.open(self.touch_dev)
-    self.input.open("/dev/input/by-path/platform-gpiokey.0-event")
-
-    -- get rotate dev by EV=d
-    local std_out = io.popen("grep -e 'Handlers\\|EV=' /proc/bus/input/devices | grep -B1 'EV=d' | grep -o 'event[0-9]'", "r")
-    if std_out then
-        local rotation_dev = std_out:read("*line")
-        std_out:close()
-        if rotation_dev then
-            self.input.open("/dev/input/"..rotation_dev)
-        end
-    end
-
-    self.input.open("fake_events")
 end
 
 -- HAL for gyro orientation switches (EV_ABS:ABS_PRESSURE (?!) w/ custom values to EV_MSC:MSC_GYRO w/ our own custom values)
-local function ZeldaGyroTranslation(this, ev)
-    -- c.f., drivers/input/misc/accel/bma2x2.c
+local function KindleGyroTransform(this, ev)
+    -- See source code:
+    -- c.f., drivers/input/misc/accel/bma2x2.c for KOA2/KOA3
+    -- c.f., drivers/input/misc/kx132/kx132.h for KS
     local UPWARD_PORTRAIT_UP_INTERRUPT_HAPPENED     = 15
     local UPWARD_PORTRAIT_DOWN_INTERRUPT_HAPPENED   = 16
     local UPWARD_LANDSCAPE_LEFT_INTERRUPT_HAPPENED  = 17
@@ -980,6 +1065,11 @@ local function ZeldaGyroTranslation(this, ev)
 end
 
 function KindleOasis2:init()
+    -- temporarily wake up awesome
+    if os.getenv("AWESOME_STOPPED") == "yes" then
+        os.execute("killall -CONT awesome")
+    end
+
     self.screen = require("ffi/framebuffer_mxcfb"):new{device = self, debug = logger.dbg}
     self.powerd = require("device/kindle/powerd"):new{
         device = self,
@@ -999,11 +1089,16 @@ function KindleOasis2:init()
         }
     }
 
-    --- @fixme When starting KOReader with the device upside down ("D"), touch input is registered wrong
+    Kindle.init(self)
+
+    --- @note When starting KOReader with the device upside down ("D"), touch input is registered wrong
     --        (i.e., probably upside down).
     --        If it's started upright ("U"), everything's okay, and turning it upside down after that works just fine.
     --        See #2206 & #2209 for the original KOA implementation, which obviously doesn't quite cut it here...
     --        See also <https://www.mobileread.com/forums/showthread.php?t=298302&page=5>
+    --        See also #11159 for details about the solution (Kindle Scribe as an example)
+    --        In regular mode, awesome is woken up for a brief moment for lipc calls.
+    --        In no-framework mode, this works as is.
     -- NOTE: It'd take some effort to actually start KOReader while in a LANDSCAPE orientation,
     --       since they're only exposed inside the stock reader, and not the Home/KUAL Booklets.
     local haslipc, lipc = pcall(require, "liblipclua")
@@ -1012,6 +1107,7 @@ function KindleOasis2:init()
         if lipc_handle then
             local orientation_code = lipc_handle:get_string_property(
                 "com.lab126.winmgr", "accelerometer")
+            logger.dbg("orientation_code =", orientation_code)
             local rotation_mode = 0
             if orientation_code then
                 if orientation_code == "U" then
@@ -1024,42 +1120,32 @@ function KindleOasis2:init()
                     rotation_mode = self.screen.DEVICE_ROTATED_COUNTER_CLOCKWISE
                 end
             end
-
             if rotation_mode > 0 then
                 self.screen.native_rotation_mode = rotation_mode
-                self.screen.cur_rotation_mode = rotation_mode
             end
-
+            self.screen:setRotationMode(rotation_mode)
             lipc_handle:close()
         end
     end
+    -- put awesome back to sleep
+    if os.getenv("AWESOME_STOPPED") == "yes" then
+        os.execute("killall -STOP awesome")
+    end
 
-    Kindle.init(self)
-
-    self.input:registerEventAdjustHook(ZeldaGyroTranslation)
+    self.input:registerEventAdjustHook(KindleGyroTransform)
     self.input.handleMiscEv = function(this, ev)
         if ev.code == C.MSC_GYRO then
             return this:handleGyroEv(ev)
         end
     end
-
-    self.input.open(self.touch_dev)
-    self.input.open("/dev/input/by-path/platform-gpio-keys-event")
-
-    -- Get accelerometer device by looking for EV=d
-    local std_out = io.popen("grep -e 'Handlers\\|EV=' /proc/bus/input/devices | grep -B1 'EV=d' | grep -o 'event[0-9]\\{1,2\\}'", "r")
-    if std_out then
-        local rotation_dev = std_out:read("*line")
-        std_out:close()
-        if rotation_dev then
-            self.input.open("/dev/input/"..rotation_dev)
-        end
-    end
-
-    self.input.open("fake_events")
 end
 
 function KindleOasis3:init()
+    -- temporarily wake up awesome
+    if os.getenv("AWESOME_STOPPED") == "yes" then
+        os.execute("killall -CONT awesome")
+    end
+
     self.screen = require("ffi/framebuffer_mxcfb"):new{device = self, debug = logger.dbg}
     self.powerd = require("device/kindle/powerd"):new{
         device = self,
@@ -1080,14 +1166,16 @@ function KindleOasis3:init()
         }
     }
 
+    Kindle.init(self)
 
-    --- @fixme The same quirks as on the Oasis 2 apply ;).
+    --- @note The same quirks as on the Oasis 2 apply ;).
     local haslipc, lipc = pcall(require, "liblipclua")
     if haslipc and lipc then
         local lipc_handle = lipc.init("com.github.koreader.screen")
         if lipc_handle then
             local orientation_code = lipc_handle:get_string_property(
                 "com.lab126.winmgr", "accelerometer")
+            logger.dbg("orientation_code =", orientation_code)
             local rotation_mode = 0
             if orientation_code then
                 if orientation_code == "U" then
@@ -1100,39 +1188,24 @@ function KindleOasis3:init()
                     rotation_mode = self.screen.DEVICE_ROTATED_COUNTER_CLOCKWISE
                 end
             end
-
             if rotation_mode > 0 then
                 self.screen.native_rotation_mode = rotation_mode
-                self.screen.cur_rotation_mode = rotation_mode
             end
-
+            self.screen:setRotationMode(rotation_mode)
             lipc_handle:close()
         end
     end
+    -- put awesome back to sleep
+    if os.getenv("AWESOME_STOPPED") == "yes" then
+        os.execute("killall -STOP awesome")
+    end
 
-    Kindle.init(self)
-
-    self.input:registerEventAdjustHook(ZeldaGyroTranslation)
+    self.input:registerEventAdjustHook(KindleGyroTransform)
     self.input.handleMiscEv = function(this, ev)
         if ev.code == C.MSC_GYRO then
             return this:handleGyroEv(ev)
         end
     end
-
-    self.input.open(self.touch_dev)
-    self.input.open("/dev/input/by-path/platform-gpio-keys-event")
-
-    -- Get accelerometer device by looking for EV=d
-    local std_out = io.popen("grep -e 'Handlers\\|EV=' /proc/bus/input/devices | grep -B1 'EV=d' | grep -o 'event[0-9]\\{1,2\\}'", "r")
-    if std_out then
-        local rotation_dev = std_out:read("*line")
-        std_out:close()
-        if rotation_dev then
-            self.input.open("/dev/input/"..rotation_dev)
-        end
-    end
-
-    self.input.open("fake_events")
 end
 
 function KindleBasic2:init()
@@ -1146,9 +1219,6 @@ function KindleBasic2:init()
     }
 
     Kindle.init(self)
-
-    self.input.open(self.touch_dev)
-    self.input.open("fake_events")
 end
 
 function KindlePaperWhite4:init()
@@ -1163,19 +1233,6 @@ function KindlePaperWhite4:init()
     }
 
     Kindle.init(self)
-
-    -- So, look for a goodix TS input device (c.f., #5110)...
-    local std_out = io.popen("grep -e 'Handlers\\|Name=' /proc/bus/input/devices | grep -A1 'goodix-ts' | grep -o 'event[0-9]'", "r")
-    if std_out then
-        local goodix_dev = std_out:read("*line")
-        std_out:close()
-        if goodix_dev then
-            self.touch_dev = "/dev/input/" .. goodix_dev
-        end
-    end
-
-    self.input.open(self.touch_dev)
-    self.input.open("fake_events")
 end
 
 function KindleBasic3:init()
@@ -1195,8 +1252,6 @@ function KindleBasic3:init()
     -- so we have to rely on contact lift detection via BTN_TOUCH:0,
     -- c.f., https://github.com/koreader/koreader/issues/5070
     self.input.snow_protocol = true
-    self.input.open(self.touch_dev)
-    self.input.open("fake_events")
 end
 
 function KindlePaperWhite5:init()
@@ -1215,14 +1270,10 @@ function KindlePaperWhite5:init()
     self.screen:_MTK_ToggleFastMode(true)
 
     Kindle.init(self)
-
-    self.input.open(self.touch_dev)
-    self.input.open("fake_events")
 end
 
 function KindleBasic4:init()
     self.screen = require("ffi/framebuffer_mxcfb"):new{device = self, debug = logger.dbg}
-    -- TBD, assume PW5 for now
     self.powerd = require("device/kindle/powerd"):new{
         device = self,
         fl_intensity_file = "/sys/class/backlight/fp9966-bl1/brightness",
@@ -1236,9 +1287,68 @@ function KindleBasic4:init()
     self.screen:_MTK_ToggleFastMode(true)
 
     Kindle.init(self)
+end
 
-    self.input.open(self.touch_dev)
-    self.input.open("fake_events")
+function KindleScribe:init()
+    -- temporarily wake up awesome
+    if os.getenv("AWESOME_STOPPED") == "yes" then
+        os.execute("killall -CONT awesome")
+    end
+
+    self.screen = require("ffi/framebuffer_mxcfb"):new{device = self, debug = logger.dbg}
+    self.powerd = require("device/kindle/powerd"):new{
+        device = self,
+        fl_intensity_file = "/sys/class/backlight/fp9966-bl1/brightness",
+        warmth_intensity_file = "/sys/class/backlight/fp9966-bl0/brightness",
+        batt_capacity_file = "/sys/class/power_supply/bd71827_bat/capacity",
+        is_charging_file = "/sys/class/power_supply/bd71827_bat/charging",
+        batt_status_file = "/sys/class/power_supply/bd71827_bat/status",
+        hall_file = "/sys/devices/platform/eink_hall/hall_enable",
+    }
+
+    -- Enable the so-called "fast" mode, so as to prevent the driver from silently promoting refreshes to REAGL.
+    self.screen:_MTK_ToggleFastMode(true)
+
+    Kindle.init(self)
+
+    --- @note The same quirks as on the Oasis 2 and 3 apply ;).
+    local haslipc, lipc = pcall(require, "liblipclua")
+    if haslipc and lipc then
+        local lipc_handle = lipc.init("com.github.koreader.screen")
+        if lipc_handle then
+            local orientation_code = lipc_handle:get_string_property(
+                "com.lab126.winmgr", "accelerometer")
+            logger.dbg("orientation_code =", orientation_code)
+            local rotation_mode = 0
+            if orientation_code then
+                if orientation_code == "U" or "L" then
+                    rotation_mode = self.screen.DEVICE_ROTATED_UPRIGHT
+                elseif orientation_code == "D" or "R" then
+                    rotation_mode = self.screen.DEVICE_ROTATED_UPSIDE_DOWN
+                end
+            end
+            if rotation_mode > 0 then
+                self.screen.native_rotation_mode = rotation_mode
+            end
+            self.screen:setRotationMode(rotation_mode)
+            lipc_handle:close()
+        end
+    end
+    -- put awesome back to sleep
+    if os.getenv("AWESOME_STOPPED") == "yes" then
+        os.execute("killall -STOP awesome")
+    end
+
+    -- Setup accelerometer rotation input
+    self.input:registerEventAdjustHook(KindleGyroTransform)
+    self.input.handleMiscEv = function(this, ev)
+        if ev.code == C.MSC_GYRO then
+            return this:handleGyroEv(ev)
+        end
+    end
+
+    -- Setup pen input
+    self.input.wacom_protocol = true
 end
 
 function KindleTouch:exit()
@@ -1248,6 +1358,9 @@ function KindleTouch:exit()
     end
 
     if self.framework_lipc_handle then
+        -- Fixes missing *stock Amazon UI* screensavers on exiting out of "no framework" started KOReader
+        -- module was unloaded in frameworkStopped() function but wasn't (re)loaded on KOReader exit
+        self.framework_lipc_handle:set_string_property("com.lab126.blanket", "load", "screensaver")
         self.framework_lipc_handle:close()
     end
 
@@ -1283,6 +1396,7 @@ KindleBasic3.exit = KindleTouch.exit
 KindleOasis3.exit = KindleTouch.exit
 KindlePaperWhite5.exit = KindleTouch.exit
 KindleBasic4.exit = KindleTouch.exit
+KindleScribe.exit = KindleTouch.exit
 
 function Kindle3:exit()
     -- send double menu key press events to trigger screen refresh
@@ -1338,6 +1452,7 @@ local kt4_set = Set { "10L", "0WF", "0WG", "0WH", "0WJ", "0VB" }
 local koa3_set = Set { "11L", "0WQ", "0WP", "0WN", "0WM", "0WL" }
 local pw5_set = Set { "1LG", "1Q0", "1PX", "1VD", "219", "21A", "2BH", "2BJ", "2DK" }
 local kt5_set = Set { "22D", "25T", "23A", "2AQ", "2AP", "1XH", "22C" }
+local ks_set = Set { "27J", "2BL", "263", "227", "2BM", "23L", "23M", "270" }
 
 if kindle_sn_lead == "B" or kindle_sn_lead == "9" then
     local kindle_devcode = string.sub(kindle_sn, 3, 4)
@@ -1384,6 +1499,8 @@ else
         return KindlePaperWhite5
     elseif kt5_set[kindle_devcode_v2] then
         return KindleBasic4
+    elseif ks_set[kindle_devcode_v2] then
+        return KindleScribe
     end
 end
 

frontend/device/kindle/powerd.lua

@@ -29,17 +29,22 @@ end
 
 -- If we start with the light off (fl_intensity is fl_min), ensure a toggle will set it to the lowest "on" step,
 -- and that we update fl_intensity (by using setIntensity and not setIntensityHW).
-function KindlePowerD:turnOnFrontlightHW()
+function KindlePowerD:turnOnFrontlightHW(done_callback)
     self:setIntensity(self.fl_intensity == self.fl_min and self.fl_min + 1 or self.fl_intensity)
+
+    return false
 end
 -- Which means we need to get rid of the insane fl_intensity == fl_min shortcut in turnOnFrontlight, too...
 -- That dates back to #2941, and I have no idea what it's supposed to help with.
-function BasePowerD:turnOnFrontlight()
+function KindlePowerD:turnOnFrontlight(done_callback)
     if not self.device:hasFrontlight() then return end
     if self:isFrontlightOn() then return false end
-    self:turnOnFrontlightHW()
+    local cb_handled = self:turnOnFrontlightHW(done_callback)
     self.is_fl_on = true
     self:stateChanged()
+    if not cb_handled and done_callback then
+        done_callback()
+    end
     return true
 end
 
@@ -79,6 +84,14 @@ function KindlePowerD:frontlightIntensityHW()
     end
 end
 
+-- Make sure isFrontlightOn reflects the actual HW state,
+-- as self.fl_intensity is kept as-is when toggling the light off,
+-- in order to be able to toggle it back on at the right intensity.
+function KindlePowerD:isFrontlightOnHW()
+    local hw_intensity = self:frontlightIntensityHW()
+    return hw_intensity > self.fl_min
+end
+
 function KindlePowerD:setIntensityHW(intensity)
     -- Handle the synthetic step switcheroo on ! canTurnFrontlightOff devices...
     local turn_it_off = false
@@ -108,6 +121,9 @@ function KindlePowerD:setIntensityHW(intensity)
             ffiUtil.writeToSysfs(intensity, self.warmth_intensity_file)
         end
     end
+
+    -- The state might have changed, make sure we don't break isFrontlightOn
+    self:_decideFrontlightState()
 end
 
 function KindlePowerD:frontlightWarmthHW()
@@ -182,22 +198,6 @@ function KindlePowerD:_readFLIntensity()
     return self:read_int_file(self.fl_intensity_file)
 end
 
-function KindlePowerD:afterResume()
-    if not self.device:hasFrontlight() then
-        return
-    end
-    if self:isFrontlightOn() then
-        -- The Kindle framework should turn the front light back on automatically.
-        -- The following statement ensures consistency of intensity, but should basically always be redundant,
-        -- since we set intensity via lipc and not sysfs ;).
-        -- NOTE: This is race-y, and we want to *lose* the race, hence the use of the scheduler (c.f., #4392)
-        UIManager:tickAfterNext(function() self:turnOnFrontlightHW() end)
-    else
-        -- But in the off case, we *do* use sysfs, so this one actually matters.
-        UIManager:tickAfterNext(function() self:turnOffFrontlightHW() end)
-    end
-end
-
 function KindlePowerD:toggleSuspend()
     if self.lipc_handle then
         self.lipc_handle:set_int_property("com.lab126.powerd", "powerButton", 1)
@@ -243,8 +243,7 @@ function KindlePowerD:initWakeupMgr()
     if not self.device:supportsScreensaver() then return end
     if self.lipc_handle == nil then return end
 
-    function KindlePowerD:wakeupFromSuspend()
-        logger.dbg("Kindle wakeupFromSuspend")
+    function KindlePowerD:wakeupFromSuspend(ts)
         -- Give the device a few seconds to settle.
         -- This filters out user input resumes -> device will resume to active
         -- Also the Kindle stays in Ready to suspend for 10 seconds
@@ -252,8 +251,7 @@ function KindlePowerD:initWakeupMgr()
         UIManager:scheduleIn(15, self.checkUnexpectedWakeup, self)
     end
 
-    function KindlePowerD:readyToSuspend()
-        logger.dbg("Kindle readyToSuspend")
+    function KindlePowerD:readyToSuspend(delay)
         if self.device.wakeup_mgr:isWakeupAlarmScheduled() then
             local now = os.time()
             local alarm = self.device.wakeup_mgr:getWakeupAlarmEpoch()
@@ -293,6 +291,20 @@ function KindlePowerD:afterResume()
 
     -- Restore user input and emit the Resume event.
     self.device:_afterResume()
+
+    if not self.device:hasFrontlight() then
+        return
+    end
+    if self:isFrontlightOn() then
+        -- The Kindle framework should turn the front light back on automatically.
+        -- The following statement ensures consistency of intensity, but should basically always be redundant,
+        -- since we set intensity via lipc and not sysfs ;).
+        -- NOTE: This is race-y, and we want to *lose* the race, hence the use of the scheduler (c.f., #4392)
+        UIManager:tickAfterNext(function() self:turnOnFrontlightHW() end)
+    else
+        -- But in the off case, we *do* use sysfs, so this one actually matters.
+        UIManager:tickAfterNext(function() self:turnOffFrontlightHW() end)
+    end
 end
 
 function KindlePowerD:UIManagerReadyHW(uimgr)

frontend/device/kobo/device.lua

@@ -2,10 +2,10 @@ local Generic = require("device/generic/device")
 local Geom = require("ui/geometry")
 local UIManager
 local WakeupMgr = require("device/wakeupmgr")
-local time = require("ui/time")
 local ffiUtil = require("ffi/util")
 local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
+local time = require("ui/time")
 local util = require("util")
 local _ = require("gettext")
 
@@ -15,6 +15,7 @@ local C = ffi.C
 require("ffi/linux_fb_h")
 require("ffi/linux_input_h")
 require("ffi/posix_h")
+require("ffi/fbink_input_h")
 
 local function yes() return true end
 local function no() return false end
@@ -31,7 +32,7 @@ local function koboEnableWifi(toggle)
 end
 
 -- checks if standby is available on the device
-local function checkStandby()
+local function checkStandby(target_state)
     logger.dbg("Kobo: checking if standby is possible ...")
     local f = io.open("/sys/power/state")
     if not f then
@@ -40,11 +41,11 @@ local function checkStandby()
     local mode = f:read()
     f:close()
     logger.dbg("Kobo: available power states:", mode)
-    if mode and mode:find("standby") then
-        logger.dbg("Kobo: standby state is supported")
+    if mode and mode:find(target_state) then
+        logger.dbg("Kobo: target standby state '" .. target_state .. "' is supported")
         return yes
     end
-    logger.dbg("Kobo: standby state is unsupported")
+    logger.dbg("Kobo: target standby state '" .. target_state .. "' is unsupported")
     return no
 end
 
@@ -140,18 +141,19 @@ local Kobo = Generic:extend{
     battery_sysfs = "/sys/class/power_supply/mc13892_bat",
     -- Stable path to the NTX input device
     ntx_dev = "/dev/input/event0",
+    ntx_fd = nil,
     -- Stable path to the Touch input device
     touch_dev = "/dev/input/event1",
-    -- Stable path to the Power Button input device
-    power_dev = nil,
     -- Event code to use to detect contact pressure
     pressure_event = nil,
     -- Device features multiple CPU cores
     isSMP = no,
     -- Device supports "eclipse" waveform modes (i.e., optimized for nightmode).
     hasEclipseWfm = no,
-    -- Device ships with various hardware revisions under the same device code, requirign automatic hardware detection...
+    -- Device ships with various hardware revisions under the same device code, requiring automatic hardware detection (PMIC & FL)...
     automagic_sysfs = false,
+    -- The standard "standby" power state
+    standby_state = "standby",
 
     unexpected_wakeup_count = 0,
 }
@@ -415,8 +417,6 @@ local KoboEuropa = Kobo:extend{
     display_dpi = 227,
     boot_rota = C.FB_ROTATE_CCW,
     battery_sysfs = "/sys/class/power_supply/battery",
-    ntx_dev = "/dev/input/by-path/platform-ntx_event0-event",
-    touch_dev = "/dev/input/by-path/platform-0-0010-event",
     isSMP = yes,
 }
 
@@ -446,8 +446,6 @@ local KoboCadmus = Kobo:extend{
     battery_sysfs = "/sys/class/power_supply/battery",
     hasAuxBattery = yes,
     aux_battery_sysfs = "/sys/class/misc/cilix",
-    ntx_dev = "/dev/input/by-path/platform-ntx_event0-event",
-    touch_dev = "/dev/input/by-path/platform-0-0010-event",
     isSMP = yes,
     -- Much like the Libra 2, there are at least two different HW revisions, with different PMICs...
     automagic_sysfs = true,
@@ -501,7 +499,6 @@ local KoboGoldfinch = Kobo:extend{
         nl_inverted = true,
     },
     battery_sysfs = "/sys/class/power_supply/battery",
-    power_dev = "/dev/input/by-path/platform-bd71828-pwrkey-event",
     -- Board is eerily similar to the Libra 2, so, it inherits the same quirks...
     -- c.f., https://github.com/koreader/koreader/issues/9552#issuecomment-1293000313
     hasReliableMxcWaitFor = no,
@@ -528,12 +525,77 @@ local KoboCondor = Kobo:extend{
         nl_inverted = true,
     },
     battery_sysfs = "/sys/class/power_supply/bd71827_bat",
-    touch_dev = "/dev/input/by-path/platform-2-0010-event",
-    ntx_dev = "/dev/input/by-path/platform-ntx_event0-event",
-    power_dev = "/dev/input/by-path/platform-bd71828-pwrkey.6.auto-event",
     isSMP = yes,
 }
 
+-- Kobo Libra Colour:
+local KoboMonza = Kobo:extend{
+    model = "Kobo_monza",
+    isMTK = yes,
+    hasEclipseWfm = yes,
+    canToggleChargingLED = yes,
+    hasFrontlight = yes,
+    hasKeys = yes,
+    hasGSensor = yes,
+    display_dpi = 300,
+    pressure_event = C.ABS_MT_PRESSURE,
+    touch_mirrored_x = false,
+    touch_mirrored_y = true,
+    hasNaturalLight = yes,
+    frontlight_settings = {
+        frontlight_white = "/sys/class/backlight/mxc_msp430.0/brightness",
+        frontlight_mixer = "/sys/class/backlight/lm3630a_led/color",
+        nl_min = 0,
+        nl_max = 10,
+        nl_inverted = true,
+    },
+    battery_sysfs = "/sys/class/power_supply/bd71827_bat",
+    isSMP = yes,
+    hasColorScreen = yes,
+}
+
+-- Kobo Clara B/W:
+local KoboSpaBW = Kobo:extend{
+    model = "Kobo_spaBW",
+    isMTK = yes,
+    hasEclipseWfm = yes,
+    canToggleChargingLED = yes,
+    hasFrontlight = yes,
+    touch_snow_protocol = true,
+    display_dpi = 300,
+    hasNaturalLight = yes,
+    frontlight_settings = {
+        frontlight_white = "/sys/class/backlight/mxc_msp430.0/brightness",
+        frontlight_mixer = "/sys/class/backlight/lm3630a_led/color",
+        nl_min = 0,
+        nl_max = 10,
+        nl_inverted = true,
+    },
+    battery_sysfs = "/sys/class/power_supply/bd71827_bat",
+}
+
+-- Kobo Clara Colour:
+local KoboSpaColour = Kobo:extend{
+    model = "Kobo_spaColour",
+    isMTK = yes,
+    hasEclipseWfm = yes,
+    canToggleChargingLED = yes,
+    hasFrontlight = yes,
+    touch_snow_protocol = true,
+    display_dpi = 300,
+    hasNaturalLight = yes,
+    frontlight_settings = {
+        frontlight_white = "/sys/class/backlight/mxc_msp430.0/brightness",
+        frontlight_mixer = "/sys/class/backlight/lm3630a_led/color",
+        nl_min = 0,
+        nl_max = 10,
+        nl_inverted = true,
+    },
+    battery_sysfs = "/sys/class/power_supply/bd71827_bat",
+    isSMP = yes,
+    hasColorScreen = yes,
+}
+
 function Kobo:setupChargingLED()
     if G_reader_settings:nilOrTrue("enable_charging_led") then
         if self:hasAuxBattery() and self.powerd:isAuxBatteryConnected() then
@@ -553,7 +615,7 @@ function Kobo:getKeyRepeat()
     self.key_repeat = ffi.new("unsigned int[?]", C.REP_CNT)
     if C.ioctl(self.ntx_fd, C.EVIOCGREP, self.key_repeat) < 0 then
         local err = ffi.errno()
-        logger.warn("Device:getKeyRepeat: EVIOCGREP ioctl failed:", ffi.string(C.strerror(err)))
+        logger.warn("Device:getKeyRepeat: EVIOCGREP ioctl on fd", self.ntx_fd, "failed:", ffi.string(C.strerror(err)))
         return false
     else
         logger.dbg("Key repeat is set up to repeat every", self.key_repeat[C.REP_PERIOD], "ms after a delay of", self.key_repeat[C.REP_DELAY], "ms")
@@ -567,14 +629,14 @@ function Kobo:disableKeyRepeat()
     local key_repeat = ffi.new("unsigned int[?]", C.REP_CNT)
     if C.ioctl(self.ntx_fd, C.EVIOCSREP, key_repeat) < 0 then
         local err = ffi.errno()
-        logger.warn("Device:disableKeyRepeat: EVIOCSREP ioctl failed:", ffi.string(C.strerror(err)))
+        logger.warn("Device:disableKeyRepeat: EVIOCSREP ioctl on fd", self.ntx_fd, "failed:", ffi.string(C.strerror(err)))
     end
 end
 
 function Kobo:restoreKeyRepeat()
     if C.ioctl(self.ntx_fd, C.EVIOCSREP, self.key_repeat) < 0 then
         local err = ffi.errno()
-        logger.warn("Device:restoreKeyRepeat: EVIOCSREP ioctl failed:", ffi.string(C.strerror(err)))
+        logger.warn("Device:restoreKeyRepeat: EVIOCSREP ioctl on fd", self.ntx_fd, "failed:", ffi.string(C.strerror(err)))
     end
 end
 
@@ -591,7 +653,7 @@ function Kobo:toggleKeyRepeat(toggle)
         -- Check the current (kernel) state to know what to do
         if C.ioctl(self.ntx_fd, C.EVIOCGREP, key_repeat) < 0 then
             local err = ffi.errno()
-            logger.warn("Device:toggleKeyRepeat: EVIOCGREP ioctl failed:", ffi.string(C.strerror(err)))
+            logger.warn("Device:toggleKeyRepeat: EVIOCGREP ioctl on fd", self.ntx_fd, "failed:", ffi.string(C.strerror(err)))
             return false
         else
             if key_repeat[C.REP_DELAY] == 0 and key_repeat[C.REP_PERIOD] == 0 then
@@ -604,7 +666,7 @@ function Kobo:toggleKeyRepeat(toggle)
 
     if C.ioctl(self.ntx_fd, C.EVIOCSREP, key_repeat) < 0 then
         local err = ffi.errno()
-        logger.warn("Device:toggleKeyRepeat: EVIOCSREP ioctl failed:", ffi.string(C.strerror(err)))
+        logger.warn("Device:toggleKeyRepeat: EVIOCSREP ioctl on fd", self.ntx_fd, "failed:", ffi.string(C.strerror(err)))
         return false
     end
 
@@ -637,8 +699,9 @@ function Kobo:init()
             debug = logger.dbg,
             is_always_portrait = self.isAlwaysPortrait(),
             mxcfb_bypass_wait_for = mxcfb_bypass_wait_for,
+            no_cfa_post_processing = G_reader_settings:isTrue("no_cfa_post_processing"),
         }
-        if self.screen.fb_bpp == 32 then
+        if self.screen.fb_bpp == 32 and not self:hasColorScreen() then
             -- Ensure we decode images properly, as our framebuffer is BGRA...
             logger.info("Enabling Kobo @ 32bpp BGR tweaks")
             self.hasBGRFrameBuffer = yes
@@ -647,10 +710,6 @@ function Kobo:init()
 
     -- So far, MTK kernels do not export a per-request inversion flag
     if self:isMTK() then
-        --- @fixme: Apparently, that doesn't quite work, at least on the Elipsa 2E, so, disable HW inversion...
-        ---         c.f., https://github.com/koreader/koreader/pull/10719#issuecomment-1693425726
-        self.canHWInvert = no
-        --[[
         -- Instead, there's a global flag that we can *set* (but not *get*) via a procfs knob...
         -- Overload the HWNightMode stuff to implement that properly, like we do on Kindle
         function self.screen:setHWNightmode(toggle)
@@ -665,7 +724,6 @@ function Kobo:init()
             -- (We want to disable this on exit, always, as it will never be used by Nickel, which does SW inversion).
             return self.hw_night_mode == true
         end
-        --]]
     end
 
     -- Automagic sysfs discovery
@@ -674,6 +732,8 @@ function Kobo:init()
         if util.pathExists("/sys/class/power_supply/battery") then
             -- Newer devices (circa sunxi)
             self.battery_sysfs = "/sys/class/power_supply/battery"
+        elseif util.fileExists("/sys/class/power_supply/bd71827_bat") then
+            self.battery_sysfs = "/sys/class/power_supply/bd71827_bat"
         else
             self.battery_sysfs = "/sys/class/power_supply/mc13892_bat"
         end
@@ -691,43 +751,6 @@ function Kobo:init()
                 self.frontlight_settings.frontlight_mixer = "/sys/class/backlight/tlc5947_bl/color"
             end
         end
-
-        -- Touch panel input
-        if util.fileExists("/dev/input/by-path/platform-2-0010-event") then
-            -- Elan (HWConfig TouchCtrl is ekth6) on i2c bus 2
-            self.touch_dev = "/dev/input/by-path/platform-2-0010-event"
-        elseif util.fileExists("/dev/input/by-path/platform-1-0010-event") then
-            -- Elan (HWConfig TouchCtrl is ekth6) on i2c bus 1
-            self.touch_dev = "/dev/input/by-path/platform-1-0010-event"
-        elseif util.fileExists("/dev/input/by-path/platform-0-0010-event") then
-            -- Elan (HWConfig TouchCtrl is ekth6) on i2c bus 0
-            self.touch_dev = "/dev/input/by-path/platform-0-0010-event"
-        else
-            self.touch_dev = "/dev/input/event1"
-        end
-
-        -- Physical buttons & synthetic NTX events
-        if util.fileExists("/dev/input/by-path/platform-gpio-keys-event") then
-            -- Libra 2 w/ a BD71828 PMIC
-            self.ntx_dev = "/dev/input/by-path/platform-gpio-keys-event"
-        elseif util.fileExists("/dev/input/by-path/platform-ntx_event0-event") then
-            -- MTK, sunxi & Mk. 7
-            self.ntx_dev = "/dev/input/by-path/platform-ntx_event0-event"
-        elseif util.fileExists("/dev/input/by-path/platform-mxckpd-event") then
-            -- circa Mk. 5 i.MX
-            self.ntx_dev = "/dev/input/by-path/platform-mxckpd-event"
-        else
-            self.ntx_dev = "/dev/input/event0"
-        end
-
-        -- Power button (this usually ends up in ntx_dev, except with some PMICs)
-        if util.fileExists("/dev/input/by-path/platform-bd71828-pwrkey-event") then
-            -- Libra 2 & Nia w/ a BD71828 PMIC
-            self.power_dev = "/dev/input/by-path/platform-bd71828-pwrkey-event"
-        elseif util.fileExists("/dev/input/by-path/platform-bd71828-pwrkey.4.auto-event") then
-            -- Sage w/ a BD71828 PMIC
-            self.power_dev = "/dev/input/by-path/platform-bd71828-pwrkey.4.auto-event"
-        end
     end
 
     -- NOTE: i.MX5 devices have a wonky RTC that doesn't like alarms set further away that UINT16_MAX seconds from now...
@@ -764,10 +787,15 @@ function Kobo:init()
         self.hasNaturalLightMixer = yes
     end
     -- Ditto
-    if self:isMk7() then
+    if self:isMk7() or self:isMTK() then
         self.canHWDither = yes
     end
 
+    -- Enable Kaleido waveform modes on supported devices
+    if self:hasColorScreen() and self:isMTK() then
+        self.hasKaleidoWfm = yes
+    end
+
     -- NOTE: Devices with an AW99703 frontlight PWM controller feature a hardware smooth ramp when setting the frontlight intensity.
     ---      A side-effect of this behavior is that if you queue a series of intensity changes ending at 0,
     ---      it won't ramp *at all*, jumping straight to zero instead.
@@ -776,6 +804,14 @@ function Kobo:init()
         self.frontlight_settings.ramp_off_delay = 0.5
     end
 
+    -- I don't know how this PWM controller behaves on earlier devices, but it's... not great here.
+    if self:hasNaturalLightMixer() and self:isMTK() and self.frontlight_settings.frontlight_mixer:find("lm3630a_led", 12, true) then
+        -- First, we need a delay between ioctls
+        self.frontlight_settings.ramp_delay =  0.025
+        -- Second, it *really* doesn't like being interleaved with screen refreshes
+        self.frontlight_settings.delay_ramp_start = true
+    end
+
     self.powerd = require("device/kobo/powerd"):new{
         device = self,
         battery_sysfs = self.battery_sysfs,
@@ -821,14 +857,41 @@ function Kobo:init()
     -- And then handle the extra shenanigans if necessary.
     self:initEventAdjustHooks()
 
-    -- Various HW Buttons, Switches & Synthetic NTX events
-    self.ntx_fd = self.input.open(self.ntx_dev)
-    -- Dedicated Power Button input device (if any)
-    if self.power_dev then
-        self.input.open(self.power_dev)
+    -- Auto-detect input devices (via FBInk's fbink_input_scan)
+    local ok, FBInkInput = pcall(ffi.load, "fbink_input")
+    if not ok then
+        -- NOP fallback for the testsuite...
+        FBInkInput = { fbink_input_scan = NOP }
+    end
+    local dev_count = ffi.new("size_t[1]")
+    -- We care about: the touchscreen, the stylus, the power button, the sleep cover, and pagination buttons
+    -- (and technically rotation events, but we'll get it with the device that provides the buttons on NTX).
+    -- We exclude keyboards to play nice with the ExternalKeyboard plugin, which will handle potential keyboards on its own.
+    local match_mask = bit.bor(C.INPUT_TOUCHSCREEN, C.INPUT_TABLET, C.INPUT_POWER_BUTTON, C.INPUT_SLEEP_COVER, C.INPUT_PAGINATION_BUTTONS)
+    local devices = FBInkInput.fbink_input_scan(match_mask, C.INPUT_KEYBOARD, 0, dev_count)
+    if devices ~= nil then
+        for i = 0, tonumber(dev_count[0]) - 1 do
+            local dev = devices[i]
+            if dev.matched then
+                -- We need to single out whichever device provides pagination buttons or sleep cover events, as we'll want to tweak key repeat there...
+                -- The first one will do, as it's extremely likely to be event0, and that's pretty fairly set in stone on NTX boards.
+                if (bit.band(dev.type, C.INPUT_PAGINATION_BUTTONS) ~= 0 or bit.band(dev.type, C.INPUT_SLEEP_COVER) ~= 0) and not self.ntx_fd then
+                    self.ntx_fd = self.input.fdopen(tonumber(dev.fd), ffi.string(dev.path), ffi.string(dev.name))
+                else
+                    self.input.fdopen(tonumber(dev.fd), ffi.string(dev.path), ffi.string(dev.name))
+                end
+            end
+        end
+        C.free(devices)
+    else
+        -- Auto-detection failed, warn and fall back to defaults
+        logger.warn("We failed to auto-detect the proper input devices, input handling may be inconsistent!")
+        -- Various HW Buttons, Switches & Synthetic NTX events
+        self.ntx_fd = self.input.open(self.ntx_dev)
+        -- Touch panel
+        self.input.open(self.touch_dev)
     end
-    -- Touch panel
-    self.input.open(self.touch_dev)
+
     -- NOTE: On devices with a gyro, there may be a dedicated input device outputting the raw accelerometer data
     --       (3-Axis Orientation/Motion Detection).
     --       We skip it because we don't need it (synthetic rotation change events are sent to the main ntx input device),
@@ -849,9 +912,12 @@ function Kobo:init()
     end
 
     -- Detect the NTX charging LED sysfs knob
-    if util.pathExists("/sys/class/leds/bd71828-green-led") then
-        -- Standard Linux LED class, wheee!
-        self.charging_led_sysfs_knob = "/sys/class/leds/bd71828-green-led"
+    if util.pathExists("/sys/class/leds/LED") then
+        self.charging_led_sysfs_knob = "/sys/class/leds/LED/brightness"
+    elseif util.pathExists("/sys/class/leds/GLED") then
+        self.charging_led_sysfs_knob = "/sys/class/leds/GLED/brightness"
+    elseif util.pathExists("/sys/class/leds/bd71828-green-led") then
+        self.charging_led_sysfs_knob = "/sys/class/leds/bd71828-green-led/brightness"
     elseif util.pathExists("/sys/devices/platform/ntx_led/lit") then
         self.ntx_lit_sysfs_knob = "/sys/devices/platform/ntx_led/lit"
     elseif util.pathExists("/sys/devices/platform/pmic_light.1/lit") then
@@ -881,8 +947,15 @@ function Kobo:init()
     -- Only enable a single core on startup
     self:enableCPUCores(1)
 
-    self.canStandby = checkStandby()
-    if self.canStandby() and (self:isMk7() or self:isSunxi() or self:isMTK())  then
+    -- On MTK, the "standby" power state is unavailable, and Nickel instead uses "mem" (and /sys/power/mem_sleep doesn't exist either)
+    if self:isMTK() then
+        self.standby_state = "mem"
+    end
+
+    self.canStandby = checkStandby(self.standby_state)
+    if self.canStandby() and (self:isMk7() or self:isSunxi()) then
+        -- NOTE: Do *NOT* enable this on MTK. What happens if you do can only be described as "shit hits the fan".
+        --       (Nickel doesn't).
         self.canPowerSaveWhileCharging = yes
     end
 
@@ -1181,7 +1254,13 @@ function Kobo:standby(max_duration)
     logger.dbg("Kobo standby: asking to enter standby . . .")
     local standby_time = time.boottime_or_realtime_coarse()
 
-    local ret = ffiUtil.writeToSysfs("standby", "/sys/power/state")
+    -- The odd Sunxi needs some time to settle before entering standby.
+    -- This will avoid the screen puzzling effect documented in
+    -- https://github.com/koreader/koreader/pull/10306#issue-1659242042 not only for
+    -- WiFi toggle, but (almost) everywhere.
+    ffiUtil.usleep(90000) -- sleep 0.09s (0.08s would also work)
+
+    local ret = ffiUtil.writeToSysfs(self.standby_state, "/sys/power/state")
 
     self.last_standby_time = time.boottime_or_realtime_coarse() - standby_time
     self.total_standby_time = self.total_standby_time + self.last_standby_time
@@ -1445,6 +1524,7 @@ function Kobo:_NTXChargingLEDToggle(toggle)
 end
 
 function Kobo:_LinuxChargingLEDToggle(toggle)
+    -- max_brightness usually says 255 for those, but 1 does the same (and matches Nickel's behavior)
     ffiUtil.writeToSysfs(toggle and "1" or "0", self.charging_led_sysfs_knob)
 end
 
@@ -1677,6 +1757,12 @@ elseif codename == "goldfinch" then
     return KoboGoldfinch
 elseif codename == "condor" then
     return KoboCondor
+elseif codename == "monza" or codename == "monzaTolino" then
+    return KoboMonza
+elseif codename == "spaBW" or codename == "spaTolinoBW" then
+    return KoboSpaBW
+elseif codename == "spaColour" or codename == "spaTolinoColour" then
+    return KoboSpaColour
 else
     error("unrecognized Kobo model ".. codename .. " with device id " .. product_id)
 end

frontend/device/kobo/powerd.lua

@@ -136,11 +136,14 @@ function KoboPowerD:init()
         self.device.frontlight_settings = self.device.frontlight_settings or {}
         -- Does this device require non-standard ramping behavior?
         self.device.frontlight_settings.ramp_off_delay = self.device.frontlight_settings.ramp_off_delay or 0.0
-        --- @note: Newer devices appear to block slightly longer on FL ioctls/sysfs, so we only really need a delay on older devices.
+        --- @note: Newer devices (or at least some PWM controllers) appear to block slightly longer on FL ioctls/sysfs,
+        ---        so we only really need a delay on older devices.
         self.device.frontlight_settings.ramp_delay = self.device.frontlight_settings.ramp_delay or (self.device:hasNaturalLight() and 0.0 or 0.025)
+        -- Some PWM controllers *really* don't like being interleaved between screen refreshes,
+        -- so we delay the *start* of the ramp on these.
+        self.device.frontlight_settings.delay_ramp_start = self.device.frontlight_settings.delay_ramp_start or false
 
-        -- If this device has natural light (currently only KA1 & Forma)
-        -- Use the SysFS interface, and ioctl otherwise.
+        -- If this device has natural light, use the sysfs interface, and ioctl otherwise.
         -- NOTE: On the Forma, nickel still appears to prefer using ntx_io to handle the FL,
         --       but it does use sysfs for the NL...
         if self.device:hasNaturalLight() then
@@ -311,7 +314,7 @@ function KoboPowerD:isChargedHW()
     return false
 end
 
-function KoboPowerD:_postponedSetIntensityHW(end_intensity, done_callback)
+function KoboPowerD:_endRampDown(end_intensity, done_callback)
     self:_setIntensityHW(end_intensity)
     self.fl_ramp_down_running = false
 
@@ -325,7 +328,8 @@ function KoboPowerD:_stopFrontlightRamp()
         -- Make sure we have no other ramp running.
         UIManager:unschedule(self.turnOffFrontlightRamp)
         UIManager:unschedule(self.turnOnFrontlightRamp)
-        UIManager:unschedule(self._postponedSetIntensityHW)
+        UIManager:unschedule(self._endRampDown)
+        UIManager:unschedule(self._endRampUp)
         self.fl_ramp_up_running = false
         self.fl_ramp_down_running = false
     end
@@ -343,7 +347,7 @@ function KoboPowerD:turnOffFrontlightRamp(curr_ramp_intensity, end_intensity, do
         UIManager:scheduleIn(self.device.frontlight_settings.ramp_delay, self.turnOffFrontlightRamp, self, curr_ramp_intensity, end_intensity, done_callback)
     else
         -- Some devices require delaying the final step, to prevent them from jumping straight to zero and messing up the ramp.
-        UIManager:scheduleIn(self.device.frontlight_settings.ramp_off_delay, self._postponedSetIntensityHW, self, end_intensity, done_callback)
+        UIManager:scheduleIn(self.device.frontlight_settings.ramp_off_delay, self._endRampDown, self, end_intensity, done_callback)
         -- no reschedule here, as we are done
     end
 end
@@ -357,8 +361,25 @@ function KoboPowerD:turnOffFrontlightHW(done_callback)
         -- We've got nothing to do if we're already ramping down
         if not self.fl_ramp_down_running then
             self:_stopFrontlightRamp()
-            self:turnOffFrontlightRamp(self.fl_intensity, self.fl_min, done_callback)
-            self.fl_ramp_down_running = true
+            -- NOTE: For devices with a ramp_off_delay, we only ramp if we start from > 2%,
+            --       otherwise you just see a single delayed step (1%) or two stuttery ones (2%) ;).
+            -- FWIW, modern devices with a different PWM controller (i.e., with no controller-specific ramp_off_delay workarounds)
+            -- deal with our 2% ramp without stuttering.
+            if self.device.frontlight_settings.ramp_off_delay > 0.0 and self.fl_intensity <= 2 then
+                UIManager:scheduleIn(self.device.frontlight_settings.ramp_delay, self._endRampDown, self, self.fl_min, done_callback)
+            else
+                -- NOTE: Similarly, some controllers *really* don't like to be interleaved with screen refreshes,
+                --       so we wait until the next UI frame for the refreshes to go through first...
+                if self.device.frontlight_settings.delay_ramp_start then
+                    UIManager:nextTick(function()
+                        self:turnOffFrontlightRamp(self.fl_intensity, self.fl_min, done_callback)
+                        self.fl_ramp_down_running = true
+                    end)
+                else
+                    self:turnOffFrontlightRamp(self.fl_intensity, self.fl_min, done_callback)
+                    self.fl_ramp_down_running = true
+                end
+            end
         end
     else
         -- If UIManager is not initialized yet, just turn it off immediately
@@ -369,6 +390,15 @@ function KoboPowerD:turnOffFrontlightHW(done_callback)
     return true
 end
 
+function KoboPowerD:_endRampUp(end_intensity, done_callback)
+    self:_setIntensityHW(end_intensity)
+    self.fl_ramp_up_running = false
+
+    if done_callback then
+        done_callback()
+    end
+end
+
 -- Similar functionality as `Kobo:turnOnFrontlightHW`, but the other way around ;).
 function KoboPowerD:turnOnFrontlightRamp(curr_ramp_intensity, end_intensity, done_callback)
     if curr_ramp_intensity == 0 then
@@ -381,12 +411,7 @@ function KoboPowerD:turnOnFrontlightRamp(curr_ramp_intensity, end_intensity, don
         self:_setIntensityHW(curr_ramp_intensity)
         UIManager:scheduleIn(self.device.frontlight_settings.ramp_delay, self.turnOnFrontlightRamp, self, curr_ramp_intensity, end_intensity, done_callback)
     else
-        self:_setIntensityHW(end_intensity)
-        self.fl_ramp_up_running = false
-
-        if done_callback then
-            done_callback()
-        end
+        UIManager:scheduleIn(self.device.frontlight_settings.ramp_delay, self._endRampUp, self, end_intensity, done_callback)
         -- no reschedule here, as we are done
     end
 end
@@ -406,8 +431,21 @@ function KoboPowerD:turnOnFrontlightHW(done_callback)
         -- We've got nothing to do if we're already ramping up
         if not self.fl_ramp_up_running then
             self:_stopFrontlightRamp()
-            self:turnOnFrontlightRamp(self.fl_min, self.fl_intensity, done_callback)
-            self.fl_ramp_up_running = true
+            if self.device.frontlight_settings.ramp_off_delay > 0.0 and self.fl_intensity <= 2 then
+                -- NOTE: Match the ramp down behavior on devices with a ramp_off_delay: jump straight to 1 or 2% intensity.
+                UIManager:scheduleIn(self.device.frontlight_settings.ramp_delay, self._endRampUp, self, self.fl_intensity, done_callback)
+            else
+                -- Same deal as in turnOffFrontlightHW
+                if self.device.frontlight_settings.delay_ramp_start then
+                    UIManager:nextTick(function()
+                        self:turnOnFrontlightRamp(self.fl_min, self.fl_intensity, done_callback)
+                        self.fl_ramp_up_running = true
+                    end)
+                else
+                    self:turnOnFrontlightRamp(self.fl_min, self.fl_intensity, done_callback)
+                    self.fl_ramp_up_running = true
+                end
+            end
         end
     else
         -- If UIManager is not initialized yet, just turn it on immediately

frontend/device/pocketbook/device.lua

@@ -1,4 +1,5 @@
 local Generic = require("device/generic/device") -- <= look at this file!
+local Geom = require("ui/geometry")
 local UIManager
 local logger = require("logger")
 local ffi = require("ffi")
@@ -55,9 +56,7 @@ local PocketBook = Generic:extend{
         -- Works same as input.event_map, but for raw input EV_KEY translation
         keymap = { [scan] = event },
     }]]
-    -- Runtime state: whether raw input is actually used
-    --- @fixme: Never actually set anywhere?
-    is_using_raw_input = nil,
+    -- We'll nil raw_input at runtime if it cannot be used.
 
     -- InkView may have started translating button codes based on rotation on newer devices...
     -- That historically wasn't the case, hence this defaulting to false.
@@ -241,7 +240,9 @@ function PocketBook:init()
     -- NOTE: This all happens in ffi/input_pocketbook.lua
 
     self._model_init()
-    if (not self.input.raw_input) or (not pcall(self.input.open, self.input, self.raw_input)) then
+    -- NOTE: This is the odd one out actually calling input.open as a *method*,
+    --       which the imp supports to get access to self.input.raw_input
+    if (not self.input.raw_input) or (not pcall(self.input.open, self.input)) then
         inkview.OpenScreen()
         -- Raw mode open failed (no permissions?), so we'll run the usual way.
         -- Disable touch coordinate translation as inkview will do that.
@@ -386,6 +387,14 @@ function PocketBook:initNetworkManager(NetworkMgr)
         return band(inkview.QueryNetwork(), C.NET_CONNECTED) ~= 0
     end
     NetworkMgr.isWifiOn = NetworkMgr.isConnected
+
+    function NetworkMgr:isOnline()
+        -- Fail early if we don't even have a default route, otherwise we're
+        -- unlikely to be online and canResolveHostnames would never succeed
+        -- again because PocketBook's glibc parses /etc/resolv.conf on first
+        -- use only. See https://sourceware.org/bugzilla/show_bug.cgi?id=984
+        return NetworkMgr:hasDefaultRoute() and NetworkMgr:canResolveHostnames()
+    end
 end
 
 function PocketBook:getSoftwareVersion()
@@ -419,6 +428,25 @@ function PocketBook:setEventHandlers(uimgr)
     end
 end
 
+local function getBrowser()
+    if util.pathExists("/usr/bin/browser.app") then
+        return true, "/usr/bin/browser.app"
+    elseif util.pathExists("/ebrmain/bin/browser.app") then
+        return true, "/ebrmain/bin/browser.app"
+    end
+    return false
+end
+
+function PocketBook:canOpenLink()
+    return inkview.MultitaskingSupported() and getBrowser()
+end
+
+function PocketBook:openLink(link)
+    local found, bin = getBrowser()
+    if not found or not link or type(link) ~= "string" then return end
+    inkview.OpenBook(bin, link, 0)
+end
+
 -- Pocketbook HW rotation modes start from landsape, CCW
 local function landscape_ccw() return {
     1, 0, 3, 2,         -- PORTRAIT, LANDSCAPE, PORTRAIT_180, LANDSCAPE_180
@@ -463,6 +491,7 @@ local PocketBook613 = PocketBook:extend{
     display_dpi = 167,
     isTouchDevice = no,
     hasWifiToggle = no,
+    hasSeamlessWifiToggle = no,
     hasFrontlight = no,
     hasDPad = yes,
     hasFewKeys = yes,
@@ -506,6 +535,12 @@ local PocketBook617 = PocketBook:extend{
     hasNaturalLight = yes,
 }
 
+-- PocketBook Basic Lux 4 (618)
+local PocketBook618 = PocketBook:extend{
+    model = "PBBLux4",
+    display_dpi = 212,
+}
+
 -- PocketBook Touch (622)
 local PocketBook622 = PocketBook:extend{
     model = "PBTouch",
@@ -555,6 +590,14 @@ local PocketBook628 = PocketBook:extend{
     hasNaturalLight = yes,
 }
 
+-- PocketBook Verse (629)
+local PocketBook629 = PocketBook:extend{
+    model = "PB629",
+    display_dpi = 212,
+    isAlwaysPortrait = yes,
+    hasNaturalLight = yes,
+}
+
 -- PocketBook Sense / Sense 2 (630)
 local PocketBook630 = PocketBook:extend{
     model = "PBSense",
@@ -582,7 +625,6 @@ local PocketBook632 = PocketBook:extend{
 local PocketBook633 = PocketBook:extend{
     model = "PBColor",
     display_dpi = 300,
-    color_saturation = 1.5,
     hasColorScreen = yes,
     canHWDither = yes, -- Adjust color saturation with inkview
     canUseCBB = no, -- 24bpp
@@ -590,6 +632,14 @@ local PocketBook633 = PocketBook:extend{
     usingForcedRotation = landscape_ccw,
 }
 
+-- PocketBook Verse Pro (634)
+local PocketBook634 = PocketBook:extend{
+    model = "PB634",
+    display_dpi = 300,
+    isAlwaysPortrait = yes,
+    hasNaturalLight = yes,
+}
+
 -- PocketBook Aqua (640)
 local PocketBook640 = PocketBook:extend{
     model = "PBAqua",
@@ -618,6 +668,24 @@ local PocketBook700 = PocketBook:extend{
     inkview_translates_buttons = true,
 }
 
+-- PocketBook Era Color (PB700K3)
+local PocketBook700K3 = PocketBook:extend{
+    model = "PBEraColor",
+    display_dpi = 300,
+    hasColorScreen = yes,
+    canHWDither = yes, -- Adjust color saturation with inkview
+    canUseCBB = no, -- 24bpp
+    isAlwaysPortrait = yes,
+    hasNaturalLight = yes,
+    -- c.f., https://github.com/koreader/koreader/issues/9556
+    inkview_translates_buttons = true,
+}
+
+function PocketBook700K3._fb_init(fb, finfo, vinfo)
+    -- Pocketbook Color Lux reports bits_per_pixel = 8, but actually uses an RGB24 framebuffer
+    vinfo.bits_per_pixel = 24
+end
+
 -- PocketBook InkPad 3 (740)
 local PocketBook740 = PocketBook:extend{
     model = "PBInkPad3",
@@ -648,7 +716,6 @@ local PocketBook740_2 = PocketBook:extend{
 local PocketBook741 = PocketBook:extend{
     model = "PBInkPadColor",
     display_dpi = 300,
-    color_saturation = 1.5,
     hasColorScreen = yes,
     canHWDither = yes, -- Adjust color saturation with inkview
     canUseCBB = no, -- 24bpp
@@ -665,7 +732,6 @@ end
 local PocketBook743C = PocketBook:extend{
     model = "PBInkPadColor2",
     display_dpi = 300,
-    color_saturation = 1.5,
     hasColorScreen = yes,
     canHWDither = yes, -- Adjust color saturation with inkview
     canUseCBB = no, -- 24bpp
@@ -679,6 +745,24 @@ function PocketBook743C._fb_init(fb, finfo, vinfo)
     vinfo.bits_per_pixel = 24
 end
 
+-- PocketBook InkPad Color 3 (743K3)
+local PocketBook743K3 = PocketBook:extend{
+    model = "PBInkPadColor3",
+    display_dpi = 300,
+    viewport = Geom:new{x=3, y=2, w=1395, h=1864},
+    hasColorScreen = yes,
+    canHWDither = yes, -- Adjust color saturation with inkview
+    canUseCBB = no, -- 24bpp
+    isAlwaysPortrait = yes,
+    usingForcedRotation = landscape_ccw,
+    hasNaturalLight = yes,
+}
+
+function PocketBook743K3._fb_init(fb, finfo, vinfo)
+    -- Pocketbook Color Lux reports bits_per_pixel = 8, but actually uses an RGB24 framebuffer
+    vinfo.bits_per_pixel = 24
+end
+
 -- PocketBook InkPad 4 (743G/743g)
 local PocketBook743G = PocketBook:extend{
     model = "PBInkPad4",
@@ -692,7 +776,6 @@ local PocketBook743G = PocketBook:extend{
 local PocketBookColorLux = PocketBook:extend{
     model = "PBColorLux",
     display_dpi = 125,
-    color_saturation = 1.5,
     hasColorScreen = yes,
     canHWDither = yes, -- Adjust color saturation with inkview
     canUseCBB = no, -- 24bpp
@@ -733,75 +816,91 @@ local PocketBook1040 = PocketBook:extend{
 
 logger.info('SoftwareVersion: ', PocketBook:getSoftwareVersion())
 
-local codename = PocketBook:getDeviceModel()
+local full_codename = PocketBook:getDeviceModel()
+
+-- Pocketbook codenames are all over the place:
+local codename = full_codename
+-- "PocketBook 615 (PB615)"
+codename = codename:match(" [(]([^()]+)[)]$") or codename
+-- "PocketBook 615"
+codename = codename:match("^PocketBook ([^ ].*)$") or codename
+-- "PB615"
+codename = codename:match("^PB(.+)$") or codename
 
-if codename == "PocketBook 515" then
+if codename == "515" then
     return PocketBook515
-elseif codename == "PB606" or codename == "PocketBook 606" then
+elseif codename == "606" then
     return PocketBook606
-elseif codename == "PocketBook 611" then
+elseif codename == "611" then
     return PocketBook611
-elseif codename == "PocketBook 613" then
+elseif codename == "613" then
     return PocketBook613
-elseif codename == "PocketBook 614" or codename == "PocketBook 614W" then
+elseif codename == "614" or codename == "614W" then
     return PocketBook614W
-elseif codename == "PB615" or codename == "PB615W" or
-    codename == "PocketBook 615" or codename == "PocketBook 615W" then
+elseif codename == "615" or codename == "615W" then
     return PocketBook615
-elseif codename == "PB616" or codename == "PB616W" or
-    codename == "PocketBook 616" or codename == "PocketBook 616W" then
+elseif codename == "616" or codename == "616W" then
     return PocketBook616
-elseif codename == "PB617" or codename == "PocketBook 617" then
+elseif codename == "617" then
     return PocketBook617
-elseif codename == "PocketBook 622" then
+elseif codename == "618" then
+    return PocketBook618
+elseif codename == "622" then
     return PocketBook622
-elseif codename == "PocketBook 623" then
+elseif codename == "623" then
     return PocketBook623
-elseif codename == "PocketBook 624" then
+elseif codename == "624" then
     return PocketBook624
-elseif codename == "PB625" then
+elseif codename == "625" then
     return PocketBook625
-elseif codename == "PB626" or codename == "PB626(2)-TL3" or
-    codename == "PocketBook 626" then
+elseif codename == "626" or codename == "626(2)-TL3" then
     return PocketBook626
-elseif codename == "PB627" then
+elseif codename == "627" then
     return PocketBook627
-elseif codename == "PB628" then
+elseif codename == "628" then
     return PocketBook628
-elseif codename == "PocketBook 630" then
+elseif codename == "629" then
+    return PocketBook629
+elseif codename == "630" then
     return PocketBook630
-elseif codename == "PB631" or codename == "PocketBook 631" then
+elseif codename == "631" then
     return PocketBook631
-elseif codename == "PB632" then
+elseif codename == "632" then
     return PocketBook632
-elseif codename == "PB633" then
+elseif codename == "633" then
     return PocketBook633
-elseif codename == "PB640" or codename == "PocketBook 640" then
+elseif codename == "634" then
+    return PocketBook634
+elseif codename == "640" then
     return PocketBook640
-elseif codename == "PB641" then
+elseif codename == "641" then
     return PocketBook641
-elseif codename == "PB650" or codename == "PocketBook 650" then
+elseif codename == "650" then
     return PocketBook650
-elseif codename == "PB700" or codename == "PocketBook 700" then
+elseif codename == "700" then
     return PocketBook700
-elseif codename == "PB740" then
+elseif codename == "700K3" then
+    return PocketBook700K3
+elseif codename == "740" then
     return PocketBook740
-elseif codename == "PB740-2" or codename == "PB740-3" then
+elseif codename == "740-2" or codename == "740-3" then
     return PocketBook740_2
-elseif codename == "PB741" then
+elseif codename == "741" then
     return PocketBook741
-elseif codename == "PB743C" then
+elseif codename == "743C" then
     return PocketBook743C
-elseif codename == "PB743G" or codename == "PB743g" or codename == "PocketBook 743G" or codename == "PocketBook 743g" then
+elseif codename == "743K3" then
+    return PocketBook743K3
+elseif codename == "743G" or codename == "743g" then
     return PocketBook743G
-elseif codename == "PocketBook 840" or codename == "Reader InkPad" then
+elseif codename == "840" or codename == "Reader InkPad" then
     return PocketBook840
-elseif codename == "PB970" then
+elseif codename == "970" then
     return PocketBook970
-elseif codename == "PB1040" then
+elseif codename == "1040" then
     return PocketBook1040
-elseif codename == "PocketBook Color Lux" then
+elseif codename == "Color Lux" then
     return PocketBookColorLux
 else
-    error("unrecognized PocketBook model " .. codename)
+    error("unrecognized PocketBook model " .. full_codename)
 end

frontend/device/pocketbook/powerd.lua

@@ -41,6 +41,9 @@ function PocketBookPowerD:setIntensityHW(intensity)
     else
         inkview.SetFrontlightState(intensity)
     end
+
+    -- We have a custom isFrontlightOn implementation, so this is redundant
+    self:_decideFrontlightState()
 end
 
 function PocketBookPowerD:isFrontlightOn()

frontend/device/sdl/device.lua

@@ -68,8 +68,10 @@ local Device = Generic:extend{
     hasBattery = SDL.getPowerInfo,
     hasKeyboard = yes,
     hasKeys = yes,
+    hasSymKey = os.getenv("DISABLE_TOUCH") == "1" and yes or no,
     hasDPad = yes,
     hasWifiToggle = no,
+    hasSeamlessWifiToggle = no,
     isTouchDevice = yes,
     isDefaultFullscreen = no,
     needsScreenRefreshAfterResume = no,
@@ -108,7 +110,6 @@ local Device = Generic:extend{
 
 local AppImage = Device:extend{
     model = "AppImage",
-    hasMultitouch = no,
     hasOTAUpdates = yes,
     isDesktop = yes,
 }
@@ -120,6 +121,12 @@ local Desktop = Device:extend{
     hasExitOptions = notOSX,
 }
 
+local Flatpak = Device:extend{
+    model = "Flatpak",
+    isDesktop = yes,
+    canExternalDictLookup = no,
+}
+
 local Emulator = Device:extend{
     model = "Emulator",
     isEmulator = yes,
@@ -230,12 +237,12 @@ function Device:init()
                 local fake_release_ev = Event:new("Gesture", fake_ges_release)
                 if scrolled_y == down then
                     fake_ges.direction = "north"
-                    UIManager:broadcastEvent(fake_pan_ev)
-                    UIManager:broadcastEvent(fake_release_ev)
+                    UIManager:sendEvent(fake_pan_ev)
+                    UIManager:sendEvent(fake_release_ev)
                 elseif scrolled_y == up then
                     fake_ges.direction = "south"
-                    UIManager:broadcastEvent(fake_pan_ev)
-                    UIManager:broadcastEvent(fake_release_ev)
+                    UIManager:sendEvent(fake_pan_ev)
+                    UIManager:sendEvent(fake_release_ev)
                 end
             elseif ev.code == SDL_MULTIGESTURE then
                 -- no-op for now
@@ -272,6 +279,9 @@ function Device:init()
                     FileManager.instance:reinit(FileManager.instance.path,
                         FileManager.instance.focused_file)
                 end
+
+                -- make sure dialogs are displayed
+                UIManager:setDirty("all", "ui")
             elseif ev.code == SDL_WINDOWEVENT_MOVED then
                 self.window.left = ev.value.data1
                 self.window.top = ev.value.data2
@@ -433,6 +443,8 @@ io.write("Starting SDL in " .. SDL.getBasePath() .. "\n")
 -------------- device probe ------------
 if os.getenv("APPIMAGE") then
     return AppImage
+elseif os.getenv("FLATPAK") then
+    return Flatpak
 elseif os.getenv("KO_MULTIUSER") then
     return Desktop
 elseif os.getenv("UBUNTU_APPLICATION_ISOLATION") then

frontend/device/sdl/event_map_sdl2.lua

@@ -61,7 +61,7 @@ return {
     [1073741893] = "F12", -- F[12]
 
     [1073742049] = "Shift", -- left shift
-    [1073742053] = "Sym", -- right shift
+    [1073742053] = os.getenv("DISABLE_TOUCH") == "1" and "Sym" or "Shift", -- right shift
     [1073742050] = "Alt", -- left alt
     [1073742054] = "AA", -- right alt key
     [1073741925] = "ContextMenu", -- Context menu key

frontend/device/sdl/powerd.lua

@@ -18,6 +18,7 @@ end
 function SDLPowerD:setIntensityHW(intensity)
     require("logger").info("set brightness to", intensity)
     self.hw_intensity = intensity or self.hw_intensity
+    self:_decideFrontlightState()
 end
 
 function SDLPowerD:frontlightWarmthHW()

frontend/device/sony-prstux/device.lua

@@ -14,7 +14,7 @@ local SonyPRSTUX = Generic:extend{
     model = "Sony PRSTUX",
     isSonyPRSTUX = yes,
     hasKeys = yes,
-    hasOTAUpdates = yes,
+    hasOTAUpdates = no,
     hasWifiManager = yes,
     canReboot = yes,
     canPowerOff = yes,

frontend/device/wakeupmgr.lua

@@ -47,6 +47,10 @@ end
 
 function WakeupMgr:init()
     self._task_queue = {}
+
+    -- Propagate rtc quirks to the RTC backend
+    self.rtc.dev_rtc = self.dev_rtc
+    self.rtc.dodgy_rtc = self.dodgy_rtc
 end
 
 -- This is a dummy task we use when working around i.MX5 RTC issues.

frontend/dispatcher.lua

@@ -32,6 +32,7 @@ local CreOptions = require("ui/data/creoptions")
 local KoptOptions = require("ui/data/koptoptions")
 local Device = require("device")
 local Event = require("ui/event")
+local FileManager = require("apps/filemanager/filemanager")
 local Notification = require("ui/widget/notification")
 local ReaderHighlight = require("apps/reader/modules/readerhighlight")
 local ReaderZooming = require("apps/reader/modules/readerzooming")
@@ -52,56 +53,62 @@ local settingsList = {
     reading_progress = {category="none", event="ShowReaderProgress", title=_("Reading progress"), general=true},
     open_previous_document = {category="none", event="OpenLastDoc", title=_("Open previous document"), general=true},
     history = {category="none", event="ShowHist", title=_("History"), general=true},
-    favorites = {category="none", event="ShowColl", arg="favorites", title=_("Favorites"), general=true},
+    history_search = {category="none", event="SearchHistory", title=_("History search"), general=true},
+    favorites = {category="none", event="ShowColl", title=_("Favorites"), general=true},
+    collections = {category="none", event="ShowCollList", title=_("Collections"), general=true},
     filemanager = {category="none", event="Home", title=_("File browser"), general=true, separator=true},
-
+    ----
     dictionary_lookup = {category="none", event="ShowDictionaryLookup", title=_("Dictionary lookup"), general=true},
-    wikipedia_lookup = {category="none", event="ShowWikipediaLookup", title=_("Wikipedia lookup"), general=true},
-    fulltext_search = {category="none", event="ShowFulltextSearchInput", title=_("Fulltext search"), general=true},
-    file_search = {category="none", event="ShowFileSearch", title=_("File search"), general=true, separator=true},
-
+    wikipedia_lookup = {category="none", event="ShowWikipediaLookup", title=_("Wikipedia lookup"), general=true, separator=true},
+    ----
     show_menu = {category="none", event="ShowMenu", title=_("Show menu"), general=true},
     menu_search = {category="none", event="MenuSearch", title=_("Menu search"), general=true},
     screenshot = {category="none", event="Screenshot", title=_("Screenshot"), general=true, separator=true},
+    ----
 
-    -- Device settings
-    exit_screensaver = {category="none", event="ExitScreensaver", title=_("Exit screensaver"), device=true},
+    -- Device
+    exit_screensaver = {category="none", event="ExitScreensaver", title=_("Exit sleep screen"), device=true},
+    start_usbms = {category="none", event="RequestUSBMS", title=_("Start USB storage"), device=true, condition=Device:canToggleMassStorage()},
     suspend = {category="none", event="RequestSuspend", title=_("Suspend"), device=true, condition=Device:canSuspend()},
-    exit = {category="none", event="Exit", title=_("Exit KOReader"), device=true},
     restart = {category="none", event="Restart", title=_("Restart KOReader"), device=true, condition=Device:canRestart()},
     reboot = {category="none", event="RequestReboot", title=_("Reboot the device"), device=true, condition=Device:canReboot()},
-    poweroff = {category="none", event="RequestPowerOff", title=_("Power off"), device=true, condition=Device:canPowerOff(), separator=true},
-
+    poweroff = {category="none", event="RequestPowerOff", title=_("Power off"), device=true, condition=Device:canPowerOff()},
+    exit = {category="none", event="Exit", title=_("Exit KOReader"), device=true, separator=true},
+    ----
     toggle_hold_corners = {category="none", event="IgnoreHoldCorners", title=_("Toggle hold corners"), device=true},
     touch_input_on = {category="none", event="IgnoreTouchInput", arg=false, title=_("Enable touch input"), device=true},
     touch_input_off = {category="none", event="IgnoreTouchInput", arg=true, title=_("Disable touch input"), device=true},
     toggle_touch_input = {category="none", event="IgnoreTouchInput", title=_("Toggle touch input"), device=true, separator=true},
+    ----
     swap_page_turn_buttons = {category="none", event="SwapPageTurnButtons", title=_("Invert page turn buttons"), device=true, condition=Device:hasKeys(), separator=true},
+    ----
     toggle_key_repeat = {category="none", event="ToggleKeyRepeat", title=_("Toggle key repeat"), device=true, condition=Device:hasKeys() and Device:canKeyRepeat(), separator=true},
     toggle_gsensor = {category="none", event="ToggleGSensor", title=_("Toggle accelerometer"), device=true, condition=Device:hasGSensor()},
+    lock_gsensor = {category="none", event="LockGSensor", title=_("Lock auto rotation to current orientation"), device=true, condition=Device:hasGSensor()},
     toggle_rotation = {category="none", event="SwapRotation", title=_("Toggle orientation"), device=true},
     invert_rotation = {category="none", event="InvertRotation", title=_("Invert rotation"), device=true},
     iterate_rotation = {category="none", event="IterateRotation", title=_("Rotate by 90° CW"), device=true},
     iterate_rotation_ccw = {category="none", event="IterateRotation", arg=true, title=_("Rotate by 90° CCW"), device=true, separator=true},
-
+    ----
     wifi_on = {category="none", event="InfoWifiOn", title=_("Turn on Wi-Fi"), device=true, condition=Device:hasWifiToggle()},
     wifi_off = {category="none", event="InfoWifiOff", title=_("Turn off Wi-Fi"), device=true, condition=Device:hasWifiToggle()},
     toggle_wifi = {category="none", event="ToggleWifi", title=_("Toggle Wi-Fi"), device=true, condition=Device:hasWifiToggle()},
     toggle_fullscreen = {category="none", event="ToggleFullscreen", title=_("Toggle Fullscreen"), device=true, condition=not Device:isAlwaysFullscreen()},
     show_network_info = {category="none", event="ShowNetworkInfo", title=_("Show network info"), device=true, separator=true},
+    ----
 
-    -- Screen & Lights
+    -- Screen and lights
     show_frontlight_dialog = {category="none", event="ShowFlDialog", title=_("Show frontlight dialog"), screen=true, condition=Device:hasFrontlight()},
     toggle_frontlight = {category="none", event="ToggleFrontlight", title=_("Toggle frontlight"), screen=true, condition=Device:hasFrontlight()},
     set_frontlight = {category="absolutenumber", event="SetFlIntensity", min=0, max=Device:getPowerDevice().fl_max, title=_("Set frontlight brightness"), screen=true, condition=Device:hasFrontlight()},
-    increase_frontlight = {category="incrementalnumber", event="IncreaseFlIntensity", min=1, max=Device:getPowerDevice().fl_max, title=_("Increase frontlight brightness"), screen=true, condition=Device:hasFrontlight()},
-    decrease_frontlight = {category="incrementalnumber", event="DecreaseFlIntensity", min=1, max=Device:getPowerDevice().fl_max, title=_("Decrease frontlight brightness"), screen=true, condition=Device:hasFrontlight()},
+    increase_frontlight = {category="incrementalnumber", event="IncreaseFlIntensity", min=0, max=Device:getPowerDevice().fl_max, title=_("Increase frontlight brightness"), screen=true, condition=Device:hasFrontlight()},
+    decrease_frontlight = {category="incrementalnumber", event="DecreaseFlIntensity", min=0, max=Device:getPowerDevice().fl_max, title=_("Decrease frontlight brightness"), screen=true, condition=Device:hasFrontlight()},
     set_frontlight_warmth = {category="absolutenumber", event="SetFlWarmth", min=0, max=100, title=_("Set frontlight warmth"), screen=true, condition=Device:hasNaturalLight()},
-    increase_frontlight_warmth = {category="incrementalnumber", event="IncreaseFlWarmth", min=1, max=Device:getPowerDevice().fl_warmth_max, title=_("Increase frontlight warmth"), screen=true, condition=Device:hasNaturalLight()},
-    decrease_frontlight_warmth = {category="incrementalnumber", event="DecreaseFlWarmth", min=1, max=Device:getPowerDevice().fl_warmth_max, title=_("Decrease frontlight warmth"), screen=true, condition=Device:hasNaturalLight(), separator=true},
-
+    increase_frontlight_warmth = {category="incrementalnumber", event="IncreaseFlWarmth", min=0, max=Device:getPowerDevice().fl_warmth_max, title=_("Increase frontlight warmth"), screen=true, condition=Device:hasNaturalLight()},
+    decrease_frontlight_warmth = {category="incrementalnumber", event="DecreaseFlWarmth", min=0, max=Device:getPowerDevice().fl_warmth_max, title=_("Decrease frontlight warmth"), screen=true, condition=Device:hasNaturalLight(), separator=true},
     night_mode = {category="none", event="ToggleNightMode", title=_("Toggle night mode"), screen=true},
     set_night_mode = {category="string", event="SetNightMode", title=_("Set night mode"), screen=true, args={true, false}, toggle={_("on"), _("off")}, separator=true},
+    ----
     full_refresh = {category="none", event="FullRefresh", title=_("Full screen refresh"), screen=true},
     set_refresh_rate = {category="absolutenumber", event="SetBothRefreshRates", min=-1, max=200, title=_("Full refresh rate (always)"), screen=true, condition=Device:hasEinkScreen()},
     set_day_refresh_rate = {category="absolutenumber", event="SetDayRefreshRate", min=-1, max=200, title=_("Full refresh rate (not in night mode)"), screen=true, condition=Device:hasEinkScreen()},
@@ -109,33 +116,48 @@ local settingsList = {
     set_flash_on_chapter_boundaries = {category="string", event="SetFlashOnChapterBoundaries", title=_("Always flash on chapter boundaries"), screen=true, condition=Device:hasEinkScreen(), args={true, false}, toggle={_("on"), _("off")}},
     toggle_flash_on_chapter_boundaries = {category="none", event="ToggleFlashOnChapterBoundaries", title=_("Toggle flashing on chapter boundaries"), screen=true, condition=Device:hasEinkScreen()},
     set_no_flash_on_second_chapter_page = {category="string", event="SetNoFlashOnSecondChapterPage", title=_("Never flash on chapter's 2nd page"), screen=true, condition=Device:hasEinkScreen(), args={true, false}, toggle={_("on"), _("off")}},
-    toggle_no_flash_on_second_chapter_page = {category="none", event="ToggleNoFlashOnSecondChapterPage", title=_("Toggle flashing on chapter's 2nd page"), screen=true, condition=Device:hasEinkScreen(), separator=true},
+    toggle_no_flash_on_second_chapter_page = {category="none", event="ToggleNoFlashOnSecondChapterPage", title=_("Toggle flashing on chapter's 2nd page"), screen=true, condition=Device:hasEinkScreen()},
     set_flash_on_pages_with_images = {category="string", event="SetFlashOnPagesWithImages", title=_("Always flash on pages with images"), screen=true, condition=Device:hasEinkScreen(), args={true, false}, toggle={_("on"), _("off")}},
-    toggle_flash_on_pages_with_images = {category="none", event="ToggleFlashOnPagesWithImages", title=_("Toggle flashing on pages with images"), screen=true, condition=Device:hasEinkScreen()},
-
-    -- filemanager settings
+    toggle_flash_on_pages_with_images = {category="none", event="ToggleFlashOnPagesWithImages", title=_("Toggle flashing on pages with images"), screen=true, condition=Device:hasEinkScreen(), separator=true},
+    ----
+
+    -- File browser
+    set_display_mode = {category="string", event="SetDisplayMode", title=_("Set display mode"), args_func=FileManager.getDisplayModeActions, filemanager=true},
+    set_sort_by = {category="string", event="SetSortBy", title=_("Sort by"), args_func=FileManager.getSortByActions, filemanager=true},
+    set_reverse_sorting = {category="string", event="SetReverseSorting", title=_("Reverse sorting"), args={true, false}, toggle={_("on"), _("off")}, filemanager=true},
+    set_mixed_sorting = {category="string", event="SetMixedSorting", title=_("Folders and files mixed"), args={true, false}, toggle={_("on"), _("off")}, filemanager=true, separator=true},
+    ----
     folder_up = {category="none", event="FolderUp", title=_("Folder up"), filemanager=true},
     show_plus_menu = {category="none", event="ShowPlusMenu", title=_("Show plus menu"), filemanager=true},
     toggle_select_mode = {category="none", event="ToggleSelectMode", title=_("Toggle select mode"), filemanager=true},
     refresh_content = {category="none", event="RefreshContent", title=_("Refresh content"), filemanager=true},
-    folder_shortcuts = {category="none", event="ShowFolderShortcutsDialog", title=_("Folder shortcuts"), filemanager=true, separator=true},
+    folder_shortcuts = {category="none", event="ShowFolderShortcutsDialog", title=_("Folder shortcuts"), filemanager=true},
+    file_search = {category="none", event="ShowFileSearch", title=_("File search"), filemanager=true},
+    ----
+    -- go_to
+    -- back
 
-    -- reader settings
+    -- Reader
     open_next_document_in_folder = {category="none", event="OpenNextDocumentInFolder", title=_("Open next document in folder"), reader=true, separator=true},
-
+    ----
     show_config_menu = {category="none", event="ShowConfigMenu", title=_("Show bottom menu"), reader=true},
-    toggle_status_bar = {category="none", event="ToggleFooterMode", title=_("Toggle status bar"), reader=true, separator=true},
-
+    toggle_status_bar = {category="none", event="ToggleFooterMode", title=_("Toggle status bar"), reader=true},
+    toggle_chapter_progress_bar = {category="none", event="ToggleChapterProgressBar", title=_("Toggle chapter progress bar"), reader=true, separator=true},
+    ----
     prev_chapter = {category="none", event="GotoPrevChapter", title=_("Previous chapter"), reader=true},
     next_chapter = {category="none", event="GotoNextChapter", title=_("Next chapter"), reader=true},
     first_page = {category="none", event="GoToBeginning", title=_("First page"), reader=true},
     last_page = {category="none", event="GoToEnd", title=_("Last page"), reader=true},
+    random_page = {category="none", event="GoToRandomPage", title=_("Random page"), reader=true},
     page_jmp = {category="absolutenumber", event="GotoViewRel", min=-100, max=100, title=_("Turn pages"), reader=true},
     go_to = {category="none", event="ShowGotoDialog", title=_("Go to page"), filemanager=true, reader=true},
     skim = {category="none", event="ShowSkimtoDialog", title=_("Skim document"), reader=true},
     prev_bookmark = {category="none", event="GotoPreviousBookmarkFromPage", title=_("Previous bookmark"), reader=true},
     next_bookmark = {category="none", event="GotoNextBookmarkFromPage", title=_("Next bookmark"), reader=true},
-    latest_bookmark = {category="none", event="GoToLatestBookmark", title=_("Go to latest bookmark"), reader=true},
+    first_bookmark = {category="none", event="GotoFirstBookmark", title=_("First bookmark"), reader=true},
+    last_bookmark = {category="none", event="GotoLastBookmark", title=_("Last bookmark"), reader=true},
+    latest_bookmark = {category="none", event="GoToLatestBookmark", title=_("Latest bookmark"), reader=true, separator=true},
+    ----
     back = {category="none", event="Back", title=_("Back"), filemanager=true, reader=true},
     previous_location = {category="none", event="GoBackLink", arg=true, title=_("Back to previous location"), reader=true},
     next_location = {category="none", event="GoForwardLink", arg=true, title=_("Forward to next location"), reader=true},
@@ -143,69 +165,78 @@ local settingsList = {
     follow_nearest_internal_link = {category="arg", event="GoToInternalPageLink", arg={pos={x=0,y=0}}, title=_("Follow nearest internal link"), reader=true},
     add_location_to_history = {category="none", event="AddCurrentLocationToStack", arg=true, title=_("Add current location to history"), reader=true},
     clear_location_history = {category="none", event="ClearLocationStack", arg=true, title=_("Clear location history"), reader=true, separator=true},
-
+    ----
+    fulltext_search = {category="none", event="ShowFulltextSearchInput", title=_("Fulltext search"), reader=true},
+    fulltext_search_findall_results = {category="none", event="ShowFindAllResults", title=_("Last fulltext search results"), reader=true},
     toc = {category="none", event="ShowToc", title=_("Table of contents"), reader=true},
     book_map = {category="none", event="ShowBookMap", title=_("Book map"), reader=true, condition=Device:isTouchDevice()},
     book_map_overview = {category="none", event="ShowBookMap", arg=true, title=_("Book map (overview)"), reader=true, condition=Device:isTouchDevice()},
     page_browser = {category="none", event="ShowPageBrowser", title=_("Page browser"), reader=true, condition=Device:isTouchDevice()},
     bookmarks = {category="none", event="ShowBookmark", title=_("Bookmarks"), reader=true},
     bookmark_search = {category="none", event="SearchBookmark", title=_("Bookmark search"), reader=true},
-
+    toggle_bookmark = {category="none", event="ToggleBookmark", title=_("Toggle bookmark"), reader=true, separator=true},
+    ----
     book_status = {category="none", event="ShowBookStatus", title=_("Book status"), reader=true},
     book_info = {category="none", event="ShowBookInfo", title=_("Book information"), reader=true},
     book_description = {category="none", event="ShowBookDescription", title=_("Book description"), reader=true},
     book_cover = {category="none", event="ShowBookCover", title=_("Book cover"), reader=true, separator=true},
-
+    ----
     translate_page = {category="none", event="TranslateCurrentPage", title=_("Translate current page"), reader=true, separator=true},
+    ----
+    toggle_page_change_animation = {category="none", event="TogglePageChangeAnimation", title=_("Toggle page turn animations"), reader=true, condition=Device:canDoSwipeAnimation()},
+    toggle_inverse_reading_order = {category="none", event="ToggleReadingOrder", title=_("Toggle page turn direction"), reader=true},
+    toggle_handmade_toc = {category="none", event="ToggleHandmadeToc", title=_("Toggle custom TOC"), reader=true},
+    toggle_handmade_flows = {category="none", event="ToggleHandmadeFlows", title=_("Toggle custom hidden flows"), reader=true, separator=true},
+    ----
+    set_highlight_action = {category="string", event="SetHighlightAction", title=_("Set highlight action"), args_func=ReaderHighlight.getHighlightActions, reader=true},
+    cycle_highlight_action = {category="none", event="CycleHighlightAction", title=_("Cycle highlight action"), reader=true},
+    cycle_highlight_style = {category="none", event="CycleHighlightStyle", title=_("Cycle highlight style"), reader=true, separator=true},
+    ----
+    flush_settings = {category="none", event="FlushSettings", arg=true, title=_("Save book metadata"), reader=true, separator=true},
+    ----
 
-    -- rolling reader settings
+    -- Reflowable documents
     set_font = {category="string", event="SetFont", title=_("Set font"), rolling=true, args_func=require("fontlist").getFontArgFunc,},
     increase_font = {category="incrementalnumber", event="IncreaseFontSize", min=0.5, max=255, step=0.5, title=_("Increase font size"), rolling=true},
     decrease_font = {category="incrementalnumber", event="DecreaseFontSize", min=0.5, max=255, step=0.5, title=_("Decrease font size"), rolling=true},
-    --
-
-    toggle_bookmark = {category="none", event="ToggleBookmark", title=_("Toggle bookmark"), reader=true},
-    toggle_page_change_animation = {category="none", event="TogglePageChangeAnimation", title=_("Toggle page turn animations"), reader=true, condition=Device:canDoSwipeAnimation()},
 
-    -- paging reader settings
+    -- Page layout documents
     toggle_page_flipping = {category="none", event="TogglePageFlipping", title=_("Toggle page flipping"), paging=true},
     toggle_bookmark_flipping = {category="none", event="ToggleBookmarkFlipping", title=_("Toggle bookmark flipping"), paging=true},
     toggle_reflow = {category="none", event="ToggleReflow", title=_("Toggle reflow"), paging=true},
     zoom = {category="string", event="SetZoomMode", title=_("Zoom mode"), args_func=ReaderZooming.getZoomModeActions, paging=true},
     zoom_factor_change = {category="none", event="ZoomFactorChange", title=_("Change zoom factor"), paging=true, separator=true},
-    --
-
-    toggle_inverse_reading_order = {category="none", event="ToggleReadingOrder", title=_("Toggle page turn direction"), reader=true, separator=true},
-    set_highlight_action = {category="string", event="SetHighlightAction", title=_("Set highlight action"), args_func=ReaderHighlight.getHighlightActions, reader=true},
-    cycle_highlight_action = {category="none", event="CycleHighlightAction", title=_("Cycle highlight action"), reader=true},
-    cycle_highlight_style = {category="none", event="CycleHighlightStyle", title=_("Cycle highlight style"), reader=true, separator=true},
-    flush_settings = {category="none", event="FlushSettings", arg=true, title=_("Save book metadata"), reader=true, separator=true},
+    ----
     panel_zoom_toggle = {category="none", event="TogglePanelZoomSetting", title=_("Toggle panel zoom"), paging=true, separator=true},
+    ----
 
     -- parsed from CreOptions
-    -- the rest of the table elements are built from their counterparts in CreOptions
     rotation_mode = {category="string", device=true},
+    font_size = {category="absolutenumber", title=_("Set font size"), rolling=true, step=0.5},
+    word_spacing = {category="string", rolling=true},
+    word_expansion = {category="string", rolling=true},
+    font_gamma = {category="string", rolling=true},
+    font_base_weight = {category="string", rolling=true},
+    font_hinting = {category="string", rolling=true},
+    font_kerning = {category="string", rolling=true, separator=true},
+    ----
     visible_pages = {category="string", rolling=true, separator=true},
+    ----
     h_page_margins = {category="string", rolling=true},
     sync_t_b_page_margins = {category="string", rolling=true},
     t_page_margin = {category="absolutenumber", rolling=true},
     b_page_margin = {category="absolutenumber", rolling=true, separator=true},
+    ----
     view_mode = {category="string", rolling=true},
     block_rendering_mode = {category="string", rolling=true},
     render_dpi = {category="string", title=_("Zoom"), rolling=true},
     line_spacing = {category="absolutenumber", rolling=true, separator=true},
-    font_size = {category="absolutenumber", title=_("Set font size"), rolling=true, step=0.5},
-    font_base_weight = {category="string", rolling=true},
-    word_spacing = {category="string", rolling=true},
-    word_expansion = {category="string", rolling=true},
-    font_gamma = {category="string", rolling=true},
-    font_hinting = {category="string", rolling=true},
-    font_kerning = {category="string", rolling=true, separator=true},
+    ----
     status_line = {category="string", rolling=true},
     embedded_css = {category="string", rolling=true},
     embedded_fonts = {category="string", rolling=true},
     smooth_scaling = {category="string", rolling=true},
-    nightmode_images = {category="string", rolling=true, separator=true},
+    nightmode_images = {category="string", rolling=true},
 
     -- parsed from KoptOptions
     kopt_trim_page = {category="string", paging=true},
@@ -213,7 +244,7 @@ local settingsList = {
     kopt_zoom_overlap_h = {category="absolutenumber", paging=true},
     kopt_zoom_overlap_v = {category="absolutenumber", paging=true},
     kopt_zoom_mode_type = {category="string", paging=true},
-    kopt_zoom_range_number = {category="string", paging=true},
+    -- kopt_zoom_range_number = {category="string", paging=true},
     kopt_zoom_factor = {category="string", paging=true},
     kopt_zoom_mode_genus = {category="string", paging=true},
     kopt_zoom_direction = {category="string", paging=true},
@@ -226,7 +257,7 @@ local settingsList = {
     kopt_font_fine_tune = {category="string", paging=true},
     kopt_word_spacing = {category="configurable", paging=true},
     kopt_text_wrap = {category="string", paging=true},
-    kopt_contrast = {category="absolutenumber", paging=true},
+    kopt_contrast = {category="string", paging=true},
     kopt_page_opt = {category="configurable", paging=true},
     kopt_hw_dithering = {category="configurable", paging=true, condition=Device:hasEinkScreen() and Device:canHWDither()},
     kopt_sw_dithering = {category="configurable", paging=true, condition=Device:hasEinkScreen() and not Device:canHWDither() and Device.screen.fb_bpp == 8},
@@ -244,48 +275,56 @@ local settingsList = {
 
 -- array for item order in menu
 local dispatcher_menu_order = {
-    -- device
+    -- General
     "reading_progress",
     "open_previous_document",
     "history",
+    "history_search",
     "favorites",
+    "collections",
     "filemanager",
-
+    ----
     "dictionary_lookup",
     "wikipedia_lookup",
-    "fulltext_search",
-    "file_search",
-
+    ----
     "show_menu",
     "menu_search",
     "screenshot",
+    ----
 
+    -- Device
     "exit_screensaver",
+    "start_usbms",
     "suspend",
-    "exit",
     "restart",
     "reboot",
     "poweroff",
-
+    "exit",
+    ----
     "toggle_hold_corners",
     "touch_input_on",
     "touch_input_off",
     "toggle_touch_input",
+    ----
     "swap_page_turn_buttons",
+    ----
     "toggle_key_repeat",
     "toggle_gsensor",
+    "lock_gsensor",
     "rotation_mode",
     "toggle_rotation",
     "invert_rotation",
     "iterate_rotation",
     "iterate_rotation_ccw",
-
+    ----
     "wifi_on",
     "wifi_off",
     "toggle_wifi",
     "toggle_fullscreen",
     "show_network_info",
+    ----
 
+    -- Screen and lights
     "show_frontlight_dialog",
     "toggle_frontlight",
     "set_frontlight",
@@ -294,10 +333,9 @@ local dispatcher_menu_order = {
     "set_frontlight_warmth",
     "increase_frontlight_warmth",
     "decrease_frontlight_warmth",
-
     "night_mode",
     "set_night_mode",
-
+    ----
     "full_refresh",
     "set_refresh_rate",
     "set_day_refresh_rate",
@@ -308,30 +346,45 @@ local dispatcher_menu_order = {
     "toggle_no_flash_on_second_chapter_page",
     "set_flash_on_pages_with_images",
     "toggle_flash_on_pages_with_images",
-
-    -- filemanager
+    ----
+
+    -- File browser
+    "set_display_mode",
+    "set_sort_by",
+    "set_reverse_sorting",
+    "set_mixed_sorting",
+    ----
     "folder_up",
     "show_plus_menu",
     "toggle_select_mode",
     "refresh_content",
     "folder_shortcuts",
+    "file_search",
+    ----
+    -- "go_to"
+    -- "back"
 
-    -- reader
+    -- Reader
     "open_next_document_in_folder",
-
+    ----
     "show_config_menu",
     "toggle_status_bar",
-
+    "toggle_chapter_progress_bar",
+    ----
     "prev_chapter",
     "next_chapter",
     "first_page",
     "last_page",
+    "random_page",
     "page_jmp",
     "go_to",
     "skim",
     "prev_bookmark",
     "next_bookmark",
+    "first_bookmark",
+    "last_bookmark",
     "latest_bookmark",
+    ----
     "back",
     "previous_location",
     "next_location",
@@ -339,21 +392,37 @@ local dispatcher_menu_order = {
     "follow_nearest_internal_link",
     "add_location_to_history",
     "clear_location_history",
-
+    ----
+    "fulltext_search",
+    "fulltext_search_findall_results",
     "toc",
     "book_map",
     "book_map_overview",
     "page_browser",
     "bookmarks",
     "bookmark_search",
-
+    "toggle_bookmark",
+    ----
     "book_status",
     "book_info",
     "book_description",
     "book_cover",
-
+    ----
     "translate_page",
+    ----
+    "toggle_page_change_animation",
+    "toggle_inverse_reading_order",
+    "toggle_handmade_toc",
+    "toggle_handmade_flows",
+    ----
+    "set_highlight_action",
+    "cycle_highlight_action",
+    "cycle_highlight_style",
+    ----
+    "flush_settings",
+    ----
 
+    -- Reflowable documents
     "set_font",
     "increase_font",
     "decrease_font",
@@ -364,42 +433,36 @@ local dispatcher_menu_order = {
     "font_base_weight",
     "font_hinting",
     "font_kerning",
-
-    "toggle_bookmark",
-    "toggle_page_change_animation",
-    "toggle_page_flipping",
-    "toggle_bookmark_flipping",
-    "toggle_reflow",
-    "toggle_inverse_reading_order",
-    "zoom",
-    "zoom_factor_change",
-    "set_highlight_action",
-    "cycle_highlight_action",
-    "cycle_highlight_style",
-    "flush_settings",
-    "panel_zoom_toggle",
-
+    ----
     "visible_pages",
-
+    ----
     "h_page_margins",
     "sync_t_b_page_margins",
     "t_page_margin",
     "b_page_margin",
-
+    ----
     "view_mode",
     "block_rendering_mode",
     "render_dpi",
     "line_spacing",
-
+    ----
     "status_line",
     "embedded_css",
     "embedded_fonts",
     "smooth_scaling",
     "nightmode_images",
 
+    -- Fixed layout documents
+    "toggle_page_flipping",
+    "toggle_bookmark_flipping",
+    "toggle_reflow",
+    "zoom",
+    "zoom_factor_change",
+    ----
+    "panel_zoom_toggle",
+    ----
     "kopt_trim_page",
     "kopt_page_margin",
-
     "kopt_zoom_overlap_h",
     "kopt_zoom_overlap_v",
     "kopt_zoom_mode_type",
@@ -407,24 +470,20 @@ local dispatcher_menu_order = {
     "kopt_zoom_factor",
     "kopt_zoom_mode_genus",
     "kopt_zoom_direction",
-
     "kopt_page_scroll",
     "kopt_page_gap_height",
     "kopt_full_screen",
     "kopt_line_spacing",
     "kopt_justification",
-
     "kopt_font_size",
     "kopt_font_fine_tune",
     "kopt_word_spacing",
     "kopt_text_wrap",
-
     "kopt_contrast",
     "kopt_page_opt",
     "kopt_hw_dithering",
     "kopt_sw_dithering",
     "kopt_quality",
-
     "kopt_doc_language",
     "kopt_forced_ocr",
     "kopt_writing_direction",
@@ -670,7 +729,7 @@ function Dispatcher:_sortActions(caller, location, settings, touchmenu_instance)
     local SortWidget = require("ui/widget/sortwidget")
     local sort_widget
     sort_widget = SortWidget:new{
-        title = _("Sort"),
+        title = _("Arrange actions"),
         item_table = display_list,
         callback = function()
             if location[settings] and next(location[settings]) ~= nil then
@@ -787,7 +846,7 @@ function Dispatcher:_addItem(caller, menu, location, settings, section)
                             value_hold_step = 5,
                             value_max = settingsList[k].max,
                             title_text = Dispatcher:getNameFromItem(k, location[settings], true),
-                            info_text = _([[If called by a gesture the amount of the gesture will be used]]),
+                            info_text = _([[When set to 0, the gesture's distance (if any) is used]]),
                             ok_always_enabled = true,
                             callback = function(spin)
                                 if location[settings] == nil then
@@ -907,6 +966,7 @@ function Dispatcher:addSubMenu(caller, menu, location, settings)
         {"rolling", _("Reflowable documents (epub, fb2, txt…)")},
         {"paging", _("Fixed layout documents (pdf, djvu, pics…)")},
     }
+    menu.max_per_page = 1 + #section_list -- settings in page 2
     for _, section in ipairs(section_list) do
         local submenu = {}
         Dispatcher:_addItem(caller, submenu, location, settings, section[1])
@@ -938,7 +998,7 @@ function Dispatcher:addSubMenu(caller, menu, location, settings)
     end
     menu[#menu].separator = true
     table.insert(menu, {
-        text = _("Sort"),
+        text = _("Arrange actions"),
         checked_func = function()
             return location[settings] ~= nil
             and location[settings].settings ~= nil
@@ -982,6 +1042,31 @@ function Dispatcher:addSubMenu(caller, menu, location, settings)
             end
         end,
     })
+    table.insert(menu, {
+        text = _("Keep QuickMenu open"),
+        checked_func = function()
+            return location[settings] ~= nil
+            and location[settings].settings ~= nil
+            and location[settings].settings.keep_open_on_apply
+        end,
+        callback = function()
+            if location[settings] then
+                if location[settings].settings then
+                    if location[settings].settings.keep_open_on_apply then
+                        location[settings].settings.keep_open_on_apply = nil
+                        if next(location[settings].settings) == nil then
+                            location[settings].settings = nil
+                        end
+                    else
+                        location[settings].settings.keep_open_on_apply = true
+                    end
+                else
+                    location[settings].settings = {["keep_open_on_apply"] = true}
+                end
+                caller.updated = true
+            end
+        end,
+    })
 end
 
 function Dispatcher:isActionEnabled(action)
@@ -1001,6 +1086,8 @@ function Dispatcher:isActionEnabled(action)
 end
 
 function Dispatcher:_showAsMenu(settings, exec_props)
+    local title = settings.settings.name or _("QuickMenu")
+    local keep_open_on_apply = settings.settings.keep_open_on_apply
     local display_list = Dispatcher:getDisplayList(settings)
     local quickmenu
     local buttons = {}
@@ -1027,6 +1114,10 @@ function Dispatcher:_showAsMenu(settings, exec_props)
             callback = function()
                 UIManager:close(quickmenu)
                 Dispatcher:execute({[v.key] = settings[v.key]})
+                if keep_open_on_apply and not util.stringStartsWith(v.key, "touch_input") then
+                    quickmenu:setTitle(title)
+                    UIManager:show(quickmenu)
+                end
             end,
             hold_callback = function()
                 if v.key:sub(1, 13) == "profile_exec_" then
@@ -1038,7 +1129,7 @@ function Dispatcher:_showAsMenu(settings, exec_props)
     end
     local ButtonDialog = require("ui/widget/buttondialog")
     quickmenu = ButtonDialog:new{
-        title = settings.settings.name or _("QuickMenu"),
+        title = title,
         title_align = "center",
         shrink_unneeded_width = true,
         shrink_min_width = math.floor(0.6 * Screen:getWidth()),

frontend/docsettings.lua

@@ -16,6 +16,40 @@ local DocSettings = LuaSettings:extend{}
 
 local HISTORY_DIR = DataStorage:getHistoryDir()
 local DOCSETTINGS_DIR = DataStorage:getDocSettingsDir()
+local DOCSETTINGS_HASH_DIR = DataStorage:getDocSettingsHashDir()
+local custom_metadata_filename = "custom_metadata.lua"
+
+function DocSettings.getSidecarStorage(location)
+    if location == "dir" then
+        return DOCSETTINGS_DIR
+    elseif location == "hash" then
+        return DOCSETTINGS_HASH_DIR
+    end
+end
+
+local function isDir(dir)
+    return lfs.attributes(dir, "mode") == "directory"
+end
+
+local function isFile(file)
+    return lfs.attributes(file, "mode") == "file"
+end
+
+local is_history_location_enabled = isDir(HISTORY_DIR)
+
+local doc_hash_cache = {}
+local is_hash_location_enabled
+
+function DocSettings.isHashLocationEnabled()
+    if is_hash_location_enabled == nil then
+        is_hash_location_enabled = isDir(DOCSETTINGS_HASH_DIR)
+    end
+    return is_hash_location_enabled
+end
+
+function DocSettings.setIsHashLocationEnabled(value)
+    is_hash_location_enabled = value
+end
 
 local function buildCandidates(list)
     local candidates = {}
@@ -23,7 +57,7 @@ local function buildCandidates(list)
 
     for i, file_path in ipairs(list) do
         -- Ignore missing files.
-        if file_path ~= "" and lfs.attributes(file_path, "mode") == "file" then
+        if file_path ~= "" and isFile(file_path) then
             local mtime = lfs.attributes(file_path, "modification")
             -- NOTE: Extra trickery: if we're inserting a "backup" file, and its primary buddy exists,
             --       make sure it will *never* sort ahead of it by using the same mtime.
@@ -64,52 +98,87 @@ local function buildCandidates(list)
     return candidates
 end
 
+local function getOrderedLocationCandidates()
+    local preferred_location = G_reader_settings:readSetting("document_metadata_folder", "doc")
+    if preferred_location == "hash" then
+        return { "hash", "doc", "dir" }
+    end
+    local candidates = preferred_location == "doc" and { "doc", "dir" } or { "dir", "doc" }
+    if DocSettings.isHashLocationEnabled() then
+        table.insert(candidates, "hash")
+    end
+    return candidates
+end
+
 --- Returns path to sidecar directory (`filename.sdr`).
 -- Sidecar directory is the file without _last_ suffix.
 -- @string doc_path path to the document (e.g., `/foo/bar.pdf`)
+-- @string force_location prefer e.g., "hash" or "dir" location over standard "doc", if available
 -- @treturn string path to the sidecar directory (e.g., `/foo/bar.sdr`)
 function DocSettings:getSidecarDir(doc_path, force_location)
     if doc_path == nil or doc_path == "" then return "" end
     local path = doc_path:match("(.*)%.") or doc_path -- file path without the last suffix
     local location = force_location or G_reader_settings:readSetting("document_metadata_folder", "doc")
     if location == "dir" then
-        path = DOCSETTINGS_DIR..path
+        path = DOCSETTINGS_DIR .. path
+    elseif location == "hash" then
+        local hsh = doc_hash_cache[doc_path]
+        if not hsh then
+            hsh = util.partialMD5(doc_path)
+            if not hsh then -- fallback to "doc"
+                return path .. ".sdr"
+            end
+            doc_hash_cache[doc_path] = hsh
+            logger.dbg("DocSettings: Caching new partial MD5 hash for", doc_path, "as", hsh)
+        else
+            logger.dbg("DocSettings: Using cached partial MD5 hash for", doc_path, "as", hsh)
+        end
+        -- converts b3fb8f4f8448160365087d6ca05c7fa2 to b3/ to avoid too many files in one dir
+        local subpath = string.format("/%s/", hsh:sub(1, 2))
+        path = DOCSETTINGS_HASH_DIR .. subpath .. hsh
     end
-    return path..".sdr"
+    return path .. ".sdr"
+end
+
+function DocSettings.getSidecarFilename(doc_path)
+    local suffix = doc_path:match(".*%.(.+)") or "_"
+    return "metadata." .. suffix .. ".lua"
 end
 
---- Returns path to `metadata.lua` file.
+--- Returns `true` if there is a `metadata.lua` file.
 -- @string doc_path path to the document (e.g., `/foo/bar.pdf`)
--- @treturn string path to `/foo/bar.sdr/metadata.lua` file
-function DocSettings:getSidecarFile(doc_path, force_location)
-    if doc_path == nil or doc_path == "" then return "" end
-    -- If the file does not have a suffix or we are working on a directory, we
-    -- should ignore the suffix part in metadata file path.
-    local suffix = doc_path:match(".*%.(.+)") or ""
-    return self:getSidecarDir(doc_path, force_location) .. "/metadata." .. suffix .. ".lua"
+-- @treturn bool
+function DocSettings:hasSidecarFile(doc_path)
+    return self:findSidecarFile(doc_path) and true or false
 end
 
 --- Returns path of `metadata.lua` file if it exists, or nil.
 -- @string doc_path path to the document (e.g., `/foo/bar.pdf`)
 -- @bool no_legacy set to true to skip check of the legacy history file
--- @treturn string
-function DocSettings:hasSidecarFile(doc_path, no_legacy)
-    local sidecar_file = self:getSidecarFile(doc_path, "doc")
-    if lfs.attributes(sidecar_file, "mode") == "file" then
-        return sidecar_file
-    end
-    sidecar_file = self:getSidecarFile(doc_path, "dir")
-    if lfs.attributes(sidecar_file, "mode") == "file" then
-        return sidecar_file
+-- @treturn string (or nil on failure)
+function DocSettings:findSidecarFile(doc_path, no_legacy)
+    if doc_path == nil or doc_path == "" then return nil end
+    local sidecar_filename = DocSettings.getSidecarFilename(doc_path)
+    local sidecar_file
+    for _, location in ipairs(getOrderedLocationCandidates()) do
+        sidecar_file = self:getSidecarDir(doc_path, location) .. "/" .. sidecar_filename
+        if isFile(sidecar_file) then
+            return sidecar_file, location
+        end
     end
-    if not no_legacy then
+    if is_history_location_enabled and not no_legacy then
         sidecar_file = self:getHistoryPath(doc_path)
-        if lfs.attributes(sidecar_file, "mode") == "file" then
-            return sidecar_file
+        if isFile(sidecar_file) then
+            return sidecar_file, "hist" -- for isSidecarFileNotInPreferredLocation() used in moveBookMetadata
         end
     end
 end
 
+function DocSettings.isSidecarFileNotInPreferredLocation(doc_path)
+    local _, location = DocSettings:findSidecarFile(doc_path)
+    return location and location ~= G_reader_settings:readSetting("document_metadata_folder", "doc")
+end
+
 function DocSettings:getHistoryPath(doc_path)
     if doc_path == nil or doc_path == "" then return "" end
     return HISTORY_DIR .. "/[" .. doc_path:gsub("(.*/)([^/]+)", "%1] %2"):gsub("/", "#") .. ".lua"
@@ -146,41 +215,6 @@ function DocSettings:getFileFromHistory(hist_name)
     end
 end
 
---- Returns path to book custom cover file if it exists, or nil.
-function DocSettings:findCoverFile(doc_path)
-    local location = G_reader_settings:readSetting("document_metadata_folder", "doc")
-    local sidecar_dir = self:getSidecarDir(doc_path, location)
-    local cover_file = self:_findCoverFileInDir(sidecar_dir)
-    if not cover_file then
-        location = location == "doc" and "dir" or "doc"
-        sidecar_dir = self:getSidecarDir(doc_path, location)
-        cover_file = self:_findCoverFileInDir(sidecar_dir)
-    end
-    return cover_file
-end
-
-function DocSettings:_findCoverFileInDir(dir)
-    local ok, iter, dir_obj = pcall(lfs.dir, dir)
-    if ok then
-        for f in iter, dir_obj do
-            if util.splitFileNameSuffix(f) == "cover" then
-                return dir .. "/" .. f
-            end
-        end
-    end
-end
-
-function DocSettings:getCoverFile(reset_cache)
-    if reset_cache then
-        self.cover_file = nil
-    else
-        if self.cover_file == nil then
-            self.cover_file = DocSettings:findCoverFile(self.data.doc_path) or false
-        end
-        return self.cover_file
-    end
-end
-
 --- Opens a document's individual settings (font, margin, dictionary, etc.)
 -- @string doc_path path to the document (e.g., `/foo/bar.pdf`)
 -- @treturn DocSettings object
@@ -188,39 +222,48 @@ function DocSettings:open(doc_path)
     -- NOTE: Beware, our new instance is new, but self is still DocSettings!
     local new = DocSettings:extend{}
 
+    new.sidecar_filename = DocSettings.getSidecarFilename(doc_path)
+
     new.doc_sidecar_dir = new:getSidecarDir(doc_path, "doc")
-    new.doc_sidecar_file = new:getSidecarFile(doc_path, "doc")
     local doc_sidecar_file, legacy_sidecar_file
-    if lfs.attributes(new.doc_sidecar_dir, "mode") == "directory" then
-        doc_sidecar_file = new.doc_sidecar_file
-        legacy_sidecar_file = new.doc_sidecar_dir.."/"..ffiutil.basename(doc_path)..".lua"
+    if isDir(new.doc_sidecar_dir) then
+        doc_sidecar_file = new.doc_sidecar_dir .. "/" .. new.sidecar_filename
+        legacy_sidecar_file = new.doc_sidecar_dir .. "/" .. ffiutil.basename(doc_path) .. ".lua"
     end
     new.dir_sidecar_dir = new:getSidecarDir(doc_path, "dir")
-    new.dir_sidecar_file = new:getSidecarFile(doc_path, "dir")
     local dir_sidecar_file
-    if lfs.attributes(new.dir_sidecar_dir, "mode") == "directory" then
-        dir_sidecar_file = new.dir_sidecar_file
+    if isDir(new.dir_sidecar_dir) then
+        dir_sidecar_file = new.dir_sidecar_dir .. "/" .. new.sidecar_filename
+    end
+    local hash_sidecar_file
+    if DocSettings.isHashLocationEnabled() then
+        new.hash_sidecar_dir = new:getSidecarDir(doc_path, "hash")
+        hash_sidecar_file = new.hash_sidecar_dir .. "/" .. new.sidecar_filename
     end
-    local history_file = new:getHistoryPath(doc_path)
+    local history_file = is_history_location_enabled and new:getHistoryPath(doc_path)
 
     -- Candidates list, in order of priority:
     local candidates_list = {
         -- New sidecar file in doc folder
         doc_sidecar_file or "",
         -- Backup file of new sidecar file in doc folder
-        doc_sidecar_file and (doc_sidecar_file..".old") or "",
+        doc_sidecar_file and (doc_sidecar_file .. ".old") or "",
         -- Legacy sidecar file
         legacy_sidecar_file or "",
         -- New sidecar file in docsettings folder
         dir_sidecar_file or "",
         -- Backup file of new sidecar file in docsettings folder
-        dir_sidecar_file and (dir_sidecar_file..".old") or "",
+        dir_sidecar_file and (dir_sidecar_file .. ".old") or "",
+        -- New sidecar file in hashdocsettings folder
+        hash_sidecar_file or "",
+        -- Backup file of new sidecar file in hashdocsettings folder
+        hash_sidecar_file and (hash_sidecar_file .. ".old") or "",
         -- Legacy history folder
-        history_file,
+        history_file or "",
         -- Backup file in legacy history folder
-        history_file..".old",
+        history_file and (history_file .. ".old") or "",
         -- Legacy kpdfview setting
-        doc_path..".kpdfview.lua",
+        doc_path .. ".kpdfview.lua",
     }
     -- We get back an array of tables for *existing* candidates, sorted MRU first (insertion order breaks ties).
     local candidates = buildCandidates(candidates_list)
@@ -252,55 +295,69 @@ function DocSettings:open(doc_path)
     return new
 end
 
+--- Light version of open(). Opens a sidecar file or a custom metadata file.
+-- Returned object cannot be used to save changes to the sidecar file (flush()).
+-- Must be used to save changes to the custom metadata file (flushCustomMetadata()).
+function DocSettings.openSettingsFile(sidecar_file)
+    local new = DocSettings:extend{}
+    local ok, stored
+    if sidecar_file then
+        ok, stored = pcall(dofile, sidecar_file)
+    end
+    if ok and next(stored) ~= nil then
+        new.data = stored
+    else
+        new.data = {}
+    end
+    new.sidecar_file = sidecar_file
+    return new
+end
+
 --- Serializes settings and writes them to `metadata.lua`.
-function DocSettings:flush(data, no_cover)
-    -- Depending on the settings, doc_settings are saved to the book folder or
-    -- to koreader/docsettings folder. The latter is also a fallback for read-only book storage.
-    local serials = G_reader_settings:readSetting("document_metadata_folder", "doc") == "doc"
-        and { {self.doc_sidecar_dir, self.doc_sidecar_file},
-              {self.dir_sidecar_dir, self.dir_sidecar_file}, }
-         or { {self.dir_sidecar_dir, self.dir_sidecar_file}, }
-
-    local s_out = dump(data or self.data, nil, true)
-    for _, s in ipairs(serials) do
-        local sidecar_dir, sidecar_file = unpack(s)
-        util.makePath(sidecar_dir)
-        local directory_updated = false
-        if lfs.attributes(sidecar_file, "mode") == "file" then
-            -- As an additional safety measure (to the ffiutil.fsync* calls used below),
-            -- we only backup the file to .old when it has not been modified in the last 60 seconds.
-            -- This should ensure in the case the fsync calls are not supported
-            -- that the OS may have itself sync'ed that file content in the meantime.
-            local mtime = lfs.attributes(sidecar_file, "modification")
-            if mtime < os.time() - 60 then
-                logger.dbg("DocSettings: Renamed", sidecar_file, "to", sidecar_file .. ".old")
-                os.rename(sidecar_file, sidecar_file .. ".old")
-                directory_updated = true -- fsync directory content too below
-            end
+function DocSettings:flush(data, no_custom_metadata)
+    data = data or self.data
+    local sidecar_dirs
+    local preferred_location = G_reader_settings:readSetting("document_metadata_folder", "doc")
+    if preferred_location == "doc" then
+        sidecar_dirs = { self.doc_sidecar_dir,  self.dir_sidecar_dir } -- fallback for read-only book storage
+    elseif preferred_location == "dir" then
+        sidecar_dirs = { self.dir_sidecar_dir }
+    elseif preferred_location == "hash" then
+        if self.hash_sidecar_dir == nil then
+            self.hash_sidecar_dir = self:getSidecarDir(data.doc_path, "hash")
         end
-        logger.dbg("DocSettings: Writing to", sidecar_file)
-        local f_out = io.open(sidecar_file, "w")
-        if f_out ~= nil then
-            f_out:write("-- we can read Lua syntax here!\nreturn ")
-            f_out:write(s_out)
-            f_out:write("\n")
-            ffiutil.fsyncOpenedFile(f_out) -- force flush to the storage device
-            f_out:close()
-
-            if directory_updated then
-                -- Ensure the file renaming is flushed to storage device
-                ffiutil.fsyncDirectory(sidecar_file)
-            end
+        sidecar_dirs = { self.hash_sidecar_dir }
+    end
 
-            -- move cover file to the metadata file location
-            if not no_cover then
-                local cover_file = self:getCoverFile()
-                if cover_file then
-                    local filepath, filename = util.splitFilePathName(cover_file)
-                    if filepath ~= sidecar_dir .. "/" then
-                        ffiutil.copyFile(cover_file, sidecar_dir .. "/" .. filename)
-                        os.remove(cover_file)
-                        self:getCoverFile(true) -- reset cache
+    local ser_data = dump(data, nil, true)
+    for _, sidecar_dir in ipairs(sidecar_dirs) do
+        local sidecar_dir_slash = sidecar_dir .. "/"
+        local sidecar_file = sidecar_dir_slash .. self.sidecar_filename
+        util.makePath(sidecar_dir)
+        logger.dbg("DocSettings: Writing to", sidecar_file)
+        local directory_updated = LuaSettings:backup(sidecar_file) -- "*.old"
+        if util.writeToFile(ser_data, sidecar_file, true, true, directory_updated) then
+            -- move custom cover file and custom metadata file to the metadata file location
+            if not no_custom_metadata then
+                local metadata_file, filepath, filename
+                -- custom cover
+                metadata_file = self:getCustomCoverFile()
+                if metadata_file then
+                    filepath, filename = util.splitFilePathName(metadata_file)
+                    if filepath ~= sidecar_dir_slash then
+                        ffiutil.copyFile(metadata_file, sidecar_dir_slash .. filename)
+                        os.remove(metadata_file)
+                        self:getCustomCoverFile(true) -- reset cache
+                    end
+                end
+                -- custom metadata
+                metadata_file = self:getCustomMetadataFile()
+                if metadata_file then
+                    filepath, filename = util.splitFilePathName(metadata_file)
+                    if filepath ~= sidecar_dir_slash then
+                        ffiutil.copyFile(metadata_file, sidecar_dir_slash .. filename)
+                        os.remove(metadata_file)
+                        self:getCustomMetadataFile(true) -- reset cache
                     end
                 end
             end
@@ -313,14 +370,27 @@ function DocSettings:flush(data, no_cover)
 end
 
 --- Purges (removes) sidecar directory.
-function DocSettings:purge(sidecar_to_keep)
+function DocSettings:purge(sidecar_to_keep, data_to_purge)
+    local custom_cover_file, custom_metadata_file
+    if sidecar_to_keep == nil then
+        custom_cover_file    = self:getCustomCoverFile()
+        custom_metadata_file = self:getCustomMetadataFile()
+    end
+    if data_to_purge == nil then -- purge all
+        data_to_purge = {
+            doc_settings         = true,
+            custom_cover_file    = custom_cover_file,
+            custom_metadata_file = custom_metadata_file,
+        }
+    end
+
     -- Remove any of the old ones we may consider as candidates in DocSettings:open()
-    if self.candidates then
+    if data_to_purge.doc_settings and self.candidates then
         for _, t in ipairs(self.candidates) do
             local candidate_path = t.path
-            if lfs.attributes(candidate_path, "mode") == "file" then
+            if isFile(candidate_path) then
                 if (not sidecar_to_keep)
-                        or (candidate_path ~= sidecar_to_keep and candidate_path ~= sidecar_to_keep..".old") then
+                        or (candidate_path ~= sidecar_to_keep and candidate_path ~= sidecar_to_keep .. ".old") then
                     os.remove(candidate_path)
                     logger.dbg("DocSettings: purged:", candidate_path)
                 end
@@ -328,73 +398,210 @@ function DocSettings:purge(sidecar_to_keep)
         end
     end
 
-    local custom_metadata_purged
-    if not sidecar_to_keep then
-        local cover_file = self:getCoverFile()
-        if cover_file then
-            os.remove(cover_file)
-            self:getCoverFile(true) -- reset cache
-            custom_metadata_purged = true
-        end
+    -- Remove custom
+    if data_to_purge.custom_cover_file then
+        os.remove(data_to_purge.custom_cover_file)
+        self:getCustomCoverFile(true) -- reset cache
     end
-    if lfs.attributes(self.doc_sidecar_dir, "mode") == "directory" then
-        os.remove(self.doc_sidecar_dir) -- keep parent folders
+    if data_to_purge.custom_metadata_file then
+        os.remove(data_to_purge.custom_metadata_file)
+        self:getCustomMetadataFile(true) -- reset cache
     end
-    if lfs.attributes(self.dir_sidecar_dir, "mode") == "directory" then
-        util.removePath(self.dir_sidecar_dir) -- remove empty parent folders
+
+    -- Remove empty sidecar dirs
+    if data_to_purge.doc_settings or data_to_purge.custom_cover_file or data_to_purge.custom_metadata_file then
+        for _, dir in ipairs({ self.doc_sidecar_dir, self.dir_sidecar_dir, self.hash_sidecar_dir }) do
+            DocSettings.removeSidecarDir(dir)
+        end
     end
 
-    return custom_metadata_purged
+    DocSettings.setIsHashLocationEnabled(nil) -- reset this in case last hash book is purged
 end
 
---- Removes empty sidecar dir.
-function DocSettings:removeSidecarDir(doc_path, sidecar_dir)
-    if sidecar_dir == self:getSidecarDir(doc_path, "doc") then
-        os.remove(sidecar_dir)
-    else
-        util.removePath(sidecar_dir)
+--- Removes sidecar dir iff empty.
+function DocSettings.removeSidecarDir(dir)
+    if dir and isDir(dir) then
+        if dir:match("^"..DOCSETTINGS_DIR) or dir:match("^"..DOCSETTINGS_HASH_DIR) then
+            util.removePath(dir) -- remove empty parent folders
+        else
+            os.remove(dir) -- keep parent folders
+        end
     end
 end
 
 --- Updates sdr location for file rename/copy/move/delete operations.
-function DocSettings:updateLocation(doc_path, new_doc_path, copy)
-    local doc_settings, new_sidecar_dir
-
-    -- update metadata
-    if self:hasSidecarFile(doc_path) then
-        doc_settings = DocSettings:open(doc_path)
-        if new_doc_path then
-            local new_doc_settings = DocSettings:open(new_doc_path)
-            -- save doc settings to the new location, no cover file yet
-            new_sidecar_dir = new_doc_settings:flush(doc_settings.data, true)
-        else
+function DocSettings.updateLocation(doc_path, new_doc_path, copy)
+    local has_sidecar_file = DocSettings:hasSidecarFile(doc_path)
+    local custom_cover_file = DocSettings:findCustomCoverFile(doc_path)
+    local custom_metadata_file = DocSettings:findCustomMetadataFile(doc_path)
+    if not (has_sidecar_file or custom_cover_file or custom_metadata_file) then return end
+
+    local doc_settings = DocSettings:open(doc_path)
+    local do_purge
+
+    if new_doc_path then -- copy/rename/move
+        if G_reader_settings:readSetting("document_metadata_folder") ~= "hash" then -- keep hash location unchanged
+            local new_sidecar_dir
+            if has_sidecar_file then
+                local new_doc_settings = DocSettings:open(new_doc_path)
+                doc_settings.data.doc_path = new_doc_path
+                new_sidecar_dir = new_doc_settings:flush(doc_settings.data, true) -- without custom
+            end
+            if not new_sidecar_dir then
+                new_sidecar_dir = DocSettings:getSidecarDir(new_doc_path)
+                util.makePath(new_sidecar_dir)
+            end
+            if custom_cover_file then
+                local _, filename = util.splitFilePathName(custom_cover_file)
+                ffiutil.copyFile(custom_cover_file, new_sidecar_dir .. "/" .. filename)
+            end
+            if custom_metadata_file then
+                ffiutil.copyFile(custom_metadata_file, new_sidecar_dir .. "/" .. custom_metadata_filename)
+            end
+            do_purge = not copy
+        end
+    else -- delete
+        if has_sidecar_file then
             local cache_file_path = doc_settings:readSetting("cache_file_path")
             if cache_file_path then
                 os.remove(cache_file_path)
             end
         end
+        do_purge = true
+    end
+
+    if do_purge then
+        doc_settings.custom_cover_file = custom_cover_file -- cache
+        doc_settings.custom_metadata_file = custom_metadata_file -- cache
+        doc_settings:purge()
     end
+end
 
-    -- update cover file
-    if not doc_settings then
-        doc_settings = DocSettings:open(doc_path)
+-- custom section
+
+function DocSettings:getCustomLocationCandidates(doc_path)
+    local sidecar_dir
+    local sidecar_file = self:findSidecarFile(doc_path, true) -- new locations only
+    if sidecar_file then -- book was opened, write custom metadata to its sidecar dir
+        sidecar_dir = util.splitFilePathName(sidecar_file):sub(1, -2)
+        return { sidecar_dir }
     end
-    local cover_file = doc_settings:getCoverFile()
-    if cover_file and new_doc_path then
-        if not new_sidecar_dir then
-            new_sidecar_dir = self:getSidecarDir(new_doc_path)
-            util.makePath(new_sidecar_dir)
+    -- new book, create sidecar dir in accordance with sdr location setting
+    local preferred_location = G_reader_settings:readSetting("document_metadata_folder", "doc")
+    if preferred_location ~= "hash" then
+        sidecar_dir = self:getSidecarDir(doc_path, "dir")
+        if preferred_location == "doc" then
+            local doc_sidecar_dir = self:getSidecarDir(doc_path, "doc")
+            return { doc_sidecar_dir, sidecar_dir } -- fallback for read-only book storage
         end
-        local _, filename = util.splitFilePathName(cover_file)
-        ffiutil.copyFile(cover_file, new_sidecar_dir .. "/" .. filename)
+    else -- "hash"
+        sidecar_dir = self:getSidecarDir(doc_path, "hash")
     end
+    return { sidecar_dir }
+end
 
-    if not copy then
-        doc_settings:purge()
+-- custom cover
+
+local function findCustomCoverFileInDir(dir)
+    local ok, iter, dir_obj = pcall(lfs.dir, dir)
+    if ok then
+        for f in iter, dir_obj do
+            if util.splitFileNameSuffix(f) == "cover" then
+                return dir .. "/" .. f
+            end
+        end
     end
-    if cover_file then
-        doc_settings:getCoverFile(true) -- reset cache
+end
+
+--- Returns path to book custom cover file if it exists, or nil.
+function DocSettings:findCustomCoverFile(doc_path)
+    doc_path = doc_path or self.data.doc_path
+    for _, location in ipairs(getOrderedLocationCandidates()) do
+        local sidecar_dir = self:getSidecarDir(doc_path, location)
+        local custom_cover_file = findCustomCoverFileInDir(sidecar_dir)
+        if custom_cover_file then
+            return custom_cover_file
+        end
+    end
+end
+
+function DocSettings:getCustomCoverFile(reset_cache)
+    if reset_cache then
+        self.custom_cover_file = nil
+    else
+        if self.custom_cover_file == nil then -- fill empty cache
+            self.custom_cover_file = self:findCustomCoverFile() or false
+        end
+        return self.custom_cover_file
+    end
+end
+
+function DocSettings:flushCustomCover(doc_path, image_file)
+    local sidecar_dirs = self:getCustomLocationCandidates(doc_path)
+    local new_cover_filename = "/cover." .. util.getFileNameSuffix(image_file):lower()
+    for _, sidecar_dir in ipairs(sidecar_dirs) do
+        util.makePath(sidecar_dir)
+        local new_cover_file = sidecar_dir .. new_cover_filename
+        if ffiutil.copyFile(image_file, new_cover_file) == nil then
+            return true
+        end
+    end
+end
+
+-- custom metadata
+
+--- Returns path to book custom metadata file if it exists, or nil.
+function DocSettings:findCustomMetadataFile(doc_path)
+    doc_path = doc_path or self.data.doc_path
+    for _, location in ipairs(getOrderedLocationCandidates()) do
+        local sidecar_dir = self:getSidecarDir(doc_path, location)
+        local custom_metadata_file = sidecar_dir .. "/" .. custom_metadata_filename
+        if isFile(custom_metadata_file) then
+            return custom_metadata_file
+        end
+    end
+end
+
+function DocSettings:getCustomMetadataFile(reset_cache)
+    if reset_cache then
+        self.custom_metadata_file = nil
+    else
+        if self.custom_metadata_file == nil then -- fill empty cache
+            self.custom_metadata_file = self:findCustomMetadataFile() or false
+        end
+        return self.custom_metadata_file
+    end
+end
+
+function DocSettings:flushCustomMetadata(doc_path)
+    local sidecar_dirs = self:getCustomLocationCandidates(doc_path)
+    local s_out = dump(self.data, nil, true)
+    for _, sidecar_dir in ipairs(sidecar_dirs) do
+        util.makePath(sidecar_dir)
+        local new_metadata_file = sidecar_dir .. "/" .. custom_metadata_filename
+        if util.writeToFile(s_out, new_metadata_file, true, true) then
+            return true
+        end
+    end
+end
+
+-- "hash" section
+
+-- Returns the list of pairs {sidecar_file, custom_metadata_file}.
+function DocSettings.findSidecarFilesInHashLocation()
+    local res = {}
+    local callback = function(fullpath, name)
+        if name:match("metadata%..+%.lua$") then
+            local sdr = { fullpath }
+            local custom_metadata_file = fullpath:gsub(name, custom_metadata_filename)
+            if isFile(custom_metadata_file) then
+                table.insert(sdr, custom_metadata_file)
+            end
+            table.insert(res, sdr)
+        end
     end
+    util.findFiles(DOCSETTINGS_HASH_DIR, callback)
+    return res
 end
 
 return DocSettings

frontend/document/credocument.lua

@@ -29,11 +29,18 @@ local CreDocument = Document:extend{
     _smooth_scaling = false,
     _nightmode_images = true,
 
-    line_space_percent = 100,
     default_font = "Noto Serif",
     monospace_font = "Droid Sans Mono",
     header_font = "Noto Sans",
 
+    prop_to_cre_prop = { -- see cre lvtinydom.h
+        title        = "doc.title",
+        authors      = "doc.authors",
+        series       = "doc.series.name",
+        series_index = "doc.series.number",
+        identifiers  = "doc.identifiers",
+    },
+
     -- Reasons for the fallback font ordering:
     -- - Noto Sans CJK SC before FreeSans/Serif, as it has nice and larger
     --   symbol glyphs for Wikipedia EPUB headings than both Free fonts)
@@ -109,6 +116,15 @@ end
 function CreDocument:engineInit()
     if not engine_initialized then
         cre = require("libs/libkoreader-cre")
+
+        -- When forking to execute any stuff in a sub-process,
+        -- as that stuff may not care about properly closing
+        -- the document, skip cre.cpp finalizer to avoid any
+        -- assertion failure.
+        require("ffi/util").addRunInSubProcessAfterForkFunc("cre_skip_teardown", function()
+            cre.setSkipTearDown(true)
+        end)
+
         -- initialize cache
         self:cacheInit()
 
@@ -150,7 +166,7 @@ function CreDocument:init()
     self.flows = {}
     self.page_in_flow = {}
 
-    local file_type = string.lower(string.match(self.file, ".+%.([^.]+)"))
+    local file_type = string.lower(string.match(self.file, ".+%.([^.]+)") or "")
     if file_type == "zip" then
         -- NuPogodi, 20.05.12: read the content of zip-file
         -- and return extention of the 1st file
@@ -167,6 +183,7 @@ function CreDocument:init()
     self.default_css = "./data/epub.css"
     if file_type == "fb2" or file_type == "fb3" then
         self.default_css = "./data/fb2.css"
+        self.is_fb2 = true -- FB2 won't look good with any html-oriented stylesheet
     end
 
     -- This mode must be the same as the default one set as ReaderView.view_mode
@@ -213,6 +230,18 @@ function CreDocument:getDocumentFormat()
     return self._document:getDocumentFormat()
 end
 
+function CreDocument:getDocumentProps()
+    return self._document:getDocumentProps()
+end
+
+function CreDocument:setAltDocumentProp(prop, value)
+    logger.dbg("CreDocument: set alt document prop", prop, value)
+    if type(value) == "number" then -- series index
+        value = tostring(value)
+    end
+    self._document:setAltDocumentProp(self.prop_to_cre_prop[prop], value)
+end
+
 function CreDocument:setupDefaultView()
     if self.loaded then
         -- Don't apply defaults if the document has already been loaded
@@ -1349,6 +1378,11 @@ function CreDocument:setBatteryState(state)
     self._document:setBatteryState(state)
 end
 
+function CreDocument:setPageInfoOverride(pageinfo)
+    logger.dbg("CreDocument: set page info", pageinfo)
+    self._document:setPageInfoOverride(pageinfo)
+end
+
 function CreDocument:isXPointerInCurrentPage(xp)
     logger.dbg("CreDocument: check xpointer in current page", xp)
     return self._document:isXPointerInCurrentPage(xp)
@@ -1380,10 +1414,14 @@ function CreDocument:getAndClearRegexSearchError()
     return retval
 end
 
-function CreDocument:findText(pattern, origin, reverse, caseInsensitive, page, regex, max_hits)
-    logger.dbg("CreDocument: find text", pattern, origin, reverse, caseInsensitive, regex, max_hits)
-    return self._document:findText(
-        pattern, origin, reverse, caseInsensitive and 1 or 0, regex and 1 or 0, max_hits or 200)
+function CreDocument:findText(pattern, origin, direction, case_insensitive, page, regex, max_hits)
+    logger.dbg("CreDocument: find text", pattern, origin, direction == 1, case_insensitive, regex, max_hits)
+    return self._document:findText(pattern, origin, direction == 1, case_insensitive, regex, max_hits)
+end
+
+function CreDocument:findAllText(pattern, case_insensitive, nb_context_words, max_hits, regex)
+    logger.dbg("CreDocument: find all text", pattern, case_insensitive, regex, max_hits, true, nb_context_words)
+    return self._document:findAllText(pattern, case_insensitive, regex, max_hits, true, nb_context_words)
 end
 
 function CreDocument:enableInternalHistory(toggle)

frontend/document/djvudocument.lua

@@ -51,23 +51,6 @@ function DjvuDocument:updateColorRendering()
     end
 end
 
-function DjvuDocument:getProps()
-    local props = self._document:getMetadata()
-    local _, _, docname = self.file:find(".*/(.*)")
-    docname = docname or self.file
-
-    -- According to djvused(1), the convention is that
-    -- BibTex keys are always lowercase and DocInfo capitalized
-    props.title = props.title or props.Title or docname:match("(.*)%.")
-    props.authors = props.author or props.Author
-    props.series = props.series or props.Series
-    props.language = props.language or props.Language
-    props.keywords = props.keywords or props.Keywords
-    props.description = props.description or props.Description
-
-    return props
-end
-
 function DjvuDocument:comparePositions(pos1, pos2)
     return self.koptinterface:comparePositions(self, pos1, pos2)
 end
@@ -140,8 +123,12 @@ function DjvuDocument:getCoverPageImage()
     return self.koptinterface:getCoverPageImage(self)
 end
 
-function DjvuDocument:findText(pattern, origin, reverse, caseInsensitive, page)
-    return self.koptinterface:findText(self, pattern, origin, reverse, caseInsensitive, page)
+function DjvuDocument:findText(pattern, origin, reverse, case_insensitive, page)
+    return self.koptinterface:findText(self, pattern, origin, reverse, case_insensitive, page)
+end
+
+function DjvuDocument:findAllText(pattern, case_insensitive, nb_context_words, max_hits)
+    return self.koptinterface:findAllText(self, pattern, case_insensitive, nb_context_words, max_hits)
 end
 
 function DjvuDocument:renderPage(pageno, rect, zoom, rotation, gamma, render_mode, hinting)

frontend/document/doccache.lua

@@ -38,7 +38,7 @@ local function computeCacheSlots()
 
     --- ...otherwise, effectively disable the cache by making it single slot...
     if mb_size < 8 then
-        logger.dbg(string.format("Setting up a minimal single slot global document cache"))
+        logger.dbg("Setting up a minimal single slot global document cache")
         return 1
     else
         return nil

frontend/document/document.lua

@@ -40,6 +40,9 @@ local Document = {
     -- bb type needed by engine for color rendering
     color_bb_type = Blitbuffer.TYPE_BBRGB32,
 
+    -- image content stats, if supported by the engine
+    _drawn_images_count = nil,
+    _drawn_images_surface_ratio = nil,
 }
 
 function Document:extend(subclass_prototype)
@@ -92,6 +95,10 @@ function Document:_init()
     self.hw_dithering = false
     -- Whether SW dithering is enabled
     self.sw_dithering = false
+
+    -- Zero-init those to be able to drop the nil guards at runtime
+    self._drawn_images_count = 0
+    self._drawn_images_surface_ratio = 0
 end
 
 -- override this method to open a document
@@ -143,58 +150,9 @@ function Document:discardChange()
     self.is_edited = false
 end
 
--- calculate partial digest of the document and store in its docsettings to avoid document saving
--- feature to change its checksum.
---
--- To the calculating mechanism itself.
--- since only PDF documents could be modified by KOReader by appending data
--- at the end of the files when highlighting, we use a non-even sampling
--- algorithm which samples with larger weight at file head and much smaller
--- weight at file tail, thus reduces the probability that appended data may change
--- the digest value.
--- Note that if PDF file size is around 1024, 4096, 16384, 65536, 262144
--- 1048576, 4194304, 16777216, 67108864, 268435456 or 1073741824, appending data
--- by highlighting in KOReader may change the digest value.
-function Document:fastDigest(docsettings)
-    if not self.file then return end
-    local file = io.open(self.file, 'rb')
-    if file then
-        local tmp_docsettings = false
-        if not docsettings then -- if not provided, open/create it
-            docsettings = require("docsettings"):open(self.file)
-            tmp_docsettings = true
-        end
-        local result = docsettings:readSetting("partial_md5_checksum")
-        if not result then
-            logger.dbg("computing and storing partial_md5_checksum")
-            local bit = require("bit")
-            local md5 = require("ffi/sha2").md5
-            local lshift = bit.lshift
-            local step, size = 1024, 1024
-            local update = md5()
-            for i = -1, 10 do
-                file:seek("set", lshift(step, 2*i))
-                local sample = file:read(size)
-                if sample then
-                    update(sample)
-                else
-                    break
-                end
-            end
-            result = update()
-            docsettings:saveSetting("partial_md5_checksum", result)
-        end
-        if tmp_docsettings then
-            docsettings:close()
-        end
-        file:close()
-        return result
-    end
-end
-
 -- this might be overridden by a document implementation
 function Document:getNativePageDimensions(pageno)
-    local hash = "pgdim|"..self.file.."|"..pageno
+    local hash = "pgdim|"..self.file.."|"..self.mod_time.."|"..pageno
     local cached = DocCache:check(hash)
     if cached then
         return cached[1]
@@ -207,8 +165,53 @@ function Document:getNativePageDimensions(pageno)
     return page_size
 end
 
-function Document:getProps()
-    return self._document:getDocumentProps()
+function Document:getDocumentProps()
+    -- pdfdocument, djvudocument
+    return self._document:getMetadata()
+    -- credocument, picdocument - overridden by a document implementation
+end
+
+function Document:getProps(cached_doc_metadata)
+    local function makeNilIfEmpty(str)
+        if str == "" then
+            return nil
+        end
+        return str
+    end
+    local props = cached_doc_metadata or self:getDocumentProps()
+    local title = makeNilIfEmpty(props.title or props.Title)
+    local authors = makeNilIfEmpty(props.authors or props.author or props.Author)
+    local series = makeNilIfEmpty(props.series or props.Series)
+    local series_index
+    if series and string.find(series, "#") then
+        -- If there's a series index in there, split it off to series_index, and only store the name in series.
+        -- This property is currently only set by:
+        --   * DjVu, for which I couldn't find a real standard for metadata fields
+        --     (we currently use Series for this field, c.f., https://exiftool.org/TagNames/DjVu.html).
+        --   * CRe, which could offer us a split getSeriesName & getSeriesNumber...
+        --     except getSeriesNumber does an atoi, so it'd murder decimal values.
+        --     So, instead, parse how it formats the whole thing as a string ;).
+        local series_name
+        series_name, series_index = series:match("(.*) #(%d+%.?%d-)$")
+        if series_index then
+            series = series_name
+            series_index = tonumber(series_index)
+        end
+    end
+    local language = makeNilIfEmpty(props.language or props.Language)
+    local keywords = makeNilIfEmpty(props.keywords or props.Keywords)
+    local description = makeNilIfEmpty(props.description or props.Description or props.subject)
+    local identifiers = makeNilIfEmpty(props.identifiers)
+    return {
+        title        = title,
+        authors      = authors,
+        series       = series,
+        series_index = series_index,
+        language     = language,
+        keywords     = keywords,
+        description  = description,
+        identifiers  = identifiers,
+    }
 end
 
 function Document:_readMetadata()
@@ -234,13 +237,13 @@ end
 
 function Document:getNextPage(page)
     local new_page = page + 1
-    return (new_page > 0 and new_page < self.info.number_of_pages) and new_page or 0
+    return (new_page > 0 and new_page <= self.info.number_of_pages) and new_page or 0
 end
 
 function Document:getPrevPage(page)
     if page == 0 then return self.info.number_of_pages end
     local new_page = page - 1
-    return (new_page > 0 and new_page < self.info.number_of_pages) and new_page or 0
+    return (new_page > 0 and new_page <= self.info.number_of_pages) and new_page or 0
 end
 
 function Document:getTotalPagesLeft(page)
@@ -372,6 +375,10 @@ function Document:findText()
     return nil
 end
 
+function Document:findAllText()
+    return nil
+end
+
 function Document:updateColorRendering()
     if self.is_color_capable and CanvasContext.is_color_rendering_enabled then
         self.render_color = true
@@ -528,6 +535,7 @@ end
 
 function Document:getDrawnImagesStatistics()
     -- For now, only set by CreDocument in CreDocument:drawCurrentView()
+    -- Returns 0, 0 (as per Document:init) otherwise.
     return self._drawn_images_count, self._drawn_images_surface_ratio
 end
 

frontend/document/documentregistry.lua

@@ -4,12 +4,12 @@ This is a registry for document providers
 
 local DocSettings = require("docsettings")
 local logger = require("logger")
-local lfs = require("libs/libkoreader-lfs")
 local util = require("util")
 
 local DocumentRegistry = {
     registry = {},
     providers = {},
+    known_providers = {}, -- hash table of registered providers { provider_key = provider }
     filetype_provider = {},
     mimetype_ext = {},
     image_ext = {
@@ -24,6 +24,10 @@ local DocumentRegistry = {
     },
 }
 
+local function getSuffix(file)
+    return util.getFileNameSuffix(file):lower()
+end
+
 function DocumentRegistry:addProvider(extension, mimetype, provider, weight)
     extension = string.lower(extension)
     table.insert(self.providers, {
@@ -37,97 +41,66 @@ function DocumentRegistry:addProvider(extension, mimetype, provider, weight)
     -- Provided we order the calls to addProvider() correctly,
     -- that means epub instead of epub3, etc.
     self.mimetype_ext[mimetype] = self.mimetype_ext[mimetype] or extension
+    if self.known_providers[provider.provider] == nil then
+        self.known_providers[provider.provider] = provider
+    end
 end
 
-function DocumentRegistry:getRandomFile(dir, opened, extension)
-    if dir:sub(-1) ~= "/" then
-        dir = dir .. "/"
-    end
-    local files = {}
-    local i = 0
-    local ok, iter, dir_obj = pcall(lfs.dir, dir)
-    if ok then
-        for entry in iter, dir_obj do
-            local file = dir .. entry
-            local file_opened = DocSettings:hasSidecarFile(file) and true or false
-            if lfs.attributes(file, "mode") == "file" and self:hasProvider(file)
-                and (opened == nil or file_opened == opened)
-                and (extension == nil or extension[util.getFileNameSuffix(entry)]) then
-                i = i + 1
-                files[i] = entry
-            end
-        end
-        if i == 0 then
-            return nil
-        end
-    else
-        return nil
-    end
-    math.randomseed(os.time())
-    return dir .. files[math.random(i)]
+-- Register an auxiliary (non-document) provider.
+-- Aux providers are modules (eg TextViewer) or plugins (eg TextEditor).
+-- It does not implement the Document API.
+-- For plugins the hash table value does not contain file handler,
+-- but only a provider_key (provider.provider) to call the corresponding
+-- plugin in FileManager:openFile().
+function DocumentRegistry:addAuxProvider(provider)
+    self.known_providers[provider.provider] = provider
 end
 
 --- Returns true if file has provider.
 -- @string file
+-- @bool include_aux include auxiliary (non-document) providers
 -- @treturn boolean
-function DocumentRegistry:hasProvider(file, mimetype)
+function DocumentRegistry:hasProvider(file, mimetype, include_aux)
     if mimetype and self.mimetype_ext[mimetype] then
         return true
     end
     if not file then return false end
 
-    local filename_suffix = string.lower(util.getFileNameSuffix(file))
-
-    local filetype_provider = G_reader_settings:readSetting("provider") or {}
-    if self.filetype_provider[filename_suffix] or filetype_provider[filename_suffix] then
+    -- registered document provider
+    local filename_suffix = getSuffix(file)
+    if self.filetype_provider[filename_suffix] then
         return true
     end
+    -- associated document or auxiliary provider for file type
+    local filetype_provider_key = G_reader_settings:readSetting("provider", {})[filename_suffix]
+    local provider = filetype_provider_key and self.known_providers[filetype_provider_key]
+    if provider and (not provider.order or include_aux) then -- excluding auxiliary by default
+        return true
+    end
+    -- associated document provider for this file
     if DocSettings:hasSidecarFile(file) then
         return DocSettings:open(file):has("provider")
     end
     return false
 end
 
---- Returns the preferred registered document handler.
+--- Returns the preferred registered document handler or fallback provider.
 -- @string file
--- @treturn table provider, or nil
-function DocumentRegistry:getProvider(file)
+-- @bool include_aux include auxiliary (non-document) providers
+-- @treturn table provider
+function DocumentRegistry:getProvider(file, include_aux)
     local providers = self:getProviders(file)
-
-    if providers then
-        -- provider for document
-        if DocSettings:hasSidecarFile(file) then
-            local doc_settings_provider = DocSettings:open(file):readSetting("provider")
-            if doc_settings_provider then
-                for _, provider in ipairs(providers) do
-                    if provider.provider.provider == doc_settings_provider then
-                        return provider.provider
-                    end
-                end
-            end
+    if providers or include_aux then
+        -- associated provider
+        local provider_key = DocumentRegistry:getAssociatedProviderKey(file)
+        local provider = provider_key and self.known_providers[provider_key]
+        if provider and (not provider.order or include_aux) then -- excluding auxiliary by default
+            return provider
         end
-
-        -- global provider for filetype
-        local filename_suffix = util.getFileNameSuffix(file)
-        local g_settings_provider = G_reader_settings:readSetting("provider")
-
-        if g_settings_provider and g_settings_provider[filename_suffix] then
-            for _, provider in ipairs(providers) do
-                if provider.provider.provider == g_settings_provider[filename_suffix] then
-                    return provider.provider
-                end
-            end
-        end
-
         -- highest weighted provider
-        return providers[1].provider
-    else
-        for _, provider in ipairs(self.providers) do
-            if provider.extension == "txt" then
-                return provider.provider
-            end
-        end
+        return providers and providers[1].provider
     end
+    return self:getFallbackProvider()
 end
 
 --- Returns the registered document handlers.
@@ -166,6 +139,59 @@ function DocumentRegistry:getProviders(file)
     end
 end
 
+function DocumentRegistry:getProviderFromKey(provider_key)
+    return self.known_providers[provider_key]
+end
+
+function DocumentRegistry:getFallbackProvider()
+    for _, provider in ipairs(self.providers) do
+        if provider.extension == "txt" then
+            return provider.provider
+        end
+    end
+end
+
+function DocumentRegistry:getAssociatedProviderKey(file, all)
+    -- all: nil - first not empty, false - this file, true - file type
+
+    if not file then -- get the full list of associated providers
+        return G_reader_settings:readSetting("provider")
+    end
+
+    -- provider for this file
+    local provider_key
+    if all ~= true then
+        if DocSettings:hasSidecarFile(file) then
+            provider_key = DocSettings:open(file):readSetting("provider")
+            if provider_key or all == false then
+                return provider_key
+            end
+        end
+        if all == false then return end
+    end
+
+    -- provider for file type
+    local providers = G_reader_settings:readSetting("provider")
+    provider_key = providers and providers[getSuffix(file)]
+    if provider_key and self.known_providers[provider_key] then
+        return provider_key
+    end
+end
+
+-- Returns array: registered auxiliary providers sorted by order.
+function DocumentRegistry:getAuxProviders()
+    local providers = {}
+    for _, provider in pairs(self.known_providers) do
+        if provider.order then -- aux
+            table.insert(providers, provider)
+        end
+    end
+    if #providers >= 1 then
+        table.sort(providers, function(a, b) return a.order < b.order end)
+        return providers
+    end
+end
+
 --- Get mapping of file extensions to providers
 -- @treturn table mapping file extensions to a list of providers
 function DocumentRegistry:getExtensions()
@@ -182,8 +208,7 @@ end
 -- @string file
 -- @bool all
 function DocumentRegistry:setProvider(file, provider, all)
-    local _, filename_suffix = util.splitFileNameSuffix(file)
-
+    provider = provider or {} -- call with nil to reset
     -- per-document
     if not all then
         local doc_settings = DocSettings:open(file)
@@ -191,9 +216,8 @@ function DocumentRegistry:setProvider(file, provider, all)
         doc_settings:flush()
     -- global
     else
-        local filetype_provider = G_reader_settings:readSetting("provider") or {}
-        filetype_provider[filename_suffix] = provider.provider
-        G_reader_settings:saveSetting("provider", filetype_provider)
+        local filetype_provider = G_reader_settings:readSetting("provider", {})
+        filetype_provider[getSuffix(file)] = provider.provider
     end
 end
 
@@ -257,7 +281,7 @@ function DocumentRegistry:getReferenceCount(file)
 end
 
 function DocumentRegistry:isImageFile(file)
-    return self.image_ext[util.getFileNameSuffix(file):lower()] and true or false
+    return self.image_ext[getSuffix(file)] and true or false
 end
 
 -- load implementations:
@@ -265,5 +289,9 @@ require("document/credocument"):register(DocumentRegistry)
 require("document/pdfdocument"):register(DocumentRegistry)
 require("document/djvudocument"):register(DocumentRegistry)
 require("document/picdocument"):register(DocumentRegistry)
+-- auxiliary built-in
+require("ui/widget/imageviewer"):register(DocumentRegistry)
+require("ui/widget/textviewer"):register(DocumentRegistry)
+-- auxiliary from plugins
 
 return DocumentRegistry

frontend/document/koptinterface.lua

@@ -12,6 +12,7 @@ local FFIUtil = require("ffi/util")
 local Geom = require("ui/geometry")
 local KOPTContext = require("ffi/koptcontext")
 local Persist = require("persist")
+local TextBoxWidget = require("ui/widget/textboxwidget")
 local TileCacheItem = require("document/tilecacheitem")
 local Utf8Proc = require("ffi/utf8proc")
 local logger = require("logger")
@@ -19,7 +20,10 @@ local util = require("util")
 
 local KoptInterface = {
     ocrengine = "ocrengine",
-    tessocr_data = DataStorage:getDataDir() .. "/data",
+    -- If `$TESSDATA_PREFIX` is set, don't override it: let libk2pdfopt honor it
+    -- (which includes checking for data in both `$TESSDATA_PREFIX/tessdata` and
+    -- in `$TESSDATA_PREFIX/` on more recent versions).
+    tessocr_data = not os.getenv('TESSDATA_PREFIX') and DataStorage:getDataDir().."/data/tessdata" or nil,
     ocr_lang = "eng",
     ocr_type = 3, -- default 0, for more accuracy use 3
     last_context_size = nil,
@@ -144,7 +148,8 @@ function KoptInterface:createContext(doc, pageno, bbox)
     kc:setZoom(doc.configurable.font_size)
     kc:setMargin(doc.configurable.page_margin)
     kc:setQuality(doc.configurable.quality)
-    kc:setContrast(doc.configurable.contrast)
+    -- k2pdfopt (for reflowing) and mupdf use different algorithms to apply gamma when rendering
+    kc:setContrast(1 / doc.configurable.contrast)
     kc:setDefectSize(doc.configurable.defect_size)
     kc:setLineSpacing(doc.configurable.line_spacing)
     kc:setWordSpacing(doc.configurable.word_spacing)
@@ -1062,6 +1067,10 @@ function KoptInterface:getWordFromReflowPosition(doc, boxes, pos)
     local pageno = pos.page
 
     local scratch_reflowed_page_boxes = self:getReflowedTextBoxesFromScratch(doc, pageno)
+    if not DEBUG.dassert(scratch_reflowed_page_boxes and next(scratch_reflowed_page_boxes) ~= nil, "scratch_reflowed_page_boxes shouldn't be nil/{}") then
+        return
+    end
+
     local scratch_reflowed_word_box = self:getWordFromBoxes(scratch_reflowed_page_boxes, pos)
 
     local reflowed_page_boxes = self:getReflowedTextBoxes(doc, pageno)
@@ -1096,49 +1105,60 @@ function KoptInterface:getWordFromNativePosition(doc, boxes, pos)
     return word_box
 end
 
-function KoptInterface:getSelectedWordContext(word, nb_words, pos)
-    local boxes = self.last_text_boxes
-    if not pos or not boxes or #boxes == 0 then return end
-    local i, j = getWordBoxIndices(boxes, pos)
-    if boxes[i][j].word ~= word then return end
-
-    local li, wi = i, j
-    local prev_count, next_count = 0, 0
-    local prev_text, next_text = {}, {}
+local function get_prev_text(boxes, i, j, nb_words)
+    local prev_count = 0
+    local prev_text = {}
     while prev_count < nb_words do
-        if li == 1 and wi == 1 then
+        if i == 1 and j == 1 then
             break
-        elseif wi == 1 then
-            li = li - 1
-            wi = #boxes[li]
+        elseif j == 1 then
+            i = i - 1
+            j = #boxes[i]
         else
-            wi = wi - 1
+            j = j - 1
         end
-        local current_word = boxes[li][wi].word
+        local current_word = boxes[i][j].word
         if #current_word > 0 then
             table.insert(prev_text, 1, current_word)
             prev_count = prev_count + 1
         end
     end
+    if #prev_text > 0 then
+        return table.concat(prev_text, " ")
+    end
+end
 
-    li, wi = i, j
+local function get_next_text(boxes, i, j, nb_words)
+    local next_count = 0
+    local next_text = {}
     while next_count < nb_words do
-        if li == #boxes and wi == #boxes[li] then
+        if i == #boxes and j == #boxes[i] then
             break
-        elseif wi == #boxes[li] then
-            li = li + 1
-            wi = 1
+        elseif j == #boxes[i] then
+            i = i + 1
+            j = 1
         else
-            wi = wi + 1
+            j = j + 1
         end
-        local current_word = boxes[li][wi].word
+        local current_word = boxes[i][j].word
         if #current_word > 0 then
             table.insert(next_text, current_word)
             next_count = next_count + 1
         end
     end
-    if #prev_text == 0 and #next_text == 0 then return end
-    return table.concat(prev_text, " "), table.concat(next_text, " ")
+    if #next_text > 0 then
+        return table.concat(next_text, " ")
+    end
+end
+
+function KoptInterface:getSelectedWordContext(word, nb_words, pos)
+    local boxes = self.last_text_boxes
+    if not pos or not boxes or #boxes == 0 then return end
+    local i, j = getWordBoxIndices(boxes, pos)
+    if boxes[i][j].word ~= word then return end
+    local prev_text = get_prev_text(boxes, i, j, nb_words)
+    local next_text = get_next_text(boxes, i, j, nb_words)
+    return prev_text, next_text
 end
 
 --[[--
@@ -1224,8 +1244,15 @@ function KoptInterface:getTextFromReflowPositions(doc, native_boxes, pos0, pos1)
     local reflowed_page_boxes = self:getReflowedTextBoxes(doc, pageno)
 
     local scratch_reflowed_word_box0 = self:getWordFromBoxes(scratch_reflowed_page_boxes, pos0)
+    if not DEBUG.dassert(scratch_reflowed_word_box0 and next(scratch_reflowed_word_box0) ~= nil, "scratch_reflowed_word_box0 shouldn't be nil/{}") then
+        return
+    end
     local reflowed_word_box0 = self:getWordFromBoxes(reflowed_page_boxes, pos0)
+
     local scratch_reflowed_word_box1 = self:getWordFromBoxes(scratch_reflowed_page_boxes, pos1)
+    if not DEBUG.dassert(scratch_reflowed_word_box1 and next(scratch_reflowed_word_box1) ~= nil, "scratch_reflowed_word_box1 shouldn't be nil/{}") then
+        return
+    end
     local reflowed_word_box1 = self:getWordFromBoxes(reflowed_page_boxes, pos1)
 
     local reflowed_pos_abs0 = scratch_reflowed_word_box0.box:center()
@@ -1335,19 +1362,23 @@ function KoptInterface:nativeToPageRectTransform(doc, pageno, rect)
     end
 end
 
-local function all_matches(boxes, pattern, caseInsensitive)
+local function get_pattern_list(pattern, case_insensitive)
     -- pattern list of single words
     local plist = {}
     -- (as in util.splitToWords(), but only splitting on spaces, keeping punctuations)
     for word in util.gsplit(pattern, "%s+") do
         if util.hasCJKChar(word) then
             for char in util.gsplit(word, "[\192-\255][\128-\191]+", true) do
-                table.insert(plist, caseInsensitive and Utf8Proc.lowercase(util.fixUtf8(char, "?")) or char)
+                table.insert(plist, case_insensitive and Utf8Proc.lowercase(util.fixUtf8(char, "?")) or char)
             end
         else
-            table.insert(plist, caseInsensitive and Utf8Proc.lowercase(util.fixUtf8(word, "?")) or word)
+            table.insert(plist, case_insensitive and Utf8Proc.lowercase(util.fixUtf8(word, "?")) or word)
         end
     end
+    return plist
+end
+
+local function all_matches(boxes, plist, case_insensitive)
     local pnb = #plist
     -- return mached word indices from index i, j
     local function match(i, j)
@@ -1361,7 +1392,7 @@ local function all_matches(boxes, pattern, caseInsensitive)
             end
             if i > #boxes then break end
             local box = boxes[i][j]
-            local word = caseInsensitive and Utf8Proc.lowercase(util.fixUtf8(box.word, "?")) or box.word
+            local word = case_insensitive and Utf8Proc.lowercase(util.fixUtf8(box.word, "?")) or box.word
             local pword = plist[pindex]
             local matched
             if pnb == 1 then -- single word in plist
@@ -1406,11 +1437,12 @@ local function all_matches(boxes, pattern, caseInsensitive)
     end)
 end
 
-function KoptInterface:findAllMatches(doc, pattern, caseInsensitive, page)
+function KoptInterface:findAllMatches(doc, pattern, case_insensitive, page)
     local text_boxes = doc:getPageTextBoxes(page)
     if not text_boxes then return end
+    local plist = get_pattern_list(pattern, case_insensitive)
     local matches = {}
-    for indices in all_matches(text_boxes or {}, pattern, caseInsensitive) do
+    for indices in all_matches(text_boxes, plist, case_insensitive) do
         for _, index in ipairs(indices) do
             local i, j = unpack(index)
             local word = text_boxes[i][j]
@@ -1426,8 +1458,8 @@ function KoptInterface:findAllMatches(doc, pattern, caseInsensitive, page)
     return matches
 end
 
-function KoptInterface:findText(doc, pattern, origin, reverse, caseInsensitive, pageno)
-    logger.dbg("Koptinterface: find text", pattern, origin, reverse, caseInsensitive, pageno)
+function KoptInterface:findText(doc, pattern, origin, reverse, case_insensitive, pageno)
+    logger.dbg("Koptinterface: find text", pattern, origin, reverse, case_insensitive, pageno)
     local last_pageno = doc:getPageCount()
     local start_page, end_page
     if reverse == 1 then
@@ -1455,7 +1487,7 @@ function KoptInterface:findText(doc, pattern, origin, reverse, caseInsensitive,
         end
     end
     for i = start_page, end_page, (reverse == 1) and -1 or 1 do
-        local matches = self:findAllMatches(doc, pattern, caseInsensitive, i)
+        local matches = self:findAllMatches(doc, pattern, case_insensitive, i)
         if #matches > 0 then
             matches.page = i
             return matches
@@ -1463,6 +1495,61 @@ function KoptInterface:findText(doc, pattern, origin, reverse, caseInsensitive,
     end
 end
 
+function KoptInterface:findAllText(doc, pattern, case_insensitive, nb_context_words, max_hits)
+    local plist = get_pattern_list(pattern, case_insensitive)
+    local res = {}
+    for page = 1, doc:getPageCount() do
+        local text_boxes = doc:getPageTextBoxes(page)
+        if text_boxes then
+            for indices in all_matches(text_boxes, plist, case_insensitive) do -- each found pattern in the page
+                local res_item = { -- item of the Menu item_table
+                    text = nil,
+                    mandatory = page,
+                    boxes = {}, -- to draw temp highlight in onMenuSelect
+                }
+                local text = {}
+                local i_prev, j_prev, i_next, j_next
+                for ind, index in ipairs(indices) do -- each word in the pattern
+                    local i, j = unpack(index)
+                    local word = text_boxes[i][j]
+                    res_item.boxes[ind] = {
+                        x = word.x0, y = word.y0,
+                        w = word.x1 - word.x0,
+                        h = word.y1 - word.y0,
+                    }
+                    text[ind] = word.word
+                    if ind == 1 then
+                        i_prev, j_prev = i, j
+                    end
+                    if ind == #indices then
+                        i_next, j_next = i, j
+                    end
+                end
+                -- Make this word bolder, using Poor Text Formatting provided by TextBoxWidget
+                -- (we know this text ends up in a TextBoxWidget).
+                text = TextBoxWidget.PTF_BOLD_START .. table.concat(text, " ") .. TextBoxWidget.PTF_BOLD_END
+                local prev_text = get_prev_text(text_boxes, i_prev, j_prev, nb_context_words)
+                if prev_text then
+                    text = prev_text .. " " .. text
+                end
+                local next_text = get_next_text(text_boxes, i_next, j_next, nb_context_words)
+                if next_text then
+                    text = text .. " " .. next_text
+                end
+                text = TextBoxWidget.PTF_HEADER .. text -- enable handling of our bold tags
+                res_item.text = text
+                table.insert(res, res_item)
+                if #res == max_hits then
+                    return res
+                end
+            end
+        end
+    end
+    if #res > 0 then
+        return res
+    end
+end
+
 --[[--
 Log reflow duration.
 --]]

frontend/document/pdfdocument.lua

@@ -211,18 +211,18 @@ local function _quadpointsFromPboxes(pboxes)
     -- will also need mupdf_h.lua to be evaluated once
     -- but this is guaranteed at this point
     local n = #pboxes
-    local quadpoints = ffi.new("float[?]", 8*n)
+    local quadpoints = ffi.new("fz_quad[?]", n)
     for i=1, n do
         -- The order must be left bottom, right bottom, left top, right top.
         -- https://bugs.ghostscript.com/show_bug.cgi?id=695130
-        quadpoints[8*i-8] = pboxes[i].x
-        quadpoints[8*i-7] = pboxes[i].y + pboxes[i].h
-        quadpoints[8*i-6] = pboxes[i].x + pboxes[i].w
-        quadpoints[8*i-5] = pboxes[i].y + pboxes[i].h
-        quadpoints[8*i-4] = pboxes[i].x
-        quadpoints[8*i-3] = pboxes[i].y
-        quadpoints[8*i-2] = pboxes[i].x + pboxes[i].w
-        quadpoints[8*i-1] = pboxes[i].y
+        quadpoints[i-1].ll.x = pboxes[i].x
+        quadpoints[i-1].ll.y = pboxes[i].y + pboxes[i].h - 1
+        quadpoints[i-1].lr.x = pboxes[i].x + pboxes[i].w - 1
+        quadpoints[i-1].lr.y = pboxes[i].y + pboxes[i].h - 1
+        quadpoints[i-1].ul.x = pboxes[i].x
+        quadpoints[i-1].ul.y = pboxes[i].y
+        quadpoints[i-1].ur.x = pboxes[i].x + pboxes[i].w - 1
+        quadpoints[i-1].ur.y = pboxes[i].y
     end
     return quadpoints, n
 end
@@ -232,10 +232,10 @@ local function _quadpointsToPboxes(quadpoints, n)
     local pboxes = {}
     for i=1, n do
         table.insert(pboxes, {
-            x = quadpoints[8*i-4],
-            y = quadpoints[8*i-3],
-            w = quadpoints[8*i-6] - quadpoints[8*i-4],
-            h = quadpoints[8*i-5] - quadpoints[8*i-3],
+            x = quadpoints[i-1].ul.x,
+            y = quadpoints[i-1].ul.y,
+            w = quadpoints[i-1].lr.x - quadpoints[i-1].ul.x + 1,
+            h = quadpoints[i-1].lr.y - quadpoints[i-1].ul.y + 1,
         })
     end
     return pboxes
@@ -313,24 +313,6 @@ function PdfDocument:close()
     Document.close(self)
 end
 
-function PdfDocument:getProps()
-    local props = self._document:getMetadata()
-    if props.title == "" then
-        local startPos = util.lastIndexOf(self.file, "%/")
-        if startPos > 0  then
-            props.title = string.sub(self.file, startPos + 1, -5) --remove extension .pdf
-        else
-            props.title = string.sub(self.file, 0, -5)
-        end
-    end
-    props.authors = props.author
-    props.series = ""
-    props.language = ""
-    props.keywords = props.keywords
-    props.description = props.subject
-    return props
-end
-
 function PdfDocument:getLinkFromPosition(pageno, pos)
     return self.koptinterface:getLinkFromPosition(self, pageno, pos)
 end
@@ -355,8 +337,12 @@ function PdfDocument:getCoverPageImage()
     return self.koptinterface:getCoverPageImage(self)
 end
 
-function PdfDocument:findText(pattern, origin, reverse, caseInsensitive, page)
-    return self.koptinterface:findText(self, pattern, origin, reverse, caseInsensitive, page)
+function PdfDocument:findText(pattern, origin, reverse, case_insensitive, page)
+    return self.koptinterface:findText(self, pattern, origin, reverse, case_insensitive, page)
+end
+
+function PdfDocument:findAllText(pattern, case_insensitive, nb_context_words, max_hits)
+    return self.koptinterface:findAllText(self, pattern, case_insensitive, nb_context_words, max_hits)
 end
 
 function PdfDocument:renderPage(pageno, rect, zoom, rotation, gamma, render_mode, hinting)

frontend/document/picdocument.lua

@@ -42,12 +42,8 @@ function PicDocument:getUsedBBox(pageno)
     return { x0 = 0, y0 = 0, x1 = self._document.width, y1 = self._document.height }
 end
 
-function PicDocument:getProps()
-    local _, _, docname = self.file:find(".*/(.*)")
-    docname = docname or self.file
-    return {
-        title = docname:match("(.*)%."),
-    }
+function PicDocument:getDocumentProps()
+    return {}
 end
 
 function PicDocument:getCoverPageImage()

frontend/fontlist.lua

@@ -142,12 +142,10 @@ function FontList:_readList(dir, mark)
         -- See if we're interested
         if file:sub(1, 1) == "." then return end
         local file_type = file:lower():match(".+%.([^.]+)") or ""
-        if not font_exts[file_type] then return end
+        if not font_exts[file_type] or isInFontsBlacklist(file) then return end
 
         -- Add it to the list
-        if not isInFontsBlacklist(file) then
-            table.insert(self.fontlist, path)
-        end
+        table.insert(self.fontlist, path)
 
         -- And into cached info table
         mark[path] = true

frontend/gettext.lua

@@ -58,7 +58,7 @@ Returns a translation.
     local translation = _("A meaningful message.")
 --]]
 function GetText_mt.__call(gettext, msgid)
-    return gettext.translation[msgid] or gettext.wrapUntranslated(msgid)
+    return gettext.translation[msgid] and gettext.translation[msgid][0] or gettext.translation[msgid] or gettext.wrapUntranslated(msgid)
 end
 
 local function c_escape(what_full, what)
@@ -275,6 +275,8 @@ function GetText_mt.__index.changeLang(new_lang)
                     -- unescape \\ or msgid won't match
                     s = s:gsub("\\\\", "\\")
                     data[what] = (data[what] or "") .. s
+                elseif what and s == "" and fuzzy then -- luacheck: ignore 542
+                    -- Ignore the likes of msgid "" and msgstr ""
                 else
                     -- Don't save this fuzzy string and unset fuzzy for the next one.
                     fuzzy = false

frontend/languagesupport.lua

@@ -194,7 +194,7 @@ function LanguageSupport:improveWordSelection(selection)
         return
     end
 
-    local language_code = self.document:getProps().language or "unknown"
+    local language_code = self.ui.doc_props.language or "unknown"
     logger.dbg("language support: improving", language_code, "selection", selection)
 
     -- Rather than requiring each language plugin to use document: methods
@@ -250,7 +250,7 @@ end
 function LanguageSupport:extraDictionaryFormCandidates(text)
     if not self:hasActiveLanguagePlugins() then return end -- nothing to do
 
-    local language_code = self.document and self.document:getProps().language or "unknown"
+    local language_code = (self.ui.doc_props and self.ui.doc_props.language) or "unknown"
     logger.dbg("language support: convert", text, "to dictionary form (marked as", language_code..")")
 
     return self:_findAndCallPlugin(

frontend/luadefaults.lua

@@ -5,7 +5,6 @@ Subclass of LuaSettings dedicated to handling the legacy global constants.
 local DataStorage = require("datastorage")
 local LuaSettings = require("luasettings")
 local dump = require("dump")
-local ffiutil = require("ffi/util")
 local util = require("util")
 local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
@@ -153,30 +152,8 @@ function LuaDefaults:reset() end
 --- Writes settings to disk.
 function LuaDefaults:flush()
     if not self.file then return end
-    local directory_updated = false
-    if lfs.attributes(self.file, "mode") == "file" then
-        -- As an additional safety measure (to the ffiutil.fsync* calls used below),
-        -- we only backup the file to .old when it has not been modified in the last 60 seconds.
-        -- This should ensure in the case the fsync calls are not supported
-        -- that the OS may have itself sync'ed that file content in the meantime.
-        local mtime = lfs.attributes(self.file, "modification")
-        if mtime < os.time() - 60 then
-            os.rename(self.file, self.file .. ".old")
-            directory_updated = true -- fsync directory content too below
-        end
-    end
-    local f_out = io.open(self.file, "w")
-    if f_out ~= nil then
-        f_out:write("-- we can read Lua syntax here!\nreturn ")
-        f_out:write(dump(self.rw, nil, true))
-        f_out:write("\n")
-        ffiutil.fsyncOpenedFile(f_out) -- force flush to the storage device
-        f_out:close()
-    end
-    if directory_updated then
-        -- Ensure the file renaming is flushed to storage device
-        ffiutil.fsyncDirectory(self.file)
-    end
+    local directory_updated = self:backup() -- LuaSettings
+    util.writeToFile(dump(self.rw, nil, true), self.file, true, true, directory_updated)
     return self
 end
 

frontend/luasettings.lua

@@ -3,9 +3,9 @@ This module handles generic settings as well as KOReader's global settings syste
 ]]
 
 local dump = require("dump")
-local ffiutil = require("ffi/util")
 local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
+local util = require("util")
 
 local LuaSettings = {}
 
@@ -249,33 +249,28 @@ function LuaSettings:reset(table)
     return self
 end
 
---- Writes settings to disk.
-function LuaSettings:flush()
-    if not self.file then return end
-    local directory_updated = false
-    if lfs.attributes(self.file, "mode") == "file" then
-        -- As an additional safety measure (to the ffiutil.fsync* calls used below),
+function LuaSettings:backup(file)
+    file = file or self.file
+    local directory_updated
+    if lfs.attributes(file, "mode") == "file" then
+        -- As an additional safety measure (to the ffiutil.fsync* calls used in util.writeToFile),
         -- we only backup the file to .old when it has not been modified in the last 60 seconds.
         -- This should ensure in the case the fsync calls are not supported
         -- that the OS may have itself sync'ed that file content in the meantime.
-        local mtime = lfs.attributes(self.file, "modification")
+        local mtime = lfs.attributes(file, "modification")
         if mtime < os.time() - 60 then
-            os.rename(self.file, self.file .. ".old")
-            directory_updated = true -- fsync directory content too below
+            os.rename(file, file .. ".old")
+            directory_updated = true -- fsync directory content
         end
     end
-    local f_out = io.open(self.file, "w")
-    if f_out ~= nil then
-        f_out:write("-- we can read Lua syntax here!\nreturn ")
-        f_out:write(dump(self.data, nil, true))
-        f_out:write("\n")
-        ffiutil.fsyncOpenedFile(f_out) -- force flush to the storage device
-        f_out:close()
-    end
-    if directory_updated then
-        -- Ensure the file renaming is flushed to storage device
-        ffiutil.fsyncDirectory(self.file)
-    end
+    return directory_updated
+end
+
+--- Writes settings to disk.
+function LuaSettings:flush()
+    if not self.file then return end
+    local directory_updated = self:backup()
+    util.writeToFile(dump(self.data, nil, true), self.file, true, true, directory_updated)
     return self
 end
 

frontend/readcollection.lua

@@ -2,149 +2,325 @@ local DataStorage = require("datastorage")
 local FFIUtil = require("ffi/util")
 local LuaSettings = require("luasettings")
 local lfs = require("libs/libkoreader-lfs")
+local logger = require("logger")
 local util = require("util")
 
-local DEFAULT_COLLECTION_NAME = "favorites"
 local collection_file = DataStorage:getSettingsDir() .. "/collection.lua"
 
-local ReadCollection = {}
+local ReadCollection = {
+    coll = nil, -- hash table
+    coll_order = nil, -- hash table
+    last_read_time = 0,
+    default_collection_name = "favorites",
+}
 
-function ReadCollection:read(collection_name)
-    if not collection_name then collection_name = DEFAULT_COLLECTION_NAME end
+-- read, write
+
+local function buildEntry(file, order, mandatory)
+    file = FFIUtil.realpath(file)
+    if not file then return end
+    if not mandatory then -- new item
+        local attr = lfs.attributes(file)
+        if not attr or attr.mode ~= "file" then return end
+        mandatory = util.getFriendlySize(attr.size or 0)
+    end
+    return {
+        file = file,
+        text = file:gsub(".*/", ""),
+        mandatory = mandatory,
+        order = order,
+    }
+end
+
+function ReadCollection:_read()
+    local collection_file_modification_time = lfs.attributes(collection_file, "modification")
+    if collection_file_modification_time then
+        if collection_file_modification_time <= self.last_read_time then return end
+        self.last_read_time = collection_file_modification_time
+    end
     local collections = LuaSettings:open(collection_file)
-    local coll = collections:readSetting(collection_name) or {}
-    local coll_max_item = 0
-    for _, v in pairs(coll) do
-        if v.order > coll_max_item then
-            coll_max_item = v.order
+    if collections:hasNot(self.default_collection_name) then
+        collections:saveSetting(self.default_collection_name, {})
+    end
+    logger.dbg("ReadCollection: reading from collection file")
+    self.coll = {}
+    self.coll_order = {}
+    for coll_name, collection in pairs(collections.data) do
+        local coll = {}
+        for _, v in ipairs(collection) do
+            local item = buildEntry(v.file, v.order)
+            if item then -- exclude deleted files
+                coll[item.file] = item
+            end
         end
+        self.coll[coll_name] = coll
+        if not collection.settings then -- favorites, first run
+            collection.settings = { order = 1 }
+        end
+        self.coll_order[coll_name] = collection.settings.order
     end
-    return coll, coll_max_item
 end
 
-function ReadCollection:readAllCollection()
-    local collection = LuaSettings:open(collection_file)
-    if collection and collection.data then
-        return collection.data
-    else
-        return {}
-    end
-end
-
-function ReadCollection:prepareList(collection_name)
-    local data = self:read(collection_name)
-    local list = {}
-    for _, v in pairs(data) do
-        local file_path = FFIUtil.realpath(v.file) or v.file -- keep orig file path of deleted files
-        local file_exists = lfs.attributes(file_path, "mode") == "file"
-        table.insert(list, {
-            order = v.order,
-            file = file_path,
-            text = v.file:gsub(".*/", ""),
-            dim = not file_exists,
-            mandatory = file_exists and util.getFriendlySize(lfs.attributes(file_path, "size") or 0) or "",
-            select_enabled = file_exists,
-        })
-    end
-    table.sort(list, function(v1,v2)
-        return v1.order < v2.order
-    end)
-    return list
-end
-
-function ReadCollection:removeItemByPath(path, is_dir)
-    local dir
-    local should_write = false
-    if is_dir then
-        path = path .. "/"
-    end
-    local coll = self:readAllCollection()
-    for i in pairs(coll) do
-        local single_collection = coll[i]
-        for item = #single_collection, 1, -1 do
-            if not is_dir and single_collection[item].file == path then
-                should_write = true
-                table.remove(single_collection, item)
-            elseif is_dir then
-                dir = util.splitFilePathName(single_collection[item].file)
-                if dir == path then
-                    should_write = true
-                    table.remove(single_collection, item)
-                end
-            end
+function ReadCollection:write(collection_name)
+    local collections = LuaSettings:open(collection_file)
+    for coll_name in pairs(collections.data) do
+        if not self.coll[coll_name] then
+            collections:delSetting(coll_name)
         end
     end
-    if should_write then
-        local collection = LuaSettings:open(collection_file)
-        collection.data = coll
-        collection:flush()
-    end
-end
-
-function ReadCollection:updateItemByPath(old_path, new_path)
-    local is_dir = false
-    local dir, file
-    if lfs.attributes(new_path, "mode") == "directory" then
-        is_dir = true
-        old_path = old_path .. "/"
-    end
-    local should_write = false
-    local coll = self:readAllCollection()
-    for i, j in pairs(coll) do
-        for k, v in pairs(j) do
-            if not is_dir and v.file == old_path then
-                should_write = true
-                coll[i][k].file = new_path
-            elseif is_dir then
-                dir, file = util.splitFilePathName(v.file)
-                if dir == old_path then
-                    should_write = true
-                    coll[i][k].file = string.format("%s/%s", new_path, file)
-                end
+    for coll_name, coll in pairs(self.coll) do
+        if not collection_name or coll_name == collection_name then
+            local data = { settings = { order = self.coll_order[coll_name] } }
+            for _, item in pairs(coll) do
+                table.insert(data, { file = item.file, order = item.order })
             end
+            collections:saveSetting(coll_name, data)
         end
     end
-    if should_write then
-        local collection = LuaSettings:open(collection_file)
-        collection.data = coll
-        collection:flush()
+    logger.dbg("ReadCollection: writing to collection file")
+    collections:flush()
+end
+
+-- info
+
+function ReadCollection:isFileInCollection(file, collection_name)
+    file = FFIUtil.realpath(file) or file
+    return self.coll[collection_name][file] and true or false
+end
+
+function ReadCollection:isFileInCollections(file)
+    file = FFIUtil.realpath(file) or file
+    for _, coll in pairs(self.coll) do
+        if coll[file] then
+            return true
+        end
     end
+    return false
 end
 
-function ReadCollection:removeItem(item, collection_name)
-    local coll = self:read(collection_name)
-    for k, v in pairs(coll) do
-        if v.file == item then
-            table.remove(coll, k)
-            break
+function ReadCollection:getCollectionsWithFile(file)
+    file = FFIUtil.realpath(file) or file
+    local collections = {}
+    for coll_name, coll in pairs(self.coll) do
+        if coll[file] then
+            collections[coll_name] = true
         end
     end
-    self:writeCollection(coll, collection_name)
+    return collections
 end
 
-function ReadCollection:writeCollection(coll_items, collection_name)
-    local collection = LuaSettings:open(collection_file)
-    collection:saveSetting(collection_name or DEFAULT_COLLECTION_NAME, coll_items)
-    collection:flush()
+function ReadCollection:getCollectionMaxOrder(collection_name)
+    local max_order = 0
+    for _, item in pairs(self.coll[collection_name]) do
+        if max_order < item.order then
+            max_order = item.order
+        end
+    end
+    return max_order
 end
 
+-- manage items
+
 function ReadCollection:addItem(file, collection_name)
-    local coll, coll_max_item = self:read(collection_name)
-    local collection_item = {
-        file = file,
-        order = coll_max_item + 1,
-    }
-    table.insert(coll, collection_item)
-    self:writeCollection(coll, collection_name)
+    local max_order = self:getCollectionMaxOrder(collection_name)
+    local item = buildEntry(file, max_order + 1)
+    self.coll[collection_name][item.file] = item
+    self:write(collection_name)
+end
+
+function ReadCollection:addRemoveItemMultiple(file, collections_to_add)
+    file = FFIUtil.realpath(file) or file
+    for coll_name, coll in pairs(self.coll) do
+        if collections_to_add[coll_name] then
+            if not coll[file] then
+                local max_order = self:getCollectionMaxOrder(coll_name)
+                coll[file] = buildEntry(file, max_order + 1)
+            end
+        else
+            if coll[file] then
+                coll[file] = nil
+            end
+        end
+    end
+    self:write()
 end
 
-function ReadCollection:checkItemExist(item, collection_name)
-    local coll = self:read(collection_name)
-    for _, v in pairs(coll) do
-        if v.file == item then
+function ReadCollection:addItemsMultiple(files, collections_to_add)
+    for file in pairs(files) do
+        file = FFIUtil.realpath(file) or file
+        for coll_name in pairs(collections_to_add) do
+            local coll = self.coll[coll_name]
+            if not coll[file] then
+                local max_order = self:getCollectionMaxOrder(coll_name)
+                coll[file] = buildEntry(file, max_order + 1)
+            end
+        end
+    end
+    self:write()
+end
+
+function ReadCollection:removeItem(file, collection_name, no_write) -- FM: delete file; FMColl: remove file
+    file = FFIUtil.realpath(file) or file
+    if collection_name then
+        if self.coll[collection_name][file] then
+            self.coll[collection_name][file] = nil
+            if not no_write then
+                self:write(collection_name)
+            end
             return true
         end
+    else
+        local do_write
+        for _, coll in pairs(self.coll) do
+            if coll[file] then
+                coll[file] = nil
+                do_write = true
+            end
+        end
+        if do_write then
+            if not no_write then
+                self:write()
+            end
+            return true
+        end
+    end
+end
+
+function ReadCollection:removeItems(files) -- FM: delete files
+    local do_write
+    for file in pairs(files) do
+        if self:removeItem(file, nil, true) then
+            do_write = true
+        end
+    end
+    if do_write then
+        self:write()
+    end
+end
+
+function ReadCollection:removeItemsByPath(path) -- FM: delete folder
+    local do_write
+    for coll_name, coll in pairs(self.coll) do
+        for file_name in pairs(coll) do
+            if util.stringStartsWith(file_name, path) then
+                coll[file_name] = nil
+                do_write = true
+            end
+        end
+    end
+    if do_write then
+        self:write()
+    end
+end
+
+function ReadCollection:_updateItem(coll_name, file_name, new_filepath, new_path)
+    local coll = self.coll[coll_name]
+    local item_old = coll[file_name]
+    local order, mandatory = item_old.order, item_old.mandatory
+    new_filepath = new_filepath or new_path .. "/" .. item_old.text
+    coll[file_name] = nil
+    local item = buildEntry(new_filepath, order, mandatory) -- no lfs call
+    coll[item.file] = item
+end
+
+function ReadCollection:updateItem(file, new_filepath) -- FM: rename file, move file
+    file = FFIUtil.realpath(file) or file
+    local do_write
+    for coll_name, coll in pairs(self.coll) do
+        if coll[file] then
+            self:_updateItem(coll_name, file, new_filepath)
+            do_write = true
+        end
+    end
+    if do_write then
+        self:write()
+    end
+end
+
+function ReadCollection:updateItems(files, new_path) -- FM: move files
+    local do_write
+    for file in pairs(files) do
+        file = FFIUtil.realpath(file) or file
+        for coll_name, coll in pairs(self.coll) do
+            if coll[file] then
+                self:_updateItem(coll_name, file, nil, new_path)
+                do_write = true
+            end
+        end
+    end
+    if do_write then
+        self:write()
+    end
+end
+
+function ReadCollection:updateItemsByPath(path, new_path) -- FM: rename folder, move folder
+    local len = #path
+    local do_write
+    for coll_name, coll in pairs(self.coll) do
+        for file_name in pairs(coll) do
+            if file_name:sub(1, len) == path then
+                self:_updateItem(coll_name, file_name, new_path .. file_name:sub(len + 1))
+                do_write = true
+            end
+        end
+    end
+    if do_write then
+        self:write()
+    end
+end
+
+function ReadCollection:getOrderedCollection(collection_name)
+    local ordered_coll = {}
+    for _, item in pairs(self.coll[collection_name]) do
+        table.insert(ordered_coll, item)
+    end
+    table.sort(ordered_coll, function(v1, v2) return v1.order < v2.order end)
+    return ordered_coll
+end
+
+function ReadCollection:updateCollectionOrder(collection_name, ordered_coll)
+    local coll = self.coll[collection_name]
+    for i, item in ipairs(ordered_coll) do
+        coll[item.file].order = i
     end
+    self:write(collection_name)
 end
 
+-- manage collections
+
+function ReadCollection:addCollection(coll_name)
+    local max_order = 0
+    for _, order in pairs(self.coll_order) do
+        if max_order < order then
+            max_order = order
+        end
+    end
+    self.coll_order[coll_name] = max_order + 1
+    self.coll[coll_name] = {}
+    self:write(coll_name)
+end
+
+function ReadCollection:renameCollection(coll_name, new_name)
+    self.coll_order[new_name] = self.coll_order[coll_name]
+    self.coll[new_name] = self.coll[coll_name]
+    self.coll_order[coll_name] = nil
+    self.coll[coll_name] = nil
+    self:write(new_name)
+end
+
+function ReadCollection:removeCollection(coll_name)
+    self.coll_order[coll_name] = nil
+    self.coll[coll_name] = nil
+    self:write()
+end
+
+function ReadCollection:updateCollectionListOrder(ordered_coll)
+    for i, item in ipairs(ordered_coll) do
+        self.coll_order[item.name] = i
+    end
+    self:write()
+end
+
+ReadCollection:_read()
+
 return ReadCollection

frontend/readhistory.lua

@@ -7,6 +7,7 @@ local util = require("util")
 local joinPath = ffiutil.joinPath
 local lfs = require("libs/libkoreader-lfs")
 local realpath = ffiutil.realpath
+local C_ = require("gettext").pgettext
 
 local history_file = joinPath(DataStorage:getDataDir(), "history.lua")
 
@@ -18,7 +19,7 @@ local ReadHistory = {
 
 local function getMandatory(date_time)
     return G_reader_settings:isTrue("history_datetime_short")
-        and datetime.secondsToDate(date_time):sub(3) or datetime.secondsToDateTime(date_time)
+        and os.date(C_("Date string", "%y-%m-%d"), date_time) or datetime.secondsToDateTime(date_time)
 end
 
 local function buildEntry(input_time, input_file)
@@ -90,15 +91,10 @@ function ReadHistory:_flush()
     for _, v in ipairs(self.hist) do
         table.insert(content, {
             time = v.time,
-            file = v.file
+            file = v.file,
         })
     end
-    local f = io.open(history_file, "w")
-    if f then
-        f:write("return " .. dump(content) .. "\n")
-        ffiutil.fsyncOpenedFile(f) -- force flush to the storage device
-        f:close()
-    end
+    util.writeToFile(dump(content), history_file, true, true)
     self:ensureLastFile()
 end
 
@@ -147,14 +143,10 @@ function ReadHistory:_readLegacyHistory()
     os.remove(history_dir)
 end
 
-function ReadHistory:_init()
-    self:reload()
-end
-
 function ReadHistory:ensureLastFile()
     local last_existing_file
     for _, v in ipairs(self.hist) do
-        if lfs.attributes(v.file, "mode") == "file" then
+        if v.select_enabled then
             last_existing_file = v.file
             break
         end
@@ -171,7 +163,7 @@ function ReadHistory:getPreviousFile(current_file)
     end
     for _, v in ipairs(self.hist) do
         -- skip current document and deleted items kept in history
-        if v.file ~= current_file and lfs.attributes(v.file, "mode") == "file" then
+        if v.file ~= current_file and v.select_enabled then
             return v.file
         end
     end
@@ -188,25 +180,102 @@ function ReadHistory:getFileByDirectory(directory, recursive)
     end
 end
 
-function ReadHistory:updateItemByPath(old_path, new_path)
-    local index = self:getIndexByFile(old_path)
+--- Updates the history list after renaming/moving a file.
+function ReadHistory:updateItem(file, new_filepath)
+    local index = self:getIndexByFile(file)
     if index then
-        self.hist[index].file = new_path
-        self.hist[index].text = new_path:gsub(".*/", "")
+        local item = self.hist[index]
+        item.file = new_filepath
+        item.text = new_filepath:gsub(".*/", "")
+        self:_flush()
+    end
+end
+
+function ReadHistory:updateItems(files, new_path) -- files = { filepath = true, }
+    local history_updated
+    for file in pairs(files) do
+        local index = self:getIndexByFile(file)
+        if index then
+            local item = self.hist[index]
+            item.file = new_path .. "/" .. item.text
+            history_updated = true
+        end
+    end
+    if history_updated then
+        self:_flush()
+    end
+end
+
+--- Updates the history list after renaming/moving a folder.
+function ReadHistory:updateItemsByPath(old_path, new_path)
+    local len = #old_path
+    local history_updated
+    for i, v in ipairs(self.hist) do
+        if v.file:sub(1, len) == old_path then
+            self.hist[i].file = new_path .. v.file:sub(len + 1)
+            history_updated = true
+        end
+    end
+    if history_updated then
         self:_flush()
-        self:reload(true)
     end
 end
 
 --- Updates the history list after deleting a file.
-function ReadHistory:fileDeleted(path)
-    local index = self:getIndexByFile(path)
+function ReadHistory:fileDeleted(path_or_index)
+    local index
+    local is_single = type(path_or_index) == "string"
+    if is_single then -- deleting single file, path passed
+        index = self:getIndexByFile(path_or_index)
+    else -- deleting folder, index passed
+        index = path_or_index
+    end
     if index then
         if G_reader_settings:isTrue("autoremove_deleted_items_from_history") then
-            self:removeItem(self.hist[index], index)
+            self:removeItem(self.hist[index], index, not is_single) -- flush immediately when deleting single file only
         else
             self.hist[index].dim = true
             self.hist[index].select_enabled = false
+            if is_single then
+                self:ensureLastFile()
+            end
+        end
+    end
+end
+
+--- Updates the history list after deleting a folder.
+function ReadHistory:folderDeleted(path)
+    local history_updated
+    for i = #self.hist, 1, -1 do
+        local file = self.hist[i].file
+        if util.stringStartsWith(file, path) then
+            self:fileDeleted(i)
+            history_updated = true
+            DocSettings.updateLocation(file) -- remove sdr if not in book location
+        end
+    end
+    if history_updated then
+        if G_reader_settings:isTrue("autoremove_deleted_items_from_history") then
+            self:_flush()
+        else
+            self:ensureLastFile()
+        end
+    end
+end
+
+function ReadHistory:removeItems(files) -- files = { filepath = true, }
+    local history_updated
+    for file in pairs(files) do
+        local index = self:getIndexByFile(file)
+        if index then
+            self:fileDeleted(index)
+            history_updated = true
+        end
+    end
+    if history_updated then
+        if G_reader_settings:isTrue("autoremove_deleted_items_from_history") then
+            self:_flush()
+        else
             self:ensureLastFile()
         end
     end
@@ -253,30 +322,32 @@ end
 --- Adds new item (last opened document) to the top of the history list.
 -- If item time (ts) is passed, add item to the history list at this time position.
 function ReadHistory:addItem(file, ts, no_flush)
-    if file ~= nil and lfs.attributes(file, "mode") == "file" then
-        local index = self:getIndexByFile(realpath(file))
-        if ts and index and self.hist[index].time == ts then
-            return -- this legacy item is in the history already
-        end
-        local now = ts or os.time()
-        local mtime = lfs.attributes(file, "modification")
-        lfs.touch(file, now, mtime) -- update book access time for sorting by last read date
-        if index == 1 and not ts then -- last book, update access time only
-            self.hist[1].time = now
-            self.hist[1].mandatory = getMandatory(now)
-        else -- old or new book
-            if index then -- old book
-                table.remove(self.hist, index)
-            end
-            index = ts and self:getIndexByTime(ts, file:gsub(".*/", "")) or 1
-            table.insert(self.hist, index, buildEntry(now, file))
-        end
-        if not no_flush then
-            self:_reduce()
-            self:_flush()
+    file = realpath(file)
+    if not file or (ts and lfs.attributes(file, "mode") ~= "file") then
+        return -- bad legacy item
+    end
+    local index = self:getIndexByFile(file)
+    if ts and index and self.hist[index].time == ts then
+        return -- legacy item already added
+    end
+    local now = ts or os.time()
+    local mtime = lfs.attributes(file, "modification")
+    lfs.touch(file, now, mtime) -- update book access time for sorting by last read date
+    if index == 1 and not ts then -- last book, update access time only
+        self.hist[1].time = now
+        self.hist[1].mandatory = getMandatory(now)
+    else -- old or new book
+        if index then -- old book
+            table.remove(self.hist, index)
         end
-        return true -- used while adding legacy items
+        index = ts and self:getIndexByTime(ts, file:gsub(".*/", "")) or 1
+        table.insert(self.hist, index, buildEntry(now, file))
+    end
+    if not no_flush then
+        self:_reduce()
+        self:_flush()
     end
+    return true -- used while adding legacy items
 end
 
 --- Updates last book access time on closing the document.
@@ -289,6 +360,12 @@ function ReadHistory:updateLastBookTime(no_flush)
     end
 end
 
+function ReadHistory:updateDateTimeString()
+    for _, v in ipairs(self.hist) do
+        v.mandatory = getMandatory(v.time)
+    end
+end
+
 --- Reloads history from history_file and legacy history folder.
 function ReadHistory:reload(force_read)
     if self:_read(force_read) then
@@ -300,6 +377,10 @@ function ReadHistory:reload(force_read)
     end
 end
 
+function ReadHistory:_init()
+    self:reload()
+end
+
 ReadHistory:_init()
 
 return ReadHistory

frontend/ui/data/creoptions.lua

@@ -51,7 +51,7 @@ local CreOptions = {
     {
         icon = "appbar.rotation",
         options = {
-            {
+            {   -- ReaderView
                 name = "rotation_mode",
                 name_text = _("Rotation"),
                 item_icons_func = function()
@@ -93,20 +93,19 @@ local CreOptions = {
                 labels = {C_("Rotation", "⤹ 90°"), C_("Rotation", "↑ 0°"), C_("Rotation", "⤸ 90°"), C_("Rotation", "↓ 180°")},
                 alternate = false,
                 values = {Screen.DEVICE_ROTATED_COUNTER_CLOCKWISE, Screen.DEVICE_ROTATED_UPRIGHT, Screen.DEVICE_ROTATED_CLOCKWISE, Screen.DEVICE_ROTATED_UPSIDE_DOWN},
+                default_value = Screen.DEVICE_ROTATED_UPRIGHT,
                 args = {Screen.DEVICE_ROTATED_COUNTER_CLOCKWISE, Screen.DEVICE_ROTATED_UPRIGHT, Screen.DEVICE_ROTATED_CLOCKWISE, Screen.DEVICE_ROTATED_UPSIDE_DOWN},
-                default_arg = 0,
                 current_func = function() return Screen:getRotationMode() end,
                 event = "SetRotationMode",
                 name_text_hold_callback = optionsutil.showValues,
             },
-            {
+            {   -- ReaderRolling
                 name = "visible_pages",
                 name_text = _("Two Columns"),
                 toggle = {_("off"), _("on")},
                 values = {1, 2},
                 default_value = 1,
                 args = {1, 2},
-                default_arg = 1,
                 event = "SetVisiblePages",
                 --[[ Commented out, to have it also available in portrait mode
                 current_func = function()
@@ -131,7 +130,7 @@ This is disabled in scroll mode. Switching from page mode with two columns to sc
     {
         icon = "appbar.crop",
         options = {
-            {
+            {   -- ReaderTypeset
                 name = "h_page_margins",
                 name_text = _("L/R Margins"),
                 buttonprogress = true,
@@ -176,7 +175,7 @@ This is disabled in scroll mode. Switching from page mode with two columns to sc
                     right_hold_step = 5,
                 },
             },
-            {
+            {   -- ReaderTypeset
                 name = "sync_t_b_page_margins",
                 name_text = _("Sync T/B Margins"),
                 toggle = {_("off"), _("on")},
@@ -184,7 +183,6 @@ This is disabled in scroll mode. Switching from page mode with two columns to sc
                 default_value = 0,
                 event = "SyncPageTopBottomMargins",
                 args = {false, true},
-                default_arg = false,
                 hide_on_apply = true,
                 name_text_hold_callback = optionsutil.showValues,
                 help_text = _([[Keep top and bottom margins synchronized.
@@ -193,7 +191,7 @@ This is disabled in scroll mode. Switching from page mode with two columns to sc
 
 In the top menu → Settings → Status bar, you can choose whether the bottom margin applies from the bottom of the screen, or from above the status bar.]]),
             },
-            {
+            {   -- ReaderTypeset
                 name = "t_page_margin",
                 name_text = _("Top Margin"),
                 buttonprogress = true,
@@ -247,7 +245,7 @@ In the top menu → Settings → Status bar, you can choose whether the bottom m
                     right_hold_step = 5,
                 },
             },
-            {
+            {   -- ReaderTypeset
                 name = "b_page_margin",
                 name_text = _("Bottom Margin"),
                 buttonprogress = true,
@@ -305,27 +303,25 @@ In the top menu → Settings → Status bar, you can choose whether the bottom m
     {
         icon = "appbar.pageview",
         options = {
-            {
+            {   -- ReaderView
                 name = "view_mode",
                 name_text = _("View Mode"),
                 toggle = {_("page"), _("continuous")},
                 values = {0, 1},
-                default_value = 0,
+                default_value = G_defaults:readSetting("DCREREADER_VIEW_MODE") == "page" and 0 or 1,
                 args = {"page", "scroll"},
-                default_arg = "page",
                 event = "SetViewMode",
                 name_text_hold_callback = optionsutil.showValues,
                 help_text = _([[- 'page' mode splits the text into pages, at the most acceptable places (page numbers and the number of pages may change when you change fonts, margins, styles, etc.).
 - 'continuous' mode allows you to scroll the text like you would in a web browser (the 'Page Overlap' setting is only available in this mode).]]),
             },
-            {
+            {   -- ReaderTypeset
                 name = "block_rendering_mode",
                 name_text = _("Render Mode"),
                 toggle = {_("legacy"), _("flat"), _("book"), _("web")},
                 values = {0, 1, 2, 3},
                 default_value = 2,
                 args = {0, 1, 2, 3},
-                default_arg = 2,
                 event = "SetBlockRenderingMode",
                 name_text_hold_callback = optionsutil.showValues,
                 help_text = _([[
@@ -334,7 +330,7 @@ In the top menu → Settings → Status bar, you can choose whether the bottom m
 - 'book' additionally allows floats, but limits style support to avoid blank spaces and overflows.
 - 'web' renders as web browsers do, allowing negative margins and possible page overflow.]]),
             },
-            {
+            {   -- ReaderTypeset
                 name = "render_dpi",
                 name_text = _("Zoom (dpi)"),
                 more_options = true,
@@ -363,7 +359,7 @@ Note that your selected font size is not affected by this setting.]]),
                 end,
 
             },
-            {
+            {   -- ReaderFont
                 name = "line_spacing",
                 name_text = _("Line Spacing"),
                 buttonprogress = true,
@@ -418,7 +414,7 @@ Note that your selected font size is not affected by this setting.]]),
     {
         icon = "appbar.textsize",
         options = {
-            {
+            {   -- ReaderFont
                 name = "font_size",
                 alt_name_text = _("Font Size"),
                 item_text = tableOfNumbersToTableOfStrings(G_defaults:readSetting("DCREREADER_CONFIG_FONT_SIZES")),
@@ -430,7 +426,7 @@ Note that your selected font size is not affected by this setting.]]),
                 args = G_defaults:readSetting("DCREREADER_CONFIG_FONT_SIZES"),
                 event = "SetFontSize",
             },
-            {
+            {   -- ReaderFont
                 name = "font_fine_tune",
                 name_text = _("Font Size"),
                 toggle = Device:isTouchDevice() and {_("decrease"), _("increase")} or nil,
@@ -458,7 +454,7 @@ Note that your selected font size is not affected by this setting.]]),
                     optionsutil.showValues(configurable, opt, prefix, nil, "pt")
                 end,
             },
-            {
+            {   -- ReaderFont
                 name = "word_spacing",
                 name_text = _("Word Spacing"),
                 more_options = true,
@@ -501,7 +497,7 @@ Note that your selected font size is not affected by this setting.]]),
                     return string.format("%d\u{202F}%%, %d\u{202F}%%", val[1], val[2]) -- use Narrow Now-Break space here
                 end,
             },
-            {
+            {   -- ReaderFont
                 name = "word_expansion",
                 name_text = _("Word Expansion"),
                 more_options = true,
@@ -540,7 +536,7 @@ Note that your selected font size is not affected by this setting.]]),
                     return string.format("%d\u{202F}%%", val) -- use Narrow No-Break space here
                 end,
             },
-            {
+            {   -- ReaderFont
                 -- This option is not shown in the bottom menu, but its fine tuning is made
                 -- available via the other_button in Word Expansion's fine tuning widget.
                 -- We still need to define it as an option here for it to be known and
@@ -572,7 +568,7 @@ Note that your selected font size is not affected by this setting.]]),
     {
         icon = "appbar.contrast",
         options = {
-            {
+            {   -- ReaderFont
                 name = "font_gamma",
                 name_text = _("Contrast"),
                 buttonprogress = true,
@@ -597,7 +593,7 @@ Note that your selected font size is not affected by this setting.]]),
                     value_table_shift = 1,
                 },
             },
-            {
+            {   -- ReaderFont
                 name = "font_base_weight",
                 name_text = _("Font Weight"),
                 toggle = { "-1", "-½", "0", "+½", "+1", "+1½", "+3" },
@@ -631,7 +627,7 @@ If a font variation is not available, as well as for fractional adjustments, it
                     return weightClassToString(400+val*100)
                 end,
             },
-            {
+            {   -- ReaderFont
                 name = "font_hinting",
                 name_text = _("Font Hinting"),
                 toggle = {C_("Font hinting", "off"), C_("Font hinting", "native"), C_("Font hinting", "auto")},
@@ -646,7 +642,7 @@ If a font variation is not available, as well as for fractional adjustments, it
 - native: use the font internal hinting instructions.
 - auto: use FreeType's hinting algorithm, ignoring font instructions.]]),
             },
-            {
+            {   -- ReaderFont
                 name = "font_kerning",
                 name_text = _("Font Kerning"),
                 toggle = {_("off"), _("fast"), _("good"), _("best")},
@@ -669,41 +665,38 @@ If a font variation is not available, as well as for fractional adjustments, it
     {
         icon = "appbar.settings",
         options = {
-            {
+            {   -- ReaderRolling
                 name = "status_line",
                 name_text = _("Alt Status Bar"),
                 toggle = {_("off"), _("on")},
                 values = {1, 0}, -- Note that 0 means crengine header status line enabled, and 1 means disabled
                 default_value = 1,
                 args = {1, 0},
-                default_arg = 1,
                 event = "SetStatusLine",
                 name_text_hold_callback = optionsutil.showValues,
                 help_text = _([[Enable or disable the rendering engine alternative status bar at the top of the screen. The items displayed can be customized via the main menu.
 
 Whether enabled or disabled, KOReader's own status bar at the bottom of the screen can be toggled by tapping.]]),
             },
-            {
+            {   -- ReaderTypeset
                 name = "embedded_css",
                 name_text = _("Embedded Style"),
                 toggle = {_("off"), _("on")},
                 values = {0, 1},
                 default_value = 1,
                 args = {false, true},
-                default_arg = nil,
                 event = "ToggleEmbeddedStyleSheet",
                 name_text_hold_callback = optionsutil.showValues,
                 help_text = _([[Enable or disable publisher stylesheets embedded in the book.
 (Note that less radical changes can be achieved via Style Tweaks in the main menu.)]]),
             },
-            {
+            {   -- ReaderTypeset
                 name = "embedded_fonts",
                 name_text = _("Embedded Fonts"),
                 toggle = {_("off"), _("on")},
                 values = {0, 1},
                 default_value = 1,
                 args = {false, true},
-                default_arg = nil,
                 event = "ToggleEmbeddedFonts",
                 enabled_func = function(configurable, document)
                     return optionsutil.enableIfEquals(configurable, "embedded_css", 1)
@@ -724,27 +717,25 @@ Whether enabled or disabled, KOReader's own status bar at the bottom of the scre
                     end
                 end,
             },
-            {
+            {   -- ReaderTypeset
                 name = "smooth_scaling",
                 name_text = _("Image Scaling"),
                 toggle = {_("fast"), _("best")},
                 values = {0, 1},
                 default_value = 0,
                 args = {false, true},
-                default_arg = nil,
                 event = "ToggleImageScaling",
                 name_text_hold_callback = optionsutil.showValues,
                 help_text = _([[- 'fast' uses a fast but inaccurate scaling algorithm when scaling images.
 - 'best' switches to a more costly but vastly more pleasing and accurate algorithm.]]),
             },
-            {
+            {   -- ReaderTypeset
                 name = "nightmode_images",
                 name_text = _("Invert Images"),
                 toggle = {_("on"), _("off")},
                 values = {1, 0},
                 default_value = 1,
                 args = {true, false},
-                default_arg = nil,
                 event = "ToggleNightmodeImages",
                 show_func = function() return Screen.night_mode end,
                 name_text_hold_callback = optionsutil.showValues,

frontend/ui/data/css_tweaks.lua

@@ -419,6 +419,13 @@ Further small adjustments can be done with 'Line Spacing' in the bottom menu.]])
         },
         {
             title = _("Font size and families"),
+            {
+                id = "font_no_presentational_hints",
+                title = _("Ignore font related HTML presentational hints"),
+                description = _("Ignore HTML attributes that contribute to styles on the elements <body> (bgcolor, text…) and <font> (face, size, color…)."),
+                css = [[body, font { -cr-hint: no-presentational; }]],
+                separator = true,
+            },
             {
                 id = "font_family_all_inherit",
                 title = _("Ignore publisher font families"),
@@ -564,6 +571,13 @@ body, h1, h2, h3, h4, h5, h6, div, li, td, th { text-indent: 0 !important; }
         title = _("Tables, links, images"),
         {
             title = _("Tables"),
+            {
+                id = "table_no_presentational_hints",
+                title = _("Ignore tables related HTML presentational hints"),
+                description = _("Ignore HTML attributes that contribute to styles on the <table> element and its sub-elements (ie. align, valign, frame, rules, border, cellpadding, cellspacing…)."),
+                css = [[table, caption, colgroup, col, thead, tbody, tfoot, tr, td, th { -cr-hint: no-presentational; }]],
+                separator = true,
+            },
             {
                 id = "table_full_width",
                 title = _("Full-width tables"),
@@ -965,6 +979,8 @@ This tweak can be duplicated as a user style tweak when books contain footnotes
 .footnote, .footnotes, .fn,
 .note, .note1, .note2, .note3,
 .ntb, .ntb-txt, .ntb-txt-j,
+.fnote, .fnote1,
+.duokan-footnote-item, /* Common chinese books */
 .przypis, .przypis1, /* Polish footnotes */
 .voetnoten /* Dutch footnotes */
 {
@@ -1114,6 +1130,21 @@ If the footnote text uses variable or absolute font sizes, line height or vertic
         font-size: inherit !important;
         line-height: inherit !important;
         vertical-align: inherit !important;
+}
+                ]],
+            },
+            {
+                id = "inpage_footnote_combine_non_linear",
+                title = _("Combine footnotes in a non-linear flow"),
+                description = _([[
+This will mark each section of consecutive footnotes (at their original location in the book) as being non-linear.
+The menu checkbox "Hide non-linear fragments" will then be available after the document is reopened, allowing to hide these sections from the regular flow: they will be skipped when turning pages and not considered in the various book & chapter progress and time to read features.]]),
+                priority = 6,
+                css = [[
+*, autoBoxing {
+    -cr-hint: late;
+    -cr-only-if: inpage-footnote;
+        -cr-hint: non-linear-combining;
 }
                 ]],
             },

frontend/ui/data/dictionaries.lua

@@ -2723,6 +2723,14 @@ local dictionaries = {
         license = "Open Vietnamese Dictionary Project, https://sourceforge.net/projects/ovdp/",
         url = "https://khoicandev.github.io/ovdp-mirror/en-vi.tar.gz"
     },
+    {
+        name = "English-Persian dictionary",
+        lang_in = "eng",
+        lang_out = "fas",
+        entries = 145075,
+        license = "MIT, https://github.com/hossein1376/English-Persian-Kindle-Custom-Dictionary",
+        url = "https://github.com/hossein1376/English-Persian-Kindle-Custom-Dictionary/raw/main/English%20Persian%20Dictionary.tar.gz"
+    },
     {
         name = "Vietnamese-English dictionary",
         lang_in = "vie",

frontend/ui/data/keyboardlayouts/cs_keyboard.lua

@@ -0,0 +1,57 @@
+local cs_keyboard = require("util").tableDeepCopy(require("ui/data/keyboardlayouts/sk_keyboard"))
+
+local keys = cs_keyboard.keys
+
+keys[1][2][1] = {
+    "2",
+    north = "ě",
+    northeast = "Ě",
+    east = "~",
+    southeast = "/",
+    south = "@",
+    southwest = "https://",
+    west = "http://",
+    northwest = "Ĺ",
+    alt_label = "ě",
+}
+keys[1][2][2] = {
+    "ě",
+    north = "2",
+    northeast = "Ě",
+    east = "~",
+    southeast = "/",
+    south = "@",
+    southwest = "https://",
+    west = "http://",
+    northwest = "ĺ",
+    alt_label = "2",
+}
+
+keys[1][5][1] = {
+    "5",
+    north = "ř",
+    northeast = "Ř",
+    east = "¾",
+    southeast = "‱",
+    south = "%",
+    southwest = "‰",
+    west = "⅔",
+    northwest = "Ŕ",
+    alt_label = "ř",
+}
+keys[1][5][2] = {
+    "ř",
+    north = "5",
+    northeast = "Ř",
+    east = "¼",
+    southeast = "‱",
+    south = "%",
+    southwest = "‰",
+    west = "½",
+    northwest = "ŕ",
+    alt_label = "5",
+}
+
+keys[5][4].label = "mezera"
+
+return cs_keyboard

frontend/ui/data/keyboardlayouts/da_keyboard.lua

@@ -0,0 +1,10 @@
+-- Start with the norwegian keyboard layout (deep copy, to not alter it)
+local da_keyboard = require("util").tableDeepCopy(require("ui/data/keyboardlayouts/no_keyboard"))
+
+local keys = da_keyboard.keys
+
+-- swap "Ø" and "Æ", and "ø" and "æ"
+keys[3][10][1], keys[3][11][1] = keys[3][11][1], keys[3][10][1]
+keys[3][10][2], keys[3][11][2] = keys[3][11][2], keys[3][10][2]
+
+return da_keyboard