From b0ceab5b2082c15211f8d08d1078a05fba6061b6 Mon Sep 17 00:00:00 2001
From: roygbyte <82218266+roygbyte@users.noreply.github.com>
Date: Mon, 13 Sep 2021 16:04:06 -0300
Subject: [PATCH] [plugin] NewsDownloader: fix XML, better error messages,
change default feed, and more (#8145)
* Fix XML, introduce some better error messages, etc.
See roygbyte/newsdownloader.koplugin for more info
* Fix feed attribute saving snafu; Change menu labels; etc.
Also:
- Change default feed
- Change "remove" label to "delete"
- Hide menu after feed sync
---
.../newsdownloader.koplugin/feed_config.lua | 7 +-
plugins/newsdownloader.koplugin/feed_view.lua | 210 ++++
plugins/newsdownloader.koplugin/main.lua | 1027 ++++++++++++++---
3 files changed, 1064 insertions(+), 180 deletions(-)
create mode 100644 plugins/newsdownloader.koplugin/feed_view.lua
diff --git a/plugins/newsdownloader.koplugin/feed_config.lua b/plugins/newsdownloader.koplugin/feed_config.lua
index 9e864acfb..2948942da 100644
--- a/plugins/newsdownloader.koplugin/feed_config.lua
+++ b/plugins/newsdownloader.koplugin/feed_config.lua
@@ -33,10 +33,7 @@ return {--do NOT change this line
-- LIST YOUR FEEDS HERE:
- { "http://feeds.reuters.com/Reuters/worldNews?format=xml", limit = 2, download_full_article=true, include_images=true, enable_filter=true},
-
- { "https://www.pcworld.com/index.rss", limit = 7 , download_full_article=false},
-
--- { "http://www.football.co.uk/international/rss.xml", limit = 2},
+ { "https://github.com/koreader/koreader/releases.atom", limit = 3, download_full_article=true, include_images=false, enable_filter=true, filter_element = "div.release-main-section"},
+ { "https://ourworldindata.org/atom.xml", limit = 5 , download_full_article=true, include_images=true, enable_filter=false, filter_element = ""},
}--do NOT change this line
diff --git a/plugins/newsdownloader.koplugin/feed_view.lua b/plugins/newsdownloader.koplugin/feed_view.lua
new file mode 100644
index 000000000..ac6cdbef7
--- /dev/null
+++ b/plugins/newsdownloader.koplugin/feed_view.lua
@@ -0,0 +1,210 @@
+local logger = require("logger")
+local _ = require("gettext")
+
+local FeedView = {
+ URL = "url",
+ LIMIT = "limit",
+ DOWNLOAD_FULL_ARTICLE = "download_full_article",
+ INCLUDE_IMAGES = "include_images",
+ ENABLE_FILTER = "enable_filter",
+ FILTER_ELEMENT = "filter_element"
+}
+
+function FeedView:getList(feed_config, callback, edit_feed_attribute_callback, delete_feed_callback)
+ local view_content = {}
+ -- Loop through the feed.
+ for idx, feed in ipairs(feed_config) do
+ local feed_item_content = {}
+ local feed_list_content = {}
+
+ local vc_feed_item = FeedView:getItem(
+ idx,
+ feed,
+ edit_feed_attribute_callback,
+ delete_feed_callback
+ )
+
+ if not vc_feed_item then
+ logger.warn('NewsDownloader: invalid feed config entry', feed)
+ else
+ feed_item_content = FeedView:flattenArray(feed_item_content, vc_feed_item)
+ local url = feed[1]
+ table.insert(
+ view_content,
+ {
+ url,
+ "",
+ callback = function()
+ -- Here is where we trigger the single feed item display
+ callback(feed_item_content)
+ end
+ }
+ )
+ -- Insert a divider.
+ table.insert(
+ view_content,
+ "---"
+ )
+ end
+ end
+ return view_content
+end
+
+function FeedView:getItem(id, feed, edit_feed_callback, delete_feed_callback)
+
+ logger.dbg("NewsDownloader:", feed)
+
+ local url = feed[1]
+ local limit = feed.limit
+
+ -- If there's no URL or limit we don't care about this
+ -- because we can't use it.
+ if not url and limit then
+ return nil
+ end
+
+ -- Collect this stuff for later, with the single view.
+ local download_full_article = feed.download_full_article == nil
+ or feed.download_full_article
+ local include_images = not never_download_images
+ and feed.include_images
+ local enable_filter = feed.enable_filter
+ or feed.enable_filter == nil
+ local filter_element = feed.filter_element
+ or feed.filter_element == nil
+
+ --- @todo: Strip the http:// or https:// from the URL
+ local sPre, sLink, sPost = url:match( "(.+)%s+(https?%S+)%s+(.*)$" )
+ local pretty_url = sLink
+
+ local view_content = {}
+ local vc = {
+ {
+ _("URL"),
+ url,
+ callback = function()
+ edit_feed_callback(
+ id,
+ FeedView.URL,
+ url
+ )
+ end
+ },
+ {
+ _("Limit"),
+ limit,
+ callback = function()
+ edit_feed_callback(
+ id,
+ FeedView.LIMIT,
+ limit
+ )
+ end
+ },
+ {
+ _("Download full article"),
+ download_full_article,
+ callback = function()
+ edit_feed_callback(
+ id,
+ FeedView.DOWNLOAD_FULL_ARTICLE,
+ download_full_article
+ )
+ end
+ },
+ {
+ _("Include images"),
+ include_images,
+ callback = function()
+ edit_feed_callback(
+ id,
+ FeedView.INCLUDE_IMAGES,
+ include_images
+ )
+ end
+ },
+ {
+ _("Enable filter"),
+ enable_filter,
+ callback = function()
+ edit_feed_callback(
+ id,
+ FeedView.ENABLE_FILTER,
+ enable_filter
+ )
+ end
+
+ },
+ {
+ _("Filter element"),
+ filter_element,
+ callback = function()
+ edit_feed_callback(
+ id,
+ FeedView.FILTER_ELEMENT,
+ filter_element
+ )
+ end
+ },
+ }
+
+ -- We don't always display this. For instance: if a feed
+ -- is being created, this button is not necessary.
+ if delete_feed_callback then
+ table.insert(
+ vc,
+ "---"
+ )
+ table.insert(
+ vc,
+ {
+ _("Delete feed"),
+ "",
+ callback = function()
+ delete_feed_callback(
+ id
+ )
+ end
+ }
+ )
+ end
+
+ return vc
+end
+
+--
+-- KeyValuePage doesn't like to get a table with sub tables.
+-- This function flattens an array, moving all nested tables
+-- up the food chain, so to speak
+--
+function FeedView:flattenArray(base_array, source_array)
+ for key, value in pairs(source_array) do
+ if value[2] == nil then
+ -- If the value is empty, then it's probably supposed to be a line
+ table.insert(
+ base_array,
+ "---"
+ )
+ else
+ if value["callback"] then
+ table.insert(
+ base_array,
+ {
+ value[1], value[2], callback = value["callback"]
+ }
+ )
+ else
+ table.insert(
+ base_array,
+ {
+ value[1], value[2]
+ }
+ )
+ end
+ end
+ end
+ return base_array
+end
+
+
+return FeedView
diff --git a/plugins/newsdownloader.koplugin/main.lua b/plugins/newsdownloader.koplugin/main.lua
index 95dddf7fb..15b1279aa 100644
--- a/plugins/newsdownloader.koplugin/main.lua
+++ b/plugins/newsdownloader.koplugin/main.lua
@@ -5,11 +5,16 @@ local DataStorage = require("datastorage")
local DownloadBackend = require("epubdownloadbackend")
local ReadHistory = require("readhistory")
local FFIUtil = require("ffi/util")
+local FeedView = require("feed_view")
local InfoMessage = require("ui/widget/infomessage")
local InputDialog = require("ui/widget/inputdialog")
local LuaSettings = require("frontend/luasettings")
local UIManager = require("ui/uimanager")
+local KeyValuePage = require("ui/widget/keyvaluepage")
+local InputDialog = require("ui/widget/inputdialog")
+local MultiConfirmBox = require("ui/widget/multiconfirmbox")
local NetworkMgr = require("ui/network/manager")
+local Persist = require("persist")
local WidgetContainer = require("ui/widget/container/widgetcontainer")
local dateparser = require("lib.dateparser")
local logger = require("logger")
@@ -18,21 +23,37 @@ local _ = require("gettext")
local T = FFIUtil.template
local NewsDownloader = WidgetContainer:new{
- name = "newsdownloader",
+ name = "news_downloader",
+ initialized = false,
+ feed_config_file = "feed_config.lua",
+ feed_config_path = nil,
+ news_config_file = "news_settings.lua",
+ settings = nil,
+ download_dir_name = "news",
+ download_dir = nil,
+ file_extension = ".epub",
+ config_key_custom_dl_dir = "custom_dl_dir",
+ empty_feed = {
+ [1] = "https://",
+ limit = 5,
+ download_full_article = true,
+ include_images = true,
+ enable_filter = false,
+ filter_element = ""
+ },
+ kv = {}
}
-local initialized = false
-local feed_config_file_name = "feed_config.lua"
-local news_downloader_config_file = "news_downloader_settings.lua"
-local news_downloader_settings
-local config_key_custom_dl_dir = "custom_dl_dir"
-local file_extension = ".epub"
-local news_download_dir_name = "news"
-local news_download_dir_path, feed_config_path
-
--- if a title looks like blabla it'll just be feed.title
--- if a title looks like blabla then we get a table
--- where [1] is the title string and the attributes are also available
+local FEED_TYPE_RSS = "rss"
+local FEED_TYPE_ATOM = "atom"
+
+--local initialized = false
+--local feed_config_file_name = "feed_config.lua"
+--local news_downloader_config_file = "news_downloader_settings.lua
+
+-- If a title looks like blabla it'll just be feed.title.
+-- If a title looks like blabla then we get a table.
+-- Where [1] is the title string and the attributes are also available.
local function getFeedTitle(possible_title)
if type(possible_title) == "string" then
return util.htmlEntitiesToUtf8(possible_title)
@@ -41,12 +62,12 @@ local function getFeedTitle(possible_title)
end
end
--- there can be multiple links
--- for now we just assume the first link is probably the right one
--- @todo write unit tests
--- some feeds that can be used for unit test
--- http://fransdejonge.com/feed/ for multiple links
--- https://github.com/koreader/koreader/commits/master.atom for single link with attributes
+-- There can be multiple links
+-- for now we just assume the first link is probably the right one.
+--- @todo Write unit tests.
+-- Some feeds that can be used for unit test.
+-- http://fransdejonge.com/feed/ for multiple links.
+-- https://github.com/koreader/koreader/commits/master.atom for single link with attributes.
local function getFeedLink(possible_link)
local E = {}
if type(possible_link) == "string" then
@@ -63,122 +84,168 @@ function NewsDownloader:init()
end
function NewsDownloader:addToMainMenu(menu_items)
- self:lazyInitialization()
menu_items.news_downloader = {
- text = _("News (RSS/Atom) downloader"),
- sub_item_table = {
- {
- text = _("Download news"),
- keep_menu_open = true,
- callback = function()
- NetworkMgr:runWhenOnline(function() self:loadConfigAndProcessFeedsWithUI() end)
- end,
- },
- {
- text = _("Go to news folder"),
- callback = function()
- local FileManager = require("apps/filemanager/filemanager")
- if self.ui.document then
- self.ui:onClose()
- end
- if FileManager.instance then
- FileManager.instance:reinit(news_download_dir_path)
- else
- FileManager:showFiles(news_download_dir_path)
- end
- end,
- },
- {
- text = _("Remove news"),
- keep_menu_open = true,
- callback = function() self:removeNewsButKeepFeedConfig() end,
- },
- {
- text = _("Never download images"),
- keep_menu_open = true,
- checked_func = function()
- return news_downloader_settings:isTrue("never_download_images")
- end,
- callback = function()
- news_downloader_settings:toggle("never_download_images")
- news_downloader_settings:flush()
- end,
- },
- {
- text = _("Settings"),
- sub_item_table = {
- {
- text = _("Change feeds configuration"),
- keep_menu_open = true,
- callback = function() self:changeFeedConfig() end,
- },
- {
- text = _("Set custom download folder"),
- keep_menu_open = true,
- callback = function() self:setCustomDownloadDirectory() end,
- },
+ text = _("News downloader (RSS/Atom)"),
+ sub_item_table_func = function()
+ return self:getSubMenuItems()
+ end,
+ }
+end
+
+function NewsDownloader:getSubMenuItems()
+ self:lazyInitialization()
+ local sub_item_table
+ sub_item_table = {
+ {
+ text = _("Go to news folder"),
+ callback = function()
+ self:openDownloadsFolder()
+ end,
+ },
+ {
+ text = _("Sync news feeds"),
+ keep_menu_open = true,
+ callback = function(touchmenu_instance)
+ NetworkMgr:runWhenOnline(function() self:loadConfigAndProcessFeedsWithUI(touchmenu_instance) end)
+ end,
+ },
+ {
+ text = _("Edit news feeds"),
+ keep_menu_open = true,
+ callback = function()
+ local Trapper = require("ui/trapper")
+ Trapper:wrap(function()
+ self:viewFeedList()
+ end)
+
+ end,
+ },
+ {
+ text = _("Settings"),
+ sub_item_table = {
+ {
+ text = _("Set download folder"),
+ keep_menu_open = true,
+ callback = function() self:setCustomDownloadDirectory() end,
+ },
+ {
+ text = _("Never download images"),
+ keep_menu_open = true,
+ checked_func = function()
+ return self.settings:isTrue("never_download_images")
+ end,
+ callback = function()
+ self.settings:toggle("never_download_images")
+ self.settings:flush()
+ end,
+ },
+ {
+ text = _("Delete all downloaded items"),
+ keep_menu_open = true,
+ callback = function()
+ local Trapper = require("ui/trapper")
+ Trapper:wrap(function()
+ local should_delete = Trapper:confirm(
+ _("Are you sure you want to delete all downloaded items?"),
+ _("Cancel"),
+ _("Delete")
+ )
+ if should_delete then
+ self:removeNewsButKeepFeedConfig()
+ Trapper:reset()
+ else
+ Trapper:reset()
+ end
+ end)
+ end,
},
},
- {
- text = _("Help"),
- keep_menu_open = true,
- callback = function()
- UIManager:show(InfoMessage:new{
- text = T(_("News downloader retrieves RSS and Atom news entries and stores them to:\n%1\n\nEach entry is a separate html file, that can be browsed by KOReader file manager.\nItems download limit can be configured in Settings."),
- BD.dirpath(news_download_dir_path))
- })
- end,
- },
+ },
+ {
+ text = _("About"),
+ keep_menu_open = true,
+ callback = function()
+ UIManager:show(InfoMessage:new{
+ text = T(_("News downloader retrieves RSS and Atom news entries and stores them to:\n%1\n\nEach entry is a separate EPUB file that can be browsed by KOReader.\nFeeds can be configured with download limits and other customization through the Edit Feeds menu item."),
+ BD.dirpath(self.download_dir))
+ })
+ end,
},
}
+ return sub_item_table
end
-
+-- lazyInitialization sets up variables that point to the
+-- downloads folder and the feeds configuration file.
function NewsDownloader:lazyInitialization()
- if not initialized then
+ if not self.initialized then
logger.dbg("NewsDownloader: obtaining news folder")
- news_downloader_settings = LuaSettings:open(("%s/%s"):format(DataStorage:getSettingsDir(), news_downloader_config_file))
- if news_downloader_settings:has(config_key_custom_dl_dir) then
- news_download_dir_path = news_downloader_settings:readSetting(config_key_custom_dl_dir)
+ self.settings = LuaSettings:open(("%s/%s"):format(DataStorage:getSettingsDir(), self.news_config_file))
+ -- Check to see if a custom download directory has been set.
+ if self.settings:has(self.config_key_custom_dl_dir) then
+ self.download_dir = self.settings:readSetting(self.config_key_custom_dl_dir)
else
- news_download_dir_path = ("%s/%s/"):format(DataStorage:getFullDataDir(), news_download_dir_name)
+ self.download_dir =
+ ("%s/%s/"):format(
+ DataStorage:getFullDataDir(),
+ self.download_dir_name)
end
-
- if not lfs.attributes(news_download_dir_path, "mode") then
+ logger.dbg("NewsDownloader: Custom directory set to:", self.download_dir)
+ -- If the directory doesn't exist we will create it.
+ if not lfs.attributes(self.download_dir, "mode") then
logger.dbg("NewsDownloader: Creating initial directory")
- lfs.mkdir(news_download_dir_path)
+ lfs.mkdir(self.download_dir)
end
- feed_config_path = news_download_dir_path .. feed_config_file_name
-
- if not lfs.attributes(feed_config_path, "mode") then
+ -- Now set the path to the feed configuration file.
+ self.feed_config_path = self.download_dir .. self.feed_config_file
+ -- If the configuration file doesn't exist create it.
+ if not lfs.attributes(self.feed_config_path, "mode") then
logger.dbg("NewsDownloader: Creating initial feed config.")
- FFIUtil.copyFile(FFIUtil.joinPath(self.path, feed_config_file_name),
- feed_config_path)
+ FFIUtil.copyFile(FFIUtil.joinPath(self.path, self.feed_config_file),
+ self.feed_config_path)
end
- initialized = true
+ self.initialized = true
end
end
-function NewsDownloader:loadConfigAndProcessFeeds()
+function NewsDownloader:loadConfigAndProcessFeeds(touchmenu_instance)
local UI = require("ui/trapper")
- UI:info("Loading news feed config…")
logger.dbg("force repaint due to upcoming blocking calls")
- local ok, feed_config = pcall(dofile, feed_config_path)
+ local ok, feed_config = pcall(dofile, self.feed_config_path)
if not ok or not feed_config then
UI:info(T(_("Invalid configuration file. Detailed error message:\n%1"), feed_config))
return
end
-
+ -- If the file contains no table elements, then the user hasn't set any feeds.
if #feed_config <= 0 then
- logger.err('NewsDownloader: empty feed list.', feed_config_path)
+ logger.err("NewsDownloader: empty feed list.", self.feed_config_path)
+ local should_edit_feed_list = UI:confirm(
+ T(_("Feed list is empty. If you want to download news, you'll have to add a feed first.")),
+ _("Close"),
+ _("Edit feed list")
+ )
+ if should_edit_feed_list then
+ -- Show the user a blank feed view so they can
+ -- add a feed to their list.
+ local feed_item_vc = FeedView:getItem(
+ 1,
+ self.empty_feed,
+ function(id, edit_key, value)
+ self:editFeedAttribute(id, edit_key, value)
+ end
+ )
+ self:viewFeedItem(
+ feed_item_vc
+ )
+ end
return
end
- local never_download_images = news_downloader_settings:isTrue("never_download_images")
-
+ local never_download_images = self.settings:isTrue("never_download_images")
local unsupported_feeds_urls = {}
-
local total_feed_entries = #feed_config
+ local feed_message
+
for idx, feed in ipairs(feed_config) do
local url = feed[1]
local limit = feed.limit
@@ -186,66 +253,179 @@ function NewsDownloader:loadConfigAndProcessFeeds()
local include_images = not never_download_images and feed.include_images
local enable_filter = feed.enable_filter or feed.enable_filter == nil
local filter_element = feed.filter_element or feed.filter_element == nil
+ -- Check if the two required attributes are set.
if url and limit then
- local feed_message = T(_("Processing %1/%2:\n%3"), idx, total_feed_entries, BD.url(url))
+ feed_message = T(_("Processing %1/%2:\n%3"), idx, total_feed_entries, BD.url(url))
UI:info(feed_message)
- NewsDownloader:processFeedSource(url, tonumber(limit), unsupported_feeds_urls, download_full_article, include_images, feed_message, enable_filter, filter_element)
+ -- Process the feed source.
+ self:processFeedSource(
+ url,
+ tonumber(limit),
+ unsupported_feeds_urls,
+ download_full_article,
+ include_images,
+ feed_message,
+ enable_filter,
+ filter_element)
else
- logger.warn('NewsDownloader: invalid feed config entry', feed)
+ logger.warn("NewsDownloader: invalid feed config entry.", feed)
end
end
if #unsupported_feeds_urls <= 0 then
- UI:info("Downloading news finished.")
+ -- When no errors are present, we get a happy message.
+ feed_message = _("Downloading news finished. ")
else
+ -- When some errors are present, we get a sour message that includes
+ -- information about the source of the error.
local unsupported_urls = ""
- for k,url in pairs(unsupported_feeds_urls) do
- unsupported_urls = unsupported_urls .. url
- if k ~= #unsupported_feeds_urls then
+ for key, value in pairs(unsupported_feeds_urls) do
+ -- Create the error message.
+ unsupported_urls = unsupported_urls .. " " .. value[1] .. " " .. value[2]
+ -- Not sure what this does.
+ if key ~= #unsupported_feeds_urls then
unsupported_urls = BD.url(unsupported_urls) .. ", "
end
end
- UI:info(T(_("Downloading news finished. Could not process some feeds. Unsupported format in: %1"), unsupported_urls))
+ -- Tell the user there were problems.
+ feed_message = _("Downloading news finished with errors. ")
+ -- Display a dialogue that requires the user to acknowledge
+ -- that errors occured.
+ UI:confirm(
+ T(_([[
+Could not process some feeds.
+Unsupported format in: %1. Please
+review your feed configuration file.]])
+ , unsupported_urls),
+ _("Continue"),
+ ""
+ )
+ end
+ -- Clear the info widgets before displaying the next ui widget.
+ UI:clear()
+ -- Check to see if this method was called from the menu. If it was,
+ -- we will have gotten a touchmenu_instance. This will context gives the user
+ -- two options about what to do next, which are handled by this block.
+ if touchmenu_instance then
+ -- Ask the user if they want to go to their downloads folder
+ -- or if they'd rather remain at the menu.
+ feed_message = feed_message .. _("Go to downloaders folder?")
+ local should_go_to_downloads = UI:confirm(
+ feed_message,
+ _("Close"),
+ _("Go to downloads")
+ )
+ if should_go_to_downloads then
+ -- Go to downloads folder.
+ UI:clear()
+ self:openDownloadsFolder()
+ touchmenu_instance:closeMenu()
+ NetworkMgr:afterWifiAction()
+ return
+ else
+ -- Return to the menu.
+ NetworkMgr:afterWifiAction()
+ return
+ end
end
- NetworkMgr:afterWifiAction()
+ return
end
-function NewsDownloader:loadConfigAndProcessFeedsWithUI()
+function NewsDownloader:loadConfigAndProcessFeedsWithUI(touchmenu_instance)
local Trapper = require("ui/trapper")
Trapper:wrap(function()
- self.loadConfigAndProcessFeeds()
+ self:loadConfigAndProcessFeeds(touchmenu_instance)
end)
end
function NewsDownloader:processFeedSource(url, limit, unsupported_feeds_urls, download_full_article, include_images, message, enable_filter, filter_element)
-
local ok, response = pcall(function()
- return DownloadBackend:getResponseAsString(url)
+ return DownloadBackend:getResponseAsString(url)
end)
local feeds
+ -- Check to see if a response is available to deserialize.
if ok then
feeds = self:deserializeXMLString(response)
end
-
+ -- If the response is not available (for a reason that we don't know),
+ -- add the URL to the unsupported feeds list.
if not ok or not feeds then
- table.insert(unsupported_feeds_urls, url)
+ local error_message
+ if not ok then
+ error_message = _("(Reason: Failed to download content)")
+ else
+ error_message = _("(Reason: Error during feed deserialization)")
+ end
+ table.insert(
+ unsupported_feeds_urls,
+ {
+ url,
+ error_message
+ }
+ )
return
end
-
- local is_rss = feeds.rss and feeds.rss.channel and feeds.rss.channel.title and feeds.rss.channel.item and feeds.rss.channel.item[1] and feeds.rss.channel.item[1].title and feeds.rss.channel.item[1].link
- local is_atom = feeds.feed and feeds.feed.title and feeds.feed.entry[1] and feeds.feed.entry[1].title and feeds.feed.entry[1].link
-
+ -- Check to see if the feed uses RSS.
+ local is_rss = feeds.rss
+ and feeds.rss.channel
+ and feeds.rss.channel.title
+ and feeds.rss.channel.item
+ and feeds.rss.channel.item[1]
+ and feeds.rss.channel.item[1].title
+ and feeds.rss.channel.item[1].link
+ -- Check to see if the feed uses Atom.
+ local is_atom = feeds.feed
+ and feeds.feed.title
+ and feeds.feed.entry[1]
+ and feeds.feed.entry[1].title
+ and feeds.feed.entry[1].link
+ -- Process the feeds accordingly.
if is_atom then
ok = pcall(function()
- return self:processAtom(feeds, limit, download_full_article, include_images, message, enable_filter, filter_element)
+ return self:processFeed(
+ FEED_TYPE_ATOM,
+ feeds,
+ limit,
+ download_full_article,
+ include_images,
+ message,
+ enable_filter,
+ filter_element
+ )
end)
elseif is_rss then
ok = pcall(function()
- return self:processRSS(feeds, limit, download_full_article, include_images, message, enable_filter, filter_element)
+ return self:processFeed(
+ FEED_TYPE_RSS,
+ feeds,
+ limit,
+ download_full_article,
+ include_images,
+ message,
+ enable_filter,
+ filter_element
+ )
end)
end
+ -- If the feed can't be processed, or it is neither
+ -- Atom or RSS, then add it to the unsupported feeds list
+ -- and return an error message.
if not ok or (not is_rss and not is_atom) then
- table.insert(unsupported_feeds_urls, url)
+ local error_message
+ if not ok then
+ error_message = _("(Reason: Failed to download content)")
+ elseif not is_rss then
+ error_message = _("(Reason: Couldn't process RSS)")
+ elseif not is_atom then
+ error_message = _("(Reason: Couldn't process Atom)")
+ end
+ table.insert(
+ unsupported_feeds_urls,
+ {
+ url,
+ error_message
+ }
+ )
end
end
@@ -256,72 +436,148 @@ function NewsDownloader:deserializeXMLString(xml_str)
-- see: koreader/plugins/newsdownloader.koplugin/lib/LICENSE_LuaXML
local treehdl = require("lib/handler")
local libxml = require("lib/xml")
-
- --Instantiate the object the states the XML file as a Lua table
+ -- Instantiate the object that parses the XML file as a Lua table.
local xmlhandler = treehdl.simpleTreeHandler()
- --Instantiate the object that parses the XML to a Lua table
+ -- Instantiate the object that parses the XML to a Lua table.
local ok = pcall(function()
- libxml.xmlParser(xmlhandler):parse(xml_str)
+ libxml.xmlParser(xmlhandler):parse(xml_str)
end)
if not ok then return end
return xmlhandler.root
end
function NewsDownloader:processAtom(feeds, limit, download_full_article, include_images, message, enable_filter, filter_element)
- local feed_output_dir = string.format("%s%s/",
- news_download_dir_path,
- util.getSafeFilename(getFeedTitle(feeds.feed.title)))
- if not lfs.attributes(feed_output_dir, "mode") then
+ -- Get the path to the output directory.
+ local feed_output_dir = string.format(
+ "%s%s/",
+ self.download_dir,
+ util.getSafeFilename(getFeedTitle(feeds.feed.title))
+ )
+ -- Create the output directory if it doesn't exist.
+ if not lfs.attributes(feed_outpnnut_dir, "mode") then
lfs.mkdir(feed_output_dir)
end
-
+ -- Download the feed.
for index, feed in pairs(feeds.feed.entry) do
+ -- If limit has been met, stop downloading feed.
if limit ~= 0 and index - 1 == limit then
break
end
- local article_message = T(_("%1\n\nFetching article %2/%3:"), message, index, limit == 0 and #feeds.rss.channel.item or limit)
+ local total_articles = limit == 0
+ and #feeds.rss.channel.item
+ or limit
+ -- Create a message to display during processing.
+ local article_message = T(
+ _("%1\n\nFetching article %2/%3:"),
+ message,
+ index,
+ total
+ )
+ -- Download the feed.
if download_full_article then
- self:downloadFeed(feed, feed_output_dir, include_images, article_message, enable_filter, filter_element)
+ self:downloadFeed(
+ feed,
+ feed_output_dir,
+ include_images,
+ article_message,
+ enable_filter,
+ filter_element
+ )
else
- self:createFromDescription(feed, feed.content[1], feed_output_dir, include_images, article_message)
+ self:createFromDescription(
+ feed,
+ feed.content[1],
+ feed_output_dir,
+ include_images,
+ article_message
+ )
end
end
end
-function NewsDownloader:processRSS(feeds, limit, download_full_article, include_images, message, enable_filter, filter_element)
+function NewsDownloader:processFeed(feed_type, feeds, limit, download_full_article, include_images, message, enable_filter, filter_element)
+ local feed_title
+ local feed_item
+ local total_feeds
+ -- Setup the above vars based on feed type.
+ if feed_type == FEED_TYPE_RSS then
+ feed_title = util.htmlEntitiesToUtf8(feeds.rss.channel.title)
+ feed_item = feeds.rss.channel.item
+ total_items = (limit == 0)
+ and #feeds.rss.channel.item
+ or limit
+ else
+ feed_title = getFeedTitle(feeds.feed.title)
+ feed_item = feeds.feed.entry
+ total_items = (limit == 0)
+ and #feeds.feed.entry
+ or limit
+ end
+ -- Get the path to the output directory.
local feed_output_dir = ("%s%s/"):format(
- news_download_dir_path, util.getSafeFilename(util.htmlEntitiesToUtf8(feeds.rss.channel.title)))
+ self.download_dir,
+ util.getSafeFilename(util.htmlEntitiesToUtf8(feed_title)))
+ -- Create the output directory if it doesn't exist.
if not lfs.attributes(feed_output_dir, "mode") then
lfs.mkdir(feed_output_dir)
end
-
- for index, feed in pairs(feeds.rss.channel.item) do
+ -- Download the feed
+ for index, feed in pairs(feed_item) do
+ -- If limit has been met, stop downloading feed.
if limit ~= 0 and index - 1 == limit then
break
end
- local article_message = T(_("%1\n\nFetching article %2/%3:"), message, index, limit == 0 and #feeds.rss.channel.item or limit)
+ -- Create a message to display during processing.
+ local article_message = T(
+ _("%1\n\nFetching article %2/%3:"),
+ message,
+ index,
+ total_items
+ )
+ -- Get the feed description.
+ local feed_description
+ if feed_type == FEED_TYPE_RSS then
+ feed_description = feed.description
+ else
+ feed_description = feed.summary
+ end
+ -- Download the article.
if download_full_article then
- self:downloadFeed(feed, feed_output_dir, include_images, article_message, enable_filter, filter_element)
+ self:downloadFeed(
+ feed,
+ feed_output_dir,
+ include_images,
+ article_message,
+ enable_filter,
+ filter_element
+ )
else
- self:createFromDescription(feed, feed.description, feed_output_dir, include_images, article_message)
+ self:createFromDescription(
+ feed,
+ feed_description,
+ feed_output_dir,
+ include_images,
+ article_message
+ )
end
end
end
local function parseDate(dateTime)
- -- uses lua-feedparser https://github.com/slact/lua-feedparser
+ -- Uses lua-feedparser https://github.com/slact/lua-feedparser
-- feedparser is available under the (new) BSD license.
-- see: koreader/plugins/newsdownloader.koplugin/lib/LICENCE_lua-feedparser
local date = dateparser.parse(dateTime)
return os.date("%y-%m-%d_%H-%M_", date)
end
+-- This appears to be used by Atom feeds in processFeed.
local function getTitleWithDate(feed)
local title = util.getSafeFilename(getFeedTitle(feed.title))
if feed.updated then
- title = parseDate(feed.updated) .. title
+ title = parseDate(feed.updated) .. title
elseif feed.pubDate then
- title = parseDate(feed.pubDate) .. title
+ title = parseDate(feed.pubDate) .. title
elseif feed.published then
title = parseDate(feed.published) .. title
end
@@ -332,7 +588,7 @@ function NewsDownloader:downloadFeed(feed, feed_output_dir, include_images, mess
local title_with_date = getTitleWithDate(feed)
local news_file_path = ("%s%s%s"):format(feed_output_dir,
title_with_date,
- file_extension)
+ self.file_extension)
local file_mode = lfs.attributes(news_file_path, "mode")
if file_mode == "file" then
@@ -350,7 +606,7 @@ function NewsDownloader:createFromDescription(feed, content, feed_output_dir, in
local title_with_date = getTitleWithDate(feed)
local news_file_path = ("%s%s%s"):format(feed_output_dir,
title_with_date,
- file_extension)
+ self.file_extension)
local file_mode = lfs.attributes(news_file_path, "mode")
if file_mode == "file" then
logger.dbg("NewsDownloader:", news_file_path, "already exists. Skipping")
@@ -372,10 +628,10 @@ function NewsDownloader:createFromDescription(feed, content, feed_output_dir, in
end
function NewsDownloader:removeNewsButKeepFeedConfig()
- logger.dbg("NewsDownloader: Removing news from :", news_download_dir_path)
- for entry in lfs.dir(news_download_dir_path) do
- if entry ~= "." and entry ~= ".." and entry ~= feed_config_file_name then
- local entry_path = news_download_dir_path .. "/" .. entry
+ logger.dbg("NewsDownloader: Removing news from :", self.download_dir)
+ for entry in lfs.dir(self.download_dir) do
+ if entry ~= "." and entry ~= ".." and entry ~= self.feed_config_file then
+ local entry_path = self.download_dir .. "/" .. entry
local entry_mode = lfs.attributes(entry_path, "mode")
if entry_mode == "file" then
os.remove(entry_path)
@@ -385,33 +641,409 @@ function NewsDownloader:removeNewsButKeepFeedConfig()
end
end
UIManager:show(InfoMessage:new{
- text = _("All news removed.")
+ text = _("All downloaded news feed items deleted.")
})
end
function NewsDownloader:setCustomDownloadDirectory()
require("ui/downloadmgr"):new{
- onConfirm = function(path)
- logger.dbg("NewsDownloader: set download directory to: ", path)
- news_downloader_settings:saveSetting(config_key_custom_dl_dir, ("%s/"):format(path))
- news_downloader_settings:flush()
-
- logger.dbg("NewsDownloader: Coping to new download folder previous feed_config_file_name from: ", feed_config_path)
- FFIUtil.copyFile(feed_config_path, ("%s/%s"):format(path, feed_config_file_name))
-
- initialized = false
- self:lazyInitialization()
- end,
- }:chooseDir()
+ onConfirm = function(path)
+ logger.dbg("NewsDownloader: set download directory to: ", path)
+ self.settings:saveSetting(self.config_key_custom_dl_dir, ("%s/"):format(path))
+ self.settings:flush()
+
+ logger.dbg("NewsDownloader: Coping to new download folder previous self.feed_config_file from: ", self.feed_config_path)
+ FFIUtil.copyFile(self.feed_config_path, ("%s/%s"):format(path, self.feed_config_file))
+
+ self.initialized = false
+ self:lazyInitialization()
+ end,
+ }:chooseDir()
+end
+
+function NewsDownloader:viewFeedList()
+ local UI = require("ui/trapper")
+ UI:info(_("Loading news feed list…"))
+ -- Protected call to see if feed config path returns a file that can be opened.
+ local ok, feed_config = pcall(dofile, self.feed_config_path)
+ if not ok or not feed_config then
+ change_feed_config = UI:confirm(
+ _("Could not open feed list. Feeds configuration file is invalid. "),
+ _("Close"),
+ _("View file")
+ )
+ if change_feed_config then
+ self:changeFeedConfig()
+ end
+ return
+ end
+ UI:clear()
+ -- See if the config file contains any feed items
+ if #feed_config <= 0 then
+ logger.err("NewsDownloader: empty feed list.", self.feed_config_path)
+ -- Why not ask the user if they want to add one?
+ -- Or, in future, move along to our list UI with an entry for new feeds
+
+ --return
+ end
+
+ local view_content = FeedView:getList(
+ feed_config,
+ function(feed_item_vc)
+ self:viewFeedItem(
+ feed_item_vc
+ )
+ end,
+ function(id, edit_key, value)
+ self:editFeedAttribute(id, edit_key, value)
+ end,
+ function(id)
+ self:deleteFeed(id)
+ end
+ )
+ -- Add a "Add new feed" button with callback
+ table.insert(
+ view_content,
+ {
+ _("Add new feed"),
+ "",
+ callback = function()
+ -- Prepare the view with all the callbacks for editing the attributes
+ local feed_item_vc = FeedView:getItem(
+ #feed_config + 1,
+ self.empty_feed,
+ function(id, edit_key, value)
+ self:editFeedAttribute(id, edit_key, value)
+ end
+ )
+ self:viewFeedItem(
+ feed_item_vc
+ )
+ end
+ }
+ )
+ -- Show the list of feeds.
+ local kv = self.kv
+ if #self.kv ~= 0 then
+ UIManager:close(self.kv)
+ end
+ self.kv = KeyValuePage:new{
+ title = _("RSS/Atom Feeds List"),
+ value_overflow_align = "right",
+ kv_pairs = view_content,
+ callback_return = function()
+ UIManager:close(self.kv)
+ end
+ }
+ UIManager:show(self.kv)
+end
+
+function NewsDownloader:viewFeedItem(data)
+ local kv = self.kv
+ if #self.kv ~= 0 then
+ UIManager:close(self.kv)
+ end
+ self.kv = KeyValuePage:new{
+ title = _("Edit Feed"),
+ value_overflow_align = "left",
+ kv_pairs = data,
+ callback_return = function()
+ self:viewFeedList()
+ end
+ }
+ UIManager:show(self.kv)
+end
+
+function NewsDownloader:editFeedAttribute(id, key, value)
+ local kv = self.kv
+ -- There are basically two types of values: string (incl. numbers)
+ -- and booleans. This block chooses what type of value our
+ -- attribute will need and displays the corresponding dialog.
+ if key == FeedView.URL
+ or key == FeedView.LIMIT
+ or key == FeedView.FILTER_ELEMENT then
+
+ local title
+ local input_type
+ local description
+
+ if key == FeedView.URL then
+ title = _("Edit feed URL")
+ input_type = "string"
+ elseif key == FeedView.LIMIT then
+ title = _("Edit feed limit")
+ description = _("Set to 0 for no limit to how many items are downloaded")
+ input_type = "number"
+ elseif key == FeedView.FILTER_ELEMENT then
+ title = _("Edit filter element.")
+ description = _("Filter based on the given CSS selector. E.g.: name_of_css.element.class")
+ input_type = "string"
+ else
+ return false
+ end
+
+ local input_dialog
+ input_dialog = InputDialog:new{
+ title = title,
+ input = value,
+ input_type = input_type,
+ description = description,
+ buttons = {
+ {
+ {
+ text = _("Cancel"),
+ callback = function()
+ UIManager:close(input_dialog)
+ UIManager:show(kv)
+ end,
+ },
+ {
+ text = _("Save"),
+ is_enter_default = true,
+ callback = function()
+ UIManager:close(input_dialog)
+ self:updateFeedConfig(id, key, input_dialog:getInputValue())
+ end,
+ },
+ }
+ },
+ }
+ UIManager:show(input_dialog)
+ input_dialog:onShowKeyboard()
+ return true
+ else
+ local text
+ if key == FeedView.DOWNLOAD_FULL_ARTICLE then
+ text = _("Download full article?")
+ elseif key == FeedView.INCLUDE_IMAGES then
+ text = _("Include images?")
+ elseif key == FeedView.ENABLE_FILTER then
+ text = _("Enable CSS filter?")
+ end
+
+ local multi_box
+ multi_box= MultiConfirmBox:new{
+ text = text,
+ choice1_text = _("Yes"),
+ choice1_callback = function()
+ UIManager:close(multi_box)
+ self:updateFeedConfig(id, key, true)
+ end,
+ choice2_text = _("No"),
+ choice2_callback = function()
+ UIManager:close(multi_box)
+ self:updateFeedConfig(id, key, false)
+ end,
+ cancel_callback = function()
+ UIManager:close(multi_box)
+ UIManager:show(kv)
+ end,
+ }
+ UIManager:show(multi_box)
+ end
+end
+
+function NewsDownloader:updateFeedConfig(id, key, value)
+ local kv = self.kv
+ -- Because this method is called at the menu,
+ -- we might not have an active view. So this conditional
+ -- statement avoids closing a null reference.
+ if #self.kv ~= 0 then
+ UIManager:close(self.kv)
+ end
+ -- It's possible that we will get a null value.
+ -- This block catches that possibility.
+ if id ~= nil and key ~= nil and value ~= nil then
+ -- This logger is a bit opaque because T() wasn't playing nice with booleans
+ logger.dbg("Newsdownloader: attempting to update config:")
+ else
+ logger.dbg("Newsdownloader: null value supplied to update. Not updating config")
+ return
+ end
+
+ local ok, feed_config = pcall(dofile, self.feed_config_path)
+ if not ok or not feed_config then
+ UI:info(T(_("Invalid configuration file. Detailed error message:\n%1"), feed_config))
+ return
+ end
+ -- If the file contains no table elements, then the user hasn't set any feeds.
+ if #feed_config <= 0 then
+ logger.dbg("NewsDownloader: empty feed list.", self.feed_config_path)
+ end
+
+ -- Check to see if the id is larger than the number of feeds. If it is,
+ -- then we know this is a new add. Insert the base array.
+ if id > #feed_config then
+ table.insert(
+ feed_config,
+ self.empty_feed
+ )
+ end
+
+ local new_config = {}
+ -- In this loop, we cycle through the feed items. A series of
+ -- conditionals checks to see if we are at the right id
+ -- and key (i.e.: the key that triggered this function.
+ -- If we are at the right spot, we overrite (or create) the value
+ for idx, feed in ipairs(feed_config) do
+ -- Check to see if this is the correct feed to update.
+ if idx == id then
+ if key == FeedView.URL then
+ if feed[1] then
+ -- If the value exists, then it's been set. So all we do
+ -- is overwrite the value.
+ feed[1] = value
+ else
+ -- If the value doesn't exist, then we need to set it.
+ -- So we insert it into the table.
+ table.insert(
+ feed,
+ {
+ value
+ }
+ )
+ end
+ elseif key == FeedView.LIMIT then
+ if feed.limit then
+ feed.limit = value
+ else
+ table.insert(
+ feed,
+ {
+ "limit",
+ value
+ }
+ )
+ end
+ elseif key == FeedView.DOWNLOAD_FULL_ARTICLE then
+ if feed.download_full_article ~= nil then
+ feed.download_full_article = value
+ else
+ table.insert(
+ feed,
+ {
+ "download_full_article",
+ value
+ }
+ )
+ end
+ elseif key == FeedView.INCLUDE_IMAGES then
+ if feed.include_images ~= nil then
+ feed.include_images = value
+ else
+ table.insert(
+ feed,
+ {
+ "include_images",
+ value
+ }
+ )
+ end
+ elseif key == FeedView.ENABLE_FILTER then
+ if feed.enable_filter ~= nil then
+ feed.enable_filter = value
+ else
+ table.insert(
+ feed,
+ {
+ "enable_filter",
+ value
+ }
+ )
+ end
+ elseif key == FeedView.FILTER_ELEMENT then
+ if feed.filter_element then
+ feed.filter_element = value
+ else
+ table.insert(
+ feed,
+ {
+ "filter_element",
+ value
+ }
+ )
+ end
+ end
+ end
+ -- Now we insert the updated (or newly created) feed into the
+ -- new config feed that we're building in this loop.
+ table.insert(
+ new_config,
+ feed
+ )
+ end
+ -- Save the config
+ logger.dbg("NewsDownloader: config to save", new_config)
+ self:saveConfig(new_config)
+ -- Refresh the view
+ local feed_item_vc = FeedView:getItem(
+ id,
+ new_config[id],
+ function(id, edit_key, value)
+ self:editFeedAttribute(id, edit_key, value)
+ end
+ )
+ self:viewFeedItem(
+ feed_item_vc
+ )
+
+end
+
+function NewsDownloader:deleteFeed(id)
+ logger.dbg("Newsdownloader: attempting to delete feed")
+ -- Check to see if we can get the config file.
+ local ok, feed_config = pcall(dofile, self.feed_config_path)
+ if not ok or not feed_config then
+ UI:info(T(_("Invalid configuration file. Detailed error message:\n%1"), feed_config))
+ return
+ end
+ -- In this loop, we cycle through the feed items. A series of
+ -- conditionals checks to see if we are at the right id
+ -- and key (i.e.: the key that triggered this function.
+ -- If we are at the right spot, we overrite (or create) the value
+ local new_config = {}
+ for idx, feed in ipairs(feed_config) do
+ -- Check to see if this is the correct feed to update.
+ if idx ~= id then
+ table.insert(
+ new_config,
+ feed
+ )
+ end
+ end
+ -- Save the config
+ local Trapper = require("ui/trapper")
+ Trapper:wrap(function()
+ logger.dbg("NewsDownloader: config to save", new_config)
+ self:saveConfig(new_config)
+ end)
+ -- Refresh the view
+ self:viewFeedList()
+end
+
+function NewsDownloader:saveConfig(config)
+ local UI = require("ui/trapper")
+ UI:info(_("Saving news feed list..."))
+ local persist = Persist:new{
+ path = self.feed_config_path
+ }
+ local ok = persist:save(config)
+ if not ok then
+ UI:info(_("Could not save news feed config."))
+ else
+ UI:info(_("News feed config updated successfully."))
+ end
+ UI:reset()
end
function NewsDownloader:changeFeedConfig()
- local feed_config_file = io.open(feed_config_path, "rb")
+ local feed_config_file = io.open(self.feed_config_path, "rb")
local config = feed_config_file:read("*all")
feed_config_file:close()
local config_editor
+ logger.info("NewsDownloader: opening configuration file", self.feed_config_path)
config_editor = InputDialog:new{
- title = T(_("Config: %1"), BD.filepath(feed_config_path)),
+ title = T(_("Config: %1"), BD.filepath(self.feed_config_path)),
input = config,
input_type = "string",
para_direction_rtl = false, -- force LTR
@@ -429,7 +1061,7 @@ function NewsDownloader:changeFeedConfig()
if not parse_error then
local syntax_okay, syntax_error = pcall(loadstring(content))
if syntax_okay then
- feed_config_file = io.open(feed_config_path, "w")
+ feed_config_file = io.open(self.feed_config_path, "w")
feed_config_file:write(content)
feed_config_file:close()
return true, _("Configuration saved")
@@ -437,8 +1069,8 @@ function NewsDownloader:changeFeedConfig()
return false, T(_("Configuration invalid: %1"), syntax_error)
end
else
- return false, T(_("Configuration invalid: %1"), parse_error)
- end
+ return false, T(_("Configuration invalid: %1"), parse_error)
+ end
end
return false, _("Configuration empty")
end,
@@ -446,12 +1078,23 @@ function NewsDownloader:changeFeedConfig()
UIManager:show(config_editor)
config_editor:onShowKeyboard()
end
+function NewsDownloader:openDownloadsFolder()
+ local FileManager = require("apps/filemanager/filemanager")
+ if self.ui.document then
+ self.ui:onClose()
+ end
+ if FileManager.instance then
+ FileManager.instance:reinit(self.download_dir)
+ else
+ FileManager:showFiles(self.download_dir)
+ end
+end
function NewsDownloader:onCloseDocument()
local document_full_path = self.ui.document.file
- if document_full_path and news_download_dir_path and news_download_dir_path == string.sub(document_full_path, 1, string.len(news_download_dir_path)) then
+ if document_full_path and self.download_dir and self.download_dir == string.sub(document_full_path, 1, string.len(self.download_dir)) then
logger.dbg("NewsDownloader: document_full_path:", document_full_path)
- logger.dbg("NewsDownloader: news_download_dir_path:", news_download_dir_path)
+ logger.dbg("NewsDownloader: self.download_dir:", self.download_dir)
logger.dbg("NewsDownloader: removing NewsDownloader file from history.")
ReadHistory:removeItemByPath(document_full_path)
local doc_dir = util.splitFilePathName(document_full_path)
@@ -459,4 +1102,38 @@ function NewsDownloader:onCloseDocument()
end
end
+--
+-- KeyValuePage doesn't like to get a table with sub tables.
+-- This function flattens an array, moving all nested tables
+-- up the food chain, so to speak
+--
+function NewsDownloader:flattenArray(base_array, source_array)
+ for key, value in pairs(source_array) do
+ if value[2] == nil then
+ -- If the value is empty, then it's probably supposed to be a line
+ table.insert(
+ base_array,
+ "---"
+ )
+ else
+ if value["callback"] then
+ table.insert(
+ base_array,
+ {
+ value[1], value[2], callback = value["callback"]
+ }
+ )
+ else
+ table.insert(
+ base_array,
+ {
+ value[1], value[2]
+ }
+ )
+ end
+ end
+ end
+ return base_array
+end
+
return NewsDownloader