diff --git a/frontend/ui/elements/filemanager_menu_order.lua b/frontend/ui/elements/filemanager_menu_order.lua index b9c463351..80bbf95e2 100644 --- a/frontend/ui/elements/filemanager_menu_order.lua +++ b/frontend/ui/elements/filemanager_menu_order.lua @@ -36,7 +36,7 @@ local order = { "storage_stat", "cloud_storage", "read_timer", - "rss_news_downloader", + "news_downloader", "synchronize_time", "terminal", "----------------------------", diff --git a/frontend/ui/elements/reader_menu_order.lua b/frontend/ui/elements/reader_menu_order.lua index 449f6c61d..392706550 100644 --- a/frontend/ui/elements/reader_menu_order.lua +++ b/frontend/ui/elements/reader_menu_order.lua @@ -59,7 +59,7 @@ local order = { "synchronize_time", "progress_sync", "zsync", - "rss_news_downloader", + "news_downloader", "terminal", }, search = { diff --git a/plugins/newsdownloader.koplugin/feed_config.lua b/plugins/newsdownloader.koplugin/feed_config.lua index 6764c58de..896d8c367 100644 --- a/plugins/newsdownloader.koplugin/feed_config.lua +++ b/plugins/newsdownloader.koplugin/feed_config.lua @@ -1,9 +1,8 @@ return { -- list your feeds here: -- only supports http URL for now - -- Atom is currently not supported, only RSS { "http://www.pcworld.com/index.rss", limit = 1 }, - { "http://www.economist.com/sections/science-technology/rss.xml", limit = 2}, + { "http://feeds.reuters.com/Reuters/worldNews?format=xml", limit = 2}, -- set limit to "0" means no download, "-1" no limit. - { "http://www.economist.com/sections/culture/rss.xml", limit = 0 }, -} \ No newline at end of file + { "http://www.football.co.uk/international/rss.xml", limit = 0 }, +} diff --git a/plugins/newsdownloader.koplugin/main.lua b/plugins/newsdownloader.koplugin/main.lua index 3d622aa2f..1440ec3f8 100644 --- a/plugins/newsdownloader.koplugin/main.lua +++ b/plugins/newsdownloader.koplugin/main.lua @@ -15,10 +15,10 @@ local ltn12 = require("ltn12") local NewsDownloader = WidgetContainer:new{} local initialized = false -- for only once lazy initialization -local FEED_CONFIG_FILE = "feed_config.lua" -local FILE_EXTENSION = ".html" -local NEWS_DL_DIR_NAME = "news" -local NEWS_DL_DIR, FEED_CONFIG_PATH +local feed_config_file = "feed_config.lua" +local file_extension = ".html" +local news_download_dir_name = "news" +local news_download_dir_path, feed_config_path local function deserializeXMLString(xml_str) -- uses LuaXML https://github.com/manoelcampos/LuaXML @@ -43,16 +43,15 @@ end function NewsDownloader:addToMainMenu(menu_items) if not initialized then - NEWS_DL_DIR = ("%s/%s/"):format(DataStorage:getDataDir(), NEWS_DL_DIR_NAME) - if not lfs.attributes(NEWS_DL_DIR, "mode") then - lfs.mkdir(NEWS_DL_DIR) + news_download_dir_path = ("%s/%s/"):format(DataStorage:getDataDir(), news_download_dir_name) + if not lfs.attributes(news_download_dir_path, "mode") then + lfs.mkdir(news_download_dir_path) end - - FEED_CONFIG_PATH = NEWS_DL_DIR .. FEED_CONFIG_FILE + feed_config_path = news_download_dir_path .. feed_config_file initialized = true end - menu_items.rss_news_downloader = { + menu_items.news_downloader = { text = _("News (RSS/Atom) downloader"), sub_item_table = { { @@ -64,9 +63,9 @@ function NewsDownloader:addToMainMenu(menu_items) callback = function() local FileManager = require("apps/filemanager/filemanager") if FileManager.instance then - FileManager.instance:reinit(NEWS_DL_DIR) + FileManager.instance:reinit(news_download_dir_path) else - FileManager:showFiles(NEWS_DL_DIR) + FileManager:showFiles(news_download_dir_path) end end, }, @@ -74,9 +73,9 @@ function NewsDownloader:addToMainMenu(menu_items) text = _("Remove news"), callback = function() -- puerge all downloaded news files, but keep the feed config - for entry in lfs.dir(NEWS_DL_DIR) do - if entry ~= "." and entry ~= ".." and entry ~= FEED_CONFIG_FILE then - local entry_path = NEWS_DL_DIR .. "/" .. entry + for entry in lfs.dir(news_download_dir_path) do + if entry ~= "." and entry ~= ".." and entry ~= feed_config_file then + local entry_path = news_download_dir_path .. "/" .. entry local entry_mode = lfs.attributes(entry_path, "mode") if entry_mode == "file" then ffi.C.remove(entry_path) @@ -95,8 +94,8 @@ function NewsDownloader:addToMainMenu(menu_items) callback = function() UIManager:show(InfoMessage:new{ text = T(_("News downloader can be configured in the feeds config file:\n%1\n\nIt downloads news items to:\n%2.\n\nTo set you own news sources edit foregoing feeds config file. Items download limit can be set there."), - FEED_CONFIG_PATH, - NEWS_DL_DIR) + feed_config_path, + news_download_dir_path) }) end, }, @@ -111,22 +110,24 @@ function NewsDownloader:loadConfigAndProcessFeeds() UIManager:forceRePaint() UIManager:close(info) - if not lfs.attributes(FEED_CONFIG_PATH, "mode") then + if not lfs.attributes(feed_config_path, "mode") then logger.dbg("NewsDownloader: Creating initial feed config.") - FFIUtil.copyFile(FFIUtil.joinPath(self.path, FEED_CONFIG_FILE), - FEED_CONFIG_PATH) + FFIUtil.copyFile(FFIUtil.joinPath(self.path, feed_config_file), + feed_config_path) end - local ok, feed_config = pcall(dofile, FEED_CONFIG_PATH) + local ok, feed_config = pcall(dofile, feed_config_path) if not ok or not feed_config then logger.info("NewsDownloader: Feed config not found.") return end if #feed_config <= 0 then - logger.info('NewsDownloader: empty feed list.', FEED_CONFIG_PATH) + logger.info('NewsDownloader: empty feed list.', feed_config_path) return end + local unsupported_feeds_urls = {} + for idx, feed in ipairs(feed_config) do local url = feed[1] local limit = feed.limit @@ -135,32 +136,73 @@ function NewsDownloader:loadConfigAndProcessFeeds() UIManager:show(info) -- processFeedSource is a blocking call, so manually force a UI refresh beforehand UIManager:forceRePaint() - self:processFeedSource(url, tonumber(limit)) + self:processFeedSource(url, tonumber(limit), unsupported_feeds_urls) UIManager:close(info) else logger.warn('NewsDownloader: invalid feed config entry', feed) end end - UIManager:show(InfoMessage:new{ - text = _("Downloading news finished."), - timeout = 1, - }) + if #unsupported_feeds_urls <= 0 then + UIManager:show(InfoMessage:new{ + text = _("Downloading news finished."), + timeout = 1, + }) + else + local unsupported_urls = "" + for k,url in pairs(unsupported_feeds_urls) do + unsupported_urls = unsupported_urls .. url + if k ~= #unsupported_feeds_urls then + unsupported_urls = unsupported_urls .. ", " + end + end + UIManager:show(InfoMessage:new{ + text = T(_("Downloading news finished. Could not process some feeds. Unsupported format in: %1"), unsupported_urls) + }) + end end -function NewsDownloader:processFeedSource(url, limit) +function NewsDownloader:processFeedSource(url, limit, unsupported_feeds_urls) local resp_lines = {} http.request({ url = url, sink = ltn12.sink.table(resp_lines), }) local feeds = deserializeXMLString(table.concat(resp_lines)) - if not feeds then return end - if not feeds.rss or not feeds.rss.channel - or not feeds.rss.channel.title or not feeds.rss.channel.item then - logger.info('NewsDownloader: Got invalid feeds', feeds) + if not feeds then + table.insert(unsupported_feeds_urls, url) + return + end + + local is_rss = feeds.rss and feeds.rss.channel and feeds.rss.channel.title and feeds.rss.channel.item and feeds.rss.channel.item[1] and feeds.rss.channel.item[1].title and feeds.rss.channel.item[1].link + local is_atom = feeds.feed and feeds.feed.title and feeds.feed.entry.title and feeds.feed.entry.link and feeds.feed.entry[1] and feeds.feed.entry[1].title and feeds.feed.entry[1].link + + if is_atom then + self:processAtom(feeds, limit) + elseif is_rss then + self:processRSS(feeds, limit) + else + table.insert(unsupported_feeds_urls, url) return end +end + +function NewsDownloader:processAtom(feeds, limit) + local feed_output_dir = string.format("%s%s/", + news_download_dir_path, + util.replaceInvalidChars(feeds.feed.title)) + if not lfs.attributes(feed_output_dir, "mode") then + lfs.mkdir(feed_output_dir) + end + + for index, feed in pairs(feeds.feed.entry) do + if index -1 == limit then + break + end + self:downloadFeed(feed, feed_output_dir) + end +end +function NewsDownloader:processRSS(feeds, limit) local feed_output_dir = ("%s%s/"):format( - NEWS_DL_DIR, util.replaceInvalidChars(feeds.rss.channel.title)) + news_download_dir_path, util.replaceInvalidChars(feeds.rss.channel.title)) if not lfs.attributes(feed_output_dir, "mode") then lfs.mkdir(feed_output_dir) end @@ -169,12 +211,17 @@ function NewsDownloader:processFeedSource(url, limit) if index -1 == limit then break end - local news_dl_path = ("%s%s%s"):format(feed_output_dir, - util.replaceInvalidChars(feed.title), - FILE_EXTENSION) - logger.dbg("NewsDownloader: News file will be stored to :", news_dl_path) - http.request({ url = url, sink = ltn12.sink.file(io.open(news_dl_path, 'w')), }) + self:downloadFeed(feed, feed_output_dir) end end +function NewsDownloader:downloadFeed(feed, feed_output_dir) + + local news_dl_path = ("%s%s%s"):format(feed_output_dir, + util.replaceInvalidChars(feed.title), + file_extension) + logger.dbg("NewsDownloader: News file will be stored to :", news_dl_path) + http.request({ url = feed.link, sink = ltn12.sink.file(io.open(news_dl_path, 'w')), }) +end + return NewsDownloader