Archives
/
koreader
mirror of https://github.com/koreader/koreader
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

News downloader atom support (#2736)

Browse Source 
* newsDownloader - atom support initial version

* NewsDownloader: update example feed config (use Reuters atom source)

* NewsDownloader: InfoMessage when unsupported urls

* NewsDownloader: update menu_item id

* NewsDownloader: change text label
pull/2880/head
mwoz123 7 years ago committed by Frans de Jonge
parent f21550b3d5
commit 4eb8664c95
4 changed files with 90 additions and 44 deletions
Show Stats Download Patch File Download Diff File

2
frontend/ui/elements/filemanager_menu_order.lua
Unescape Escape View File

@ -36,7 +36,7 @@ local order = {
"storage_stat",
"cloud_storage",
"read_timer",
"rss_news_downloader",
"news_downloader",
"synchronize_time",
"terminal",
"----------------------------",

2
frontend/ui/elements/reader_menu_order.lua
Unescape Escape View File

@ -59,7 +59,7 @@ local order = {
"synchronize_time",
"progress_sync",
"zsync",
"rss_news_downloader",
"news_downloader",
"terminal",
},
search = {

7
plugins/newsdownloader.koplugin/feed_config.lua
Unescape Escape View File

@ -1,9 +1,8 @@
return {
-- list your feeds here:
-- only supports http URL for now
-- Atom is currently not supported, only RSS
{ "http://www.pcworld.com/index.rss", limit = 1 },
{ "http://www.economist.com/sections/science-technology/rss.xml", limit = 2},
{ "http://feeds.reuters.com/Reuters/worldNews?format=xml", limit = 2},
-- set limit to "0" means no download, "-1" no limit.
{ "http://www.economist.com/sections/culture/rss.xml", limit = 0 },
}
{ "http://www.football.co.uk/international/rss.xml", limit = 0 },
}

123
plugins/newsdownloader.koplugin/main.lua
Unescape Escape View File

@ -15,10 +15,10 @@ local ltn12 = require("ltn12")
local NewsDownloader = WidgetContainer:new{}
local initialized = false -- for only once lazy initialization
local FEED_CONFIG_FILE = "feed_config.lua"
local FILE_EXTENSION = ".html"
local NEWS_DL_DIR_NAME = "news"
local NEWS_DL_DIR, FEED_CONFIG_PATH
local feed_config_file = "feed_config.lua"
local file_extension = ".html"
local news_download_dir_name = "news"
local news_download_dir_path, feed_config_path
local function deserializeXMLString(xml_str)
-- uses LuaXML https://github.com/manoelcampos/LuaXML
@ -43,16 +43,15 @@ end
function NewsDownloader:addToMainMenu(menu_items)
if not initialized then
NEWS_DL_DIR = ("%s/%s/"):format(DataStorage:getDataDir(), NEWS_DL_DIR_NAME)
if not lfs.attributes(NEWS_DL_DIR, "mode") then
lfs.mkdir(NEWS_DL_DIR)
news_download_dir_path = ("%s/%s/"):format(DataStorage:getDataDir(), news_download_dir_name)
if not lfs.attributes(news_download_dir_path, "mode") then
lfs.mkdir(news_download_dir_path)
end
FEED_CONFIG_PATH = NEWS_DL_DIR .. FEED_CONFIG_FILE
feed_config_path = news_download_dir_path .. feed_config_file
initialized = true
end
menu_items.rss_news_downloader = {
menu_items.news_downloader = {
text = _("News (RSS/Atom) downloader"),
sub_item_table = {
{
@ -64,9 +63,9 @@ function NewsDownloader:addToMainMenu(menu_items)
callback = function()
local FileManager = require("apps/filemanager/filemanager")
if FileManager.instance then
FileManager.instance:reinit(NEWS_DL_DIR)
FileManager.instance:reinit(news_download_dir_path)
else
FileManager:showFiles(NEWS_DL_DIR)
FileManager:showFiles(news_download_dir_path)
end
end,
},
@ -74,9 +73,9 @@ function NewsDownloader:addToMainMenu(menu_items)
text = _("Remove news"),
callback = function()
-- puerge all downloaded news files, but keep the feed config
for entry in lfs.dir(NEWS_DL_DIR) do
if entry ~= "." and entry ~= ".." and entry ~= FEED_CONFIG_FILE then
local entry_path = NEWS_DL_DIR .. "/" .. entry
for entry in lfs.dir(news_download_dir_path) do
if entry ~= "." and entry ~= ".." and entry ~= feed_config_file then
local entry_path = news_download_dir_path .. "/" .. entry
local entry_mode = lfs.attributes(entry_path, "mode")
if entry_mode == "file" then
ffi.C.remove(entry_path)
@ -95,8 +94,8 @@ function NewsDownloader:addToMainMenu(menu_items)
callback = function()
UIManager:show(InfoMessage:new{
text = T(_("News downloader can be configured in the feeds config file:\n%1\n\nIt downloads news items to:\n%2.\n\nTo set you own news sources edit foregoing feeds config file. Items download limit can be set there."),
FEED_CONFIG_PATH,
NEWS_DL_DIR)
feed_config_path,
news_download_dir_path)
})
end,
},
@ -111,22 +110,24 @@ function NewsDownloader:loadConfigAndProcessFeeds()
UIManager:forceRePaint()
UIManager:close(info)
if not lfs.attributes(FEED_CONFIG_PATH, "mode") then
if not lfs.attributes(feed_config_path, "mode") then
logger.dbg("NewsDownloader: Creating initial feed config.")
FFIUtil.copyFile(FFIUtil.joinPath(self.path, FEED_CONFIG_FILE),
FEED_CONFIG_PATH)
FFIUtil.copyFile(FFIUtil.joinPath(self.path, feed_config_file),
feed_config_path)
end
local ok, feed_config = pcall(dofile, FEED_CONFIG_PATH)
local ok, feed_config = pcall(dofile, feed_config_path)
if not ok or not feed_config then
logger.info("NewsDownloader: Feed config not found.")
return
end
if #feed_config <= 0 then
logger.info('NewsDownloader: empty feed list.', FEED_CONFIG_PATH)
logger.info('NewsDownloader: empty feed list.', feed_config_path)
return
end
local unsupported_feeds_urls = {}
for idx, feed in ipairs(feed_config) do
local url = feed[1]
local limit = feed.limit
@ -135,32 +136,73 @@ function NewsDownloader:loadConfigAndProcessFeeds()
UIManager:show(info)
-- processFeedSource is a blocking call, so manually force a UI refresh beforehand
UIManager:forceRePaint()
self:processFeedSource(url, tonumber(limit))
self:processFeedSource(url, tonumber(limit), unsupported_feeds_urls)
UIManager:close(info)
else
logger.warn('NewsDownloader: invalid feed config entry', feed)
end
end
UIManager:show(InfoMessage:new{
text = _("Downloading news finished."),
timeout = 1,
})
if #unsupported_feeds_urls <= 0 then
UIManager:show(InfoMessage:new{
text = _("Downloading news finished."),
timeout = 1,
})
else
local unsupported_urls = ""
for k,url in pairs(unsupported_feeds_urls) do
unsupported_urls = unsupported_urls .. url
if k ~= #unsupported_feeds_urls then
unsupported_urls = unsupported_urls .. ", "
end
end
UIManager:show(InfoMessage:new{
text = T(_("Downloading news finished. Could not process some feeds. Unsupported format in: %1"), unsupported_urls)
})
end
end
function NewsDownloader:processFeedSource(url, limit)
function NewsDownloader:processFeedSource(url, limit, unsupported_feeds_urls)
local resp_lines = {}
http.request({ url = url, sink = ltn12.sink.table(resp_lines), })
local feeds = deserializeXMLString(table.concat(resp_lines))
if not feeds then return end
if not feeds.rss or not feeds.rss.channel
or not feeds.rss.channel.title or not feeds.rss.channel.item then
logger.info('NewsDownloader: Got invalid feeds', feeds)
if not feeds then
table.insert(unsupported_feeds_urls, url)
return
end
local is_rss = feeds.rss and feeds.rss.channel and feeds.rss.channel.title and feeds.rss.channel.item and feeds.rss.channel.item[1] and feeds.rss.channel.item[1].title and feeds.rss.channel.item[1].link
local is_atom = feeds.feed and feeds.feed.title and feeds.feed.entry.title and feeds.feed.entry.link and feeds.feed.entry[1] and feeds.feed.entry[1].title and feeds.feed.entry[1].link
if is_atom then
self:processAtom(feeds, limit)
elseif is_rss then
self:processRSS(feeds, limit)
else
table.insert(unsupported_feeds_urls, url)
return
end
end
function NewsDownloader:processAtom(feeds, limit)
local feed_output_dir = string.format("%s%s/",
news_download_dir_path,
util.replaceInvalidChars(feeds.feed.title))
if not lfs.attributes(feed_output_dir, "mode") then
lfs.mkdir(feed_output_dir)
end
for index, feed in pairs(feeds.feed.entry) do
if index -1 == limit then
break
end
self:downloadFeed(feed, feed_output_dir)
end
end
function NewsDownloader:processRSS(feeds, limit)
local feed_output_dir = ("%s%s/"):format(
NEWS_DL_DIR, util.replaceInvalidChars(feeds.rss.channel.title))
news_download_dir_path, util.replaceInvalidChars(feeds.rss.channel.title))
if not lfs.attributes(feed_output_dir, "mode") then
lfs.mkdir(feed_output_dir)
end
@ -169,12 +211,17 @@ function NewsDownloader:processFeedSource(url, limit)
if index -1 == limit then
break
end
local news_dl_path = ("%s%s%s"):format(feed_output_dir,
util.replaceInvalidChars(feed.title),
FILE_EXTENSION)
logger.dbg("NewsDownloader: News file will be stored to :", news_dl_path)
http.request({ url = url, sink = ltn12.sink.file(io.open(news_dl_path, 'w')), })
self:downloadFeed(feed, feed_output_dir)
end
end
function NewsDownloader:downloadFeed(feed, feed_output_dir)
local news_dl_path = ("%s%s%s"):format(feed_output_dir,
util.replaceInvalidChars(feed.title),
file_extension)
logger.dbg("NewsDownloader: News file will be stored to :", news_dl_path)
http.request({ url = feed.link, sink = ltn12.sink.file(io.open(news_dl_path, 'w')), })
end
return NewsDownloader

Write Preview
Loading…
Cancel
Save