local DataStorage = require ( " datastorage " )
local DownloadBackend = require ( " internaldownloadbackend " )
--local DownloadBackend = require("luahttpdownloadbackend")
local ReadHistory = require ( " readhistory " )
local FFIUtil = require ( " ffi/util " )
local InfoMessage = require ( " ui/widget/infomessage " )
local InputDialog = require ( " ui/widget/inputdialog " )
local LuaSettings = require ( " frontend/luasettings " )
local UIManager = require ( " ui/uimanager " )
local NetworkMgr = require ( " ui/network/manager " )
local WidgetContainer = require ( " ui/widget/container/widgetcontainer " )
local dateparser = require ( " lib.dateparser " )
local logger = require ( " logger " )
local util = require ( " util " )
local _ = require ( " gettext " )
local T = FFIUtil.template
local NewsDownloader = WidgetContainer : new {
name = " newsdownloader " ,
}
local initialized = false
local wifi_enabled_before_action = true
local feed_config_file_name = " feed_config.lua "
local news_downloader_config_file = " news_downloader_settings.lua "
local config_key_custom_dl_dir = " custom_dl_dir " ;
local file_extension = " .html "
local news_download_dir_name = " news "
local news_download_dir_path , feed_config_path
-- if a title looks like <title>blabla</title> it'll just be feed.title
-- if a title looks like <title attr="alb">blabla</title> then we get a table
-- where [1] is the title string and the attributes are also available
local function getFeedTitle ( possible_title )
if type ( possible_title ) == " string " then
return possible_title
elseif possible_title [ 1 ] and type ( possible_title [ 1 ] ) == " string " then
return possible_title [ 1 ]
end
end
-- there can be multiple links
-- for now we just assume the first link is probably the right one
-- @todo write unit tests
-- some feeds that can be used for unit test
-- http://fransdejonge.com/feed/ for multiple links
-- https://github.com/koreader/koreader/commits/master.atom for single link with attributes
local function getFeedLink ( possible_link )
local E = { }
if type ( possible_link ) == " string " then
return possible_link
elseif ( possible_link._attr or E ) . href then
return possible_link._attr . href
elseif ( ( possible_link [ 1 ] or E ) . _attr or E ) . href then
return possible_link [ 1 ] . _attr.href
end
end
-- TODO: implement as NetworkMgr:afterWifiAction with configuration options
function NewsDownloader : afterWifiAction ( )
if not wifi_enabled_before_action then
NetworkMgr : promptWifiOff ( )
end
end
function NewsDownloader : init ( )
self.ui . menu : registerToMainMenu ( self )
end
function NewsDownloader : addToMainMenu ( menu_items )
self : lazyInitialization ( )
menu_items.news_downloader = {
text = _ ( " News (RSS/Atom) downloader " ) ,
sub_item_table = {
{
text = _ ( " Download news " ) ,
keep_menu_open = true ,
callback = function ( )
if not NetworkMgr : isOnline ( ) then
wifi_enabled_before_action = false
NetworkMgr : beforeWifiAction ( self.loadConfigAndProcessFeeds )
else
self : loadConfigAndProcessFeeds ( )
end
end ,
} ,
{
text = _ ( " Go to news folder " ) ,
callback = function ( )
local FileManager = require ( " apps/filemanager/filemanager " )
if FileManager.instance then
FileManager.instance : reinit ( news_download_dir_path )
else
FileManager : showFiles ( news_download_dir_path )
end
end ,
} ,
{
text = _ ( " Remove news " ) ,
keep_menu_open = true ,
callback = function ( ) self : removeNewsButKeepFeedConfig ( ) end ,
} ,
{
text = _ ( " Settings " ) ,
sub_item_table = {
{
text = _ ( " Change feeds configuration " ) ,
keep_menu_open = true ,
callback = function ( ) self : changeFeedConfig ( ) end ,
} ,
{
text = _ ( " Set custom download directory " ) ,
keep_menu_open = true ,
callback = function ( ) self : setCustomDownloadDirectory ( ) end ,
} ,
} ,
} ,
{
text = _ ( " Help " ) ,
keep_menu_open = true ,
callback = function ( )
UIManager : show ( InfoMessage : new {
text = T ( _ ( " News downloader retrieves RSS and Atom news entries and stores them to: \n %1 \n \n Each entry is a separate html file, that can be browsed by KOReader file manager. \n Items download limit can be configured in Settings. " ) ,
news_download_dir_path )
} )
end ,
} ,
} ,
}
end
function NewsDownloader : lazyInitialization ( )
if not initialized then
logger.dbg ( " NewsDownloader: obtaining news folder " )
local news_downloader_settings = LuaSettings : open ( ( " %s/%s " ) : format ( DataStorage : getSettingsDir ( ) , news_downloader_config_file ) )
if news_downloader_settings : has ( config_key_custom_dl_dir ) then
news_download_dir_path = news_downloader_settings : readSetting ( config_key_custom_dl_dir )
else
news_download_dir_path = ( " %s/%s/ " ) : format ( DataStorage : getFullDataDir ( ) , news_download_dir_name )
end
if not lfs.attributes ( news_download_dir_path , " mode " ) then
logger.dbg ( " NewsDownloader: Creating initial directory " )
lfs.mkdir ( news_download_dir_path )
end
feed_config_path = news_download_dir_path .. feed_config_file_name
if not lfs.attributes ( feed_config_path , " mode " ) then
logger.dbg ( " NewsDownloader: Creating initial feed config. " )
FFIUtil.copyFile ( FFIUtil.joinPath ( self.path , feed_config_file_name ) ,
feed_config_path )
end
initialized = true
end
end
function NewsDownloader : loadConfigAndProcessFeeds ( )
local info = InfoMessage : new { text = _ ( " Loading news feed config… " ) }
UIManager : show ( info )
logger.dbg ( " force repaint due to upcoming blocking calls " )
UIManager : forceRePaint ( )
UIManager : close ( info )
local ok , feed_config = pcall ( dofile , feed_config_path )
if not ok or not feed_config then
info = InfoMessage : new { text = T ( _ ( " Invalid configuration file. Detailed error message: \n %1 " ) , feed_config ) }
UIManager : show ( info )
return
end
if # feed_config <= 0 then
logger.err ( ' NewsDownloader: empty feed list. ' , feed_config_path )
return
end
local unsupported_feeds_urls = { }
local total_feed_entries = table.getn ( feed_config )
for idx , feed in ipairs ( feed_config ) do
local url = feed [ 1 ]
local limit = feed.limit
local download_full_article = feed.download_full_article == nil or feed.download_full_article
if url and limit then
info = InfoMessage : new { text = T ( _ ( " Processing %1/%2: \n %3 " ) , idx , total_feed_entries , url ) }
UIManager : show ( info )
-- processFeedSource is a blocking call, so manually force a UI refresh beforehand
UIManager : forceRePaint ( )
NewsDownloader : processFeedSource ( url , tonumber ( limit ) , unsupported_feeds_urls , download_full_article )
UIManager : close ( info )
else
logger.warn ( ' NewsDownloader: invalid feed config entry ' , feed )
end
end
if # unsupported_feeds_urls <= 0 then
UIManager : show ( InfoMessage : new {
text = _ ( " Downloading news finished. " ) ,
timeout = 1 ,
} )
else
local unsupported_urls = " "
for k , url in pairs ( unsupported_feeds_urls ) do
unsupported_urls = unsupported_urls .. url
if k ~= # unsupported_feeds_urls then
unsupported_urls = unsupported_urls .. " , "
end
end
UIManager : show ( InfoMessage : new {
text = T ( _ ( " Downloading news finished. Could not process some feeds. Unsupported format in: %1 " ) , unsupported_urls )
} )
end
NewsDownloader : afterWifiAction ( )
end
function NewsDownloader : processFeedSource ( url , limit , unsupported_feeds_urls , download_full_article )
local ok , response = pcall ( function ( )
return DownloadBackend : getResponseAsString ( url )
end )
local feeds
if ok then
feeds = self : deserializeXMLString ( response )
end
if not ok or not feeds then
table.insert ( unsupported_feeds_urls , url )
return
end
local is_rss = feeds.rss and feeds.rss . channel and feeds.rss . channel.title and feeds.rss . channel.item and feeds.rss . channel.item [ 1 ] and feeds.rss . channel.item [ 1 ] . title and feeds.rss . channel.item [ 1 ] . link
local is_atom = feeds.feed and feeds.feed . title and feeds.feed . entry [ 1 ] and feeds.feed . entry [ 1 ] . title and feeds.feed . entry [ 1 ] . link
if is_atom then
ok = pcall ( function ( )
return self : processAtom ( feeds , limit , download_full_article )
end )
elseif is_rss then
ok = pcall ( function ( )
return self : processRSS ( feeds , limit , download_full_article )
end )
end
if not ok or ( not is_rss and not is_atom ) then
table.insert ( unsupported_feeds_urls , url )
end
end
function NewsDownloader : deserializeXMLString ( xml_str )
-- uses LuaXML https://github.com/manoelcampos/LuaXML
-- The MIT License (MIT)
-- Copyright (c) 2016 Manoel Campos da Silva Filho
-- see: koreader/plugins/newsdownloader.koplugin/lib/LICENSE_LuaXML
local treehdl = require ( " lib/handler " )
local libxml = require ( " lib/xml " )
--Instantiate the object the states the XML file as a Lua table
local xmlhandler = treehdl.simpleTreeHandler ( )
--Instantiate the object that parses the XML to a Lua table
local ok = pcall ( function ( )
libxml.xmlParser ( xmlhandler ) : parse ( xml_str )
end )
if not ok then return end
return xmlhandler.root
end
function NewsDownloader : processAtom ( feeds , limit , download_full_article )
local feed_output_dir = string.format ( " %s%s/ " ,
news_download_dir_path ,
util.replaceInvalidChars ( getFeedTitle ( feeds.feed . title ) ) )
if not lfs.attributes ( feed_output_dir , " mode " ) then
lfs.mkdir ( feed_output_dir )
end
for index , feed in pairs ( feeds.feed . entry ) do
if limit ~= 0 and index - 1 == limit then
break
end
if download_full_article then
self : downloadFeed ( feed , feed_output_dir )
else
self : createFromDescription ( feed , feed.context , feed_output_dir )
end
end
end
function NewsDownloader : processRSS ( feeds , limit , download_full_article )
local feed_output_dir = ( " %s%s/ " ) : format (
news_download_dir_path , util.replaceInvalidChars ( feeds.rss . channel.title ) )
if not lfs.attributes ( feed_output_dir , " mode " ) then
lfs.mkdir ( feed_output_dir )
end
for index , feed in pairs ( feeds.rss . channel.item ) do
if limit ~= 0 and index - 1 == limit then
break
end
if download_full_article then
self : downloadFeed ( feed , feed_output_dir )
else
self : createFromDescription ( feed , feed.description , feed_output_dir )
end
end
end
local function parseDate ( dateTime )
-- uses lua-feedparser https://github.com/slact/lua-feedparser
-- feedparser is available under the (new) BSD license.
-- see: koreader/plugins/newsdownloader.koplugin/lib/LICENCE_lua-feedparser
local date = dateparser.parse ( dateTime )
return os.date ( " %y-%m-%d_%H-%M_ " , date )
end
local function getTitleWithDate ( feed )
local title = util.replaceInvalidChars ( getFeedTitle ( feed.title ) )
if feed.updated then
title = parseDate ( feed.updated ) .. title
elseif feed.pubDate then
title = parseDate ( feed.pubDate ) .. title
elseif feed.published then
title = parseDate ( feed.published ) .. title
end
return title
end
function NewsDownloader : downloadFeed ( feed , feed_output_dir )
local link = getFeedLink ( feed.link )
local news_dl_path = ( " %s%s%s " ) : format ( feed_output_dir ,
getTitleWithDate ( feed ) ,
file_extension )
logger.dbg ( " NewsDownloader: News file will be stored to : " , news_dl_path )
DownloadBackend : download ( link , news_dl_path )
end
function NewsDownloader : createFromDescription ( feed , context , feed_output_dir )
local news_file_path = ( " %s%s%s " ) : format ( feed_output_dir ,
getTitleWithDate ( feed ) ,
file_extension )
logger.dbg ( " NewsDownloader: News file will be created : " , news_file_path )
local file = io.open ( news_file_path , " w " )
local footer = _ ( " This is just description of the feed. To download full article go to News Downloader settings and change 'download_full_article' to 'true' " )
local html = string.format ( [ [ < ! DOCTYPE html >
< html >
< head >< meta charset = ' UTF-8 ' >< title >% s </ title ></ head >
< body >< header >< h2 >% s </ h2 ></ header >< article >% s </ article >
< br >< footer >< small >% s </ small ></ footer >
</ body >
</ html > ] ] , feed.title , feed.title , context , footer )
file : write ( html )
file : close ( )
end
function NewsDownloader : removeNewsButKeepFeedConfig ( )
logger.dbg ( " NewsDownloader: Removing news from : " , news_download_dir_path )
for entry in lfs.dir ( news_download_dir_path ) do
if entry ~= " . " and entry ~= " .. " and entry ~= feed_config_file_name then
local entry_path = news_download_dir_path .. " / " .. entry
local entry_mode = lfs.attributes ( entry_path , " mode " )
if entry_mode == " file " then
os.remove ( entry_path )
elseif entry_mode == " directory " then
FFIUtil.purgeDir ( entry_path )
end
end
end
UIManager : show ( InfoMessage : new {
text = _ ( " All news removed. " )
} )
end
function NewsDownloader : setCustomDownloadDirectory ( )
require ( " ui/downloadmgr " ) : new {
onConfirm = function ( path )
logger.dbg ( " NewsDownloader: set download directory to: " , path )
local news_downloader_settings = LuaSettings : open ( ( " %s/%s " ) : format ( DataStorage : getSettingsDir ( ) , news_downloader_config_file ) )
news_downloader_settings : saveSetting ( config_key_custom_dl_dir , ( " %s/ " ) : format ( path ) )
news_downloader_settings : flush ( )
logger.dbg ( " NewsDownloader: Coping to new download folder previous feed_config_file_name from: " , feed_config_path )
FFIUtil.copyFile ( feed_config_path , ( " %s/%s " ) : format ( path , feed_config_file_name ) )
initialized = false
self : lazyInitialization ( )
end ,
} : chooseDir ( )
end
function NewsDownloader : changeFeedConfig ( )
local feed_config_file = io.open ( feed_config_path , " rb " )
local config = feed_config_file : read ( " *all " )
feed_config_file : close ( )
local config_editor
config_editor = InputDialog : new {
title = T ( _ ( " Config: %1 " ) , feed_config_path ) ,
input = config ,
input_type = " string " ,
fullscreen = true ,
condensed = true ,
allow_newline = true ,
cursor_at_end = false ,
add_nav_bar = true ,
reset_callback = function ( )
return config
end ,
save_callback = function ( content )
if content and # content > 0 then
local parse_error = util.checkLuaSyntax ( content )
if not parse_error then
local syntax_okay , syntax_error = pcall ( loadstring ( content ) )
if syntax_okay then
feed_config_file = io.open ( feed_config_path , " w " )
feed_config_file : write ( content )
feed_config_file : close ( )
return true , _ ( " Configuration saved " )
else
return false , T ( _ ( " Configuration invalid: %1 " ) , syntax_error )
end
else
return false , T ( _ ( " Configuration invalid: %1 " ) , parse_error )
end
end
return false , _ ( " Configuration empty " )
end ,
}
UIManager : show ( config_editor )
config_editor : onShowKeyboard ( )
end
function NewsDownloader : onCloseDocument ( )
local document_full_path = self.ui . document.file
if document_full_path and news_download_dir_path and news_download_dir_path == string.sub ( document_full_path , 1 , string.len ( news_download_dir_path ) ) then
logger.dbg ( " NewsDownloader: document_full_path: " , document_full_path )
logger.dbg ( " NewsDownloader: news_download_dir_path: " , news_download_dir_path )
logger.dbg ( " NewsDownloader: removing NewsDownloader file from history. " )
ReadHistory : removeItemByPath ( document_full_path )
end
end
return NewsDownloader