2
0
mirror of https://github.com/koreader/koreader synced 2024-10-31 21:20:20 +00:00
koreader/plugins/newsdownloader.koplugin/internaldownloadbackend.lua
NiLuJe 2f9db25969
Unify LuaSocket usage (#7405)
* Add a new socketutil module with a few helper functions that allow us to:
  * Always use a sane User-Agent (previously, only Wikipedia did so)
  * Set timeouts in an almost sane manner. Doing it explicitly prevents an interaction with KOSync that does crazy stuff I don't even want to try to understand.
* Unified said timeouts based on the request's intended usage (except for Wikipedia, which already had meaningful timeout values).
* Stopped using LuaSec directly, LuaSocket defers to LuaSec sanely on its own. Everything now transparently supports HTTPS without code duplication.
2021-03-15 01:25:10 +01:00

47 lines
1.7 KiB
Lua

local http = require("socket.http")
local logger = require("logger")
local ltn12 = require("ltn12")
local socket = require("socket")
local socketutil = require("socketutil")
local InternalDownloadBackend = {}
local max_redirects = 5; --prevent infinite redirects
function InternalDownloadBackend:getResponseAsString(url, redirectCount)
if not redirectCount then
redirectCount = 0
elseif redirectCount == max_redirects then
error("InternalDownloadBackend: reached max redirects: ", redirectCount)
end
logger.dbg("InternalDownloadBackend: url :", url)
local sink = {}
socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
local request = {
url = url,
sink = ltn12.sink.table(sink),
}
local code, headers, status = socket.skip(1, http.request(request))
socketutil:reset_timeout()
if code ~= 200 then
logger.dbg("InternalDownloadBackend: HTTP response code <> 200. Response status: ", status)
if code and code > 299 and code < 400 and headers and headers["location"] then -- handle 301, 302...
local redirected_url = headers["location"]
logger.dbg("InternalDownloadBackend: Redirecting to url: ", redirected_url)
return self:getResponseAsString(redirected_url, redirectCount + 1)
else
error("InternalDownloadBackend: Don't know how to handle HTTP response status: ", status)
end
end
return table.concat(sink)
end
function InternalDownloadBackend:download(url, path)
local response = self:getResponseAsString(url)
local file = io.open(path, 'w')
file:write(response)
file:close()
end
return InternalDownloadBackend