Add: use https:// for content-service connections (#10448)

This requires the use of WinHTTP (for Windows) or libcurl (for all
others except Emscripten). Emscripten does not support http(s)
calls currently.

On Linux it requires ca-certificates to be installed, so the HTTPS
certificate can be validated. It is really likely this is installed
on any modern machine, as most connections these days are HTTPS.

(On MacOS and Windows the certificate store is filled by default)

Reminder: in case the http(s):// connection cannot be established,
OpenTTD falls back to a custom TCP-based connection to fetch the
content from the content-service. Emscripten will always do this.
pull/491/head
Patric Stout 1 year ago committed by GitHub
parent 09f7f32b8d
commit 64523709bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -102,6 +102,7 @@ jobs:
echo "::group::Install dependencies"
sudo apt-get install -y --no-install-recommends \
liballegro4-dev \
libcurl4-openssl-dev \
libfontconfig-dev \
libicu-dev \
liblzma-dev \
@ -184,6 +185,7 @@ jobs:
- name: Prepare vcpkg
run: |
vcpkg install --triplet=${{ matrix.arch }}-osx \
curl \
liblzma \
libpng \
lzo \

@ -31,6 +31,7 @@ jobs:
echo "::group::Install dependencies"
sudo apt-get install -y --no-install-recommends \
liballegro4-dev \
libcurl4-openssl-dev \
libfontconfig-dev \
libicu-dev \
liblzma-dev \

@ -29,6 +29,7 @@ jobs:
yum install -y \
fontconfig-devel \
freetype-devel \
libcurl4-openssl-dev \
libicu-devel \
libpng-devel \
lzo-devel \

@ -45,6 +45,8 @@ jobs:
- name: Prepare vcpkg
run: |
vcpkg install \
curl:x64-osx \
curl:arm64-osx \
liblzma:x64-osx \
liblzma:arm64-osx \
libpng:x64-osx \

@ -124,6 +124,14 @@ find_package(LibLZMA)
find_package(LZO)
find_package(PNG)
if(WIN32 OR EMSCRIPTEN)
# Windows uses WinHttp for HTTP requests.
# Emscripten uses Javascript for HTTP requests.
else()
# All other targets use libcurl.
find_package(CURL)
endif()
if(NOT OPTION_DEDICATED)
if(NOT WIN32)
find_package(Allegro)
@ -259,6 +267,10 @@ link_package(ZLIB TARGET ZLIB::ZLIB ENCOURAGED)
link_package(LIBLZMA TARGET LibLZMA::LibLZMA ENCOURAGED)
link_package(LZO)
if(NOT WIN32 AND NOT EMSCRIPTEN)
link_package(CURL ENCOURAGED)
endif()
if(NOT OPTION_DEDICATED)
link_package(Fluidsynth)
link_package(SDL)
@ -368,6 +380,7 @@ if(WIN32)
imm32
usp10
psapi
winhttp
)
endif()

@ -10,14 +10,17 @@ OpenTTD makes use of the following external libraries:
- (encouraged) libpng: making screenshots and loading heightmaps
- (optional) liblzo2: (de)compressing of old (pre 0.3.0) savegames
For Linux, the following additional libraries are used (for non-dedicated only):
For Linux, the following additional libraries are used:
- (encouraged) libcurl: content downloads
- libSDL2: hardware access (video, sound, mouse)
- libfreetype: loading generic fonts and rendering them
- libfontconfig: searching for fonts, resolving font names to actual fonts
- libicu: handling of right-to-left scripts (e.g. Arabic and Persian) and
natural sorting of strings
If you are building a dedicated-server only, you don't need the last four.
OpenTTD does not require any of the libraries to be present, but without
liblzma you cannot open most recent savegames and without zlib you cannot
open most older savegames or use the content downloading system.

@ -64,6 +64,9 @@
#ifdef WITH_ZLIB
# include <zlib.h>
#endif
#ifdef WITH_CURL
# include <curl/curl.h>
#endif
#include "safeguards.h"
@ -273,6 +276,16 @@ char *CrashLog::LogLibraries(char *buffer, const char *last) const
buffer += seprintf(buffer, last, " Zlib: %s\n", zlibVersion());
#endif
#ifdef WITH_CURL
auto *curl_v = curl_version_info(CURLVERSION_NOW);
buffer += seprintf(buffer, last, " Curl: %s\n", curl_v->version);
if (curl_v->ssl_version != nullptr) {
buffer += seprintf(buffer, last, " Curl SSL: %s\n", curl_v->ssl_version);
} else {
buffer += seprintf(buffer, last, " Curl SSL: none\n");
}
#endif
buffer += seprintf(buffer, last, "\n");
return buffer;
}

@ -9,6 +9,7 @@ add_files(
game_info.h
host.cpp
host.h
http.h
os_abstraction.cpp
os_abstraction.h
packet.cpp
@ -25,8 +26,6 @@ add_files(
tcp_coordinator.h
tcp_game.cpp
tcp_game.h
tcp_http.cpp
tcp_http.h
tcp_listen.h
tcp_stun.cpp
tcp_stun.h
@ -35,3 +34,16 @@ add_files(
udp.cpp
udp.h
)
add_files(
http_curl.cpp
CONDITION CURL_FOUND
)
add_files(
http_winhttp.cpp
CONDITION WIN32
)
add_files(
http_none.cpp
CONDITION NOT CURL_FOUND AND NOT WIN32
)

@ -59,11 +59,11 @@ const char *NetworkContentServerConnectionString()
}
/**
* Get the connection string for the content mirror from the environment variable OTTD_CONTENT_MIRROR_CS,
* or when it has not been set a hard coded default DNS hostname of the production server.
* @return The content mirror's connection string.
* Get the URI string for the content mirror from the environment variable OTTD_CONTENT_MIRROR_URI,
* or when it has not been set a hard coded URI of the production server.
* @return The content mirror's URI string.
*/
const char *NetworkContentMirrorConnectionString()
const char *NetworkContentMirrorUriString()
{
return GetEnv("OTTD_CONTENT_MIRROR_CS", "binaries.openttd.org");
return GetEnv("OTTD_CONTENT_MIRROR_URI", "https://binaries.openttd.org/bananas");
}

@ -15,10 +15,7 @@
const char *NetworkCoordinatorConnectionString();
const char *NetworkStunConnectionString();
const char *NetworkContentServerConnectionString();
const char *NetworkContentMirrorConnectionString();
/** URL of the HTTP mirror system */
static const char * const NETWORK_CONTENT_MIRROR_URL = "/bananas";
const char *NetworkContentMirrorUriString();
static const uint16 NETWORK_COORDINATOR_SERVER_PORT = 3976; ///< The default port of the Game Coordinator server (TCP)
static const uint16 NETWORK_STUN_SERVER_PORT = 3975; ///< The default port of the STUN server (TCP)

@ -0,0 +1,65 @@
/*
* This file is part of OpenTTD.
* OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
* OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* @file http.h Basic functions to send and receive HTTP packets.
*/
#ifndef NETWORK_CORE_HTTP_H
#define NETWORK_CORE_HTTP_H
#include "tcp.h"
/** Callback for when the HTTP handler has something to tell us. */
struct HTTPCallback {
/**
* An error has occurred and the connection has been closed.
* @note HTTP socket handler is closed/freed.
*/
virtual void OnFailure() = 0;
/**
* We're receiving data.
* @param data the received data, nullptr when all data has been received.
* @param length the amount of received data, 0 when all data has been received.
* @note When nullptr is sent the HTTP socket handler is closed/freed.
*/
virtual void OnReceiveData(const char *data, size_t length) = 0;
/** Silentium */
virtual ~HTTPCallback() {}
};
/** Base socket handler for HTTP traffic. */
class NetworkHTTPSocketHandler {
public:
/**
* Connect to the given URI.
*
* @param uri the URI to connect to (https://.../..).
* @param callback the callback to send data back on.
* @param data optionally, the data we want to send. When set, this will be a POST request, otherwise a GET request.
*/
static void Connect(const std::string &uri, HTTPCallback *callback, const char *data = nullptr);
/**
* Do the receiving for all HTTP connections.
*/
static void HTTPReceive();
};
/**
* Initialize the HTTP socket handler.
*/
void NetworkHTTPInitialize();
/**
* Uninitialize the HTTP socket handler.
*/
void NetworkHTTPUninitialize();
#endif /* NETWORK_CORE_HTTP_H */

@ -0,0 +1,227 @@
/*
* This file is part of OpenTTD.
* OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
* OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* @file http_curl.cpp CURL-based implementation for HTTP requests.
*/
#include "../../stdafx.h"
#include "../../debug.h"
#include "../../rev.h"
#include "../network_internal.h"
#include "http.h"
#include <curl/curl.h>
#include "../../safeguards.h"
/** Single HTTP request. */
class NetworkHTTPRequest {
private:
std::string uri; ///< URI to connect to.
HTTPCallback *callback; ///< Callback to send data back on.
const char *data; ///< Data to send, if any.
CURL *curl = nullptr; ///< CURL handle.
CURLM *multi_handle = nullptr; ///< CURL multi-handle.
public:
NetworkHTTPRequest(const std::string &uri, HTTPCallback *callback, const char *data = nullptr);
~NetworkHTTPRequest();
void Connect();
bool Receive();
};
static std::vector<NetworkHTTPRequest *> _http_requests;
static std::vector<NetworkHTTPRequest *> _new_http_requests;
static CURLSH *_curl_share = nullptr;
/**
* Create a new HTTP request.
*
* @param uri the URI to connect to (https://.../..).
* @param callback the callback to send data back on.
* @param data optionally, the data we want to send. When set, this will be a POST request, otherwise a GET request.
*/
NetworkHTTPRequest::NetworkHTTPRequest(const std::string &uri, HTTPCallback *callback, const char *data) :
uri(uri),
callback(callback),
data(data)
{
}
/**
* Start the HTTP request handling.
*
* This is done in an async manner, so we can do other things while waiting for
* the HTTP request to finish. The actual receiving of the data is done in
* Receive().
*/
void NetworkHTTPRequest::Connect()
{
Debug(net, 1, "HTTP request to {}", uri);
this->curl = curl_easy_init();
assert(this->curl != nullptr);
if (_debug_net_level >= 5) {
curl_easy_setopt(this->curl, CURLOPT_VERBOSE, 1L);
}
curl_easy_setopt(curl, CURLOPT_SHARE, _curl_share);
if (this->data != nullptr) {
curl_easy_setopt(this->curl, CURLOPT_POST, 1L);
curl_easy_setopt(this->curl, CURLOPT_POSTFIELDS, this->data);
}
curl_easy_setopt(this->curl, CURLOPT_URL, this->uri.c_str());
/* Setup our (C-style) callback function which we pipe back into the callback. */
curl_easy_setopt(this->curl, CURLOPT_WRITEFUNCTION, +[](char *ptr, size_t size, size_t nmemb, void *userdata) -> size_t {
Debug(net, 4, "HTTP callback: {} bytes", size * nmemb);
HTTPCallback *callback = static_cast<HTTPCallback *>(userdata);
callback->OnReceiveData(ptr, size * nmemb);
return size * nmemb;
});
curl_easy_setopt(this->curl, CURLOPT_WRITEDATA, this->callback);
/* Setup some default options. */
std::string user_agent = fmt::format("OpenTTD/{}", GetNetworkRevisionString());
curl_easy_setopt(this->curl, CURLOPT_USERAGENT, user_agent.c_str());
curl_easy_setopt(this->curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(this->curl, CURLOPT_MAXREDIRS, 5L);
/* Give the connection about 10 seconds to complete. */
curl_easy_setopt(this->curl, CURLOPT_CONNECTTIMEOUT, 10L);
/* Set a buffer of 100KiB, as the default of 16KiB seems a bit small. */
curl_easy_setopt(this->curl, CURLOPT_BUFFERSIZE, 100L * 1024L);
/* Fail our call if we don't receive a 2XX return value. */
curl_easy_setopt(this->curl, CURLOPT_FAILONERROR, 1L);
/* Create a multi-handle so we can do the call async. */
this->multi_handle = curl_multi_init();
curl_multi_add_handle(this->multi_handle, this->curl);
/* Trigger it for the first time so it becomes active. */
int still_running;
curl_multi_perform(this->multi_handle, &still_running);
}
/**
* Poll and process the HTTP request/response.
*
* @return True iff the request is done; no call to Receive() should be done after it returns true.
*/
bool NetworkHTTPRequest::Receive()
{
int still_running = 0;
/* Check for as long as there is activity on the socket, but once in a while return.
* This allows the GUI to always update, even on really fast downloads. */
for (int count = 0; count < 100; count++) {
/* Check if there was activity in the multi-handle. */
int numfds;
curl_multi_poll(this->multi_handle, NULL, 0, 0, &numfds);
if (numfds == 0) return false;
/* Let CURL process the activity. */
curl_multi_perform(this->multi_handle, &still_running);
if (still_running == 0) break;
}
/* The download is still pending (so the count is reached). Update GUI. */
if (still_running != 0) return false;
/* The request is done; check the result and close up. */
int msgq;
CURLMsg *msg = curl_multi_info_read(this->multi_handle, &msgq);
/* We can safely assume this returns something, as otherwise the multi-handle wouldn't be empty. */
assert(msg != nullptr);
assert(msg->msg == CURLMSG_DONE);
CURLcode res = msg->data.result;
if (res == CURLE_OK) {
Debug(net, 1, "HTTP request succeeded");
this->callback->OnReceiveData(nullptr, 0);
} else {
Debug(net, 0, "HTTP request failed: {}", curl_easy_strerror(res));
this->callback->OnFailure();
}
return true;
}
/**
* Destructor of the HTTP request.
*
* Makes sure all handlers are closed, and all memory is free'd.
*/
NetworkHTTPRequest::~NetworkHTTPRequest()
{
if (this->curl) {
curl_multi_remove_handle(this->multi_handle, this->curl);
curl_multi_cleanup(this->multi_handle);
curl_easy_cleanup(this->curl);
}
free(this->data);
}
/* static */ void NetworkHTTPSocketHandler::Connect(const std::string &uri, HTTPCallback *callback, const char *data)
{
auto request = new NetworkHTTPRequest(uri, callback, data);
request->Connect();
_new_http_requests.push_back(request);
}
/* static */ void NetworkHTTPSocketHandler::HTTPReceive()
{
if (!_new_http_requests.empty()) {
/* We delay adding new requests, as Receive() below can cause a callback which adds a new requests. */
_http_requests.insert(_http_requests.end(), _new_http_requests.begin(), _new_http_requests.end());
_new_http_requests.clear();
}
if (_http_requests.empty()) return;
for (auto it = _http_requests.begin(); it != _http_requests.end(); /* nothing */) {
NetworkHTTPRequest *cur = *it;
if (cur->Receive()) {
it = _http_requests.erase(it);
delete cur;
continue;
}
++it;
}
}
void NetworkHTTPInitialize()
{
curl_global_init(CURL_GLOBAL_DEFAULT);
/* Create a share that tracks DNS, SSL session, and connections. As this
* always runs in the same thread, sharing a connection should be fine. */
_curl_share = curl_share_init();
curl_share_setopt(_curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
curl_share_setopt(_curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
curl_share_setopt(_curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT);
}
void NetworkHTTPUninitialize()
{
curl_share_cleanup(_curl_share);
curl_global_cleanup();
}

@ -0,0 +1,37 @@
/*
* This file is part of OpenTTD.
* OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
* OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* @file http_emscripten.cpp Emscripten-based implementation for HTTP requests.
*/
#include "../../stdafx.h"
#include "../../debug.h"
#include "../../rev.h"
#include "../network_internal.h"
#include "http.h"
#include "../../safeguards.h"
/* static */ void NetworkHTTPSocketHandler::Connect(const std::string &uri, HTTPCallback *callback, const char *data)
{
/* No valid HTTP backend was compiled in, so we fail all HTTP requests. */
callback->OnFailure();
}
/* static */ void NetworkHTTPSocketHandler::HTTPReceive()
{
}
void NetworkHTTPInitialize()
{
}
void NetworkHTTPUninitialize()
{
}

@ -0,0 +1,316 @@
/*
* This file is part of OpenTTD.
* OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
* OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* @file http_winhttp.cpp WinHTTP-based implementation for HTTP requests.
*/
#include "../../stdafx.h"
#include "../../debug.h"
#include "../../rev.h"
#include "../network_internal.h"
#include "http.h"
#include <winhttp.h>
#include "../../safeguards.h"
static HINTERNET _winhttp_session = nullptr;
/** Single HTTP request. */
class NetworkHTTPRequest {
private:
std::wstring uri; ///< URI to connect to.
HTTPCallback *callback; ///< Callback to send data back on.
const char *data; ///< Data to send, if any.
HINTERNET connection = nullptr; ///< Current connection object.
HINTERNET request = nullptr; ///< Current request object.
bool finished = false; ///< Whether we are finished with the request.
int depth = 0; ///< Current redirect depth we are in.
public:
NetworkHTTPRequest(const std::wstring &uri, HTTPCallback *callback, const char *data = nullptr);
~NetworkHTTPRequest();
void Connect();
bool Receive();
void WinHttpCallback(DWORD code, void *info, DWORD length);
};
static std::vector<NetworkHTTPRequest *> _http_requests;
static std::vector<NetworkHTTPRequest *> _new_http_requests;
/**
* Create a new HTTP request.
*
* @param uri the URI to connect to (https://.../..).
* @param callback the callback to send data back on.
* @param data optionally, the data we want to send. When set, this will be a POST request, otherwise a GET request.
*/
NetworkHTTPRequest::NetworkHTTPRequest(const std::wstring &uri, HTTPCallback *callback, const char *data) :
uri(uri),
callback(callback),
data(data)
{
}
static std::string GetLastErrorAsString()
{
char buffer[512];
DWORD error_code = GetLastError();
if (FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_FROM_HMODULE | FORMAT_MESSAGE_IGNORE_INSERTS, GetModuleHandleA("winhttp.dll"), error_code,
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buffer, sizeof(buffer), NULL) == 0) {
return fmt::format("unknown error {}", error_code);
}
return buffer;
}
/**
* Callback from the WinHTTP library, called when-ever something changes about the HTTP request status.
*
* The callback needs to call some WinHttp functions for certain states, so WinHttp continues
* to read the request. This also allows us to abort when things go wrong, by simply not calling
* those functions.
* Comments with "Next step:" mark where WinHttp needs a call to continue.
*
* @param code The code of the event.
* @param info The information about the event.
* @param length The length of the information.
*/
void NetworkHTTPRequest::WinHttpCallback(DWORD code, void *info, DWORD length)
{
switch (code) {
case WINHTTP_CALLBACK_STATUS_RESOLVING_NAME:
case WINHTTP_CALLBACK_STATUS_NAME_RESOLVED:
case WINHTTP_CALLBACK_STATUS_CONNECTING_TO_SERVER:
case WINHTTP_CALLBACK_STATUS_CONNECTED_TO_SERVER:
case WINHTTP_CALLBACK_STATUS_SENDING_REQUEST:
case WINHTTP_CALLBACK_STATUS_REQUEST_SENT:
case WINHTTP_CALLBACK_STATUS_RECEIVING_RESPONSE:
case WINHTTP_CALLBACK_STATUS_RESPONSE_RECEIVED:
case WINHTTP_CALLBACK_STATUS_CLOSING_CONNECTION:
case WINHTTP_CALLBACK_STATUS_CONNECTION_CLOSED:
case WINHTTP_CALLBACK_STATUS_HANDLE_CREATED:
case WINHTTP_CALLBACK_STATUS_HANDLE_CLOSING:
/* We don't care about these events, and explicitly ignore them. */
break;
case WINHTTP_CALLBACK_STATUS_REDIRECT:
/* Make sure we are not in a redirect loop. */
if (this->depth++ > 5) {
Debug(net, 0, "HTTP request failed: too many redirects");
this->callback->OnFailure();
this->finished = true;
return;
}
break;
case WINHTTP_CALLBACK_STATUS_SENDREQUEST_COMPLETE:
/* Next step: read response. */
WinHttpReceiveResponse(this->request, nullptr);
break;
case WINHTTP_CALLBACK_STATUS_HEADERS_AVAILABLE:
{
/* Retrieve the status code. */
DWORD status_code = 0;
DWORD status_code_size = sizeof(status_code);
WinHttpQueryHeaders(this->request, WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, WINHTTP_HEADER_NAME_BY_INDEX, &status_code, &status_code_size, WINHTTP_NO_HEADER_INDEX);
Debug(net, 3, "HTTP request status code: {}", status_code);
/* If there is any error, we simply abort the request. */
if (status_code >= 400) {
Debug(net, 0, "HTTP request failed: status-code {}", status_code);
this->callback->OnFailure();
this->finished = true;
return;
}
/* Next step: query for any data. */
WinHttpQueryDataAvailable(this->request, nullptr);
} break;
case WINHTTP_CALLBACK_STATUS_DATA_AVAILABLE:
{
/* Retrieve the amount of data available to process. */
DWORD size = *(DWORD *)info;
/* Next step: read the data in a temporary allocated buffer.
* The buffer will be free'd in the next step. */
char *buffer = size == 0 ? nullptr : MallocT<char>(size);
WinHttpReadData(this->request, buffer, size, 0);
} break;
case WINHTTP_CALLBACK_STATUS_READ_COMPLETE:
Debug(net, 4, "HTTP callback: {} bytes", length);
this->callback->OnReceiveData(static_cast<char *>(info), length);
/* Free the temporary buffer that was allocated in the previous step. */
free(info);
if (length == 0) {
/* Next step: no more data available: request is finished. */
this->finished = true;
Debug(net, 1, "HTTP request succeeded");
} else {
/* Next step: query for more data. */
WinHttpQueryDataAvailable(this->request, nullptr);
}
break;
case WINHTTP_CALLBACK_STATUS_SECURE_FAILURE:
case WINHTTP_CALLBACK_STATUS_REQUEST_ERROR:
Debug(net, 0, "HTTP request failed: {}", GetLastErrorAsString());
this->callback->OnFailure();
this->finished = true;
break;
default:
Debug(net, 0, "HTTP request failed: unexepected callback code 0x{:x}", code);
this->callback->OnFailure();
this->finished = true;
return;
}
}
static void CALLBACK StaticWinHttpCallback(HINTERNET handle, DWORD_PTR context, DWORD code, void *info, DWORD length)
{
NetworkHTTPRequest *request = (NetworkHTTPRequest *)context;
request->WinHttpCallback(code, info, length);
}
/**
* Start the HTTP request handling.
*
* This is done in an async manner, so we can do other things while waiting for
* the HTTP request to finish. The actual receiving of the data is done in
* Receive().
*/
void NetworkHTTPRequest::Connect()
{
Debug(net, 1, "HTTP request to {}", std::string(uri.begin(), uri.end()));
URL_COMPONENTS url_components = {};
wchar_t scheme[32];
wchar_t hostname[128];
wchar_t url_path[4096];
/* Convert the URL to its components. */
url_components.dwStructSize = sizeof(url_components);
url_components.lpszScheme = scheme;
url_components.dwSchemeLength = lengthof(scheme);
url_components.lpszHostName = hostname;
url_components.dwHostNameLength = lengthof(hostname);
url_components.lpszUrlPath = url_path;
url_components.dwUrlPathLength = lengthof(url_path);
WinHttpCrackUrl(this->uri.c_str(), 0, 0, &url_components);
/* Create the HTTP connection. */
this->connection = WinHttpConnect(_winhttp_session, url_components.lpszHostName, url_components.nPort, 0);
if (this->connection == nullptr) {
Debug(net, 0, "HTTP request failed: {}", GetLastErrorAsString());
this->callback->OnFailure();
this->finished = true;
return;
}
this->request = WinHttpOpenRequest(connection, data == nullptr ? L"GET" : L"POST", url_components.lpszUrlPath, nullptr, WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, url_components.nScheme == INTERNET_SCHEME_HTTPS ? WINHTTP_FLAG_SECURE : 0);
if (this->request == nullptr) {
WinHttpCloseHandle(this->connection);
Debug(net, 0, "HTTP request failed: {}", GetLastErrorAsString());
this->callback->OnFailure();
this->finished = true;
return;
}
/* Send the request (possibly with a payload). */
if (data == nullptr) {
WinHttpSendRequest(this->request, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, reinterpret_cast<DWORD_PTR>(this));
} else {
WinHttpSendRequest(this->request, L"Content-Type: application/x-www-form-urlencoded\r\n", -1, const_cast<char *>(data), static_cast<DWORD>(strlen(data)), static_cast<DWORD>(strlen(data)), reinterpret_cast<DWORD_PTR>(this));
}
}
/**
* Poll and process the HTTP request/response.
*
* @return True iff the request is done; no call to Receive() should be done after it returns true.
*/
bool NetworkHTTPRequest::Receive()
{
return this->finished;
}
/**
* Destructor of the HTTP request.
*
* Makes sure all handlers are closed, and all memory is free'd.
*/
NetworkHTTPRequest::~NetworkHTTPRequest()
{
if (this->request) {
WinHttpCloseHandle(this->request);
WinHttpCloseHandle(this->connection);
}
free(this->data);
}
/* static */ void NetworkHTTPSocketHandler::Connect(const std::string &uri, HTTPCallback *callback, const char *data)
{
auto request = new NetworkHTTPRequest(std::wstring(uri.begin(), uri.end()), callback, data);
request->Connect();
_new_http_requests.push_back(request);
}
/* static */ void NetworkHTTPSocketHandler::HTTPReceive()
{
if (!_new_http_requests.empty()) {
/* We delay adding new requests, as Receive() below can cause a callback which adds a new requests. */
_http_requests.insert(_http_requests.end(), _new_http_requests.begin(), _new_http_requests.end());
_new_http_requests.clear();
}
if (_http_requests.empty()) return;
for (auto it = _http_requests.begin(); it != _http_requests.end(); /* nothing */) {
NetworkHTTPRequest *cur = *it;
if (cur->Receive()) {
it = _http_requests.erase(it);
delete cur;
continue;
}
++it;
}
}
void NetworkHTTPInitialize()
{
/* We create a single session, from which we build up every other request. */
std::string user_agent = fmt::format("OpenTTD/{}", GetNetworkRevisionString());
_winhttp_session = WinHttpOpen(std::wstring(user_agent.begin(), user_agent.end()).c_str(), WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, WINHTTP_FLAG_ASYNC);
/* Set the callback function for all requests. The "context" maps it back into the actual request instance. */
WinHttpSetStatusCallback(_winhttp_session, StaticWinHttpCallback, WINHTTP_CALLBACK_FLAG_ALL_NOTIFICATIONS, 0);
/* 10 seconds timeout for requests. */
WinHttpSetTimeouts(_winhttp_session, 10000, 10000, 10000, 10000);
}
void NetworkHTTPUninitialize()
{
WinHttpCloseHandle(_winhttp_session);
}

@ -1,322 +0,0 @@
/*
* This file is part of OpenTTD.
* OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
* OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* @file tcp_http.cpp Basic functions to receive and send HTTP TCP packets.
*/
#include "../../stdafx.h"
#include "../../debug.h"
#include "../../rev.h"
#include "../network_internal.h"
#include "game_info.h"
#include "tcp_http.h"
#include "../../safeguards.h"
/** List of open HTTP connections. */
static std::vector<NetworkHTTPSocketHandler *> _http_connections;
/**
* Start the querying
* @param s the socket of this connection
* @param callback the callback for HTTP retrieval
* @param host the hostname of the server to connect to
* @param url the url at the server
* @param data the data to send
* @param depth the depth (redirect recursion) of the queries
*/
NetworkHTTPSocketHandler::NetworkHTTPSocketHandler(SOCKET s,
HTTPCallback *callback, const std::string &host, const char *url,
const char *data, int depth) :
NetworkSocketHandler(),
recv_pos(0),
recv_length(0),
callback(callback),
data(data),
redirect_depth(depth),
sock(s)
{
Debug(net, 5, "[tcp/http] Requesting {}{}", host, url);
std::string request;
if (data != nullptr) {
request = fmt::format("POST {} HTTP/1.0\r\nHost: {}\r\nUser-Agent: OpenTTD/{}\r\nContent-Type: text/plain\r\nContent-Length: {}\r\n\r\n{}\r\n", url, host, GetNetworkRevisionString(), strlen(data), data);
} else {
request = fmt::format("GET {} HTTP/1.0\r\nHost: {}\r\nUser-Agent: OpenTTD/{}\r\n\r\n", url, host, GetNetworkRevisionString());
}
ssize_t res = send(this->sock, request.data(), (int)request.size(), 0);
if (res != (ssize_t)request.size()) {
/* Sending all data failed. Socket can't handle this little bit
* of information? Just fall back to the old system! */
this->callback->OnFailure();
delete this;
return;
}
_http_connections.push_back(this);
}
/** Free whatever needs to be freed. */
NetworkHTTPSocketHandler::~NetworkHTTPSocketHandler()
{
this->CloseSocket();
free(this->data);
}
/**
* Close the actual socket of the connection.
*/
void NetworkHTTPSocketHandler::CloseSocket()
{
if (this->sock != INVALID_SOCKET) closesocket(this->sock);
this->sock = INVALID_SOCKET;
}
/**
* Helper to simplify the error handling.
* @param msg the error message to show.
*/
#define return_error(msg) { Debug(net, 1, msg); return -1; }
static const char * const NEWLINE = "\r\n"; ///< End of line marker
static const char * const END_OF_HEADER = "\r\n\r\n"; ///< End of header marker
static const char * const HTTP_1_0 = "HTTP/1.0 "; ///< Preamble for HTTP 1.0 servers
static const char * const HTTP_1_1 = "HTTP/1.1 "; ///< Preamble for HTTP 1.1 servers
static const char * const CONTENT_LENGTH = "Content-Length: "; ///< Header for the length of the content
static const char * const LOCATION = "Location: "; ///< Header for location
/**
* Handle the header of a HTTP reply.
* @return amount of data to continue downloading.
* > 0: we need to download N bytes.
* = 0: we're being redirected.
* < 0: an error occurred. Downloading failed.
* @note if an error occurred the header might not be in its
* original state. No effort is undertaken to bring
* the header in its original state.
*/
int NetworkHTTPSocketHandler::HandleHeader()
{
assert(strlen(HTTP_1_0) == strlen(HTTP_1_1));
assert(strstr(this->recv_buffer, END_OF_HEADER) != nullptr);
/* We expect a HTTP/1.[01] reply */
if (strncmp(this->recv_buffer, HTTP_1_0, strlen(HTTP_1_0)) != 0 &&
strncmp(this->recv_buffer, HTTP_1_1, strlen(HTTP_1_1)) != 0) {
return_error("[tcp/http] Received invalid HTTP reply");
}
char *status = this->recv_buffer + strlen(HTTP_1_0);
if (strncmp(status, "200", 3) == 0) {
/* We are going to receive a document. */
/* Get the length of the document to receive */
char *length = strcasestr(this->recv_buffer, CONTENT_LENGTH);
if (length == nullptr) return_error("[tcp/http] Missing 'content-length' header");
/* Skip the header */
length += strlen(CONTENT_LENGTH);
/* Search the end of the line. This is safe because the header will
* always end with two newlines. */
char *end_of_line = strstr(length, NEWLINE);
/* Read the length */
*end_of_line = '\0';
int len = atoi(length);
/* Restore the header. */
*end_of_line = '\r';
/* Make sure we're going to download at least something;
* zero sized files are, for OpenTTD's purposes, always
* wrong. You can't have gzips of 0 bytes! */
if (len == 0) return_error("[tcp/http] Refusing to download 0 bytes");
Debug(net, 7, "[tcp/http] Downloading {} bytes", len);
return len;
}
if (strncmp(status, "301", 3) != 0 &&
strncmp(status, "302", 3) != 0 &&
strncmp(status, "303", 3) != 0 &&
strncmp(status, "307", 3) != 0) {
/* We are not going to be redirected :(. */
/* Search the end of the line. This is safe because the header will
* always end with two newlines. */
*strstr(status, NEWLINE) = '\0';
Debug(net, 1, "[tcp/http] Unhandled status reply {}", status);
return -1;
}
if (this->redirect_depth == 5) return_error("[tcp/http] Too many redirects, looping redirects?");
/* Redirect to other URL */
char *uri = strcasestr(this->recv_buffer, LOCATION);
if (uri == nullptr) return_error("[tcp/http] Missing 'location' header for redirect");
uri += strlen(LOCATION);
/* Search the end of the line. This is safe because the header will
* always end with two newlines. */
char *end_of_line = strstr(uri, NEWLINE);
*end_of_line = '\0';
Debug(net, 7, "[tcp/http] Redirecting to {}", uri);
int ret = NetworkHTTPSocketHandler::Connect(uri, this->callback, this->data, this->redirect_depth + 1);
if (ret != 0) return ret;
/* We've relinquished control of data now. */
this->data = nullptr;
/* Restore the header. */
*end_of_line = '\r';
return 0;
}
/**
* Connect to the given URI.
* @param uri the URI to connect to.
* @param callback the callback to send data back on.
* @param data the data we want to send (as POST).
* @param depth the recursion/redirect depth.
*/
/* static */ int NetworkHTTPSocketHandler::Connect(char *uri, HTTPCallback *callback, const char *data, int depth)
{
char *hname = strstr(uri, "://");
if (hname == nullptr) return_error("[tcp/http] Invalid location");
hname += 3;
char *url = strchr(hname, '/');
if (url == nullptr) return_error("[tcp/http] Invalid location");
*url = '\0';
std::string hostname = std::string(hname);
/* Restore the URL. */
*url = '/';
new NetworkHTTPContentConnecter(hostname, callback, url, data, depth);
return 0;
}
#undef return_error
/**
* Handle receiving of HTTP data.
* @return state of the receival of HTTP data.
* > 0: we need more cycles for downloading
* = 0: we are done downloading
* < 0: we have hit an error
*/
int NetworkHTTPSocketHandler::Receive()
{
for (;;) {
ssize_t res = recv(this->sock, (char *)this->recv_buffer + this->recv_pos, lengthof(this->recv_buffer) - this->recv_pos, 0);
if (res == -1) {
NetworkError err = NetworkError::GetLast();
if (!err.WouldBlock()) {
/* Something went wrong... */
if (!err.IsConnectionReset()) Debug(net, 0, "Recv failed: {}", err.AsString());
return -1;
}
/* Connection would block, so stop for now */
return 1;
}
/* No more data... did we get everything we wanted? */
if (res == 0) {
if (this->recv_length != 0) return -1;
this->callback->OnReceiveData(nullptr, 0);
return 0;
}
/* Wait till we read the end-of-header identifier */
if (this->recv_length == 0) {
ssize_t read = this->recv_pos + res;
ssize_t end = std::min<ssize_t>(read, lengthof(this->recv_buffer) - 1);
/* Do a 'safe' search for the end of the header. */
char prev = this->recv_buffer[end];
this->recv_buffer[end] = '\0';
char *end_of_header = strstr(this->recv_buffer, END_OF_HEADER);
this->recv_buffer[end] = prev;
if (end_of_header == nullptr) {
if (read == lengthof(this->recv_buffer)) {
Debug(net, 1, "[tcp/http] Header too big");
return -1;
}
this->recv_pos = read;
} else {
int ret = this->HandleHeader();
if (ret <= 0) return ret;
this->recv_length = ret;
end_of_header += strlen(END_OF_HEADER);
int len = std::min(read - (end_of_header - this->recv_buffer), res);
if (len != 0) {
this->callback->OnReceiveData(end_of_header, len);
this->recv_length -= len;
}
this->recv_pos = 0;
}
} else {
res = std::min<ssize_t>(this->recv_length, res);
/* Receive whatever we're expecting. */
this->callback->OnReceiveData(this->recv_buffer, res);
this->recv_length -= res;
}
}
}
/**
* Do the receiving for all HTTP connections.
*/
/* static */ void NetworkHTTPSocketHandler::HTTPReceive()
{
/* No connections, just bail out. */
if (_http_connections.size() == 0) return;
fd_set read_fd;
struct timeval tv;
FD_ZERO(&read_fd);
for (NetworkHTTPSocketHandler *handler : _http_connections) {
FD_SET(handler->sock, &read_fd);
}
tv.tv_sec = tv.tv_usec = 0; // don't block at all.
int n = select(FD_SETSIZE, &read_fd, nullptr, nullptr, &tv);
if (n == -1) return;
for (auto iter = _http_connections.begin(); iter < _http_connections.end(); /* nothing */) {
NetworkHTTPSocketHandler *cur = *iter;
if (FD_ISSET(cur->sock, &read_fd)) {
int ret = cur->Receive();
/* First send the failure. */
if (ret < 0) cur->callback->OnFailure();
if (ret <= 0) {
/* Then... the connection can be closed */
cur->CloseSocket();
iter = _http_connections.erase(iter);
delete cur;
continue;
}
}
iter++;
}
}

@ -1,121 +0,0 @@
/*
* This file is part of OpenTTD.
* OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
* OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* @file tcp_http.h Basic functions to receive and send HTTP TCP packets.
*/
#ifndef NETWORK_CORE_TCP_HTTP_H
#define NETWORK_CORE_TCP_HTTP_H
#include "tcp.h"
/** Callback for when the HTTP handler has something to tell us. */
struct HTTPCallback {
/**
* An error has occurred and the connection has been closed.
* @note HTTP socket handler is closed/freed.
*/
virtual void OnFailure() = 0;
/**
* We're receiving data.
* @param data the received data, nullptr when all data has been received.
* @param length the amount of received data, 0 when all data has been received.
* @note When nullptr is sent the HTTP socket handler is closed/freed.
*/
virtual void OnReceiveData(const char *data, size_t length) = 0;
/** Silentium */
virtual ~HTTPCallback() {}
};
/** Base socket handler for HTTP traffic. */
class NetworkHTTPSocketHandler : public NetworkSocketHandler {
private:
char recv_buffer[4096]; ///< Partially received message.
int recv_pos; ///< Current position in buffer.
int recv_length; ///< Length of the data still retrieving.
HTTPCallback *callback; ///< The callback to call for the incoming data.
const char *data; ///< The (POST) data we might want to forward (to a redirect).
int redirect_depth; ///< The depth of the redirection.
int HandleHeader();
int Receive();
public:
SOCKET sock; ///< The socket currently connected to
/**
* Whether this socket is currently bound to a socket.
* @return true when the socket is bound, false otherwise
*/
bool IsConnected() const
{
return this->sock != INVALID_SOCKET;
}
void CloseSocket();
NetworkHTTPSocketHandler(SOCKET sock, HTTPCallback *callback,
const std::string &host, const char *url, const char *data, int depth);
~NetworkHTTPSocketHandler();
static int Connect(char *uri, HTTPCallback *callback,
const char *data = nullptr, int depth = 0);
static void HTTPReceive();
};
/** Connect with a HTTP server and do ONE query. */
class NetworkHTTPContentConnecter : TCPConnecter {
std::string hostname; ///< Hostname we are connecting to.
HTTPCallback *callback; ///< Callback to tell that we received some data (or won't).
const char *url; ///< The URL we want to get at the server.
const char *data; ///< The data to send
int depth; ///< How far we have recursed
public:
/**
* Start the connecting.
* @param hostname The hostname to connect to.
* @param callback The callback for HTTP retrieval.
* @param url The url at the server.
* @param data The data to send.
* @param depth The depth (redirect recursion) of the queries.
*/
NetworkHTTPContentConnecter(const std::string &hostname, HTTPCallback *callback, const char *url, const char *data = nullptr, int depth = 0) :
TCPConnecter(hostname, 80),
hostname(hostname),
callback(callback),
url(stredup(url)),
data(data),
depth(depth)
{
}
/** Free all our allocated data. */
~NetworkHTTPContentConnecter()
{
free(this->url);
}
void OnFailure() override
{
this->callback->OnFailure();
free(this->data);
}
void OnConnect(SOCKET s) override
{
new NetworkHTTPSocketHandler(s, this->callback, this->hostname, this->url, this->data, this->depth);
/* We've relinquished control of data now. */
this->data = nullptr;
}
};
#endif /* NETWORK_CORE_TCP_HTTP_H */

@ -1268,12 +1268,14 @@ void NetworkStartUp()
NetworkUDPInitialize();
Debug(net, 3, "Network online, multiplayer available");
NetworkFindBroadcastIPs(&_broadcast_list);
NetworkHTTPInitialize();
}
/** This shuts the network down */
void NetworkShutDown()
{
NetworkDisconnect(true);
NetworkHTTPUninitialize();
NetworkUDPClose();
Debug(net, 3, "Shutting down network");

@ -311,13 +311,6 @@ void ClientNetworkContentSocketHandler::DownloadSelectedContent(uint &files, uin
{
bytes = 0;
#ifdef __EMSCRIPTEN__
/* Emscripten is loaded via an HTTPS connection. As such, it is very
* difficult to make HTTP connections. So always use the TCP method of
* downloading content. */
fallback = true;
#endif
ContentIDList content;
for (const ContentInfo *ci : this->infos) {
if (!ci->IsSelected() || ci->state == ContentInfo::ALREADY_HERE) continue;
@ -361,7 +354,7 @@ void ClientNetworkContentSocketHandler::DownloadSelectedContentHTTP(const Conten
this->http_response_index = -1;
new NetworkHTTPContentConnecter(NetworkContentMirrorConnectionString(), this, NETWORK_CONTENT_MIRROR_URL, content_request);
NetworkHTTPSocketHandler::Connect(NetworkContentMirrorUriString(), this, content_request);
/* NetworkHTTPContentConnecter takes over freeing of content_request! */
}

@ -11,7 +11,7 @@
#define NETWORK_CONTENT_H
#include "core/tcp_content.h"
#include "core/tcp_http.h"
#include "core/http.h"
#include <unordered_map>
/** Vector with content info */

Loading…
Cancel
Save