add disk worker based file flusher logger

make format

remove package.json
This commit is contained in:
Jeff Becker 2019-04-16 09:20:48 -04:00
parent a45d6db0e0
commit 9503cc66f0
No known key found for this signature in database
GPG Key ID: F357B3B42F6F9B05
21 changed files with 225 additions and 38 deletions

View File

@ -1,12 +0,0 @@
{
"name": "lokinet",
"version": "0.0.0",
"description": "lokinet god awful node binding",
"main": "lokinet.js",
"private": true,
"gypfile": true,
"dependencies": {
"bindings": "~1.2.1",
"node-addon-api": "^1.0.0"
}
}

View File

@ -16,6 +16,7 @@ set(LIB_UTIL_SRC
util/json.cpp util/json.cpp
util/logger.cpp util/logger.cpp
util/android_logger.cpp util/android_logger.cpp
util/file_logger.cpp
util/ostream_logger.cpp util/ostream_logger.cpp
util/syslog_logger.cpp util/syslog_logger.cpp
util/win32_logger.cpp util/win32_logger.cpp

View File

@ -54,6 +54,9 @@ llarp_ev_loop_run_single_process(llarp_ev_loop_ptr ev,
ev->update_time(); ev->update_time();
logic->tick_async(ev->time_now()); logic->tick_async(ev->time_now());
llarp_threadpool_tick(tp); llarp_threadpool_tick(tp);
// tick log stream at the VERY END of the tick cycle so that all logs
// flush
llarp::LogContext::Instance().logStream->Tick(ev->time_now());
} }
} }
} }

View File

@ -78,9 +78,8 @@ namespace llarp
}; };
}; // namespace llarp }; // namespace llarp
struct llarp_poll_loop struct llarp_poll_loop : public llarp_ev_loop,
: public llarp_ev_loop, public std::enable_shared_from_this< llarp_poll_loop >
public std::enable_shared_from_this< llarp_poll_loop >
{ {
upoll_t* upollfd; upoll_t* upollfd;

View File

@ -281,7 +281,7 @@ namespace llarp
using namespace std::placeholders; using namespace std::placeholders;
if(self->record.work if(self->record.work
&& self->record.work->IsValid( && self->record.work->IsValid(
std::bind(&Crypto::shorthash, crypto, _1, _2), now)) std::bind(&Crypto::shorthash, crypto, _1, _2), now))
{ {
llarp::LogDebug("LRCM extended lifetime by ", llarp::LogDebug("LRCM extended lifetime by ",
self->record.work->extendedLifetime, " seconds for ", self->record.work->extendedLifetime, " seconds for ",

View File

@ -76,7 +76,8 @@ namespace llarp
return (pathSuccessCount * chances) > pathFailCount; return (pathSuccessCount * chances) > pathFailCount;
} }
static bool constexpr checkIsGood(uint64_t fails, uint64_t success, uint64_t chances) static bool constexpr checkIsGood(uint64_t fails, uint64_t success,
uint64_t chances)
{ {
if(fails > 0) if(fails > 0)
return success / fails > chances; return success / fails > chances;
@ -90,7 +91,7 @@ namespace llarp
{ {
return checkIsGood(connectTimeoutCount, connectGoodCount, chances); return checkIsGood(connectTimeoutCount, connectGoodCount, chances);
} }
bool bool
RouterProfile::IsGoodForPath(uint64_t chances) const RouterProfile::IsGoodForPath(uint64_t chances) const
{ {
@ -98,7 +99,7 @@ namespace llarp
} }
bool bool
Profiling::IsBadForConnect(const RouterID & r, uint64_t chances) Profiling::IsBadForConnect(const RouterID& r, uint64_t chances)
{ {
lock_t lock(&m_ProfilesMutex); lock_t lock(&m_ProfilesMutex);
auto itr = m_Profiles.find(r); auto itr = m_Profiles.find(r);
@ -108,7 +109,7 @@ namespace llarp
} }
bool bool
Profiling::IsBadForPath(const RouterID & r, uint64_t chances) Profiling::IsBadForPath(const RouterID& r, uint64_t chances)
{ {
lock_t lock(&m_ProfilesMutex); lock_t lock(&m_ProfilesMutex);
auto itr = m_Profiles.find(r); auto itr = m_Profiles.find(r);

View File

@ -64,13 +64,13 @@ namespace llarp
LOCKS_EXCLUDED(m_ProfilesMutex); LOCKS_EXCLUDED(m_ProfilesMutex);
/// check if this rotuer should have paths built over it /// check if this rotuer should have paths built over it
bool bool
IsBadForPath(const RouterID & r, uint64_t chances = 8) IsBadForPath(const RouterID& r, uint64_t chances = 8)
LOCK_RETURNED(m_ProfilesMutex); LOCK_RETURNED(m_ProfilesMutex);
/// check if this router should be connected directly to /// check if this router should be connected directly to
bool bool
IsBadForConnect(const RouterID & r, uint64_t chances = 8) IsBadForConnect(const RouterID& r, uint64_t chances = 8)
LOCKS_EXCLUDED(m_ProfilesMutex); LOCKS_EXCLUDED(m_ProfilesMutex);
void void

View File

@ -14,6 +14,7 @@
#include <util/buffer.hpp> #include <util/buffer.hpp>
#include <util/encode.hpp> #include <util/encode.hpp>
#include <util/logger.hpp> #include <util/logger.hpp>
#include <util/file_logger.hpp>
#include <util/logger_syslog.hpp> #include <util/logger_syslog.hpp>
#include <util/metrics.hpp> #include <util/metrics.hpp>
#include <util/str.hpp> #include <util/str.hpp>
@ -847,6 +848,28 @@ namespace llarp
LogContext::Instance().logStream = std::make_unique< SysLogStream >(); LogContext::Instance().logStream = std::make_unique< SysLogStream >();
#endif #endif
} }
if(StrEq(key, "file"))
{
LogInfo("open log file: ", val);
FILE *logfile = ::fopen(val, "a");
if(logfile)
{
LogContext::Instance().logStream =
std::make_unique< FileLogStream >(diskworker(), logfile, 500);
LogInfo("started logging to ", val);
}
else if(errno)
{
LogError("could not open log file at '", val, "': ", strerror(errno));
errno = 0;
}
else
{
LogError("failed to open log file at '", val,
"' for an unknown reason, bailing tf out kbai");
::abort();
}
}
} }
else if(StrEq(section, "lokid")) else if(StrEq(section, "lokid"))
{ {

View File

@ -86,6 +86,11 @@ namespace llarp
// path to write our self signed rc to // path to write our self signed rc to
fs::path our_rc_file = "rc.signed"; fs::path our_rc_file = "rc.signed";
// use file based logging?
bool m_UseFileLogging = false;
// default log file path
fs::path logfile = "lokinet.log";
// our router contact // our router contact
RouterContact _rc; RouterContact _rc;

View File

@ -1460,11 +1460,12 @@ namespace llarp
size_t tries = 5; size_t tries = 5;
do do
{ {
nodedb->select_random_hop_excluding(hops[hop], {hops[hop - 1].pubkey, remote}); nodedb->select_random_hop_excluding(hops[hop],
{hops[hop - 1].pubkey, remote});
--tries; --tries;
} while( } while(m_Endpoint->Router()->routerProfiling().IsBadForPath(
m_Endpoint->Router()->routerProfiling().IsBadForPath(hops[hop].pubkey) hops[hop].pubkey)
&& tries > 0); && tries > 0);
return tries > 0; return tries > 0;
} }
return false; return false;

View File

@ -36,8 +36,7 @@ namespace llarp
} }
void void
AndroidLogStream::Print(LogLevel lvl, const char* tag, AndroidLogStream::Print(LogLevel lvl, const char* tag, const std::string& msg)
const std::string& msg) const
{ {
std::string str("lokinet|"); std::string str("lokinet|");
str += tag; str += tag;

View File

@ -13,7 +13,12 @@ namespace llarp
int lineno) const override; int lineno) const override;
void void
Log(LogLevel lvl, const std::string& msg) const override; Log(LogLevel lvl, const std::string& msg) override;
void
PostLog(std::stringstream&) const override{};
void Tick(llarp_time_t) override;
}; };
} // namespace llarp } // namespace llarp

View File

@ -0,0 +1,89 @@
#include <util/file_logger.hpp>
#include <util/logger_internal.hpp>
namespace llarp
{
FileLogStream::FileLogStream(llarp_threadpool *disk, FILE *f,
llarp_time_t flushInterval)
: m_Disk(disk), m_File(f), m_FlushInterval(flushInterval)
{
}
FileLogStream::~FileLogStream()
{
fflush(m_File);
fclose(m_File);
}
bool
FileLogStream::ShouldFlush(llarp_time_t now) const
{
if(m_LastFlush >= now)
return false;
const auto dlt = now - m_LastFlush;
return dlt >= m_FlushInterval;
}
void
FileLogStream::PreLog(std::stringstream &ss, LogLevel lvl, const char *fname,
int lineno) const
{
switch(lvl)
{
case eLogNone:
break;
case eLogDebug:
ss << "[DBG] ";
break;
case eLogInfo:
ss << "[NFO] ";
break;
case eLogWarn:
ss << "[WRN] ";
break;
case eLogError:
ss << "[ERR] ";
break;
}
ss << "(" << thread_id_string() << ") " << log_timestamp() << " " << fname
<< ":" << lineno << "\t";
}
void
FileLogStream::Print(LogLevel, const char *, const std::string &msg)
{
m_Lines.emplace_back(msg);
}
void
FileLogStream::Tick(llarp_time_t now)
{
if(ShouldFlush(now))
FlushLinesToDisk(now);
}
void
FileLogStream::FlushLinesToDisk(llarp_time_t now)
{
FlushEvent *ev = new FlushEvent(std::move(m_Lines), m_File);
llarp_threadpool_queue_job(m_Disk, {ev, &FlushEvent::HandleFlush});
m_LastFlush = now;
}
void
FileLogStream::FlushEvent::HandleFlush(void *user)
{
static_cast< FileLogStream::FlushEvent * >(user)->Flush();
}
void
FileLogStream::FlushEvent::Flush()
{
for(const auto &line : lines)
fprintf(f, "%s\n", line.c_str());
fflush(f);
delete this;
}
} // namespace llarp

View File

@ -0,0 +1,61 @@
#ifndef LLARP_UTIL_FILE_LOGGER_HPP
#define LLARP_UTIL_FILE_LOGGER_HPP
#include <util/logstream.hpp>
#include <util/threadpool.h>
#include <util/time.hpp>
namespace llarp
{
/// fluhsable file based log stream
struct FileLogStream : public ILogStream
{
FileLogStream(llarp_threadpool* disk, FILE* f, llarp_time_t flushInterval);
~FileLogStream();
void
PreLog(std::stringstream& out, LogLevel lvl, const char* fname,
int lineno) const override;
void
Print(LogLevel, const char*, const std::string& msg) override;
void
Tick(llarp_time_t now) override;
void
PostLog(std::stringstream&) const override{};
private:
struct FlushEvent
{
FlushEvent(std::deque< std::string > l, FILE* file)
: lines(std::move(l)), f(file)
{
}
const std::deque< std::string > lines;
FILE* const f;
void
Flush();
static void
HandleFlush(void*);
};
bool
ShouldFlush(llarp_time_t now) const;
void
FlushLinesToDisk(llarp_time_t now);
llarp_threadpool* m_Disk;
FILE* m_File;
const llarp_time_t m_FlushInterval;
llarp_time_t m_LastFlush = 0;
std::deque< std::string > m_Lines;
};
} // namespace llarp
#endif

View File

@ -12,10 +12,12 @@ namespace llarp
int lineno) const override; int lineno) const override;
void void
Print(LogLevel lvl, const char* tag, const std::string& msg) const override; Print(LogLevel lvl, const char* tag, const std::string& msg) override;
void void
PostLog(std::stringstream& ss) const override; PostLog(std::stringstream& ss) const override;
void Tick(llarp_time_t) override{};
}; };
} // namespace llarp } // namespace llarp
#endif #endif

View File

@ -4,6 +4,8 @@
#include <string> #include <string>
#include <util/loglevel.hpp> #include <util/loglevel.hpp>
#include <sstream> #include <sstream>
#include <util/time.hpp>
namespace llarp namespace llarp
{ {
/// logger stream interface /// logger stream interface
@ -15,10 +17,14 @@ namespace llarp
PreLog(std::stringstream& out, LogLevel lvl, const char* fname, PreLog(std::stringstream& out, LogLevel lvl, const char* fname,
int lineno) const = 0; int lineno) const = 0;
virtual void virtual void
Print(LogLevel lvl, const char* filename, const std::string& msg) const = 0; Print(LogLevel lvl, const char* filename, const std::string& msg) = 0;
virtual void virtual void
PostLog(std::stringstream& out) const = 0; PostLog(std::stringstream& out) const = 0;
/// called every end of event loop tick
virtual void
Tick(llarp_time_t now) = 0;
}; };
using ILogStream_ptr = std::unique_ptr< ILogStream >; using ILogStream_ptr = std::unique_ptr< ILogStream >;

View File

@ -44,7 +44,7 @@ namespace llarp
} }
void void
OStreamLogStream::Print(LogLevel, const char*, const std::string& msg) const OStreamLogStream::Print(LogLevel, const char*, const std::string& msg)
{ {
m_Out << msg; m_Out << msg;
} }

View File

@ -19,11 +19,13 @@ namespace llarp
int lineno) const override; int lineno) const override;
void void
Print(LogLevel lvl, const char* tag, const std::string& msg) const override; Print(LogLevel lvl, const char* tag, const std::string& msg) override;
virtual void virtual void
PostLog(std::stringstream& ss) const override; PostLog(std::stringstream& ss) const override;
void Tick(llarp_time_t) override{};
private: private:
std::ostream& m_Out; std::ostream& m_Out;
}; };

View File

@ -31,7 +31,7 @@ namespace llarp
} }
void void
SysLogStream::Print(LogLevel lvl, const char*, const std::string& msg) const SysLogStream::Print(LogLevel lvl, const char*, const std::string& msg)
{ {
switch(lvl) switch(lvl)
{ {

View File

@ -16,8 +16,8 @@ struct llarp_threadpool
std::queue< std::function< void(void) > > jobs GUARDED_BY(m_access); std::queue< std::function< void(void) > > jobs GUARDED_BY(m_access);
llarp_threadpool(int workers, const char *name) llarp_threadpool(int workers, const char *name)
: impl( : impl(std::make_unique< llarp::thread::ThreadPool >(workers,
std::make_unique< llarp::thread::ThreadPool >(workers, workers * 128)) workers * 128))
{ {
(void)name; (void)name;
} }

View File

@ -18,6 +18,8 @@ namespace llarp
void void
PostLog(std::stringstream& s) const override; PostLog(std::stringstream& s) const override;
void Tick(llarp_time_t) override{};
bool isConsoleModern = bool isConsoleModern =
true; // qol fix so oldfag clients don't see ugly escapes true; // qol fix so oldfag clients don't see ugly escapes
HANDLE fd1 = GetStdHandle(STD_OUTPUT_HANDLE); HANDLE fd1 = GetStdHandle(STD_OUTPUT_HANDLE);