[logfile] add more options when opening files

src/command_executor.cc

@@ -477,8 +477,8 @@ void execute_init_commands(vector<pair<string, string> > &msgs)
                 lnav_data.ld_pt_search.substr(3),
                 lnav_data.ld_pt_min_time,
                 lnav_data.ld_pt_max_time));
-        lnav_data.ld_file_names.insert(
+        lnav_data.ld_file_names[lnav_data.ld_pt_search]
-                make_pair(lnav_data.ld_pt_search, pt->copy_fd().release()));
+            .with_fd(pt->copy_fd());
         lnav_data.ld_curl_looper.add_request(pt.release());
 #endif
     }

src/lnav.cc

@@ -451,7 +451,9 @@ public:
 
     void closed_file(logfile *lf) {
         log_info("closed text file: %s", lf->get_filename().c_str());
-        lnav_data.ld_file_names.erase(make_pair(lf->get_filename(), lf->get_fd()));
+        if (!lf->is_valid_filename()) {
+            lnav_data.ld_file_names.erase(lf->get_filename());
+        }
         lnav_data.ld_files.remove(lf);
         delete lf;
     };
@@ -562,7 +564,9 @@ void rebuild_indexes(bool force)
 
         if (!lf->exists() || lf->is_closed()) {
             log_info("closed log file: %s", lf->get_filename().c_str());
-            lnav_data.ld_file_names.erase(make_pair(lf->get_filename(), lf->get_fd()));
+            if (!lf->is_valid_filename()) {
+                lnav_data.ld_file_names.erase(lf->get_filename());
+            }
             lnav_data.ld_text_source.remove(lf);
             lnav_data.ld_log_source.remove_file(lf);
             file_iter = lnav_data.ld_files.erase(file_iter);
@@ -649,8 +653,9 @@ static bool append_default_files(lnav_flags_t flag)
                     perror("Unable to resolve path");
                 }
                 else {
-                    lnav_data.ld_file_names.insert(make_pair(abspath.in(),
+                    logfile_open_options default_loo;
-                                                             -1));
+
+                    lnav_data.ld_file_names[abspath.in()] = default_loo;
                 }
             }
             else if (stat(path.c_str(), &st) == 0) {
@@ -1145,7 +1150,7 @@ struct same_file {
  * @param fd       An already-opened descriptor for 'filename'.
  * @param required Specifies whether or not the file must exist and be valid.
  */
-static bool watch_logfile(string filename, int fd, bool required)
+static bool watch_logfile(string filename, logfile_open_options &loo, bool required)
 {
     static loading_observer obs;
     list<logfile *>::iterator file_iter;
@@ -1157,8 +1162,8 @@ static bool watch_logfile(string filename, int fd, bool required)
         return retval;
     }
 
-    if (fd != -1) {
+    if (loo.loo_fd != -1) {
-        rc = fstat(fd, &st);
+        rc = fstat(loo.loo_fd, &st);
     }
     else {
         rc = stat(filename.c_str(), &st);
@@ -1203,10 +1208,10 @@ static bool watch_logfile(string filename, int fd, bool required)
 
             default:
                 /* It's a new file, load it in. */
-                logfile *lf = new logfile(filename, fd);
+                logfile *lf = new logfile(filename, loo);
 
-                log_info("loading new file: fd=%d; filename=%s",
+                log_info("loading new file: filename=%s",
-                         fd, filename.c_str());
+                         filename.c_str());
                 lf->set_logfile_observer(&obs);
                 lnav_data.ld_files.push_back(lf);
                 lnav_data.ld_text_source.push_back(lf);
@@ -1263,8 +1268,10 @@ static void expand_filename(string path, bool required)
                         gl->gl_pathv[lpc], strerror(errno));
                 }
             }
-            else if (required || access(abspath.in(), R_OK) == 0){
+            else if (required || access(abspath.in(), R_OK) == 0) {
-                watch_logfile(abspath.in(), -1, required);
+                logfile_open_options loo;
+
+                watch_logfile(abspath.in(), loo, required);
             }
         }
     }
@@ -1272,14 +1279,14 @@ static void expand_filename(string path, bool required)
 
 bool rescan_files(bool required)
 {
-    set<pair<string, int> >::iterator iter;
+    map<string, logfile_open_options>::iterator iter;
     list<logfile *>::iterator         file_iter;
     bool retval = false;
 
     for (iter = lnav_data.ld_file_names.begin();
          iter != lnav_data.ld_file_names.end();
          iter++) {
-        if (iter->second == -1) {
+        if (iter->second.loo_fd == -1) {
             expand_filename(iter->first, required);
             if (lnav_data.ld_flags & LNF_ROTATED) {
                 string path = iter->first + ".*";
@@ -1297,6 +1304,8 @@ bool rescan_files(bool required)
         logfile *lf = *file_iter;
 
         if (!lf->exists() || lf->is_closed()) {
+            log_info("Log file no longer exists or is closed: %s",
+                     lf->get_filename().c_str());
             return true;
         }
         else {
@@ -1410,9 +1419,8 @@ static string execute_action(log_data_helper &ldh,
                     sizeof(desc), "[%d] Output of %s",
                     exec_count++,
                     action.ad_cmdline[0].c_str());
-                lnav_data.ld_file_names.insert(make_pair(
+                lnav_data.ld_file_names[desc]
-                    desc,
+                    .with_fd(pp->get_fd());
-                    pp->get_fd()));
                 lnav_data.ld_files_to_front.push_back(make_pair(desc, 0));
             }
 
@@ -2590,6 +2598,7 @@ int main(int argc, char *argv[])
     }
 
     for (lpc = 0; lpc < argc; lpc++) {
+        logfile_open_options default_loo;
         auto_mem<char> abspath;
         struct stat    st;
 
@@ -2605,12 +2614,13 @@ int main(int argc, char *argv[])
         else if (is_url(argv[lpc])) {
             auto_ptr<url_loader> ul(new url_loader(argv[lpc]));
 
-            lnav_data.ld_file_names.insert(make_pair(argv[lpc], ul->copy_fd().release()));
+            lnav_data.ld_file_names[argv[lpc]]
+                .with_fd(ul->copy_fd());
             lnav_data.ld_curl_looper.add_request(ul.release());
         }
 #endif
         else if (is_glob(argv[lpc])) {
-            lnav_data.ld_file_names.insert(make_pair(argv[lpc], -1));
+            lnav_data.ld_file_names[argv[lpc]] = default_loo;
         }
         else if (stat(argv[lpc], &st) == -1) {
             fprintf(stderr,
@@ -2629,10 +2639,10 @@ int main(int argc, char *argv[])
             if (dir_wild[dir_wild.size() - 1] == '/') {
                 dir_wild.resize(dir_wild.size() - 1);
             }
-            lnav_data.ld_file_names.insert(make_pair(dir_wild + "/*", -1));
+            lnav_data.ld_file_names[dir_wild + "/*"] = default_loo;
         }
         else {
-            lnav_data.ld_file_names.insert(make_pair(abspath.in(), -1));
+            lnav_data.ld_file_names[abspath.in()] = default_loo;
         }
     }
 
@@ -2706,7 +2716,8 @@ int main(int argc, char *argv[])
                                                 lnav_data.ld_flags &
                                                 LNF_TIMESTAMP, stdin_out));
         stdin_out_fd = stdin_reader->get_fd();
-        lnav_data.ld_file_names.insert(make_pair("stdin", stdin_out_fd));
+        lnav_data.ld_file_names["stdin"]
+            .with_fd(stdin_out_fd);
         if (dup2(STDOUT_FILENO, STDIN_FILENO) == -1) {
             perror("cannot dup stdout to stdin");
         }
@@ -2751,7 +2762,7 @@ int main(int argc, char *argv[])
                 log_info("    %s", cmd_iter->c_str());
             }
             log_info("  files:");
-            for (std::set<pair<string, int> >::iterator file_iter =
+            for (map<string, logfile_open_options>::iterator file_iter =
                  lnav_data.ld_file_names.begin();
                  file_iter != lnav_data.ld_file_names.end();
                  ++file_iter) {

src/lnav.hh

@@ -186,7 +186,7 @@ struct _lnav_data {
     std::list<std::string>                  ld_commands;
     bool                                    ld_cmd_init_done;
     std::vector<std::string>                ld_config_paths;
-    std::set<std::pair<std::string, int> >  ld_file_names;
+    std::map<std::string, logfile_open_options> ld_file_names;
     std::list<logfile *>                    ld_files;
     std::list<std::string>                  ld_other_files;
     std::set<std::string>                   ld_closed_files;

src/lnav_commands.cc

@@ -102,8 +102,8 @@ static string refresh_pt_search()
             lnav_data.ld_pt_search.substr(3),
             lnav_data.ld_pt_min_time,
             lnav_data.ld_pt_max_time));
-    lnav_data.ld_file_names.insert(
+    lnav_data.ld_file_names[lnav_data.ld_pt_search]
-            make_pair(lnav_data.ld_pt_search, pt->copy_fd().release()));
+        .with_fd(pt->copy_fd());
     lnav_data.ld_curl_looper.add_request(pt.release());
 
     ensure_view(&lnav_data.ld_views[LNV_LOG]);
@@ -735,9 +735,9 @@ static string com_pipe_to(string cmdline, vector<string> &args)
                         sizeof(desc), "[%d] Output of %s",
                         exec_count++,
                         cmdline.c_str());
-                lnav_data.ld_file_names.insert(make_pair(
+                lnav_data.ld_file_names[desc]
-                        desc,
+                    .with_fd(pp->get_fd())
-                        pp->get_fd()));
+                    .with_detect_format(false);
                 lnav_data.ld_files_to_front.push_back(make_pair(desc, 0));
                 if (lnav_data.ld_rl_view != NULL) {
                     lnav_data.ld_rl_view->set_alt_value(HELP_MSG_1(
@@ -1395,6 +1395,7 @@ static string com_open(string cmdline, vector<string> &args)
             }
         }
         if (file_iter == lnav_data.ld_files.end()) {
+            logfile_open_options default_loo;
             auto_mem<char> abspath;
             struct stat    st;
 
@@ -1404,7 +1405,8 @@ static string com_open(string cmdline, vector<string> &args)
 #else
                 auto_ptr<url_loader> ul(new url_loader(fn));
 
-                lnav_data.ld_file_names.insert(make_pair(fn, ul->copy_fd().release()));
+                lnav_data.ld_file_names[fn]
+                    .with_fd(ul->copy_fd());
                 lnav_data.ld_curl_looper.add_request(ul.release());
                 lnav_data.ld_files_to_front.push_back(make_pair(fn, top));
 
@@ -1412,7 +1414,7 @@ static string com_open(string cmdline, vector<string> &args)
 #endif
             }
             else if (is_glob(fn.c_str())) {
-                lnav_data.ld_file_names.insert(make_pair(fn, -1));
+                lnav_data.ld_file_names[fn] = default_loo;
                 retval = "info: watching -- " + fn;
             }
             else if (stat(fn.c_str(), &st) == -1) {
@@ -1428,7 +1430,7 @@ static string com_open(string cmdline, vector<string> &args)
                 if (dir_wild[dir_wild.size() - 1] == '/') {
                     dir_wild.resize(dir_wild.size() - 1);
                 }
-                lnav_data.ld_file_names.insert(make_pair(dir_wild + "/*", -1));
+                lnav_data.ld_file_names[dir_wild + "/*"] = default_loo;
                 retval = "info: watching -- " + dir_wild;
             }
             else if (!S_ISREG(st.st_mode)) {
@@ -1440,7 +1442,7 @@ static string com_open(string cmdline, vector<string> &args)
             }
             else {
                 fn = abspath.in();
-                lnav_data.ld_file_names.insert(make_pair(fn, -1));
+                lnav_data.ld_file_names[fn] = default_loo;
                 retval = "info: opened -- " + fn;
                 lnav_data.ld_files_to_front.push_back(make_pair(fn, top));
 
@@ -1500,7 +1502,7 @@ static string com_close(string cmdline, vector<string> &args)
             if (is_url(fn.c_str())) {
                 lnav_data.ld_curl_looper.close_request(fn);
             }
-            lnav_data.ld_file_names.erase(make_pair(fn, -1));
+            lnav_data.ld_file_names.erase(fn);
             lnav_data.ld_closed_files.insert(fn);
             retval = "info: closed -- " + fn;
         }

src/logfile.cc

@@ -52,7 +52,7 @@ using namespace std;
 static const size_t MAX_UNRECOGNIZED_LINES = 1000;
 static const size_t INDEX_RESERVE_INCREMENT = 1024;
 
-logfile::logfile(const string &filename, auto_fd fd)
+logfile::logfile(const string &filename, logfile_open_options &loo)
 throw (error)
     : lf_filename(filename),
       lf_index_time(0),
@@ -68,7 +68,7 @@ throw (error)
     this->lf_time_offset.tv_usec = 0;
     
     memset(&this->lf_stat, 0, sizeof(this->lf_stat));
-    if (fd == -1) {
+    if (loo.loo_fd == -1) {
         char resolved_path[PATH_MAX];
 
         errno = 0;
@@ -84,14 +84,14 @@ throw (error)
             throw error(filename, EINVAL);
         }
 
-        if ((fd = open(resolved_path, O_RDONLY)) == -1) {
+        if ((loo.loo_fd = open(resolved_path, O_RDONLY)) == -1) {
             throw error(filename, errno);
         }
 
-        fd.close_on_exec();
+        loo.loo_fd.close_on_exec();
 
         log_info("Creating logfile: fd=%d; size=%d; mtime=%d; filename=%s",
-                 (int) fd,
+                 (int) loo.loo_fd,
                  this->lf_stat.st_size,
                  this->lf_stat.st_mtime,
                  filename.c_str());
@@ -99,14 +99,16 @@ throw (error)
         this->lf_valid_filename = true;
     }
     else {
-        log_perror(fstat(fd, &this->lf_stat));
+        log_perror(fstat(loo.loo_fd, &this->lf_stat));
         this->lf_valid_filename = false;
     }
 
     this->lf_content_id = hash_string(this->lf_filename);
-    this->lf_line_buffer.set_fd(fd);
+    this->lf_line_buffer.set_fd(loo.loo_fd);
     this->lf_index.reserve(INDEX_RESERVE_INCREMENT);
 
+    this->lf_options = loo;
+
     ensure(this->invariant());
 }
 
@@ -149,7 +151,8 @@ void logfile::process_prefix(off_t offset, shared_buffer_ref &sbr)
         /* We've locked onto a format, just use that scanner. */
         found = this->lf_format->scan(this->lf_index, offset, sbr);
     }
-    else if (this->lf_index.size() < MAX_UNRECOGNIZED_LINES) {
+    else if (this->lf_options.loo_detect_format &&
+             this->lf_index.size() < MAX_UNRECOGNIZED_LINES) {
         vector<log_format *> &root_formats =
             log_format::get_root_formats();
         vector<log_format *>::iterator iter;

src/logfile.hh

@@ -67,6 +67,26 @@ public:
     virtual void logfile_indexing(logfile &lf, off_t off, size_t total) = 0;
 };
 
+struct logfile_open_options {
+    logfile_open_options() : loo_detect_format(true) {
+    };
+
+    logfile_open_options &with_fd(auto_fd fd) {
+        this->loo_fd = fd;
+
+        return *this;
+    };
+
+    logfile_open_options &with_detect_format(bool val) {
+        this->loo_detect_format = val;
+
+        return *this;
+    };
+
+    auto_fd loo_fd;
+    bool loo_detect_format;
+};
+
 /**
  * Container for the lines in a log file and some metadata.
  */
@@ -94,7 +114,7 @@ public:
      * constructor should open the file specified by 'filename'.  The
      * descriptor needs to be seekable.
      */
-    logfile(const std::string &filename, auto_fd fd = -1) throw (error);
+    logfile(const std::string &filename, logfile_open_options &loo) throw (error);
 
     virtual ~logfile();
 
@@ -122,6 +142,10 @@ public:
         return this->lf_line_buffer.is_compressed();
     };
 
+    bool is_valid_filename() const {
+        return this->lf_valid_filename;
+    };
+
     /**
      * @return The detected format, rebuild_index() must be called before this
      * will return a value other than NULL.
@@ -351,6 +375,7 @@ protected:
 
     void set_format_base_time(log_format *lf);
 
+    logfile_open_options lf_options;
     bool        lf_valid_filename;
     std::string lf_filename;
     std::string lf_content_id;

src/readline_callbacks.cc

@@ -311,8 +311,9 @@ void rl_callback(void *dummy, readline_curses *rc)
                          "Output of %s (%s)",
                          path_and_args.c_str(),
                          timestamp);
-                lnav_data.ld_file_names.insert(
+                lnav_data.ld_file_names[desc]
-                    make_pair(desc, fd_copy.release()));
+                    .with_fd(fd_copy)
+                    .with_detect_format(false);
                 lnav_data.ld_files_to_front.push_back(make_pair(desc, 0));
 
                 if (lnav_data.ld_rl_view != NULL) {

src/session_data.cc

@@ -262,9 +262,11 @@ void init_session(void)
 
     context.Init(0, 0);
     hash_updater updater(&context);
-    for_each(lnav_data.ld_file_names.begin(),
+    for (map<string, logfile_open_options>::iterator iter = lnav_data.ld_file_names.begin();
-             lnav_data.ld_file_names.end(),
+         iter != lnav_data.ld_file_names.end();
-             object_field(updater, &pair<string, int>::first));
+         ++iter) {
+        updater(iter->first);
+    }
     context.Final(hash.out(0), hash.out(1));
 
     lnav_data.ld_session_id = hash.to_string();
@@ -1224,10 +1226,11 @@ void save_session(void)
             {
                 yajlpp_array file_list(handle);
 
-                for_each(lnav_data.ld_file_names.begin(),
+                for (map<string, logfile_open_options>::iterator iter = lnav_data.ld_file_names.begin();
-                         lnav_data.ld_file_names.end(),
+                     iter != lnav_data.ld_file_names.end();
-                         object_field(file_list.gen,
+                     ++iter) {
-                                      &pair<string, int>::first));
+                    file_list.gen(iter->first);
+                }
             }
 
             root_map.gen("views");

test/drive_logfile.cc

@@ -102,7 +102,8 @@ int main(int argc, char *argv[])
     fprintf(stderr, "error: expecting log file name\n");
   }
   else {
-    logfile lf(argv[0]);
+    logfile_open_options default_loo;
+    logfile lf(argv[0], default_loo);
     struct stat st;
 
     stat(argv[0], &st);