strgen: Split non-upstream english.txt strings into separate file

src/lang/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(MASTER_LANG_FILE
         ${CMAKE_CURRENT_SOURCE_DIR}/english.txt
+        ${CMAKE_CURRENT_SOURCE_DIR}/extra/english.txt
 )
 
 set(LANG_SOURCE_FILES

src/strgen/strgen.cpp

@@ -96,6 +96,8 @@ void NORETURN CDECL error(const char *s, ...)
 /** A reader that simply reads using fopen. */
 struct FileStringReader : StringReader {
 	FILE *fh; ///< The file we are reading.
+	FILE *fh2 = nullptr; ///< The file we are reading.
+	std::string file2;
 
 	/**
 	 * Create the reader.
@@ -104,22 +106,42 @@ struct FileStringReader : StringReader {
 	 * @param master      Are we reading the master file?
 	 * @param translation Are we reading a translation?
 	 */
-	FileStringReader(StringData &data, const char *file, bool master, bool translation) :
+	FileStringReader(StringData &data, const char *file, const char *file2, bool master, bool translation) :
 			StringReader(data, file, master, translation)
 	{
 		this->fh = fopen(file, "rb");
 		if (this->fh == nullptr) error("Could not open %s", file);
+
+		if (file2 != nullptr) {
+			this->file2.assign(file2);
+			this->fh2 = fopen(file2, "rb");
+			if (this->fh2 == nullptr) error("Could not open %s", file2);
+		}
 	}
 
+	FileStringReader(StringData &data, const char *file, bool master, bool translation) :
+			FileStringReader(data, file, nullptr, master, translation) {}
+
 	/** Free/close the file. */
 	virtual ~FileStringReader()
 	{
 		fclose(this->fh);
+		if (this->fh2 != nullptr) fclose(this->fh2);
 	}
 
 	char *ReadLine(char *buffer, const char *last) override
 	{
-		return fgets(buffer, ClampToU16(last - buffer + 1), this->fh);
+		char *result = fgets(buffer, ClampToU16(last - buffer + 1), this->fh);
+		if (result == nullptr && this->fh2 != nullptr) {
+			fclose(this->fh);
+			this->fh = this->fh2;
+			this->fh2 = nullptr;
+			this->file = std::move(this->file2);
+			_file = this->file.c_str();
+			_cur_line = 1;
+			return this->FileStringReader::ReadLine(buffer, last);
+		}
+		return result;
 	}
 
 	void HandlePragma(char *str) override;
@@ -199,6 +221,35 @@ void FileStringReader::HandlePragma(char *str)
 			strecpy(_lang.cases[_lang.num_cases], s, lastof(_lang.cases[_lang.num_cases]));
 			_lang.num_cases++;
 		}
+	} else if (!memcmp(str, "override ", 9)) {
+		if (this->translation) error("Overrides are only allowed in the base translation.");
+		if (!memcmp(str + 9, "on", 2)) {
+			this->data.override_mode = true;
+		} else if (!memcmp(str + 9, "off", 3)) {
+			this->data.override_mode = false;
+		} else {
+			error("Invalid override mode %s", str + 9);
+		}
+	} else if (!memcmp(str, "override ", 9)) {
+		if (this->translation) error("Overrides are only allowed in the base translation.");
+		if (!memcmp(str + 9, "on", 2)) {
+			this->data.override_mode = true;
+		} else if (!memcmp(str + 9, "off", 3)) {
+			this->data.override_mode = false;
+		} else {
+			error("Invalid override mode %s", str + 9);
+		}
+	} else if (!memcmp(str, "after ", 6)) {
+		if (this->translation) error("Insert after is only allowed in the base translation.");
+		LangString *ent = this->data.Find(str + 6);
+		if (ent != nullptr) {
+			this->data.insert_after = ent;
+		} else {
+			error("Can't find string to insert after: '%s'", str + 6);
+		}
+	} else if (!memcmp(str, "end-after", 10)) {
+		if (this->translation) error("Insert after is only allowed in the base translation.");
+		this->data.insert_after = nullptr;
 	} else {
 		StringReader::HandlePragma(str);
 	}
@@ -389,16 +440,33 @@ static inline void ottd_mkdir(const char *directory)
  * path separator and the filename. The separator is only appended if the path
  * does not already end with a separator
  */
-static inline char *mkpath(char *buf, const char *last, const char *path, const char *file)
+static inline char *mkpath2(char *buf, const char *last, const char *path, const char *path2, const char *file)
 {
 	strecpy(buf, path, last); // copy directory into buffer
 
 	char *p = strchr(buf, '\0'); // add path separator if necessary
+
+	if (path2 != nullptr) {
+		if (p[-1] != PATHSEPCHAR && p != last) *p++ = PATHSEPCHAR;
+		strecpy(p, path2, last); // concatenate filename at end of buffer
+		p = strchr(buf, '\0');
+	}
+
 	if (p[-1] != PATHSEPCHAR && p != last) *p++ = PATHSEPCHAR;
 	strecpy(p, file, last); // concatenate filename at end of buffer
 	return buf;
 }
 
+/**
+ * Create a path consisting of an already existing path, a possible
+ * path separator and the filename. The separator is only appended if the path
+ * does not already end with a separator
+ */
+static inline char *mkpath(char *buf, const char *last, const char *path, const char *file)
+{
+	return mkpath2(buf, last, path, nullptr, file);
+}
+
 #if defined(_WIN32)
 /**
  * On MingW, it is common that both / as \ are accepted in the
@@ -434,6 +502,7 @@ static const OptionData _opts[] = {
 int CDECL main(int argc, char *argv[])
 {
 	char pathbuf[MAX_PATH];
+	char pathbuf2[MAX_PATH];
 	const char *src_dir = ".";
 	const char *dest_dir = nullptr;
 
@@ -528,10 +597,11 @@ int CDECL main(int argc, char *argv[])
 		 * directory. As input english.txt is parsed from the source directory */
 		if (mgo.numleft == 0) {
 			mkpath(pathbuf, lastof(pathbuf), src_dir, "english.txt");
+			mkpath2(pathbuf2, lastof(pathbuf2), src_dir, "extra", "english.txt");
 
 			/* parse master file */
 			StringData data(TEXT_TAB_END);
-			FileStringReader master_reader(data, pathbuf, true, false);
+			FileStringReader master_reader(data, pathbuf, pathbuf2, true, false);
 			master_reader.ParseFile();
 			if (_errors != 0) return 1;
 
@@ -547,10 +617,11 @@ int CDECL main(int argc, char *argv[])
 			char *r;
 
 			mkpath(pathbuf, lastof(pathbuf), src_dir, "english.txt");
+			mkpath2(pathbuf2, lastof(pathbuf2), src_dir, "extra", "english.txt");
 
 			StringData data(TEXT_TAB_END);
 			/* parse master file and check if target file is correct */
-			FileStringReader master_reader(data, pathbuf, true, false);
+			FileStringReader master_reader(data, pathbuf, pathbuf2, true, false);
 			master_reader.ParseFile();
 
 			for (int i = 0; i < mgo.numleft; i++) {

src/strgen/strgen.h

@@ -12,6 +12,10 @@
 
 #include "../language.h"
 
+#include <memory>
+#include <string>
+#include <vector>
+
 /** Container for the different cases of a string. */
 struct Case {
 	int caseidx;  ///< The index of the case.
@@ -27,12 +31,14 @@ struct LangString {
 	char *name;            ///< Name of the string.
 	char *english;         ///< English text.
 	char *translated;      ///< Translated text.
-	size_t hash_next;      ///< Next hash entry.
-	size_t index;          ///< The index in the language file.
+	LangString *hash_next; ///< Next hash entry.
+	int index;             ///< The index in the language file.
 	int line;              ///< Line of string in source-file.
 	Case *translated_case; ///< Cases of the translation.
+	std::unique_ptr<LangString> chain_next;
 
 	LangString(const char *name, const char *english, size_t index, int line);
+	void ReplaceDefinition(const char *name, const char *english, int line);
 	~LangString();
 	void FreeTranslation();
 };
@@ -40,10 +46,14 @@ struct LangString {
 /** Information about the currently known strings. */
 struct StringData {
 	LangString **strings; ///< Array of all known strings.
-	size_t *hash_heads;   ///< Hash table for the strings.
+	LangString **hash_heads; ///< Hash table for the strings.
 	size_t tabs;          ///< The number of 'tabs' of strings.
 	size_t max_strings;   ///< The maximum number of strings.
-	size_t next_string_id;///< The next string ID to allocate.
+	int next_string_id;   ///< The next string ID to allocate.
+
+	std::vector<std::unique_ptr<LangString>> string_store;
+	LangString *insert_after = nullptr;
+	bool override_mode = false;
 
 	StringData(size_t tabs);
 	~StringData();
@@ -59,7 +69,7 @@ struct StringData {
 /** Helper for reading strings. */
 struct StringReader {
 	StringData &data; ///< The data to fill during reading.
-	const char *file; ///< The file we are reading.
+	std::string file; ///< The file we are reading.
 	bool master;      ///< Are we reading the master file?
 	bool translation; ///< Are we reading a translation, implies !master. However, the base translation will have this false.
 

src/strgen/strgen_base.cpp

@@ -71,6 +71,15 @@ LangString::~LangString()
 	delete this->translated_case;
 }
 
+void LangString::ReplaceDefinition(const char *name, const char *english, int line)
+{
+	free(this->name);
+	free(this->english);
+	this->name = stredup(name);
+	this->english = stredup(english);
+	this->line = line;
+}
+
 /** Free all data related to the translation. */
 void LangString::FreeTranslation()
 {
@@ -88,14 +97,13 @@ void LangString::FreeTranslation()
 StringData::StringData(size_t tabs) : tabs(tabs), max_strings(tabs * TAB_SIZE)
 {
 	this->strings = CallocT<LangString *>(max_strings);
-	this->hash_heads = CallocT<size_t>(max_strings);
+	this->hash_heads = CallocT<LangString *>(max_strings);
 	this->next_string_id = 0;
 }
 
 /** Free everything we allocated. */
 StringData::~StringData()
 {
-	for (size_t i = 0; i < this->max_strings; i++) delete this->strings[i];
 	free(this->strings);
 	free(this->hash_heads);
 }
@@ -131,8 +139,7 @@ void StringData::Add(const char *s, LangString *ls)
 	uint hash = this->HashStr(s);
 	ls->hash_next = this->hash_heads[hash];
 	/* Off-by-one for hash find. */
-	this->hash_heads[hash] = ls->index + 1;
-	this->strings[ls->index] = ls;
+	this->hash_heads[hash] = ls;
 }
 
 /**
@@ -142,13 +149,11 @@ void StringData::Add(const char *s, LangString *ls)
  */
 LangString *StringData::Find(const char *s)
 {
-	size_t idx = this->hash_heads[this->HashStr(s)];
-
-	while (idx-- > 0) {
-		LangString *ls = this->strings[idx];
+	LangString *ls = this->hash_heads[this->HashStr(s)];
 
+	while (ls != nullptr) {
 		if (strcmp(ls->name, s) == 0) return ls;
-		idx = ls->hash_next;
+		ls = ls->hash_next;
 	}
 	return nullptr;
 }
@@ -591,7 +596,6 @@ StringReader::StringReader(StringData &data, const char *file, bool master, bool
 /** Make sure the right reader gets freed. */
 StringReader::~StringReader()
 {
-	free(file);
 }
 
 static void ExtractCommandString(ParsedCommandStruct *p, const char *s, bool warnings)
@@ -758,17 +762,33 @@ void StringReader::HandleString(char *str)
 		}
 
 		if (ent != nullptr) {
+			if (this->data.override_mode) {
+				ent->ReplaceDefinition(str, s, _cur_line);
+				return;
+			}
 			strgen_error("String name '%s' is used multiple times", str);
 			return;
+		} else if (this->data.override_mode) {
+			strgen_error("String '%s' marked as overriding, but does not override", str);
+			return;
 		}
 
-		if (this->data.strings[this->data.next_string_id] != nullptr) {
-			strgen_error("String ID 0x" PRINTF_SIZEX " for '%s' already in use by '%s'", this->data.next_string_id, str, this->data.strings[this->data.next_string_id]->name);
-			return;
+		if (this->data.next_string_id >= 0 && this->data.insert_after != nullptr) {
+			strgen_error("Cannot use insert_after and id at the same time: '%s'", str);
 		}
 
 		/* Allocate a new LangString */
-		this->data.Add(str, new LangString(str, s, this->data.next_string_id++, _cur_line));
+		std::unique_ptr<LangString> ls(new LangString(str, s, this->data.next_string_id, _cur_line));
+		this->data.next_string_id = -1;
+		this->data.Add(str, ls.get());
+
+		if (this->data.insert_after != nullptr) {
+			LangString *cur = ls.get();
+			this->data.insert_after->chain_next = std::move(ls);
+			this->data.insert_after = cur;
+		} else {
+			this->data.string_store.push_back(std::move(ls));
+		}
 	} else {
 		if (ent == nullptr) {
 			strgen_warning("String name '%s' does not exist in master file", str);
@@ -820,7 +840,7 @@ void StringReader::ParseFile()
 	_warnings = _errors = 0;
 
 	_translation = this->translation;
-	_file = this->file;
+	_file = this->file.c_str();
 
 	/* Abusing _show_todo to replace "warning" with "info" for translations. */
 	_show_todo &= 3;
@@ -833,14 +853,38 @@ void StringReader::ParseFile()
 	strecpy(_lang.digit_decimal_separator, ".", lastof(_lang.digit_decimal_separator));
 
 	_cur_line = 1;
-	while (this->data.next_string_id < this->data.max_strings && this->ReadLine(buf, lastof(buf)) != nullptr) {
+	while (this->ReadLine(buf, lastof(buf)) != nullptr) {
 		rstrip(buf);
 		this->HandleString(buf);
 		_cur_line++;
 	}
 
-	if (this->data.next_string_id == this->data.max_strings) {
-		strgen_error("Too many strings, maximum allowed is " PRINTF_SIZE, this->data.max_strings);
+	if (this->master) {
+		/* Allocate IDs */
+		size_t next_id = 0;
+		for (const std::unique_ptr<LangString> &item : this->data.string_store) {
+			LangString *ls = item.get();
+			do {
+				if (ls->index >= 0) {
+					next_id = ls->index;
+				} else {
+					ls->index = next_id;
+				}
+
+				if ((size_t)ls->index >= this->data.max_strings) {
+					strgen_error("Too many strings, maximum allowed is " PRINTF_SIZE, this->data.max_strings);
+					return;
+				} else if (this->data.strings[ls->index] != nullptr) {
+					strgen_error("String ID 0x%X for '%s' already in use by '%s'", (uint)ls->index, ls->name, this->data.strings[ls->index]->name);
+					return;
+				} else {
+					this->data.strings[ls->index] = ls;
+				}
+
+				next_id++;
+				ls = ls->chain_next.get();
+			} while (ls != nullptr);
+		}
 	}
 }