mirror of https://github.com/searxng/searxng
[mod] reduce memory footprint by not calling babel.Locale.parse at runtime
babel.Locale.parse loads more than 60MB in RAM. The only purpose is to get: LOCALE_NAMES - searx.data.LOCALES["LOCALE_NAMES"] RTL_LOCALES - searx.data.LOCALES["RTL_LOCALES"] This commit calls babel.Locale.parse when the translations are update from weblate and stored in:: searx/data/locales.json This file can be build by:: ./manage data.locales By store these variables in searx.data when the translations are updated we save round about 65MB (usually 4 worker = 260MB of RAM saved. Suggested-by: https://github.com/searxng/searxng/discussions/2633#discussioncomment-8490494 Co-authored-by: Markus Heiser <markus.heiser@darmarit.de>pull/3233/head
parent
76845ea42c
commit
ed66ed758d
@ -0,0 +1,69 @@
|
||||
{
|
||||
"LOCALE_NAMES": {
|
||||
"af": "Afrikaans",
|
||||
"ar": "العربية (Arabic)",
|
||||
"bg": "Български (Bulgarian)",
|
||||
"bn": "বাংলা (Bangla)",
|
||||
"bo": "བོད་སྐད་ (Tibetan)",
|
||||
"ca": "Català (Catalan)",
|
||||
"cs": "Čeština (Czech)",
|
||||
"cy": "Cymraeg (Welsh)",
|
||||
"da": "Dansk (Danish)",
|
||||
"de": "Deutsch (German)",
|
||||
"dv": "ދިވެހި (Dhivehi)",
|
||||
"el-GR": "Ελληνικά, Ελλάδα (Greek, Greece)",
|
||||
"en": "English",
|
||||
"eo": "Esperanto",
|
||||
"es": "Español (Spanish)",
|
||||
"et": "Eesti (Estonian)",
|
||||
"eu": "Euskara (Basque)",
|
||||
"fa-IR": "فارسی, ایران (Persian, Iran)",
|
||||
"fi": "Suomi (Finnish)",
|
||||
"fil": "Filipino",
|
||||
"fr": "Français (French)",
|
||||
"gl": "Galego (Galician)",
|
||||
"he": "עברית (Hebrew)",
|
||||
"hr": "Hrvatski (Croatian)",
|
||||
"hu": "Magyar (Hungarian)",
|
||||
"ia": "Interlingua",
|
||||
"id": "Indonesia (Indonesian)",
|
||||
"it": "Italiano (Italian)",
|
||||
"ja": "日本語 (Japanese)",
|
||||
"ko": "한국어 (Korean)",
|
||||
"lt": "Lietuvių (Lithuanian)",
|
||||
"lv": "Latviešu (Latvian)",
|
||||
"ml": "മലയാളം (Malayalam)",
|
||||
"ms": "Melayu (Malay)",
|
||||
"nb-NO": "Norsk bokmål, Norge (Norwegian bokmål, Norway)",
|
||||
"nl": "Nederlands (Dutch)",
|
||||
"nl-BE": "Nederlands, België (Dutch, Belgium)",
|
||||
"oc": "Occitan",
|
||||
"pa": "ਪੰਜਾਬੀ (Punjabi)",
|
||||
"pap": "Papiamento",
|
||||
"pl": "Polski (Polish)",
|
||||
"pt": "Português (Portuguese)",
|
||||
"pt-BR": "Português, Brasil (Portuguese, Brazil)",
|
||||
"ro": "Română (Romanian)",
|
||||
"ru": "Русский (Russian)",
|
||||
"si": "සිංහල (Sinhala)",
|
||||
"sk": "Slovenčina (Slovak)",
|
||||
"sl": "Slovenščina (Slovenian)",
|
||||
"sr": "Српски (Serbian)",
|
||||
"sv": "Svenska (Swedish)",
|
||||
"szl": "Ślōnski (Silesian)",
|
||||
"ta": "தமிழ் (Tamil)",
|
||||
"te": "తెలుగు (Telugu)",
|
||||
"th": "ไทย (Thai)",
|
||||
"tr": "Türkçe (Turkish)",
|
||||
"uk": "Українська (Ukrainian)",
|
||||
"vi": "Tiếng việt (Vietnamese)",
|
||||
"zh-HK": "中文, 中國香港特別行政區 (Chinese, Hong Kong SAR China)",
|
||||
"zh-Hans-CN": "中文, 中国 (Chinese, China)",
|
||||
"zh-Hant-TW": "中文, 台灣 (Chinese, Taiwan)"
|
||||
},
|
||||
"RTL_LOCALES": [
|
||||
"fa-IR",
|
||||
"ar",
|
||||
"he"
|
||||
]
|
||||
}
|
@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Update locale names in :origin:`searx/data/locales.json` used by
|
||||
:ref:`searx.locales`
|
||||
|
||||
- :py:obj:`searx.locales.RTL_LOCALES`
|
||||
- :py:obj:`searx.locales.LOCALE_NAMES`
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Set
|
||||
import json
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
import babel
|
||||
import babel.languages
|
||||
import babel.core
|
||||
|
||||
from searx import searx_dir
|
||||
from searx.locales import (
|
||||
ADDITIONAL_TRANSLATIONS,
|
||||
LOCALE_BEST_MATCH,
|
||||
get_translation_locales,
|
||||
)
|
||||
|
||||
LOCALE_DATA_FILE = Path(searx_dir) / 'data' / 'locales.json'
|
||||
TRANSLATOINS_FOLDER = Path(searx_dir) / 'translations'
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
LOCALE_NAMES = {}
|
||||
RTL_LOCALES: Set[str] = set()
|
||||
|
||||
for tag, descr in ADDITIONAL_TRANSLATIONS.items():
|
||||
locale = babel.Locale.parse(LOCALE_BEST_MATCH[tag], sep='-')
|
||||
LOCALE_NAMES[tag] = descr
|
||||
if locale.text_direction == 'rtl':
|
||||
RTL_LOCALES.add(tag)
|
||||
|
||||
for tag in LOCALE_BEST_MATCH:
|
||||
descr = LOCALE_NAMES.get(tag)
|
||||
if not descr:
|
||||
locale = babel.Locale.parse(tag, sep='-')
|
||||
LOCALE_NAMES[tag] = get_locale_descr(locale, tag.replace('-', '_'))
|
||||
if locale.text_direction == 'rtl':
|
||||
RTL_LOCALES.add(tag)
|
||||
|
||||
for tr_locale in get_translation_locales():
|
||||
sxng_tag = tr_locale.replace('_', '-')
|
||||
descr = LOCALE_NAMES.get(sxng_tag)
|
||||
if not descr:
|
||||
locale = babel.Locale.parse(tr_locale)
|
||||
LOCALE_NAMES[sxng_tag] = get_locale_descr(locale, tr_locale)
|
||||
if locale.text_direction == 'rtl':
|
||||
RTL_LOCALES.add(sxng_tag)
|
||||
|
||||
content = {
|
||||
"LOCALE_NAMES": LOCALE_NAMES,
|
||||
"RTL_LOCALES": list(RTL_LOCALES),
|
||||
}
|
||||
|
||||
with open(LOCALE_DATA_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(content, f, indent=2, sort_keys=True, ensure_ascii=False)
|
||||
|
||||
|
||||
def get_locale_descr(locale: babel.Locale, tr_locale):
|
||||
"""Get locale name e.g. 'Français - fr' or 'Português (Brasil) - pt-BR'
|
||||
|
||||
:param locale: instance of :py:class:`Locale`
|
||||
:param tr_locale: name e.g. 'fr' or 'pt_BR' (delimiter is *underscore*)
|
||||
"""
|
||||
|
||||
native_language, native_territory = _get_locale_descr(locale, tr_locale)
|
||||
english_language, english_territory = _get_locale_descr(locale, 'en')
|
||||
|
||||
if native_territory == english_territory:
|
||||
english_territory = None
|
||||
|
||||
if not native_territory and not english_territory:
|
||||
# none territory name
|
||||
if native_language == english_language:
|
||||
return native_language
|
||||
return native_language + ' (' + english_language + ')'
|
||||
|
||||
else:
|
||||
result = native_language + ', ' + native_territory + ' (' + english_language
|
||||
if english_territory:
|
||||
return result + ', ' + english_territory + ')'
|
||||
return result + ')'
|
||||
|
||||
|
||||
def _get_locale_descr(locale: babel.Locale, tr_locale: str) -> tuple[str, str]:
|
||||
language_name = locale.get_language_name(tr_locale).capitalize() # type: ignore
|
||||
if language_name and ('a' <= language_name[0] <= 'z'):
|
||||
language_name = language_name.capitalize()
|
||||
territory_name: str = locale.get_territory_name(tr_locale) # type: ignore
|
||||
return language_name, territory_name
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
Loading…
Reference in New Issue