Update translations (#1348)

* Show translations in own tongue

* Fix flake8

* Update

* Update

* Update

* Update

* fix mypy

* Update

* Update

Co-authored-by: Daniel Girtler <girtler.daniel@gmail.com>

10 files changed, 219 insertions, 198 deletions

diff --git a/archinstall/__init__.py b/archinstall/__init__.py
index 1a360c67..f607a922 100644
--- a/archinstall/__init__.py
+++ b/archinstall/__init__.py
@@ -40,7 +40,7 @@ from .lib.menu.selection_menu import (
 	Selector,
 	GeneralMenu
 )
-from .lib.translation import Translation, DeferredTranslation
+from .lib.translationhandler import TranslationHandler, DeferredTranslation
 from .lib.plugins import plugins, load_plugin # This initiates the plugin loading ceremony
 from .lib.configuration import *
 from .lib.udev import udevadm_info
diff --git a/archinstall/lib/menu/global_menu.py b/archinstall/lib/menu/global_menu.py
index 1a292476..1badc052 100644
--- a/archinstall/lib/menu/global_menu.py
+++ b/archinstall/lib/menu/global_menu.py
@@ -50,7 +50,8 @@ class GlobalMenu(GeneralMenu):
 			Selector(
 				_('Archinstall language'),
 				lambda x: self._select_archinstall_language(x),
-				default='English')
+				display_func=lambda x: x.display_name,
+				default=self.translation_handler.get_language('en'))
 		self._menu_options['keyboard-layout'] = \
 			Selector(
 				_('Keyboard layout'),
diff --git a/archinstall/lib/menu/selection_menu.py b/archinstall/lib/menu/selection_menu.py
index c6ac5852..8a08812c 100644
--- a/archinstall/lib/menu/selection_menu.py
+++ b/archinstall/lib/menu/selection_menu.py
@@ -8,9 +8,11 @@ from typing import Callable, Any, List, Iterator, Tuple, Optional, Dict, TYPE_CH
 from .menu import Menu, MenuSelectionType
 from ..locale_helpers import set_keyboard_language
 from ..output import log
-from ..translation import Translation
+from ..translationhandler import TranslationHandler, Language
 from ..hsm.fido import get_fido2_devices
 
+from ..user_interaction.general_conf import select_archinstall_language
+
 if TYPE_CHECKING:
 	_: Any
 
@@ -181,7 +183,7 @@ class GeneralMenu:
 
 		"""
 		self._enabled_order :List[str] = []
-		self._translation = Translation.load_nationalization()
+		self._translation_handler = TranslationHandler()
 		self.is_context_mgr = False
 		self._data_store = data_store if data_store is not None else {}
 		self.auto_cursor = auto_cursor
@@ -213,6 +215,10 @@ class GeneralMenu:
 
 		self.exit_callback()
 
+	@property
+	def translation_handler(self) -> TranslationHandler:
+		return self._translation_handler
+
 	def _setup_selection_menu_options(self):
 		""" Define the menu options.
 			Menu options can be defined here in a subclass or done per program calling self.set_option()
@@ -461,14 +467,10 @@ class GeneralMenu:
 					mandatory_waiting += 1
 		return mandatory_fields, mandatory_waiting
 
-	def _select_archinstall_language(self, preset_value: str) -> str:
-		from ... import select_archinstall_language
-		language = select_archinstall_language(preset_value)
-		if language is not None:
-			self._translation.activate(language)
-			return language
-
-		return preset_value
+	def _select_archinstall_language(self, preset_value: Language) -> Language:
+		language = select_archinstall_language(self.translation_handler.translated_languages, preset_value)
+		self._translation_handler.activate(language)
+		return language
 
 	def _select_hsm(self, preset :Optional[pathlib.Path] = None) -> Optional[pathlib.Path]:
 		title = _('Select which partitions to mark for formatting:')
diff --git a/archinstall/lib/translation.py b/archinstall/lib/translation.py
deleted file mode 100644
index c20a4285..00000000
--- a/archinstall/lib/translation.py
+++ /dev/null
@@ -1,144 +0,0 @@
-from __future__ import annotations
-
-import json
-import logging
-import os
-import gettext
-
-from pathlib import Path
-from typing import List, Dict, Any, TYPE_CHECKING, Tuple
-from .exceptions import TranslationError
-
-if TYPE_CHECKING:
-	_: Any
-
-
-class LanguageDefinitions:
-	_languages = 'languages.json'
-	_cyrillic = 'cyrillic.json'
-
-	def __init__(self):
-		self._mappings = self._get_language_mappings()
-		self._cyrillic_languages = self._get_cyrillic_languages()
-
-	def is_cyrillic(self, language: str) -> bool:
-		return language in self._cyrillic_languages
-
-	def _get_language_mappings(self) -> List[Dict[str, str]]:
-		locales_dir = Translation.get_locales_dir()
-		languages = Path.joinpath(locales_dir, self._languages)
-
-		with open(languages, 'r') as fp:
-			return json.load(fp)
-
-	def get_language(self, abbr: str) -> str:
-		for entry in self._mappings:
-			if entry['abbr'] == abbr:
-				return entry['lang']
-
-		raise ValueError(f'No language with abbreviation "{abbr}" found')
-
-	def _get_cyrillic_languages(self) -> List[str]:
-		locales_dir = Translation.get_locales_dir()
-		languages = Path.joinpath(locales_dir, self._cyrillic)
-
-		with open(languages, 'r') as fp:
-			data = json.load(fp)
-			return data['languages']
-
-
-class DeferredTranslation:
-	def __init__(self, message: str):
-		self.message = message
-
-	def __len__(self) -> int:
-		return len(self.message)
-
-	def __str__(self) -> str:
-		translate = _
-		if translate is DeferredTranslation:
-			return self.message
-		return translate(self.message)
-
-	def __lt__(self, other) -> bool:
-		return self.message < other
-
-	def __gt__(self, other) -> bool:
-		return self.message > other
-
-	def __add__(self, other) -> DeferredTranslation:
-		if isinstance(other, str):
-			other = DeferredTranslation(other)
-
-		concat = self.message + other.message
-		return DeferredTranslation(concat)
-
-	def format(self, *args) -> str:
-		return self.message.format(*args)
-
-	@classmethod
-	def install(cls):
-		import builtins
-		builtins._ = cls
-
-
-class Translation:
-	def __init__(self, locales_dir):
-		self._languages = {}
-
-		for names in self._get_translation_lang():
-			try:
-				self._languages[names[0]] = gettext.translation('base', localedir=locales_dir, languages=names)
-			except FileNotFoundError as error:
-				raise TranslationError(f"Could not locate language file for '{names}': {error}")
-
-	def activate(self, name):
-		if language := self._languages.get(name, None):
-			languages = LanguageDefinitions()
-
-			if languages.is_cyrillic(name):
-				self._set_font('UniCyr_8x16')
-			else:
-				# this will reset a possible previously set font to a default font
-				self._set_font('')
-
-			language.install()
-		else:
-			raise ValueError(f'Language not supported: {name}')
-
-	def _set_font(self, font: str):
-		from archinstall import SysCommand, log
-		try:
-			log(f'Setting new font: {font}', level=logging.DEBUG)
-			SysCommand(f'setfont {font}')
-		except Exception:
-			log(f'Unable to set font {font}', level=logging.ERROR)
-
-	@classmethod
-	def load_nationalization(cls) -> Translation:
-		locales_dir = cls.get_locales_dir()
-		return Translation(locales_dir)
-
-	@classmethod
-	def get_locales_dir(cls) -> Path:
-		cur_path = Path(__file__).parent.parent
-		locales_dir = Path.joinpath(cur_path, 'locales')
-		return locales_dir
-
-	@classmethod
-	def _defined_languages(cls) -> List[str]:
-		locales_dir = cls.get_locales_dir()
-		filenames = os.listdir(locales_dir)
-		return list(filter(lambda x: len(x) == 2, filenames))
-
-	@classmethod
-	def _get_translation_lang(cls) -> List[Tuple[str, str]]:
-		def_languages = cls._defined_languages()
-		languages = LanguageDefinitions()
-		return [(languages.get_language(lang), lang) for lang in def_languages]
-
-	@classmethod
-	def get_available_lang(cls) -> List[str]:
-		def_languages = cls._defined_languages()
-		languages = LanguageDefinitions()
-		return [languages.get_language(lang) for lang in def_languages]
diff --git a/archinstall/lib/translationhandler.py b/archinstall/lib/translationhandler.py
new file mode 100644
index 00000000..12c8da4a
--- /dev/null
+++ b/archinstall/lib/translationhandler.py
@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import gettext
+from dataclasses import dataclass
+
+from pathlib import Path
+from typing import List, Dict, Any, TYPE_CHECKING, Optional
+from .exceptions import TranslationError
+
+if TYPE_CHECKING:
+	_: Any
+
+
+@dataclass
+class Language:
+	abbr: str
+	lang: str
+	translation: gettext.NullTranslations
+	translation_percent: int
+	translated_lang: Optional[str]
+
+	@property
+	def display_name(self) -> str:
+		if self.translated_lang:
+			name = self.translated_lang
+		else:
+			name = self.lang
+		return f'{name} ({self.translation_percent}%)'
+
+	def is_match(self, lang_or_translated_lang: str) -> bool:
+		if self.lang == lang_or_translated_lang:
+			return True
+		elif self.translated_lang == lang_or_translated_lang:
+			return True
+		return False
+
+
+class TranslationHandler:
+	_base_pot = 'base.pot'
+	_languages = 'languages.json'
+
+	def __init__(self):
+		# to display cyrillic languages correctly
+		self._set_font('UniCyr_8x16')
+
+		self._total_messages = self._get_total_messages()
+		self._translated_languages = self._get_translations()
+
+	@property
+	def translated_languages(self) -> List[Language]:
+		return self._translated_languages
+
+	def _get_translations(self) -> List[Language]:
+		mappings = self._load_language_mappings()
+		defined_languages = self._defined_languages()
+
+		languages = []
+
+		for short_form in defined_languages:
+			mapping_entry: Dict[str, Any] = next(filter(lambda x: x['abbr'] == short_form, mappings))
+			abbr = mapping_entry['abbr']
+			lang = mapping_entry['lang']
+			translated_lang = mapping_entry.get('translated_lang', None)
+
+			try:
+				translation = gettext.translation('base', localedir=self._get_locales_dir(), languages=(abbr, lang))
+
+				if abbr == 'en':
+					percent = 100
+				else:
+					num_translations = self._get_catalog_size(translation)
+					percent = int((num_translations / self._total_messages) * 100)
+
+				language = Language(abbr, lang, translation, percent, translated_lang)
+				languages.append(language)
+			except FileNotFoundError as error:
+				raise TranslationError(f"Could not locate language file for '{lang}': {error}")
+
+		return languages
+
+	def _set_font(self, font: str):
+		from archinstall import SysCommand, log
+		try:
+			log(f'Setting font: {font}', level=logging.DEBUG)
+			SysCommand(f'setfont {font}')
+		except Exception:
+			log(f'Unable to set font {font}', level=logging.ERROR)
+
+	def _load_language_mappings(self) -> List[Dict[str, Any]]:
+		locales_dir = self._get_locales_dir()
+		languages = Path.joinpath(locales_dir, self._languages)
+
+		with open(languages, 'r') as fp:
+			return json.load(fp)
+
+	def _get_catalog_size(self, translation: gettext.NullTranslations) -> int:
+		# this is a ery naughty way of retrieving the data but
+		# there's no alternative method exposed unfortunately
+		catalog = translation._catalog  # type: ignore
+		messages = {k: v for k, v in catalog.items() if k and v}
+		return len(messages)
+
+	def _get_total_messages(self) -> int:
+		locales = self._get_locales_dir()
+		with open(f'{locales}/{self._base_pot}', 'r') as fp:
+			lines = fp.readlines()
+			msgid_lines = [line for line in lines if 'msgid' in line]
+		return len(msgid_lines) - 1  # don't count the first line which contains the metadata
+
+	def get_language(self, abbr: str) -> Language:
+		try:
+			return next(filter(lambda x: x.abbr == abbr, self._translated_languages))
+		except Exception:
+			raise ValueError(f'No language with abbreviation "{abbr}" found')
+
+	def activate(self, language: Language):
+		language.translation.install()
+
+	def _get_locales_dir(self) -> Path:
+		cur_path = Path(__file__).parent.parent
+		locales_dir = Path.joinpath(cur_path, 'locales')
+		return locales_dir
+
+	def _defined_languages(self) -> List[str]:
+		locales_dir = self._get_locales_dir()
+		filenames = os.listdir(locales_dir)
+		return list(filter(lambda x: len(x) == 2 or x == 'pt_BR', filenames))
+
+
+class DeferredTranslation:
+	def __init__(self, message: str):
+		self.message = message
+
+	def __len__(self) -> int:
+		return len(self.message)
+
+	def __str__(self) -> str:
+		translate = _
+		if translate is DeferredTranslation:
+			return self.message
+		return translate(self.message)
+
+	def __lt__(self, other) -> bool:
+		return self.message < other
+
+	def __gt__(self, other) -> bool:
+		return self.message > other
+
+	def __add__(self, other) -> DeferredTranslation:
+		if isinstance(other, str):
+			other = DeferredTranslation(other)
+
+		concat = self.message + other.message
+		return DeferredTranslation(concat)
+
+	def format(self, *args) -> str:
+		return self.message.format(*args)
+
+	@classmethod
+	def install(cls):
+		import builtins
+		builtins._ = cls
diff --git a/archinstall/lib/user_interaction/general_conf.py b/archinstall/lib/user_interaction/general_conf.py
index 15c42b86..bdc602b3 100644
--- a/archinstall/lib/user_interaction/general_conf.py
+++ b/archinstall/lib/user_interaction/general_conf.py
@@ -12,7 +12,7 @@ from ..output import log
 from ..profiles import Profile, list_profiles
 from ..mirrors import list_mirrors
 
-from ..translation import Translation
+from ..translationhandler import Language
 from ..packages.packages import validate_package_list
 
 from ..storage import storage
@@ -118,13 +118,22 @@ def select_mirror_regions(preset_values: Dict[str, Any] = {}) -> Dict[str, Any]:
 		case _: return {selected: mirrors[selected] for selected in selected_mirror.value}
 
 
-def select_archinstall_language(preset_values: str):
-	languages = Translation.get_available_lang()
-	choice = Menu(_('Archinstall language'), languages, default_option=preset_values).run()
+def select_archinstall_language(languages: List[Language], preset_value: Language) -> Language:
+	# these are the displayed language names which can either be
+	# the english name of a language or, if present, the
+	# name of the language in its own language
+	options = {lang.display_name: lang for lang in languages}
+
+	choice = Menu(
+		_('Archinstall language'),
+		list(options.keys()),
+		default_option=preset_value.display_name
+	).run()
 
 	match choice.type_:
-		case MenuSelectionType.Esc: return preset_values
-		case MenuSelectionType.Selection: return choice.value
+		case MenuSelectionType.Esc: return preset_value
+		case MenuSelectionType.Selection:
+			return options[choice.value]
 
 
 def select_profile(preset) -> Optional[Profile]:
diff --git a/archinstall/lib/user_interaction/manage_users_conf.py b/archinstall/lib/user_interaction/manage_users_conf.py
index a97328c2..84ce3556 100644
--- a/archinstall/lib/user_interaction/manage_users_conf.py
+++ b/archinstall/lib/user_interaction/manage_users_conf.py
@@ -57,10 +57,10 @@ class UserList(ListManager):
 			prompt = str(_('Password for user "{}": ').format(entry.username))
 			new_password = get_password(prompt=prompt)
 			if new_password:
-				user = next(filter(lambda x: x == entry, data), 1)
+				user = next(filter(lambda x: x == entry, data))
 				user.password = new_password
 		elif action == self._actions[2]:  # promote/demote
-			user = next(filter(lambda x: x == entry, data), 1)
+			user = next(filter(lambda x: x == entry, data))
 			user.sudo = False if user.sudo else True
 		elif action == self._actions[3]:  # delete
 			data = [d for d in data if d != entry]
diff --git a/archinstall/locales/README.md b/archinstall/locales/README.md
index 70822c05..51662702 100644
--- a/archinstall/locales/README.md
+++ b/archinstall/locales/README.md
@@ -5,7 +5,7 @@ Archinstall supports multiple languages, which depend on translations coming fro
 New languages can be added simply by creating a new folder with the proper language abbrevation (see list `languages.json` if unsure).  
 Run the following command to create a new template for a language
 ```
-    mkdir -p <abbr>/LC_MESSAGES/ && touch <abbr>/LC_MESSAGES/base.po
+mkdir -p <abbr>/LC_MESSAGES/ && touch <abbr>/LC_MESSAGES/base.po
 ```
 
 After that run the script `./locales_generator.sh` it will automatically populate the new `base.po` file with the strings that 
@@ -31,3 +31,10 @@ msgstr "Wollen sie wirklich abbrechen?"
 
 After the translations have been written, run the script once more `./locales_generator.sh` and it will auto-generate the `base.mo` file with the included translations.
 After that you're all ready to go and enjoy Archinstall in the new language :)
+
+To display the language inside Archinstall in your own tongue, please edit the file `languages.json` and 
+add a `translated_lang` entry to the respective language, e.g. 
+
+```
+ {"abbr": "pl", "lang": "Polish", "translated_lang": "Polskie"}
+```
diff --git a/archinstall/locales/cyrillic.json b/archinstall/locales/cyrillic.json
deleted file mode 100644
index 13f11ad0..00000000
--- a/archinstall/locales/cyrillic.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
-  "languages": [
-    "Abkhazian",
-    "Azerbaijani",
-    "Bashkir",
-    "Belarusian",
-    "Bulgarian",
-    "Chuvash",
-    "Komi",
-    "Macedonian",
-    "Mongolian",
-    "Russian",
-    "Serbo-Croatian",
-    "Tajik",
-    "Tatar",
-    "Ukrainian",
-    "Uzbek"
-  ]
-}
diff --git a/archinstall/locales/languages.json b/archinstall/locales/languages.json
index c3c9d2c2..883042e8 100644
--- a/archinstall/locales/languages.json
+++ b/archinstall/locales/languages.json
@@ -20,7 +20,7 @@
  {"abbr": "br", "lang": "Breton"},
  {"abbr": "bg", "lang": "Bulgarian"},
  {"abbr": "ca", "lang": "Catalan"},
- {"abbr": "cs", "lang": "Czech"},
+ {"abbr": "cs", "lang": "Czech", "translated_lang": "čeština"},
  {"abbr": "ch", "lang": "Chamorro"},
  {"abbr": "ce", "lang": "Chechen"},
  {"abbr": "cu", "lang": "Church Slavic"},
@@ -29,8 +29,8 @@
  {"abbr": "co", "lang": "Corsican"},
  {"abbr": "cr", "lang": "Cree"},
  {"abbr": "cy", "lang": "Welsh"},
- {"abbr": "da", "lang": "Danish"},
- {"abbr": "de", "lang": "German"},
+ {"abbr": "da", "lang": "Danish", "translated_lang": "Dansk"},
+ {"abbr": "de", "lang": "German", "translated_lang":  "Deutsch"},
  {"abbr": "dv", "lang": "Dhivehi"},
  {"abbr": "dz", "lang": "Dzongkha"},
  {"abbr": "el", "lang": "Modern Greek (1453-)"},
@@ -43,7 +43,7 @@
  {"abbr": "fa", "lang": "Persian"},
  {"abbr": "fj", "lang": "Fijian"},
  {"abbr": "fi", "lang": "Finnish"},
- {"abbr": "fr", "lang": "French"},
+ {"abbr": "fr", "lang": "French", "translated_lang": "Française"},
  {"abbr": "fy", "lang": "Western Frisian"},
  {"abbr": "ff", "lang": "Fulah"},
  {"abbr": "gd", "lang": "Scottish Gaelic"},
@@ -71,7 +71,7 @@
  {"abbr": "id", "lang": "Indonesian"},
  {"abbr": "ik", "lang": "Inupiaq"},
  {"abbr": "is", "lang": "Icelandic"},
- {"abbr": "it", "lang": "Italian"},
+ {"abbr": "it", "lang": "Italian", "translated_lang":  "Italiano"},
  {"abbr": "jv", "lang": "Javanese"},
  {"abbr": "ja", "lang": "Japanese"},
  {"abbr": "kl", "lang": "Kalaallisut"},
@@ -114,7 +114,7 @@
  {"abbr": "nd", "lang": "North Ndebele"},
  {"abbr": "ng", "lang": "Ndonga"},
  {"abbr": "ne", "lang": "Nepali (macrolanguage)"},
- {"abbr": "nl", "lang": "Dutch"},
+ {"abbr": "nl", "lang": "Dutch", "translated_lang": "Nederlands"},
  {"abbr": "nn", "lang": "Norwegian Nynorsk"},
  {"abbr": "nb", "lang": "Norwegian Bokmål"},
  {"abbr": "no", "lang": "Norwegian"},
@@ -126,15 +126,15 @@
  {"abbr": "os", "lang": "Ossetian"},
  {"abbr": "pa", "lang": "Panjabi"},
  {"abbr": "pi", "lang": "Pali"},
- {"abbr": "pl", "lang": "Polish"},
- {"abbr": "pt", "lang": "Portuguese"},
- {"abbr": "pt_BR", "lang": "Brazilian Portuguese"},
+ {"abbr": "pl", "lang": "Polish", "translated_lang": "Polskie"},
+ {"abbr": "pt", "lang": "Portuguese", "translated_lang": "Português"},
+ {"abbr": "pt_BR", "lang": "Brazilian Portuguese", "translated_lang": "Portugues do Brasil"},
  {"abbr": "ps", "lang": "Pushto"},
  {"abbr": "qu", "lang": "Quechua"},
  {"abbr": "rm", "lang": "Romansh"},
  {"abbr": "ro", "lang": "Romanian"},
  {"abbr": "rn", "lang": "Rundi"},
- {"abbr": "ru", "lang": "Russian"},
+ {"abbr": "ru", "lang": "Russian", "translated_lang":  "русский"},
  {"abbr": "sg", "lang": "Sango"},
  {"abbr": "sa", "lang": "Sanskrit"},
  {"abbr": "si", "lang": "Sinhala"},
@@ -146,14 +146,14 @@
  {"abbr": "sd", "lang": "Sindhi"},
  {"abbr": "so", "lang": "Somali"},
  {"abbr": "st", "lang": "Southern Sotho"},
- {"abbr": "es", "lang": "Spanish"},
+ {"abbr": "es", "lang": "Spanish", "translated_lang": "Española"},
  {"abbr": "sq", "lang": "Albanian"},
  {"abbr": "sc", "lang": "Sardinian"},
  {"abbr": "sr", "lang": "Serbian"},
  {"abbr": "ss", "lang": "Swati"},
  {"abbr": "su", "lang": "Sundanese"},
  {"abbr": "sw", "lang": "Swahili (macrolanguage)"},
- {"abbr": "sv", "lang": "Swedish"},
+ {"abbr": "sv", "lang": "Swedish", "translated_lang":  "Svenska"},
  {"abbr": "ty", "lang": "Tahitian"},
  {"abbr": "ta", "lang": "Tamil"},
  {"abbr": "tt", "lang": "Tatar"},
@@ -166,11 +166,11 @@
  {"abbr": "tn", "lang": "Tswana"},
  {"abbr": "ts", "lang": "Tsonga"},
  {"abbr": "tk", "lang": "Turkmen"},
- {"abbr": "tr", "lang": "Turkish"},
+ {"abbr": "tr", "lang": "Turkish", "translated_lang" :  "Türk"},
  {"abbr": "tw", "lang": "Twi"},
  {"abbr": "ug", "lang": "Uighur"},
  {"abbr": "uk", "lang": "Ukrainian"},
- {"abbr": "ur", "lang": "Urdu"},
+ {"abbr": "ur", "lang": "Urdu", "translated_lang": "اردو"},
  {"abbr": "uz", "lang": "Uzbek"},
  {"abbr": "ve", "lang": "Venda"},
  {"abbr": "vi", "lang": "Vietnamese"},