Merging in latest changes from master.

12 files changed, 269 insertions, 47 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 00000000..ca54c552
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,92 @@
+# This file contains GitLab CI/CD configuration for the ArchInstall project.
+# It defines several jobs that get run when a new commit is made, and is comparable to the GitHub workflows.
+# There is an expectation that a runner exists that has the --privileged flag enabled for the build ISO job to run correctly.
+# These jobs should leverage the same tag as that runner. If necessary, change the tag from 'docker' to the one it uses.
+# All jobs will be run in the official archlinux container image, so we will declare that here.
+
+image: archlinux:latest
+
+# This can be used to handle common actions. In this case, we do a pacman -Sy to make sure repos are ready to use.
+before_script:
+  - pacman -Sy
+
+stages:
+  - lint
+  - test
+  - build
+  - publish
+
+mypy:
+  stage: lint
+  tags:
+    - docker
+  script:
+    - pacman --noconfirm -Syu python mypy
+    - mypy . --ignore-missing-imports || exit 0
+
+flake8:
+  stage: lint
+  tags:
+    - docker
+  script:
+    - pacman --noconfirm -Syu python python-pip
+    - python -m pip install --upgrade pip
+    - pip install flake8
+    - flake8 . --count --select=E9,F63,F7 --show-source --statistics
+    - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+# We currently do not have unit tests implemented but this stage is written in anticipation of their future usage.
+# When a stage name is preceeded with a '.' it's treated as "disabled" by GitLab and is not executed, so it's fine for it to be declared.
+.pytest:
+  stage: test
+  tags:
+    - docker
+  script:
+    - pacman --noconfirm -Syu python python-pip
+    - python -m pip install --upgrade pip
+    - pip install pytest
+    - pytest
+
+# This stage might fail with exit code 137 on a shared runner. This is probably due to the CPU/memory consumption needed to run the build.
+build_iso:
+  stage: build
+  tags:
+    - docker
+  script:
+    - pwd
+    - find .
+    - cat /etc/os-release
+    - mkdir -p /tmp/archlive/airootfs/root/archinstall-git; cp -r . /tmp/archlive/airootfs/root/archinstall-git
+    - echo "pip uninstall archinstall -y; cd archinstall-git; python setup.py install" > /tmp/archlive/airootfs/root/.zprofile
+    - echo "echo \"This is an unofficial ISO for development and testing of archinstall. No support will be provided.\"" >> /tmp/archlive/airootfs/root/.zprofile
+    - echo "echo \"This ISO was built from Git SHA $CI_COMMIT_SHA\"" >> /tmp/archlive/airootfs/root/.zprofile
+    - echo "echo \"Type archinstall to launch the installer.\"" >> /tmp/archlive/airootfs/root/.zprofile
+    - cat /tmp/archlive/airootfs/root/.zprofile
+    - pacman --noconfirm -S git archiso
+    - cp -r /usr/share/archiso/configs/releng/* /tmp/archlive
+    - echo -e "git\npython\npython-pip\npython-setuptools" >> /tmp/archlive/packages.x86_64
+    - find /tmp/archlive
+    - cd /tmp/archlive; mkarchiso -v -w work/ -o out/ ./
+  artifacts:
+    name: "Arch Live ISO"
+    paths:
+      - /tmp/archlive/out/*.iso
+    expire_in: 1 week
+
+## This job only runs when a tag is created on the master branch. This is because we do not want to try to publish to PyPi every time we commit.
+## The following CI/CD variables need to be set to the PyPi username and password in the GitLab project's settings for this stage to work.
+# * FLIT_USERNAME
+# * FLIT_PASSWORD
+publish_pypi:
+  stage: publish
+  tags:
+    - docker
+  script:
+    - pacman --noconfirm -S python python-pip
+    - python -m pip install --upgrade pip
+    - pip install setuptools wheel flit
+    - flit
+  only:
+    - tags
+  except:
+    - branches
diff --git a/PKGBUILD b/PKGBUILD
index 7a5da658..cc7a669f 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -10,16 +10,18 @@ arch=('any')
 url="https://github.com/archlinux/archinstall"
 license=('GPL')
 depends=('python')
-makedepends=('python-setuptools')
+makedepends=('python-setuptools' 'python-sphinx')
 provides=('python-archinstall')
 conflicts=('archinstall' 'python-archinstall' 'python-archinstall-git')
 
 build() {
         cd "$startdir"
         python setup.py build
+        make man -C docs
 }
 
 package() {
         cd "$startdir"
         python setup.py install --root="${pkgdir}" --optimize=1 --skip-build
+        install -vDm 644 docs/_build/man/archinstall.1 -t "${pkgdir}/usr/share/man/man1/"
 }
diff --git a/archinstall/__init__.py b/archinstall/__init__.py
index 0b799d5b..84595528 100644
--- a/archinstall/__init__.py
+++ b/archinstall/__init__.py
@@ -30,8 +30,11 @@ storage['__version__'] = __version__
 def initialize_arguments():
 	config = {}
 	parser.add_argument("--config", nargs="?", help="JSON configuration file or URL")
+	parser.add_argument("--creds", nargs="?", help="JSON credentials configuration file")
 	parser.add_argument("--silent", action="store_true",
-						help="Warning!!! No prompts, ignored if config is not passed")
+						help="WARNING: Disables all prompts for input and confirmation. If no configuration is provided, this is ignored")
+	parser.add_argument("--dry-run", action="store_true",
+						help="Generates a configuration file and then exits instead of performing an installation")
 	parser.add_argument("--script", default="guided", nargs="?", help="Script to run for installation", type=str)
 	args, unknowns = parser.parse_known_args()
 	if args.config is not None:
@@ -47,6 +50,9 @@ def initialize_arguments():
 					config = json.loads(response.read())
 		except Exception as e:
 			print(e)
+		if args.creds is not None:
+			with open(args.creds) as file:
+				config.update(json.load(file))
 		# Installation can't be silent if config is not passed
 		config["silent"] = args.silent
 	for arg in unknowns:
@@ -57,6 +63,8 @@ def initialize_arguments():
 				key, val = arg[2:], True
 			config[key] = val
 	config["script"] = args.script
+	if args.dry_run is not None:
+		config["dry_run"] = args.dry_run
 	return config
 
 
diff --git a/archinstall/lib/disk.py b/archinstall/lib/disk.py
index efcf1844..e0d9b423 100644
--- a/archinstall/lib/disk.py
+++ b/archinstall/lib/disk.py
@@ -539,16 +539,16 @@ class Filesystem:
 		if self.blockdevice.keep_partitions is False:
 			log(f'Wiping {self.blockdevice} by using partition format {self.mode}', level=logging.DEBUG)
 			if self.mode == GPT:
-				if self.raw_parted(f'{self.blockdevice.device} mklabel gpt').exit_code == 0:
+				if self.parted_mklabel(self.blockdevice.device, "gpt"):
 					self.blockdevice.flush_cache()
 					return self
 				else:
-					raise DiskError('Problem setting the partition format to GPT:', f'/usr/bin/parted -s {self.blockdevice.device} mklabel gpt')
+					raise DiskError('Problem setting the disk label type to GPT:', f'/usr/bin/parted -s {self.blockdevice.device} mklabel gpt')
 			elif self.mode == MBR:
-				if SysCommand(f"/usr/bin/parted -s {self.blockdevice.device} mklabel msdos").exit_code == 0:
+				if self.parted_mklabel(self.blockdevice.device, "msdos"):
 					return self
 				else:
-					raise DiskError('Problem setting the partition format to MBR:', f'/usr/bin/parted -s {self.blockdevice.device} mklabel msdos')
+					raise DiskError('Problem setting the disk label type to msdos:', f'/usr/bin/parted -s {self.blockdevice.device} mklabel msdos')
 			else:
 				raise DiskError(f'Unknown mode selected to format in: {self.mode}')
 
@@ -655,6 +655,14 @@ class Filesystem:
 	def set(self, partition: int, string: str):
 		return self.parted(f'{self.blockdevice.device} set {partition + 1} {string}') == 0
 
+	def parted_mklabel(self, device: str, disk_label: str):
+		# Try to unmount devices before attempting to run mklabel
+		try:
+			SysCommand(f'bash -c "umount {device}?"')
+		except:
+			pass
+		return self.raw_parted(f'{device} mklabel {disk_label}').exit_code == 0
+
 
 def device_state(name, *args, **kwargs):
 	# Based out of: https://askubuntu.com/questions/528690/how-to-get-list-of-all-non-removable-disk-device-names-ssd-hdd-and-sata-ide-onl/528709#528709
diff --git a/archinstall/lib/general.py b/archinstall/lib/general.py
index 6bb3b101..4d3257ba 100644
--- a/archinstall/lib/general.py
+++ b/archinstall/lib/general.py
@@ -2,14 +2,41 @@ import hashlib
 import json
 import logging
 import os
-import pty
 import shlex
 import subprocess
 import sys
 import time
 from datetime import datetime, date
-from select import epoll, EPOLLIN, EPOLLHUP
 from typing import Union
+try:
+	from select import epoll, EPOLLIN, EPOLLHUP
+except:
+	import select
+	EPOLLIN = 0
+	EPOLLHUP = 0
+	class epoll():
+		""" #!if windows
+		Create a epoll() implementation that simulates the epoll() behavior.
+		This so that the rest of the code doesn't need to worry weither we're using select() or epoll().
+		"""
+		def __init__(self):
+			self.sockets = {}
+			self.monitoring = {}
+
+		def unregister(self, fileno, *args, **kwargs):
+			try:
+				del(self.monitoring[fileno])
+			except:
+				pass
+
+		def register(self, fileno, *args, **kwargs):
+			self.monitoring[fileno] = True
+
+		def poll(self, timeout=0.05, *args, **kwargs):
+			try:
+				return [[fileno, 1] for fileno in select.select(list(self.monitoring.keys()), [], [], timeout)[0]]
+			except OSError:
+				return []
 
 from .exceptions import *
 from .output import log
@@ -203,27 +230,6 @@ class SysCommandWorker:
 				except UnicodeDecodeError:
 					return False
 
-			output = output.strip('\r\n ')
-			if len(output) <= 0:
-				return False
-
-			from .user_interaction import get_terminal_width
-
-			# Move back to the beginning of the terminal
-			sys.stdout.flush()
-			sys.stdout.write("\033[%dG" % 0)
-			sys.stdout.flush()
-
-			# Clear the line
-			sys.stdout.write(" " * get_terminal_width())
-			sys.stdout.flush()
-
-			# Move back to the beginning again
-			sys.stdout.flush()
-			sys.stdout.write("\033[%dG" % 0)
-			sys.stdout.flush()
-
-			# And print the new output we're peaking on:
 			sys.stdout.write(output)
 			sys.stdout.flush()
 		return True
@@ -253,6 +259,8 @@ class SysCommandWorker:
 					self.exit_code = 1
 
 	def execute(self) -> bool:
+		import pty
+
 		if (old_dir := os.getcwd()) != self.working_directory:
 			os.chdir(self.working_directory)
 
diff --git a/archinstall/lib/hardware.py b/archinstall/lib/hardware.py
index 45e042db..a63155f5 100644
--- a/archinstall/lib/hardware.py
+++ b/archinstall/lib/hardware.py
@@ -48,10 +48,12 @@ AVAILABLE_GFX_DRIVERS = {
 		"intel-media-driver",
 		"vulkan-intel",
 	],
-	"Nvidia": {
-		"open-source": ["mesa", "xf86-video-nouveau", "libva-mesa-driver"],
-		"proprietary": ["nvidia"],
-	},
+	"Nvidia (open-source)": [
+		"mesa",
+		"xf86-video-nouveau",
+		"libva-mesa-driver"
+	],
+	"Nvidia (proprietary)": ["nvidia"],
 	"VMware / VirtualBox (open-source)": ["mesa", "xf86-video-vmware"],
 }
 
diff --git a/archinstall/lib/mirrors.py b/archinstall/lib/mirrors.py
index ccfc2808..95fb5ac6 100644
--- a/archinstall/lib/mirrors.py
+++ b/archinstall/lib/mirrors.py
@@ -1,11 +1,57 @@
 import urllib.error
 import urllib.request
+from typing import Union
 
 from .general import *
 from .output import log
 
+def sort_mirrorlist(raw_data :bytes, sort_order=["https", "http"]) -> bytes:
+	"""
+	This function can sort /etc/pacman.d/mirrorlist according to the
+	mirror's URL prefix. By default places HTTPS before HTTP but it also
+	preserves the country/rank-order.
+
+	This assumes /etc/pacman.d/mirrorlist looks like the following:
+
+	## Comment
+	Server = url
+
+	or
+
+	## Comment
+	#Server = url
+
+	But the Comments need to start with double-hashmarks to be distringuished
+	from server url definitions (commented or uncommented).
+	"""
+	comments_and_whitespaces = b""
+
+	categories = {key: [] for key in sort_order+["Unknown"]}
+	for line in raw_data.split(b"\n"):
+		if line[0:2] in (b'##', b''):
+			comments_and_whitespaces += line + b'\n'
+		elif line[:6].lower() == b'server' or line[:7].lower() == b'#server':
+			opening, url = line.split(b'=', 1)
+			opening, url = opening.strip(), url.strip()
+			if (category := url.split(b'://',1)[0].decode('UTF-8')) in categories:
+				categories[category].append(comments_and_whitespaces)
+				categories[category].append(opening+b' = '+url+b'\n')
+			else:
+				categories["Unknown"].append(comments_and_whitespaces)
+				categories["Unknown"].append(opening+b' = '+url+b'\n')
+
+			comments_and_whitespaces = b""
 
-def filter_mirrors_by_region(regions, destination='/etc/pacman.d/mirrorlist', *args, **kwargs):
+
+	new_raw_data = b''
+	for category in sort_order+["Unknown"]:
+		for line in categories[category]:
+			new_raw_data += line
+
+	return new_raw_data
+
+
+def filter_mirrors_by_region(regions, destination='/etc/pacman.d/mirrorlist', sort_order=["https", "http"], *args, **kwargs) -> Union[bool, bytes]:
 	"""
 	This function will change the active mirrors on the live medium by
 	filtering which regions are active based on `regions`.
@@ -16,12 +62,19 @@ def filter_mirrors_by_region(regions, destination='/etc/pacman.d/mirrorlist', *a
 	region_list = []
 	for region in regions.split(','):
 		region_list.append(f'country={region}')
-	response = urllib.request.urlopen(urllib.request.Request(f"https://archlinux.org/mirrorlist/?{'&'.join(region_list)}&protocol=https&ip_version=4&ip_version=6&use_mirror_status=on'", headers={'User-Agent': 'ArchInstall'}))
+	response = urllib.request.urlopen(urllib.request.Request(f"https://archlinux.org/mirrorlist/?{'&'.join(region_list)}&protocol=https&protocol=http&ip_version=4&ip_version=6&use_mirror_status=on'", headers={'User-Agent': 'ArchInstall'}))
 	new_list = response.read().replace(b"#Server", b"Server")
-	with open(destination, "wb") as mirrorlist:
-		mirrorlist.write(new_list)
 
-	return True
+	if sort_order:
+		new_list = sort_mirrorlist(new_list, sort_order=sort_order)
+
+	if destination:
+		with open(destination, "wb") as mirrorlist:
+			mirrorlist.write(new_list)
+
+		return True
+	else:
+		return new_list.decode('UTF-8')
 
 
 def add_custom_mirrors(mirrors: list, *args, **kwargs):
@@ -78,8 +131,8 @@ def re_rank_mirrors(top=10, *positionals, **kwargs):
 	return False
 
 
-def list_mirrors():
-	url = "https://archlinux.org/mirrorlist/?protocol=https&ip_version=4&ip_version=6&use_mirror_status=on"
+def list_mirrors(sort_order=["https", "http"]):
+	url = "https://archlinux.org/mirrorlist/?protocol=https&protocol=http&ip_version=4&ip_version=6&use_mirror_status=on"
 	regions = {}
 
 	try:
@@ -88,8 +141,12 @@ def list_mirrors():
 		log(f'Could not fetch an active mirror-list: {err}', level=logging.WARNING, fg="yellow")
 		return regions
 
+	mirrorlist = response.read()
+	if sort_order:
+		mirrorlist = sort_mirrorlist(mirrorlist, sort_order=sort_order)
+
 	region = 'Unknown region'
-	for line in response.readlines():
+	for line in mirrorlist.split(b'\n'):
 		if len(line.strip()) == 0:
 			continue
 
diff --git a/archinstall/lib/networking.py b/archinstall/lib/networking.py
index 0643c9cf..64bcb58e 100644
--- a/archinstall/lib/networking.py
+++ b/archinstall/lib/networking.py
@@ -1,4 +1,3 @@
-import fcntl
 import logging
 import os
 import socket
@@ -12,6 +11,7 @@ from .storage import storage
 
 
 def get_hw_addr(ifname):
+	import fcntl
 	s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
 	info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', bytes(ifname, 'utf-8')[:15]))
 	return ':'.join('%02x' % b for b in info[18:24])
@@ -31,7 +31,7 @@ def list_interfaces(skip_loopback=True):
 def check_mirror_reachable():
 	if (exit_code := SysCommand("pacman -Sy").exit_code) == 0:
 		return True
-	elif exit_code == 256:
+	elif os.geteuid() != 0:
 		log("check_mirror_reachable() uses 'pacman -Sy' which requires root.", level=logging.ERROR, fg="red")
 
 	return False
diff --git a/archinstall/lib/user_interaction.py b/archinstall/lib/user_interaction.py
index 50f3be9a..4f99820d 100644
--- a/archinstall/lib/user_interaction.py
+++ b/archinstall/lib/user_interaction.py
@@ -7,9 +7,7 @@ import select  # Used for char by char polling of sys.stdin
 import shutil
 import signal
 import sys
-import termios
 import time
-import tty
 
 from .disk import BlockDevice
 from .exceptions import *
@@ -285,6 +283,8 @@ class MiniCurses:
 
 	def get_keyboard_input(self, strip_rowbreaks=True, end='\n'):
 		assert end in ['\r', '\n', None]
+		import termios
+		import tty
 
 		poller = select.epoll()
 		response = ''
diff --git a/docs/conf.py b/docs/conf.py
index 375ff434..add1c5e7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -41,7 +41,7 @@ copyright = '2020, Anton Hvornum'
 author = 'Anton Hvornum'
 
 # The full version, including alpha/beta/rc tags
-release = 'v2.1.0'
+release = 'v2.3.0.dev0'
 
 # -- General configuration ---------------------------------------------------
 
diff --git a/examples/guided.py b/examples/guided.py
index 57d4818b..2eb5fede 100644
--- a/examples/guided.py
+++ b/examples/guided.py
@@ -195,7 +195,10 @@ def perform_filesystem_operations():
 	with open("/var/log/archinstall/user_configuration.json", "w") as config_file:
 		config_file.write(user_configuration)
 	print()
-
+	
+	if archinstall.arguments.get('dry_run'):
+		exit(0)
+	
 	if not archinstall.arguments.get('silent'):
 		input('Press Enter to continue.')
 
diff --git a/profiles/cutefish.py b/profiles/cutefish.py
new file mode 100644
index 00000000..1df2467a
--- /dev/null
+++ b/profiles/cutefish.py
@@ -0,0 +1,42 @@
+# A desktop environment using "Cutefish"
+
+import archinstall
+
+is_top_level_profile = False
+
+__packages__ = [
+	"cutefish",
+	"noto-fonts",
+	"konsole",
+	"sddm"
+]
+
+
+def _prep_function(*args, **kwargs):
+	"""
+	Magic function called by the importing installer
+	before continuing any further. It also avoids executing any
+	other code in this stage. So it's a safe way to ask the user
+	for more input before any other installer steps start.
+	"""
+
+	# Cutefish requires a functional xorg installation.
+	profile = archinstall.Profile(None, "xorg")
+	with profile.load_instructions(namespace="xorg.py") as imported:
+		if hasattr(imported, "_prep_function"):
+			return imported._prep_function()
+		else:
+			print("Deprecated (??): xorg profile has no _prep_function() anymore")
+
+
+# Ensures that this code only gets executed if executed
+# through importlib.util.spec_from_file_location("cutefish", "/somewhere/cutefish.py")
+# or through conventional import cutefish
+if __name__ == "cutefish":
+	# Install dependency profiles
+	archinstall.storage["installation_session"].install_profile("xorg")
+
+	# Install the Cutefish packages
+	archinstall.storage["installation_session"].add_additional_packages(__packages__)
+
+	archinstall.storage["installation_session"].enable_service("sddm")