Btrfs2 (#787)

* All the changes needed to make btrfs subvolumes work. It boils down to two points;
the handling of the addressing of subvolumes re. physical partitions, and the small changes at the bootloader level
* We added a new script only_hd for testing purposes. It only handles hadrd drive management
* restoring an escape hatch during subvolume processing
* hipercommented manage_btrfs_subvolumes
* Ready to be able to select and process options in subvolume mounting
* Separte nodatacow processing
* Solving a flake8 complain
* Use of bind names @ get_filesystem_type
* compress mount option bypass
* Preparations for encryption handling
* Compatibility to master version re. encrypted btrfs volumes
* Now we can create subvolumes and mountpoints inside an encrypted btrfs partition
* changes for entries file generation with systemd-bootctl
* flake8 corrections plus some comments

Co-authored-by: Anton Hvornum <anton@hvornum.se>

4 files changed, 168 insertions, 21 deletions

diff --git a/archinstall/lib/disk/btrfs.py b/archinstall/lib/disk/btrfs.py
index 7ae4f6a6..fb9712f8 100644
--- a/archinstall/lib/disk/btrfs.py
+++ b/archinstall/lib/disk/btrfs.py
@@ -6,6 +6,8 @@ from .helpers import get_mount_info
 from ..exceptions import DiskError
 from ..general import SysCommand
 from ..output import log
+from .partition import Partition
+
 
 def mount_subvolume(installation, subvolume_location :Union[pathlib.Path, str], force=False) -> bool:
 	"""
@@ -72,3 +74,105 @@ def create_subvolume(installation, subvolume_location :Union[pathlib.Path, str])
 	log(f"Creating a subvolume on {target}", level=logging.INFO)
 	if (cmd := SysCommand(f"btrfs subvolume create {target}")).exit_code != 0:
 		raise DiskError(f"Could not create a subvolume at {target}: {cmd}")
+
+def manage_btrfs_subvolumes(installation, partition :dict, mountpoints :dict, subvolumes :dict, unlocked_device :dict = None):
+	""" we do the magic with subvolumes in a centralized place
+	parameters:
+	* the installation object
+	* the partition dictionary entry which represents the physical partition
+	* mountpoinst, the dictionary which contains all the partititon to be mounted
+	* subvolumes is the dictionary with the names of the subvolumes and its location
+	We expect the partition has been mounted as / , and it to be unmounted after the processing
+	Then we create all the subvolumes inside btrfs as demand
+	We clone then, both the partition dictionary and the object inside it and adapt it to the subvolume needs
+	Then we add it them to the mountpoints dictionary to be processed as "normal" partitions
+	# TODO For encrypted devices we need some special processing prior to it
+	"""
+	# We process each of the pairs <subvolume name: mount point | None | mount info dict>
+	# th mount info dict has an entry for the path of the mountpoint (named 'mountpoint') and 'options' which is a list
+	# of mount options (or similar used by brtfs)
+	for name, right_hand in subvolumes.items():
+		try:
+			# we normalize the subvolume name (getting rid of slash at the start if exists. In our implemenation has no semantic load - every subvolume is created from the top of the hierarchy- and simplifies its further use
+			if name.startswith('/'):
+				name = name[1:]
+			# renormalize the right hand.
+			location = None
+			mount_options = []
+			# no contents, so it is not to be mounted
+			if not right_hand:
+				location = None
+			# just a string. per backward compatibility the mount point
+			elif isinstance(right_hand,str):
+				location = right_hand
+			# a dict. two elements 'mountpoint' (obvious) and and a mount options list ¿?
+			elif isinstance(right_hand,dict):
+				location = right_hand.get('mountpoint',None)
+				mount_options = right_hand.get('options',[])
+			# we create the subvolume
+			create_subvolume(installation,name)
+			# Make the nodatacow processing now
+			# It will be the main cause of creation of subvolumes which are not to be mounted
+			# it is not an options which can be established by subvolume (but for whole file systems), and can be
+			# set up via a simple attribute change in a directory (if empty). And here the directories are brand new
+			if 'nodatacow' in mount_options:
+				if (cmd := SysCommand(f"chattr +C {installation.target}/{name}")).exit_code != 0:
+					raise DiskError(f"Could not set  nodatacow attribute at {installation.target}/{name}: {cmd}")
+				# entry is deleted so nodatacow doesn't propagate to the mount options
+				del mount_options[mount_options.index('nodatacow')]
+			# Make the compress processing now
+			# it is not an options which can be established by subvolume (but for whole file systems), and can be
+			# set up via a simple attribute change in a directory (if empty). And here the directories are brand new
+			# in this way only zstd compression is activaded
+			# TODO WARNING it is not clear if it should be a standard feature, so it might need to be deactivated
+			if 'compress' in mount_options:
+				if (cmd := SysCommand(f"chattr +c {installation.target}/{name}")).exit_code != 0:
+					raise DiskError(f"Could not set compress attribute at {installation.target}/{name}: {cmd}")
+				# entry is deleted so nodatacow doesn't propagate to the mount options
+				del mount_options[mount_options.index('compress')]
+			# END compress processing.
+			# we do not mount if THE basic partition will be mounted or if we exclude explicitly this subvolume
+			if not partition['mountpoint'] and location is not None:
+				# we begin to create a fake partition entry. First we copy the original -the one that corresponds to
+				# the primary partition
+				fake_partition = partition.copy()
+				# we start to modify entries in the "fake partition" to match the needs of the subvolumes
+				#
+				# to avoid any chance of entering in a loop (not expected) we delete the list of subvolumes in the copy
+				# and reset the encryption parameters
+				del fake_partition['btrfs']
+				fake_partition['encrypted'] = False
+				fake_partition['generate-encryption-key-file'] = False
+				# Mount destination. As of now the right hand part
+				fake_partition['mountpoint'] = location
+				# we load the name in an attribute called subvolume, but i think it is not needed anymore, 'cause the mount logic uses a different path.
+				fake_partition['subvolume'] = name
+				# here we add the mount options
+				fake_partition['options'] = mount_options
+				# Here comes the most exotic part. The dictionary attribute 'device_instance' contains an instance of Partition. This instance will be queried along the mount process at the installer.
+				# We instanciate a new object with following attributes coming / adapted from the instance which was in the primary partition entry (the one we are coping - partition['device_instance']
+				# * path, which will be expanded with the subvolume name to use the bind mount syntax the system uses for naming mounted subvolumes
+				# * size. When the OS queries all the subvolumes share the same size as the full partititon
+				# * uuid. All the subvolumes on a partition share the same uuid
+				if not unlocked_device:
+					fake_partition['device_instance'] = Partition(f"{partition['device_instance'].path}[/{name}]",partition['device_instance'].size,partition['device_instance'].uuid)
+				else:
+					# for subvolumes IN an encrypted partition we make our device instance from unlocked device instead of the raw partition.
+					# This time we make a copy (we should to the same above TODO) and alter the path by hand
+					from copy import copy
+					# KIDS DONT'T DO THIS AT HOME
+					fake_partition['device_instance'] = copy(unlocked_device)
+					fake_partition['device_instance'].path = f"{unlocked_device.path}[/{name}]"
+				# we reset this attribute, which holds where the partition is actually mounted. Remember, the physical partition is mounted at this moment and therefore has the value '/'.
+				# If i don't reset it, process will abort as "already mounted' .
+				# TODO It works for this purpose, but the fact that this bevahiour can happed, should make think twice
+				fake_partition['device_instance'].mountpoint = None
+				#
+				# Well, now that this "fake partition" is ready, we add it to the list of the ones which are to be mounted,
+				# as "normal" ones
+				mountpoints[fake_partition['mountpoint']] = fake_partition
+		except Exception as e:
+			raise e
+	# if the physical partition has been selected to be mounted, we include it at the list. Remmeber, all the above treatement won't happen except the creation of the subvolume
+	if partition['mountpoint']:
+		mountpoints[partition['mountpoint']] = partition
diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py
index 72be7e70..51ef949b 100644
--- a/archinstall/lib/disk/filesystem.py
+++ b/archinstall/lib/disk/filesystem.py
@@ -37,10 +37,10 @@ class Filesystem:
 		for i in range(storage['DISK_RETRY_ATTEMPTS']):
 			self.partprobe()
 			time.sleep(5)
-			
+
 			# TODO: Convert to blkid (or something similar, but blkid doesn't support traversing to list sub-PARTUUIDs based on blockdevice path?)
 			output = json.loads(SysCommand(f"lsblk --json -o+PARTUUID {self.blockdevice.device}").decode('UTF-8'))
-			
+
 			for device in output['blockdevices']:
 				for index, partition in enumerate(device['children']):
 					if (partuuid := partition.get('partuuid', None)) and partuuid.lower() == uuid:
@@ -93,8 +93,12 @@ class Filesystem:
 							storage['arguments']['!encryption-password'] = get_password(f"Enter a encryption password for {partition['device_instance']}")
 
 						partition['!password'] = storage['arguments']['!encryption-password']
-
-					loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['mountpoint']).name}loop"
+					# to be able to generate an unique name in case the partition will not be mounted
+					if partition.get('mountpoint',None):
+						ppath = partition['mountpoint']
+					else:
+						ppath = partition['device_instance'].path
+					loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(ppath).name}loop"
 
 					partition['device_instance'].encrypt(password=partition['!password'])
 
@@ -207,9 +211,9 @@ class Filesystem:
 			SysCommand(f'bash -c "umount {device}?"')
 		except:
 			pass
-		
+
 		self.partprobe()
 		worked = self.raw_parted(f'{device} mklabel {disk_label}').exit_code == 0
 		self.partprobe()
-		
+
 		return worked
diff --git a/archinstall/lib/disk/helpers.py b/archinstall/lib/disk/helpers.py
index 8e6a79e4..ba29744f 100644
--- a/archinstall/lib/disk/helpers.py
+++ b/archinstall/lib/disk/helpers.py
@@ -123,9 +123,19 @@ def harddrive(size=None, model=None, fuzzy=False):
 
 		return collection[drive]
 
+def split_bind_name(path :Union[pathlib.Path, str]) -> list:
+	# we check for the bind notation. if exist we'll only use the "true" device path
+	if '[' in str(path) :  # is a bind path (btrfs subvolume path)
+		device_path, bind_path = str(path).split('[')
+		bind_path = bind_path[:-1].strip() # remove the ]
+	else:
+		device_path = path
+		bind_path = None
+	return device_path,bind_path
 
 def get_mount_info(path :Union[pathlib.Path, str], traverse=False, return_real_path=False) -> dict:
-	for traversal in list(map(str, [str(path)] + list(pathlib.Path(str(path)).parents))):
+	device_path,bind_path = split_bind_name(path)
+	for traversal in list(map(str, [str(device_path)] + list(pathlib.Path(str(device_path)).parents))):
 		try:
 			log(f"Getting mount information for device path {traversal}", level=logging.INFO)
 			output = SysCommand(f'/usr/bin/findmnt --json {traversal}').decode('UTF-8')
@@ -141,6 +151,10 @@ def get_mount_info(path :Union[pathlib.Path, str], traverse=False, return_real_p
 		raise DiskError(f"Could not get mount information for device path {path}")
 
 	output = json.loads(output)
+	# for btrfs partitions we redice the filesystem list to the one with the source equals to the parameter
+	# i.e. the subvolume filesystem we're searching for
+	if 'filesystems' in output and len(output['filesystems']) > 1 and bind_path is not None:
+		output['filesystems'] = [entry for entry in output['filesystems'] if entry['source'] == str(path)]
 	if 'filesystems' in output:
 		if len(output['filesystems']) > 1:
 			raise DiskError(f"Path '{path}' contains multiple mountpoints: {output['filesystems']}")
@@ -180,8 +194,9 @@ def get_partitions_in_use(mountpoint) -> list:
 
 
 def get_filesystem_type(path):
+	device_name, bind_name = split_bind_name(path)
 	try:
-		return SysCommand(f"blkid -o value -s TYPE {path}").decode('UTF-8').strip()
+		return SysCommand(f"blkid -o value -s TYPE {device_name}").decode('UTF-8').strip()
 	except SysCallError:
 		return None
 
@@ -217,12 +232,13 @@ def partprobe():
 	time.sleep(5)
 
 def convert_device_to_uuid(path :str) -> str:
+	device_name, bind_name = split_bind_name(path)
 	for i in range(storage['DISK_RETRY_ATTEMPTS']):
 		partprobe()
-		
+
 		# TODO: Convert lsblk to blkid
 		# (lsblk supports BlockDev and Partition UUID grabbing, blkid requires you to pick PTUUID and PARTUUID)
-		output = json.loads(SysCommand(f"lsblk --json -o+UUID {path}").decode('UTF-8'))
+		output = json.loads(SysCommand(f"lsblk --json -o+UUID {device_name}").decode('UTF-8'))
 
 		for device in output['blockdevices']:
 			if (dev_uuid := device.get('uuid', None)):
diff --git a/archinstall/lib/disk/partition.py b/archinstall/lib/disk/partition.py
index b696d9dd..bb6f2d53 100644
--- a/archinstall/lib/disk/partition.py
+++ b/archinstall/lib/disk/partition.py
@@ -7,7 +7,7 @@ import os
 import hashlib
 from typing import Optional
 from .blockdevice import BlockDevice
-from .helpers import get_mount_info, get_filesystem_type, convert_size_to_gb
+from .helpers import get_mount_info, get_filesystem_type, convert_size_to_gb, split_bind_name
 from ..storage import storage
 from ..exceptions import DiskError, SysCallError, UnknownFilesystemFormat
 from ..output import log
@@ -87,7 +87,7 @@ class Partition:
 
 	@property
 	def sector_size(self):
-		output = json.loads(SysCommand(f"lsblk --json -o+LOG-SEC {self.path}").decode('UTF-8'))
+		output = json.loads(SysCommand(f"lsblk --json -o+LOG-SEC {self.device_path}").decode('UTF-8'))
 
 		for device in output['blockdevices']:
 			return device.get('log-sec', None)
@@ -114,7 +114,7 @@ class Partition:
 		for i in range(storage['DISK_RETRY_ATTEMPTS']):
 			self.partprobe()
 
-			if (handle := SysCommand(f"lsblk --json -b -o+SIZE {self.path}")).exit_code == 0:
+			if (handle := SysCommand(f"lsblk --json -b -o+SIZE {self.device_path}")).exit_code == 0:
 				lsblk = json.loads(handle.decode('UTF-8'))
 
 				for device in lsblk['blockdevices']:
@@ -144,7 +144,7 @@ class Partition:
 
 	@property
 	def partition_type(self):
-		lsblk = json.loads(SysCommand(f"lsblk --json -o+PTTYPE {self.path}").decode('UTF-8'))
+		lsblk = json.loads(SysCommand(f"lsblk --json -o+PTTYPE {self.device_path}").decode('UTF-8'))
 
 		for device in lsblk['blockdevices']:
 			return device['pttype']
@@ -155,6 +155,7 @@ class Partition:
 		Returns the PARTUUID as returned by lsblk.
 		This is more reliable than relying on /dev/disk/by-partuuid as
 		it doesn't seam to be able to detect md raid partitions.
+		For bind mounts all the subvolumes share the same uuid
 		"""
 		for i in range(storage['DISK_RETRY_ATTEMPTS']):
 			self.partprobe()
@@ -175,8 +176,7 @@ class Partition:
 		For instance when you want to get a __repr__ of the class.
 		"""
 		self.partprobe()
-
-		return SysCommand(f'blkid -s PARTUUID -o value {self.path}').decode('UTF-8').strip()
+		return SysCommand(f'blkid -s PARTUUID -o value {self.device_path}').decode('UTF-8').strip()
 
 	@property
 	def encrypted(self):
@@ -184,7 +184,6 @@ class Partition:
 
 	@encrypted.setter
 	def encrypted(self, value: bool):
-
 		self._encrypted = value
 
 	@property
@@ -194,11 +193,26 @@ class Partition:
 	@property
 	def real_device(self):
 		for blockdevice in json.loads(SysCommand('lsblk -J').decode('UTF-8'))['blockdevices']:
-			if parent := self.find_parent_of(blockdevice, os.path.basename(self.path)):
+			if parent := self.find_parent_of(blockdevice, os.path.basename(self.device_path)):
 				return f"/dev/{parent}"
 		# 	raise DiskError(f'Could not find appropriate parent for encrypted partition {self}')
 		return self.path
 
+	@property
+	def device_path(self):
+		""" for bind mounts returns the phisical path of the partition
+		"""
+		device_path, bind_name = split_bind_name(self.path)
+		return device_path
+
+	@property
+	def bind_name(self):
+		""" for bind mounts returns the bind name (subvolume path).
+		Returns none if this property does not exist
+		"""
+		device_path, bind_name = split_bind_name(self.path)
+		return bind_name
+
 	def partprobe(self):
 		SysCommand(f'bash -c "partprobe"')
 		time.sleep(1)
@@ -348,11 +362,20 @@ class Partition:
 
 			pathlib.Path(target).mkdir(parents=True, exist_ok=True)
 
+			if self.bind_name:
+				device_path = self.device_path
+				# TODO options should be better be a list than a string
+				if options:
+					options = f"{options},subvol={self.bind_name}"
+				else:
+					options = f"subvol={self.bind_name}"
+			else:
+				device_path = self.path
 			try:
 				if options:
-					mnt_handle = SysCommand(f"/usr/bin/mount -t {fs_type} -o {options} {self.path} {target}")
+					mnt_handle = SysCommand(f"/usr/bin/mount -t {fs_type} -o {options} {device_path} {target}")
 				else:
-					mnt_handle = SysCommand(f"/usr/bin/mount -t {fs_type} {self.path} {target}")
+					mnt_handle = SysCommand(f"/usr/bin/mount -t {fs_type} {device_path} {target}")
 
 				# TODO: Should be redundant to check for exit_code
 				if mnt_handle.exit_code != 0:
@@ -401,5 +424,5 @@ def get_mount_fs_type(fs):
 	if fs == 'ntfs':
 		return 'ntfs3'  # Needed to use the Paragon R/W NTFS driver
 	elif fs == 'fat32':
-		return 'vfat'  # This is the actual type used for fat32 mounting.
+		return 'vfat'  # This is the actual type used for fat32 mounting
 	return fs