From 7ac06d75d0785da07d270dc38a7581d74c285cc5 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Fri, 22 Oct 2021 20:43:01 +0200 Subject: Restructured disk.py into lib/disk/.py instead. Shouldn't be any broken links as we expose all the functions through __init__.py - but you never know so I'll keep an eye for issues with this. --- archinstall/lib/disk/partition.py | 332 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100644 archinstall/lib/disk/partition.py (limited to 'archinstall/lib/disk/partition.py') diff --git a/archinstall/lib/disk/partition.py b/archinstall/lib/disk/partition.py new file mode 100644 index 00000000..6b60347f --- /dev/null +++ b/archinstall/lib/disk/partition.py @@ -0,0 +1,332 @@ +import glob +import pathlib +import time +from typing import Optional +from .blockdevice import BlockDevice +from ..output import log + +class Partition: + def __init__(self, path: str, block_device: BlockDevice, part_id=None, size=-1, filesystem=None, mountpoint=None, encrypted=False, autodetect_filesystem=True): + if not part_id: + part_id = os.path.basename(path) + + self.block_device = block_device + self.path = path + self.part_id = part_id + self.mountpoint = mountpoint + self.target_mountpoint = mountpoint + self.filesystem = filesystem + self.size = size # TODO: Refresh? + self._encrypted = None + self.encrypted = encrypted + self.allow_formatting = False + + if mountpoint: + self.mount(mountpoint) + + mount_information = get_mount_info(self.path) + + if self.mountpoint != mount_information.get('target', None) and mountpoint: + raise DiskError(f"{self} was given a mountpoint but the actual mountpoint differs: {mount_information.get('target', None)}") + + if target := mount_information.get('target', None): + self.mountpoint = target + + if not self.filesystem and autodetect_filesystem: + if fstype := mount_information.get('fstype', get_filesystem_type(path)): + self.filesystem = fstype + + if self.filesystem == 'crypto_LUKS': + self.encrypted = True + + def __lt__(self, left_comparitor): + if type(left_comparitor) == Partition: + left_comparitor = left_comparitor.path + else: + left_comparitor = str(left_comparitor) + return self.path < left_comparitor # Not quite sure the order here is correct. But /dev/nvme0n1p1 comes before /dev/nvme0n1p5 so seems correct. + + def __repr__(self, *args, **kwargs): + mount_repr = '' + if self.mountpoint: + mount_repr = f", mounted={self.mountpoint}" + elif self.target_mountpoint: + mount_repr = f", rel_mountpoint={self.target_mountpoint}" + + if self._encrypted: + return f'Partition(path={self.path}, size={self.size}, PARTUUID={self.uuid}, parent={self.real_device}, fs={self.filesystem}{mount_repr})' + else: + return f'Partition(path={self.path}, size={self.size}, PARTUUID={self.uuid}, fs={self.filesystem}{mount_repr})' + + def __dump__(self): + return { + 'type' : 'primary', + 'PARTUUID' : self.uuid, + 'wipe' : self.allow_formatting, + 'boot' : self.boot, + 'ESP' : self.boot, + 'mountpoint' : self.target_mountpoint, + 'encrypted' : self._encrypted, + 'start' : self.start, + 'size' : self.end, + 'filesystem' : { + 'format' : get_filesystem_type(self.path) + } + } + + @property + def sector_size(self): + output = json.loads(SysCommand(f"lsblk --json -o+LOG-SEC {self.path}").decode('UTF-8')) + + for device in output['blockdevices']: + return device.get('log-sec', None) + + @property + def start(self): + output = json.loads(SysCommand(f"sfdisk --json {self.block_device.path}").decode('UTF-8')) + + for partition in output.get('partitiontable', {}).get('partitions', []): + if partition['node'] == self.path: + return partition['start']# * self.sector_size + + @property + def end(self): + # TODO: Verify that the logic holds up, that 'size' is the size without 'start' added to it. + output = json.loads(SysCommand(f"sfdisk --json {self.block_device.path}").decode('UTF-8')) + + for partition in output.get('partitiontable', {}).get('partitions', []): + if partition['node'] == self.path: + return partition['size']# * self.sector_size + + @property + def boot(self): + output = json.loads(SysCommand(f"sfdisk --json {self.block_device.path}").decode('UTF-8')) + + # Get the bootable flag from the sfdisk output: + # { + # "partitiontable": { + # "label":"dos", + # "id":"0xd202c10a", + # "device":"/dev/loop0", + # "unit":"sectors", + # "sectorsize":512, + # "partitions": [ + # {"node":"/dev/loop0p1", "start":2048, "size":10483712, "type":"83", "bootable":true} + # ] + # } + # } + + for partition in output.get('partitiontable', {}).get('partitions', []): + if partition['node'] == self.path: + return partition.get('bootable', False) + + return False + + @property + def partition_type(self): + lsblk = json.loads(SysCommand(f"lsblk --json -o+PTTYPE {self.path}").decode('UTF-8')) + + for device in lsblk['blockdevices']: + return device['pttype'] + + @property + def uuid(self) -> Optional[str]: + """ + Returns the PARTUUID as returned by lsblk. + This is more reliable than relying on /dev/disk/by-partuuid as + it doesn't seam to be able to detect md raid partitions. + """ + + lsblk = json.loads(SysCommand(f'lsblk -J -o+PARTUUID {self.path}').decode('UTF-8')) + for partition in lsblk['blockdevices']: + return partition.get('partuuid', None) + return None + + @property + def encrypted(self): + return self._encrypted + + @encrypted.setter + def encrypted(self, value: bool): + + self._encrypted = value + + @property + def parent(self): + return self.real_device + + @property + def real_device(self): + for blockdevice in json.loads(SysCommand('lsblk -J').decode('UTF-8'))['blockdevices']: + if parent := self.find_parent_of(blockdevice, os.path.basename(self.path)): + return f"/dev/{parent}" + # raise DiskError(f'Could not find appropriate parent for encrypted partition {self}') + return self.path + + def detect_inner_filesystem(self, password): + log(f'Trying to detect inner filesystem format on {self} (This might take a while)', level=logging.INFO) + from .luks import luks2 + + try: + with luks2(self, storage.get('ENC_IDENTIFIER', 'ai')+'loop', password, auto_unmount=True) as unlocked_device: + return unlocked_device.filesystem + except SysCallError: + return None + + def has_content(self): + fs_type = get_filesystem_type(self.path) + if not fs_type or "swap" in fs_type: + return False + + temporary_mountpoint = '/tmp/' + hashlib.md5(bytes(f"{time.time()}", 'UTF-8') + os.urandom(12)).hexdigest() + temporary_path = pathlib.Path(temporary_mountpoint) + + temporary_path.mkdir(parents=True, exist_ok=True) + if (handle := SysCommand(f'/usr/bin/mount {self.path} {temporary_mountpoint}')).exit_code != 0: + raise DiskError(f'Could not mount and check for content on {self.path} because: {b"".join(handle)}') + + files = len(glob.glob(f"{temporary_mountpoint}/*")) + iterations = 0 + while SysCommand(f"/usr/bin/umount -R {temporary_mountpoint}").exit_code != 0 and (iterations := iterations + 1) < 10: + time.sleep(1) + + temporary_path.rmdir() + + return True if files > 0 else False + + def encrypt(self, *args, **kwargs): + """ + A wrapper function for luks2() instances and the .encrypt() method of that instance. + """ + from .luks import luks2 + + handle = luks2(self, None, None) + return handle.encrypt(self, *args, **kwargs) + + def format(self, filesystem=None, path=None, log_formatting=True): + """ + Format can be given an overriding path, for instance /dev/null to test + the formatting functionality and in essence the support for the given filesystem. + """ + if filesystem is None: + filesystem = self.filesystem + + if path is None: + path = self.path + + # To avoid "unable to open /dev/x: No such file or directory" + start_wait = time.time() + while pathlib.Path(path).exists() is False and time.time() - start_wait < 10: + time.sleep(0.025) + + if log_formatting: + log(f'Formatting {path} -> {filesystem}', level=logging.INFO) + + if filesystem == 'btrfs': + if 'UUID:' not in (mkfs := SysCommand(f'/usr/bin/mkfs.btrfs -f {path}').decode('UTF-8')): + raise DiskError(f'Could not format {path} with {filesystem} because: {mkfs}') + self.filesystem = filesystem + + elif filesystem == 'fat32': + mkfs = SysCommand(f'/usr/bin/mkfs.vfat -F32 {path}').decode('UTF-8') + if ('mkfs.fat' not in mkfs and 'mkfs.vfat' not in mkfs) or 'command not found' in mkfs: + raise DiskError(f"Could not format {path} with {filesystem} because: {mkfs}") + self.filesystem = filesystem + + elif filesystem == 'ext4': + if (handle := SysCommand(f'/usr/bin/mkfs.ext4 -F {path}')).exit_code != 0: + raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") + self.filesystem = filesystem + + elif filesystem == 'ext2': + if (handle := SysCommand(f'/usr/bin/mkfs.ext2 -F {path}')).exit_code != 0: + raise DiskError(f'Could not format {path} with {filesystem} because: {b"".join(handle)}') + self.filesystem = 'ext2' + + elif filesystem == 'xfs': + if (handle := SysCommand(f'/usr/bin/mkfs.xfs -f {path}')).exit_code != 0: + raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") + self.filesystem = filesystem + + elif filesystem == 'f2fs': + if (handle := SysCommand(f'/usr/bin/mkfs.f2fs -f {path}')).exit_code != 0: + raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") + self.filesystem = filesystem + + elif filesystem == 'crypto_LUKS': + # from .luks import luks2 + # encrypted_partition = luks2(self, None, None) + # encrypted_partition.format(path) + self.filesystem = filesystem + + else: + raise UnknownFilesystemFormat(f"Fileformat '{filesystem}' is not yet implemented.") + + if get_filesystem_type(path) == 'crypto_LUKS' or get_filesystem_type(self.real_device) == 'crypto_LUKS': + self.encrypted = True + else: + self.encrypted = False + + return True + + def find_parent_of(self, data, name, parent=None): + if data['name'] == name: + return parent + elif 'children' in data: + for child in data['children']: + if parent := self.find_parent_of(child, name, parent=data['name']): + return parent + + def mount(self, target, fs=None, options=''): + if not self.mountpoint: + log(f'Mounting {self} to {target}', level=logging.INFO) + if not fs: + if not self.filesystem: + raise DiskError(f'Need to format (or define) the filesystem on {self} before mounting.') + fs = self.filesystem + + pathlib.Path(target).mkdir(parents=True, exist_ok=True) + + try: + if options: + SysCommand(f"/usr/bin/mount -o {options} {self.path} {target}") + else: + SysCommand(f"/usr/bin/mount {self.path} {target}") + except SysCallError as err: + raise err + + self.mountpoint = target + return True + + def unmount(self): + try: + SysCommand(f"/usr/bin/umount {self.path}") + except SysCallError as err: + exit_code = err.exit_code + + # Without to much research, it seams that low error codes are errors. + # And above 8k is indicators such as "/dev/x not mounted.". + # So anything in between 0 and 8k are errors (?). + if 0 < exit_code < 8000: + raise err + + self.mountpoint = None + return True + + def umount(self): + return self.unmount() + + def filesystem_supported(self): + """ + The support for a filesystem (this partition) is tested by calling + partition.format() with a path set to '/dev/null' which returns two exceptions: + 1. SysCallError saying that /dev/null is not formattable - but the filesystem is supported + 2. UnknownFilesystemFormat that indicates that we don't support the given filesystem type + """ + try: + self.format(self.filesystem, '/dev/null', log_formatting=False, allow_formatting=True) + except (SysCallError, DiskError): + pass # We supported it, but /dev/null is not formatable as expected so the mkfs call exited with an error code + except UnknownFilesystemFormat as err: + raise err + return True \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 8b96080ec81939f3a69c28585c21c43e496d087b Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Fri, 22 Oct 2021 21:02:39 +0200 Subject: Forgot some imports that didn't show up on a static run without going through a few of the menu's --- archinstall/lib/disk/blockdevice.py | 1 + archinstall/lib/disk/partition.py | 1 + archinstall/lib/disk/user_guides.py | 1 + 3 files changed, 3 insertions(+) (limited to 'archinstall/lib/disk/partition.py') diff --git a/archinstall/lib/disk/blockdevice.py b/archinstall/lib/disk/blockdevice.py index 422f35aa..daa65323 100644 --- a/archinstall/lib/disk/blockdevice.py +++ b/archinstall/lib/disk/blockdevice.py @@ -1,4 +1,5 @@ import json +import logging from ..output import log from ..general import SysCommand diff --git a/archinstall/lib/disk/partition.py b/archinstall/lib/disk/partition.py index 6b60347f..30151583 100644 --- a/archinstall/lib/disk/partition.py +++ b/archinstall/lib/disk/partition.py @@ -1,6 +1,7 @@ import glob import pathlib import time +import logging from typing import Optional from .blockdevice import BlockDevice from ..output import log diff --git a/archinstall/lib/disk/user_guides.py b/archinstall/lib/disk/user_guides.py index f6466268..0a975149 100644 --- a/archinstall/lib/disk/user_guides.py +++ b/archinstall/lib/disk/user_guides.py @@ -1,3 +1,4 @@ +import logging from ..output import log def suggest_single_disk_layout(block_device, default_filesystem=None): -- cgit v1.2.3-70-g09d2 From 7149b76f3bd3163938fe7413546e5f678f98851f Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Fri, 22 Oct 2021 21:54:16 +0200 Subject: Forgot some imports that didn't show up on a static run without going through a few of the menu's --- archinstall/lib/disk/__init__.py | 2 +- archinstall/lib/disk/blockdevice.py | 3 +++ archinstall/lib/disk/filesystem.py | 6 +++++- archinstall/lib/disk/helpers.py | 2 ++ archinstall/lib/disk/partition.py | 4 ++++ archinstall/lib/disk/user_guides.py | 4 ++-- examples/guided.py | 6 +++--- 7 files changed, 20 insertions(+), 7 deletions(-) (limited to 'archinstall/lib/disk/partition.py') diff --git a/archinstall/lib/disk/__init__.py b/archinstall/lib/disk/__init__.py index 8237f774..352d04b9 100644 --- a/archinstall/lib/disk/__init__.py +++ b/archinstall/lib/disk/__init__.py @@ -1,7 +1,7 @@ from .btrfs import * from .helpers import * from .blockdevice import BlockDevice -from .filesystem import Filesystem +from .filesystem import Filesystem, MBR, GPT from .partition import * from .user_guides import * from .validators import * \ No newline at end of file diff --git a/archinstall/lib/disk/blockdevice.py b/archinstall/lib/disk/blockdevice.py index daa65323..57cbcfa6 100644 --- a/archinstall/lib/disk/blockdevice.py +++ b/archinstall/lib/disk/blockdevice.py @@ -1,3 +1,4 @@ +import os import json import logging from ..output import log @@ -94,6 +95,7 @@ class BlockDevice: @property def partitions(self): + from .filesystem import Partition SysCommand(['partprobe', self.path]) result = SysCommand(['/usr/bin/lsblk', '-J', self.path]) @@ -123,6 +125,7 @@ class BlockDevice: @property def partition_table_type(self): + from .filesystem import GPT return GPT @property diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py index b53d8451..28846764 100644 --- a/archinstall/lib/disk/filesystem.py +++ b/archinstall/lib/disk/filesystem.py @@ -1,7 +1,11 @@ import time +import logging +import json from .partition import Partition from .blockdevice import BlockDevice +from ..general import SysCommand from ..output import log +from ..storage import storage GPT = 0b00000001 MBR = 0b00000010 @@ -58,7 +62,7 @@ class Filesystem: return index def load_layout(self, layout :dict): - from .luks import luks2 + from ..luks import luks2 # If the layout tells us to wipe the drive, we do so if layout.get('wipe', False): diff --git a/archinstall/lib/disk/helpers.py b/archinstall/lib/disk/helpers.py index 8b372f73..65abdea2 100644 --- a/archinstall/lib/disk/helpers.py +++ b/archinstall/lib/disk/helpers.py @@ -132,6 +132,8 @@ def get_mount_info(path) -> dict: def get_partitions_in_use(mountpoint) -> list: + from .partition import Partition + try: output = SysCommand(f"/usr/bin/findmnt --json -R {mountpoint}").decode('UTF-8') except SysCallError: diff --git a/archinstall/lib/disk/partition.py b/archinstall/lib/disk/partition.py index 30151583..3bb2982b 100644 --- a/archinstall/lib/disk/partition.py +++ b/archinstall/lib/disk/partition.py @@ -2,9 +2,13 @@ import glob import pathlib import time import logging +import json +import os from typing import Optional from .blockdevice import BlockDevice +from .helpers import get_mount_info, get_filesystem_type from ..output import log +from ..general import SysCommand class Partition: def __init__(self, path: str, block_device: BlockDevice, part_id=None, size=-1, filesystem=None, mountpoint=None, encrypted=False, autodetect_filesystem=True): diff --git a/archinstall/lib/disk/user_guides.py b/archinstall/lib/disk/user_guides.py index 0a975149..79b9d48f 100644 --- a/archinstall/lib/disk/user_guides.py +++ b/archinstall/lib/disk/user_guides.py @@ -3,7 +3,7 @@ from ..output import log def suggest_single_disk_layout(block_device, default_filesystem=None): if not default_filesystem: - from .user_interaction import ask_for_main_filesystem_format + from ..user_interaction import ask_for_main_filesystem_format default_filesystem = ask_for_main_filesystem_format() MIN_SIZE_TO_ALLOW_HOME_PART = 40 # Gb @@ -76,7 +76,7 @@ def suggest_single_disk_layout(block_device, default_filesystem=None): def suggest_multi_disk_layout(block_devices, default_filesystem=None): if not default_filesystem: - from .user_interaction import ask_for_main_filesystem_format + from ..user_interaction import ask_for_main_filesystem_format default_filesystem = ask_for_main_filesystem_format() # Not really a rock solid foundation of information to stand on, but it's a start: diff --git a/examples/guided.py b/examples/guided.py index b7c75b30..2efb4972 100644 --- a/examples/guided.py +++ b/examples/guided.py @@ -140,7 +140,7 @@ def ask_user_questions(): # Ask for a root password (optional, but triggers requirement for super-user if skipped) if not archinstall.arguments.get('!root-password', None): - archinstall.arguments['!root-password'] = archinstall.get_password(prompt='Enter root password (Recommendation: leave blank to leave root disabled): ') + archinstall.arguments['!root-password'] = archinstall.get_password(prompt='Enter root password (leave blank to disable disabled & create superuser): ') # Ask for additional users (super-user if root pw was not set) @@ -245,9 +245,9 @@ def perform_filesystem_operations(): Setup the blockdevice, filesystem (and optionally encryption). Once that's done, we'll hand over to perform_installation() """ - mode = archinstall.GPT + mode = archinstall.disk.GPT if has_uefi() is False: - mode = archinstall.MBR + mode = archinstall.disk.MBR for drive in archinstall.arguments['harddrives']: with archinstall.Filesystem(drive, mode) as fs: -- cgit v1.2.3-70-g09d2