"""
Photon installer
"""
#
#    Author: Mahmoud Bassiouny <mbassiouny@vmware.com>

import subprocess
import os
import re
import shutil
import signal
import sys
import glob
import modules.commons
import random
import curses
import stat
import tempfile
from logger import Logger
from commandutils import CommandUtils
from jsonwrapper import JsonWrapper
from progressbar import ProgressBar
from window import Window
from actionresult import ActionResult
from networkmanager import NetworkManager
from enum import Enum

class PartitionType(Enum):
    SWAP = 1
    LINUX = 2
    LVM = 3
    ESP = 4
    BIOS = 5

class Installer(object):
    """
    Photon installer
    """

    # List of allowed keys in kickstart config file.
    # Please keep ks_config.txt file updated.
    known_keys = {
        'additional_files',
        'additional_packages',
        'additional_rpms_path',
        'arch',
        'autopartition',
        'bootmode',
        'disk',
        'eject_cdrom',
        'hostname',
        'install_linux_esx',
        'linux_flavor',
        'live',
        'log_level',
        'ostree',
        'packages',
        'packagelist_file',
        'partition_type',
        'partitions',
        'network',
        'password',
        'postinstall',
        'postinstallscripts',
        'public_key',
        'search_path',
        'setup_grub_script',
        'shadow_password',
        'type',
        'ui'
    }

    default_partitions = [{"mountpoint": "/", "size": 0, "filesystem": "ext4"}]
    all_linux_flavors = ["linux", "linux-esx", "linux-aws", "linux-secure", "linux-rt"]
    linux_dependencies = ["devel", "drivers", "docs", "oprofile", "dtb", "hmacgen"]

    def __init__(self, working_directory="/mnt/photon-root",
                 rpm_path=os.path.dirname(__file__)+"/../stage/RPMS", log_path=os.path.dirname(__file__)+"/../stage/LOGS"):
        self.exiting = False
        self.interactive = False
        self.install_config = None
        self.rpm_path = rpm_path
        self.log_path = log_path
        self.logger = None
        self.cmd = None
        self.working_directory = working_directory

        if os.path.exists(self.working_directory) and os.path.isdir(self.working_directory) and working_directory == '/mnt/photon-root':
            shutil.rmtree(self.working_directory)
        if not os.path.exists(self.working_directory):
            os.mkdir(self.working_directory)

        self.photon_root = self.working_directory + "/photon-chroot"
        self.installer_path = os.path.dirname(os.path.abspath(__file__))
        self.tdnf_conf_path = self.working_directory + "/tdnf.conf"
        self.tdnf_repo_path = self.working_directory + "/photon-local.repo"
        self.rpm_cache_dir = self.photon_root + '/cache/tdnf/photon-local/rpms'
        # used by tdnf.conf as cachedir=, tdnf will append the rest
        self.rpm_cache_dir_short = self.photon_root + '/cache/tdnf'

        self.setup_grub_command = os.path.dirname(__file__)+"/mk-setup-grub.sh"

        signal.signal(signal.SIGINT, self.exit_gracefully)
        self.lvs_to_detach = {'vgs': [], 'pvs': []}

    """
    create, append and validate configuration date - install_config
    """
    def configure(self, install_config, ui_config = None):
        # Initialize logger and cmd first
        if not install_config:
            # UI installation
            log_level = 'debug'
            console = False
        else:
            log_level = install_config.get('log_level', 'info')
            console = not install_config.get('ui', False)
        self.logger = Logger.get_logger(self.log_path, log_level, console)
        self.cmd = CommandUtils(self.logger)

        # run UI configurator iff install_config param is None
        if not install_config and ui_config:
            from iso_config import IsoConfig
            self.interactive = True
            config = IsoConfig()
            install_config = curses.wrapper(config.configure, ui_config)

        self._add_defaults(install_config)

        issue = self._check_install_config(install_config)
        if issue:
            self.logger.error(issue)
            raise Exception(issue)

        self.install_config = install_config


    def execute(self):
        if 'setup_grub_script' in self.install_config:
            self.setup_grub_command = self.install_config['setup_grub_script']

        if self.install_config['ui']:
            curses.wrapper(self._install)
        else:
            self._install()

    def _add_defaults(self, install_config):
        """
        Add default install_config settings if not specified
        """
        # set arch to host's one if not defined
        arch = subprocess.check_output(['uname', '-m'], universal_newlines=True).rstrip('\n')
        if 'arch' not in install_config:
            install_config['arch'] = arch

        # 'bootmode' mode
        if 'bootmode' not in install_config:
            if "x86_64" in arch:
                install_config['bootmode'] = 'dualboot'
            else:
                install_config['bootmode'] = 'efi'

        # extend 'packages' by 'packagelist_file' and 'additional_packages'
        packages = []
        if 'packagelist_file' in install_config:
            plf = install_config['packagelist_file']
            if not plf.startswith('/'):
                plf = os.path.join(os.path.dirname(__file__), plf)
            json_wrapper_package_list = JsonWrapper(plf)
            package_list_json = json_wrapper_package_list.read()
            packages.extend(package_list_json["packages"])

        if 'additional_packages' in install_config:
            packages.extend(install_config['additional_packages'])

        # add bootloader packages after bootmode set
        if install_config['bootmode'] in ['dualboot', 'efi']:
            packages.append('grub2-efi-image')

        if 'packages' in install_config:
            install_config['packages'] = list(set(packages + install_config['packages']))
        else:
            install_config['packages'] = packages

        # live means online system. When you create an image for
        # target system, live should be set to false.
        if 'live' not in install_config:
            install_config['live'] = 'loop' not in install_config['disk']

        # default partition
        if 'partitions' not in install_config:
            install_config['partitions'] = Installer.default_partitions

        # define 'hostname' as 'photon-<RANDOM STRING>'
        if "hostname" not in install_config or install_config['hostname'] == "":
            install_config['hostname'] = 'photon-%12x' % random.randrange(16**12)

        # Set password if needed.
        # Installer uses 'shadow_password' and optionally 'password'/'age'
        # to set aging if present. See modules/m_updaterootpassword.py
        if 'shadow_password' not in install_config:
            if 'password' not in install_config:
                install_config['password'] = {'crypted': True, 'text': '*', 'age': -1}

            if install_config['password']['crypted']:
                install_config['shadow_password'] = install_config['password']['text']
            else:
                install_config['shadow_password'] = CommandUtils.generate_password_hash(install_config['password']['text'])

        # Do not show UI progress by default
        if 'ui' not in install_config:
            install_config['ui'] = False

        # Log level
        if 'log_level' not in install_config:
            install_config['log_level'] = 'info'

        # Extend search_path by current dir and script dir
        if 'search_path' not in install_config:
            install_config['search_path'] = []
        for dirname in [os.getcwd(), os.path.abspath(os.path.dirname(__file__))]:
            if dirname not in install_config['search_path']:
                install_config['search_path'].append(dirname)

        if 'linux_flavor' not in install_config:
            if install_config.get('install_linux_esx', False) == True:
                install_config['linux_flavor'] = "linux-esx"
            else:
                available_flavors=[]
                for flavor in self.all_linux_flavors:
                    if flavor in install_config['packages']:
                        available_flavors.append(flavor)
                if len(available_flavors) == 1:
                    install_config['linux_flavor'] = available_flavors[0]


        install_config['install_linux_esx'] = False

    def _check_install_config(self, install_config):
        """
        Sanity check of install_config before its execution.
        Return error string or None
        """

        unknown_keys = install_config.keys() - Installer.known_keys
        if len(unknown_keys) > 0:
            return "Unknown install_config keys: " + ", ".join(unknown_keys)

        if not 'disk' in install_config:
            return "No disk configured"

        # For Ostree install_config['packages'] will be empty list, because ostree
        # uses preinstalled tree ostree-repo.tar.gz for installation
        if 'ostree' not in install_config and 'linux_flavor' not in install_config:
            return "Attempting to install more than one linux flavor"

        # Perform following checks here:
        # 1) Only one extensible partition is allowed per disk
        # 2) /boot can not be LVM
        # 3) / must present
        # 4) Duplicate mountpoints should not be present
        has_extensible = {}
        has_root = False
        mountpoints = []
        default_disk = install_config['disk']
        for partition in install_config['partitions']:
            disk = partition.get('disk', default_disk)
            mntpoint = partition.get('mountpoint', '')
            if disk not in has_extensible:
                has_extensible[disk] = False
            size = partition['size']
            if size == 0:
                if has_extensible[disk]:
                    return "Disk {} has more than one extensible partition".format(disk)
                else:
                    has_extensible[disk] = True
            if mntpoint != '':
                mountpoints.append(mntpoint)
            if mntpoint == '/boot' and 'lvm' in partition:
                return "/boot on LVM is not supported"
            elif mntpoint == '/':
                has_root = True
        if not has_root:
            return "There is no partition assigned to root '/'"

        if len(mountpoints) != len(set(mountpoints)):
            return "Duplicate mountpoints exist in partition table!!"

        if install_config['arch'] not in ["aarch64", 'x86_64']:
            return "Unsupported target architecture {}".format(install_config['arch'])

        # No BIOS for aarch64
        if install_config['arch'] == 'aarch64' and install_config['bootmode'] in ['dualboot', 'bios']:
            return "Aarch64 targets do not support BIOS boot. Set 'bootmode' to 'efi'."

        if 'age' in install_config.get('password', {}):
            if install_config['password']['age'] < -1:
                return "Password age should be -1, 0 or positive"

        return None

    def _install(self, stdscreen=None):
        """
        Install photon system and handle exception
        """
        if self.install_config['ui']:
            # init the screen
            curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE)
            curses.init_pair(2, curses.COLOR_BLACK, curses.COLOR_WHITE)
            curses.init_pair(3, curses.COLOR_BLACK, curses.COLOR_GREEN)
            curses.init_pair(4, curses.COLOR_RED, curses.COLOR_WHITE)
            stdscreen.bkgd(' ', curses.color_pair(1))
            maxy, maxx = stdscreen.getmaxyx()
            curses.curs_set(0)

            # initializing windows
            height = 10
            width = 75
            progress_padding = 5

            progress_width = width - progress_padding
            starty = (maxy - height) // 2
            startx = (maxx - width) // 2
            self.window = Window(height, width, maxy, maxx,
                                 'Installing Photon', False)
            self.progress_bar = ProgressBar(starty + 3,
                                            startx + progress_padding // 2,
                                            progress_width)
            self.window.show_window()
            self.progress_bar.initialize('Initializing installation...')
            self.progress_bar.show()

        try:
            self._unsafe_install()
        except Exception as inst:
            self.logger.exception(repr(inst))
            self.exit_gracefully()

        # Congratulation screen
        if self.install_config['ui']:
            self.progress_bar.hide()
            self.window.addstr(0, 0, 'Congratulations, Photon has been installed in {0} secs.\n\n'
                               'Press any key to continue to boot...'
                               .format(self.progress_bar.time_elapsed))
            if self.interactive:
                self.window.content_window().getch()

        if self.install_config['live']:
            self._eject_cdrom()

    def _unsafe_install(self):
        """
        Install photon system
        """
        self._partition_disk()
        self._format_partitions()
        self._mount_partitions()
        if 'ostree' in self.install_config:
            from ostreeinstaller import OstreeInstaller
            ostree = OstreeInstaller(self)
            ostree.install()
        else:
            self._setup_install_repo()
            self._initialize_system()
            self._mount_special_folders()
            self._install_packages()
            self._install_additional_rpms()
            self._enable_network_in_chroot()
            self._setup_network()
            self._finalize_system()
            self._cleanup_install_repo()
            self._setup_grub()
            self._create_fstab()
        self._execute_modules(modules.commons.POST_INSTALL)
        self._disable_network_in_chroot()
        self._unmount_all()

    def exit_gracefully(self, signal1=None, frame1=None):
        """
        This will be called if the installer interrupted by Ctrl+C, exception
        or other failures
        """
        del signal1
        del frame1
        if not self.exiting and self.install_config:
            self.exiting = True
            if self.install_config['ui']:
                self.progress_bar.hide()
                self.window.addstr(0, 0, 'Oops, Installer got interrupted.\n\n' +
                                   'Press any key to get to the bash...')
                self.window.content_window().getch()

            self._cleanup_install_repo()
            self._unmount_all()
        sys.exit(1)

    def _setup_network(self):
        if 'network' not in self.install_config:
            return
        # setup network config files in chroot
        nm = NetworkManager(self.install_config, self.photon_root)
        if not nm.setup_network():
            self.logger.error("Failed to setup network!")
            self.exit_gracefully()

        # Configure network when in live mode (ISO) and when network is not
        # already configured (typically in KS flow).
        if ('live' in self.install_config and
                'conf_files' not in self.install_config['network']):
            nm = NetworkManager(self.install_config)
            if not nm.setup_network():
                self.logger.error("Failed to setup network in ISO system")
                self.exit_gracefully()
            nm.restart_networkd()

    def _unmount_all(self):
        """
        Unmount partitions and special folders
        """
        for d in ["/tmp", "/run", "/sys", "/dev/pts", "/dev", "/proc"]:
            if os.path.exists(self.photon_root + d):
                retval = self.cmd.run(['umount', '-l', self.photon_root + d])
                if retval != 0:
                    self.logger.error("Failed to unmount {}".format(d))

        for partition in self.install_config['partitions'][::-1]:
            if self._get_partition_type(partition) in [PartitionType.BIOS, PartitionType.SWAP]:
                continue
            mountpoint = self.photon_root + partition["mountpoint"]
            if os.path.exists(mountpoint):
                retval = self.cmd.run(['umount', '-l', mountpoint])
                if retval != 0:
                    self.logger.error("Failed to unmount partition {}".format(mountpoint))

        # need to call it twice, because of internal bind mounts
        if 'ostree' in self.install_config:
            if os.path.exists(self.photon_root):
                retval = self.cmd.run(['umount', '-R', self.photon_root])
                retval = self.cmd.run(['umount', '-R', self.photon_root])
                if retval != 0:
                    self.logger.error("Failed to unmount disks in photon root")

        self.cmd.run(['sync'])
        if os.path.exists(self.photon_root):
            shutil.rmtree(self.photon_root)

        # Deactivate LVM VGs
        for vg in self.lvs_to_detach['vgs']:
            retval = self.cmd.run(["vgchange", "-v", "-an", vg])
            if retval != 0:
                self.logger.error("Failed to deactivate LVM volume group: {}".format(vg))

        # Get the disks from partition table
        disks = set(partition.get('disk', self.install_config['disk']) for partition in self.install_config['partitions'])
        for disk in disks:
            if 'loop' in disk:
                # Simulate partition hot remove to notify LVM
                for pv in self.lvs_to_detach['pvs']:
                    retval = self.cmd.run(["dmsetup", "remove", pv])
                    if retval != 0:
                        self.logger.error("Failed to detach LVM physical volume: {}".format(pv))
                # Uninitialize device paritions mapping
                retval = self.cmd.run(['kpartx', '-d', disk])
                if retval != 0:
                    self.logger.error("Failed to unmap partitions of the disk image {}". format(disk))
                    return None

    def _bind_installer(self):
        """
        Make the photon_root/installer directory if not exits
        The function finalize_system will access the file /installer/mk-finalize-system.sh
        after chroot to photon_root.
        Bind the /installer folder to self.photon_root/installer, so that after chroot
        to photon_root,
        the file can still be accessed as /installer/mk-finalize-system.sh.
        """
        # Make the photon_root/installer directory if not exits
        if(self.cmd.run(['mkdir', '-p',
                            os.path.join(self.photon_root, "installer")]) != 0 or
           self.cmd.run(['mount', '--bind', self.installer_path,
                            os.path.join(self.photon_root, "installer")]) != 0):
            self.logger.error("Fail to bind installer")
            self.exit_gracefully()

    def _unbind_installer(self):
        # unmount the installer directory
        if os.path.exists(os.path.join(self.photon_root, "installer")):
            retval = self.cmd.run(['umount', os.path.join(self.photon_root, "installer")])
            if retval != 0:
                self.logger.error("Fail to unbind the installer directory")
            # remove the installer directory
            retval = self.cmd.run(['rm', '-rf', os.path.join(self.photon_root, "installer")])
            if retval != 0:
                self.logger.error("Fail to remove the installer directory")

    def _bind_repo_dir(self):
        """
        Bind repo dir for tdnf installation
        """
        if self.rpm_path.startswith("https://") or self.rpm_path.startswith("http://"):
            return
        if (self.cmd.run(['mkdir', '-p', self.rpm_cache_dir]) != 0 or
                self.cmd.run(['mount', '--bind', self.rpm_path, self.rpm_cache_dir]) != 0):
            self.logger.error("Fail to bind cache rpms")
            self.exit_gracefully()

    def _unbind_repo_dir(self):
        """
        Unbind repo dir after installation
        """
        if self.rpm_path.startswith("https://") or self.rpm_path.startswith("http://"):
            return
        if os.path.exists(self.rpm_cache_dir):
            if (self.cmd.run(['umount', self.rpm_cache_dir]) != 0 or
                    self.cmd.run(['rm', '-rf', self.rpm_cache_dir]) != 0):
                self.logger.error("Fail to unbind cache rpms")

    def _get_partuuid(self, path):
        partuuid = subprocess.check_output(['blkid', '-s', 'PARTUUID', '-o', 'value', path],
                                       universal_newlines=True).rstrip('\n')
        # Backup way to get uuid/partuuid. Leave it here for later use.
        #if partuuidval == '':
        #    sgdiskout = Utils.runshellcommand(
        #        "sgdisk -i 2 {} ".format(disk_device))
        #    partuuidval = (re.findall(r'Partition unique GUID.*',
        #                          sgdiskout))[0].split(':')[1].strip(' ').lower()
        return partuuid

    def _get_uuid(self, path):
        return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', path],
                                       universal_newlines=True).rstrip('\n')

    def _create_fstab(self, fstab_path = None):
        """
        update fstab
        """
        if not fstab_path:
            fstab_path = os.path.join(self.photon_root, "etc/fstab")
        with open(fstab_path, "w") as fstab_file:
            fstab_file.write("#system\tmnt-pt\ttype\toptions\tdump\tfsck\n")

            for partition in self.install_config['partitions']:
                ptype = self._get_partition_type(partition)
                if ptype == PartitionType.BIOS:
                    continue

                options = 'defaults'
                dump = 1
                fsck = 2

                if partition.get('mountpoint', '') == '/':
                    options = options + ',barrier,noatime,noacl,data=ordered'
                    fsck = 1

                if ptype == PartitionType.SWAP:
                    mountpoint = 'swap'
                    dump = 0
                    fsck = 0
                else:
                    mountpoint = partition['mountpoint']

                # Use PARTUUID/UUID instead of bare path.
                # Prefer PARTUUID over UUID as it is supported by kernel
                # and UUID only by initrd.
                path = partition['path']
                mnt_src = None
                partuuid = self._get_partuuid(path)
                if partuuid != '':
                    mnt_src = "PARTUUID={}".format(partuuid)
                else:
                    uuid = self._get_uuid(path)
                    if uuid != '':
                        mnt_src = "UUID={}".format(uuid)
                if not mnt_src:
                    raise RuntimeError("Cannot get PARTUUID/UUID of: {}".format(path))

                fstab_file.write("{}\t{}\t{}\t{}\t{}\t{}\n".format(
                    mnt_src,
                    mountpoint,
                    partition['filesystem'],
                    options,
                    dump,
                    fsck
                    ))
            # Add the cdrom entry
            fstab_file.write("/dev/cdrom\t/mnt/cdrom\tiso9660\tro,noauto\t0\t0\n")

    def _generate_partitions_param(self, reverse=False):
        """
        Generate partition param for mount command
        """
        if reverse:
            step = -1
        else:
            step = 1
        params = []
        for partition in self.install_config['partitions'][::step]:
            if self._get_partition_type(partition) in [PartitionType.BIOS, PartitionType.SWAP]:
                continue

            params.extend(['--partitionmountpoint', partition["path"], partition["mountpoint"]])
        return params

    def _mount_partitions(self):
        for partition in self.install_config['partitions'][::1]:
            if self._get_partition_type(partition) in [PartitionType.BIOS, PartitionType.SWAP]:
                continue
            mountpoint = self.photon_root + partition["mountpoint"]
            self.cmd.run(['mkdir', '-p', mountpoint])
            retval = self.cmd.run(['mount', '-v', partition["path"], mountpoint])
            if retval != 0:
                self.logger.error("Failed to mount partition {}".format(partition["path"]))
                self.exit_gracefully()

    def _initialize_system(self):
        """
        Prepare the system to install photon
        """
        if self.install_config['ui']:
            self.progress_bar.update_message('Initializing system...')
        self._bind_installer()
        self._bind_repo_dir()

        # Initialize rpm DB
        self.cmd.run(['mkdir', '-p', os.path.join(self.photon_root, "var/lib/rpm")])
        retval = self.cmd.run(['rpm', '--root', self.photon_root, '--initdb',
                               '--dbpath', '/var/lib/rpm'])
        if retval != 0:
            self.logger.error("Failed to initialize rpm DB")
            self.exit_gracefully()

        # Install filesystem rpm
        tdnf_cmd = "tdnf install filesystem --installroot {0} --assumeyes -c {1}".format(self.photon_root,
                        self.tdnf_conf_path)
        retval = self.cmd.run(tdnf_cmd)
        if retval != 0:
            retval = self.cmd.run(['docker', 'run',
                                   '-v', self.rpm_cache_dir+':'+self.rpm_cache_dir,
                                   '-v', self.working_directory+':'+self.working_directory,
                                   'photon:3.0', '/bin/sh', '-c', tdnf_cmd])
            if retval != 0:
                self.logger.error("Failed to install filesystem rpm")
                self.exit_gracefully()

        # Create special devices. We need it when devtpmfs is not mounted yet.
        devices = {
            'console': (600, stat.S_IFCHR, 5, 1),
            'null': (666, stat.S_IFCHR, 1, 3),
            'random': (444, stat.S_IFCHR, 1, 8),
            'urandom': (444, stat.S_IFCHR, 1, 9)
        }
        for device, (mode, dev_type, major, minor) in devices.items():
            os.mknod(os.path.join(self.photon_root, "dev", device),
                    mode | dev_type, os.makedev(major, minor))


    def _mount_special_folders(self):
        for d in ["/proc", "/dev", "/dev/pts", "/sys"]:
            retval = self.cmd.run(['mount', '-o', 'bind', d, self.photon_root + d])
            if retval != 0:
                self.logger.error("Failed to bind mount {}".format(d))
                self.exit_gracefully()

        for d in ["/tmp", "/run"]:
            retval = self.cmd.run(['mount', '-t', 'tmpfs', 'tmpfs', self.photon_root + d])
            if retval != 0:
                self.logger.error("Failed to bind mount {}".format(d))
                self.exit_gracefully()

    def _copy_additional_files(self):
        if 'additional_files' in self.install_config:
            for filetuples in self.install_config['additional_files']:
                for src, dest in filetuples.items():
                    if src.startswith('http://') or src.startswith('https://'):
                        temp_file = tempfile.mktemp()
                        result, msg = CommandUtils.wget(src, temp_file, False)
                        if result:
                            shutil.copyfile(temp_file, self.photon_root + dest)
                        else:
                            self.logger.error("Download failed URL: {} got error: {}".format(src, msg))
                    else:
                        srcpath = self.getfile(src)
                        if (os.path.isdir(srcpath)):
                            shutil.copytree(srcpath, self.photon_root + dest, True)
                        else:
                            shutil.copyfile(srcpath, self.photon_root + dest)

    def _finalize_system(self):
        """
        Finalize the system after the installation
        """
        if self.install_config['ui']:
            self.progress_bar.show_loading('Finalizing installation')

        self._copy_additional_files()

        self.cmd.run_in_chroot(self.photon_root, "/sbin/ldconfig")

        # Importing the pubkey
        self.cmd.run_in_chroot(self.photon_root, "rpm --import /etc/pki/rpm-gpg/*")

    def _cleanup_install_repo(self):
        self._unbind_installer()
        self._unbind_repo_dir()
        # remove the tdnf cache directory.
        retval = self.cmd.run(['rm', '-rf', os.path.join(self.photon_root, "cache")])
        if retval != 0:
            self.logger.error("Fail to remove the cache")
        if os.path.exists(self.tdnf_conf_path):
            os.remove(self.tdnf_conf_path)
        if os.path.exists(self.tdnf_repo_path):
            os.remove(self.tdnf_repo_path)

    def _setup_grub(self):
        bootmode = self.install_config['bootmode']

        # Setup bios grub
        if bootmode == 'dualboot' or bootmode == 'bios':
            retval = self.cmd.run('grub2-install --target=i386-pc --force --boot-directory={} {}'.format(self.photon_root + "/boot", self.install_config['disk']))
            if retval != 0:
                retval = self.cmd.run(['grub-install', '--target=i386-pc', '--force',
                                   '--boot-directory={}'.format(self.photon_root + "/boot"),
                                   self.install_config['disk']])
                if retval != 0:
                    raise Exception("Unable to setup grub")

        # Setup efi grub
        if bootmode == 'dualboot' or bootmode == 'efi':
            esp_pn = '1'
            if bootmode == 'dualboot':
                esp_pn = '2'

            self.cmd.run(['mkdir', '-p', self.photon_root + '/boot/efi/boot/grub2'])
            with open(os.path.join(self.photon_root, 'boot/efi/boot/grub2/grub.cfg'), "w") as grub_cfg:
                grub_cfg.write("search -n -u {} -s\n".format(self._get_uuid(self.install_config['partitions_data']['boot'])))
                grub_cfg.write("set prefix=($root){}grub2\n".format(self.install_config['partitions_data']['bootdirectory']))
                grub_cfg.write("configfile {}grub2/grub.cfg\n".format(self.install_config['partitions_data']['bootdirectory']))

            if self.install_config['live']:
                arch = self.install_config['arch']
                # 'x86_64' -> 'bootx64.efi', 'aarch64' -> 'bootaa64.efi'
                exe_name = 'boot'+arch[:-5]+arch[-2:]+'.efi'
                # Some platforms do not support adding boot entry. Thus, ignore failures
                self.cmd.run(['efibootmgr', '--create', '--remove-dups', '--disk', self.install_config['disk'],
                              '--part', esp_pn, '--loader', '/EFI/BOOT/' + exe_name, '--label', 'Photon'])

        # Create custom grub.cfg
        retval = self.cmd.run(
            [self.setup_grub_command, self.photon_root,
             self.install_config['partitions_data']['root'],
             self.install_config['partitions_data']['boot'],
             self.install_config['partitions_data']['bootdirectory']])

        if retval != 0:
            raise Exception("Bootloader (grub2) setup failed")

    def _execute_modules(self, phase):
        """
        Execute the scripts in the modules folder
        """
        sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "modules")))
        modules_paths = glob.glob(os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) + '/m_*.py')
        for mod_path in modules_paths:
            module = os.path.splitext(os.path.basename(mod_path))[0]
            try:
                __import__(module)
                mod = sys.modules[module]
            except ImportError:
                self.logger.error('Error importing module {}'.format(module))
                continue

            # the module default is disabled
            if not hasattr(mod, 'enabled') or mod.enabled is False:
                self.logger.info("module {} is not enabled".format(module))
                continue
            # check for the install phase
            if not hasattr(mod, 'install_phase'):
                self.logger.error("Error: can not defind module {} phase".format(module))
                continue
            if mod.install_phase != phase:
                self.logger.info("Skipping module {0} for phase {1}".format(module, phase))
                continue
            if not hasattr(mod, 'execute'):
                self.logger.error("Error: not able to execute module {}".format(module))
                continue
            self.logger.info("Executing: " + module)
            mod.execute(self)

    def _adjust_packages_based_on_selected_flavor(self):
        """
        Install slected linux flavor only
        """
        redundant_linux_flavors = []
        def filter_packages(package):
            package = package.split('-')
            if len(package) > 1:
                flavor = package[1]
            else:
                flavor = ""
            if(package[0] != "linux"):
                return True
            elif("" in redundant_linux_flavors and flavor in self.linux_dependencies):
                return False
            elif(flavor in redundant_linux_flavors):
                return False
            else:
                return True

        for flavor in self.all_linux_flavors:
            if(flavor != self.install_config['linux_flavor']):
                flavor = flavor.split('-')
                if len(flavor) > 1:
                    flavor = flavor[1]
                else:
                    flavor = ""
                redundant_linux_flavors.append(flavor)
        self.install_config['packages'] = list(filter(filter_packages,self.install_config['packages']))

    def _add_packages_to_install(self, package):
        """
        Install packages on Vmware virtual machine if requested
        """
        self.install_config['packages'].append(package)

    def _setup_install_repo(self):
        """
        Setup the tdnf repo for installation
        """
        keepcache = False
        with open(self.tdnf_repo_path, "w") as repo_file:
            repo_file.write("[photon-local]\n")
            repo_file.write("name=VMWare Photon installer repo\n")
            if self.rpm_path.startswith("https://") or self.rpm_path.startswith("http://"):
                repo_file.write("baseurl={}\n".format(self.rpm_path))
            else:
                repo_file.write("baseurl=file://{}\n".format(self.rpm_cache_dir))
                keepcache = True
            repo_file.write("gpgcheck=0\nenabled=1\n")
        with open(self.tdnf_conf_path, "w") as conf_file:
            conf_file.writelines([
                "[main]\n",
                "gpgcheck=0\n",
                "installonly_limit=3\n",
                "clean_requirements_on_remove=true\n"])
            # baseurl and cachedir are bindmounted to rpm_path, we do not
            # want input RPMS to be removed after installation.
            if keepcache:
                conf_file.write("keepcache=1\n")
            conf_file.write("repodir={}\n".format(self.working_directory))
            conf_file.write("cachedir={}\n".format(self.rpm_cache_dir_short))

    def _install_additional_rpms(self):
        rpms_path = self.install_config.get('additional_rpms_path', None)

        if not rpms_path or not os.path.exists(rpms_path):
            return

        if self.cmd.run([ 'rpm', '--root', self.photon_root, '-U', rpms_path + '/*.rpm' ]) != 0:
            self.logger.info('Failed to install additional_rpms from ' + rpms_path)
            self.exit_gracefully()

    def _install_packages(self):
        """
        Install packages using tdnf command
        """
        self._adjust_packages_based_on_selected_flavor()
        selected_packages = self.install_config['packages']
        state = 0
        packages_to_install = {}
        total_size = 0
        stderr = None
        tdnf_cmd = "tdnf install --installroot {0} --assumeyes -c {1} {2}".format(self.photon_root,
                        self.tdnf_conf_path, " ".join(selected_packages))
        self.logger.debug(tdnf_cmd)

        # run in shell to do not throw exception if tdnf not found
        process = subprocess.Popen(tdnf_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        if self.install_config['ui']:
            while True:
                output = process.stdout.readline().decode()
                if output == '':
                    retval = process.poll()
                    if retval is not None:
                        stderr = process.communicate()[1]
                        break
                if state == 0:
                    if output == 'Installing:\n':
                        state = 1
                elif state == 1: #N A EVR Size(readable) Size(in bytes)
                    if output == '\n':
                        state = 2
                        self.progress_bar.update_num_items(total_size)
                    else:
                        info = output.split()
                        package = '{0}-{1}.{2}'.format(info[0], info[2], info[1])
                        packages_to_install[package] = int(info[5])
                        total_size += int(info[5])
                elif state == 2:
                    if output == 'Downloading:\n':
                        self.progress_bar.update_message('Preparing ...')
                        state = 3
                elif state == 3:
                    self.progress_bar.update_message(output)
                    if output == 'Running transaction\n':
                        state = 4
                else:
                    self.logger.info("[tdnf] {0}".format(output))
                    prefix = 'Installing/Updating: '
                    if output.startswith(prefix):
                        package = output[len(prefix):].rstrip('\n')
                        self.progress_bar.increment(packages_to_install[package])

                    self.progress_bar.update_message(output)
        else:
            stdout,stderr = process.communicate()
            self.logger.info(stdout.decode())
            retval = process.returncode
            # image creation. host's tdnf might not be available or can be outdated (Photon 1.0)
            # retry with docker container
            if retval != 0 and retval != 137:
                self.logger.error(stderr.decode())
                stderr = None
                self.logger.info("Retry 'tdnf install' using docker image")
                retval = self.cmd.run(['docker', 'run',
                                   '-v', self.rpm_cache_dir+':'+self.rpm_cache_dir,
                                   '-v', self.working_directory+':'+self.working_directory,
                                   'photon:3.0', '/bin/sh', '-c', tdnf_cmd])

        # 0 : succeed; 137 : package already installed; 65 : package not found in repo.
        if retval != 0 and retval != 137:
            self.logger.error("Failed to install some packages")
            if stderr:
                self.logger.error(stderr.decode())
            self.exit_gracefully()

    def _eject_cdrom(self):
        """
        Eject the cdrom on request
        """
        if self.install_config.get('eject_cdrom', True):
            self.cmd.run(['eject', '-r'])

    def _enable_network_in_chroot(self):
        """
        Enable network in chroot
        """
        if os.path.exists("/etc/resolv.conf"):
            shutil.copy("/etc/resolv.conf", self.photon_root + '/etc/.')

    def _disable_network_in_chroot(self):
        """
        disable network in chroot
        """
        if os.path.exists(self.photon_root + '/etc/resolv.conf'):
            os.remove(self.photon_root + '/etc/resolv.conf')

    def partition_compare(self, p):
        if 'mountpoint' in p:
            return (1, len(p['mountpoint']), p['mountpoint'])
        return (0, 0, "A")

    def _get_partition_path(self, disk, part_idx):
        prefix = ''
        if 'nvme' in disk or 'mmcblk' in disk or 'loop' in disk:
            prefix = 'p'

        # loop partitions device names are /dev/mapper/loopXpY instead of /dev/loopXpY
        if 'loop' in disk:
            path = '/dev/mapper' + disk[4:] + prefix + repr(part_idx)
        else:
            path = disk + prefix + repr(part_idx)

        return path

    def _get_partition_type(self, partition):
        if partition['filesystem'] == 'bios':
            return PartitionType.BIOS
        if partition['filesystem'] == 'swap':
            return PartitionType.SWAP
        if partition.get('mountpoint', '') == '/boot/efi' and partition['filesystem'] == 'vfat':
            return PartitionType.ESP
        if partition.get('lvm', None):
            return PartitionType.LVM
        return PartitionType.LINUX

    def _partition_type_to_string(self, ptype):
        if ptype == PartitionType.BIOS:
            return 'ef02'
        if ptype == PartitionType.SWAP:
            return '8200'
        if ptype == PartitionType.ESP:
            return 'ef00'
        if ptype == PartitionType.LVM:
            return '8e00'
        if ptype == PartitionType.LINUX:
            return '8300'
        raise Exception("Unknown partition type: {}".format(ptype))

    def _create_logical_volumes(self, physical_partition, vg_name, lv_partitions, extensible):
        """
        Create logical volumes
        """
        #Remove LVM logical volumes and volume groups if already exists
        #Existing lvs & vg should be removed to continue re-installation
        #else pvcreate command fails to create physical volumes even if executes forcefully
        retval = self.cmd.run(['bash', '-c', 'pvs | grep {}'. format(vg_name)])
        if retval == 0:
            #Remove LV's associated to VG and VG
            retval = self.cmd.run(["vgremove", "-f", vg_name])
            if retval != 0:
                self.logger.error("Error: Failed to remove existing vg before installation {}". format(vg_name))
        # if vg is not extensible (all lvs inside are known size) then make last lv
        # extensible, i.e. shrink it. Srinking last partition is important. We will
        # not be able to provide specified size because given physical partition is
        # also used by LVM header.
        extensible_logical_volume = None
        if not extensible:
            extensible_logical_volume = lv_partitions[-1]
            extensible_logical_volume['size'] = 0

        # create physical volume
        command = ['pvcreate', '-ff', '-y', physical_partition]
        retval = self.cmd.run(command)
        if retval != 0:
            raise Exception("Error: Failed to create physical volume, command : {}".format(command))

        # create volume group
        command = ['vgcreate', vg_name, physical_partition]
        retval = self.cmd.run(command)
        if retval != 0:
            raise Exception("Error: Failed to create volume group, command = {}".format(command))

        # create logical volumes
        for partition in lv_partitions:
            lv_cmd = ['lvcreate', '-y']
            lv_name = partition['lvm']['lv_name']
            size = partition['size']
            if partition['size'] == 0:
                # Each volume group can have only one extensible logical volume
                if not extensible_logical_volume:
                    extensible_logical_volume = partition
            else:
                lv_cmd.extend(['-L', '{}M'.format(partition['size']), '-n', lv_name, vg_name ])
                retval = self.cmd.run(lv_cmd)
                if retval != 0:
                    raise Exception("Error: Failed to create logical volumes , command: {}".format(lv_cmd))
            partition['path'] = '/dev/' + vg_name + '/' + lv_name

        # create extensible logical volume
        if not extensible_logical_volume:
            raise Exception("Can not fully partition VG: " + vg_name)

        lv_name = extensible_logical_volume['lvm']['lv_name']
        lv_cmd = ['lvcreate', '-y']
        lv_cmd.extend(['-l', '100%FREE', '-n', lv_name, vg_name ])

        retval = self.cmd.run(lv_cmd)
        if retval != 0:
            raise Exception("Error: Failed to create extensible logical volume, command = {}". format(lv_cmd))

        # remember pv/vg for detaching it later.
        self.lvs_to_detach['pvs'].append(os.path.basename(physical_partition))
        self.lvs_to_detach['vgs'].append(vg_name)

    def _get_partition_tree_view(self):
        # Tree View of partitions list, to be returned.
        # 1st level: dict of disks
        # 2nd level: list of physical partitions, with all information necessary to partition the disk
        # 3rd level: list of logical partitions (LVM) or detailed partition information needed to format partition
        ptv = {}

        # Dict of VG's per disk. Purpose of this dict is:
        # 1) to collect its LV's
        # 2) to accumulate total size
        # 3) to create physical partition representation for VG
        vg_partitions = {}

        default_disk = self.install_config['disk']
        partitions = self.install_config['partitions']
        for partition in partitions:
            disk = partition.get('disk', default_disk)
            if disk not in ptv:
                ptv[disk] = []
            if disk not in vg_partitions:
                vg_partitions[disk] = {}

            if partition.get('lvm', None):
                vg_name = partition['lvm']['vg_name']
                if vg_name not in vg_partitions[disk]:
                    vg_partitions[disk][vg_name] = {
                        'size': 0,
                        'type': self._partition_type_to_string(PartitionType.LVM),
                        'extensible': False,
                        'lvs': [],
                        'vg_name': vg_name
                    }
                vg_partitions[disk][vg_name]['lvs'].append(partition)
                if partition['size'] == 0:
                    vg_partitions[disk][vg_name]['extensible'] = True
                    vg_partitions[disk][vg_name]['size'] = 0
                else:
                    if not vg_partitions[disk][vg_name]['extensible']:
                        vg_partitions[disk][vg_name]['size'] = vg_partitions[disk][vg_name]['size'] + partition['size']
            else:
                if 'type' in partition:
                    ptype_code = partition['type']
                else:
                    ptype_code = self._partition_type_to_string(self._get_partition_type(partition))

                l2entry = {
                    'size': partition['size'],
                    'type': ptype_code,
                    'partition': partition
                }
                ptv[disk].append(l2entry)

        # Add accumulated VG partitions
        for disk, vg_list in vg_partitions.items():
                ptv[disk].extend(vg_list.values())
        return ptv

    def _insert_boot_partitions(self):
        bios_found = False
        esp_found = False
        for partition in self.install_config['partitions']:
            ptype = self._get_partition_type(partition)
            if ptype == PartitionType.BIOS:
                bios_found = True
            if ptype == PartitionType.ESP:
                esp_found = True

       # Adding boot partition required for ostree if already not present in partitions table
        if 'ostree' in self.install_config:
            mount_points = [partition['mountpoint'] for partition in self.install_config['partitions'] if 'mountpoint' in partition]
            if '/boot' not in mount_points:
                boot_partition = {'size': 300, 'filesystem': 'ext4', 'mountpoint': '/boot'}
                self.install_config['partitions'].insert(0, boot_partition)

        bootmode = self.install_config.get('bootmode', 'bios')

        # Insert efi special partition
        if not esp_found and (bootmode == 'dualboot' or bootmode == 'efi'):
            efi_partition = { 'size': 10, 'filesystem': 'vfat', 'mountpoint': '/boot/efi' }
            self.install_config['partitions'].insert(0, efi_partition)

        # Insert bios partition last to be very first
        if not bios_found and (bootmode == 'dualboot' or bootmode == 'bios'):
            bios_partition = { 'size': 4, 'filesystem': 'bios' }
            self.install_config['partitions'].insert(0, bios_partition)

    def _partition_disk(self):
        """
        Partition the disk
        """

        if self.install_config['ui']:
            self.progress_bar.update_message('Partitioning...')

        self._insert_boot_partitions()
        ptv = self._get_partition_tree_view()

        partitions = self.install_config['partitions']
        partitions_data = {}
        lvm_present = False

        # Partitioning disks
        for disk, l2entries in ptv.items():

            # Clear the disk first
            retval = self.cmd.run(['sgdisk', '-o', '-g', disk])
            if retval != 0:
                raise Exception("Failed clearing disk {0}".format(disk))

            # Build partition command and insert 'part' into 'partitions'
            partition_cmd = ['sgdisk']
            part_idx = 1
            # command option for extensible partition
            last_partition = None
            for l2 in l2entries:
                if 'lvs' in l2:
                    # will be used for _create_logical_volumes() invocation
                    l2['path'] = self._get_partition_path(disk, part_idx)
                else:
                    l2['partition']['path'] = self._get_partition_path(disk, part_idx)

                if l2['size'] == 0:
                    last_partition = []
                    last_partition.extend(['-n{}'.format(part_idx)])
                    last_partition.extend(['-t{}:{}'.format(part_idx, l2['type'])])
                else:
                    partition_cmd.extend(['-n{}::+{}M'.format(part_idx, l2['size'])])
                    partition_cmd.extend(['-t{}:{}'.format(part_idx, l2['type'])])
                part_idx = part_idx + 1
            # if extensible partition present, add it to the end of the disk
            if last_partition:
                partition_cmd.extend(last_partition)
            partition_cmd.extend(['-p', disk])

            # Run the partitioning command (all physical partitions in one shot)
            retval = self.cmd.run(partition_cmd)
            if retval != 0:
                raise Exception("Failed partition disk, command: {0}".format(partition_cmd))

            # For RPi image we used 'parted' instead of 'sgdisk':
            # parted -s $IMAGE_NAME mklabel msdos mkpart primary fat32 1M 30M mkpart primary ext4 30M 100%
            # Try to use 'sgdisk -m' to convert GPT to MBR and see whether it works.
            if self.install_config.get('partition_type', 'gpt') == 'msdos':
                # m - colon separated partitions list
                m = ":".join([str(i) for i in range(1,part_idx)])
                retval = self.cmd.run(['sgdisk', '-m', m, disk])
                if retval != 0:
                    raise Exception("Failed to setup efi partition")

            # Make loop disk partitions available
            if 'loop' in disk:
                retval = self.cmd.run(['kpartx', '-avs', disk])
                if retval != 0:
                    raise Exception("Failed to rescan partitions of the disk image {}". format(disk))

            # Go through l2 entries again and create logical partitions
            for l2 in l2entries:
                if 'lvs' not in l2:
                    continue
                lvm_present = True
                self._create_logical_volumes(l2['path'], l2['vg_name'], l2['lvs'], l2['extensible'])

        if lvm_present:
            # add lvm2 package to install list
            self._add_packages_to_install('lvm2')

        # Create partitions_data (needed for mk-setup-grub.sh)
        for partition in partitions:
            if "mountpoint" in partition:
                if partition['mountpoint'] == '/':
                    partitions_data['root'] = partition['path']
                elif partition['mountpoint'] == '/boot':
                    partitions_data['boot'] = partition['path']
                    partitions_data['bootdirectory'] = '/'

        # If no separate boot partition, then use /boot folder from root partition
        if 'boot' not in partitions_data:
            partitions_data['boot'] = partitions_data['root']
            partitions_data['bootdirectory'] = '/boot/'

        # Sort partitions by mountpoint to be able to mount and
        # unmount it in proper sequence
        partitions.sort(key=lambda p: self.partition_compare(p))

        self.install_config['partitions_data'] = partitions_data

    def _format_partitions(self):
        partitions = self.install_config['partitions']
        self.logger.info(partitions)

        # Format the filesystem
        for partition in partitions:
            ptype = self._get_partition_type(partition)
            # Do not format BIOS boot partition
            if ptype == PartitionType.BIOS:
                continue
            if ptype == PartitionType.SWAP:
                mkfs_cmd = ['mkswap']
            else:
                mkfs_cmd = ['mkfs', '-t', partition['filesystem']]

            if 'fs_options' in partition:
                options = re.sub("[^\S]", " ", partition['fs_options']).split()
                mkfs_cmd.extend(options)

            mkfs_cmd.extend([partition['path']])
            retval = self.cmd.run(mkfs_cmd)

            if retval != 0:
                raise Exception(
                    "Failed to format {} partition @ {}".format(partition['filesystem'],
                                                         partition['path']))

    def getfile(self, filename):
        """
        Returns absolute filepath by filename.
        """
        for dirname in self.install_config['search_path']:
            filepath = os.path.join(dirname, filename)
            if os.path.exists(filepath):
                return filepath
        raise Exception("File {} not found in the following directories {}".format(filename, self.install_config['search_path']))

