#!/usr/bin/env bash
set -Eeuo pipefail


# Constants
readonly CURR_DIR="$(dirname "$(readlink -f "$BASH_SOURCE")")"
readonly TMP_DIR="${TMP_DIR:-/tmp}"
QEMU_OPTS=( )


# Main function
main () {
    get_os
    get_arch
    set_defaults
    get_opts "$@"
    set_binaries
    qemu_opt_ignition
    qemu_opt_pxe
    qemu_opt_disk
    qemu_opt_uefi
    qemu_opt_cpu
    qemu_opt_memory
    qemu_opt_monitor
    qemu_opt_remote_control
    qemu_opt_network
    qemu_opt_tpm
    qemu_opt_misc
    qemu_status
    qemu_execute
}


# Evaluate operating system
get_os () {
    # Obtain the currently used OS
    host_os="$(uname -s)"

    if [ $host_os = "Darwin" ]; then
        os="macos"
    elif [ $host_os = "Linux" ]; then
        os="linux"

        # Validate if this process is running inside a
        # a container (like Docker) by checking what
        # init process is used and set a default OS in
        # case it is not systemd and a container then.
        # Otherwise, If this is nativly running
        # on a host, evaluate the running OS
        init_system="$(cat /proc/1/sched | awk 'NR==1{print $1}')"
        if [ $init_system != "systemd" ]; then
            host_dist="debian"
        else
            host_dist="$(hostnamectl | grep "Operating System:" | awk '{print tolower($3)}')"
        fi

        # Evaluate Distribution
        if [ "$host_dist" = "centos" ]; then
            os="centos"
        elif [ "$host_dist" = "fedora" ]; then
            os="fedora"
        elif [ "$host_dist" = "debian" ] || [ "$host_dist" = "garden" ]; then
            os="debian"
        elif [ "$host_dist" = "ubuntu" ]; then
            os="ubuntu"
        elif [ "$host_dist" = "arch" ] || [ "$host_dist" = "manjaro" ]; then
            os="arch"
        elif [ "$host_dist" = "nixos" ]; then
            os="nixos"
        elif [ "$host_dist" = "opensuse" ]; then
            os="opensuse"
        else
            echo "Error: Unsupported Linux distribution [$host_dist]."
            exit 1
        fi

    else
        echo "Error: Unsupported operating system."
        exit 1
    fi
}


# Evaluate system architecture
get_arch () {
    # Include "get_arch.sh" for only maintaining a single
    # file that evaluates the current used arch
    arch=$(${CURR_DIR}/get_arch.sh)
    # Create an immutable var including the original
    # system architecture of the host system
    readonly arch_orig=$arch
}


# Set default values
set_defaults () {
    # Set CRE
    gardenlinux_build_cre=${GARDENLINUX_BUILD_CRE:-"sudo podman"}

    # Set default vars
    cpu="2"
    memory="2Gi"
    mac="$(printf '02:%02x:%02x:%02x:%02x:%02x' $((RANDOM%256)) $((RANDOM%256)) $((RANDOM%256)) $((RANDOM%256)) $((RANDOM%256)))"
    mac="$(printf "%012s" | tr -d ":" <<< ${mac,,})"
    macfull="$(sed 's/../&:/g;s/:$//' <<< $mac)"
    uuid="12345678-0000-0000-0000-${mac}"
    monitor=1
    bridge=
    daemonize=
    pxe=
    pxe_binary=
    pxefile=${CURR_DIR}/../examples/ipxe/start-vm.ipxe
    ignfile=
    pidfile=
    uefi=1
    uefi_code=
    uefi_vars=
    tpm=1
    vnc=
    keypress=1
    port_dest=22
    port_base=2223
    vnc_base=5900
    tpm2=

    # Evaluate free and usable ports
    if [ $os != "macos" ]; then
        # Set port(s) on Linux based systems
        port=$port_base; while ss -tul | grep :$port &> /dev/null; do (( ++port )); done
        vnc_port=0; while ss -tul | grep :$(( vnc_port + $vnc_base )) &> /dev/null; do (( ++vnc_port )); done
    else
        # Set port(s) on macOS based systems
        port=$port_base; while netstat -an -ptcp | grep LISTEN | awk {'print $4'} | grep :$port &> /dev/null; do (( ++port )); done
        vnc_port=0; while netstat -an -ptcp | grep LISTEN | awk {'print $4'} | grep :$(( vnc_port + $vnc_base )) &> /dev/null; do (( ++vnc_port )); done
    fi

    # Set QEMU defaults
    # Run as user "nobody" when started as root user
    if [ $(id -u) == 0 ]; then
        QEMU_OPTS+=("-runas nobody")
    fi

    # Do not use defaults on QEMU
    QEMU_OPTS+=("-nodefaults")
    # UUID is expected by systemd and Garden Linux
    QEMU_OPTS+=("-uuid $uuid" )
}


# Get (external) options that are
# passed to this script
get_opts () {
source "${CURR_DIR}/.constants.sh" \
    --flags 'daemonize,uefi,legacy-bios,skipkp,vnc,tpm2' \
    --flags 'cpu:,mem:,bridge:,port:,mac:,arch:' \
    --flags 'pxe:' \
    --flags 'ignfile:' \
    --flags 'pidfile:' \
    --flags 'ueficode:,uefivars:' \
    --flags 'destport:' \
    --usage '[ --daemonize ] [ --cpu:<#> ] [ --mem:<size> ] [ --pxe:<dir> ] [<image file>[,<size>]]*' \
    --sample '.build/rootfs.raw' \
    --sample ',1G ubuntu-16.04.7-server-amd64.iso' \
    --help  "Starts a virtual machine using qemu. Made for running Garden Linux vms for testing. On Linux hosts, it should be able to use kvm if possible.

--cpu       <int>         Number of vCPUs to start the VM. (default: $cpu)
--arch      <platform>    Set the emulated platform. i386, x86_64, x86_64-microvm, arm, aarch64 are supported. (default: $arch)
--mem       <int>         Memory provided to the virtual machine (default: $memory) in MB if unit is omitted.
--uefi      <bool>        Boot with uefi bios enabled, needs \`apt-get install ovmf\`. (default: yes)
--legacy-bios <bool>        Boot with uefi bios disabled, can't be used with uefi. (default: no)
--ueficode  <path>        Defines the uefi code used. The file is readonly. (default: $uefi_code)
--uefivars  <path>        Defines the uefi variables used. The file will be !modified! if variables are set. (default: $uefi_vars)
--port      <int>         Specifies the local ssh port. This port is mounted to the running machine, not if --bridge is specified. (default: $port_base)
--destport  <int>         Specifies the remote ssh port. This port is used to connect, to a specific sshd. (default: $port_dest)
--mac       <macaddr>     The mac address is usually randomized. It is used to identify the monitoring port, the mac address of the machine and the UUID. Can be set to this value.
--pxe       <bool>        Enables pxe boot on the vm. Minimum one image file must be a directory.
--ignfile   <path>        Provide an ignition file when pxe booting. Can be used standalone.
--pidfile   <path>        Provide a file where to store the pid of the started qemu vm.
--daemonize <bool>        Start the virtual machine in background, console deactivated (default: no).
--skipkp    <bool>        Skip the key press to the verify status before execute.
--vnc       <bool>        Switches from serial console to vnc graphics console / enables vnc in --daemonize.
--tpm2      <bool>        Enable TPM2 emulation and QEMU passthrough using swtpm.
<image file>              A file containing the image to boot. Format is determined by the extension (raw, vdi, vpc, vhd, vhdx, vmdk, qcow2, iso).
                          An optional suffix of the form <image file>,qcow=<size> can be specified to write changes to a copy-on-write device instead"

eval "$dgetopt"
while true; do
	flag="$1"; shift
	dgetopt-case "$flag"
	case "$flag" in
        --cpu)          cpu="$1";       shift ;;
        --arch)         arch="$1";      shift ;;
        --mem)          memory="$1";    shift ;;
        --daemonize)    daemonize=1;    keypress=; ;;
        --vnc)          vnc=1;                ;;
        --uefi)         uefi=1;               ;;
        --legacy-bios)  uefi=;                ;;
        --mac)          mac="$1";       shift ;;
        --ueficode)     uefi_code="$1"; shift ;;
        --uefivars)     uefi_vars="$1"; shift ;;
        --bridge)       bridge="$1";    shift ;;
        --pxe )         pxe="$(realpath $1)";      shift ;;
        --ignfile )     ignfile="$(realpath $1)";  shift ;;
        --pidfile)      pidfile="$(realpath $1)";  shift ;;
        --port)         port="$1";      shift ;;
        --destport)     port_dest="$1"; shift ;;
        --skipkp)       keypress=;            ;;
        --tpm2)         tpm2=1;               ;;
        --) break ;;
        *) eusage "Unknown flag '$flag'"      ;;
    esac
done

# Get positional args in a dedicated var for
# processing positional arguments in "qemu_opt_disk"
images=$@
}


# Set dedicated binaries for supported
# Operating Systems
set_binaries () {
    # Retranslate the architecture to match
    # the QEMU binaries if needed and support legacy inputs
    if [ $arch = "amd64" ] || [ $arch = "x86_64" ]; then
        # Depending on the used distribution QEMU may ship
        # architecture related binaries
        qemu_arch="x86_64"
        arch="amd64"
    elif [ $arch = "arm64" ] || [ $arch = "aarch64" ]; then
        # Depending on the used distribution QEMU may ship
        # architecture related binaries
        qemu_arch="aarch64"
        arch="arm64"
    else
        echo "Error: Unsupported architecture $arch"
        exit 1
    fi

    # Binaries
    bin_head="head"
    bin_stat="stat"
    bin_sed="sed"
    bin_uuid="cat /proc/sys/kernel/random/uuid"
    bin_realpath="realpath"
    bin_qemu="qemu-system-$qemu_arch"

    # CentOS specific binaries
    if [ $os = "centos" ]; then
        # Binary naming: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html-single/8.0_release_notes/index#virtualization
        bin_qemu="/usr/libexec/qemu-kvm"
    fi

    # macOS specific binaries
    if [ $os = "macos" ]; then
        bin_head="ghead"
        bin_stat="gstat"
        bin_sed="gsed"
        bin_uuid="/usr/bin/uuidgen"
        bin_realpath="grealpath"
    fi
}


# Setup ignition to prepare PXE boot or standalone ignition usage
qemu_opt_ignition () {
    if [ $pxe ]; then

        # Check for ignition file
        if [ "$ignfile" ]; then
            pxefile=$CURR_DIR/../examples/ipxe/start-vm-ignition.ipxe
        fi

        [ -e "$CURR_DIR/$mac.ipxe" ] || cp $pxefile $CURR_DIR/$mac.ipxe

        glbuilds=()
        glbuild=""

        # Get all vmlinuz files
        for v in $(find "$pxe" -type f -name '*.vmlinuz' -exec basename {} \; | cut -d. -f 1); do
            glbuilds+=(${v})
        done

        if [ "${#glbuilds[@]}" == "0" ]; then
            echo "Error: No vmlinuz found!" 1>&2
            exit 1
        elif [ "${#glbuilds[@]}" == "1" ]; then
            glbuild="${glbuilds[0]}"
        else
            echo "Error: Multiple builds found, which one should be used?"

            for i in ${!glbuilds[@]}; do
                echo "[$i] ${glbuilds[i]}"
            done

            echo -n "ENTER entry number : "
            read n

            if [[ ! "$n" =~ ^[0-9]+$ ]] ; then
                echo "Error: Not a valid entry number"
                exit 1
            fi

            if [[ "$n" -gt "${#glbuilds[@]}" ]]; then
                echo "Error: Not a valid entry"
                exit 1
            fi

            glbuild="${glbuilds[${n}]}"

        fi

        # modify the boot.ipxe to load the proper kernel and initramfs
        ${bin_sed} -i "s/PATHGOESHERE//g;s/IPADDRESSGOESHERE/10.0.2.2/g" $CURR_DIR/$mac.ipxe
        QEMU_OPTS+=( -boot order=nc )
    else
	# add neccessary ignfile config for qemu start
	# https://docs.fedoraproject.org/en-US/fedora-coreos/provisioning-qemu/#_setting_up_a_new_vm
	if [ "$ignfile" ]; then
	    QEMU_OPTS+=( -fw_cfg name=opt/com.coreos/config,file=${ignfile} )
	fi
    fi
}


# Validate and set the defined PXE options
# and spawn a helper container within a CRE
qemu_opt_pxe () {
    if [ "$pxe" ] && [ "$uefi" ]; then
        pxe_binary="$pxe"
        pxe=
    fi

    if [ $pxe ]; then
        container_name=$($bin_uuid)
        function stop(){
            rm  "$pxe/root."{vmlinuz,initrd,squashfs}
            ${gardenlinux_build_cre} stop -t 0 $1
            [[ ! -s "$pxe/ignition.json" ]] && rm -f "$pxe/ignition.json"
            echo "INFO: PXE container stopped."
        }
        echo

        # Link VMLinuz
        if [[ -f "${pxe}/${glbuild}.vmlinuz" ]]; then
            ln -sf "$glbuild.vmlinuz" "$pxe/root.vmlinuz"
        else
            echo "Error: Missing ${glbuild}.vmlinuz, exiting"; exit 1
        fi

        # Link Initrd
        if [[ -f "${pxe}/${glbuild}.initrd" ]]; then
            ln -sf "$glbuild.initrd" "$pxe/root.initrd"
        else
            echo "Error: Missing ${glbuild}.initrd, exiting"; exit 1
        fi

        # Link squashFS
        if [[ -f "${pxe}/${glbuild}.squashfs" ]]; then
            ln -sf "$glbuild.squashfs" "$pxe/root.squashfs"
        else
            echo "Error: Missing ${glbuild}.squashfs, exiting"; exit 1
        fi

        trap 'stop $container_name' EXIT
        echo "INFO: PXE container starting."

        if [ "$ignfile" ]; then
            ${gardenlinux_build_cre} run -it --rm -d -p 127.0.0.1:8888:80 --name ${container_name} -v ${pxe}:/usr/share/nginx/html -v ${ignfile}:/usr/share/nginx/html/ignition.json nginx
        else
            ${gardenlinux_build_cre} run -it --rm -d -p 127.0.0.1:8888:80 --name ${container_name} -v ${pxe}:/usr/share/nginx/html:ro nginx
        fi

        if [ "$pxe_binary" ]; then
            cp "$pxe_binary" "$CURR_DIR/$mac.efi"
            QEMU_OPTS+=( -boot order=nc )
        fi

    fi
}


# Validate and set the defined image disk to
# start
qemu_opt_disk () {
    inflate_list=( )
    disk_count=0

    if [ "$arch" = "amd64" ]; then
        [ "$images" == "" ] || QEMU_OPTS+=("-device virtio-scsi-pci,id=scsi0")
    fi

    # Iterate over all user defined images / disks
    # and add them to QEMU
    for i in $images; do
        image_file=$i
        image_size=0
        image_direct=""
        use_qcow=0
        if [[ $image_file == *","* ]]; then
            image_size=${image_file##*,}
            image_file=${image_file%%,$image_size}
            if [[ "$image_size" == 'qcow='* ]]; then
                image_size=${image_size#'qcow='}
                use_qcow=1
            fi
        fi

        # If no filename is left point to tmp
        [ "$image_file" == "" ]    &&  image_file=$(mktemp --suff=.raw)
        [[ $image_file == *"."* ]] &&  image_ext=${image_file##*.};
        [ -e $image_file ] || eusage "Error: File \"$image_file\" does not exist."

        # Calculate in bytes
        image_size_bytes=$(numfmt --from=iec --suffix="B" --format "%f" ${image_size} | $bin_head -c-2)
        image_ext=${image_ext/^vhd$/vpc}

        # Validate for .iso file as CD rom image
        if [ "$image_ext" == "iso" ]; then
            QEMU_OPTS+=(	"-drive media=cdrom,file=$image_file,readonly" )
        elif [ -d $image_file ]; then
            targetDir=$image_file
        elif [ "$use_qcow" = 1 ]; then
            qcow_relative_path=$("$bin_realpath" --relative-to="$CURR_DIR" "$image_file")
            qemu-img create -f qcow2 -F "$image_ext" -b "$qcow_relative_path" "$CURR_DIR/$mac.qcow2" "$image_size_bytes"
            if [ "$arch" = amd64 ]; then
                QEMU_OPTS+=("-device scsi-hd,drive=drive$disk_count,bus=scsi0.0"
                            "-drive format=qcow2,if=none,id=drive$disk_count,file=$CURR_DIR/$mac.qcow2")
            else
                QEMU_OPTS+=("-drive if=virtio,format=qcow2,file=$CURR_DIR/$mac.qcow2")
            fi
            (( ++disk_count ))
        else
            # Test if direct access is possible and image is not already in use
            dd if=$image_file of=/dev/null count=1 iflag=direct 2> /dev/null && imagedirect="aio=native,cache.direct=on,"

            # if there is a bigger size, we need to inflate
            [ $($bin_stat --printf="%s" $image_file) -lt ${image_size_bytes} ] && inflate_list+=( "${image_file},${image_size_bytes}" )
            if [ "$arch" = amd64 ]; then
                QEMU_OPTS+=("-device scsi-hd,drive=drive${disk_count},bus=scsi0.0"
                            "-drive format=${image_ext},if=none,discard=unmap,${image_direct}id=drive${disk_count},file=${image_file}")
            else
                QEMU_OPTS+=("-drive if=virtio,format=${image_ext},file=$image_file")
            fi
            (( ++disk_count ))
        fi
    done

    [ $disk_count -gt 0 -o "$pxe" -o "$pxe_binary" ] || eusage "Error: Missing bootdisk. boot via --pxe, provide tmpdisk via --disk or provide bootdisk image file."

    # inflating selected files
    for i in "${inflate_list[@]}"; do
        dd if=/dev/zero of=$(cut -d, -f1 <<< "$i") count=0 bs=1 seek=$(cut -d, -f2 <<< "$i") 2> /dev/null
    done
}


# Validate and set UEFI related options
qemu_opt_uefi () {
    # Evaluate UEFI code and vars files for realted architecture and OS
    if [ "$arch" = amd64 ]; then
        # UEFI is optional on x86_64
        if [ $uefi ]; then
            if [ "$os" = macos ]; then
                # Test for HomeBrew path
                homebrew_prefix=$(brew --prefix qemu)
                uefi_code="${uefi_code:-$homebrew_prefix/share/qemu/edk2-x86_64-code.fd}"
                uefi_vars="${uefi_vars:-$homebrew_prefix/share/qemu/edk2-i386-vars.fd}"
            elif [ "$os" = debian ] || [ "$os" = ubuntu ] ; then
		local usz=""
		if [ ! -f /usr/share/OVMF/OVMF_CODE.fd ] || [ ! -f /usr/share/OVMF/OVMF_VARS.fd ]; then
                    usz="_4M"
                fi
                uefi_code="${uefi_code:-/usr/share/OVMF/OVMF_CODE${usz}.fd}"
                uefi_vars="${uefi_vars:-/usr/share/OVMF/OVMF_VARS${usz}.fd}"
            elif [ "$os" = centos ] || [ "$os" = fedora ] ; then
                uefi_code="${uefi_code:-/usr/share/OVMF/OVMF_CODE.secboot.fd}"
                uefi_vars="${uefi_vars:-/usr/share/OVMF/OVMF_VARS.secboot.fd}"
            elif [ "$os" = arch ]; then
                uefi_code="${uefi_code:-/usr/share/OVMF/x64/OVMF_CODE.fd}"
                uefi_vars="${uefi_vars:-/usr/share/OVMF/x64/OVMF_VARS.fd}"
            elif [ "$os" = nixos ]; then
                # Requires the following NixOS configuration:
                # virtualisation.libvirtd.enable = true;
                uefi_code="${uefi_code:-/run/libvirt/nix-ovmf/OVMF_CODE.fd}"
                uefi_vars="${uefi_vars:-/run/libvirt/nix-ovmf/OVMF_VARS.fd}"
            elif [ "$os" = opensuse ]; then
                uefi_code="${uefi_code:-/usr/share/qemu/ovmf-x86_64.bin}"
                uefi_vars="${uefi_vars:-/usr/share/qemu/ovmf-x86_64-vars.bin}"
            else
                echo "Error: Could not find UEFI code and vars file."
                exit 1
            fi
        fi

    elif [ "$arch" = arm64 ]; then
        # Always enable UEFI mode on aarch64
        if [[ -z "$uefi" ]]; then
            echo "WARNING: --legacy-boot is not supported on AARCH64. Enabling UEFI."
        fi
        uefi=1

        # Obtain UEFI code and vars from
        # Homebrew path on macOS
        if [ "$os" = macos ]; then
            # Test for HomeBrew path
            homebrew_prefix=$(brew --prefix qemu)
            uefi_code="${uefi_code:-$homebrew_prefix/share/qemu/edk2-aarch64-code.fd}"
            uefi_vars="${uefi_vars:-$homebrew_prefix/share/qemu/edk2-arm-vars.fd}"
        elif [ "$os" = debian ] || [ "$os" = ubuntu ] ; then
            uefi_code="${uefi_code:-/usr/share/AAVMF/AAVMF_CODE.fd}"
            uefi_vars="${uefi_vars:-/usr/share/AAVMF/AAVMF_VARS.fd}"
        elif [ "$os" = centos ] || [ "$os" = fedora ] ; then
            # Note: Running `AARCH64` on `x86_64` requires `qemu-system-aarch64`
            # package which is not present in official repositories
            uefi_code="${uefi_code:-/usr/share/AAVMF/AAVMF_CODE.fd}"
            uefi_vars="${uefi_vars:-/usr/share/AAVMF/AAVMF_VARS.fd}"
	elif [ "$os" = arch ]; then
            # Note: Running `AARCH64` on `x86_64` requires `qemu-system-aarch64` and `edk2-armvirt`
            # package which is *present* in official repositories
            uefi_code="${uefi_code:-/usr/share/AAVMF/AAVMF_CODE.fd}"
            uefi_vars="${uefi_vars:-/usr/share/AAVMF/AAVMF_VARS.fd}"
        elif [ "$os" = nixos ]; then
            # Requires the following NixOS configuration to have AArch64 UEFI firmware on x86 hosts:
            #
            # virtualisation.libvirtd.qemu.ovmf.packages = [
            #   pkgs.OVMFFull.fd
            #   pkgs.pkgsCross.aarch64-multiplatform.OVMF.fd
            # ];
            uefi_code="${uefi_code:-/run/libvirt/nix-ovmf/AAVMF_CODE.fd}"
            uefi_vars="${uefi_vars:-/run/libvirt/nix-ovmf/AAVMF_VARS.fd}"
        else
            echo "Error: Could not find UEFI code and vars file."
            exit 1
        fi

    fi

    # Define UEFI options for QEMU
    if [ $uefi ]; then

        [ -r $uefi_code ] || eusage "Missing uefi code at $uefi_code.\n Run: apt-get install ovmf qemu-efi-aarch64"
        [ -r $uefi_vars ] || eusage "Missing uefi vars at $uefi_vars.\n Run: apt-get install ovmf qemu-efi-aarch64"

        # The source file might be read-only.
        [ -e $CURR_DIR/$mac.vars ] || cat "$uefi_vars" > "$CURR_DIR"/"$mac".vars

        # Add support for amd64 architecture
        if [ "$arch" = amd64 ]; then
            QEMU_OPTS+=("-global driver=cfi.pflash01,property=secure,value=on")
        fi

        # Set UEFI code and vars files
	    QEMU_OPTS+=("-drive if=pflash,format=raw,unit=0,file=${uefi_code},readonly=on"
                    "-drive if=pflash,format=raw,unit=1,file=$CURR_DIR/$mac.vars")

    fi
}


# Validate and set the CPU architecture
# cores and chipsets
qemu_opt_cpu () {
    # Set CPU count
    QEMU_OPTS+=("-smp $cpu")

    # Validate amd64 arch
    if [ "$arch" = amd64 ]; then

        # Validate for native or emulated arch
        if [ "$arch" = "$arch_orig" ]; then

            # Running on native arch
            # Check for acceleration support
            if [ "$os" = macos ]; then
                # macOS HVF support is always given on all new Intel Macs
                # that are supported by the current HomeBrew version
                QEMU_OPTS+=("-cpu host" "-accel hvf")

            else

                # Validate for KVM support on AMD64
                # If no KVM support is present QEMU handles
                # the -cpu option itself
                if [ -w "/dev/kvm" ]; then
                    QEMU_OPTS+=("-enable-kvm" "-cpu host" "-machine q35,smm=on")
                else
                    echo -e "WARNING: Can not use KVM acceleration. Please see:\n https://github.com/gardenlinux/gardenlinux/tree/main/bin#start-vm \n\n"
                fi

            fi

        else
            # Running on emulated arch
            # Set default CPU on emulated arch
            QEMU_OPTS+=("-cpu Broadwell")

        fi

    fi

    # Validate AARCH64/ARM64 arch
    if [ "$arch" = arm64 ]; then

        # Validate for native or emulated arch
        if [ "$arch" = "$arch_orig" ]; then

            # Running on native arch
            # Check for acceleration support
            if [ "$os" = macos ]; then
                # macOS HVF support is always given on all new Intel Macs
                # that are supported by the current HomeBrew version
                QEMU_OPTS+=("-cpu host" "-machine virt" "-accel hvf")

            else

                # Validate for KVM support on AMD64
                if [ -w "/dev/kvm" ]; then
                    QEMU_OPTS+=("-cpu host" "-machine virt" "-accel kvm")
                else
                    echo -e "WARNING: Can not use KVM acceleration. Please see:\nhttps://github.com/gardenlinux/gardenlinux/tree/main/bin#start-vm \n\n"
                    QEMU_OPTS+=("-cpu max" "-machine virt")
                fi

            fi

        else
            # Running on emulated arch
            # Set default CPU on emulated arch
            QEMU_OPTS+=("-cpu max" "-machine virt")

        fi

    fi
}


# Validate and define memory options
# for VM
qemu_opt_memory () {
    [[ "${memory: -1}" =~ [0-9] ]] && memory="${memory}Mi"
    memory=$(numfmt --from=auto --suffix="B" --format "%f" ${memory} | $bin_head -c-2)
    QEMU_OPTS+=("-m $(( $memory / 1048576 ))")
}


# Validate and define monitor port for
# extended commands
qemu_opt_monitor () {
    # Add Monitor
    if [ $monitor ]; then
        [ -e $TMP_DIR/$mac.monitor ] && eusage "Error: Monitor to this MAC address already exists $CURR_DIR/$mac.monitor"
        QEMU_OPTS+=("-monitor unix:$TMP_DIR/$mac.monitor,server,nowait")
    fi
}

# Validate and define commands for remote
# control of VM
qemu_opt_remote_control () {
    # VNC
    if [ $vnc ]; then
        ( while [ ! -e $CURR_DIR/$mac.monitor ]; do sleep 0.1; done
        echo "printf "WARNING: Change VNC password\n%s\n" MYPASSWORD | socat - UNIX-CONNECT:$CURR_DIR/$mac.monitor &> /dev/null )&"
        exit1
        printf "WARNING: Change VNC password\n%s\n" MYPASSWORD | socat - UNIX-CONNECT:$CURR_DIR/$mac.monitor &> /dev/null )&
        QEMU_OPTS+=("-vnc :${vnc_port},password")
    fi

    # IPMI
    if [ "centos" != $os ]; then
        if [ "$arch" = amd64 ]; then
            # Add a BMC simulator
            QEMU_OPTS+=("-device ipmi-bmc-sim,id=bmc0"
                        "-device isa-ipmi-kcs,bmc=bmc0,ioport=0xca2")

            # Add QEMU guest agent support
            QEMU_OPTS+=("-chardev socket,path=$TMP_DIR/$mac.guest,server=on,wait=off,id=qga0"
                        "-device virtio-serial"
                        "-device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0")
        fi
    fi
}


# Validate and define monitor port for
# extended commands
qemu_opt_network () {
    # Create network bridge (requires root).
    # (currently unsupported on macOS)
    if [ $os != macos ]; then
        if [ $bridge ]; then
            [ $(id -u) == 0 ] || eusage "Error: For bridging you must be root. More information can be found on: https://github.com/gardenlinux/gardenlinux/tree/main/bin#start-vm"
            if [ ! -d /etc/qemu ]; then
                mkdir -p /etc/qemu
                chown root:kvm /etc/qemu
            fi
            if [ ! -e /etc/qemu/bridge.conf ]; then
                touch /etc/qemu/bridge.conf
                chown root:kvm /etc/qemu/bridge.conf
                chmod 0640 /etc/qemu/bridge.conf
            fi

            printf "%s\nallow %s\n" "$(cat /etc/qemu/bridge.conf)" $bridge > $CURR_DIR/$mac.bridge
            awk '!seen[$0]++' < $CURR_DIR/$mac.bridge > /etc/qemu/bridge.conf
            rm -f $CURR_DIR/$mac.bridge
        fi
    fi

    # Set rom file on PXE boot
    if [ $pxe_binary ]; then
        QEMU_OPTS+=("-device virtio-net-pci,romfile=,netdev=net0,mac=$macfull")
    else
        QEMU_OPTS+=("-device virtio-net-pci,netdev=net0,mac=$macfull")
    fi

    # Set bridge or host fwd network
    if [ $bridge ]; then
        QEMU_OPTS+=("-netdev bridge,id=net0,br=${bridge}")
    elif [ $pxe ]; then
        QEMU_OPTS+=("-netdev user,id=net0,hostfwd=tcp::$port-:$port_dest,hostname=garden,tftp=$CURR_DIR,bootfile=$mac.ipxe")
    elif [ $pxe_binary ]; then
        QEMU_OPTS+=("-netdev user,id=net0,hostfwd=tcp::$port-:$port_dest,hostname=garden,tftp=$CURR_DIR,bootfile=$mac.efi")
    else
        QEMU_OPTS+=("-netdev user,id=net0,hostfwd=tcp::$port-:$port_dest,hostname=garden")
    fi
}


# Setup TPM2 emulation support using swtpm
qemu_opt_tpm () {
    if [ "$tpm2" ]; then
        if [ ! "$(command -v swtpm)" ]; then
            eusage "Error: Could not find swtpm on the system"
        else
            swtpm socket --tpmstate backend-uri=file://$CURR_DIR/$mac.permall --ctrl type=unixio,path=$CURR_DIR/swtpm-sock --tpm2 --daemon --terminate
        fi

        QEMU_OPTS+=("-chardev socket,id=chrtpm,path=$CURR_DIR/swtpm-sock")

        # 'tpm-tis-device' for aarch64 versus 'tpm-tis' for x86
        # Reference: https://listman.redhat.com/archives/libvir-list/2021-February/msg00647.html
        if [ $arch = "amd64" ]; then
            QEMU_OPTS+=("-tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0")
        elif [ $arch = "arm64" ]; then
            QEMU_OPTS+=("-tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis-device,tpmdev=tpm0")
        fi
    fi
}


qemu_opt_misc () {
    if [ "$arch" = "amd64" ]; then
        # Use minimal memory
        QEMU_OPTS+=("-device virtio-balloon")

        # Add random number generator (RNG) to the host
        QEMU_OPTS+=("-device virtio-rng-pci,rng=rng0"
                    "-object rng-random,id=rng0,filename=/dev/random")
    fi

    if [ $pidfile ]; then
	QEMU_OPTS+=("-pidfile $pidfile")
    fi

    # Set QEMU start options depending on
    # daemonizing the VM
    if [ $daemonize ]; then
        QEMU_OPTS+=("-daemonize"
                    "-display none")
    else
        QEMU_OPTS+=("-nographic"
                    "-serial mon:stdio")
    fi
}


# Print out status of QEMU execution command including
# all defined options
qemu_status () {
    # Print status
    printf "Status:\n"
    printf "  starting VM(UUID:%s) with MAC:%s in %s\n" $uuid $macfull $CURR_DIR
    [ $monitor ]    && printf "  monitor: %s.monitor\tconnect: socat - UNIX-CONNECT:%s\n" $mac $CURR_DIR/$mac.monitor
    [ $pxe ]        && printf "  pxeboot: %s.ipxe\n" $mac
    [ $pxe ]        && printf "  pxeboot: files served from %s\n" "$pxe"
    [ $bridge ]     && printf "  interface: %s  bridged\n" $bridge
    [ $bridge ]     || printf "  sshport: (host) tcp/%s -> (vm) tcp/%s  (unbridged)\n"  $port $port_dest
    [ $vnc ]        && printf "  vncport: %s\n" $(( vnc_port + 5900 ))
    [ $uefi ]       && printf "  uefi boot enabled. %s.vars stores efivars\n" $mac
    [ $tpm2 ]       && printf "  tpm2 emulation enabled\n"

    # Print inflated disks
    for i in "${inflatelist[@]}"; do
        printf "  file: %s will be inflated to %s\n" $(cut -d, -f1 <<< "$i") $(cut -d, -f2 <<< "$i")
    done

    # Print QEMU command line output
    ( printf "\n  commandline: %s " $bin_qemu
    printf '%s ' "${QEMU_OPTS[@]}";printf "\n" ) | sed 's/ /!/g;s/!-/ -/g' | fold -s -w $(( $(tput cols) - 4 )) | sed 's/!/ /g;3,$ s|^|    |'

    # Wait for input in not --skipkp is set
    if [ $keypress ]; then
        read -n 1 -r -s -p $'Press any key to continue...\n'
    fi
}


# Start QEMU with QEMU_OPTS
qemu_execute () {
    # Execute Arch/Distro realted binary with according QEMU opts
    $bin_qemu ${QEMU_OPTS[@]}
}


# Main
main "$@"
