noid-mklive/mkrootfs.sh.in

#!/bin/sh
#-
# Copyright (c) 2013-2015 Juan Romero Pardines.
# Copyright (c) 2017 Google
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#-

readonly PROGNAME=$(basename "$0")
readonly ARCH=$(uname -m)

trap 'die "Interrupted! exiting..."' INT TERM HUP


info_msg() {
    printf "\033[1m%s\n\033[m" "$@"
}

die() {
    printf "FATAL: %s\n" "$@"
    umount_pseudofs
    [ -d "$ROOTFS" ] && rm -rf "$ROOTFS"
    exit 1
}


# Even though we only support really one target for most of these
# architectures this lets us refer to these quickly and easily by
# XBPS_ARCH.  This makes it a lot more obvious what is happening later
# in the script, and it makes it easier to consume the contents of
# these down the road in later scripts.
usage() {
    cat <<_EOF
Usage: $PROGNAME [options] <arch>

Supported architectures: i686, i686-musl, x86_64, x86_64-musl,
                         armv5tel, armv5tel-musl, armv6l, armv6l-musl, armv7l, armv7l-musl
                         aarch64, aarch64-musl,
                         mipsel, mipsel-musl


Options
    -b <syspkg> Set an alternative base-system package (defaults to base-system)
    -c <dir>    Set XBPS cache directory (defaults to \$PWD/xbps-cachedir-<arch>)
    -C <file>   Full path to the XBPS configuration file
    -h          Show this help
    -r <repo>   Set XBPS repository (may be set multiple times)
    -V          Show version
_EOF
}

mount_pseudofs() {
    # This function ensures that the psuedofs mountpoints are present
    # in the chroot.  Strictly they are not necessary to have for many
    # commands, but bind-mounts are cheap and it isn't too bad to just
    # mount them all the time.
    for f in dev proc sys; do
        # In a naked chroot there is nothing to bind the mounts to, so
        # we need to create directories for these first.
        [ ! -d "$ROOTFS/$f" ] && mkdir -p "$ROOTFS/$f"
        if ! mountpoint -q "$ROOTFS/$f" ; then
            # It is VERY important that this only happen if the
            # pseudofs isn't already mounted.  If it already is then
            # this is virtually impossible to troubleshoot because it
            # looks like the subsequent umount just isn't working.
            mount -r --bind /$f "$ROOTFS/$f"
        fi
    done
}

umount_pseudofs() {
    # This function cleans up the mounts in the chroot.  Failure to
    # clean up these mounts will prevent the tmpdir from being
    # deletable instead throwing the error "Device or Resource Busy".
    # The '-f' option is passed to umount to account for the
    # contingency where the psuedofs mounts are not present.
    if [ -d "${ROOTFS}" ]; then
        for f in dev proc sys; do
            umount -f "$ROOTFS/$f" >/dev/null 2>&1
        done
    fi
}

run_cmd_target() {
    info_msg "Running $* for target $XBPS_TARGET_ARCH ..."
    if [ "$XBPS_TARGET_ARCH" = "$(xbps-uhelper arch)" ] ; then
        # This is being run on the same architecture as the host,
        # therefore we should set XBPS_ARCH.
        if ! eval XBPS_ARCH="$XBPS_TARGET_ARCH" "$@" ; then
            die "Could not run command $*"
        fi
    else
        # This is being run on a foriegn arch, therefore we should set
        # XBPS_TARGET_ARCH.  In this case XBPS will not attempt
        # certain actions and will require reconfiguration later.
        if ! eval XBPS_TARGET_ARCH="$XBPS_TARGET_ARCH" "$@" ; then
            die "Could not run command $*"
        fi
    fi
}

run_cmd() {
    info_msg "Running $*"
    eval "$@"
}

run_cmd_chroot() {
    # General purpose chroot function which makes sure the chroot is
    # prepared.  This function takes 2 arguments, the location to
    # chroot to and the command to run.

    # This is an idempotent function, it is safe to call every time
    # before entering the chroot.  This has the advantage of making
    # execution in the chroot appear as though it "Just Works(tm)".
    register_binfmt

    # Before we step into the chroot we need to make sure the
    # pseudo-filesystems are ready to go.  Not all commands will need
    # this, but its still a good idea to call it here anyway.
    mount_pseudofs

    # With assurance that things will run now we can jump into the
    # chroot and run stuff!
    chroot "$1" sh -c "$2"
}

cleanup_chroot() {
    # This function cleans up the chroot shims that are used by QEMU
    # to allow builds on alien platforms.  It takes no arguments but
    # expects the global $ROOTFS variable to be set.

    # Un-Mount the pseudofs mounts if they were mounted
    umount_pseudofs

    # If a QEMU binary was copied in, remove that as well
    if [ -x "$ROOTFS/usr/bin/$QEMU_BIN" ] ; then
        rm "$ROOTFS/usr/bin/$QEMU_BIN"
    fi
}

# TODO: Figure out how to register the binfmt for x86_64 and for i686
# to facilitate building on alien build systems.
register_binfmt() {
    # This function sets up everything that is needed to be able to
    # chroot into a ROOTFS and be able to run commands there.  This
    # really matters on platforms where the host architecture is
    # different from the target, and you wouldn't be able to run
    # things like xbps-reconfigure -a.  This function is idempotent
    # (You can run it multiple times without modifying state).  This
    # function takes no arguments, but does expect the global variable
    # $XBPS_TARGET_ARCH to be set.

    # This select sets up the "magic" bytes in /proc that let the
    # kernel select an alternate interpreter.  More values for this
    # map can be obtained from here:
    # https://github.com/qemu/qemu/blob/master/scripts/qemu-binfmt-conf.sh
    case "${XBPS_TARGET_ARCH}" in
        armv*)
            _cpu=arm
            _magic="\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00"
            _mask="\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff"
            QEMU_BIN=qemu-arm-static
            ;;
        aarch64*)
            _cpu=aarch64
            _magic="\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7"
            _mask="\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff"
            QEMU_BIN=qemu-aarch64-static
            ;;
        mipsel*)
            _cpu=mipsel
            _magic="\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00"
            _mask="\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff"
            QEMU_BIN=qemu-mipsel-static
            ;;
        *86*)
            info_msg "FIXME: Assuming that x86 instructions are native"
            QEMU_BIN=NATIVE
            ;;
        *)
            die "Unknown target architecture!"
            ;;
    esac

    # In the special case where the build is native we can return
    # without doing anything else
    if [ "$QEMU_BIN" = "NATIVE" ] ; then
        return
    fi
    
    # For builds that do not match the host architecture, the correct
    # qemu binary will be required.
    if ! $QEMU_BIN -version >/dev/null 2>&1; then
        die "$QEMU_BIN binary is missing in your system, exiting."
    fi

    # In order to use the binfmt system the binfmt_misc mountpoint
    # must exist inside of proc
    if ! mountpoint -q /proc/sys/fs/binfmt_misc ; then
        modprobe -q binfmt_misc
        mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc 2>/dev/null
    fi

    # Only register if the map is incomplete
    if [ ! -f /proc/sys/fs/binfmt_misc/qemu-$_cpu ] ; then
        echo ":qemu-$_cpu:M::$_magic:$_mask:$QEMU_BIN:" > /proc/sys/fs/binfmt_misc/register 2>/dev/null
    fi

    # If the static binary isn't in the chroot then the chroot will
    # fail.  The kernel knows about the map but without the static
    # version there's no interpreter in the chroot, only the
    # dynamically linked one in the host.  To simplify things we just
    # use the static one always and make sure it shows up at the same
    # place in the host and the chroot.
    if [ ! -x "$ROOTFS/usr/bin/$QEMU_BIN" ] ; then
        cp -f "$(which "$QEMU_BIN")" "$ROOTFS/usr/bin" ||
            die "Could not install $QEMU_BIN to $ROOTFS/usr/bin/"
    fi
}

#
# main()
#
while getopts "C:c:h:r:V" opt; do
    case $opt in
        C) XBPS_CONFFILE="-C $OPTARG";;
        c) XBPS_CACHEDIR="--cachedir=$OPTARG";;
        h) usage; exit 0;;
        r) XBPS_REPOSITORY="$XBPS_REPOSITORY --repository=$OPTARG";;
        V) echo "$PROGNAME @@MKLIVE_VERSION@@"; exit 0;;
    esac
done
shift $((OPTIND - 1))
XBPS_TARGET_ARCH="$1"

# This is an aweful hack since the script isn't using privesc
# mechanisms selectively.  This is a TODO item.
if [ "$(id -u)" -ne 0 ]; then
    die "need root perms to continue, exiting."
fi

# If the arch wasn't set let's bail out now, nothing else in this
# script will work without knowing what we're trying to build for.
if [ -z "$XBPS_TARGET_ARCH" ]; then
    echo "$PROGNAME: arch was not set!"
    usage; exit 1
fi

# If the repository hasn't already been set, we set it to a sane value
# here.  These should all resolve even if they won't have the
# appropriate repodata files for the selected architecture.
: "${XBPS_REPOSITORY:=--repository=http://repo.voidlinux.eu/current \
                      --repository=http://repo.voidlinux.eu/current/musl \
                      --repository=http://repo.voidlinux.eu/current/aarch64}"

# The package artifacts are cacheable, but they need to be isolated
# from the host cache.
: "${XBPS_CACHEDIR:=--cachedir=$PWD/xbps-cache/${XBPS_TARGET_ARCH}}"

# The following binaries are required to proceed
for f in chroot tar xbps-install xbps-reconfigure xbps-query; do
    if ! which $f >/dev/null ; then
        die "$f binary is missing in your system, exiting."
    fi
done

# We need to operate on a tempdir, if this fails to create, it is
# absolutely crucial to bail out so that we don't hose the system that
# is running the script.
ROOTFS=$(mktemp -d) || die "failed to create tempdir, exiting..."

# This maintains the chain of trust, the keys in the repo are known to
# be good and so we copy those.  Why don't we just use the ones on the
# host system?  That's a good point, but there's no promise that the
# system running the script is Void, or that those keys haven't been
# tampered with.  Its much easier to use these since the will always
# exist.
mkdir -p "$ROOTFS/var/db/xbps/keys"
cp keys/*.plist "$ROOTFS/var/db/xbps/keys"

# This sets up files that are important for XBPS to work on the new
# filesystem.  It does not actually install anything.
run_cmd_target "xbps-install -S $XBPS_CONFFILE $XBPS_CACHEDIR $XBPS_REPOSITORY -r $ROOTFS"

# Later scripts expect the permissions on / to be the canonical 755,
# so we set this here.
chmod 755 "$ROOTFS"

# The pseudofs mountpoints are needed for the qemu support in cases
# where we are running things that aren't natively executable.
mount_pseudofs

# With everything setup, we can now run the install to load the
# base-voidstrap package into the rootfs.  This will not produce a
# bootable system but will instead produce a base component that can
# be quickly expanded to perform other actions on.
run_cmd_target "xbps-install -S $XBPS_CONFFILE $XBPS_CACHEDIR $XBPS_REPOSITORY -r $ROOTFS -y base-voidstrap"

# Enable en_US.UTF-8 locale and generate it into the target ROOTFS.
# This is a bit of a hack since some glibc stuff doesn't really work
# correctly without a locale being generated.  While some could argue
# that this is an arbitrary or naive choice to enable the en_US
# locale, most people using Void are able to work with the English
# language at least enough to enable thier preferred locale.  If this
# truly becomes an issue in the future this hack can be revisited.
if [ -e "$ROOTFS/etc/default/libc-locales" ]; then
    LOCALE=en_US.UTF-8
    sed -e "s/\#\(${LOCALE}.*\)/\1/g" -i "$ROOTFS/etc/default/libc-locales"
fi

# The reconfigure step needs to execute code that's been compiled for
# the target architecture.  Since the target isn't garanteed to be the
# same as the host, this needs to be done via qemu.
info_msg "Reconfiguring packages for ${XBPS_TARGET_ARCH} ..."
case "$XBPS_TARGET_ARCH" in
    # TODO: Rather than asserting that x86 code will work, check
    # instead if the system that is hosting this script is the same as
    # the target, using binfmt if it is not.
    i686*|x86_64*)
        run_cmd "XBPS_ARCH=${XBPS_TARGET_ARCH} xbps-reconfigure -r $ROOTFS base-files"
        ;;
    *)
        # This step sets up enough of the base-files that the chroot
        # will work and they can be reconfigured natively.  Without
        # this step there isn't enough configured for ld to work.
        # This step runs as the host architecture.
        run_cmd "xbps-reconfigure -r $ROOTFS base-files"

        # Now running as the target system, this step reconfigures the
        # base-files completely.  Certain things just won't work in
        # the first pass, so this cleans up any issues that linger.
        run_cmd_chroot "$ROOTFS" "env -i xbps-reconfigure -f base-files"

        # TODO: determine why these lines are here.  What is the harm
        # in having them and what do they remove.  Do they interact
        # adversely with the alien build support discussed above.
        rmdir "$ROOTFS/usr/lib32" 2>/dev/null
        rm -f "$ROOTFS/lib32" "$ROOTFS/lib64" "$ROOTFS/usr/lib64"
        ;;
esac

# Once base-files is configured and functional its possible to
# configure the rest of the system.
run_cmd_chroot "$ROOTFS" "xbps-reconfigure -a"

# At this point we're done running things in the chroot and we can
# clean up the shims.  Failure to do this can result in things hanging
# when we try to delete the tmpdir.
cleanup_chroot

# Set the default password.  Previous versions of this script used a
# chroot to do this, but that is unnecessary since chpasswd
# understands how to operate on chroots without actually needing to be
# chrooted.  We also remove the lock file in this step to clean up the
# lock on the passwd database, lest it be left in the system and
# propogated to other points.
echo root:voidlinux | chpasswd -c SHA512 --root "$ROOTFS" || die "Could not set default credentials"
rm -f "$ROOTFS/etc/.pwd.lock"

# The cache isn't that useful since by the time the ROOTFS will be
# used it is likely to be out of date.  Rather than shipping it around
# only for it to be out of date, we remove it now.
rm -rf "$ROOTFS/var/cache/*" 2>/dev/null

# Finally we can compress the tarball, the name will include the
# architecture and the date on which the tarball was built.
tarball=void-${XBPS_TARGET_ARCH}-ROOTFS-$(date '+%Y%m%d').tar.xz
run_cmd "tar -cp --posix --xattrs -C $ROOTFS . | xz -T0 -9 > $tarball "

# Now that we have the tarball we don't need the rootfs anymore, so we
# can get rid of it.
rm -rf "$ROOTFS"

# Last thing to do before closing out is to let the user know that
# this succeeded.  This also ensures that there's something visible
# that the user can look for at the end of the script, which can make
# it easier to see what's going on if something above failed.
info_msg "Successfully created $tarball ($XBPS_TARGET_ARCH)"