smartos-live/configure

#!/usr/bin/bash
#
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright 2021 Joyent, Inc.
# Copyright 2022 MNX Cloud, Inc.
#
shopt -s xpg_echo
unalias -a

PATH=/usr/bin:/usr/sbin:/sbin:/opt/local/bin:/opt/local/sbin
export PATH

conf_arg0=$(basename $0)
conf_arg0_dir=$(dirname $0)
conf_root=$PWD
conf_ips=
conf_priv="pfexec"

#
# Build defaults which are unlikely to change, but may be overridden by
# configure-build.
#
source default.configure-build
if [[ -f configure-build ]]; then
	source configure-build || fatal "failed to source configure-build"
fi

#
# We configure the projects framework using a 'configure-projects' file. This
# is a plain text file which contains the project name, the branch we wish to
# build it at, and either a URL to a git repository or 'origin' to denote the
# github.com/joyent/<repo> location. See create_projects.
#
read -r -d '' configure_projects < default.configure-projects

function warn
{
	local msg="$*"
	[[ -z "$msg" ]] && msg="failed"
	echo "$conf_arg0: $msg" >&2
}

function fatal
{
	local msg="$*"
	[[ -z "$msg" ]] && msg="failed"
	echo "$conf_arg0: $msg" >&2
	exit 1
}

function check_loc
{
	[[ "$(readlink -f "$PWD")" != "$(readlink -f "$conf_arg0_dir")" ]] && \
	    fatal "configure must be run from inside $conf_arg0_dir"
	[[ "$(uname -s)" != "SunOS" ]] && fatal "build only works on SunOS"
	if [[ "$(uname -v)" =~ "joyent_" ]]; then
		[[ "$(zonename)" == "global" ]] && fatal \
		    "building in a GZ on SmartOS is not supported"
	else
		echo "Not on SmartOS, assuming IPS and sudo"
		conf_ips=1
		conf_priv="sudo"
	fi
}

#
# Checkout a project at the specific branch if the repository doesn't exist,
# or isn't already at the required branch. Warn if the repository contains
# uncommitted contents.
#
function checkout_project
{
	local dir="$1"
	local branch="$2"
	local url="$3"

	echo "Using $url at $branch in $dir"

	if [[ ! -d "$dir" ]]; then
		git clone "$url" "${dir}" || fatal "unable to clone $dir"
	fi

	local cur=$(git -C ${dir} branch | grep "^*" | \
	    awk 'BEGIN{ FS=" " } { print $2 }')

	# If our repository is already at the desired branch, then do nothing,
	# intentionally preserving whatever's in the repository.
	if [[ "$cur " != "$branch" ]]; then
		git -C ${dir} checkout ${branch}
		if [[ $? -ne 0 ]]; then

			# If this is a remote branch, say 'refs/changes/34/1234/1',
			# then we might need to fetch it before being able to check it out.
			echo "unable to checkout $branch, attempting to fetch it ..."

			# Use a local branch name without '/' chars so a detached head
			# doesn't confuse our standard parsing of 'git describe' output
			local branch_desc=$(echo $branch | sed -e 's/\//_/g')

			# We assume the remote is named 'origin'.
			git -C ${dir} fetch origin ${branch}:${branch_desc}

			# If the above fails, see if the project appears to be a GitHub
			# PR-xxx branch name, and if so, add the relevant refspec for
			# pull requests.
			if [[ $? -ne 0 ]]; then
				PR_NUMBER=$(echo "${branch}" | sed -nEe 's/^PR-([0-9]+)$/\1/p')
				if [[ -n "${PR_NUMBER}" ]]; then
					echo "Branch appears to be from a PR, adding GitHub refspec"
					echo "    +refs/pull/${PR_NUMBER}/head:refs/remotes/origin/${branch}"
					git -C ${dir} config --add remote.origin.fetch \
					    +refs/pull/${PR_NUMBER}/head:refs/remotes/origin/${branch}
					git -C ${dir} fetch || \
					    fatal "unable to fetch using PR refspec"
				else
					fatal "unable to fetch ${branch} from origin"
				fi
			fi
			git -C ${dir} checkout ${branch_desc} || \
				fatal "unable to checkout ${branch_desc}"
		fi
	fi

	git -C ${dir} --no-pager show -s
	git -C ${dir} describe --all --long --dirty | egrep -e '-dirty$' &&
	    echo "Warning: ${dir} contains uncommitted changes!"
	echo

	return 0
}

#
# Read a flat file called 'configure-projects' of the form:
# <project name>:<project branch>:[project git repo URL or path]
#
# The special token 'origin' can be used in place of a full git
# repo URL to denote the standard github.com URL for that project.
# If no URL is given, we default to github.com.
#
# Once the configure-projects file has been read (or we've parsed the
# defaults), invoke checkout_project for each one.
#
function create_projects
{
	local dir="projects/local"
	local git_stem="https://github.com/joyent"

	[[ -d "$dir" ]] || mkdir -p "$dir" || fatal "failed to create $dir"

	if [[ -f "configure-projects" ]]; then
		# note we're not attempting to merge configure-projects with the default
		read -r -d '' configure_projects  < configure-projects
		echo "Info: using the following configure-projects file:"
		echo "$configure_projects"
	else
		echo "Info: no configure-projects file found, using defaults."
	fi
	echo "$configure_projects" | grep -v '^#' | while read -r line; do
		local project_dir=$(echo $line | cut -d: -f 1 | sed -e 's/ //g')
		local project_name=$(basename $project_dir)
		local git_name="${project_name}.git"
		local project_branch=$(echo $line | cut -d: -f 2 | sed -e 's/ //g')
		local project_url=$(echo $line | cut -d: -f 3- | sed -e 's/ //g')

		# special cases for some projects
		case ${project_name} in
		'kvm'|'kvm-cmd')
			git_name="illumos-${project_name}.git"
			;;
		'illumos')
			git_name="illumos-joyent.git"
			;;
		'ur-agent')
			git_name="sdc-ur-agent.git"
			;;
		esac

		# process URL keywords
		case $project_url in
		'origin'|'')
			project_url="$git_stem/$git_name"
			;;
		esac
		checkout_project "projects/$project_dir" "$project_branch" \
		    "$project_url"
	done || fatal "Error encountered when creating projects directory"
}

function install_pkgin
{
	local pkglist

	pkglist="build-essential flex libxslt openjdk11 nodejs"
	pkglist="$pkglist p5-XML-Parser gettext python27 py39-expat python39"
	pkglist="$pkglist coreutils gsed pkg_alternatives cdrtools"
	pkglist="$pkglist py27-sqlite3 nasm pigz smartos-build-tools"

	$conf_priv pkgin -f update || fatal "failed to update pkgsrc repository"

	#
	# Remove legacy packages replaced by the smartos-build-tools package.
	#
	for pkg in astmsgtools dmake rpcgen sgstools; do
		if pkg_info -qe $pkg; then
			$conf_priv pkg_delete ${pkg}
		fi
	done

	for pkg in $pkglist; do
		if ! pkg_info -qe $pkg; then
			$conf_priv pkgin -y install $pkg || fatal \
		    "failed to install $pkg"
		fi
	done
}

function install_ips
{
	fatal "Building on ips based systems has yet to be implemented"
}

function install_packages
{
	if [[ -z "$conf_ips" ]]; then
		install_pkgin
	else
		install_ips
	fi
	[[ $? -eq 0 ]] || fatal "failed to install packages"
}

function fetch_adjuncts
{
	local tgz
	[[ -z "$ILLUMOS_ADJUNCT_TARBALL_URL" ]] && fatal \
	    "ILLUMOS_ADJUNCT_TARBALL_URL missing from configure"
	curl -O $ILLUMOS_ADJUNCT_TARBALL_URL
	[[ $? -eq 0 ]] || fatal "failed to fetch adjuncts tarball"
}

function fetch_closed
{
	local ildir="projects/illumos"
	local cld="on-closed-bin.i386.tar.bz2"
	local clnd="on-closed-bin-nd.i386.tar.bz2"
	[[ -z "$ON_CLOSED_BINS_URL" ]] && fatal \
	    "missing ON_CLOSED_BINS_URL from configure"
	[[ -z "$ON_CLOSED_BINS_ND_URL" ]] && fatal \
	    "missing ON_CLOSED_BINS_ND_URL from configure"
	if [[ ! -f $ildir/$cld ]]; then
		curl $ON_CLOSED_BINS_URL -o $ildir/$cld || fatal \
		"failed to fetch closed bins (debug)"
	fi
	if [[ ! -f $ildir/$clnd ]]; then
		curl $ON_CLOSED_BINS_ND_URL -o $ildir/$clnd || fatal \
		"failed to fetch closed bins (non-debug)"
	fi
	cd $ildir >/dev/null 2>&1 || fatal "failed to cd into $ildir"
	tar xpjf $cld || fatal \
	    "failed to extract closed bins (debug)"
	tar xpjf $clnd || fatal \
	    "failed to extract closed bins (non-debug)"
	cd - >/dev/null 2>&1
}

function generate_env
{
	#
	# Note: since a nightly clobber build removes the whole proto, and we're
	# installing non-Illumos things there, we always use the -i option, and
	# ILLUMOS_CLOBBER is handled separately in Makefile.
	#
	local nopts="-CiLmMNnt"
	local lprefix

	case $ILLUMOS_ENABLE_DEBUG in
	exclusive)
		nopts="${nopts}DF" ;;
	yes)
		nopts="${nopts}D" ;;
	no|"")
		;;
	*)
		echo "unknown debug option $ILLUMOS_ENABLE_DEBUG" >&2
		exit 2 ;;
	esac

	GNUC_ROOT="$conf_root/proto.strap/usr/gcc/$PRIMARY_COMPILER_VER"
	PRIMARY_CC="gcc$PRIMARY_COMPILER_VER,$GNUC_ROOT/bin/gcc,gnu"
	PRIMARY_CCC="gcc$PRIMARY_COMPILER_VER,$GNUC_ROOT/bin/g++,gnu"

	SHADOW_CCS=
	SHADOW_CCCS=

	IFS=,
	for cc in $SHADOW_COMPILERS; do
		gcc_ver=$(echo $cc | sed 's/^gcc//')
		root="$conf_root/proto.strap/usr/gcc/$gcc_ver"
		SHADOW_CCS+=" gcc${gcc_ver},$root/bin/gcc,gnu"
		SHADOW_CCCS+=" gcc${gcc_ver},$root/bin/g++,gnu"
	done
	unset IFS

	if [[ "$ENABLE_SMATCH" = "yes" ]]; then
		SHADOW_CCS+=" smatch,\$BUILD_TOOLS/onbld/bin/\$MACH/smatch,smatch"
	fi

	lprefix=$(echo $conf_root | tr / _)
	[[ $? -eq 0 ]] || fatal "failed to create lock prefix"

cat > "projects/illumos/illumos.sh" <<EOF
NIGHTLY_OPTIONS="$nopts";			export NIGHTLY_OPTIONS
GATE="${RELEASE_VER}";				export GATE
CODEMGR_WS="$conf_root/projects/illumos";	export CODEMGR_WS
# NOTE: build_illumos modifies this initial constant
MAX_JOBS=128
maxjobs() {
        ncpu=\`kstat -p cpu_info:::state | grep -c on-line\`
	zcpucap=\`pfexec mdata-get sdc:cpu_cap\`
	if [[ \$zcpucap -eq 0 || \$zcpucap == "" ]]; then
		zcpu=\${MAX_JOBS}
	else
		zcpu=\`expr \$zcpucap / 100\`
	fi
	if [[ \$zcpu -lt \$ncpu ]]; then
		expr \$zcpu + 2
        elif [[ \$(( \$ncpu + 2 )) -lt \${MAX_JOBS} ]]; then
                expr \$ncpu + 2
        else
                printf "%d\n" \${MAX_JOBS}
        fi
}
DMAKE_MAX_JOBS=\`maxjobs\`;			export DMAKE_MAX_JOBS

PARENT_WS="";					export PARENT_WS
CLONE_WS="http://hg.illumos.org/illumos-gate"	export CLONE_WS
STAFFER="nobody";				export STAFFER
BUILD_PROJECT="";				export BUILD_PROJECT
LOCKNAME="\`whoami\`_${lprefix}_nightly.lock";	export LOCKNAME
ATLOG="\$CODEMGR_WS/log";			export ATLOG
LOGFILE="\$ATLOG/nightly.log";			export LOGFILE
MACH=\`uname -p\`;				export MACH
ON_CLOSED_BINS="\$CODEMGR_WS/closed";		export ON_CLOSED_BINS
REF_PROTO_LIST="\$PARENT_WS/usr/src/proto_list_\${MACH}";
						export REF_PROTO_LIST
ROOT="$conf_root/proto";			export ROOT
ADJUNCT_PROTO="$conf_root/proto.strap";		export ADJUNCT_PROTO
NATIVE_ADJUNCT="/opt/local";			export NATIVE_ADJUNCT
SRC="\$CODEMGR_WS/usr/src";			export SRC
VERSION="\$GATE";				export VERSION
PARENT_ROOT="$conf_root/proto";			export PARENT_ROOT
PARENT_TOOLS_ROOT="\$PARENT_WS/usr/src/tools/proto/root_\$MACH-nd"
						export PARENT_TOOLS_ROOT
PKGARCHIVE="\${CODEMGR_WS}/packages/\${MACH}/nightly";
						export PKGARCHIVE
PKGPUBLISHER_REDIST="${PUBLISHER}";		export PKGPUBLISHER_REDIST
MAKEFLAGS=k;					export MAKEFLAGS
UT_NO_USAGE_TRACKING="1";			export UT_NO_USAGE_TRACKING
MULTI_PROTO="no";				export MULTI_PROTO
BUILD_TOOLS="\$SRC/tools/proto/root_\${MACH}-nd/opt";
						export BUILD_TOOLS
SPRO_ROOT=/opt/SUNWspro;			export SPRO_ROOT
SPRO_VROOT=\$SPRO_ROOT;				export SPRO_VROOT
GNU_ROOT="$conf_root/proto.strap/usr/gnu"	export GNU_ROOT
# Use GCC as the primary compiler
__GNUC="";					export __GNUC
# Use GCC4 or later specific flags
__GNUC4="";					export __GNUC4
# root of \$PRIMARY_CC
GNUC_ROOT="$GNUC_ROOT";				export GNUC_ROOT
PRIMARY_CC="$PRIMARY_CC";			export PRIMARY_CC
PRIMARY_CCC="$PRIMARY_CCC";			export PRIMARY_CCC
SHADOW_CCS="$SHADOW_CCS";			export SHADOW_CCS
SHADOW_CCCS="$SHADOW_CCCS";			export SHADOW_CCCS
JAVA_ROOT=/opt/local/java/openjdk11;		export JAVA_ROOT
FLEX=/opt/local/bin/flex;			export FLEX
GNUXGETTEXT=/opt/local/bin/xgettext;		export GNUXGETTEXT
PYTHON3=/opt/local/bin/python3.9;		export PYTHON3
PYTHON3_VERSION=3.9;				export PYTHON3_VERSION
PYTHON3_PKGVERS=-39;				export PYTHON3_PKGVERS
PYTHON3_SUFFIX='';				export PYTHON3_SUFFIX
BLD_JAVA_11='';					export BLD_JAVA_11

# Disable the split 32/64 perl module builds since smartos doesn't exec perl
# via isaexec
# export BUILDPERL32='#'
export BUILDPERL64='#'

#
# Note that this isn't the strap perl: that only really exists for the benefit
# of projects/illumos-extra/openssl
#
PERL=/opt/local/bin/perl;			export PERL
ELFDUMP=/usr/bin/elfdump;			export ELFDUMP
LORDER=/usr/bin/lorder;				export LORDER
MCS=/usr/bin/mcs;				export MCS
NM=/usr/bin/nm;					export NM
STRIP=/usr/bin/strip;				export STRIP
TSORT=/usr/bin/tsort;				export TSORT
AR=/usr/bin/ar;					export AR
#
# We override $MAKE in ./tools/build_illumos so we can properly
# bootstrap the tools build.
#
if [[ -z "\$MAKE" ]]; then
MAKE="\$SRC/tools/proto/root_i386-nd/opt/onbld/bin/i386/dmake";	export MAKE
fi
LEX=/opt/local/bin/lex;				export LEX
YACC=/opt/local/bin/yacc;			export YACC
BISON=/opt/local/bin/bison;			export BISON
GM4=/opt/local/bin/gm4;				export GM4
ASTBINDIR=/opt/local/ast/bin;			export ASTBINDIR
LD_TOXIC_PATH="\$ROOT/lib:\$ROOT/usr/lib";	export LD_TOXIC_PATH
# generate buildversion for illumos buildversion module
BUILDVERSION_EXEC="${conf_root}/tools/build_etcrelease -g";
						export BUILDVERSION_EXEC

EOF
	[[ $? -eq 0 ]] || fatal "failed to write illumos nightly env file"
}

#
# This construction lets us place a block of text verbatim into $usage.
#
read -r -d '' usage <<EOF
./configure [options]:
	-c
		clobber Illumos before each build [default: no]
	-d
		build Illumos in DEBUG mode only [default: no]
	-h
		this message
	-p gcc7
		primary compiler version [default: gcc7]
	-P password
		platform root password [default: randomly chosen]
	-r
		full strap build (no cache) [default: no]
	-S
		do *not* run smatch [default is to run smatch]
	-s gcc10
		shadow compilers, comma delimited (gcc10,gcc#) [default: none]
EOF

while getopts "cdhp:P:rSs:" arg; do
	case $arg in
	c)
		ILLUMOS_CLOBBER=yes ;;

	#
	# We have only one proto area: we'll skip building non-debug here, as
	# the results are over-written by nightly.  If the user really wants a
	# sanity check build, they can specify =yes via the env var, or by
	# directly editing illumos.sh.
	#
	d)
		ILLUMOS_ENABLE_DEBUG=exclusive ;;
	h)
		echo "$usage"
		exit 0 ;;
	p)
		PRIMARY_COMPILER=$OPTARG
		FORCE_STRAP_REBUILD=yes ;;
	P)
		PLATFORM_PASSWORD="$OPTARG" ;;
	r)
		FORCE_STRAP_REBUILD=yes ;;
	S)
		ENABLE_SMATCH=no ;;
	s)
		SHADOW_COMPILERS=$OPTARG
		FORCE_STRAP_REBUILD=yes ;;
	?)
		echo "$usage" >&2
		exit 2 ;;
	esac
done

[[ -n "$ILLUMOS_ENABLE_DEBUG" ]] || ILLUMOS_ENABLE_DEBUG=no
[[ -n "$ILLUMOS_CLOBBER" ]] || ILLUMOS_CLOBBER=no
[[ -n "$PRIMARY_COMPILER" ]] || PRIMARY_COMPILER=gcc7
[[ -n "$ENABLE_SMATCH" ]] || ENABLE_SMATCH=yes

PRIMARY_COMPILER_VER=$(echo $PRIMARY_COMPILER | sed 's/^gcc//')

cat >build.env <<EOF
FORCE_STRAP_REBUILD=$FORCE_STRAP_REBUILD
ILLUMOS_CLOBBER=$ILLUMOS_CLOBBER
ILLUMOS_ENABLE_DEBUG=$ILLUMOS_ENABLE_DEBUG
PRIMARY_COMPILER=$PRIMARY_COMPILER
PRIMARY_COMPILER_VER=$PRIMARY_COMPILER_VER
SHADOW_COMPILERS=$SHADOW_COMPILERS
HOST_GCC4_PREFIX=$HOST_GCC4_PREFIX
ENABLE_SMATCH=$ENABLE_SMATCH
EOF

[[ -n "$PLATFORM_PASSWORD" ]] && \
	echo "PLATFORM_PASSWORD=$PLATFORM_PASSWORD" >> build.env

echo "Doing pre-flight checks... \c "
check_loc
echo "done."
echo "Creating and updating projects directory ..."
create_projects
echo "done."
echo "Installing packages ... \c "
install_packages
echo "done."
echo "Fetching adjuncts tgz... \c "
fetch_adjuncts
echo "done."
echo "Fetching and extracting closed bins ... \c "
fetch_closed
echo "done."
echo "Generating illumos environment file... \c "
generate_env
echo "done."

cat <<EOF
Configuration complete. To build the live image run:
gmake world && gmake live
EOF

# Congrats, we made it
exit 0