Local Mirror Of SmartOS Live Build Environment
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

536 lines
15 KiB

#!/usr/bin/bash
#
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright 2021 Joyent, Inc.
# Copyright 2022 MNX Cloud, Inc.
#
shopt -s xpg_echo
unalias -a
PATH=/usr/bin:/usr/sbin:/sbin:/opt/local/bin:/opt/local/sbin
export PATH
conf_arg0=$(basename $0)
conf_arg0_dir=$(dirname $0)
conf_root=$PWD
conf_ips=
conf_priv="pfexec"
#
# Build defaults which are unlikely to change, but may be overridden by
# configure-build.
#
source default.configure-build
if [[ -f configure-build ]]; then
source configure-build || fatal "failed to source configure-build"
fi
#
# We configure the projects framework using a 'configure-projects' file. This
# is a plain text file which contains the project name, the branch we wish to
# build it at, and either a URL to a git repository or 'origin' to denote the
# github.com/joyent/<repo> location. See create_projects.
#
read -r -d '' configure_projects < default.configure-projects
function warn
{
local msg="$*"
[[ -z "$msg" ]] && msg="failed"
echo "$conf_arg0: $msg" >&2
}
function fatal
{
local msg="$*"
[[ -z "$msg" ]] && msg="failed"
echo "$conf_arg0: $msg" >&2
exit 1
}
function check_loc
{
[[ "$(readlink -f "$PWD")" != "$(readlink -f "$conf_arg0_dir")" ]] && \
fatal "configure must be run from inside $conf_arg0_dir"
[[ "$(uname -s)" != "SunOS" ]] && fatal "build only works on SunOS"
if [[ "$(uname -v)" =~ "joyent_" ]]; then
[[ "$(zonename)" == "global" ]] && fatal \
"building in a GZ on SmartOS is not supported"
else
echo "Not on SmartOS, assuming IPS and sudo"
conf_ips=1
conf_priv="sudo"
fi
}
#
# Checkout a project at the specific branch if the repository doesn't exist,
# or isn't already at the required branch. Warn if the repository contains
# uncommitted contents.
#
function checkout_project
{
local dir="$1"
local branch="$2"
local url="$3"
echo "Using $url at $branch in $dir"
if [[ ! -d "$dir" ]]; then
git clone "$url" "${dir}" || fatal "unable to clone $dir"
fi
local cur=$(git -C ${dir} branch | grep "^*" | \
awk 'BEGIN{ FS=" " } { print $2 }')
# If our repository is already at the desired branch, then do nothing,
# intentionally preserving whatever's in the repository.
if [[ "$cur " != "$branch" ]]; then
git -C ${dir} checkout ${branch}
if [[ $? -ne 0 ]]; then
# If this is a remote branch, say 'refs/changes/34/1234/1',
# then we might need to fetch it before being able to check it out.
echo "unable to checkout $branch, attempting to fetch it ..."
# Use a local branch name without '/' chars so a detached head
# doesn't confuse our standard parsing of 'git describe' output
local branch_desc=$(echo $branch | sed -e 's/\//_/g')
# We assume the remote is named 'origin'.
git -C ${dir} fetch origin ${branch}:${branch_desc}
# If the above fails, see if the project appears to be a GitHub
# PR-xxx branch name, and if so, add the relevant refspec for
# pull requests.
if [[ $? -ne 0 ]]; then
PR_NUMBER=$(echo "${branch}" | sed -nEe 's/^PR-([0-9]+)$/\1/p')
if [[ -n "${PR_NUMBER}" ]]; then
echo "Branch appears to be from a PR, adding GitHub refspec"
echo " +refs/pull/${PR_NUMBER}/head:refs/remotes/origin/${branch}"
git -C ${dir} config --add remote.origin.fetch \
+refs/pull/${PR_NUMBER}/head:refs/remotes/origin/${branch}
git -C ${dir} fetch || \
fatal "unable to fetch using PR refspec"
else
fatal "unable to fetch ${branch} from origin"
fi
fi
git -C ${dir} checkout ${branch_desc} || \
fatal "unable to checkout ${branch_desc}"
fi
fi
git -C ${dir} --no-pager show -s
git -C ${dir} describe --all --long --dirty | egrep -e '-dirty$' &&
echo "Warning: ${dir} contains uncommitted changes!"
echo
return 0
}
#
# Read a flat file called 'configure-projects' of the form:
# <project name>:<project branch>:[project git repo URL or path]
#
# The special token 'origin' can be used in place of a full git
# repo URL to denote the standard github.com URL for that project.
# If no URL is given, we default to github.com.
#
# Once the configure-projects file has been read (or we've parsed the
# defaults), invoke checkout_project for each one.
#
function create_projects
{
local dir="projects/local"
local git_stem="https://github.com/joyent"
[[ -d "$dir" ]] || mkdir -p "$dir" || fatal "failed to create $dir"
if [[ -f "configure-projects" ]]; then
# note we're not attempting to merge configure-projects with the default
read -r -d '' configure_projects < configure-projects
echo "Info: using the following configure-projects file:"
echo "$configure_projects"
else
echo "Info: no configure-projects file found, using defaults."
fi
echo "$configure_projects" | grep -v '^#' | while read -r line; do
local project_dir=$(echo $line | cut -d: -f 1 | sed -e 's/ //g')
local project_name=$(basename $project_dir)
local git_name="${project_name}.git"
local project_branch=$(echo $line | cut -d: -f 2 | sed -e 's/ //g')
local project_url=$(echo $line | cut -d: -f 3- | sed -e 's/ //g')
# special cases for some projects
case ${project_name} in
'kvm'|'kvm-cmd')
git_name="illumos-${project_name}.git"
;;
'illumos')
git_name="illumos-joyent.git"
;;
'ur-agent')
git_name="sdc-ur-agent.git"
;;
esac
# process URL keywords
case $project_url in
'origin'|'')
project_url="$git_stem/$git_name"
;;
esac
checkout_project "projects/$project_dir" "$project_branch" \
"$project_url"
done || fatal "Error encountered when creating projects directory"
}
function install_pkgin
{
local pkglist
pkglist="build-essential flex libxslt openjdk11 nodejs"
pkglist="$pkglist p5-XML-Parser gettext python27 py27-expat python39"
pkglist="$pkglist coreutils gsed pkg_alternatives cdrtools"
pkglist="$pkglist py27-sqlite3 nasm pigz smartos-build-tools"
$conf_priv pkgin -f update || fatal "failed to update pkgsrc repository"
#
# Remove legacy packages replaced by the smartos-build-tools package.
#
for pkg in astmsgtools dmake rpcgen sgstools; do
if pkg_info -qe $pkg; then
$conf_priv pkg_delete ${pkg}
fi
done
for pkg in $pkglist; do
if ! pkg_info -qe $pkg; then
$conf_priv pkgin -y install $pkg || fatal \
"failed to install $pkg"
fi
done
}
function install_ips
{
fatal "Building on ips based systems has yet to be implemented"
}
function install_packages
{
if [[ -z "$conf_ips" ]]; then
install_pkgin
else
install_ips
fi
[[ $? -eq 0 ]] || fatal "failed to install packages"
}
function fetch_adjuncts
{
local tgz
[[ -z "$ILLUMOS_ADJUNCT_TARBALL_URL" ]] && fatal \
"ILLUMOS_ADJUNCT_TARBALL_URL missing from configure"
curl -O $ILLUMOS_ADJUNCT_TARBALL_URL
[[ $? -eq 0 ]] || fatal "failed to fetch adjuncts tarball"
}
function fetch_closed
{
local ildir="projects/illumos"
local cld="on-closed-bin.i386.tar.bz2"
local clnd="on-closed-bin-nd.i386.tar.bz2"
[[ -z "$ON_CLOSED_BINS_URL" ]] && fatal \
"missing ON_CLOSED_BINS_URL from configure"
[[ -z "$ON_CLOSED_BINS_ND_URL" ]] && fatal \
"missing ON_CLOSED_BINS_ND_URL from configure"
if [[ ! -f $ildir/$cld ]]; then
curl $ON_CLOSED_BINS_URL -o $ildir/$cld || fatal \
"failed to fetch closed bins (debug)"
fi
if [[ ! -f $ildir/$clnd ]]; then
curl $ON_CLOSED_BINS_ND_URL -o $ildir/$clnd || fatal \
"failed to fetch closed bins (non-debug)"
fi
cd $ildir >/dev/null 2>&1 || fatal "failed to cd into $ildir"
tar xpjf $cld || fatal \
"failed to extract closed bins (debug)"
tar xpjf $clnd || fatal \
"failed to extract closed bins (non-debug)"
cd - >/dev/null 2>&1
}
function generate_env
{
#
# Note: since a nightly clobber build removes the whole proto, and we're
# installing non-Illumos things there, we always use the -i option, and
# ILLUMOS_CLOBBER is handled separately in Makefile.
#
local nopts="-CiLmMNnt"
local lprefix
case $ILLUMOS_ENABLE_DEBUG in
exclusive)
nopts="${nopts}DF" ;;
yes)
nopts="${nopts}D" ;;
no|"")
;;
*)
echo "unknown debug option $ILLUMOS_ENABLE_DEBUG" >&2
exit 2 ;;
esac
GNUC_ROOT="$conf_root/proto.strap/usr/gcc/$PRIMARY_COMPILER_VER"
PRIMARY_CC="gcc$PRIMARY_COMPILER_VER,$GNUC_ROOT/bin/gcc,gnu"
PRIMARY_CCC="gcc$PRIMARY_COMPILER_VER,$GNUC_ROOT/bin/g++,gnu"
SHADOW_CCS=
SHADOW_CCCS=
IFS=,
for cc in $SHADOW_COMPILERS; do
gcc_ver=$(echo $cc | sed 's/^gcc//')
root="$conf_root/proto.strap/usr/gcc/$gcc_ver"
SHADOW_CCS+=" gcc${gcc_ver},$root/bin/gcc,gnu"
SHADOW_CCCS+=" gcc${gcc_ver},$root/bin/g++,gnu"
done
unset IFS
if [[ "$ENABLE_SMATCH" = "yes" ]]; then
SHADOW_CCS+=" smatch,\$BUILD_TOOLS/onbld/bin/\$MACH/smatch,smatch"
fi
lprefix=$(echo $conf_root | tr / _)
[[ $? -eq 0 ]] || fatal "failed to create lock prefix"
cat > "projects/illumos/illumos.sh" <<EOF
NIGHTLY_OPTIONS="$nopts"; export NIGHTLY_OPTIONS
GATE="${RELEASE_VER}"; export GATE
CODEMGR_WS="$conf_root/projects/illumos"; export CODEMGR_WS
# NOTE: build_illumos modifies this initial constant
MAX_JOBS=128
maxjobs() {
ncpu=\`kstat -p cpu_info:::state | grep -c on-line\`
zcpucap=\`pfexec mdata-get sdc:cpu_cap\`
if [[ \$zcpucap -eq 0 || \$zcpucap == "" ]]; then
zcpu=\${MAX_JOBS}
else
zcpu=\`expr \$zcpucap / 100\`
fi
if [[ \$zcpu -lt \$ncpu ]]; then
expr \$zcpu + 2
elif [[ \$(( \$ncpu + 2 )) -lt \${MAX_JOBS} ]]; then
expr \$ncpu + 2
else
printf "%d\n" \${MAX_JOBS}
fi
}
DMAKE_MAX_JOBS=\`maxjobs\`; export DMAKE_MAX_JOBS
PARENT_WS=""; export PARENT_WS
CLONE_WS="http://hg.illumos.org/illumos-gate" export CLONE_WS
STAFFER="nobody"; export STAFFER
BUILD_PROJECT=""; export BUILD_PROJECT
LOCKNAME="\`whoami\`_${lprefix}_nightly.lock"; export LOCKNAME
ATLOG="\$CODEMGR_WS/log"; export ATLOG
LOGFILE="\$ATLOG/nightly.log"; export LOGFILE
MACH=\`uname -p\`; export MACH
ON_CLOSED_BINS="\$CODEMGR_WS/closed"; export ON_CLOSED_BINS
REF_PROTO_LIST="\$PARENT_WS/usr/src/proto_list_\${MACH}";
export REF_PROTO_LIST
ROOT="$conf_root/proto"; export ROOT
ADJUNCT_PROTO="$conf_root/proto.strap"; export ADJUNCT_PROTO
NATIVE_ADJUNCT="/opt/local"; export NATIVE_ADJUNCT
SRC="\$CODEMGR_WS/usr/src"; export SRC
VERSION="\$GATE"; export VERSION
PARENT_ROOT="$conf_root/proto"; export PARENT_ROOT
PARENT_TOOLS_ROOT="\$PARENT_WS/usr/src/tools/proto/root_\$MACH-nd"
export PARENT_TOOLS_ROOT
PKGARCHIVE="\${CODEMGR_WS}/packages/\${MACH}/nightly";
export PKGARCHIVE
PKGPUBLISHER_REDIST="${PUBLISHER}"; export PKGPUBLISHER_REDIST
MAKEFLAGS=k; export MAKEFLAGS
UT_NO_USAGE_TRACKING="1"; export UT_NO_USAGE_TRACKING
MULTI_PROTO="no"; export MULTI_PROTO
BUILD_TOOLS="\$SRC/tools/proto/root_\${MACH}-nd/opt";
export BUILD_TOOLS
SPRO_ROOT=/opt/SUNWspro; export SPRO_ROOT
SPRO_VROOT=\$SPRO_ROOT; export SPRO_VROOT
GNU_ROOT="$conf_root/proto.strap/usr/gnu" export GNU_ROOT
# Use GCC as the primary compiler
__GNUC=""; export __GNUC
# Use GCC4 or later specific flags
__GNUC4=""; export __GNUC4
# root of \$PRIMARY_CC
GNUC_ROOT="$GNUC_ROOT"; export GNUC_ROOT
PRIMARY_CC="$PRIMARY_CC"; export PRIMARY_CC
PRIMARY_CCC="$PRIMARY_CCC"; export PRIMARY_CCC
SHADOW_CCS="$SHADOW_CCS"; export SHADOW_CCS
SHADOW_CCCS="$SHADOW_CCCS"; export SHADOW_CCCS
JAVA_ROOT=/opt/local/java/openjdk11; export JAVA_ROOT
FLEX=/opt/local/bin/flex; export FLEX
GNUXGETTEXT=/opt/local/bin/xgettext; export GNUXGETTEXT
PYTHON3=/opt/local/bin/python3.9; export PYTHON3
PYTHON3_VERSION=3.9; export PYTHON3_VERSION
PYTHON3_PKGVERS=-39; export PYTHON3_PKGVERS
PYTHON3_SUFFIX=''; export PYTHON3_SUFFIX
BLD_JAVA_11=''; export BLD_JAVA_11
# Disable the split 32/64 perl module builds since smartos doesn't exec perl
# via isaexec
# export BUILDPERL32='#'
export BUILDPERL64='#'
#
# Note that this isn't the strap perl: that only really exists for the benefit
# of projects/illumos-extra/openssl
#
PERL=/opt/local/bin/perl; export PERL
ELFDUMP=/usr/bin/elfdump; export ELFDUMP
LORDER=/usr/bin/lorder; export LORDER
MCS=/usr/bin/mcs; export MCS
NM=/usr/bin/nm; export NM
STRIP=/usr/bin/strip; export STRIP
TSORT=/usr/bin/tsort; export TSORT
AR=/usr/bin/ar; export AR
#
# We override $MAKE in ./tools/build_illumos so we can properly
# bootstrap the tools build.
#
if [[ -z "\$MAKE" ]]; then
MAKE="\$SRC/tools/proto/root_i386-nd/opt/onbld/bin/i386/dmake"; export MAKE
fi
LEX=/opt/local/bin/lex; export LEX
YACC=/opt/local/bin/yacc; export YACC
BISON=/opt/local/bin/bison; export BISON
GM4=/opt/local/bin/gm4; export GM4
ASTBINDIR=/opt/local/ast/bin; export ASTBINDIR
LD_TOXIC_PATH="\$ROOT/lib:\$ROOT/usr/lib"; export LD_TOXIC_PATH
# generate buildversion for illumos buildversion module
BUILDVERSION_EXEC="${conf_root}/tools/build_etcrelease -g";
export BUILDVERSION_EXEC
EOF
[[ $? -eq 0 ]] || fatal "failed to write illumos nightly env file"
}
#
# This construction lets us place a block of text verbatim into $usage.
#
read -r -d '' usage <<EOF
./configure [options]:
-c
clobber Illumos before each build [default: no]
-d
build Illumos in DEBUG mode only [default: no]
-h
this message
-p gcc7
primary compiler version [default: gcc7]
-P password
platform root password [default: randomly chosen]
-r
full strap build (no cache) [default: no]
-S
do *not* run smatch [default is to run smatch]
-s gcc10
shadow compilers, comma delimited (gcc10,gcc#) [default: none]
EOF
while getopts "cdhp:P:rSs:" arg; do
case $arg in
c)
ILLUMOS_CLOBBER=yes ;;
#
# We have only one proto area: we'll skip building non-debug here, as
# the results are over-written by nightly. If the user really wants a
# sanity check build, they can specify =yes via the env var, or by
# directly editing illumos.sh.
#
d)
ILLUMOS_ENABLE_DEBUG=exclusive ;;
h)
echo "$usage"
exit 0 ;;
p)
PRIMARY_COMPILER=$OPTARG
FORCE_STRAP_REBUILD=yes ;;
P)
PLATFORM_PASSWORD="$OPTARG" ;;
r)
FORCE_STRAP_REBUILD=yes ;;
S)
ENABLE_SMATCH=no ;;
s)
SHADOW_COMPILERS=$OPTARG
FORCE_STRAP_REBUILD=yes ;;
?)
echo "$usage" >&2
exit 2 ;;
esac
done
[[ -n "$ILLUMOS_ENABLE_DEBUG" ]] || ILLUMOS_ENABLE_DEBUG=no
[[ -n "$ILLUMOS_CLOBBER" ]] || ILLUMOS_CLOBBER=no
[[ -n "$PRIMARY_COMPILER" ]] || PRIMARY_COMPILER=gcc7
[[ -n "$ENABLE_SMATCH" ]] || ENABLE_SMATCH=yes
PRIMARY_COMPILER_VER=$(echo $PRIMARY_COMPILER | sed 's/^gcc//')
cat >build.env <<EOF
FORCE_STRAP_REBUILD=$FORCE_STRAP_REBUILD
ILLUMOS_CLOBBER=$ILLUMOS_CLOBBER
ILLUMOS_ENABLE_DEBUG=$ILLUMOS_ENABLE_DEBUG
PRIMARY_COMPILER=$PRIMARY_COMPILER
PRIMARY_COMPILER_VER=$PRIMARY_COMPILER_VER
SHADOW_COMPILERS=$SHADOW_COMPILERS
HOST_GCC4_PREFIX=$HOST_GCC4_PREFIX
ENABLE_SMATCH=$ENABLE_SMATCH
EOF
[[ -n "$PLATFORM_PASSWORD" ]] && \
echo "PLATFORM_PASSWORD=$PLATFORM_PASSWORD" >> build.env
echo "Doing pre-flight checks... \c "
check_loc
echo "done."
echo "Creating and updating projects directory ..."
create_projects
echo "done."
echo "Installing packages ... \c "
install_packages
echo "done."
echo "Fetching adjuncts tgz... \c "
fetch_adjuncts
echo "done."
echo "Fetching and extracting closed bins ... \c "
fetch_closed
echo "done."
echo "Generating illumos environment file... \c "
generate_env
echo "done."
cat <<EOF
Configuration complete. To build the live image run:
gmake world && gmake live
EOF
# Congrats, we made it
exit 0