You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
536 lines
15 KiB
536 lines
15 KiB
#!/usr/bin/bash |
|
# |
|
# |
|
# This file and its contents are supplied under the terms of the |
|
# Common Development and Distribution License ("CDDL"), version 1.0. |
|
# You may only use this file in accordance with the terms of version |
|
# 1.0 of the CDDL. |
|
# |
|
# A full copy of the text of the CDDL should have accompanied this |
|
# source. A copy of the CDDL is also available via the Internet at |
|
# http://www.illumos.org/license/CDDL. |
|
# |
|
|
|
# |
|
# Copyright 2021 Joyent, Inc. |
|
# Copyright 2022 MNX Cloud, Inc. |
|
# |
|
shopt -s xpg_echo |
|
unalias -a |
|
|
|
PATH=/usr/bin:/usr/sbin:/sbin:/opt/local/bin:/opt/local/sbin |
|
export PATH |
|
|
|
conf_arg0=$(basename $0) |
|
conf_arg0_dir=$(dirname $0) |
|
conf_root=$PWD |
|
conf_ips= |
|
conf_priv="pfexec" |
|
|
|
# |
|
# Build defaults which are unlikely to change, but may be overridden by |
|
# configure-build. |
|
# |
|
source default.configure-build |
|
if [[ -f configure-build ]]; then |
|
source configure-build || fatal "failed to source configure-build" |
|
fi |
|
|
|
# |
|
# We configure the projects framework using a 'configure-projects' file. This |
|
# is a plain text file which contains the project name, the branch we wish to |
|
# build it at, and either a URL to a git repository or 'origin' to denote the |
|
# github.com/joyent/<repo> location. See create_projects. |
|
# |
|
read -r -d '' configure_projects < default.configure-projects |
|
|
|
function warn |
|
{ |
|
local msg="$*" |
|
[[ -z "$msg" ]] && msg="failed" |
|
echo "$conf_arg0: $msg" >&2 |
|
} |
|
|
|
function fatal |
|
{ |
|
local msg="$*" |
|
[[ -z "$msg" ]] && msg="failed" |
|
echo "$conf_arg0: $msg" >&2 |
|
exit 1 |
|
} |
|
|
|
function check_loc |
|
{ |
|
[[ "$(readlink -f "$PWD")" != "$(readlink -f "$conf_arg0_dir")" ]] && \ |
|
fatal "configure must be run from inside $conf_arg0_dir" |
|
[[ "$(uname -s)" != "SunOS" ]] && fatal "build only works on SunOS" |
|
if [[ "$(uname -v)" =~ "joyent_" ]]; then |
|
[[ "$(zonename)" == "global" ]] && fatal \ |
|
"building in a GZ on SmartOS is not supported" |
|
else |
|
echo "Not on SmartOS, assuming IPS and sudo" |
|
conf_ips=1 |
|
conf_priv="sudo" |
|
fi |
|
} |
|
|
|
# |
|
# Checkout a project at the specific branch if the repository doesn't exist, |
|
# or isn't already at the required branch. Warn if the repository contains |
|
# uncommitted contents. |
|
# |
|
function checkout_project |
|
{ |
|
local dir="$1" |
|
local branch="$2" |
|
local url="$3" |
|
|
|
echo "Using $url at $branch in $dir" |
|
|
|
if [[ ! -d "$dir" ]]; then |
|
git clone "$url" "${dir}" || fatal "unable to clone $dir" |
|
fi |
|
|
|
local cur=$(git -C ${dir} branch | grep "^*" | \ |
|
awk 'BEGIN{ FS=" " } { print $2 }') |
|
|
|
# If our repository is already at the desired branch, then do nothing, |
|
# intentionally preserving whatever's in the repository. |
|
if [[ "$cur " != "$branch" ]]; then |
|
git -C ${dir} checkout ${branch} |
|
if [[ $? -ne 0 ]]; then |
|
|
|
# If this is a remote branch, say 'refs/changes/34/1234/1', |
|
# then we might need to fetch it before being able to check it out. |
|
echo "unable to checkout $branch, attempting to fetch it ..." |
|
|
|
# Use a local branch name without '/' chars so a detached head |
|
# doesn't confuse our standard parsing of 'git describe' output |
|
local branch_desc=$(echo $branch | sed -e 's/\//_/g') |
|
|
|
# We assume the remote is named 'origin'. |
|
git -C ${dir} fetch origin ${branch}:${branch_desc} |
|
|
|
# If the above fails, see if the project appears to be a GitHub |
|
# PR-xxx branch name, and if so, add the relevant refspec for |
|
# pull requests. |
|
if [[ $? -ne 0 ]]; then |
|
PR_NUMBER=$(echo "${branch}" | sed -nEe 's/^PR-([0-9]+)$/\1/p') |
|
if [[ -n "${PR_NUMBER}" ]]; then |
|
echo "Branch appears to be from a PR, adding GitHub refspec" |
|
echo " +refs/pull/${PR_NUMBER}/head:refs/remotes/origin/${branch}" |
|
git -C ${dir} config --add remote.origin.fetch \ |
|
+refs/pull/${PR_NUMBER}/head:refs/remotes/origin/${branch} |
|
git -C ${dir} fetch || \ |
|
fatal "unable to fetch using PR refspec" |
|
else |
|
fatal "unable to fetch ${branch} from origin" |
|
fi |
|
fi |
|
git -C ${dir} checkout ${branch_desc} || \ |
|
fatal "unable to checkout ${branch_desc}" |
|
fi |
|
fi |
|
|
|
git -C ${dir} --no-pager show -s |
|
git -C ${dir} describe --all --long --dirty | egrep -e '-dirty$' && |
|
echo "Warning: ${dir} contains uncommitted changes!" |
|
echo |
|
|
|
return 0 |
|
} |
|
|
|
# |
|
# Read a flat file called 'configure-projects' of the form: |
|
# <project name>:<project branch>:[project git repo URL or path] |
|
# |
|
# The special token 'origin' can be used in place of a full git |
|
# repo URL to denote the standard github.com URL for that project. |
|
# If no URL is given, we default to github.com. |
|
# |
|
# Once the configure-projects file has been read (or we've parsed the |
|
# defaults), invoke checkout_project for each one. |
|
# |
|
function create_projects |
|
{ |
|
local dir="projects/local" |
|
local git_stem="https://github.com/joyent" |
|
|
|
[[ -d "$dir" ]] || mkdir -p "$dir" || fatal "failed to create $dir" |
|
|
|
if [[ -f "configure-projects" ]]; then |
|
# note we're not attempting to merge configure-projects with the default |
|
read -r -d '' configure_projects < configure-projects |
|
echo "Info: using the following configure-projects file:" |
|
echo "$configure_projects" |
|
else |
|
echo "Info: no configure-projects file found, using defaults." |
|
fi |
|
echo "$configure_projects" | grep -v '^#' | while read -r line; do |
|
local project_dir=$(echo $line | cut -d: -f 1 | sed -e 's/ //g') |
|
local project_name=$(basename $project_dir) |
|
local git_name="${project_name}.git" |
|
local project_branch=$(echo $line | cut -d: -f 2 | sed -e 's/ //g') |
|
local project_url=$(echo $line | cut -d: -f 3- | sed -e 's/ //g') |
|
|
|
# special cases for some projects |
|
case ${project_name} in |
|
'kvm'|'kvm-cmd') |
|
git_name="illumos-${project_name}.git" |
|
;; |
|
'illumos') |
|
git_name="illumos-joyent.git" |
|
;; |
|
'ur-agent') |
|
git_name="sdc-ur-agent.git" |
|
;; |
|
esac |
|
|
|
# process URL keywords |
|
case $project_url in |
|
'origin'|'') |
|
project_url="$git_stem/$git_name" |
|
;; |
|
esac |
|
checkout_project "projects/$project_dir" "$project_branch" \ |
|
"$project_url" |
|
done || fatal "Error encountered when creating projects directory" |
|
} |
|
|
|
function install_pkgin |
|
{ |
|
local pkglist |
|
|
|
pkglist="build-essential flex libxslt openjdk11 nodejs" |
|
pkglist="$pkglist p5-XML-Parser gettext python27 py39-expat python39" |
|
pkglist="$pkglist coreutils gsed pkg_alternatives cdrtools" |
|
pkglist="$pkglist py27-sqlite3 nasm pigz smartos-build-tools" |
|
|
|
$conf_priv pkgin -f update || fatal "failed to update pkgsrc repository" |
|
|
|
# |
|
# Remove legacy packages replaced by the smartos-build-tools package. |
|
# |
|
for pkg in astmsgtools dmake rpcgen sgstools; do |
|
if pkg_info -qe $pkg; then |
|
$conf_priv pkg_delete ${pkg} |
|
fi |
|
done |
|
|
|
for pkg in $pkglist; do |
|
if ! pkg_info -qe $pkg; then |
|
$conf_priv pkgin -y install $pkg || fatal \ |
|
"failed to install $pkg" |
|
fi |
|
done |
|
} |
|
|
|
function install_ips |
|
{ |
|
fatal "Building on ips based systems has yet to be implemented" |
|
} |
|
|
|
function install_packages |
|
{ |
|
if [[ -z "$conf_ips" ]]; then |
|
install_pkgin |
|
else |
|
install_ips |
|
fi |
|
[[ $? -eq 0 ]] || fatal "failed to install packages" |
|
} |
|
|
|
function fetch_adjuncts |
|
{ |
|
local tgz |
|
[[ -z "$ILLUMOS_ADJUNCT_TARBALL_URL" ]] && fatal \ |
|
"ILLUMOS_ADJUNCT_TARBALL_URL missing from configure" |
|
curl -O $ILLUMOS_ADJUNCT_TARBALL_URL |
|
[[ $? -eq 0 ]] || fatal "failed to fetch adjuncts tarball" |
|
} |
|
|
|
function fetch_closed |
|
{ |
|
local ildir="projects/illumos" |
|
local cld="on-closed-bin.i386.tar.bz2" |
|
local clnd="on-closed-bin-nd.i386.tar.bz2" |
|
[[ -z "$ON_CLOSED_BINS_URL" ]] && fatal \ |
|
"missing ON_CLOSED_BINS_URL from configure" |
|
[[ -z "$ON_CLOSED_BINS_ND_URL" ]] && fatal \ |
|
"missing ON_CLOSED_BINS_ND_URL from configure" |
|
if [[ ! -f $ildir/$cld ]]; then |
|
curl $ON_CLOSED_BINS_URL -o $ildir/$cld || fatal \ |
|
"failed to fetch closed bins (debug)" |
|
fi |
|
if [[ ! -f $ildir/$clnd ]]; then |
|
curl $ON_CLOSED_BINS_ND_URL -o $ildir/$clnd || fatal \ |
|
"failed to fetch closed bins (non-debug)" |
|
fi |
|
cd $ildir >/dev/null 2>&1 || fatal "failed to cd into $ildir" |
|
tar xpjf $cld || fatal \ |
|
"failed to extract closed bins (debug)" |
|
tar xpjf $clnd || fatal \ |
|
"failed to extract closed bins (non-debug)" |
|
cd - >/dev/null 2>&1 |
|
} |
|
|
|
function generate_env |
|
{ |
|
# |
|
# Note: since a nightly clobber build removes the whole proto, and we're |
|
# installing non-Illumos things there, we always use the -i option, and |
|
# ILLUMOS_CLOBBER is handled separately in Makefile. |
|
# |
|
local nopts="-CiLmMNnt" |
|
local lprefix |
|
|
|
case $ILLUMOS_ENABLE_DEBUG in |
|
exclusive) |
|
nopts="${nopts}DF" ;; |
|
yes) |
|
nopts="${nopts}D" ;; |
|
no|"") |
|
;; |
|
*) |
|
echo "unknown debug option $ILLUMOS_ENABLE_DEBUG" >&2 |
|
exit 2 ;; |
|
esac |
|
|
|
GNUC_ROOT="$conf_root/proto.strap/usr/gcc/$PRIMARY_COMPILER_VER" |
|
PRIMARY_CC="gcc$PRIMARY_COMPILER_VER,$GNUC_ROOT/bin/gcc,gnu" |
|
PRIMARY_CCC="gcc$PRIMARY_COMPILER_VER,$GNUC_ROOT/bin/g++,gnu" |
|
|
|
SHADOW_CCS= |
|
SHADOW_CCCS= |
|
|
|
IFS=, |
|
for cc in $SHADOW_COMPILERS; do |
|
gcc_ver=$(echo $cc | sed 's/^gcc//') |
|
root="$conf_root/proto.strap/usr/gcc/$gcc_ver" |
|
SHADOW_CCS+=" gcc${gcc_ver},$root/bin/gcc,gnu" |
|
SHADOW_CCCS+=" gcc${gcc_ver},$root/bin/g++,gnu" |
|
done |
|
unset IFS |
|
|
|
if [[ "$ENABLE_SMATCH" = "yes" ]]; then |
|
SHADOW_CCS+=" smatch,\$BUILD_TOOLS/onbld/bin/\$MACH/smatch,smatch" |
|
fi |
|
|
|
lprefix=$(echo $conf_root | tr / _) |
|
[[ $? -eq 0 ]] || fatal "failed to create lock prefix" |
|
|
|
cat > "projects/illumos/illumos.sh" <<EOF |
|
NIGHTLY_OPTIONS="$nopts"; export NIGHTLY_OPTIONS |
|
GATE="${RELEASE_VER}"; export GATE |
|
CODEMGR_WS="$conf_root/projects/illumos"; export CODEMGR_WS |
|
# NOTE: build_illumos modifies this initial constant |
|
MAX_JOBS=128 |
|
maxjobs() { |
|
ncpu=\`kstat -p cpu_info:::state | grep -c on-line\` |
|
zcpucap=\`pfexec mdata-get sdc:cpu_cap\` |
|
if [[ \$zcpucap -eq 0 || \$zcpucap == "" ]]; then |
|
zcpu=\${MAX_JOBS} |
|
else |
|
zcpu=\`expr \$zcpucap / 100\` |
|
fi |
|
if [[ \$zcpu -lt \$ncpu ]]; then |
|
expr \$zcpu + 2 |
|
elif [[ \$(( \$ncpu + 2 )) -lt \${MAX_JOBS} ]]; then |
|
expr \$ncpu + 2 |
|
else |
|
printf "%d\n" \${MAX_JOBS} |
|
fi |
|
} |
|
DMAKE_MAX_JOBS=\`maxjobs\`; export DMAKE_MAX_JOBS |
|
|
|
PARENT_WS=""; export PARENT_WS |
|
CLONE_WS="http://hg.illumos.org/illumos-gate" export CLONE_WS |
|
STAFFER="nobody"; export STAFFER |
|
BUILD_PROJECT=""; export BUILD_PROJECT |
|
LOCKNAME="\`whoami\`_${lprefix}_nightly.lock"; export LOCKNAME |
|
ATLOG="\$CODEMGR_WS/log"; export ATLOG |
|
LOGFILE="\$ATLOG/nightly.log"; export LOGFILE |
|
MACH=\`uname -p\`; export MACH |
|
ON_CLOSED_BINS="\$CODEMGR_WS/closed"; export ON_CLOSED_BINS |
|
REF_PROTO_LIST="\$PARENT_WS/usr/src/proto_list_\${MACH}"; |
|
export REF_PROTO_LIST |
|
ROOT="$conf_root/proto"; export ROOT |
|
ADJUNCT_PROTO="$conf_root/proto.strap"; export ADJUNCT_PROTO |
|
NATIVE_ADJUNCT="/opt/local"; export NATIVE_ADJUNCT |
|
SRC="\$CODEMGR_WS/usr/src"; export SRC |
|
VERSION="\$GATE"; export VERSION |
|
PARENT_ROOT="$conf_root/proto"; export PARENT_ROOT |
|
PARENT_TOOLS_ROOT="\$PARENT_WS/usr/src/tools/proto/root_\$MACH-nd" |
|
export PARENT_TOOLS_ROOT |
|
PKGARCHIVE="\${CODEMGR_WS}/packages/\${MACH}/nightly"; |
|
export PKGARCHIVE |
|
PKGPUBLISHER_REDIST="${PUBLISHER}"; export PKGPUBLISHER_REDIST |
|
MAKEFLAGS=k; export MAKEFLAGS |
|
UT_NO_USAGE_TRACKING="1"; export UT_NO_USAGE_TRACKING |
|
MULTI_PROTO="no"; export MULTI_PROTO |
|
BUILD_TOOLS="\$SRC/tools/proto/root_\${MACH}-nd/opt"; |
|
export BUILD_TOOLS |
|
SPRO_ROOT=/opt/SUNWspro; export SPRO_ROOT |
|
SPRO_VROOT=\$SPRO_ROOT; export SPRO_VROOT |
|
GNU_ROOT="$conf_root/proto.strap/usr/gnu" export GNU_ROOT |
|
# Use GCC as the primary compiler |
|
__GNUC=""; export __GNUC |
|
# Use GCC4 or later specific flags |
|
__GNUC4=""; export __GNUC4 |
|
# root of \$PRIMARY_CC |
|
GNUC_ROOT="$GNUC_ROOT"; export GNUC_ROOT |
|
PRIMARY_CC="$PRIMARY_CC"; export PRIMARY_CC |
|
PRIMARY_CCC="$PRIMARY_CCC"; export PRIMARY_CCC |
|
SHADOW_CCS="$SHADOW_CCS"; export SHADOW_CCS |
|
SHADOW_CCCS="$SHADOW_CCCS"; export SHADOW_CCCS |
|
JAVA_ROOT=/opt/local/java/openjdk11; export JAVA_ROOT |
|
FLEX=/opt/local/bin/flex; export FLEX |
|
GNUXGETTEXT=/opt/local/bin/xgettext; export GNUXGETTEXT |
|
PYTHON3=/opt/local/bin/python3.9; export PYTHON3 |
|
PYTHON3_VERSION=3.9; export PYTHON3_VERSION |
|
PYTHON3_PKGVERS=-39; export PYTHON3_PKGVERS |
|
PYTHON3_SUFFIX=''; export PYTHON3_SUFFIX |
|
BLD_JAVA_11=''; export BLD_JAVA_11 |
|
|
|
# Disable the split 32/64 perl module builds since smartos doesn't exec perl |
|
# via isaexec |
|
# export BUILDPERL32='#' |
|
export BUILDPERL64='#' |
|
|
|
# |
|
# Note that this isn't the strap perl: that only really exists for the benefit |
|
# of projects/illumos-extra/openssl |
|
# |
|
PERL=/opt/local/bin/perl; export PERL |
|
ELFDUMP=/usr/bin/elfdump; export ELFDUMP |
|
LORDER=/usr/bin/lorder; export LORDER |
|
MCS=/usr/bin/mcs; export MCS |
|
NM=/usr/bin/nm; export NM |
|
STRIP=/usr/bin/strip; export STRIP |
|
TSORT=/usr/bin/tsort; export TSORT |
|
AR=/usr/bin/ar; export AR |
|
# |
|
# We override $MAKE in ./tools/build_illumos so we can properly |
|
# bootstrap the tools build. |
|
# |
|
if [[ -z "\$MAKE" ]]; then |
|
MAKE="\$SRC/tools/proto/root_i386-nd/opt/onbld/bin/i386/dmake"; export MAKE |
|
fi |
|
LEX=/opt/local/bin/lex; export LEX |
|
YACC=/opt/local/bin/yacc; export YACC |
|
BISON=/opt/local/bin/bison; export BISON |
|
GM4=/opt/local/bin/gm4; export GM4 |
|
ASTBINDIR=/opt/local/ast/bin; export ASTBINDIR |
|
LD_TOXIC_PATH="\$ROOT/lib:\$ROOT/usr/lib"; export LD_TOXIC_PATH |
|
# generate buildversion for illumos buildversion module |
|
BUILDVERSION_EXEC="${conf_root}/tools/build_etcrelease -g"; |
|
export BUILDVERSION_EXEC |
|
|
|
EOF |
|
[[ $? -eq 0 ]] || fatal "failed to write illumos nightly env file" |
|
} |
|
|
|
# |
|
# This construction lets us place a block of text verbatim into $usage. |
|
# |
|
read -r -d '' usage <<EOF |
|
./configure [options]: |
|
-c |
|
clobber Illumos before each build [default: no] |
|
-d |
|
build Illumos in DEBUG mode only [default: no] |
|
-h |
|
this message |
|
-p gcc7 |
|
primary compiler version [default: gcc7] |
|
-P password |
|
platform root password [default: randomly chosen] |
|
-r |
|
full strap build (no cache) [default: no] |
|
-S |
|
do *not* run smatch [default is to run smatch] |
|
-s gcc10 |
|
shadow compilers, comma delimited (gcc10,gcc#) [default: none] |
|
EOF |
|
|
|
while getopts "cdhp:P:rSs:" arg; do |
|
case $arg in |
|
c) |
|
ILLUMOS_CLOBBER=yes ;; |
|
|
|
# |
|
# We have only one proto area: we'll skip building non-debug here, as |
|
# the results are over-written by nightly. If the user really wants a |
|
# sanity check build, they can specify =yes via the env var, or by |
|
# directly editing illumos.sh. |
|
# |
|
d) |
|
ILLUMOS_ENABLE_DEBUG=exclusive ;; |
|
h) |
|
echo "$usage" |
|
exit 0 ;; |
|
p) |
|
PRIMARY_COMPILER=$OPTARG |
|
FORCE_STRAP_REBUILD=yes ;; |
|
P) |
|
PLATFORM_PASSWORD="$OPTARG" ;; |
|
r) |
|
FORCE_STRAP_REBUILD=yes ;; |
|
S) |
|
ENABLE_SMATCH=no ;; |
|
s) |
|
SHADOW_COMPILERS=$OPTARG |
|
FORCE_STRAP_REBUILD=yes ;; |
|
?) |
|
echo "$usage" >&2 |
|
exit 2 ;; |
|
esac |
|
done |
|
|
|
[[ -n "$ILLUMOS_ENABLE_DEBUG" ]] || ILLUMOS_ENABLE_DEBUG=no |
|
[[ -n "$ILLUMOS_CLOBBER" ]] || ILLUMOS_CLOBBER=no |
|
[[ -n "$PRIMARY_COMPILER" ]] || PRIMARY_COMPILER=gcc7 |
|
[[ -n "$ENABLE_SMATCH" ]] || ENABLE_SMATCH=yes |
|
|
|
PRIMARY_COMPILER_VER=$(echo $PRIMARY_COMPILER | sed 's/^gcc//') |
|
|
|
cat >build.env <<EOF |
|
FORCE_STRAP_REBUILD=$FORCE_STRAP_REBUILD |
|
ILLUMOS_CLOBBER=$ILLUMOS_CLOBBER |
|
ILLUMOS_ENABLE_DEBUG=$ILLUMOS_ENABLE_DEBUG |
|
PRIMARY_COMPILER=$PRIMARY_COMPILER |
|
PRIMARY_COMPILER_VER=$PRIMARY_COMPILER_VER |
|
SHADOW_COMPILERS=$SHADOW_COMPILERS |
|
HOST_GCC4_PREFIX=$HOST_GCC4_PREFIX |
|
ENABLE_SMATCH=$ENABLE_SMATCH |
|
EOF |
|
|
|
[[ -n "$PLATFORM_PASSWORD" ]] && \ |
|
echo "PLATFORM_PASSWORD=$PLATFORM_PASSWORD" >> build.env |
|
|
|
echo "Doing pre-flight checks... \c " |
|
check_loc |
|
echo "done." |
|
echo "Creating and updating projects directory ..." |
|
create_projects |
|
echo "done." |
|
echo "Installing packages ... \c " |
|
install_packages |
|
echo "done." |
|
echo "Fetching adjuncts tgz... \c " |
|
fetch_adjuncts |
|
echo "done." |
|
echo "Fetching and extracting closed bins ... \c " |
|
fetch_closed |
|
echo "done." |
|
echo "Generating illumos environment file... \c " |
|
generate_env |
|
echo "done." |
|
|
|
cat <<EOF |
|
Configuration complete. To build the live image run: |
|
gmake world && gmake live |
|
EOF |
|
|
|
# Congrats, we made it |
|
exit 0
|
|
|