#!/bin/bash # Copyright (c) 2012 The Native Client Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. set -o nounset set -o errexit # Turn on/off debugging mode readonly PNACL_DEBUG=${PNACL_DEBUG:-false} # True if the scripts are running on the build bots. readonly PNACL_BUILDBOT=${PNACL_BUILDBOT:-false} # Dump all build output to stdout readonly PNACL_VERBOSE=${PNACL_VERBOSE:-false} readonly TIME_AT_STARTUP=$(date '+%s') SetScriptPath() { SCRIPT_PATH="$1" } SetLogDirectory() { TC_LOG="$1" TC_LOG_ALL="${TC_LOG}/ALL" } ###################################################################### # Detect system type ###################################################################### BUILD_PLATFORM=$(uname | tr '[A-Z]' '[a-z]') BUILD_PLATFORM_LINUX=false BUILD_PLATFORM_MAC=false BUILD_PLATFORM_WIN=false if [ "${BUILD_PLATFORM}" == "linux" ] ; then BUILD_PLATFORM_LINUX=true SCONS_BUILD_PLATFORM=linux BUILD_ARCH=${BUILD_ARCH:-$(uname -m)} EXEC_EXT= SO_EXT=.so SO_DIR=lib elif [[ "${BUILD_PLATFORM}" =~ cygwin_nt ]]; then BUILD_PLATFORM=win BUILD_PLATFORM_WIN=true SCONS_BUILD_PLATFORM=win # force 32 bit host because build is also 32 bit on windows. HOST_ARCH=${HOST_ARCH:-x86_32} BUILD_ARCH=${BUILD_ARCH:-x86_32} EXEC_EXT=.exe SO_EXT=.dll SO_DIR=bin # On Windows, DLLs are placed in bin/ # because the dynamic loader searches %PATH% elif [ "${BUILD_PLATFORM}" == "darwin" ] ; then BUILD_PLATFORM=mac BUILD_PLATFORM_MAC=true SCONS_BUILD_PLATFORM=mac # On mac, uname -m is a lie. We always do 64 bit host builds, on # 64 bit build machines HOST_ARCH=${HOST_ARCH:-x86_64} BUILD_ARCH=${BUILD_ARCH:-x86_64} EXEC_EXT= SO_EXT=.dylib SO_DIR=lib else echo "Unknown system '${BUILD_PLATFORM}'" exit -1 fi readonly BUILD_PLATFORM readonly BUILD_PLATFORM_LINUX readonly BUILD_PLATFORM_MAC readonly BUILD_PLATFORM_WIN readonly SCONS_BUILD_PLATFORM readonly SO_EXT readonly SO_DIR BUILD_ARCH_SHORT=${BUILD_ARCH} BUILD_ARCH_X8632=false BUILD_ARCH_X8664=false BUILD_ARCH_ARM=false BUILD_ARCH_MIPS=false if [ "${BUILD_ARCH}" == "x86_32" ] || [ "${BUILD_ARCH}" == "i386" ] || [ "${BUILD_ARCH}" == "i686" ] ; then BUILD_ARCH=x86_32 BUILD_ARCH_X8632=true BUILD_ARCH_SHORT=x86 elif [ "${BUILD_ARCH}" == "x86_64" ] ; then BUILD_ARCH_X8664=true BUILD_ARCH_SHORT=x86 elif [ "${BUILD_ARCH}" == "armv7l" ] ; then BUILD_ARCH_ARM=true BUILD_ARCH_SHORT=arm elif [ "${BUILD_ARCH}" == "mips32" ] || [ "${BUILD_ARCH}" == "mips" ] ; then BUILD_ARCH_MIPS=true BUILD_ARCH_SHORT=mips else echo "Unknown build arch '${BUILD_ARCH}'" exit -1 fi readonly BUILD_ARCH readonly BUILD_ARCH_SHORT readonly BUILD_ARCH_X8632 readonly BUILD_ARCH_X8664 readonly BUILD_ARCH_ARM readonly BUILD_ARCH_MIPS HOST_ARCH=${HOST_ARCH:-${BUILD_ARCH}} HOST_ARCH_X8632=false HOST_ARCH_X8664=false HOST_ARCH_ARM=false HOST_ARCH_MIPS=false if [ "${HOST_ARCH}" == "x86_32" ] || [ "${HOST_ARCH}" == "i386" ] || [ "${HOST_ARCH}" == "i686" ] ; then HOST_ARCH=x86_32 HOST_ARCH_X8632=true elif [ "${HOST_ARCH}" == "x86_64" ] ; then HOST_ARCH_X8664=true elif [ "${HOST_ARCH}" == "armv7l" ] ; then HOST_ARCH_ARM=true elif [ "${HOST_ARCH}" == "mips32" ] || [ "${HOST_ARCH}" == "mips" ] ; then HOST_ARCH_MIPS=true else echo "Unknown host arch '${HOST_ARCH}'" exit -1 fi readonly HOST_ARCH readonly HOST_ARCH_X8632 readonly HOST_ARCH_X8664 readonly HOST_ARCH_ARM readonly HOST_ARCH_MIPS if [ "${BUILD_ARCH}" != "${HOST_ARCH}" ]; then if ! { ${BUILD_ARCH_X8664} && ${HOST_ARCH_X8632}; }; then echo "Cross builds other than build=x86_64 with host=x86_32 not supported" exit -1 fi fi if ${BUILD_PLATFORM_WIN}; then readonly GCLIENT="gclient.bat" readonly GIT="git.bat" readonly HG="hg.bat" readonly SVN="svn.bat" else readonly GCLIENT="gclient" readonly GIT="git" readonly HG="hg" readonly SVN="svn" fi # On Windows, scons expects Windows-style paths (C:\foo\bar) # This function converts cygwin posix paths to Windows-style paths. # On all other platforms, this function does nothing to the path. PosixToSysPath() { local path="$1" if ${BUILD_PLATFORM_WIN}; then cygpath -w "$(GetAbsolutePath "${path}")" else echo "${path}" fi } ###################################################################### # Git repository tools ###################################################################### git-has-changes() { local dir=$1 spushd "${dir}" local status=$(${GIT} status --porcelain --untracked-files=no) spopd [[ ${#status} > 0 ]] return $? } git-assert-no-changes() { local dir=$1 if git-has-changes "${dir}"; then Banner "ERROR: Repository ${dir} has local changes" exit -1 fi } ###################################################################### # Subversion repository tools ###################################################################### svn-at-revision() { local dir="$1" local rev="$2" local repo_rev=$(svn-get-revision "${dir}") [ "${repo_rev}" == "${rev}" ] return $? } #+ svn-get-revision svn-get-revision() { local dir="$1" spushd "${dir}" local rev=$(${SVN} info | grep 'Revision: ' | cut -b 11-) if ! [[ "${rev}" =~ ^[0-9]+$ ]]; then echo "Invalid revision number '${rev}' or invalid repository '${dir}'" 1>&2 exit -1 fi spopd echo "${rev}" } #+ svn-checkout - Checkout an SVN repository svn-checkout() { local url="$1" local dir="$2" local rev="$3" if [ ! -d "${dir}" ]; then StepBanner "SVN-CHECKOUT" "Checking out ${url}" RunWithLog "svn-checkout" ${SVN} co "${url}" "${dir}" -r "${rev}" else SkipBanner "SVN-CHECKOUT" "Using existing SVN repository for ${url}" fi } #+ svn-update - Update an SVN repository svn-update() { local dir="$1" local rev="$2" assert-dir "$dir" \ "SVN repository $(basename "${dir}") doesn't exist." spushd "${dir}" if [[ "$rev" == "tip" ]]; then RunWithLog "svn-update" ${SVN} update else RunWithLog "svn-update" ${SVN} update -r ${rev} fi spopd } svn-has-changes() { local dir="$1" spushd "${dir}" local STATUS=$(${SVN} status) spopd [ "${STATUS}" != "" ] return $? } #+ svn-assert-no-changes - Assert an svn repo has no local changes svn-assert-no-changes() { local dir="$1" if svn-has-changes "${dir}" ; then local name=$(basename "${dir}") Banner "ERROR: Repository ${name} has local changes" exit -1 fi } ###################################################################### # vcs rev info helper ###################################################################### get-field () { cut -d " " -f $1 } git-one-line-rev-info() { local commit=$(${GIT} log -n 1 | head -1 | get-field 2) local url=$(egrep "^[^a-z]+url = " .git/config | head -1 | get-field 3) # variable bypass does implicit whitespace strip echo "[GIT] ${url}: ${commit}" } svn-one-line-rev-info() { local url=$(${SVN} info | egrep 'URL:' | get-field 2) local rev=$(${SVN} info | egrep 'Revision:' | get-field 2) echo "[SVN] ${url}: ${rev}" } #+ one-line-rev-info - show one line summmary for one-line-rev-info() { spushd $1 if [ -d .svn ]; then svn-one-line-rev-info elif [ -d .git ]; then # we currently only git-one-line-rev-info else echo "[$1] Unknown version control system" fi spopd } ###################################################################### # Logging tools ###################################################################### #@ progress - Show build progress (open in another window) progress() { StepBanner "PROGRESS WINDOW" while true; do if [ -f "${TC_LOG_ALL}" ]; then if tail --version > /dev/null; then # GNU coreutils tail tail -s 0.05 --max-unchanged-stats=20 --follow=name "${TC_LOG_ALL}" else # MacOS tail tail -F "${TC_LOG_ALL}" fi fi sleep 1 done } #+ clean-logs - Clean all logs clean-logs() { rm -rf "${TC_LOG}" } # Logged pushd spushd() { LogEcho "-------------------------------------------------------------------" LogEcho "ENTERING: $1" pushd "$1" > /dev/null } # Logged popd spopd() { LogEcho "LEAVING: $(pwd)" popd > /dev/null LogEcho "-------------------------------------------------------------------" LogEcho "ENTERING: $(pwd)" } LogEcho() { mkdir -p "${TC_LOG}" echo "$*" >> "${TC_LOG_ALL}" } RunWithLog() { local log="${TC_LOG}/$1" mkdir -p "${TC_LOG}" shift 1 local ret=1 if ${PNACL_VERBOSE}; then echo "RUNNING: " "$@" | tee "${log}" | tee -a "${TC_LOG_ALL}" "$@" 2>&1 | tee "${log}" | tee -a "${TC_LOG_ALL}" ret=${PIPESTATUS[0]} else echo "RUNNING: " "$@" | tee -a "${TC_LOG_ALL}" &> "${log}" "$@" 2>&1 | tee -a "${TC_LOG_ALL}" &> "${log}" ret=${PIPESTATUS[0]} fi if [ ${ret} -ne 0 ]; then echo Banner "ERROR" echo -n "COMMAND:" PrettyPrint "$@" echo echo "LOGFILE: ${log}" echo echo "PWD: $(pwd)" echo if ${PNACL_BUILDBOT}; then echo "BEGIN LOGFILE Contents." cat "${log}" echo "END LOGFILE Contents." fi return 1 fi return 0 } PrettyPrint() { # Pretty print, respecting parameter grouping for I in "$@"; do local has_space=$(echo "$I" | grep " ") if [ ${#has_space} -gt 0 ]; then echo -n ' "' echo -n "$I" echo -n '"' else echo -n " $I" fi done echo } assert-dir() { local dir="$1" local msg="$2" if [ ! -d "${dir}" ]; then Banner "ERROR: ${msg}" exit -1 fi } assert-file() { local fn="$1" local msg="$2" if [ ! -f "${fn}" ]; then Banner "ERROR: ${fn} does not exist. ${msg}" exit -1 fi } Usage() { egrep "^#@" "${SCRIPT_PATH}" | cut -b 3- } Usage2() { egrep "^#(@|\+)" "${SCRIPT_PATH}" | cut -b 3- } Banner() { echo "" echo " *********************************************************************" echo " | " for arg in "$@" ; do echo " | ${arg}" done echo " | " echo " *********************************************************************" } StepBanner() { local module="$1" if [ $# -eq 1 ]; then echo "" echo "-------------------------------------------------------------------" local padding=$(RepeatStr ' ' 28) echo "${padding}${module}" echo "-------------------------------------------------------------------" else shift 1 local padding=$(RepeatStr ' ' $((20-${#module})) ) echo "[$(TimeStamp)] ${module}${padding}" "$@" fi } TimeStamp() { if date --version &> /dev/null ; then # GNU 'date' date -d "now - ${TIME_AT_STARTUP}sec" '+%M:%S' else # Other 'date' (assuming BSD for now) local time_now=$(date '+%s') local time_delta=$[ ${time_now} - ${TIME_AT_STARTUP} ] date -j -f "%s" "${time_delta}" "+%M:%S" fi } SubBanner() { echo "----------------------------------------------------------------------" echo " $@" echo "----------------------------------------------------------------------" } SkipBanner() { StepBanner "$1" "Skipping $2, already up to date." } RepeatStr() { local str="$1" local count=$2 local ret="" while [ $count -gt 0 ]; do ret="${ret}${str}" count=$((count-1)) done echo "$ret" } Fatal() { echo 1>&2 echo "$@" 1>&2 echo 1>&2 exit -1 } is-ELF() { local F=$(file -b "$1") [[ "${F}" =~ "ELF" ]] } is-Mach() { local F=$(file -b "$1") [[ "${F}" =~ "Mach-O" ]] } is-shell-script() { local F=$(file -b "$1") [[ "${F}" =~ "shell" ]] } get_dir_size_in_mb() { du -msc "$1" | tail -1 | egrep -o "[0-9]+" } confirm-yes() { local msg="$1" while true; do echo -n "${msg} [Y/N]? " local YESNO read YESNO if [ "${YESNO}" == "N" ] || [ "${YESNO}" == "n" ]; then return 1 fi if [ "${YESNO}" == "Y" ] || [ "${YESNO}" == "y" ]; then return 0 fi done } # On Linux with GNU readlink, "readlink -f" would be a quick way # of getting an absolute path, but MacOS has BSD readlink. GetAbsolutePath() { local relpath=$1 local reldir local relname if [ -d "${relpath}" ]; then reldir="${relpath}" relname="" else reldir="$(dirname "${relpath}")" relname="/$(basename "${relpath}")" fi local absdir="$(cd "${reldir}" && pwd)" echo "${absdir}${relname}" } # The Queue* functions provide a simple way of keeping track # of multiple background processes in order to ensure that: # 1) they all finish successfully (with return value 0), and # 2) they terminate if the master process is killed/interrupted. # (This is done using a bash "trap" handler) # # Example Usage: # command1 & # QueueLastProcess # command2 & # QueueLastProcess # command3 & # QueueLastProcess # echo "Waiting for commands finish..." # QueueWait # # TODO(pdox): Right now, this abstraction is only used for # paralellizing translations in the self-build. If we're going # to use this for anything more complex, then throttling the # number of active processes to exactly PNACL_CONCURRENCY would # be a useful feature. CT_WAIT_QUEUE="" QueueLastProcess() { local pid=$! CT_WAIT_QUEUE+=" ${pid}" if ! QueueConcurrent ; then QueueWait fi } QueueConcurrent() { [ ${PNACL_CONCURRENCY} -gt 1 ] } QueueWait() { for pid in ${CT_WAIT_QUEUE} ; do wait ${pid} done CT_WAIT_QUEUE="" } # Add a trap so that if the user Ctrl-C's or kills # this script while background processes are running, # the background processes don't keep going. trap 'QueueKill' SIGINT SIGTERM SIGHUP QueueKill() { echo echo "Killing queued processes: ${CT_WAIT_QUEUE}" for pid in ${CT_WAIT_QUEUE} ; do kill ${pid} &> /dev/null || true done echo CT_WAIT_QUEUE="" } QueueEmpty() { [ "${CT_WAIT_QUEUE}" == "" ] }