kumquat-buildroot/support/download/git

96 lines
3.0 KiB
Plaintext
Raw Normal View History

#!/usr/bin/env bash
# We want to catch any unexpected failure, and exit immediately
set -e
# Download helper for git, to be called from the download wrapper script
#
# Call it as:
# .../git [-q] [-r] OUT_FILE REPO_URL CSET BASENAME
#
# -q Be quiet.
# -r Clone and archive sub-modules.
#
# Environment:
pkg-infra: don't use DL_DIR as scratchpad for temporary downloads DL_DIR can be a very precious place for some users: they use it to store all the downloaded archives to share across all their Buildroot (and maybe non-Buildroot) builds. We do not want to trash this location with our temporary downloads (e.g. git, Hg, svn, cvs repository clones/checkouts, or wget, bzr tep tarballs). Turns out that we already have some kind of scratchpad, the BUILD_DIR. Although it is not really a disposable location, that's the best we have so far. Also, we create the temporary tarballs with mktemp using the final tarball, as template, since we want the temporary to be on the same filesystem as the final location, so the 'mv' is just a plain, atomic rename(2), and we are not left with a half-copied file as the final location. Using mktemp ensures all temp file names are unique, so it allows for parallel downloads from different build dirs at the same time, without cloberring each downloads. Note: we're using neither ${TMP} nor ${TMPDIR} since they are shared locations, sometime with little place (eg. tmpfs), and some of the repositories we clone/checkout can be very big. Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr> Cc: Samuel Martin <s.martin49@gmail.com> Cc: Arnout Vandecappelle <arnout@mind.be> Cc: Thomas Petazzoni <thomas.petazzoni@free-electrons.com> Cc: Thomas De Schampheleire <patrickdepinguin@gmail.com> Tested-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com> [tested a particular scenario that used to fail: two separate builds using a shared DL_DIR, ccache enabled, so that they run almost synchronously. These would download the same file at the same time, corrupting each other. With the patches in this series, all works fine.] Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
2014-07-03 21:36:20 +02:00
# GIT : the git command to call
verbose=
recurse=0
while getopts :qr OPT; do
case "${OPT}" in
q) verbose=-q; exec >/dev/null;;
r) recurse=1;;
\?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
esac
done
shift $((OPTIND-1))
output="${1}"
repo="${2}"
cset="${3}"
basename="${4}"
support/download: protect from custom commands with spaces in args Some users may provide custom download commands with spaces in their arguments, like so: BR2_HG="hg --config foo.bar='some space-separated value'" However, the way we currently call those commands does not account for the extra quotes, and each space-separated part of the command is interpreted as separate arguments. Fix that by calling 'eval' on the commands. Because of the eval, we must further quote our own arguments, to avoid the eval further splitting them in case there are spaces (even though we do not support paths with spaces, better be clean from the onset to avoid breakage in the future). We change all the wrappers to use a wrapper-function, even those with a single call, so they all look alike. Note that we do not single-quote some of the variables, like ${verbose} because it can be empty and we really do not want to generate an empty-string argument. That's not a problem, as ${verbose} would not normally contain space-separated values (it could get set to something like '-q -v' but in that case we'd still want two arguments, so that's fine). Reported-by: Thomas De Schampheleire <patrickdepinguin@gmail.com> Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr> Cc: Thomas De Schampheleire <patrickdepinguin@gmail.com> Reviewed-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com> Tested-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com> Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
2015-12-07 10:26:55 +01:00
# Caller needs to single-quote its arguments to prevent them from
# being expanded a second time (in case there are spaces in them)
_git() {
eval ${GIT} "${@}"
}
# Try a shallow clone, since it is faster than a full clone - but that only
# works if the version is a ref (tag or branch). Before trying to do a shallow
# clone we check if ${cset} is in the list provided by git ls-remote. If not
# we fall back on a full clone.
#
# Messages for the type of clone used are provided to ease debugging in case of
# problems
git_done=0
support/download: protect from custom commands with spaces in args Some users may provide custom download commands with spaces in their arguments, like so: BR2_HG="hg --config foo.bar='some space-separated value'" However, the way we currently call those commands does not account for the extra quotes, and each space-separated part of the command is interpreted as separate arguments. Fix that by calling 'eval' on the commands. Because of the eval, we must further quote our own arguments, to avoid the eval further splitting them in case there are spaces (even though we do not support paths with spaces, better be clean from the onset to avoid breakage in the future). We change all the wrappers to use a wrapper-function, even those with a single call, so they all look alike. Note that we do not single-quote some of the variables, like ${verbose} because it can be empty and we really do not want to generate an empty-string argument. That's not a problem, as ${verbose} would not normally contain space-separated values (it could get set to something like '-q -v' but in that case we'd still want two arguments, so that's fine). Reported-by: Thomas De Schampheleire <patrickdepinguin@gmail.com> Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr> Cc: Thomas De Schampheleire <patrickdepinguin@gmail.com> Reviewed-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com> Tested-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com> Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
2015-12-07 10:26:55 +01:00
if [ -n "$(_git ls-remote "'${repo}'" "'${cset}'" 2>&1)" ]; then
printf "Doing shallow clone\n"
if _git clone ${verbose} --depth 1 -b "'${cset}'" "'${repo}'" "'${basename}'"; then
git_done=1
else
printf "Shallow clone failed, falling back to doing a full clone\n"
fi
fi
if [ ${git_done} -eq 0 ]; then
printf "Doing full clone\n"
_git clone ${verbose} "'${repo}'" "'${basename}'"
fi
pushd "${basename}" >/dev/null
# Try to get the special refs exposed by some forges (pull-requests for
# github, changes for gerrit...). There is no easy way to know whether
# the cset the user passed us is such a special ref or a tag or a sha1
# or whatever else. We'll eventually fail at checking out that cset,
# below, if there is an issue anyway. Since most of the cset we're gonna
# have to clone are not such special refs, consign the output to oblivion
# so as not to alarm unsuspecting users, but still trace it as a warning.
if ! _git fetch origin "'${cset}:${cset}'" >/dev/null 2>&1; then
printf "Could not fetch special ref '%s'; assuming it is not special.\n" "${cset}"
fi
# Checkout the required changeset, so that we can update the required
# submodules.
_git checkout -q "'${cset}'"
# Get date of commit to generate a reproducible archive.
# %cD is RFC2822, so it's fully qualified, with TZ and all.
date="$( _git log -1 --pretty=format:%cD )"
# There might be submodules, so fetch them.
if [ ${recurse} -eq 1 ]; then
_git submodule update --init --recursive
fi
# We do not need the .git dir; we keep other .git files, in case they
# are the only files in their directory.
rm -rf .git
popd >/dev/null
# Generate the archive, sort with the C locale so that it is reproducible
tar cf - --numeric-owner --owner=0 --group=0 --mtime="${date}" \
-T <(find "${basename}" -not -type d |LC_ALL=C sort) \
|gzip -n >"${output}"