220 lines
8.3 KiB
Bash
Executable File
220 lines
8.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# This script is a wrapper to the other download backends.
|
|
# Its role is to ensure atomicity when saving downloaded files
|
|
# back to BR2_DL_DIR, and not clutter BR2_DL_DIR with partial,
|
|
# failed downloads.
|
|
|
|
# To avoid cluttering BR2_DL_DIR, we download to a trashable
|
|
# location, namely in $(BUILD_DIR).
|
|
# Then, we move the downloaded file to a temporary file in the
|
|
# same directory as the final output file.
|
|
# This allows us to finally atomically rename it to its final
|
|
# name.
|
|
# If anything goes wrong, we just remove all the temporaries
|
|
# created so far.
|
|
|
|
# We want to catch any unexpected failure, and exit immediately.
|
|
set -e
|
|
|
|
export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:ru:qf:e"
|
|
|
|
main() {
|
|
local OPT OPTARG
|
|
local backend output hfile recurse quiet rc
|
|
local -a uris
|
|
|
|
# Parse our options; anything after '--' is for the backend
|
|
while getopts ":c:d:D:o:n:N:H:rf:u:q" OPT; do
|
|
case "${OPT}" in
|
|
c) cset="${OPTARG}";;
|
|
d) dl_dir="${OPTARG}";;
|
|
D) old_dl_dir="${OPTARG}";;
|
|
o) output="${OPTARG}";;
|
|
n) raw_base_name="${OPTARG}";;
|
|
N) base_name="${OPTARG}";;
|
|
H) hfile="${OPTARG}";;
|
|
r) recurse="-r";;
|
|
f) filename="${OPTARG}";;
|
|
u) uris+=( "${OPTARG}" );;
|
|
q) quiet="-q";;
|
|
:) error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
|
|
\?) error "unknown option '%s'\n" "${OPTARG}";;
|
|
esac
|
|
done
|
|
|
|
# Forget our options, and keep only those for the backend
|
|
shift $((OPTIND-1))
|
|
|
|
if [ -z "${output}" ]; then
|
|
error "no output specified, use -o\n"
|
|
fi
|
|
|
|
# Legacy handling: check if the file already exists in the global
|
|
# download directory. If it does, hard-link it. If it turns out it
|
|
# was an incorrect download, we'd still check it below anyway.
|
|
# If we can neither link nor copy, fallback to doing a download.
|
|
# NOTE! This is not atomic, is subject to TOCTTOU, but the whole
|
|
# dl-wrapper runs under an flock, so we're safe.
|
|
if [ ! -e "${output}" -a -e "${old_dl_dir}/${filename}" ]; then
|
|
ln "${old_dl_dir}/${filename}" "${output}" || \
|
|
cp "${old_dl_dir}/${filename}" "${output}" || \
|
|
true
|
|
fi
|
|
|
|
# If the output file already exists and:
|
|
# - there's no .hash file: do not download it again and exit promptly
|
|
# - matches all its hashes: do not download it again and exit promptly
|
|
# - fails at least one of its hashes: force a re-download
|
|
# - there's no hash (but a .hash file): consider it a hard error
|
|
if [ -e "${output}" ]; then
|
|
if support/download/check-hash ${quiet} "${hfile}" "${output}" "${output##*/}"; then
|
|
exit 0
|
|
elif [ ${?} -ne 2 ]; then
|
|
# Do not remove the file, otherwise it might get re-downloaded
|
|
# from a later location (i.e. primary -> upstream -> mirror).
|
|
# Do not print a message, check-hash already did.
|
|
exit 1
|
|
fi
|
|
rm -f "${output}"
|
|
warn "Re-downloading '%s'...\n" "${output##*/}"
|
|
fi
|
|
|
|
# Look through all the uris that we were given to download the package
|
|
# source
|
|
download_and_check=0
|
|
rc=1
|
|
for uri in "${uris[@]}"; do
|
|
backend_urlencode="${uri%%+*}"
|
|
backend="${backend_urlencode%|*}"
|
|
case "${backend}" in
|
|
git|svn|cvs|bzr|file|scp|hg) ;;
|
|
*) backend="wget" ;;
|
|
esac
|
|
uri=${uri#*+}
|
|
|
|
urlencode=${backend#*|}
|
|
# urlencode must be "urlencode"
|
|
[ "${urlencode}" != "urlencode" ] && urlencode=""
|
|
|
|
# tmpd is a temporary directory in which backends may store
|
|
# intermediate by-products of the download.
|
|
# tmpf is the file in which the backends should put the downloaded
|
|
# content.
|
|
# tmpd is located in $(BUILD_DIR), so as not to clutter the (precious)
|
|
# $(BR2_DL_DIR)
|
|
# We let the backends create tmpf, so they are able to set whatever
|
|
# permission bits they want (although we're only really interested in
|
|
# the executable bit.)
|
|
tmpd="$(mktemp -d "${BUILD_DIR}/.${output##*/}.XXXXXX")"
|
|
tmpf="${tmpd}/output"
|
|
|
|
# Helpers expect to run in a directory that is *really* trashable, so
|
|
# they are free to create whatever files and/or sub-dirs they might need.
|
|
# Doing the 'cd' here rather than in all backends is easier.
|
|
cd "${tmpd}"
|
|
|
|
# If the backend fails, we can just remove the content of the temporary
|
|
# directory to remove all the cruft it may have left behind, and try
|
|
# the next URI until it succeeds. Once out of URI to try, we need to
|
|
# cleanup and exit.
|
|
if ! "${OLDPWD}/support/download/${backend}" \
|
|
$([ -n "${urlencode}" ] && printf %s '-e') \
|
|
-c "${cset}" \
|
|
-d "${dl_dir}" \
|
|
-n "${raw_base_name}" \
|
|
-N "${base_name}" \
|
|
-f "${filename}" \
|
|
-u "${uri}" \
|
|
-o "${tmpf}" \
|
|
${quiet} ${recurse} -- "${@}"
|
|
then
|
|
# cd back to keep path coherence
|
|
cd "${OLDPWD}"
|
|
rm -rf "${tmpd}"
|
|
continue
|
|
fi
|
|
|
|
# cd back to free the temp-dir, so we can remove it later
|
|
cd "${OLDPWD}"
|
|
|
|
# Check if the downloaded file is sane, and matches the stored hashes
|
|
# for that file
|
|
if support/download/check-hash ${quiet} "${hfile}" "${tmpf}" "${output##*/}"; then
|
|
rc=0
|
|
else
|
|
if [ ${?} -ne 3 ]; then
|
|
rm -rf "${tmpd}"
|
|
continue
|
|
fi
|
|
|
|
# the hash file exists and there was no hash to check the file
|
|
# against
|
|
rc=1
|
|
fi
|
|
download_and_check=1
|
|
break
|
|
done
|
|
|
|
# We tried every URI possible, none seems to work or to check against the
|
|
# available hash. *ABORT MISSION*
|
|
if [ "${download_and_check}" -eq 0 ]; then
|
|
rm -rf "${tmpd}"
|
|
exit 1
|
|
fi
|
|
|
|
# tmp_output is in the same directory as the final output, so we can
|
|
# later move it atomically.
|
|
tmp_output="$(mktemp "${output}.XXXXXX")"
|
|
|
|
# 'mktemp' creates files with 'go=-rwx', so the files are not accessible
|
|
# to users other than the one doing the download (and root, of course).
|
|
# This can be problematic when a shared BR2_DL_DIR is used by different
|
|
# users (e.g. on a build server), where all users may write to the shared
|
|
# location, since other users would not be allowed to read the files
|
|
# another user downloaded.
|
|
# So, we restore the 'go' access rights to a more sensible value, while
|
|
# still abiding by the current user's umask. We must do that before the
|
|
# final 'mv', so just do it now.
|
|
# Some backends (cp and scp) may create executable files, so we need to
|
|
# carry the executable bit if needed.
|
|
[ -x "${tmpf}" ] && new_mode=755 || new_mode=644
|
|
new_mode=$(printf "%04o" $((0${new_mode} & ~0$(umask))))
|
|
chmod ${new_mode} "${tmp_output}"
|
|
|
|
# We must *not* unlink tmp_output, otherwise there is a small window
|
|
# during which another download process may create the same tmp_output
|
|
# name (very, very unlikely; but not impossible.)
|
|
# Using 'cp' is not reliable, since 'cp' may unlink the destination file
|
|
# if it is unable to open it with O_WRONLY|O_TRUNC; see:
|
|
# http://pubs.opengroup.org/onlinepubs/9699919799/utilities/cp.html
|
|
# Since the destination filesystem can be anything, it might not support
|
|
# O_TRUNC, so 'cp' would unlink it first.
|
|
# Use 'cat' and append-redirection '>>' to save to the final location,
|
|
# since that is the only way we can be 100% sure of the behaviour.
|
|
if ! cat "${tmpf}" >>"${tmp_output}"; then
|
|
rm -rf "${tmpd}" "${tmp_output}"
|
|
exit 1
|
|
fi
|
|
rm -rf "${tmpd}"
|
|
|
|
# tmp_output and output are on the same filesystem, so POSIX guarantees
|
|
# that 'mv' is atomic, because it then uses rename() that POSIX mandates
|
|
# to be atomic, see:
|
|
# http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html
|
|
if ! mv -f "${tmp_output}" "${output}"; then
|
|
rm -f "${tmp_output}"
|
|
exit 1
|
|
fi
|
|
|
|
return ${rc}
|
|
}
|
|
|
|
trace() { local msg="${1}"; shift; printf "%s: ${msg}" "${my_name}" "${@}"; }
|
|
warn() { trace "${@}" >&2; }
|
|
errorN() { local ret="${1}"; shift; warn "${@}"; exit ${ret}; }
|
|
error() { errorN 1 "${@}"; }
|
|
|
|
my_name="${0##*/}"
|
|
main "${@}"
|