2017-07-20 16:35:14 +02:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
|
|
# Copyright (C) 2016 Samuel Martin <s.martin49@gmail.com>
|
|
|
|
# Copyright (C) 2017 Wolfgang Grandegger <wg@grandegger.com>
|
|
|
|
#
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
|
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
|
|
# (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
# General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
# along with this program; if not, write to the Free Software
|
|
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
|
|
|
|
usage() {
|
|
|
|
cat <<EOF >&2
|
|
|
|
Usage: ${0} TREE_KIND
|
|
|
|
|
|
|
|
Description:
|
|
|
|
|
|
|
|
This script scans a tree and sanitize ELF files' RPATH found in there.
|
|
|
|
|
|
|
|
Sanitization behaves the same whatever the kind of the processed tree,
|
|
|
|
but the resulting RPATH differs. The rpath sanitization is done using
|
2017-07-22 13:15:42 +02:00
|
|
|
"patchelf --make-rpath-relative".
|
2017-07-20 16:35:14 +02:00
|
|
|
|
|
|
|
Arguments:
|
|
|
|
|
|
|
|
TREE_KIND Kind of tree to be processed.
|
|
|
|
Allowed values: host, target, staging
|
|
|
|
|
|
|
|
Environment:
|
|
|
|
|
|
|
|
PATCHELF patchelf program to use
|
|
|
|
(default: HOST_DIR/bin/patchelf)
|
|
|
|
|
|
|
|
HOST_DIR host directory
|
|
|
|
STAGING_DIR staging directory
|
|
|
|
TARGET_DIR target directory
|
|
|
|
|
|
|
|
TOOLCHAIN_EXTERNAL_DOWNLOAD_INSTALL_DIR
|
|
|
|
(default HOST_DIR/opt/ext-toolchain)
|
2017-07-22 13:15:41 +02:00
|
|
|
|
|
|
|
Returns: 0 if success or 1 in case of error
|
|
|
|
|
2017-07-20 16:35:14 +02:00
|
|
|
EOF
|
|
|
|
}
|
|
|
|
|
|
|
|
: ${PATCHELF:=${HOST_DIR}/bin/patchelf}
|
|
|
|
|
|
|
|
# ELF files should not be in these sub-directories
|
|
|
|
HOST_EXCLUDEPATHS="/share/terminfo"
|
|
|
|
STAGING_EXCLUDEPATHS="/usr/include /usr/share/terminfo"
|
support/scripts/fix-rpath: exclude /lib/firmware in the target
The /lib/firmware directory contains random firmware for various
devices. It happens that some of them might be or appear to be ELF
files, but they shouldn't be checked by fix-rpath. For example, one of
the Qualcomm VPU firmware file appears to be an ELF file, but patchelf
isn't happy about it:
$ ./output/host/bin/patchelf --print-rpath output/target/lib/firmware/qcom/venus-4.2/venus.b00
patchelf: patchelf.cc:387: void ElfFile<Elf_Ehdr, Elf_Phdr, Elf_Shdr, Elf_Addr, Elf_Off, Elf_Dyn, Elf_Sym>::parse() [with Elf_Ehdr = Elf32_Ehdr; Elf_Phdr = Elf32_Phdr; Elf_Shdr = Elf32_Shdr; Elf_Addr = unsigned int; Elf_Off = unsigned int; Elf_Dyn = Elf32_Dyn; Elf_Sym = Elf32_Sym]: Assertion `shstrtabIndex < shdrs.size()' failed.
Aborted (core dumped)
Even though patchelf definitely shouldn't crash, it anyway doesn't
make sense to check ELF files in /lib/firmware, so let's exclude this
directory from our check.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Acked-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
2018-04-12 13:50:09 +02:00
|
|
|
TARGET_EXCLUDEPATHS="/lib/firmware"
|
2017-07-20 16:35:14 +02:00
|
|
|
|
|
|
|
main() {
|
|
|
|
local rootdir
|
|
|
|
local tree="${1}"
|
|
|
|
local find_args=( )
|
|
|
|
local sanitize_extra_args=( )
|
|
|
|
|
2017-07-22 13:15:41 +02:00
|
|
|
if ! "${PATCHELF}" --version > /dev/null 2>&1; then
|
|
|
|
echo "Error: can't execute patchelf utility '${PATCHELF}'"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2017-07-20 16:35:14 +02:00
|
|
|
case "${tree}" in
|
|
|
|
host)
|
|
|
|
rootdir="${HOST_DIR}"
|
|
|
|
|
|
|
|
# do not process the sysroot (only contains target binaries)
|
|
|
|
find_args+=( "-path" "${STAGING_DIR}" "-prune" "-o" )
|
|
|
|
|
|
|
|
# do not process the external toolchain installation directory to
|
|
|
|
# avoid breaking it.
|
|
|
|
test "${TOOLCHAIN_EXTERNAL_DOWNLOAD_INSTALL_DIR}" != "" && \
|
|
|
|
find_args+=( "-path" "${TOOLCHAIN_EXTERNAL_DOWNLOAD_INSTALL_DIR}" "-prune" "-o" )
|
|
|
|
|
|
|
|
for excludepath in ${HOST_EXCLUDEPATHS}; do
|
2017-07-22 13:15:42 +02:00
|
|
|
find_args+=( "-path" "${HOST_DIR}""${excludepath}" "-prune" "-o" )
|
2017-07-20 16:35:14 +02:00
|
|
|
done
|
|
|
|
|
|
|
|
# do not process the patchelf binary but a copy to work-around "file in use"
|
|
|
|
find_args+=( "-path" "${PATCHELF}" "-prune" "-o" )
|
|
|
|
cp "${PATCHELF}" "${PATCHELF}.__to_be_patched"
|
|
|
|
|
|
|
|
# we always want $ORIGIN-based rpaths to make it relocatable.
|
|
|
|
sanitize_extra_args+=( "--relative-to-file" )
|
|
|
|
;;
|
|
|
|
|
|
|
|
staging)
|
|
|
|
rootdir="${STAGING_DIR}"
|
|
|
|
|
|
|
|
# ELF files should not be in these sub-directories
|
|
|
|
for excludepath in ${STAGING_EXCLUDEPATHS}; do
|
|
|
|
find_args+=( "-path" "${STAGING_DIR}""${excludepath}" "-prune" "-o" )
|
|
|
|
done
|
|
|
|
|
|
|
|
# should be like for the target tree below
|
|
|
|
sanitize_extra_args+=( "--no-standard-lib-dirs" )
|
|
|
|
;;
|
|
|
|
|
|
|
|
target)
|
|
|
|
rootdir="${TARGET_DIR}"
|
support/scripts/fix-rpath: exclude /lib/firmware in the target
The /lib/firmware directory contains random firmware for various
devices. It happens that some of them might be or appear to be ELF
files, but they shouldn't be checked by fix-rpath. For example, one of
the Qualcomm VPU firmware file appears to be an ELF file, but patchelf
isn't happy about it:
$ ./output/host/bin/patchelf --print-rpath output/target/lib/firmware/qcom/venus-4.2/venus.b00
patchelf: patchelf.cc:387: void ElfFile<Elf_Ehdr, Elf_Phdr, Elf_Shdr, Elf_Addr, Elf_Off, Elf_Dyn, Elf_Sym>::parse() [with Elf_Ehdr = Elf32_Ehdr; Elf_Phdr = Elf32_Phdr; Elf_Shdr = Elf32_Shdr; Elf_Addr = unsigned int; Elf_Off = unsigned int; Elf_Dyn = Elf32_Dyn; Elf_Sym = Elf32_Sym]: Assertion `shstrtabIndex < shdrs.size()' failed.
Aborted (core dumped)
Even though patchelf definitely shouldn't crash, it anyway doesn't
make sense to check ELF files in /lib/firmware, so let's exclude this
directory from our check.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Acked-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
2018-04-12 13:50:09 +02:00
|
|
|
|
|
|
|
for excludepath in ${TARGET_EXCLUDEPATHS}; do
|
|
|
|
find_args+=( "-path" "${TARGET_DIR}""${excludepath}" "-prune" "-o" )
|
|
|
|
done
|
|
|
|
|
2017-07-20 16:35:14 +02:00
|
|
|
# we don't want $ORIGIN-based rpaths but absolute paths without rootdir.
|
|
|
|
# we also want to remove rpaths pointing to /lib or /usr/lib.
|
|
|
|
sanitize_extra_args+=( "--no-standard-lib-dirs" )
|
|
|
|
;;
|
|
|
|
|
|
|
|
*)
|
|
|
|
usage
|
|
|
|
exit 1
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
|
|
|
find_args+=( "-type" "f" "-print" )
|
|
|
|
|
|
|
|
while read file ; do
|
|
|
|
# check if it's an ELF file
|
core: implement per-package SDK and target
This commit implements the core of the move to per-package SDK and
target directories. The main idea is that instead of having a global
output/host and output/target in which all packages install files, we
switch to per-package host and target directories, that only contain
their explicit dependencies.
There are two main benefits:
- Packages will now see only the dependencies they explicitly list in
their <pkg>_DEPENDENCIES variable, and the recursive dependencies
thereof.
- We can support top-level parallel build properly, because a package
only "sees" its own host directory and target directory, isolated
from the build of other packages that can happen in parallel.
It works as follows:
- A new output/per-package/ directory is created, which will contain
one sub-directory per package, and inside it, a "host" directory
and a "target" directory:
output/per-package/busybox/target
output/per-package/busybox/host
output/per-package/host-fakeroot/target
output/per-package/host-fakeroot/host
This output/per-package/ directory is PER_PACKAGE_DIR.
- The global TARGET_DIR and HOST_DIR variable now automatically point
to the per-package directory when PKG is defined. So whenever a
package references $(HOST_DIR) or $(TARGET_DIR) in its build
process, it effectively references the per-package host/target
directories. Note that STAGING_DIR is a sub-dir of HOST_DIR, so it
is handled as well.
- Of course, packages have dependencies, so those dependencies must
be installed in the per-package host and target directories. To do
so, we simply rsync (using hard links to save space and time) the
host and target directories of the direct dependencies of the
package to the current package host and target directories.
We only need to take care of direct dependencies (and not
recursively all dependencies), because we accumulate into those
per-package host and target directories the files installed by the
dependencies. Note that this only works because we make the
assumption that one package does *not* overwrite files installed by
another package.
This is done for "extract dependencies" at the beginning of the
extract step, and for "normal dependencies" at the beginning of the
configure step.
This is basically enough to make per-package SDK and target work. The
only gotcha is that at the end of the build, output/target and
output/host are empty, which means that:
- The filesystem image creation code cannot work.
- We don't have a SDK to build code outside of Buildroot.
In order to fix this, this commit extends the target-finalize step so
that it starts by populating output/target and output/host by
rsync-ing into them the target and host directories of all packages
listed in the $(PACKAGES) variable. It is necessary to do this
sequentially in the target-finalize step and not in each
package. Doing it in package installation means that it can be done in
parallel. In that case, there is a chance that two rsyncs are creating
the same hardlink or directory at the same time, which makes one of
them fail.
This change to per-package directories has an impact on the RPATH
built into the host binaries, as those RPATH now point to various
per-package host directories, and no longer to the global host
directory. We do not try to rewrite such RPATHs during the build as
having such RPATHs is perfectly fine, but we still need to handle two
fallouts from this change:
- The check-host-rpath script, which verifies at the end of each
package installation that it has the appropriate RPATH, is modified
to understand that a RPATH to $(PER_PACKAGE_DIR)/<pkg>/host/lib is
a correct RPAT.
- The fix-rpath script, which mungles the RPATH mainly for the SDK
preparation, is modified to rewrite the RPATH to not point to
per-package directories. Indeed the patchelf --make-rpath-relative
call only works if the RPATH points to the ROOTDIR passed as
argument, and this ROOTDIR is the global host directory. Rewriting
the RPATH to not point to per-package host directories prior to
this is an easy solution to this issue.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
2019-11-05 17:46:40 +01:00
|
|
|
rpath=$(${PATCHELF} --print-rpath "${file}" 2>&1)
|
|
|
|
if test $? -ne 0 ; then
|
|
|
|
continue
|
2017-07-20 16:35:14 +02:00
|
|
|
fi
|
core: implement per-package SDK and target
This commit implements the core of the move to per-package SDK and
target directories. The main idea is that instead of having a global
output/host and output/target in which all packages install files, we
switch to per-package host and target directories, that only contain
their explicit dependencies.
There are two main benefits:
- Packages will now see only the dependencies they explicitly list in
their <pkg>_DEPENDENCIES variable, and the recursive dependencies
thereof.
- We can support top-level parallel build properly, because a package
only "sees" its own host directory and target directory, isolated
from the build of other packages that can happen in parallel.
It works as follows:
- A new output/per-package/ directory is created, which will contain
one sub-directory per package, and inside it, a "host" directory
and a "target" directory:
output/per-package/busybox/target
output/per-package/busybox/host
output/per-package/host-fakeroot/target
output/per-package/host-fakeroot/host
This output/per-package/ directory is PER_PACKAGE_DIR.
- The global TARGET_DIR and HOST_DIR variable now automatically point
to the per-package directory when PKG is defined. So whenever a
package references $(HOST_DIR) or $(TARGET_DIR) in its build
process, it effectively references the per-package host/target
directories. Note that STAGING_DIR is a sub-dir of HOST_DIR, so it
is handled as well.
- Of course, packages have dependencies, so those dependencies must
be installed in the per-package host and target directories. To do
so, we simply rsync (using hard links to save space and time) the
host and target directories of the direct dependencies of the
package to the current package host and target directories.
We only need to take care of direct dependencies (and not
recursively all dependencies), because we accumulate into those
per-package host and target directories the files installed by the
dependencies. Note that this only works because we make the
assumption that one package does *not* overwrite files installed by
another package.
This is done for "extract dependencies" at the beginning of the
extract step, and for "normal dependencies" at the beginning of the
configure step.
This is basically enough to make per-package SDK and target work. The
only gotcha is that at the end of the build, output/target and
output/host are empty, which means that:
- The filesystem image creation code cannot work.
- We don't have a SDK to build code outside of Buildroot.
In order to fix this, this commit extends the target-finalize step so
that it starts by populating output/target and output/host by
rsync-ing into them the target and host directories of all packages
listed in the $(PACKAGES) variable. It is necessary to do this
sequentially in the target-finalize step and not in each
package. Doing it in package installation means that it can be done in
parallel. In that case, there is a chance that two rsyncs are creating
the same hardlink or directory at the same time, which makes one of
them fail.
This change to per-package directories has an impact on the RPATH
built into the host binaries, as those RPATH now point to various
per-package host directories, and no longer to the global host
directory. We do not try to rewrite such RPATHs during the build as
having such RPATHs is perfectly fine, but we still need to handle two
fallouts from this change:
- The check-host-rpath script, which verifies at the end of each
package installation that it has the appropriate RPATH, is modified
to understand that a RPATH to $(PER_PACKAGE_DIR)/<pkg>/host/lib is
a correct RPAT.
- The fix-rpath script, which mungles the RPATH mainly for the SDK
preparation, is modified to rewrite the RPATH to not point to
per-package directories. Indeed the patchelf --make-rpath-relative
call only works if the RPATH points to the ROOTDIR passed as
argument, and this ROOTDIR is the global host directory. Rewriting
the RPATH to not point to per-package host directories prior to
this is an easy solution to this issue.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
2019-11-05 17:46:40 +01:00
|
|
|
|
|
|
|
# make files writable if necessary
|
|
|
|
changed=$(chmod -c u+w "${file}")
|
|
|
|
|
|
|
|
# With per-package directory support, most RPATH of host
|
|
|
|
# binaries will point to per-package directories. This won't
|
|
|
|
# work with the --make-rpath-relative ${rootdir} invocation as
|
|
|
|
# the per-package host directory is not within ${rootdir}. So,
|
|
|
|
# we rewrite all RPATHs pointing to per-package directories so
|
|
|
|
# that they point to the global host directry.
|
package/pkg-generic.mk, support/scripts/fix-rpath: fix per-package regexp
Commit c4e6d5c8be6ada8e7c60950e3b499c55d48761cb ("core: implement
per-package SDK and target") had a mistake on the regexp that is used
to match $(PER_PACKAGE_DIR)/<something>/, and due to this, the regexp
was never matched.
The + sign in [^/]+ which was suggested by Yann E. Morin during the
review of the per-package patch series (instead of [^/]*) needs to be
escaped to be taken into account correctly. Without this, the regexp
doesn't match, and the replacement is not done, causing:
(1) For the libtool fixup in pkg-generic.mk, the lack of replacement
causes libtool .la files to not be tweaked as expected, which it
turn causes build failures reported by the autobuilder.
(2) For the fix-rpath, the RPATH of host binaries in the SDK were not
correct.
Interestingly, we have the same regexp in
support/scripts/check-host-rpath, but here the + sign does not need to
be escaped.
Fixes:
http://autobuild.buildroot.net/results/d4d996f3923699e266afd40cc7180de0f7257d99/ (libsvg-cairo)
http://autobuild.buildroot.net/results/56330f86872f67a2ce328e09b4c7b12aa835a432/ (bind)
http://autobuild.buildroot.net/results/9e0fc42d2c9f856b92954b08019b83ce668ef289/ (ibrcommon)
and probably a number of other similar issues
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
2019-12-11 13:25:00 +01:00
|
|
|
changed_rpath=$(echo ${rpath} | sed "s@${PER_PACKAGE_DIR}/[^/]\+/host@${HOST_DIR}@")
|
core: implement per-package SDK and target
This commit implements the core of the move to per-package SDK and
target directories. The main idea is that instead of having a global
output/host and output/target in which all packages install files, we
switch to per-package host and target directories, that only contain
their explicit dependencies.
There are two main benefits:
- Packages will now see only the dependencies they explicitly list in
their <pkg>_DEPENDENCIES variable, and the recursive dependencies
thereof.
- We can support top-level parallel build properly, because a package
only "sees" its own host directory and target directory, isolated
from the build of other packages that can happen in parallel.
It works as follows:
- A new output/per-package/ directory is created, which will contain
one sub-directory per package, and inside it, a "host" directory
and a "target" directory:
output/per-package/busybox/target
output/per-package/busybox/host
output/per-package/host-fakeroot/target
output/per-package/host-fakeroot/host
This output/per-package/ directory is PER_PACKAGE_DIR.
- The global TARGET_DIR and HOST_DIR variable now automatically point
to the per-package directory when PKG is defined. So whenever a
package references $(HOST_DIR) or $(TARGET_DIR) in its build
process, it effectively references the per-package host/target
directories. Note that STAGING_DIR is a sub-dir of HOST_DIR, so it
is handled as well.
- Of course, packages have dependencies, so those dependencies must
be installed in the per-package host and target directories. To do
so, we simply rsync (using hard links to save space and time) the
host and target directories of the direct dependencies of the
package to the current package host and target directories.
We only need to take care of direct dependencies (and not
recursively all dependencies), because we accumulate into those
per-package host and target directories the files installed by the
dependencies. Note that this only works because we make the
assumption that one package does *not* overwrite files installed by
another package.
This is done for "extract dependencies" at the beginning of the
extract step, and for "normal dependencies" at the beginning of the
configure step.
This is basically enough to make per-package SDK and target work. The
only gotcha is that at the end of the build, output/target and
output/host are empty, which means that:
- The filesystem image creation code cannot work.
- We don't have a SDK to build code outside of Buildroot.
In order to fix this, this commit extends the target-finalize step so
that it starts by populating output/target and output/host by
rsync-ing into them the target and host directories of all packages
listed in the $(PACKAGES) variable. It is necessary to do this
sequentially in the target-finalize step and not in each
package. Doing it in package installation means that it can be done in
parallel. In that case, there is a chance that two rsyncs are creating
the same hardlink or directory at the same time, which makes one of
them fail.
This change to per-package directories has an impact on the RPATH
built into the host binaries, as those RPATH now point to various
per-package host directories, and no longer to the global host
directory. We do not try to rewrite such RPATHs during the build as
having such RPATHs is perfectly fine, but we still need to handle two
fallouts from this change:
- The check-host-rpath script, which verifies at the end of each
package installation that it has the appropriate RPATH, is modified
to understand that a RPATH to $(PER_PACKAGE_DIR)/<pkg>/host/lib is
a correct RPAT.
- The fix-rpath script, which mungles the RPATH mainly for the SDK
preparation, is modified to rewrite the RPATH to not point to
per-package directories. Indeed the patchelf --make-rpath-relative
call only works if the RPATH points to the ROOTDIR passed as
argument, and this ROOTDIR is the global host directory. Rewriting
the RPATH to not point to per-package host directories prior to
this is an easy solution to this issue.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
2019-11-05 17:46:40 +01:00
|
|
|
if test "${rpath}" != "${changed_rpath}" ; then
|
|
|
|
${PATCHELF} --set-rpath ${changed_rpath} "${file}"
|
|
|
|
fi
|
|
|
|
|
|
|
|
# call patchelf to sanitize the rpath
|
|
|
|
${PATCHELF} --make-rpath-relative "${rootdir}" ${sanitize_extra_args[@]} "${file}"
|
|
|
|
# restore the original permission
|
|
|
|
test "${changed}" != "" && chmod u-w "${file}"
|
2017-07-20 16:35:14 +02:00
|
|
|
done < <(find "${rootdir}" ${find_args[@]})
|
|
|
|
|
|
|
|
# Restore patched patchelf utility
|
|
|
|
test "${tree}" = "host" && mv "${PATCHELF}.__to_be_patched" "${PATCHELF}"
|
|
|
|
|
|
|
|
# ignore errors
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
main ${@}
|