package/python3: use the provided pyc compiler

Previously, we used support/scripts/pycompile.py to generate the pyc
files for the python libraries.

While the script worked, it did not follow the PEP 3147 layout
requirements for py+pyc deployments.

Now, use the package's own compileall.py script. This will follow
PEP 3147 guidelines. It also supports "legacy" pyc only deployments as
described here:
  https://peps.python.org/pep-3147/#case-4-legacy-pyc-files-and-source-less-imports

With this change, we no longer need to hack support for side-by-side pyc
files because files will be deployed as appropriate.

This also has the added benefit of not requiring python3 on the host to
build host-python3.

Fixes: #14911

Signed-off-by: Vincent Fazio <vfazio@xes-inc.com>
[yann.morin.1998@free.fr:
  - build-tested in a python-less environment
  - build+run-tested with the runtime-test infra
]
Tested-by: Yann E. MORIN <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Signed-off-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>

package/python3/0011-Add-importlib-fix-for-PEP-3147-issue.patch

@@ -1,105 +0,0 @@
-From e2ea659eac1849db471d3c01a0d0af9d6fca2e9a Mon Sep 17 00:00:00 2001
-From: Christophe Vu-Brugier <cvubrugier@fastmail.fm>
-Date: Wed, 22 Feb 2017 16:48:49 -0800
-Subject: [PATCH] Add importlib fix for PEP 3147 issue
-
-Python 3 has a new standard for installing .pyc file, called PEP
-3147. Unfortunately, this standard requires both the .py and .pyc
-files to be installed for a Python module to be found. This is quite
-annoying on space-constrained embedded systems, since the .py file is
-technically not required for execution.
-
-This patch changes cache_from_source() and source_from_cache() in
-importlib to get rid of the "__pycache__" directory.
-This effectively disables PEP 3147 for:
-
-* The python standard library
-* Packages built with distutils or setuptools
-* Packages built with automake that use the `py-compile` helper
-
-Signed-off-by: Christophe Vu-Brugier <cvubrugier@fastmail.fm>
-[ Andrey Smirnov: ported to Python 3.6 ]
-Signed-off-by: Andrey Smirnov <andrew.smirnov@gmail.com>
----
- Lib/importlib/_bootstrap_external.py | 44 ++++------------------------
- 1 file changed, 5 insertions(+), 39 deletions(-)
-
-diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
-index 25a3f8c0e0..2cb9a9aa52 100644
---- a/Lib/importlib/_bootstrap_external.py
-+++ b/Lib/importlib/_bootstrap_external.py
-@@ -392,8 +392,6 @@ def cache_from_source(path, debug_override=None, *, optimization=None):
-     a True value is the same as setting 'optimization' to the empty string
-     while a False value is equivalent to setting 'optimization' to '1'.
- 
--    If sys.implementation.cache_tag is None then NotImplementedError is raised.
--
-     """
-     if debug_override is not None:
-         _warnings.warn('the debug_override parameter is deprecated; use '
-@@ -405,10 +403,7 @@ def cache_from_source(path, debug_override=None, *, optimization=None):
-     path = _os.fspath(path)
-     head, tail = _path_split(path)
-     base, sep, rest = tail.rpartition('.')
--    tag = sys.implementation.cache_tag
--    if tag is None:
--        raise NotImplementedError('sys.implementation.cache_tag is None')
--    almost_filename = ''.join([(base if base else rest), sep, tag])
-+    almost_filename = ''.join([(base if base else rest)])
-     if optimization is None:
-         if sys.flags.optimize == 0:
-             optimization = ''
-@@ -445,46 +440,17 @@ def cache_from_source(path, debug_override=None, *, optimization=None):
-             head.lstrip(path_separators),
-             filename,
-         )
--    return _path_join(head, _PYCACHE, filename)
-+    return _path_join(head, filename)
- 
- 
- def source_from_cache(path):
-     """Given the path to a .pyc. file, return the path to its .py file.
- 
-     The .pyc file does not need to exist; this simply returns the path to
--    the .py file calculated to correspond to the .pyc file.  If path does
--    not conform to PEP 3147/488 format, ValueError will be raised. If
--    sys.implementation.cache_tag is None then NotImplementedError is raised.
--
-+    the .py file calculated to correspond to the .pyc file.
-     """
--    if sys.implementation.cache_tag is None:
--        raise NotImplementedError('sys.implementation.cache_tag is None')
--    path = _os.fspath(path)
--    head, pycache_filename = _path_split(path)
--    found_in_pycache_prefix = False
--    if sys.pycache_prefix is not None:
--        stripped_path = sys.pycache_prefix.rstrip(path_separators)
--        if head.startswith(stripped_path + path_sep):
--            head = head[len(stripped_path):]
--            found_in_pycache_prefix = True
--    if not found_in_pycache_prefix:
--        head, pycache = _path_split(head)
--        if pycache != _PYCACHE:
--            raise ValueError(f'{_PYCACHE} not bottom-level directory in '
--                             f'{path!r}')
--    dot_count = pycache_filename.count('.')
--    if dot_count not in {2, 3}:
--        raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}')
--    elif dot_count == 3:
--        optimization = pycache_filename.rsplit('.', 2)[-2]
--        if not optimization.startswith(_OPT):
--            raise ValueError("optimization portion of filename does not start "
--                             f"with {_OPT!r}")
--        opt_level = optimization[len(_OPT):]
--        if not opt_level.isalnum():
--            raise ValueError(f"optimization level {optimization!r} is not an "
--                             "alphanumeric value")
--    base_filename = pycache_filename.partition('.')[0]
-+    head, filename = _path_split(path)
-+    base_filename = filename.partition('.')[0]
-     return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
- 
- 
--- 
-2.25.1
-

package/python3/python3.mk

@@ -182,47 +182,6 @@ PYTHON3_CONF_OPTS += \
 	--disable-idle3 \
 	--disable-pyc-build
 
-#
-# Some of CPython's source code is generated using Python interpreter
-# and some helper tools such as "Programs/_freeze_importlib" or
-# "Parser/pgen" (look for regen-* targets in Makefile.pre.in for more
-# info). Normally CPython codebase ships with those files
-# pre-generated, so just regular "make" with no additional steps
-# should be sufficient for a succesfull build, however due to
-# Buildroot's "Add importlib fix for PEP 3147 issue" custom patch we
-# end up modifying "Lib/importlib/_bootstrap_external.py" which means
-# we have to do "regen-importlib" step before building CPython
-# (Importlib is a builtin module that needs to be "frozen"/converted
-# to a C array of bytecode using "Programs/_freeze_importlib")
-#
-# To achive that we add pre-build steps to host-python3 as well as
-# python3 that execute "regen-importlib" target.
-#
-# Unfortunately, for the target Python, "Programs/_freeze_importlib"
-# is built for the target, while we need to run them at build time. So
-# when installing host-python3, we copy them to $(HOST_DIR)/bin...
-#
-define HOST_PYTHON3_MAKE_REGEN_IMPORTLIB
-	$(HOST_MAKE_ENV) $(PYTHON3_CONF_ENV) $(MAKE) $(HOST_CONFIGURE_OPTS) -C $(@D) regen-importlib
-	cp $(@D)/Programs/_freeze_importlib $(HOST_DIR)/bin/python-freeze-importlib
-endef
-
-HOST_PYTHON3_PRE_BUILD_HOOKS += HOST_PYTHON3_MAKE_REGEN_IMPORTLIB
-#
-# ... And then, when building the target python we first buid
-# 'Programs/_freeze_importlib' to force GNU Make to update all of the
-# prerequisites of 'Programs/_freeze_importlib', then copy our stashed
-# "host-usable" version over the one that was just build and then
-# build "regen-importlib" target
-#
-define PYTHON3_MAKE_REGEN_IMPORTLIB
-	$(TARGET_MAKE_ENV) $(PYTHON3_CONF_ENV) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) Programs/_freeze_importlib
-	cp $(HOST_DIR)/bin/python-freeze-importlib $(@D)/Programs/_freeze_importlib
-	$(TARGET_MAKE_ENV) $(PYTHON3_CONF_ENV) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) regen-importlib
-endef
-
-PYTHON3_PRE_BUILD_HOOKS += PYTHON3_MAKE_REGEN_IMPORTLIB
-
 #
 # Remove useless files. In the config/ directory, only the Makefile
 # and the pyconfig.h files are needed at runtime.
@@ -292,9 +251,9 @@ define PYTHON3_CREATE_PYC_FILES
 	$(PYTHON3_FIX_TIME)
 	PYTHONPATH="$(PYTHON3_PATH)" \
 	$(HOST_DIR)/bin/python$(PYTHON3_VERSION_MAJOR) \
-		$(TOPDIR)/support/scripts/pycompile.py \
-		$(if $(VERBOSE),--verbose) \
-		--strip-root $(TARGET_DIR) \
+		$(PYTHON3_DIR)/Lib/compileall.py \
+		$(if $(BR2_PACKAGE_PYTHON3_PYC_ONLY),-b) \
+		-s $(TARGET_DIR) \
 		$(TARGET_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)
 endef
 

support/scripts/pycompile.py

@@ -1,91 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-Byte compile all .py files from provided directories. This script is an
-alternative implementation of compileall.compile_dir written with
-cross-compilation in mind.
-"""
-
-import argparse
-import os
-import py_compile
-import re
-import sys
-
-
-def compile_one(host_path, strip_root=None, verbose=False):
-    """
-    Compile a .py file into a .pyc file located next to it.
-
-    :arg host_path:
-        Absolute path to the file to compile on the host running the build.
-    :arg strip_root:
-        Prefix to remove from the original source paths encoded in compiled
-        files.
-    :arg verbose:
-        Print compiled file paths.
-    """
-    if os.path.islink(host_path) or not os.path.isfile(host_path):
-        return  # only compile real files
-
-    if not re.match(r"^[_A-Za-z][_A-Za-z0-9]*\.py$",
-                    os.path.basename(host_path)):
-        return  # only compile "importable" python modules
-
-    if strip_root is not None:
-        # determine the runtime path of the file (i.e.: relative path to root
-        # dir prepended with "/").
-        runtime_path = os.path.join("/", os.path.relpath(host_path, strip_root))
-    else:
-        runtime_path = host_path
-
-    if verbose:
-        print("  PYC  {}".format(runtime_path))
-
-    # will raise an error if the file cannot be compiled
-    py_compile.compile(host_path, cfile=host_path + "c",
-                       dfile=runtime_path, doraise=True)
-
-
-def existing_dir_abs(arg):
-    """
-    argparse type callback that checks that argument is a directory and returns
-    its absolute path.
-    """
-    if not os.path.isdir(arg):
-        raise argparse.ArgumentTypeError('no such directory: {!r}'.format(arg))
-    return os.path.abspath(arg)
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("dirs", metavar="DIR", nargs="+", type=existing_dir_abs,
-                        help="Directory to recursively scan and compile")
-    parser.add_argument("--strip-root", metavar="ROOT", type=existing_dir_abs,
-                        help="""
-                        Prefix to remove from the original source paths encoded
-                        in compiled files
-                        """)
-    parser.add_argument("--verbose", action="store_true",
-                        help="Print compiled files")
-
-    args = parser.parse_args()
-
-    try:
-        for d in args.dirs:
-            if args.strip_root and ".." in os.path.relpath(d, args.strip_root):
-                parser.error("DIR: not inside ROOT dir: {!r}".format(d))
-            for parent, _, files in os.walk(d):
-                for f in files:
-                    compile_one(os.path.join(parent, f), args.strip_root,
-                                args.verbose)
-
-    except Exception as e:
-        print("error: {}".format(e))
-        return 1
-
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())