From: Christophe Vu-Brugier Date: Sat, 14 Nov 2015 15:54:30 +0000 (+0100) Subject: python3: add patch to fix PEP 3147 issue with automake built packages X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=c3d65a4dad707db83122ac082202f3e8dc70e60f;p=buildroot.git python3: add patch to fix PEP 3147 issue with automake built packages Packages built with automake use a `py-compile` helper to byte-compile Python source files. This script uses the "py_compile" module from the standard library. In turn, the compile() function in the "py_compile" module invokes the cache_from_source() function provided by importlib. This commit adds a new patch named "020-importlib-no-pep3147.patch" that changes cache_from_source() and source_from_cache() in importlib to get rid of the "__pycache__" directory. This commit fixes the following import error in kmod when the module is built for Python 3: >>> from kmod import Kmod Traceback (most recent call last): File "", line 1, in ImportError: cannot import name 'Kmod' Moreover, this commit removes two patches that are no longer necessary since modifying cache_from_source() and source_from_cache() disables PEP 3147 for the standard library and distutils / setuptools. * 004-old-stdlib-cache.patch * 016-distutils-no-pep3147.patch Signed-off-by: Christophe Vu-Brugier Signed-off-by: Thomas Petazzoni Tested-by: Thomas Petazzoni --- diff --git a/package/python3/004-old-stdlib-cache.patch b/package/python3/004-old-stdlib-cache.patch deleted file mode 100644 index 9ed7e59a9b..0000000000 --- a/package/python3/004-old-stdlib-cache.patch +++ /dev/null @@ -1,78 +0,0 @@ -python3: Fix pyc-only related runtime exceptions - -Python3 changes the pyc lookup strategy, ignoring the -__pycache__ directory if the .py file is missing. Change -install location to enable use of .pyc without their parent .py - -See http://www.python.org/dev/peps/pep-3147 - -Signed-off-by: Daniel Nelson - -Index: b/configure.ac -=================================================================== ---- a/configure.ac -+++ b/configure.ac -@@ -352,6 +352,23 @@ - AC_SUBST(FRAMEWORKUNIXTOOLSPREFIX) - AC_SUBST(FRAMEWORKINSTALLAPPSPREFIX) - -+STDLIB_CACHE_FLAGS= -+AC_MSG_CHECKING(for --enable-old-stdlib-cache) -+AC_ARG_ENABLE(old-stdlib-cache, -+ AS_HELP_STRING([--enable-old-stdlib-cache], [enable pre-pep3147 stdlib cache]), -+[ -+ if test "$enableval" = "yes" -+ then -+ STDLIB_CACHE_FLAGS="-b" -+ else -+ STDLIB_CACHE_FLAGS="" -+ fi -+], -+[ -+ STDLIB_CACHE_FLAGS="" -+]) -+AC_SUBST(STDLIB_CACHE_FLAGS) -+ - ##AC_ARG_WITH(dyld, - ## AS_HELP_STRING([--with-dyld], - ## [Use (OpenStep|Rhapsody) dynamic linker])) -Index: b/Makefile.pre.in -=================================================================== ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -162,6 +162,9 @@ - # Options to enable prebinding (for fast startup prior to Mac OS X 10.3) - OTHER_LIBTOOL_OPT=@OTHER_LIBTOOL_OPT@ - -+# Option to enable old-style precompiled stdlib -+STDLIB_CACHE_FLAGS=@STDLIB_CACHE_FLAGS@ -+ - # Environment to run shared python without installed libraries - RUNSHARED= @RUNSHARED@ - -@@ -1247,21 +1250,21 @@ - fi - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ - $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ -- -d $(LIBDEST) -f \ -+ -d $(LIBDEST) -f $(STDLIB_CACHE_FLAGS) \ - -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \ - $(DESTDIR)$(LIBDEST) - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ - $(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \ -- -d $(LIBDEST) -f \ -+ -d $(LIBDEST) -f $(STDLIB_CACHE_FLAGS) \ - -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \ - $(DESTDIR)$(LIBDEST) - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ - $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ -- -d $(LIBDEST)/site-packages -f \ -+ -d $(LIBDEST)/site-packages -f $(STDLIB_CACHE_FLAGS) \ - -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ - $(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \ -- -d $(LIBDEST)/site-packages -f \ -+ -d $(LIBDEST)/site-packages -f $(STDLIB_CACHE_FLAGS) \ - -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ - $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt diff --git a/package/python3/005-pyc-pyo-conditional.patch b/package/python3/005-pyc-pyo-conditional.patch index c3b159da76..0d60ff6949 100644 --- a/package/python3/005-pyc-pyo-conditional.patch +++ b/package/python3/005-pyc-pyo-conditional.patch @@ -9,27 +9,27 @@ Index: b/Makefile.pre.in +ifeq (@PYC_BUILD@,yes) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ - -d $(LIBDEST) -f $(STDLIB_CACHE_FLAGS) \ + -d $(LIBDEST) -f \ -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \ $(DESTDIR)$(LIBDEST) +endif +ifeq (@PYO_BUILD@,yes) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \ - -d $(LIBDEST) -f $(STDLIB_CACHE_FLAGS) \ + -d $(LIBDEST) -f \ -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \ $(DESTDIR)$(LIBDEST) +endif +ifeq (@PYC_BUILD@,yes) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ - -d $(LIBDEST)/site-packages -f $(STDLIB_CACHE_FLAGS) \ + -d $(LIBDEST)/site-packages -f \ -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages +endif +ifeq (@PYO_BUILD@,yes) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \ - -d $(LIBDEST)/site-packages -f $(STDLIB_CACHE_FLAGS) \ + -d $(LIBDEST)/site-packages -f \ -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages +endif -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ diff --git a/package/python3/016-distutils-no-pep3147.patch b/package/python3/016-distutils-no-pep3147.patch deleted file mode 100644 index 0fdfeb24ca..0000000000 --- a/package/python3/016-distutils-no-pep3147.patch +++ /dev/null @@ -1,36 +0,0 @@ -Add distutils fix for PEP 3147 issue - -Python 3 has a new standard for installing .pyc file, called PEP -3147. Unfortunately, this standard requires both the .py and .pyc -files to be installed for a Python module to be found. This is quite -annoying on space-constrained embedded systems, since the .py file is -technically not required for execution. - -For the Python standard library, our Python 3 package already contains -a patch named python3-004-old-stdlib-cache.patch, which allows to -disable the PEP 3147 installation. - -But that lives the distutils/setuptools package an unsolved -problem. This patch therefore adds a new patch to Python, which makes -distutils package use the traditional installation path when byte -compiling, rather than the PEP 3147 installation path. Since -setuptools relies on distutils internally, it also fixes setuptools -based packages. - -Signed-off-by: Thomas Petazzoni - -Index: b/Lib/distutils/util.py -=================================================================== ---- a/Lib/distutils/util.py -+++ b/Lib/distutils/util.py -@@ -437,7 +437,9 @@ - # Terminology from the py_compile module: - # cfile - byte-compiled file - # dfile - purported source filename (same as 'file' by default) -- if optimize >= 0: -+ if "_python_sysroot" in os.environ: -+ cfile = file + (__debug__ and "c" or "o") -+ elif optimize >= 0: - cfile = importlib.util.cache_from_source( - file, debug_override=not optimize) - else: diff --git a/package/python3/020-importlib-no-pep3147.patch b/package/python3/020-importlib-no-pep3147.patch new file mode 100644 index 0000000000..779b5e27c3 --- /dev/null +++ b/package/python3/020-importlib-no-pep3147.patch @@ -0,0 +1,69 @@ +Add importlib fix for PEP 3147 issue + +Python 3 has a new standard for installing .pyc file, called PEP +3147. Unfortunately, this standard requires both the .py and .pyc +files to be installed for a Python module to be found. This is quite +annoying on space-constrained embedded systems, since the .py file is +technically not required for execution. + +This patch changes cache_from_source() and source_from_cache() in +importlib to get rid of the "__pycache__" directory. +This effectively disables PEP 3147 for: + +* The python standard library +* Packages built with distutils or setuptools +* Packages built with automake that use the `py-compile` helper + +Signed-off-by: Christophe Vu-Brugier + +diff -purN a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py +--- a/Lib/importlib/_bootstrap.py 2015-02-25 12:27:44.000000000 +0100 ++++ b/Lib/importlib/_bootstrap.py 2015-08-27 17:33:26.167191059 +0200 +@@ -444,8 +444,6 @@ def cache_from_source(path, debug_overri + If debug_override is not None, then it must be a boolean and is used in + place of sys.flags.optimize. + +- If sys.implementation.cache_tag is None then NotImplementedError is raised. +- + """ + debug = not sys.flags.optimize if debug_override is None else debug_override + if debug: +@@ -454,33 +452,19 @@ def cache_from_source(path, debug_overri + suffixes = OPTIMIZED_BYTECODE_SUFFIXES + head, tail = _path_split(path) + base, sep, rest = tail.rpartition('.') +- tag = sys.implementation.cache_tag +- if tag is None: +- raise NotImplementedError('sys.implementation.cache_tag is None') +- filename = ''.join([(base if base else rest), sep, tag, suffixes[0]]) +- return _path_join(head, _PYCACHE, filename) ++ filename = ''.join([(base if base else rest), suffixes[0]]) ++ return _path_join(head, filename) + + + def source_from_cache(path): + """Given the path to a .pyc./.pyo file, return the path to its .py file. + + The .pyc/.pyo file does not need to exist; this simply returns the path to +- the .py file calculated to correspond to the .pyc/.pyo file. If path does +- not conform to PEP 3147 format, ValueError will be raised. If +- sys.implementation.cache_tag is None then NotImplementedError is raised. ++ the .py file calculated to correspond to the .pyc/.pyo file. + + """ +- if sys.implementation.cache_tag is None: +- raise NotImplementedError('sys.implementation.cache_tag is None') +- head, pycache_filename = _path_split(path) +- head, pycache = _path_split(head) +- if pycache != _PYCACHE: +- raise ValueError('{} not bottom-level directory in ' +- '{!r}'.format(_PYCACHE, path)) +- if pycache_filename.count('.') != 2: +- raise ValueError('expected only 2 dots in ' +- '{!r}'.format(pycache_filename)) +- base_filename = pycache_filename.partition('.')[0] ++ head, filename = _path_split(path) ++ base_filename = filename.partition('.')[0] + return _path_join(head, base_filename + SOURCE_SUFFIXES[0]) + + diff --git a/package/python3/python3.mk b/package/python3/python3.mk index fd5709a92e..a37b791a87 100644 --- a/package/python3/python3.mk +++ b/package/python3/python3.mk @@ -147,7 +147,17 @@ define PYTHON3_TOUCH_GRAMMAR_FILES touch $(@D)/Include/graminit.h $(@D)/Python/graminit.c endef +# This prevents the Python Makefile from regenerating the +# Python/importlib.h header if Lib/importlib/_bootstrap.py has changed +# because its generation is broken in a cross-compilation environment +# and importlib.h is not used. + +define PYTHON3_TOUCH_IMPORTLIB_H + touch $(@D)/Python/importlib.h +endef + PYTHON3_POST_PATCH_HOOKS += PYTHON3_TOUCH_GRAMMAR_FILES +PYTHON3_POST_PATCH_HOOKS += PYTHON3_TOUCH_IMPORTLIB_H # # Remove useless files. In the config/ directory, only the Makefile