diff --git a/pypy.spec b/pypy.spec index 76397b4..7196eeb 100644 --- a/pypy.spec +++ b/pypy.spec @@ -1,7 +1,7 @@ Name: pypy Version: 1.4 -Release: 3%{?dist} -Summary: Implementation of the Python language, using Python itself +Release: 4%{?dist} +Summary: Python implementation with a Just-In-Time compiler Group: Development/Languages # LGPL and another free license we'd need to ask spot about are present in some @@ -14,9 +14,28 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) # High-level configuration of the build: -# How many different builds do we want? +# PyPy consists of an implementation of an interpreter (with JIT compilation) +# for the full Python language written in a high-level language, leaving many +# of the implementation details as "pluggable" policies. +# +# The implementation language is then compiled down to .c code, from which we +# obtain a binary. +# +# This allows us to build a near-arbitrary collection of different +# implementations of Python with differing tradeoffs +# +# (As it happens, the implementation language is itself Python, albeit a +# restricted subset "RPython", chosen to making it amenable to being compiled. +# The result implements the full Python language though) + +# We could build many different implementations of Python. +# For now, let's focus on the implementation that appears to be receiving the +# most attention upstream: the JIT-enabled build, with all standard +# optimizations -# A build of pypy-jit on i686 took 77 mins: +# Building a configuration can take significant time: + +# A build of pypy (with jit) on i686 took 77 mins: # [Timer] Timings: # [Timer] annotate --- 583.3 s # [Timer] rtype_lltype --- 760.9 s @@ -29,7 +48,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) # [Timer] =========================================== # [Timer] Total: --- 4620.5 s # -# A build of pypy-nojit on x86_64 took about an hour: +# A build of pypy (nojit) on x86_64 took about an hour: # [Timer] Timings: # [Timer] annotate --- 537.5 s # [Timer] rtype_lltype --- 667.3 s @@ -42,18 +61,6 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) # [Timer] Total: --- 3572.0 s # # -# A build of pypy-sandbox on i686 took about 40 mins: -# [Timer] Timings: -# [Timer] annotate --- 436.4 s -# [Timer] rtype_lltype --- 452.6 s -# [Timer] backendopt_lltype --- 267.2 s -# [Timer] stackcheckinsertion_lltype --- 26.1 s -# [Timer] database_c --- 317.6 s -# [Timer] source_c --- 635.4 s -# [Timer] compile_c --- 270.3 s -# [Timer] =========================================== -# [Timer] Total: --- 2405.5 s -# # A build of pypy-stackless on i686 took about 87 mins: # [Timer] Timings: # [Timer] annotate --- 584.2 s @@ -71,15 +78,14 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) # or 3.25 hours without the jit -# Should we build a "pypy-jit" binary? -# pypy-1.2/pypy/jit/backend/detect_cpu.py:getcpuclass currently supports the +# Should we build a "pypy" binary? (with jit) +# pypy-1.4/pypy/jit/backend/detect_cpu.py:getcpuclassname currently supports the # following options: # 'i386', 'x86' # 'x86-without-sse2': +# 'x86_64' # 'cli' # 'llvm' -# We may eventually want 'llvm' on x86_64 for the jit build; -# turning off for now for simplicity %ifarch %{ix86} x86_64 # FIXME: is there a better way of expressing "intel" here? %global with_jit 1 @@ -87,45 +93,19 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) %global with_jit 0 %endif -# Should we build a "pypy-nojit" binary? -%global with_nojit 1 +# Should we build a "pypy-stackless" binary? +%global with_stackless 0 -# Should we build a "pypy-sandbox" binary? -%global with_sandbox 1 -# Should we build a "pypy-stackless" binary? -%global with_stackless 1 +# Easy way to enable/disable verbose logging: +%global verbose_logs 1 -# Turn off the brp-python-bytecompile script for now; ultimately we'll want to -# precompile the .pyc files using one of the freshly built pypy binaries +# Turn off the brp-python-bytecompile postprocessing script +# We manually invoke it later on, using the freshly built pypy binary %global __os_install_post \ %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g') -# pypy uses .pyc files be default (--objspace-usepycfiles) -# and pypy-1.2/pypy/module/imp/importing.py has this comment: - # XXX picking a magic number is a mess. So far it works because we - # have only two extra opcodes, which bump the magic number by +1 and - # +2 respectively, and CPython leaves a gap of 10 when it increases - # its own magic number. To avoid assigning exactly the same numbers - # as CPython we always add a +2. We'll have to think again when we - # get at the fourth new opcode :-( - # - # * CALL_LIKELY_BUILTIN +1 - # * CALL_METHOD +2 - # - # In other words: - # - # default_magic -- used by CPython without the -U option - # default_magic + 1 -- used by CPython with the -U option - # default_magic + 2 -- used by PyPy without any extra opcode - # ... - # default_magic + 5 -- used by PyPy with both extra opcodes -# - - - - # Source and patches: Source0: http://pypy.org/download/pypy-%{version}-src.tar.bz2 @@ -156,14 +136,21 @@ BuildRequires: expat-devel BuildRequires: openssl-devel BuildRequires: valgrind-devel +BuildRequires: /usr/bin/execstack + # pypy is bundling these so we delete them in %%prep. I don't think they are # needed unless we build pypy targetted at running on the jvm. #BuildRequires: jna #BuildRequires: jasmin # Not yet in Fedora + +# Metadata for the core package (the JIT build): +Requires: pypy-libs = %{version}-%{release} + %description -pypy is a reimplementation of the Python language in Python itself, which is -then compiled into implementations with particular properties. +PyPy's implementation of Python, featuring a Just-In-Time compiler, and various +optimized implementations of the standard types (strings, dictionaries, etc) + %package libs Group: Development/Languages @@ -171,35 +158,13 @@ Summary: Run-time libraries used by PyPy implementations of Python %description libs Libraries required by the various PyPy implementations of Python. -%if 0%{with_jit} -%package jit -Group: Development/Languages -Summary: Python implementation with a Just-In-Time compiler -Requires: pypy-libs = %{version}-%{release} -%description jit -Python implementation with a Just-In-Time compiler -%endif - -%if 0%{with_nojit} -%package nojit -Group: Development/Languages -Summary: Python interpreter built using PyPy -Requires: pypy-libs = %{version}-%{release} -%description nojit -This is a Python interpreter built using PyPy, without a Just-in-Time -compiler. It is typically slower than CPython, but may use 1.5x or 2x less -RAM for programs requiring hundred of MB to run. -%endif - -%if 0%{with_sandbox} -%package sandbox +%if 0%{with_stackless} +%package stackless Group: Development/Languages -Summary: Sandboxed Python interpreter built using PyPy +Summary: Stackless Python interpreter built using PyPy Requires: pypy-libs = %{version}-%{release} -%description sandbox -Experimental build of PyPy that replace all calls to external libraries (C -or platform) with a stub that communicates with an external process handling -the policy. +%description stackless +Build of PyPy with support for micro-threads for massive concurrency %endif %if 0%{with_stackless} @@ -226,6 +191,9 @@ find -name "*.py" -exec \ find . -name '*.jar' -exec rm \{\} \; +# Remove DOS batch files: +find -name "*.bat"|xargs rm -f + %build BuildPyPy() { @@ -241,9 +209,6 @@ BuildPyPy() { echo "--------------------------------------------------------------" pushd pypy/translator/goal - # For now, filter our CFLAGS of everything that could be conflicting with - # pypy. Need to check these and reenable ones that are okay later. - export CFLAGS=$(echo "$RPM_OPT_FLAGS" | sed -e 's/-Wp,-D_FORTIFY_SOURCE=2//' -e 's/-fexceptions//' -e 's/-fstack-protector//' -e 's/--param=ssp-buffer-size=4//' -e 's/-O2//' -e 's/-g//' -e 's/-fasynchronous-unwind-tables//' -e 's/-march=i686//' -e 's/-mtune=atom//') # The generated binary embeds copies of the values of all environment # variables. We need to unset "RPM_BUILD_ROOT" to avoid a fatal error from @@ -258,14 +223,40 @@ BuildPyPy() { # and it appears that this stops rpm from extracting the source code to the # debuginfo package # - # The logic in pypy-1.2/pypy/tool/udir.py indicates that it is generated in: + # The logic in pypy-1.4/pypy/tool/udir.py indicates that it is generated in: # $PYPY_USESSION_DIR/usession-$PYPY_USESSION_BASENAME-N # and so we set PYPY_USESSION_DIR so that this tempdir is within the build # location, and set $PYPY_USESSION_BASENAME so that the tempdir is unique # for each invocation of BuildPyPy - if test -x './pypy-jit' ; then - INTERP='./pypy-jit' + # Compilation flags for C code: + # pypy-1.4/pypy/translator/c/genc.py:gen_makefile + # assembles a Makefile within + # THE_UDIR/testing_1/Makefile + # calling out to platform.gen_makefile + # For us, that's + # pypy-1.4/pypy/translator/platform/linux.py: class BaseLinux(BasePosix): + # which by default has: + # CFLAGS = ['-O3', '-pthread', '-fomit-frame-pointer', + # '-Wall', '-Wno-unused'] + # plus all substrings from CFLAGS in the environment. + # This is used to generate a value for CFLAGS that's written into the Makefile + + # https://bugzilla.redhat.com/show_bug.cgi?id=588941#c18 + # The generated Makefile compiles the .c files into assembler (.s), rather + # than direct to .o It then post-processes this assembler to locate + # garbage-collection roots (building .lbl.s and .gcmap files, and a + # "gcmaptable.s"). (The modified .lbl.s files have extra code injected + # within them). + # Unfortunately, the code to do this: + # pypy-1.4/pypy/translator/c/gcc/trackgcroot.py + # doesn't interract well with the results of using our standard build flags. + # For now, filter our CFLAGS of everything that could be conflicting with + # pypy. Need to check these and reenable ones that are okay later. + export CFLAGS=$(echo "$RPM_OPT_FLAGS" | sed -e 's/-Wp,-D_FORTIFY_SOURCE=2//' -e 's/-fexceptions//' -e 's/-fstack-protector//' -e 's/--param=ssp-buffer-size=4//' -e 's/-O2//' -e 's/-fasynchronous-unwind-tables//' -e 's/-march=i686//' -e 's/-mtune=atom//') + + if test -x './pypy' ; then + INTERP='./pypy' else INTERP='python' fi @@ -275,6 +266,9 @@ BuildPyPy() { PYPY_USESSION_DIR=$(pwd) \ PYPY_USESSION_BASENAME=$ExeName \ $INTERP translate.py \ +%if verbose_logs + --translation-verbose \ +%endif --cflags="$CFLAGS" \ --batch \ --output=$ExeName \ @@ -293,22 +287,10 @@ BuildPyPy() { %if 0%{with_jit} BuildPyPy \ - pypy-jit \ + pypy \ "-Ojit" %endif -%if 0%{with_nojit} -BuildPyPy \ - pypy-nojit \ - "-O2" -%endif - -%if 0%{with_sandbox} -BuildPyPy \ - pypy-sandbox \ - "--sandbox" -%endif - %if 0%{with_stackless} BuildPyPy \ pypy-stackless \ @@ -323,21 +305,24 @@ rm -rf $RPM_BUILD_ROOT InstallPyPy() { ExeName=$1 + install -m 755 pypy/translator/goal/$ExeName %{buildroot}/%{_bindir} + + # The generated machine code doesn't need an executable stack, but + # going through assembler means that the toolchain can't know this, and thus + # by default pessimistically assumes the assembler files do need one. + # + # Reported upstream as: https://codespeak.net/issue/pypy-dev/issue610 + # + # I tried various approaches involving fixing the build, but the simplest + # approach is to postprocess the ELF file: + execstack --clear-execstack %{buildroot}/%{_bindir}/$ExeName } mkdir -p %{buildroot}/%{_bindir} %if 0%{with_jit} -InstallPyPy pypy-jit -%endif - -%if 0%{with_nojit} -InstallPyPy pypy-nojit -%endif - -%if 0%{with_sandbox} -InstallPyPy pypy-sandbox +InstallPyPy pypy %endif %if 0%{with_stackless} @@ -368,8 +353,86 @@ cp -a lib_pypy %{buildroot}/%{pypyprefix} # Remove a text file that documents which selftests fail on Win32: rm %{buildroot}/%{pypyprefix}/lib-python/win32-failures.txt +# Remove shebang lines from .py files that aren't executable, and +# remove executability from .py files that don't have a shebang line: +find \ + %{buildroot} demo \ + -name "*.py" \ + \( \ + \( \! -perm /u+x,g+x,o+x -exec sed -e '/^#!/Q 0' -e 'Q 1' {} \; \ + -print -exec sed -i '1d' {} \; \ + \) \ + -o \ + \( \ + -perm /u+x,g+x,o+x ! -exec grep -m 1 -q '^#!' {} \; \ + -exec chmod a-x {} \; \ + \) \ + \) + mkdir -p %{buildroot}/%{pypyprefix}/site-packages + +# pypy uses .pyc files by default (--objspace-usepycfiles), but has a slightly +# different bytecode format to CPython. It doesn't use .pyo files: the -O flag +# is treated as a "dummy optimization flag for compatibility with C Python" +# +# pypy-1.4/pypy/module/imp/importing.py has this comment: + # XXX picking a magic number is a mess. So far it works because we + # have only two extra opcodes, which bump the magic number by +1 and + # +2 respectively, and CPython leaves a gap of 10 when it increases + # its own magic number. To avoid assigning exactly the same numbers + # as CPython we always add a +2. We'll have to think again when we + # get at the fourth new opcode :-( + # + # * CALL_LIKELY_BUILTIN +1 + # * CALL_METHOD +2 + # + # In other words: + # + # default_magic -- used by CPython without the -U option + # default_magic + 1 -- used by CPython with the -U option + # default_magic + 2 -- used by PyPy without any extra opcode + # ... + # default_magic + 5 -- used by PyPy with both extra opcodes +# + +# pypy-1.4/pypy/interpreter/pycode.py has: +# +# default_magic = (62141+2) | 0x0a0d0000 # this PyPy's magic +# # (62131=CPython 2.5.1) +# giving a value for "default_magic" for PyPy of 0xa0df2bf. +# Note that this corresponds to the "default_magic + 2" from the comment above + +# In my builds: +# $ ./pypy --info | grep objspace.opcodes +# objspace.opcodes.CALL_LIKELY_BUILTIN: False +# objspace.opcodes.CALL_METHOD: True +# so I'd expect the magic number to be: +# 0x0a0df2bf + 2 (the flag for CALL_METHOD) +# giving +# 0x0a0df2c1 +# +# I'm seeing +# c1 f2 0d 0a +# as the first four bytes of the .pyc files, which is consistent with this. + + +# Bytecompile all of the .py files we ship, using our pypy binary, giving us +# .pyc files for pypy. The script actually does the work twice (passing in -O +# the second time) but it's simplest to reuse that script. +# +# The script has special-casing for .py files below +# /usr/lib{64}/python[0-9].[0-9] +# but given that we're installing into a different path, the supplied "default" +# implementation gets used instead. +# +# Note that some of the test files deliberately contain syntax errors, so +# we pass 0 for the second argument ("errors_terminate"): +/usr/lib/rpm/brp-python-bytecompile \ + %{buildroot}/%{_bindir}/pypy \ + 0 + + %clean rm -rf $RPM_BUILD_ROOT @@ -382,36 +445,43 @@ rm -rf $RPM_BUILD_ROOT %dir %{pypyprefix}/lib-python %{pypyprefix}/lib-python/%{pylibver}/ %{pypyprefix}/lib-python/modified-%{pylibver}/ -%{pypyprefix}/lib-python/conftest.py +%{pypyprefix}/lib-python/conftest.py* %{pypyprefix}/lib_pypy/ %{pypyprefix}/site-packages/ %if 0%{with_jit} -%files jit +%files %defattr(-,root,root,-) -%{_bindir}/pypy-jit -%endif - -%if 0%{with_nojit} -%files nojit -%defattr(-,root,root,-) -%{_bindir}/pypy-nojit -%endif - -%if 0%{with_sandbox} -%files sandbox -%defattr(-,root,root,-) -%{_bindir}/pypy-sandbox +%doc LICENSE README +%{_bindir}/pypy %endif %if 0%{with_stackless} %files stackless %defattr(-,root,root,-) +%doc LICENSE README %{_bindir}/pypy-stackless %endif %changelog +* Wed Dec 15 2010 David Malcolm - 1.4-4 +- rename the jit build and subpackge to just "pypy", and remove the nojit and +sandbox builds, as upstream now seems to be focussing on the JIT build (with +only stackless called out in the getting-started-python docs); disable +stackless for now +- add a verbose_logs specfile boolean; leave it enabled for now (whilst fixing +build issues) +- add more comments, and update others to reflect 1.2 -> 1.4 changes +- re-enable debuginfo within CFLAGS ("-g") +- add the LICENSE and README to all subpackages +- ensure the built binaries don't have the "I need an executable stack" flag +- remove DOS batch files during %%prep (idlelib.bat) +- remove shebang lines from .py files that aren't executable, and remove +executability from .py files that don't have a shebang line (taken from +our python3.spec) +- bytecompile the .py files into .pyc files in pypy's bytecode format + * Sun Nov 28 2010 Toshio Kuratomi - 1.4-3 - BuildRequire valgrind-devel - Install pypy library from the new directory