Staging
v0.8.1
v0.8.1
https://github.com/python/cpython
Revision 66dd5338a1ca98921c8e6c51228541ef8ed8076a authored by Miss Islington (bot) on 18 November 2020, 16:28:57 UTC, committed by GitHub on 18 November 2020, 16:28:57 UTC
Fix a race condition in "make regen-all" when make -jN option is used to run jobs in parallel. The clinic.py script now only use atomic write to write files. Moveover, generated files are now left unchanged if the content does not change, to not change the file modification time. The "make regen-all" command runs "make clinic" and "make regen-importlib" targets: * "make regen-importlib" builds object files (ex: Modules/_weakref.o) from source files (ex: Modules/_weakref.c) and clinic files (ex: Modules/clinic/_weakref.c.h) * "make clinic" always rewrites all clinic files (ex: Modules/clinic/_weakref.c.h) Since there is no dependency between "clinic" and "regen-importlib" Makefile targets, these two targets can be run in parallel. Moreover, half of clinic.py file writes are not atomic and so there is a race condition when "make regen-all" runs jobs in parallel using make -jN option (which can be passed in MAKEFLAGS environment variable). Fix clinic.py to make all file writes atomic: * Add write_file() function to ensure that all file writes are atomic: write into a temporary file and then use os.replace(). * Moreover, write_file() doesn't recreate or modify the file if the content does not change to avoid modifying the file modification file. * Update test_clinic to verify these assertions with a functional test. * Remove Clinic.force attribute which was no longer used, whereas Clinic.verify remains useful. (cherry picked from commit 8fba9523cf08029dc2e280d9f48fdd57ab178c9d) (cherry picked from commit c53c3f400050a7edc92ccb7285a6d7eeb4c37fd2) Co-authored-by: Victor Stinner <vstinner@python.org>
1 parent 73e02ff
Tip revision: 66dd5338a1ca98921c8e6c51228541ef8ed8076a authored by Miss Islington (bot) on 18 November 2020, 16:28:57 UTC
bpo-42398: Fix "make regen-all" race condition (GH-23362) (GH-23367)
bpo-42398: Fix "make regen-all" race condition (GH-23362) (GH-23367)
Tip revision: 66dd533
.travis.yml
language: c
dist: xenial
# To cache doc-building dependencies and C compiler output.
cache:
- pip
- ccache
- directories:
- $HOME/multissl
env:
global:
- OPENSSL=1.1.1f
- OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}"
- PATH="${OPENSSL_DIR}/bin:$PATH"
- CFLAGS="-I${OPENSSL_DIR}/include"
- LDFLAGS="-L${OPENSSL_DIR}/lib"
# Set rpath with env var instead of -Wl,-rpath linker flag
# OpenSSL ignores LDFLAGS when linking bin/openssl
- LD_RUN_PATH="${OPENSSL_DIR}/lib"
branches:
only:
- master
- /^\d\.\d$/
- buildbot-custom
matrix:
fast_finish: true
allow_failures:
- env: OPTIONAL=true
include:
- name: "CPython tests"
os: linux
language: c
compiler: clang
# gcc also works, but to keep the # of concurrent builds down, we use one C
# compiler here and the other to run the coverage build. Clang is preferred
# in this instance for its better error messages.
env: TESTING=cpython
addons:
apt:
packages:
- gdb
- xvfb
- name: "Documentation build"
os: linux
language: python
# Build the docs against a stable version of Python so code bugs don't hold up doc-related PRs.
python: 3.6
env: TESTING=docs
before_script:
- cd Doc
- make venv PYTHON=python
script:
- make check suspicious html SPHINXOPTS="-q -W -j4"
- name: "Documentation tests"
os: linux
language: c
compiler: clang
env: TESTING=doctest
addons:
apt:
packages:
- xvfb
before_script:
- ./configure
- make -j4
- make -C Doc/ PYTHON=../python venv
script:
xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest
- name: "Test code coverage (Python)"
os: linux
language: c
compiler: gcc
env: OPTIONAL=true
addons:
apt:
packages:
- xvfb
before_script:
- |
if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]
then
echo "Don't run Python coverage on pull requests."
exit
fi
- ./configure
- make -j4
# Need a venv that can parse covered code.
- ./python -m venv venv
- ./venv/bin/python -m pip install -U coverage
- ./venv/bin/python -m test.pythoninfo
script:
# Skip tests that re-run the entire test suite.
- xvfb-run ./venv/bin/python -m coverage run --branch --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures || true
after_script: # Probably should be after_success once test suite updated to run under coverage.py.
# Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
- source ./venv/bin/activate
- bash <(curl -s https://codecov.io/bash)
- name: "Test code coverage (C)"
os: linux
language: c
compiler: gcc
env: OPTIONAL=true
addons:
apt:
packages:
- lcov
- xvfb
before_script:
- |
if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]
then
echo "Don't run C coverage on pull requests."
exit
fi
- ./configure
script:
- xvfb-run make -j4 coverage-report
after_script: # Probably should be after_success once test suite updated to run under coverage.py.
- make pythoninfo
- bash <(curl -s https://codecov.io/bash)
before_install:
- set -e
- |
# Check short-circuit conditions
if [[ "${TESTING}" != "docs" && "${TESTING}" != "doctest" ]]
then
if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]
then
echo "Not a PR, doing full build."
else
# Pull requests are slightly complicated because $TRAVIS_COMMIT_RANGE
# may include more changes than desired if the history is convoluted.
# Instead, explicitly fetch the base branch and compare against the
# merge-base commit.
git fetch -q origin +refs/heads/$TRAVIS_BRANCH
changes=$(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD))
echo "Files changed:"
echo "$changes"
if ! echo "$changes" | grep -qvE '(\.rst$)|(^Doc)|(^Misc)'
then
echo "Only docs were updated, stopping build process."
exit
fi
fi
fi
install:
- |
# Install OpenSSL as necessary
# Note: doctest needs OpenSSL
if [[ "${TESTING}" != "docs" ]]
then
# clang complains about unused-parameter a lot, redirect stderr
python3 Tools/ssl/multissltests.py --steps=library \
--base-directory ${HOME}/multissl \
--openssl ${OPENSSL} >/dev/null 2>&1
fi
- openssl version
# Travis provides only 2 cores, so don't overdo the parallelism and waste memory.
before_script:
# -Og is much faster than -O0
- CFLAGS="${CFLAGS} -Og" ./configure --with-pydebug
- make -j4 regen-all
- changes=`git status --porcelain`
- |
# Check for changes in regenerated files
if ! test -z "$changes"
then
echo "Generated files not up to date"
echo "$changes"
exit 1
fi
- make -j4
- make pythoninfo
script:
# Using the built Python as patchcheck.py is built around the idea of using
# a checkout-build of CPython to know things like what base branch the changes
# should be compared against.
# Only run on Linux as the check only needs to be run once.
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi
# Check that all symbols exported by libpython start with "Py" or "_Py"
- make smelly
# `-r -w` implicitly provided through `make buildbottest`.
- |
if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
XVFB_RUN=xvfb-run;
fi
$XVFB_RUN make buildbottest TESTOPTS="-j4 -uall,-cpu"
notifications:
email: false
irc:
channels:
# This is set to a secure variable to prevent forks from notifying the
# IRC channel whenever they fail a build. This can be removed when travis
# implements https://github.com/travis-ci/travis-ci/issues/1094.
# The actual value here is: irc.freenode.net#python-dev
- secure: "s7kAkpcom2yUJ8XqyjFI0obJmhAGrn1xmoivdaPdgBIA++X47TBp1x4pgDsbEsoalef7bEwa4l07KdT4qa+DOd/c4QxaWom7fbN3BuLVsZuVfODnl79+gYq/TAbGfyH+yDs18DXrUfPgwD7C5aW32ugsqAOd4iWzfGJQ5OrOZzqzGjYdYQUEkJFXgxDEIb4aHvxNDWGO3Po9uKISrhb5saQ0l776yLo1Ur7M4oxl8RTbCdgX0vf5TzPg52BgvZpOgt3DHOUYPeiJLKNjAE6ibg0U95sEvMfHX77nz4aFY4/3UI6FFaRla34rZ+mYKrn0TdxOhera1QOgPmM6HzdO4K44FpfK1DS0Xxk9U9/uApq+cG0bU3W+cVUHDBe5+90lpRBAXHeHCgT7TI8gec614aiT8lEr3+yH8OBRYGzkjNK8E2LJZ/SxnVxDe7aLF6AWcoWLfS6/ziAIBFQ5Nc4U72CT8fGVSkl8ywPiRlvixKdvTODMSZo0jMqlfZSNaAPTsNRx4wu5Uis4qekwe32Fz4aB6KGpsuuVjBi+H6v0RKxNJNGY3JKDiEH2TK0UE2auJ5GvLW48aUVFcQMB7euCWYXlSWVRHh3WLU8QXF29Dw4JduRZqUpOdRgMHU79UHRq+mkE0jAS/nBcS6CvsmxCpTSrfVYuMOu32yt18QQoTyU="
on_success: change
on_failure: always
skip_join: true
webhooks:
urls:
# For the same reasons as above for IRC, we encrypt the webhook address
# for Zulip. The actual value is:
# https://python.zulipchat.com/api/v1/external/travis?api_key=<api-key-redacted>&stream=core%2Ftest+runs
- secure: "vLz2TodSL7wQ8DsIu86koRS9i4dsK40PH8+wzY93PBCCAzJAz113LTxK3/9PamMv+ObDRUSe5OpXcquE3d5Gwpu8IymF113qK0I3uNr+O3FtmKlj/Kd1P/V+z4pTpS3zh3lW9gyKV9EMWXIWS0IYKKZQU144XqUlIiucWK2jHJF/cSz2cRAx/6Kx68X4mZwEC7hiMOF4ZsWUMbCglM89ybeS0pr0kK9mmh88qsPyRQov3mRKswmVPlePk7clVLNAL43qSe3SzmrmACZfQ9KJYmpYnr/cjo2b6svYHcQBAwAUarZZ9KBMXeV7HwGWsSXAvHH2ynR0P++braBHGEMTGMSitdVWzFTmeiHnrkp08WAB+uFs54iEx3VklTk9bCzozTm2S94TRxbrsG9SypMvQxG570JV6P2XYuR+taCb/GMtMqrtGQm2e1Ht+nDLtiUb+/+rwEbicJJ13knptOQZI4tPOZESI/kXkORkSNwFfLSNLSl9jTlMmO7KjAAPApURHEdx26RbItAn8mIX2NcHTRjKn2qV4h3C54nmHmKWn/ZudHHJc6ieZSEUBoaLGAYmcWJRqrM6jiy2h9I9TRrCKAiGh5jT47FYKLwosTtV245l/ZhDb6eTVfEFT6TSLEoyfx9cCtTUvfMtXYl8eN9wlFYYpH8MSWbMD14eEkKBTWg="
on_success: change
on_failure: always
Computing file changes ...