Staging
v0.5.1
https://github.com/python/cpython
Revision 66dd5338a1ca98921c8e6c51228541ef8ed8076a authored by Miss Islington (bot) on 18 November 2020, 16:28:57 UTC, committed by GitHub on 18 November 2020, 16:28:57 UTC

Fix a race condition in "make regen-all" when make -jN option is used
to run jobs in parallel. The clinic.py script now only use atomic
write to write files. Moveover, generated files are now left
unchanged if the content does not change, to not change the file
modification time.

The "make regen-all" command runs "make clinic" and "make
regen-importlib" targets:

* "make regen-importlib" builds object files (ex: Modules/_weakref.o)
  from source files (ex: Modules/_weakref.c) and clinic files (ex:
  Modules/clinic/_weakref.c.h)
* "make clinic" always rewrites all clinic files
  (ex: Modules/clinic/_weakref.c.h)

Since there is no dependency between "clinic" and "regen-importlib"
Makefile targets, these two targets can be run in parallel. Moreover,
half of clinic.py file writes are not atomic and so there is a race
condition when "make regen-all" runs jobs in parallel using make -jN
option (which can be passed in MAKEFLAGS environment variable).

Fix clinic.py to make all file writes atomic:

* Add write_file() function to ensure that all file writes are
  atomic: write into a temporary file and then use os.replace().
* Moreover, write_file() doesn't recreate or modify the file if the
  content does not change to avoid modifying the file modification
  file.
* Update test_clinic to verify these assertions with a functional
  test.
* Remove Clinic.force attribute which was no longer used, whereas
  Clinic.verify remains useful.

(cherry picked from commit 8fba9523cf08029dc2e280d9f48fdd57ab178c9d)
(cherry picked from commit c53c3f400050a7edc92ccb7285a6d7eeb4c37fd2)

Co-authored-by: Victor Stinner <vstinner@python.org>
1 parent 73e02ff
Raw File
Tip revision: 66dd5338a1ca98921c8e6c51228541ef8ed8076a authored by Miss Islington (bot) on 18 November 2020, 16:28:57 UTC
bpo-42398: Fix "make regen-all" race condition (GH-23362) (GH-23367)
Tip revision: 66dd533
valgrind-python.supp
#
# This is a valgrind suppression file that should be used when using valgrind.
#
#  Here's an example of running valgrind:
#
#	cd python/dist/src
#	valgrind --tool=memcheck --suppressions=Misc/valgrind-python.supp \
#		./python -E ./Lib/test/regrtest.py -u gui,network
#
# You must edit Objects/obmalloc.c and uncomment Py_USING_MEMORY_DEBUGGER
# to use the preferred suppressions with address_in_range.
#
# If you do not want to recompile Python, you can uncomment
# suppressions for _PyObject_Free and _PyObject_Realloc.
#
# See Misc/README.valgrind for more information.

# all tool names: Addrcheck,Memcheck,cachegrind,helgrind,massif
{
   ADDRESS_IN_RANGE/Invalid read of size 4
   Memcheck:Addr4
   fun:address_in_range
}

{
   ADDRESS_IN_RANGE/Invalid read of size 4
   Memcheck:Value4
   fun:address_in_range
}

{
   ADDRESS_IN_RANGE/Invalid read of size 8 (x86_64 aka amd64)
   Memcheck:Value8
   fun:address_in_range
}

{
   ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value
   Memcheck:Cond
   fun:address_in_range
}

#
# Leaks (including possible leaks)
#    Hmmm, I wonder if this masks some real leaks.  I think it does.
#    Will need to fix that.
#

{
   Suppress leaking the GIL.  Happens once per process, see comment in ceval.c.
   Memcheck:Leak
   fun:malloc
   fun:PyThread_allocate_lock
   fun:PyEval_InitThreads
}

{
   Suppress leaking the GIL after a fork.
   Memcheck:Leak
   fun:malloc
   fun:PyThread_allocate_lock
   fun:PyEval_ReInitThreads
}

{
   Suppress leaking the autoTLSkey.  This looks like it shouldn't leak though.
   Memcheck:Leak
   fun:malloc
   fun:PyThread_create_key
   fun:_PyGILState_Init
   fun:Py_InitializeEx
   fun:Py_Main
}

{
   Hmmm, is this a real leak or like the GIL?
   Memcheck:Leak
   fun:malloc
   fun:PyThread_ReInitTLS
}

{
   Handle PyMalloc confusing valgrind (possibly leaked)
   Memcheck:Leak
   fun:realloc
   fun:_PyObject_GC_Resize
   fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING
}

{
   Handle PyMalloc confusing valgrind (possibly leaked)
   Memcheck:Leak
   fun:malloc
   fun:_PyObject_GC_New
   fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING
}

{
   Handle PyMalloc confusing valgrind (possibly leaked)
   Memcheck:Leak
   fun:malloc
   fun:_PyObject_GC_NewVar
   fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING
}

#
# Non-python specific leaks
#

{
   Handle pthread issue (possibly leaked)
   Memcheck:Leak
   fun:calloc
   fun:allocate_dtv
   fun:_dl_allocate_tls_storage
   fun:_dl_allocate_tls
}

{
   Handle pthread issue (possibly leaked)
   Memcheck:Leak
   fun:memalign
   fun:_dl_allocate_tls_storage
   fun:_dl_allocate_tls
}

###{
###   ADDRESS_IN_RANGE/Invalid read of size 4
###   Memcheck:Addr4
###   fun:_PyObject_Free
###}
###
###{
###   ADDRESS_IN_RANGE/Invalid read of size 4
###   Memcheck:Value4
###   fun:_PyObject_Free
###}
###
###{
###   ADDRESS_IN_RANGE/Use of uninitialised value of size 8
###   Memcheck:Addr8
###   fun:_PyObject_Free
###}
###
###{
###   ADDRESS_IN_RANGE/Use of uninitialised value of size 8
###   Memcheck:Value8
###   fun:_PyObject_Free
###}
###
###{
###   ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value
###   Memcheck:Cond
###   fun:_PyObject_Free
###}

###{
###   ADDRESS_IN_RANGE/Invalid read of size 4
###   Memcheck:Addr4
###   fun:_PyObject_Realloc
###}
###
###{
###   ADDRESS_IN_RANGE/Invalid read of size 4
###   Memcheck:Value4
###   fun:_PyObject_Realloc
###}
###
###{
###   ADDRESS_IN_RANGE/Use of uninitialised value of size 8
###   Memcheck:Addr8
###   fun:_PyObject_Realloc
###}
###
###{
###   ADDRESS_IN_RANGE/Use of uninitialised value of size 8
###   Memcheck:Value8
###   fun:_PyObject_Realloc
###}
###
###{
###   ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value
###   Memcheck:Cond
###   fun:_PyObject_Realloc
###}

###
### All the suppressions below are for errors that occur within libraries
### that Python uses.  The problems to not appear to be related to Python's
### use of the libraries.
###

{
   Generic ubuntu ld problems
   Memcheck:Addr8
   obj:/lib/ld-2.4.so
   obj:/lib/ld-2.4.so
   obj:/lib/ld-2.4.so
   obj:/lib/ld-2.4.so
}

{
   Generic gentoo ld problems
   Memcheck:Cond
   obj:/lib/ld-2.3.4.so
   obj:/lib/ld-2.3.4.so
   obj:/lib/ld-2.3.4.so
   obj:/lib/ld-2.3.4.so
}

{
   DBM problems, see test_dbm
   Memcheck:Param
   write(buf)
   fun:write
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   fun:dbm_close
}

{
   DBM problems, see test_dbm
   Memcheck:Value8
   fun:memmove
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   fun:dbm_store
   fun:dbm_ass_sub
}

{
   DBM problems, see test_dbm
   Memcheck:Cond
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   fun:dbm_store
   fun:dbm_ass_sub
}

{
   DBM problems, see test_dbm
   Memcheck:Cond
   fun:memmove
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   obj:/usr/lib/libdb1.so.2
   fun:dbm_store
   fun:dbm_ass_sub
}

{
   GDBM problems, see test_gdbm
   Memcheck:Param
   write(buf)
   fun:write
   fun:gdbm_open

}

{
   Uninitialised byte(s) false alarm, see bpo-35561
   Memcheck:Param
   epoll_ctl(event)
   fun:epoll_ctl
   fun:pyepoll_internal_ctl
}

{
   ZLIB problems, see test_gzip
   Memcheck:Cond
   obj:/lib/libz.so.1.2.3
   obj:/lib/libz.so.1.2.3
   fun:deflate
}

{
   Avoid problems w/readline doing a putenv and leaking on exit
   Memcheck:Leak
   fun:malloc
   fun:xmalloc
   fun:sh_set_lines_and_columns
   fun:_rl_get_screen_size
   fun:_rl_init_terminal_io
   obj:/lib/libreadline.so.4.3
   fun:rl_initialize
}

# Valgrind emits "Conditional jump or move depends on uninitialised value(s)"
# false alarms on GCC builtin strcmp() function. The GCC code is correct.
#
# Valgrind bug: https://bugs.kde.org/show_bug.cgi?id=264936
{
   bpo-38118: Valgrind emits false alarm on GCC builtin strcmp()
   Memcheck:Cond
   fun:PyUnicode_Decode
}


###
### These occur from somewhere within the SSL, when running
###  test_socket_sll.  They are too general to leave on by default.
###
###{
###   somewhere in SSL stuff
###   Memcheck:Cond
###   fun:memset
###}
###{
###   somewhere in SSL stuff
###   Memcheck:Value4
###   fun:memset
###}
###
###{
###   somewhere in SSL stuff
###   Memcheck:Cond
###   fun:MD5_Update
###}
###
###{
###   somewhere in SSL stuff
###   Memcheck:Value4
###   fun:MD5_Update
###}

# Fedora's package "openssl-1.0.1-0.1.beta2.fc17.x86_64" on x86_64
# See http://bugs.python.org/issue14171
{
   openssl 1.0.1 prng 1
   Memcheck:Cond
   fun:bcmp
   fun:fips_get_entropy
   fun:FIPS_drbg_instantiate
   fun:RAND_init_fips
   fun:OPENSSL_init_library
   fun:SSL_library_init
   fun:init_hashlib
}

{
   openssl 1.0.1 prng 2
   Memcheck:Cond
   fun:fips_get_entropy
   fun:FIPS_drbg_instantiate
   fun:RAND_init_fips
   fun:OPENSSL_init_library
   fun:SSL_library_init
   fun:init_hashlib
}

{
   openssl 1.0.1 prng 3
   Memcheck:Value8
   fun:_x86_64_AES_encrypt_compact
   fun:AES_encrypt
}

#
# All of these problems come from using test_socket_ssl
#
{
   from test_socket_ssl
   Memcheck:Cond
   fun:BN_bin2bn
}

{
   from test_socket_ssl
   Memcheck:Cond
   fun:BN_num_bits_word
}

{
   from test_socket_ssl
   Memcheck:Value4
   fun:BN_num_bits_word
}

{
   from test_socket_ssl
   Memcheck:Cond
   fun:BN_mod_exp_mont_word
}

{
   from test_socket_ssl
   Memcheck:Cond
   fun:BN_mod_exp_mont
}

{
   from test_socket_ssl
   Memcheck:Param
   write(buf)
   fun:write
   obj:/usr/lib/libcrypto.so.0.9.7
}

{
   from test_socket_ssl
   Memcheck:Cond
   fun:RSA_verify
}

{
   from test_socket_ssl
   Memcheck:Value4
   fun:RSA_verify
}

{
   from test_socket_ssl
   Memcheck:Value4
   fun:DES_set_key_unchecked
}

{
   from test_socket_ssl
   Memcheck:Value4
   fun:DES_encrypt2
}

{
   from test_socket_ssl
   Memcheck:Cond
   obj:/usr/lib/libssl.so.0.9.7
}

{
   from test_socket_ssl
   Memcheck:Value4
   obj:/usr/lib/libssl.so.0.9.7
}

{
   from test_socket_ssl
   Memcheck:Cond
   fun:BUF_MEM_grow_clean
}

{
   from test_socket_ssl
   Memcheck:Cond
   fun:memcpy
   fun:ssl3_read_bytes
}

{
   from test_socket_ssl
   Memcheck:Cond
   fun:SHA1_Update
}

{
   from test_socket_ssl
   Memcheck:Value4
   fun:SHA1_Update
}

{
   test_buffer_non_debug
   Memcheck:Addr4
   fun:PyUnicodeUCS2_FSConverter
}

{
   test_buffer_non_debug
   Memcheck:Addr4
   fun:PyUnicode_FSConverter
}

{
   wcscmp_false_positive
   Memcheck:Addr8
   fun:wcscmp
   fun:_PyOS_GetOpt
   fun:Py_Main
   fun:main
}

# Additional suppressions for the unified decimal tests:
{
   test_decimal
   Memcheck:Addr4
   fun:PyUnicodeUCS2_FSConverter
}

{
   test_decimal2
   Memcheck:Addr4
   fun:PyUnicode_FSConverter
}

back to top