Staging
v0.8.1
v0.8.1
Revision 070fae6d0ff49e63bfd5f2bdc66f8eb1df3b6557 authored by Christian Heimes on 02 July 2019, 18:39:42 UTC, committed by Ned Deily on 02 July 2019, 18:42:08 UTC
ssl.match_hostname() no longer accepts IPv4 addresses with additional text after the address and only quad-dotted notation without trailing whitespaces. Some inet_aton() implementations ignore whitespace and all data after whitespace, e.g. '127.0.0.1 whatever'. Short notations like '127.1' for '127.0.0.1' were already filtered out. The bug was initially found by Dominik Czarnota and reported by Paul Kehrer. Signed-off-by: Christian Heimes <christian@python.org> https://bugs.python.org/issue37463
1 parent dcc0eb3
main.py
"""
Generates a layout of Python for Windows from a build.
See python make_layout.py --help for usage.
"""
__author__ = "Steve Dower <steve.dower@python.org>"
__version__ = "3.8"
import argparse
import functools
import os
import re
import shutil
import subprocess
import sys
import tempfile
import zipfile
from pathlib import Path
if __name__ == "__main__":
# Started directly, so enable relative imports
__path__ = [str(Path(__file__).resolve().parent)]
from .support.appxmanifest import *
from .support.catalog import *
from .support.constants import *
from .support.filesets import *
from .support.logging import *
from .support.options import *
from .support.pip import *
from .support.props import *
BDIST_WININST_FILES_ONLY = FileNameSet("wininst-*", "bdist_wininst.py")
BDIST_WININST_STUB = "PC/layout/support/distutils.command.bdist_wininst.py"
TEST_PYDS_ONLY = FileStemSet("xxlimited", "_ctypes_test", "_test*")
TEST_DIRS_ONLY = FileNameSet("test", "tests")
IDLE_DIRS_ONLY = FileNameSet("idlelib")
TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter")
TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo")
TCLTK_FILES_ONLY = FileNameSet("turtle.py")
VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip")
EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*")
EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle")
EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt")
EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*")
EXCLUDE_FROM_CATALOG = FileSuffixSet(".exe", ".pyd", ".dll")
REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*")
LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt")
PY_FILES = FileSuffixSet(".py")
PYC_FILES = FileSuffixSet(".pyc")
CAT_FILES = FileSuffixSet(".cat")
CDF_FILES = FileSuffixSet(".cdf")
DATA_DIRS = FileNameSet("data")
TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser")
TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt")
def get_lib_layout(ns):
def _c(f):
if f in EXCLUDE_FROM_LIB:
return False
if f.is_dir():
if f in TEST_DIRS_ONLY:
return ns.include_tests
if f in TCLTK_DIRS_ONLY:
return ns.include_tcltk
if f in IDLE_DIRS_ONLY:
return ns.include_idle
if f in VENV_DIRS_ONLY:
return ns.include_venv
else:
if f in TCLTK_FILES_ONLY:
return ns.include_tcltk
if f in BDIST_WININST_FILES_ONLY:
return ns.include_bdist_wininst
return True
for dest, src in rglob(ns.source / "Lib", "**/*", _c):
yield dest, src
if not ns.include_bdist_wininst:
src = ns.source / BDIST_WININST_STUB
yield Path("distutils/command/bdist_wininst.py"), src
def get_tcltk_lib(ns):
if not ns.include_tcltk:
return
tcl_lib = os.getenv("TCL_LIBRARY")
if not tcl_lib or not os.path.isdir(tcl_lib):
try:
with open(ns.build / "TCL_LIBRARY.env", "r", encoding="utf-8-sig") as f:
tcl_lib = f.read().strip()
except FileNotFoundError:
pass
if not tcl_lib or not os.path.isdir(tcl_lib):
warn("Failed to find TCL_LIBRARY")
return
for dest, src in rglob(Path(tcl_lib).parent, "**/*"):
yield "tcl/{}".format(dest), src
def get_layout(ns):
def in_build(f, dest="", new_name=None):
n, _, x = f.rpartition(".")
n = new_name or n
src = ns.build / f
if ns.debug and src not in REQUIRED_DLLS:
if not src.stem.endswith("_d"):
src = src.parent / (src.stem + "_d" + src.suffix)
if not n.endswith("_d"):
n += "_d"
f = n + "." + x
yield dest + n + "." + x, src
if ns.include_symbols:
pdb = src.with_suffix(".pdb")
if pdb.is_file():
yield dest + n + ".pdb", pdb
if ns.include_dev:
lib = src.with_suffix(".lib")
if lib.is_file():
yield "libs/" + n + ".lib", lib
if ns.include_appxmanifest:
yield from in_build("python_uwp.exe", new_name="python")
yield from in_build("pythonw_uwp.exe", new_name="pythonw")
else:
yield from in_build("python.exe", new_name="python")
yield from in_build("pythonw.exe", new_name="pythonw")
yield from in_build(PYTHON_DLL_NAME)
if ns.include_launchers and ns.include_appxmanifest:
if ns.include_pip:
yield from in_build("python_uwp.exe", new_name="pip")
if ns.include_idle:
yield from in_build("pythonw_uwp.exe", new_name="idle")
if ns.include_stable:
yield from in_build(PYTHON_STABLE_DLL_NAME)
for dest, src in rglob(ns.build, "vcruntime*.dll"):
yield dest, src
yield "LICENSE.txt", ns.source / "LICENSE"
for dest, src in rglob(ns.build, ("*.pyd", "*.dll")):
if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS:
continue
if src in EXCLUDE_FROM_PYDS:
continue
if src in TEST_PYDS_ONLY and not ns.include_tests:
continue
if src in TCLTK_PYDS_ONLY and not ns.include_tcltk:
continue
yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/")
if ns.zip_lib:
zip_name = PYTHON_ZIP_NAME
yield zip_name, ns.temp / zip_name
else:
for dest, src in get_lib_layout(ns):
yield "Lib/{}".format(dest), src
if ns.include_venv:
yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python")
yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw")
if ns.include_tools:
def _c(d):
if d.is_dir():
return d in TOOLS_DIRS
return d in TOOLS_FILES
for dest, src in rglob(ns.source / "Tools", "**/*", _c):
yield "Tools/{}".format(dest), src
if ns.include_underpth:
yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME
if ns.include_dev:
def _c(d):
if d.is_dir():
return d.name != "internal"
return True
for dest, src in rglob(ns.source / "Include", "**/*.h", _c):
yield "include/{}".format(dest), src
src = ns.source / "PC" / "pyconfig.h"
yield "include/pyconfig.h", src
for dest, src in get_tcltk_lib(ns):
yield dest, src
if ns.include_pip:
pip_dir = get_pip_dir(ns)
if not pip_dir.is_dir():
log_warning("Failed to find {} - pip will not be included", pip_dir)
else:
pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}"
for dest, src in rglob(pip_dir, "**/*"):
if src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB:
continue
yield pkg_root.format(dest), src
if ns.include_chm:
for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME):
yield "Doc/{}".format(dest), src
if ns.include_html_doc:
for dest, src in rglob(ns.doc_build / "html", "**/*"):
yield "Doc/html/{}".format(dest), src
if ns.include_props:
for dest, src in get_props_layout(ns):
yield dest, src
for dest, src in get_appx_layout(ns):
yield dest, src
if ns.include_cat:
if ns.flat_dlls:
yield ns.include_cat.name, ns.include_cat
else:
yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat
def _compile_one_py(src, dest, name, optimize, checked=True):
import py_compile
if dest is not None:
dest = str(dest)
mode = (
py_compile.PycInvalidationMode.CHECKED_HASH
if checked
else py_compile.PycInvalidationMode.UNCHECKED_HASH
)
try:
return Path(
py_compile.compile(
str(src),
dest,
str(name),
doraise=True,
optimize=optimize,
invalidation_mode=mode,
)
)
except py_compile.PyCompileError:
log_warning("Failed to compile {}", src)
return None
def _py_temp_compile(src, ns, dest_dir=None, checked=True):
if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS:
return None
dest = (dest_dir or ns.temp) / (src.stem + ".py")
return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked)
def _write_to_zip(zf, dest, src, ns, checked=True):
pyc = _py_temp_compile(src, ns, checked=checked)
if pyc:
try:
zf.write(str(pyc), dest.with_suffix(".pyc"))
finally:
try:
pyc.unlink()
except:
log_exception("Failed to delete {}", pyc)
return
if src in LIB2TO3_GRAMMAR_FILES:
from lib2to3.pgen2.driver import load_grammar
tmp = ns.temp / src.name
try:
shutil.copy(src, tmp)
load_grammar(str(tmp))
for f in ns.temp.glob(src.stem + "*.pickle"):
zf.write(str(f), str(dest.parent / f.name))
try:
f.unlink()
except:
log_exception("Failed to delete {}", f)
except:
log_exception("Failed to compile {}", src)
finally:
try:
tmp.unlink()
except:
log_exception("Failed to delete {}", tmp)
zf.write(str(src), str(dest))
def generate_source_files(ns):
if ns.zip_lib:
zip_name = PYTHON_ZIP_NAME
zip_path = ns.temp / zip_name
if zip_path.is_file():
zip_path.unlink()
elif zip_path.is_dir():
log_error(
"Cannot create zip file because a directory exists by the same name"
)
return
log_info("Generating {} in {}", zip_name, ns.temp)
ns.temp.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
for dest, src in get_lib_layout(ns):
_write_to_zip(zf, dest, src, ns, checked=False)
if ns.include_underpth:
log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp)
ns.temp.mkdir(parents=True, exist_ok=True)
with open(ns.temp / PYTHON_PTH_NAME, "w", encoding="utf-8") as f:
if ns.zip_lib:
print(PYTHON_ZIP_NAME, file=f)
if ns.include_pip:
print("packages", file=f)
else:
print("Lib", file=f)
print("Lib/site-packages", file=f)
if not ns.flat_dlls:
print("DLLs", file=f)
print(".", file=f)
print(file=f)
print("# Uncomment to run site.main() automatically", file=f)
print("#import site", file=f)
if ns.include_appxmanifest:
log_info("Generating AppxManifest.xml in {}", ns.temp)
ns.temp.mkdir(parents=True, exist_ok=True)
with open(ns.temp / "AppxManifest.xml", "wb") as f:
f.write(get_appxmanifest(ns))
with open(ns.temp / "_resources.xml", "wb") as f:
f.write(get_resources_xml(ns))
if ns.include_pip:
pip_dir = get_pip_dir(ns)
if not (pip_dir / "pip").is_dir():
log_info("Extracting pip to {}", pip_dir)
pip_dir.mkdir(parents=True, exist_ok=True)
extract_pip_files(ns)
if ns.include_props:
log_info("Generating {} in {}", PYTHON_PROPS_NAME, ns.temp)
ns.temp.mkdir(parents=True, exist_ok=True)
with open(ns.temp / PYTHON_PROPS_NAME, "wb") as f:
f.write(get_props(ns))
def _create_zip_file(ns):
if not ns.zip:
return None
if ns.zip.is_file():
try:
ns.zip.unlink()
except OSError:
log_exception("Unable to remove {}", ns.zip)
sys.exit(8)
elif ns.zip.is_dir():
log_error("Cannot create ZIP file because {} is a directory", ns.zip)
sys.exit(8)
ns.zip.parent.mkdir(parents=True, exist_ok=True)
return zipfile.ZipFile(ns.zip, "w", zipfile.ZIP_DEFLATED)
def copy_files(files, ns):
if ns.copy:
ns.copy.mkdir(parents=True, exist_ok=True)
try:
total = len(files)
except TypeError:
total = None
count = 0
zip_file = _create_zip_file(ns)
try:
need_compile = []
in_catalog = []
for dest, src in files:
count += 1
if count % 10 == 0:
if total:
log_info("Processed {:>4} of {} files", count, total)
else:
log_info("Processed {} files", count)
log_debug("Processing {!s}", src)
if (
ns.precompile
and src in PY_FILES
and src not in EXCLUDE_FROM_COMPILE
and src.parent not in DATA_DIRS
and os.path.normcase(str(dest)).startswith(os.path.normcase("Lib"))
):
if ns.copy:
need_compile.append((dest, ns.copy / dest))
else:
(ns.temp / "Lib" / dest).parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, ns.temp / "Lib" / dest)
need_compile.append((dest, ns.temp / "Lib" / dest))
if src not in EXCLUDE_FROM_CATALOG:
in_catalog.append((src.name, src))
if ns.copy:
log_debug("Copy {} -> {}", src, ns.copy / dest)
(ns.copy / dest).parent.mkdir(parents=True, exist_ok=True)
try:
shutil.copy2(src, ns.copy / dest)
except shutil.SameFileError:
pass
if ns.zip:
log_debug("Zip {} into {}", src, ns.zip)
zip_file.write(src, str(dest))
if need_compile:
for dest, src in need_compile:
compiled = [
_compile_one_py(src, None, dest, optimize=0),
_compile_one_py(src, None, dest, optimize=1),
_compile_one_py(src, None, dest, optimize=2),
]
for c in compiled:
if not c:
continue
cdest = Path(dest).parent / Path(c).relative_to(src.parent)
if ns.zip:
log_debug("Zip {} into {}", c, ns.zip)
zip_file.write(c, str(cdest))
in_catalog.append((cdest.name, cdest))
if ns.catalog:
# Just write out the CDF now. Compilation and signing is
# an extra step
log_info("Generating {}", ns.catalog)
ns.catalog.parent.mkdir(parents=True, exist_ok=True)
write_catalog(ns.catalog, in_catalog)
finally:
if zip_file:
zip_file.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-v", help="Increase verbosity", action="count")
parser.add_argument(
"-s",
"--source",
metavar="dir",
help="The directory containing the repository root",
type=Path,
default=None,
)
parser.add_argument(
"-b", "--build", metavar="dir", help="Specify the build directory", type=Path
)
parser.add_argument(
"--doc-build",
metavar="dir",
help="Specify the docs build directory",
type=Path,
default=None,
)
parser.add_argument(
"--copy",
metavar="directory",
help="The name of the directory to copy an extracted layout to",
type=Path,
default=None,
)
parser.add_argument(
"--zip",
metavar="file",
help="The ZIP file to write all files to",
type=Path,
default=None,
)
parser.add_argument(
"--catalog",
metavar="file",
help="The CDF file to write catalog entries to",
type=Path,
default=None,
)
parser.add_argument(
"--log",
metavar="file",
help="Write all operations to the specified file",
type=Path,
default=None,
)
parser.add_argument(
"-t",
"--temp",
metavar="file",
help="A temporary working directory",
type=Path,
default=None,
)
parser.add_argument(
"-d", "--debug", help="Include debug build", action="store_true"
)
parser.add_argument(
"-p",
"--precompile",
help="Include .pyc files instead of .py",
action="store_true",
)
parser.add_argument(
"-z", "--zip-lib", help="Include library in a ZIP file", action="store_true"
)
parser.add_argument(
"--flat-dlls", help="Does not create a DLLs directory", action="store_true"
)
parser.add_argument(
"-a",
"--include-all",
help="Include all optional components",
action="store_true",
)
parser.add_argument(
"--include-cat",
metavar="file",
help="Specify the catalog file to include",
type=Path,
default=None,
)
for opt, help in get_argparse_options():
parser.add_argument(opt, help=help, action="store_true")
ns = parser.parse_args()
update_presets(ns)
ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent)
ns.build = ns.build or Path(sys.executable).parent
ns.temp = ns.temp or Path(tempfile.mkdtemp())
ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build")
if not ns.source.is_absolute():
ns.source = (Path.cwd() / ns.source).resolve()
if not ns.build.is_absolute():
ns.build = (Path.cwd() / ns.build).resolve()
if not ns.temp.is_absolute():
ns.temp = (Path.cwd() / ns.temp).resolve()
if not ns.doc_build.is_absolute():
ns.doc_build = (Path.cwd() / ns.doc_build).resolve()
if ns.include_cat and not ns.include_cat.is_absolute():
ns.include_cat = (Path.cwd() / ns.include_cat).resolve()
if ns.copy and not ns.copy.is_absolute():
ns.copy = (Path.cwd() / ns.copy).resolve()
if ns.zip and not ns.zip.is_absolute():
ns.zip = (Path.cwd() / ns.zip).resolve()
if ns.catalog and not ns.catalog.is_absolute():
ns.catalog = (Path.cwd() / ns.catalog).resolve()
configure_logger(ns)
log_info(
"""OPTIONS
Source: {ns.source}
Build: {ns.build}
Temp: {ns.temp}
Copy to: {ns.copy}
Zip to: {ns.zip}
Catalog: {ns.catalog}""",
ns=ns,
)
if ns.include_idle and not ns.include_tcltk:
log_warning("Assuming --include-tcltk to support --include-idle")
ns.include_tcltk = True
try:
generate_source_files(ns)
files = list(get_layout(ns))
copy_files(files, ns)
except KeyboardInterrupt:
log_info("Interrupted by Ctrl+C")
return 3
except SystemExit:
raise
except:
log_exception("Unhandled error")
if error_was_logged():
log_error("Errors occurred.")
return 1
if __name__ == "__main__":
sys.exit(int(main() or 0))
Computing file changes ...