Staging
v0.8.1
v0.8.1
https://github.com/python/cpython
Tip revision: 8ad7d506ca59a5025216b583df3ff069c66bc5a3 authored by Ćukasz Langa on 20 July 2020, 17:47:09 UTC
Python 3.9.0b5
Python 3.9.0b5
Tip revision: 8ad7d50
test_pypi_packages.py
#!/usr/bin/env python3.8
import argparse
import os
import glob
import tarfile
import zipfile
import shutil
import pathlib
import sys
from typing import Generator, Any
sys.path.insert(0, ".")
from pegen import build
from scripts import test_parse_directory
HERE = pathlib.Path(__file__).resolve().parent
argparser = argparse.ArgumentParser(
prog="test_pypi_packages", description="Helper program to test parsing PyPI packages",
)
argparser.add_argument(
"-t", "--tree", action="count", help="Compare parse tree to official AST", default=0
)
def get_packages() -> Generator[str, None, None]:
all_packages = (
glob.glob("./data/pypi/*.tar.gz")
+ glob.glob("./data/pypi/*.zip")
+ glob.glob("./data/pypi/*.tgz")
)
for package in all_packages:
yield package
def extract_files(filename: str) -> None:
savedir = os.path.join("data", "pypi")
if tarfile.is_tarfile(filename):
tarfile.open(filename).extractall(savedir)
elif zipfile.is_zipfile(filename):
zipfile.ZipFile(filename).extractall(savedir)
else:
raise ValueError(f"Could not identify type of compressed file {filename}")
def find_dirname(package_name: str) -> str:
for name in os.listdir(os.path.join("data", "pypi")):
full_path = os.path.join("data", "pypi", name)
if os.path.isdir(full_path) and name in package_name:
return full_path
assert False # This is to fix mypy, should never be reached
def run_tests(dirname: str, tree: int) -> int:
return test_parse_directory.parse_directory(
dirname,
verbose=False,
excluded_files=[],
tree_arg=tree,
short=True,
mode=1 if tree else 0,
parser="pegen",
)
def main() -> None:
args = argparser.parse_args()
tree = args.tree
for package in get_packages():
print(f"Extracting files from {package}... ", end="")
try:
extract_files(package)
print("Done")
except ValueError as e:
print(e)
continue
print(f"Trying to parse all python files ... ")
dirname = find_dirname(package)
status = run_tests(dirname, tree)
if status == 0:
shutil.rmtree(dirname)
else:
print(f"Failed to parse {dirname}")
if __name__ == "__main__":
main()