Staging
v0.5.1
https://github.com/python/cpython
Revision 96c8475362acb41decd1d7db9243f328973e5de7 authored by Victor Stinner on 26 September 2019, 14:17:34 UTC, committed by GitHub on 26 September 2019, 14:17:34 UTC
* bpo-38234: Py_SetPath() uses the program full path (GH-16357)

Py_SetPath() now sets sys.executable to the program full path
(Py_GetProgramFullPath()), rather than to the program name
(Py_GetProgramName()).

Fix also memory leaks in pathconfig_set_from_config().

(cherry picked from commit 1ce152a42eaa917d7763bce93f1e1ca72530d7ca)

* bpo-38234: Add tests for Python init path config (GH-16358)


(cherry picked from commit bb6bf7d342b4503a6227fd209fac934905b6a1aa)

* bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)

Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed
to test pyvenv.cfg and pybuilddir.txt configuration files.

Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8
encoding, not ASCII.

(cherry picked from commit 52ad33abbfb6637d74932617c7013bae0ccf6e32)

* bpo-38234: Cleanup getpath.c (GH-16367)

* search_for_prefix() directly calls reduce() if found is greater
  than 0.
* Add calculate_pybuilddir() subfunction.
* search_for_prefix(): add path string buffer for readability.
* Fix some error handling code paths: release resources on error.
* calculate_read_pyenv(): rename tmpbuffer to filename.
* test.pythoninfo now also logs windows.dll_path

(cherry picked from commit 221fd84703c545408bbb4a6e0b58459651331f5c)

* bpo-38234: Fix test_embed pathconfig tests (GH-16390)

bpo-38234: On macOS and FreeBSD, the temporary directory can be
symbolic link. For example, /tmp can be a symbolic link to /var/tmp.
Call realpath() to resolve all symbolic links.

(cherry picked from commit 00508a7407d7d300b487532e2271534b20e378a7)

* bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402)

* Add test_embed.test_init_setpath_config(): test Py_SetPath()
  with PyConfig.
* test_init_setpath() and test_init_setpythonhome() no longer call
  Py_SetProgramName(), but use the default program name.
* _PyPathConfig: isolated, site_import  and base_executable
  fields are now only available on Windows.
* If executable is set explicitly in the configuration, ignore
  calculated base_executable: _PyConfig_InitPathConfig() copies
  executable to base_executable.
* Complete path config documentation.

(cherry picked from commit 8bf39b606ef7b02c0279a80789f3c4824b0da5e9)

* bpo-38234: Complete init config documentation (GH-16404)


(cherry picked from commit 88feaecd46a8f427e30ef7ad8cfcddfe392a2402)

* bpo-38234: Fix test_embed.test_init_setpath_config() on FreeBSD (GH-16406)

Explicitly preinitializes with a Python preconfiguration to avoid
Py_SetPath() implicit preinitialization with a compat
preconfiguration.

Fix also test_init_setpath() and test_init_setpythonhome() on macOS:
use self.test_exe as the executable (and base_executable), rather
than shutil.which('python3').

(cherry picked from commit 49d99f01e6e51acec5ca57a02e857f0796bc418b)

* bpo-38234: Py_Initialize() sets global path configuration (GH-16421)

* Py_InitializeFromConfig() now writes PyConfig path configuration to
  the global path configuration (_Py_path_config).
* Add test_embed.test_get_pathconfig().
* Fix typo in _PyWideStringList_Join().

(cherry picked from commit 12f2f177fc483723406d7917194e7f655a20631b)
1 parent 68040ed
Raw File
Tip revision: 96c8475362acb41decd1d7db9243f328973e5de7 authored by Victor Stinner on 26 September 2019, 14:17:34 UTC
[3.8] bpo-38234: Backport init path config changes from master (GH-16423)
Tip revision: 96c8475
dynload_win.c

/* Support for dynamic loading of extension modules */

#include "Python.h"

#ifdef HAVE_DIRECT_H
#include <direct.h>
#endif
#include <ctype.h>

#include "importdl.h"
#include "patchlevel.h"
#include <windows.h>

// "activation context" magic - see dl_nt.c...
#if HAVE_SXS
extern ULONG_PTR _Py_ActivateActCtx();
void _Py_DeactivateActCtx(ULONG_PTR cookie);
#endif

#ifdef _DEBUG
#define PYD_DEBUG_SUFFIX "_d"
#else
#define PYD_DEBUG_SUFFIX ""
#endif

#ifdef PYD_PLATFORM_TAG
#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) "-" PYD_PLATFORM_TAG ".pyd"
#else
#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) ".pyd"
#endif

#define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd"

const char *_PyImport_DynLoadFiletab[] = {
    PYD_TAGGED_SUFFIX,
    PYD_UNTAGGED_SUFFIX,
    NULL
};

/* Function to return the name of the "python" DLL that the supplied module
   directly imports.  Looks through the list of imported modules and
   returns the first entry that starts with "python" (case sensitive) and
   is followed by nothing but numbers until the separator (period).

   Returns a pointer to the import name, or NULL if no matching name was
   located.

   This function parses through the PE header for the module as loaded in
   memory by the system loader.  The PE header is accessed as documented by
   Microsoft in the MSDN PE and COFF specification (2/99), and handles
   both PE32 and PE32+.  It only worries about the direct import table and
   not the delay load import table since it's unlikely an extension is
   going to be delay loading Python (after all, it's already loaded).

   If any magic values are not found (e.g., the PE header or optional
   header magic), then this function simply returns NULL. */

#define DWORD_AT(mem) (*(DWORD *)(mem))
#define WORD_AT(mem)  (*(WORD *)(mem))

static char *GetPythonImport (HINSTANCE hModule)
{
    unsigned char *dllbase, *import_data, *import_name;
    DWORD pe_offset, opt_offset;
    WORD opt_magic;
    int num_dict_off, import_off;

    /* Safety check input */
    if (hModule == NULL) {
        return NULL;
    }

    /* Module instance is also the base load address.  First portion of
       memory is the MS-DOS loader, which holds the offset to the PE
       header (from the load base) at 0x3C */
    dllbase = (unsigned char *)hModule;
    pe_offset = DWORD_AT(dllbase + 0x3C);

    /* The PE signature must be "PE\0\0" */
    if (memcmp(dllbase+pe_offset,"PE\0\0",4)) {
        return NULL;
    }

    /* Following the PE signature is the standard COFF header (20
       bytes) and then the optional header.  The optional header starts
       with a magic value of 0x10B for PE32 or 0x20B for PE32+ (PE32+
       uses 64-bits for some fields).  It might also be 0x107 for a ROM
       image, but we don't process that here.

       The optional header ends with a data dictionary that directly
       points to certain types of data, among them the import entries
       (in the second table entry). Based on the header type, we
       determine offsets for the data dictionary count and the entry
       within the dictionary pointing to the imports. */

    opt_offset = pe_offset + 4 + 20;
    opt_magic = WORD_AT(dllbase+opt_offset);
    if (opt_magic == 0x10B) {
        /* PE32 */
        num_dict_off = 92;
        import_off   = 104;
    } else if (opt_magic == 0x20B) {
        /* PE32+ */
        num_dict_off = 108;
        import_off   = 120;
    } else {
        /* Unsupported */
        return NULL;
    }

    /* Now if an import table exists, offset to it and walk the list of
       imports.  The import table is an array (ending when an entry has
       empty values) of structures (20 bytes each), which contains (at
       offset 12) a relative address (to the module base) at which a
       string constant holding the import name is located. */

    if (DWORD_AT(dllbase + opt_offset + num_dict_off) >= 2) {
        /* We have at least 2 tables - the import table is the second
           one.  But still it may be that the table size is zero */
        if (0 == DWORD_AT(dllbase + opt_offset + import_off + sizeof(DWORD)))
            return NULL;
        import_data = dllbase + DWORD_AT(dllbase +
                                         opt_offset +
                                         import_off);
        while (DWORD_AT(import_data)) {
            import_name = dllbase + DWORD_AT(import_data+12);
            if (strlen(import_name) >= 6 &&
                !strncmp(import_name,"python",6)) {
                char *pch;

#ifndef _DEBUG
                /* In a release version, don't claim that python3.dll is
                   a Python DLL. */
                if (strcmp(import_name, "python3.dll") == 0) {
                    import_data += 20;
                    continue;
                }
#endif

                /* Ensure python prefix is followed only
                   by numbers to the end of the basename */
                pch = import_name + 6;
#ifdef _DEBUG
                while (*pch && pch[0] != '_' && pch[1] != 'd' && pch[2] != '.') {
#else
                while (*pch && *pch != '.') {
#endif
                    if (*pch >= '0' && *pch <= '9') {
                        pch++;
                    } else {
                        pch = NULL;
                        break;
                    }
                }

                if (pch) {
                    /* Found it - return the name */
                    return import_name;
                }
            }
            import_data += 20;
        }
    }

    return NULL;
}

dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix,
                                              const char *shortname,
                                              PyObject *pathname, FILE *fp)
{
    dl_funcptr p;
    char funcname[258], *import_python;
    const wchar_t *wpathname;

#ifndef _DEBUG
    _Py_CheckPython3();
#endif

    wpathname = _PyUnicode_AsUnicode(pathname);
    if (wpathname == NULL)
        return NULL;

    PyOS_snprintf(funcname, sizeof(funcname), "%.20s_%.200s", prefix, shortname);

    {
        HINSTANCE hDLL = NULL;
        unsigned int old_mode;
#if HAVE_SXS
        ULONG_PTR cookie = 0;
#endif

        /* Don't display a message box when Python can't load a DLL */
        old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);

#if HAVE_SXS
        cookie = _Py_ActivateActCtx();
#endif
        /* bpo-36085: We use LoadLibraryEx with restricted search paths
           to avoid DLL preloading attacks and enable use of the
           AddDllDirectory function. We add SEARCH_DLL_LOAD_DIR to
           ensure DLLs adjacent to the PYD are preferred. */
        Py_BEGIN_ALLOW_THREADS
        hDLL = LoadLibraryExW(wpathname, NULL,
                              LOAD_LIBRARY_SEARCH_DEFAULT_DIRS |
                              LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
        Py_END_ALLOW_THREADS
#if HAVE_SXS
        _Py_DeactivateActCtx(cookie);
#endif

        /* restore old error mode settings */
        SetErrorMode(old_mode);

        if (hDLL==NULL){
            PyObject *message;
            unsigned int errorCode;

            /* Get an error string from Win32 error code */
            wchar_t theInfo[256]; /* Pointer to error text
                                  from system */
            int theLength; /* Length of error text */

            errorCode = GetLastError();

            theLength = FormatMessageW(
                FORMAT_MESSAGE_FROM_SYSTEM |
                FORMAT_MESSAGE_IGNORE_INSERTS, /* flags */
                NULL, /* message source */
                errorCode, /* the message (error) ID */
                MAKELANGID(LANG_NEUTRAL,
                           SUBLANG_DEFAULT),
                           /* Default language */
                theInfo, /* the buffer */
                sizeof(theInfo) / sizeof(wchar_t), /* size in wchars */
                NULL); /* no additional format args. */

            /* Problem: could not get the error message.
               This should not happen if called correctly. */
            if (theLength == 0) {
                message = PyUnicode_FromFormat(
                    "DLL load failed with error code %u while importing %s",
                    errorCode, shortname);
            } else {
                /* For some reason a \r\n
                   is appended to the text */
                if (theLength >= 2 &&
                    theInfo[theLength-2] == '\r' &&
                    theInfo[theLength-1] == '\n') {
                    theLength -= 2;
                    theInfo[theLength] = '\0';
                }
                message = PyUnicode_FromFormat(
                    "DLL load failed while importing %s: ", shortname);

                PyUnicode_AppendAndDel(&message,
                    PyUnicode_FromWideChar(
                        theInfo,
                        theLength));
            }
            if (message != NULL) {
                PyObject *shortname_obj = PyUnicode_FromString(shortname);
                PyErr_SetImportError(message, shortname_obj, pathname);
                Py_XDECREF(shortname_obj);
                Py_DECREF(message);
            }
            return NULL;
        } else {
            char buffer[256];

            PyOS_snprintf(buffer, sizeof(buffer),
#ifdef _DEBUG
                          "python%d%d_d.dll",
#else
                          "python%d%d.dll",
#endif
                          PY_MAJOR_VERSION,PY_MINOR_VERSION);
            import_python = GetPythonImport(hDLL);

            if (import_python &&
                _stricmp(buffer,import_python)) {
                PyErr_Format(PyExc_ImportError,
                             "Module use of %.150s conflicts "
                             "with this version of Python.",
                             import_python);
                Py_BEGIN_ALLOW_THREADS
                FreeLibrary(hDLL);
                Py_END_ALLOW_THREADS
                return NULL;
            }
        }
        Py_BEGIN_ALLOW_THREADS
        p = GetProcAddress(hDLL, funcname);
        Py_END_ALLOW_THREADS
    }

    return p;
}
back to top