import.c

/* Module definition and import implementation */

#include "Python.h"

#include "Python-ast.h"
#undef Yield /* undefine macro conflicting with winbase.h */
#include "pyarena.h"
#include "pythonrun.h"
#include "errcode.h"
#include "marshal.h"
#include "code.h"
#include "compile.h"
#include "eval.h"
#include "osdefs.h"
#include "importdl.h"

#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef __cplusplus
extern "C" {
#endif

#ifdef MS_WINDOWS
/* for stat.st_mode */
typedef unsigned short mode_t;
#endif


/* Magic word to reject .pyc files generated by other Python versions.
   It should change for each incompatible change to the bytecode.

   The value of CR and LF is incorporated so if you ever read or write
   a .pyc file in text mode the magic number will be wrong; also, the
   Apple MPW compiler swaps their values, botching string constants.

   The magic numbers must be spaced apart atleast 2 values, as the
   -U interpeter flag will cause MAGIC+1 being used. They have been
   odd numbers for some time now.

   There were a variety of old schemes for setting the magic number.
   The current working scheme is to increment the previous value by
   10.

   Known values:
       Python 1.5:   20121
       Python 1.5.1: 20121
       Python 1.5.2: 20121
       Python 1.6:   50428
       Python 2.0:   50823
       Python 2.0.1: 50823
       Python 2.1:   60202
       Python 2.1.1: 60202
       Python 2.1.2: 60202
       Python 2.2:   60717
       Python 2.3a0: 62011
       Python 2.3a0: 62021
       Python 2.3a0: 62011 (!)
       Python 2.4a0: 62041
       Python 2.4a3: 62051
       Python 2.4b1: 62061
       Python 2.5a0: 62071
       Python 2.5a0: 62081 (ast-branch)
       Python 2.5a0: 62091 (with)
       Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
       Python 2.5b3: 62101 (fix wrong code: for x, in ...)
       Python 2.5b3: 62111 (fix wrong code: x += yield)
       Python 2.5c1: 62121 (fix wrong lnotab with for loops and
                            storing constants that should have been removed)
       Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
       Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
       Python 2.6a1: 62161 (WITH_CLEANUP optimization)
       Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
       Python 2.7a0: 62181 (optimize conditional branches:
                introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
       Python 2.7a0  62191 (introduce SETUP_WITH)
       Python 2.7a0  62201 (introduce BUILD_SET)
       Python 2.7a0  62211 (introduce MAP_ADD and SET_ADD)
.
*/
#define MAGIC (62211 | ((long)'\r'<<16) | ((long)'\n'<<24))

/* Magic word as global; note that _PyImport_Init() can change the
   value of this global to accommodate for alterations of how the
   compiler works which are enabled by command line switches. */
static long pyc_magic = MAGIC;

/* See _PyImport_FixupExtension() below */
static PyObject *extensions = NULL;

/* This table is defined in config.c: */
extern struct _inittab _PyImport_Inittab[];

struct _inittab *PyImport_Inittab = _PyImport_Inittab;

/* these tables define the module suffixes that Python recognizes */
struct filedescr * _PyImport_Filetab = NULL;

#ifdef RISCOS
static const struct filedescr _PyImport_StandardFiletab[] = {
    {"/py", "U", PY_SOURCE},
    {"/pyc", "rb", PY_COMPILED},
    {0, 0}
};
#else
static const struct filedescr _PyImport_StandardFiletab[] = {
    {".py", "U", PY_SOURCE},
#ifdef MS_WINDOWS
    {".pyw", "U", PY_SOURCE},
#endif
    {".pyc", "rb", PY_COMPILED},
    {0, 0}
};
#endif

#ifdef MS_WINDOWS
static int isdir(char *path) {
    DWORD rv;
    /* see issue1293 and issue3677:
     * stat() on Windows doesn't recognise paths like
     * "e:\\shared\\" and "\\\\whiterab-c2znlh\\shared" as dirs.
     * Also reference issue6727:
     * stat() on Windows is broken and doesn't resolve symlinks properly.
     */
    rv = GetFileAttributesA(path);
    return rv != INVALID_FILE_ATTRIBUTES && rv & FILE_ATTRIBUTE_DIRECTORY;
}
#else
#ifdef HAVE_STAT
static int isdir(char *path) {
    struct stat statbuf;
    return stat(path, &statbuf) == 0 && S_ISDIR(statbuf.st_mode);
}
#else
#ifdef RISCOS
/* with RISCOS, isdir is in unixstuff */
#else
int isdir(char *path) {
    return 0;
}
#endif /* RISCOS */
#endif /* HAVE_STAT */
#endif /* MS_WINDOWS */

/* Initialize things */

void
_PyImport_Init(void)
{
    const struct filedescr *scan;
    struct filedescr *filetab;
    int countD = 0;
    int countS = 0;

    /* prepare _PyImport_Filetab: copy entries from
       _PyImport_DynLoadFiletab and _PyImport_StandardFiletab.
     */
#ifdef HAVE_DYNAMIC_LOADING
    for (scan = _PyImport_DynLoadFiletab; scan->suffix != NULL; ++scan)
        ++countD;
#endif
    for (scan = _PyImport_StandardFiletab; scan->suffix != NULL; ++scan)
        ++countS;
    filetab = PyMem_NEW(struct filedescr, countD + countS + 1);
    if (filetab == NULL)
        Py_FatalError("Can't initialize import file table.");
#ifdef HAVE_DYNAMIC_LOADING
    memcpy(filetab, _PyImport_DynLoadFiletab,
           countD * sizeof(struct filedescr));
#endif
    memcpy(filetab + countD, _PyImport_StandardFiletab,
           countS * sizeof(struct filedescr));
    filetab[countD + countS].suffix = NULL;

    _PyImport_Filetab = filetab;

    if (Py_OptimizeFlag) {
        /* Replace ".pyc" with ".pyo" in _PyImport_Filetab */
        for (; filetab->suffix != NULL; filetab++) {
#ifndef RISCOS
            if (strcmp(filetab->suffix, ".pyc") == 0)
                filetab->suffix = ".pyo";
#else
            if (strcmp(filetab->suffix, "/pyc") == 0)
                filetab->suffix = "/pyo";
#endif
        }
    }

    if (Py_UnicodeFlag) {
        /* Fix the pyc_magic so that byte compiled code created
           using the all-Unicode method doesn't interfere with
           code created in normal operation mode. */
        pyc_magic = MAGIC + 1;
    }
}

void
_PyImportHooks_Init(void)
{
    PyObject *v, *path_hooks = NULL, *zimpimport;
    int err = 0;

    /* adding sys.path_hooks and sys.path_importer_cache, setting up
       zipimport */
    if (PyType_Ready(&PyNullImporter_Type) < 0)
        goto error;

    if (Py_VerboseFlag)
        PySys_WriteStderr("# installing zipimport hook\n");

    v = PyList_New(0);
    if (v == NULL)
        goto error;
    err = PySys_SetObject("meta_path", v);
    Py_DECREF(v);
    if (err)
        goto error;
    v = PyDict_New();
    if (v == NULL)
        goto error;
    err = PySys_SetObject("path_importer_cache", v);
    Py_DECREF(v);
    if (err)
        goto error;
    path_hooks = PyList_New(0);
    if (path_hooks == NULL)
        goto error;
    err = PySys_SetObject("path_hooks", path_hooks);
    if (err) {
  error:
        PyErr_Print();
        Py_FatalError("initializing sys.meta_path, sys.path_hooks, "
                      "path_importer_cache, or NullImporter failed"
                      );
    }

    zimpimport = PyImport_ImportModule("zipimport");
    if (zimpimport == NULL) {
        PyErr_Clear(); /* No zip import module -- okay */
        if (Py_VerboseFlag)
            PySys_WriteStderr("# can't import zipimport\n");
    }
    else {
        PyObject *zipimporter = PyObject_GetAttrString(zimpimport,
                                                       "zipimporter");
        Py_DECREF(zimpimport);
        if (zipimporter == NULL) {
            PyErr_Clear(); /* No zipimporter object -- okay */
            if (Py_VerboseFlag)
                PySys_WriteStderr(
                    "# can't import zipimport.zipimporter\n");
        }
        else {
            /* sys.path_hooks.append(zipimporter) */
            err = PyList_Append(path_hooks, zipimporter);
            Py_DECREF(zipimporter);
            if (err)
                goto error;
            if (Py_VerboseFlag)
                PySys_WriteStderr(
                    "# installed zipimport hook\n");
        }
    }
    Py_DECREF(path_hooks);
}

void
_PyImport_Fini(void)
{
    Py_XDECREF(extensions);
    extensions = NULL;
    PyMem_DEL(_PyImport_Filetab);
    _PyImport_Filetab = NULL;
}


/* Locking primitives to prevent parallel imports of the same module
   in different threads to return with a partially loaded module.
   These calls are serialized by the global interpreter lock. */

#ifdef WITH_THREAD

#include "pythread.h"

static PyThread_type_lock import_lock = 0;
static long import_lock_thread = -1;
static int import_lock_level = 0;

void
_PyImport_AcquireLock(void)
{
    long me = PyThread_get_thread_ident();
    if (me == -1)
        return; /* Too bad */
    if (import_lock == NULL) {
        import_lock = PyThread_allocate_lock();
        if (import_lock == NULL)
            return;  /* Nothing much we can do. */
    }
    if (import_lock_thread == me) {
        import_lock_level++;
        return;
    }
    if (import_lock_thread != -1 || !PyThread_acquire_lock(import_lock, 0))
    {
        PyThreadState *tstate = PyEval_SaveThread();
        PyThread_acquire_lock(import_lock, 1);
        PyEval_RestoreThread(tstate);
    }
    import_lock_thread = me;
    import_lock_level = 1;
}

int
_PyImport_ReleaseLock(void)
{
    long me = PyThread_get_thread_ident();
    if (me == -1 || import_lock == NULL)
        return 0; /* Too bad */
    if (import_lock_thread != me)
        return -1;
    import_lock_level--;
    if (import_lock_level == 0) {
        import_lock_thread = -1;
        PyThread_release_lock(import_lock);
    }
    return 1;
}

/* This function is called from PyOS_AfterFork to ensure that newly
   created child processes do not share locks with the parent.
   We now acquire the import lock around fork() calls but on some platforms
   (Solaris 9 and earlier? see isue7242) that still left us with problems. */

void
_PyImport_ReInitLock(void)
{
    if (import_lock != NULL) {
        import_lock = PyThread_allocate_lock();
        if (import_lock == NULL) {
            Py_FatalError("PyImport_ReInitLock failed to create a new lock");
        }
    }
    import_lock_thread = -1;
    import_lock_level = 0;
}

#endif

static PyObject *
imp_lock_held(PyObject *self, PyObject *noargs)
{
#ifdef WITH_THREAD
    return PyBool_FromLong(import_lock_thread != -1);
#else
    return PyBool_FromLong(0);
#endif
}

static PyObject *
imp_acquire_lock(PyObject *self, PyObject *noargs)
{
#ifdef WITH_THREAD
    _PyImport_AcquireLock();
#endif
    Py_INCREF(Py_None);
    return Py_None;
}

static PyObject *
imp_release_lock(PyObject *self, PyObject *noargs)
{
#ifdef WITH_THREAD
    if (_PyImport_ReleaseLock() < 0) {
        PyErr_SetString(PyExc_RuntimeError,
                        "not holding the import lock");
        return NULL;
    }
#endif
    Py_INCREF(Py_None);
    return Py_None;
}

static void
imp_modules_reloading_clear(void)
{
    PyInterpreterState *interp = PyThreadState_Get()->interp;
    if (interp->modules_reloading != NULL)
        PyDict_Clear(interp->modules_reloading);
}

/* Helper for sys */

PyObject *
PyImport_GetModuleDict(void)
{
    PyInterpreterState *interp = PyThreadState_GET()->interp;
    if (interp->modules == NULL)
        Py_FatalError("PyImport_GetModuleDict: no module dictionary!");
    return interp->modules;
}


/* List of names to clear in sys */
static char* sys_deletes[] = {
    "path", "argv", "ps1", "ps2", "exitfunc",
    "exc_type", "exc_value", "exc_traceback",
    "last_type", "last_value", "last_traceback",
    "path_hooks", "path_importer_cache", "meta_path",
    /* misc stuff */
    "flags", "float_info",
    NULL
};

static char* sys_files[] = {
    "stdin", "__stdin__",
    "stdout", "__stdout__",
    "stderr", "__stderr__",
    NULL
};


/* Un-initialize things, as good as we can */

void
PyImport_Cleanup(void)
{
    Py_ssize_t pos, ndone;
    char *name;
    PyObject *key, *value, *dict;
    PyInterpreterState *interp = PyThreadState_GET()->interp;
    PyObject *modules = interp->modules;

    if (modules == NULL)
        return; /* Already done */

    /* Delete some special variables first.  These are common
       places where user values hide and people complain when their
       destructors fail.  Since the modules containing them are
       deleted *last* of all, they would come too late in the normal
       destruction order.  Sigh. */

    value = PyDict_GetItemString(modules, "__builtin__");
    if (value != NULL && PyModule_Check(value)) {
        dict = PyModule_GetDict(value);
        if (Py_VerboseFlag)
            PySys_WriteStderr("# clear __builtin__._\n");
        PyDict_SetItemString(dict, "_", Py_None);
    }
    value = PyDict_GetItemString(modules, "sys");
    if (value != NULL && PyModule_Check(value)) {
        char **p;
        PyObject *v;
        dict = PyModule_GetDict(value);
        for (p = sys_deletes; *p != NULL; p++) {
            if (Py_VerboseFlag)
                PySys_WriteStderr("# clear sys.%s\n", *p);
            PyDict_SetItemString(dict, *p, Py_None);
        }
        for (p = sys_files; *p != NULL; p+=2) {
            if (Py_VerboseFlag)
                PySys_WriteStderr("# restore sys.%s\n", *p);
            v = PyDict_GetItemString(dict, *(p+1));
            if (v == NULL)
                v = Py_None;
            PyDict_SetItemString(dict, *p, v);
        }
    }

    /* First, delete __main__ */
    value = PyDict_GetItemString(modules, "__main__");
    if (value != NULL && PyModule_Check(value)) {
        if (Py_VerboseFlag)
            PySys_WriteStderr("# cleanup __main__\n");
        _PyModule_Clear(value);
        PyDict_SetItemString(modules, "__main__", Py_None);
    }

    /* The special treatment of __builtin__ here is because even
       when it's not referenced as a module, its dictionary is
       referenced by almost every module's __builtins__.  Since
       deleting a module clears its dictionary (even if there are
       references left to it), we need to delete the __builtin__
       module last.  Likewise, we don't delete sys until the very
       end because it is implicitly referenced (e.g. by print).

       Also note that we 'delete' modules by replacing their entry
       in the modules dict with None, rather than really deleting
       them; this avoids a rehash of the modules dictionary and
       also marks them as "non existent" so they won't be
       re-imported. */

    /* Next, repeatedly delete modules with a reference count of
       one (skipping __builtin__ and sys) and delete them */
    do {
        ndone = 0;
        pos = 0;
        while (PyDict_Next(modules, &pos, &key, &value)) {
            if (value->ob_refcnt != 1)
                continue;
            if (PyString_Check(key) && PyModule_Check(value)) {
                name = PyString_AS_STRING(key);
                if (strcmp(name, "__builtin__") == 0)
                    continue;
                if (strcmp(name, "sys") == 0)
                    continue;
                if (Py_VerboseFlag)
                    PySys_WriteStderr(
                        "# cleanup[1] %s\n", name);
                _PyModule_Clear(value);
                PyDict_SetItem(modules, key, Py_None);
                ndone++;
            }
        }
    } while (ndone > 0);

    /* Next, delete all modules (still skipping __builtin__ and sys) */
    pos = 0;
    while (PyDict_Next(modules, &pos, &key, &value)) {
        if (PyString_Check(key) && PyModule_Check(value)) {
            name = PyString_AS_STRING(key);
            if (strcmp(name, "__builtin__") == 0)
                continue;
            if (strcmp(name, "sys") == 0)
                continue;
            if (Py_VerboseFlag)
                PySys_WriteStderr("# cleanup[2] %s\n", name);
            _PyModule_Clear(value);
            PyDict_SetItem(modules, key, Py_None);
        }
    }

    /* Next, delete sys and __builtin__ (in that order) */
    value = PyDict_GetItemString(modules, "sys");
    if (value != NULL && PyModule_Check(value)) {
        if (Py_VerboseFlag)
            PySys_WriteStderr("# cleanup sys\n");
        _PyModule_Clear(value);
        PyDict_SetItemString(modules, "sys", Py_None);
    }
    value = PyDict_GetItemString(modules, "__builtin__");
    if (value != NULL && PyModule_Check(value)) {
        if (Py_VerboseFlag)
            PySys_WriteStderr("# cleanup __builtin__\n");
        _PyModule_Clear(value);
        PyDict_SetItemString(modules, "__builtin__", Py_None);
    }

    /* Finally, clear and delete the modules directory */
    PyDict_Clear(modules);
    interp->modules = NULL;
    Py_DECREF(modules);
    Py_CLEAR(interp->modules_reloading);
}


/* Helper for pythonrun.c -- return magic number */

long
PyImport_GetMagicNumber(void)
{
    return pyc_magic;
}


/* Magic for extension modules (built-in as well as dynamically
   loaded).  To prevent initializing an extension module more than
   once, we keep a static dictionary 'extensions' keyed by module name
   (for built-in modules) or by filename (for dynamically loaded
   modules), containing these modules.  A copy of the module's
   dictionary is stored by calling _PyImport_FixupExtension()
   immediately after the module initialization function succeeds.  A
   copy can be retrieved from there by calling
   _PyImport_FindExtension(). */

PyObject *
_PyImport_FixupExtension(char *name, char *filename)
{
    PyObject *modules, *mod, *dict, *copy;
    if (extensions == NULL) {
        extensions = PyDict_New();
        if (extensions == NULL)
            return NULL;
    }
    modules = PyImport_GetModuleDict();
    mod = PyDict_GetItemString(modules, name);
    if (mod == NULL || !PyModule_Check(mod)) {
        PyErr_Format(PyExc_SystemError,
          "_PyImport_FixupExtension: module %.200s not loaded", name);
        return NULL;
    }
    dict = PyModule_GetDict(mod);
    if (dict == NULL)
        return NULL;
    copy = PyDict_Copy(dict);
    if (copy == NULL)
        return NULL;
    PyDict_SetItemString(extensions, filename, copy);
    Py_DECREF(copy);
    return copy;
}

PyObject *
_PyImport_FindExtension(char *name, char *filename)
{
    PyObject *dict, *mod, *mdict;
    if (extensions == NULL)
        return NULL;
    dict = PyDict_GetItemString(extensions, filename);
    if (dict == NULL)
        return NULL;
    mod = PyImport_AddModule(name);
    if (mod == NULL)
        return NULL;
    mdict = PyModule_GetDict(mod);
    if (mdict == NULL)
        return NULL;
    if (PyDict_Update(mdict, dict))
        return NULL;
    if (Py_VerboseFlag)
        PySys_WriteStderr("import %s # previously loaded (%s)\n",
            name, filename);
    return mod;
}


/* Get the module object corresponding to a module name.
   First check the modules dictionary if there's one there,
   if not, create a new one and insert it in the modules dictionary.
   Because the former action is most common, THIS DOES NOT RETURN A
   'NEW' REFERENCE! */

static PyObject *
_PyImport_AddModuleObject(PyObject *name)
{
    PyObject *modules = PyImport_GetModuleDict();
    PyObject *m;

    if ((m = _PyDict_GetItemWithError(modules, name)) != NULL &&
        PyModule_Check(m)) {
        return m;
    }
    if (PyErr_Occurred()) {
        return NULL;
    }
    m = PyModule_New(PyString_AS_STRING(name));
    if (m == NULL) {
        return NULL;
    }
    if (PyDict_SetItem(modules, name, m) != 0) {
        Py_DECREF(m);
        return NULL;
    }
    assert(Py_REFCNT(m) > 1);
    Py_DECREF(m); /* Yes, it still exists, in modules! */

    return m;
}

PyObject *
PyImport_AddModule(const char *name)
{
    PyObject *nameobj, *module;
    nameobj = PyString_FromString(name);
    if (nameobj == NULL)
        return NULL;
    module = _PyImport_AddModuleObject(nameobj);
    Py_DECREF(nameobj);
    return module;
}

/* Remove name from sys.modules, if it's there. */
static void
remove_module(const char *name)
{
    PyObject *modules = PyImport_GetModuleDict();
    if (PyDict_GetItemString(modules, name) == NULL)
        return;
    if (PyDict_DelItemString(modules, name) < 0)
        Py_FatalError("import:  deleting existing key in"
                      "sys.modules failed");
}

/* Execute a code object in a module and return the module object
 * WITH INCREMENTED REFERENCE COUNT.  If an error occurs, name is
 * removed from sys.modules, to avoid leaving damaged module objects
 * in sys.modules.  The caller may wish to restore the original
 * module object (if any) in this case; PyImport_ReloadModule is an
 * example.
 */
PyObject *
PyImport_ExecCodeModule(char *name, PyObject *co)
{
    return PyImport_ExecCodeModuleEx(name, co, (char *)NULL);
}

PyObject *
PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname)
{
    PyObject *modules = PyImport_GetModuleDict();
    PyObject *m, *d, *v;

    m = PyImport_AddModule(name);
    if (m == NULL)
        return NULL;
    /* If the module is being reloaded, we get the old module back
       and re-use its dict to exec the new code. */
    d = PyModule_GetDict(m);
    if (PyDict_GetItemString(d, "__builtins__") == NULL) {
        if (PyDict_SetItemString(d, "__builtins__",
                                 PyEval_GetBuiltins()) != 0)
            goto error;
    }
    /* Remember the filename as the __file__ attribute */
    v = NULL;
    if (pathname != NULL) {
        v = PyString_FromString(pathname);
        if (v == NULL)
            PyErr_Clear();
    }
    if (v == NULL) {
        v = ((PyCodeObject *)co)->co_filename;
        Py_INCREF(v);
    }
    if (PyDict_SetItemString(d, "__file__", v) != 0)
        PyErr_Clear(); /* Not important enough to report */
    Py_DECREF(v);

    v = PyEval_EvalCode((PyCodeObject *)co, d, d);
    if (v == NULL)
        goto error;
    Py_DECREF(v);

    if ((m = PyDict_GetItemString(modules, name)) == NULL) {
        PyErr_Format(PyExc_ImportError,
                     "Loaded module %.200s not found in sys.modules",
                     name);
        return NULL;
    }

    Py_INCREF(m);

    return m;

  error:
    remove_module(name);
    return NULL;
}


/* Given a pathname for a Python source file, fill a buffer with the
   pathname for the corresponding compiled file.  Return the pathname
   for the compiled file, or NULL if there's no space in the buffer.
   Doesn't set an exception. */

static char *
make_compiled_pathname(char *pathname, char *buf, size_t buflen)
{
    size_t len = strlen(pathname);
    if (len+2 > buflen)
        return NULL;

#ifdef MS_WINDOWS
    /* Treat .pyw as if it were .py.  The case of ".pyw" must match
       that used in _PyImport_StandardFiletab. */
    if (len >= 4 && strcmp(&pathname[len-4], ".pyw") == 0)
        --len;          /* pretend 'w' isn't there */
#endif
    memcpy(buf, pathname, len);
    buf[len] = Py_OptimizeFlag ? 'o' : 'c';
    buf[len+1] = '\0';

    return buf;
}


/* Given a pathname for a Python source file, its time of last
   modification, and a pathname for a compiled file, check whether the
   compiled file represents the same version of the source.  If so,
   return a FILE pointer for the compiled file, positioned just after
   the header; if not, return NULL.
   Doesn't set an exception. */

static FILE *
check_compiled_module(char *pathname, time_t mtime, char *cpathname)
{
    FILE *fp;
    long magic;
    long pyc_mtime;

    fp = fopen(cpathname, "rb");
    if (fp == NULL)
        return NULL;
    magic = PyMarshal_ReadLongFromFile(fp);
    if (magic != pyc_magic) {
        if (Py_VerboseFlag)
            PySys_WriteStderr("# %s has bad magic\n", cpathname);
        fclose(fp);
        return NULL;
    }
    pyc_mtime = PyMarshal_ReadLongFromFile(fp);
    if (pyc_mtime != mtime) {
        if (Py_VerboseFlag)
            PySys_WriteStderr("# %s has bad mtime\n", cpathname);
        fclose(fp);
        return NULL;
    }
    if (Py_VerboseFlag)
        PySys_WriteStderr("# %s matches %s\n", cpathname, pathname);
    return fp;
}


/* Read a code object from a file and check it for validity */

static PyCodeObject *
read_compiled_module(char *cpathname, FILE *fp)
{
    PyObject *co;

    co = PyMarshal_ReadLastObjectFromFile(fp);
    if (co == NULL)
        return NULL;
    if (!PyCode_Check(co)) {
        PyErr_Format(PyExc_ImportError,
                     "Non-code object in %.200s", cpathname);
        Py_DECREF(co);
        return NULL;
    }
    return (PyCodeObject *)co;
}


/* Load a module from a compiled file, execute it, and return its
   module object WITH INCREMENTED REFERENCE COUNT */

static PyObject *
load_compiled_module(char *name, char *cpathname, FILE *fp)
{
    long magic;
    PyCodeObject *co;
    PyObject *m;

    magic = PyMarshal_ReadLongFromFile(fp);
    if (magic != pyc_magic) {
        PyErr_Format(PyExc_ImportError,
                     "Bad magic number in %.200s", cpathname);
        return NULL;
    }
    (void) PyMarshal_ReadLongFromFile(fp);
    co = read_compiled_module(cpathname, fp);
    if (co == NULL)
        return NULL;
    if (Py_VerboseFlag)
        PySys_WriteStderr("import %s # precompiled from %s\n",
            name, cpathname);
    m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, cpathname);
    Py_DECREF(co);

    return m;
}

/* Parse a source file and return the corresponding code object */

static PyCodeObject *
parse_source_module(const char *pathname, FILE *fp)
{
    PyCodeObject *co = NULL;
    mod_ty mod;
    PyCompilerFlags flags;
    PyArena *arena = PyArena_New();
    if (arena == NULL)
        return NULL;

    flags.cf_flags = 0;

    mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, &flags,
                               NULL, arena);
    if (mod) {
        co = PyAST_Compile(mod, pathname, NULL, arena);
    }
    PyArena_Free(arena);
    return co;
}


/* Helper to open a bytecode file for writing in exclusive mode */

static FILE *
open_exclusive(char *filename, mode_t mode)
{
#if defined(O_EXCL)&&defined(O_CREAT)&&defined(O_WRONLY)&&defined(O_TRUNC)
    /* Use O_EXCL to avoid a race condition when another process tries to
       write the same file.  When that happens, our open() call fails,
       which is just fine (since it's only a cache).
       XXX If the file exists and is writable but the directory is not
       writable, the file will never be written.  Oh well.
    */
    int fd;
    (void) unlink(filename);
    fd = open(filename, O_EXCL|O_CREAT|O_WRONLY|O_TRUNC
#ifdef O_BINARY
                            |O_BINARY   /* necessary for Windows */
#endif
#ifdef __VMS
            , mode, "ctxt=bin", "shr=nil"
#else
            , mode
#endif
          );
    if (fd < 0)
        return NULL;
    return fdopen(fd, "wb");
#else
    /* Best we can do -- on Windows this can't happen anyway */
    return fopen(filename, "wb");
#endif
}


/* Write a compiled module to a file, placing the time of last
   modification of its source into the header.
   Errors are ignored, if a write error occurs an attempt is made to
   remove the file. */

static void
write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, time_t mtime)
{
    FILE *fp;
#ifdef MS_WINDOWS   /* since Windows uses different permissions  */
    mode_t mode = srcstat->st_mode & ~S_IEXEC;
    /* Issue #6074: We ensure user write access, so we can delete it later
     * when the source file changes. (On POSIX, this only requires write
     * access to the directory, on Windows, we need write access to the file
     * as well)
     */
    mode |= _S_IWRITE;
#else
    mode_t mode = srcstat->st_mode & ~S_IXUSR & ~S_IXGRP & ~S_IXOTH;
#endif

    fp = open_exclusive(cpathname, mode);
    if (fp == NULL) {
        if (Py_VerboseFlag)
            PySys_WriteStderr(
                "# can't create %s\n", cpathname);
        return;
    }
    PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION);
    /* First write a 0 for mtime */
    PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
    PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION);
    if (fflush(fp) != 0 || ferror(fp)) {
        if (Py_VerboseFlag)
            PySys_WriteStderr("# can't write %s\n", cpathname);
        /* Don't keep partial file */
        fclose(fp);
        (void) unlink(cpathname);
        return;
    }
    /* Now write the true mtime (as a 32-bit field) */
    fseek(fp, 4L, 0);
    assert(mtime <= 0xFFFFFFFF);
    PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION);
    fflush(fp);
    fclose(fp);
    if (Py_VerboseFlag)
        PySys_WriteStderr("# wrote %s\n", cpathname);
}

static void
update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname)
{
    PyObject *constants, *tmp;
    Py_ssize_t i, n;

    if (!_PyString_Eq(co->co_filename, oldname))
        return;

    tmp = co->co_filename;
    co->co_filename = newname;
    Py_INCREF(co->co_filename);
    Py_DECREF(tmp);

    constants = co->co_consts;
    n = PyTuple_GET_SIZE(constants);
    for (i = 0; i < n; i++) {
        tmp = PyTuple_GET_ITEM(constants, i);
        if (PyCode_Check(tmp))
            update_code_filenames((PyCodeObject *)tmp,
                                  oldname, newname);
    }
}

static int
update_compiled_module(PyCodeObject *co, char *pathname)
{
    PyObject *oldname, *newname;