Python_import

Python
Python
  • reference: Python 源码剖析
  • Python_import
    • import 前奏曲
    • Python中import机制的黑盒探测
      • Python 内建module
      • 用户自定义module
      • 嵌套import
      • import package
      • from与import
      • 符号重命名
      • 符号的销毁与重载
    • import机制的实现
      • 解析 module/package树状结构
      • 加载 module/package
      • 搜索module
      • from与import
    • Python中的import操作
    • 与module有关的名字空间问题

1.Python模块的动态加载机制

a.import 前奏曲

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[IMPORT_NAME]
w = GETITEM(names,oparg); "sys"
x = PyDict_GetItemString(f->f_builtins,"__import__");  //__import__PyCFunctionObject对象
v = POP(); // Py_None
u = TOP(); // -1

//将python的import动作需要使用的信息打包到tuple中
if(PyInt_AsLong(u) != -1 || PyErr_Occurred()){
  w = PyTuple_Pack(5,w,f->f_globals,f->f_locals == NULL ? Py_None : f->f_locals, v, u);
}else{
  w = PyTuple_Pack(4,w,f->f_globals,f->f_locals == NULL ? Py_None : f->f_locals, v);
}

x = PyEval_CallObject(x,w);
SET_TOP(x);
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
[cevcl.c]
#define PyEval_CallObject(func,arg) PyEval_CallObjectWithKeywords(func, arg,(PyObject *)NULL)

PyObject* PyEval_CallObjectWithKeywords(PyObject* func, PyObject* arg, PyObject* kw){
  PyObject* result;

  if(arg == NULL)
    arg = PyTuple_New(0);
  else if(!PyTuple_Check(arg)){
    PyErr_SetString(PyExc_TypeError,"argument list must be a tuple");
    return NULL;
  }

  if(kw != NULL &&  !PyDict_Check(kw)){
    PrErr_SetString(PyExc_TypeError,"keyword list must be a dictionary");
    return NULL;
  }

  result = PyObject_Call(func,arg,kw);
  return result;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
[methodobject.c]
PyTypeObject PyCFunction_Type = {
  PyObject_HEAD_INIT(&PyType_Type)
  0,
  "builtin_function_or_method",
  ...
  PyCFunction_Call, /* tp_call */
  ...
}

[methodobject.c]
PyObject* PyCFunction_Call(PyObject* func, PyObject* ARG, PyObject* kw){
  PyCFunctionObject* f = (PyCFunctionObject *)func;
  PyCFunction meth = PyCFuntion_GET_FUNCTION(func);
  PyObject* self = PyCFuntion_GET_SELF(func);
  int size;

  switch(PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)){
    case METH_VARARGS:
      if(kw == NULL || PyDict_Size(kw) == 0)
        return (*meth)(self,arg);
      break;
    case METH_VARARGS | METH_KEYWORDS:
    case METH_OLDARFS | METH_KEYWORDS:
      //函数调用
      return (*(PyCFunctionWithKeywords)meth)(self,arg,kw);
    ...
  }
  PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", f->m_ml->ml_name);

  return NULL;
}

b.Python中import机制的黑盒探测

  • 标准import
    • Python 内建module
    • 用户自定义module
    • 嵌套import
    • import package
    • from与import
    • 符号重命名
    • 符号的销毁与重载

path_module

如果该module不在pool中，这是python才执行动态加载的动作,如果希望动态改变新的module，则oython通过重新加载reload

c.import机制的实现

Python运行时的全局module pool的维护和搜索

解析与搜索module路径的树状结构

对不同文件格式的module的动态加载机制

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
[bltinmodule.c]
static PyObject* builtin__import__(PyObject* self, PyObject* args, PyObject*kwds){
  static char* kwlist[] = {"name","globals","locals","fromlist","level",0};
  char* name;
  PyObject* globals = NULL;
  PyObject* locals = NULL;
  PyObject* fromlist = NULL;
  int level = -1;

  //从tuple中解析出需要的信息
  if(!PyArg_ParseTupleAndKeywords(args,kwds,"s|oooi:__import__",kwlist,&name,&globals,&locals,&fromlist,&level))
    return NULL;
  return PyImport_ImportModuleLevel(name,globals,locals,fromlist,level);
}

int PyArg_ParseTupleAndKeywords(PyObject* args, PyObject*kw,const char* format,char* keywords[],...) //拆包
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
[import.c]
PyObject* PyImport_ImportModuleLevel(char* name, PyObject* globals, PyObject* locals,PyObject* fromlist, int level){
  PyObject* result;
  lock_import();
  result = import_module_level(name,globals,locals,fromlist,level);
  unlock_import();
  return result;
}

[import.c]
static PyObject* import_module_level(char* name,PyObject* globals, PyObject* locals, PyObject* fromlist, int level){
  char buf[MAXPATHLEN + 1];
  int buflen = 0;
  PyObject* parent,*head,*next,*tail;
  //获得import动作发生的package环境
  parent = get_parent(globals,buf,&buflen,level);
  //解析module的“路径”结构,依次加载每一个package/module
  head = load_next(parent, Py_None, &name, buf, &buflen);
  tail = head;
  while(name){
    next = load_next(tail,tail,&name,buf,&buflen);
    tail = next;
  }

  //处理from ** import ***语句
  if(fromlist != NULL){
    if(fromlist == Py_None || !PyObject_IsTrue(fromlist))
      fromlist = NULL;
  }

  //import 的形式不是from *** import **, 返回head
  if(fromlist == NULL){
    return head;
  }

  //import 的形式是from *** import ** 返回tail
  if(!ensure_fromlist(tail,fromlist.buf,buflen,0)){
    return NULL;
  }

  return tail;
}

fromlist

  • 解析 module/package树状结构
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
[import.c]
static PyObject* get_parent(PyObject* globals, char* buf, Py_ssize_t* p_buflen, int level){
  PyObject* namestr = NULL;
  PyObject* pathstr = NULL;
  PyObject* modname,*modpath,*modules,*parent;

  //获得当前module的名字
  namestr = PyString_InternFromString("__name__");
  pathstr = PyString_InternFromString("__path__");
  *buf = '\0';
  *p_buflen = 0;
  modname = PyDict_GetItem(globals,namestr);
  modpath = PyDict_GetItem(globals,pathstr);
  if(modpath != NULL){
    //在package的__init__.py中进行import动作
    Py_ssize_t len = PyString_GET_SIZE(modname);
    strcpy(buf,PyString_AS_STRING(modname));
  }else{
    //在package中的module中进行import动作
    char* start = PyString_AS_STRING(modname);
    char* lastdot = strrchr(start,'.');
    size_t len;
    len = lastdot - start;
    buf[len] = '\0'
  }

  while (--level > 0) {
    char* dot = strrchr(buf,'.');
    *dot = '\0';
  }
  *p_buflen = len;

  //在sys.modules中查找当前package的名字对应的module对象
  modules = PyImport_GetModuleDict();
  parent = PyDict_GetItemString(modules,buf);
  return parent;
}

Python中的import动作都是发生在某一个package的环境中

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
[import.c]
static PyObject* load_next(PyObject* mod, PyObject* altmod, char** p_name, cha* buf, int* p_buflen){
  char* name = *p_name;
  char* dot = strrchr(name,'.');
  size_t len;
  char* p;
  PyObject* result;

  ... // 获得下一个需要加载的package或module的名字

  // 对package或module进行import动作
  result = import_submodule(mod,p,buf);
  if(result == PyNone && altmod != mod){  //altmod == mod or altmod == Py_None
    result = import_submodule(altmod,p,p);
  }
  return result;
}

buf_p

  • 加载 module/package
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
static PyObject* import_submodule(PyObject* mod, char* subname, char* fullname){
  //获得sys.modules
  PyObject* modules = PyImport_GetModuleDict();
  PyObject* m = NULL;

  /* 约束:
    if mod == None: subname == fullname
    else: mod.__name__ + "." + subname == fullname
  */

  //在sys.modules中查找module/package是否已经被加载
  if((m = PyDict_GetItemString(modules,fullname)) != NULL){
    Py_INCREF(m);
  }
  else{
    PyObject* path, *loader = NULL;
    char* buf[MAXPATHLEN + 1];
    struct filedescr* fdp;
    FILE* fp = NULL;

    //获得import动作的路径信息
    if(mod == PyNone){
      path = NULL;
    }else{
      path = PyObject_GetAttrString(mod,"__path__");
    }

    buf[0] = '\0';

    //搜索module/package
    fdp = find_module(fullname,subname,path,buf,MAXPATHLEN +1 , &fp, &loader);
    //加载module/package
    m = load_module(fullname,fp,buf,fdp->type,loader);
    //将加载的module放入到sys.modules中
    add_submodule(mod,m,fullname,subname,modules);
  }
}
  • 搜索module(subname.py/subname.pyc/subname.pyd/subname.pyo/subname.dll)

find_module

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[importdl.h]
/* Definitions for dynamic loading of extension modules */
enum filetype{
  SEARCH_ERROR,
  PY_SOURCE,
  PY_COMPILED;
  C_EXTENSION,
  PY_RESOURCE, /* Mac only */
  PKG_DIRECTORY,
  C_BUILTIN,
  PY_FROZEN,
  PY_CODERESOURCE, /* Mac only */
  IMP_HOOK
};

struct filedescr{
  char* suffix;
  char* mode;
  enum filetype type;
};

extern struct filedescr* _PyImport_Filetab;
  • 加载module
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
[import.c]
static PyObject* load_module(char* name,FILE* fp,char* buf, int type, PyObject* loader){
  PyObject* modules;
  PyObject* m;
  int err;

  switch (type) {
    //py
    case PY_SOURCE:
      m = load_source_module(name,buf,fp);
      break;
    //pyc
    case PY_COMPILED:
      m = load_compiled_module(name,buf,fp);
      break;
    //dll(pyd)
    case C_EXTENSION:
      m = _PyImport_LoadDynamicModule(name,buf,fp);
      break;
    //加载package
      m = load_package(name,buf);
      break;
    case C_BUILTIN:
      if(buf != NUll && buf[0] != '\0')
        name = buf;
      //创建内建module
      init_builtin(name);
      //确认内建module出现在sys.modules中，如果没有则抛出异常
      modules = PyImport_GetModuleDict();
      m = PyDict_GetItemString(modules,name);
      if(m == NULL){
        ...//抛出异常
        return NULL;
      }
      Py_INCREF(m);
      break;
      ...
      return m;
  }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
//load package
[import.c]
static PyObject* load_package(char* name, char* pathname){
  PyObject* m,*d;
  PyObject* file = NULL;
  PyObject* path = NULL;
  int err;
  char buf[MAXPATHLEN + 1];
  FILE* fp = NULL;
  struct filedescr* fdp;

  //创建PyModuleObject对象，并加入sys.modules中
  m = PyImport_AddModule(name);
  d = PyModule_GetDict(m);
  ...

  //在package的目录夏寻找并加载__init__.py文件
  fdp = find_module(name, "__init__", path,buf,sizeof(buf), &fp, NULL);
  m = load_module(name,fp,buf,fdp->type,NULL);
  ...
  return m;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
//内建module
//init_builtin("math")
[import.c]
static int init_builtin(char* name){
  struct _inittab* p;

  //在内建module的备份中查找名为name的module
  if(_PyImport_FindExtension(name,name) != NULL)
    return 1;

  //遍历内建module集合，寻找匹配的module
  for(p = PyImport_Inittab; p->name != NULL; p++){  //PyImport_Inittab是一个全局变量，维护着一个内建module的完整列表
    if(strcmp(name,p->name) == 0){
      //初始化内建module
      (*p->initfunc)();
      //加入到内建module备份中
      _PyImport_FixupExtension(name,name);
      return 1;
    }
  }
  return 0;
}

[import.h]
struct _inittab{
  char* name;
  void (*initfunc)(void);
}

[import.c]
struct _inittab* PyImport_Inittab = _PyImport_Inittab;

[PC/config.c]
struct _inittab _PyImport_Inittab[] = {
  ...
  {"math",initmath},
  {"nt",initnt},
  ...
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
//C 扩展module
[importdl.c]
PyObject* _PyImport_LoadDynamicModule(char* name,char* pathname, FILE* fp){
  PyObject* m;
  char* lastdot,*shortname,*packagecontext,*oldcontext;
  dl_funcptr p;
  //在python的module备份中检查是否有名为name的module
  if((m = _PyImport_FindExtension(name,pathname)) != NULL){
    Py_INCREF(m);
    return m;
  }
  ...

  //从dll中获得module的初始化函数的其实地址
  p = _PyImport_GetDynLoadFunc(name,shortname,pathname,fp);

  //调用module的初始化函数
  (*p)();
  //从sys.modules中获得已经被加载的module
  m = PyDict_GetItemString(PyImport_GetModuleDict(),name);
  //设置module的__file__属性
  PyModule_AddStringConstant(m,"__file__",pathname);
  //将module加入到python的module备份中
  _PyImport_FixupExtension(name,pathname);
  return m;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
[dynload_win.c]
dl_funcptr _PyImport_GetDynLoadFunc(char* fqname,char* shortname,char* pathname, FILE* fp){
  dl_funcptr p;
  char funcname[258], *import_python;
  //获得module的初始化函数名
  PyOS_snprintf(funcname,sizeof(funcname),"init%.200s",shortname);
  {
    HINSTANCE hDLL = NULL;
    //使用win32 API加载dll文件
    hDLL = LoadLibraryEx(pathname,NULL,LOAD_WITH_ALTERED_SEARCH_PATH);
    if(hDLL == NULL){
      ...//加载dll文件失败。抛出异常
    }else{
      char buffer[256];
      //获得当前python对应的dll文件名
      #ifdef_DEBUG
        PyOS_snprintf(buffer,sizeof(buffer),"python%d%d_d.dll",
      #else
        PyOS_snprintf(buffer,sizeof(buffer),"python%d%d.dll",
      #endif
        PY_MAJOR_VERSION,PY_MINOR_VERSION);
      //获得module中所引用的Python的dll文件名
      import_python = GetPythonImport(hDLL);
      //确保当前Python对应的dll即是module所引用的dll
      if(import_python && strcasecmp(buffer,import_python)){
        ...//dll文件不匹配,抛出异常
      }
    }
    //调用Win32 API获得module初始化函数的地址
    p = GetProcAddress(hDLL,funcname);
  }
  return p;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
[C extension module : abc.dll]
static PyMethodDef abc_methods[] = {
  {"hello",Hello,METH_VARARGS,"say hello"},
  {NULL,NULL}
};

EXPORT int initabc(void){
  Py_InitModule("abc",abc_methods);
  return 0;
}

[modsuport.h]
#define Py_InitModule(name,methods)
  Py_InitModule4(name,methods,(char *)NULL,(PyObject *)NULL,PYTHON_API_VERSION)
  • from与import
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
[import.c]
static PyObject* import_module_level(char* name, PyObject* globals, PyObject* locals, PyObject* fromlist, int level){
  ...

  //处理from *** import ** 语句
  if(fromlist != NULL){
    if(fromlist == Py_None || !PyObject_IsTrue(fromlist))
      fromlist = NULL;
  }

  //import的形式不是from *** import **返回head
  if(fromlist == NULL){
    return head;
  }

  //import的形式是from ** import ** 返回tail
  if(!ensure_fromlist(tail,fromlist,buf,buflen,0)){
    return NULL;
  }
  return tail;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
static int ensure_fromlist(PyObject* mod, PyObject* fromlist, char* buf, Py_ssize_t buflen, int recursive){
  int i;
  for(i = 0; ; i++){
    PyObject* item = PySequence_GetItem(fromlist,i);
    //若item为NULL，则结束ensure动作
    if(item == NULL){
      return 0;
    }

    if(PyString_AS_STRING(item)[0] == '*'){
      PyObject* all;
      /* See if the package defines __all__ */
      all = PyObject_GetAttrString(mod,"__all___");
      int ret = ensure_fromlist(mod,all,buf,buflen,1);
      continue;
    }
    hasit = PyObject_HasAttr(mod,item);
    //hasit为false，意味着出现"from A import mod2"这样的情形
    if(!hasit){
      char* subname = PyString_AS_STRING(item);
      PyObject* submod;
      char* p;
      p = buf + buflen;
      *p++ = '.';
      strcpy(p,subname);
      submod = import_submodule(mod,subname,buf);
    }
  }
}

e.Python中的import操作

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
//from xml.sax import xmlreader
[IMPORT_FROM]
w = GETITEM(names,oparg); //xmlreader
v = TOP(); //xml.sax
x = import_from(v,w);
PUSH(x);

[ceval.c]
static PyObject* import_from(PyObject*v, PyObject* name){
  PyObject* x;

  x = PyObject_GetAttr(v,name);
  if(x == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)){
    PyErr_Format(PyExc_ImportError,"cannot import name %.230s", PyString_AsString(name));
  }
  return x;
}

f.与module有关的名字空间问题

Share