Python C API cheatsheet

Performance of ctypes

// fib.c
unsigned int fib(unsigned int n)
{
    if ( n < 2) {
        return n;
    }
    return fib(n-1) + fib(n-2);
}

Building a libfib.dylib (Mac OSX)

clang -Wall -Werror -shared -fPIC -o libfib.dylib fib.c

Comparing the performance

>>> import time
>>> from ctypes import *
>>> def fib(n):
...     if n < 2:
...         return n
...     return fib(n-1) + fib(n-2)
...
>>> s = time.time(); fib(35); e = time.time()
9227465
>>> print("cost time: {} sec".format(e - s))
cost time: 4.09563493729 sec
>>> libfib = CDLL("./libfib.dylib")
>>> s = time.time(); libfib.fib(35); e = time.time()
9227465
>>> print("cost time: {} sec".format(e - s))
cost time: 0.0819959640503 sec

Error handling when use ctypes

from __future__ import print_function

import errno
import os

from ctypes import *
from sys import platform, maxsize

is_64bits = maxsize > 2**32

if is_64bits and platform == 'darwin':
    libc = CDLL("libc.dylib", use_errno=True)
else:
    raise RuntimeError("Not support platform: {}".format(platform))

stat = libc.stat

class Stat(Structure):
    '''
    From /usr/include/sys/stat.h

    struct stat {
        dev_t         st_dev;
        ino_t         st_ino;
        mode_t        st_mode;
        nlink_t       st_nlink;
        uid_t         st_uid;
        gid_t         st_gid;
        dev_t         st_rdev;
    #ifndef _POSIX_SOURCE
        struct      timespec st_atimespec;
        struct      timespec st_mtimespec;
        struct      timespec st_ctimespec;
    #else
        time_t        st_atime;
        long          st_atimensec;
        time_t        st_mtime;
        long          st_mtimensec;
        time_t        st_ctime;
        long          st_ctimensec;
    #endif
        off_t         st_size;
        int64_t       st_blocks;
        u_int32_t     st_blksize;
        u_int32_t     st_flags;
        u_int32_t     st_gen;
        int32_t       st_lspare;
        int64_t       st_qspare[2];
    };
    '''
    _fields_ = [('st_dev',        c_ulong),
                ('st_ino',        c_ulong),
                ('st_mode',       c_ushort),
                ('st_nlink',      c_uint),
                ('st_uid',        c_uint),
                ('st_gid',        c_uint),
                ('st_rdev',       c_ulong),
                ('st_atime',      c_longlong),
                ('st_atimendesc', c_long),
                ('st_mtime',      c_longlong),
                ('st_mtimendesc', c_long),
                ('st_ctime',      c_longlong),
                ('st_ctimendesc', c_long),
                ('st_size',       c_ulonglong),
                ('st_blocks',     c_int64),
                ('st_blksize',    c_uint32),
                ('st_flags',      c_uint32),
                ('st_gen',        c_uint32),
                ('st_lspare',     c_int32),
                ('st_qspare',     POINTER(c_int64) * 2)]

# stat success
path = create_string_buffer(b"/etc/passwd")
st = Stat()
ret = stat(path, byref(st))
assert ret == 0

# if stat fail, check errno
path = create_string_buffer(b"&%$#@!")
st = Stat()
ret = stat(path, byref(st))
if ret != 0:
    errno_ = get_errno() # get errno
    errmsg = "stat({}) failed. {}".format(path.raw, os.strerror(errno_))
    raise OSError(errno_, errmsg)

output:

$ python err_handling.py   # python2
Traceback (most recent call last):
  File "err_handling.py", line 85, in <module>
    raise OSError(errno_, errmsg)
OSError: [Errno 2] stat(&%$#@!) failed. No such file or directory

$ python3 err_handling.py  # python3
Traceback (most recent call last):
  File "err_handling.py", line 85, in <module>
    raise OSError(errno_, errmsg)
FileNotFoundError: [Errno 2] stat(b'&%$#@!\x00') failed. No such file or directory

Getting File System Type

from __future__ import print_function

from ctypes import *
from sys import platform

if platform not in ('linux', 'linux2'):
    raise RuntimeError("Not support '{}'".format(platform))


# from Linux/include/uapi/linux/magic.h

EXT_SUPER_MAGIC      = 0x137D
EXT2_OLD_SUPER_MAGIC = 0xEF51
EXT2_SUPER_MAGIC     = 0xEF53
EXT3_SUPER_MAGIC     = 0xEF53
EXT4_SUPER_MAGIC     = 0xEF53
BTRFS_SUPER_MAGIC    = 0x9123683E


class KernelFsid(Structure):
    '''
    From Linux/arch/mips/include/asm/posix_types.h

    typedef struct {
            long    val[2];
    } __kernel_fsid_t;
    '''
    _fields_ = [('val', POINTER(c_long) * 2)]

class Statfs(Structure):
    '''
    From Linux/arch/mips/include/asm/statfs.h

    struct statfs {
            long            f_type;
    #define f_fstyp f_type
            long            f_bsize;
            long            f_frsize;
            long            f_blocks;
            long            f_bfree;
            long            f_files;
            long            f_ffree;
            long            f_bavail;

            /* Linux specials */
            __kernel_fsid_t f_fsid;
            long            f_namelen;
            long            f_flags;
            long            f_spare[5];
    };
    '''
    _fields_ = [('f_type',    c_long),
                ('f_bsize',   c_long),
                ('f_frsize',  c_long),
                ('f_block',   c_long),
                ('f_bfree',   c_long),
                ('f_files',   c_long),
                ('f_ffree',   c_long),
                ('f_fsid',    KernelFsid),
                ('f_namelen', c_long),
                ('f_flags',   c_long),
                ('f_spare',   POINTER(c_long) * 5)]


libc = CDLL('libc.so.6', use_errno=True)
statfs = libc.statfs

path = create_string_buffer(b'/etc')
fst = Statfs()
ret = statfs(path, byref(fst))
assert ret == 0

print('Is ext4? {}'.format(fst.f_type == EXT4_SUPER_MAGIC))

output:

$ python3 statfs.py
Is ext4? True

Doing Zero-copy via sendfile

from __future__ import print_function, unicode_literals

import os
import sys
import errno
import platform

from ctypes import *

# check os
p = platform.system()
if p != "Linux":
    raise OSError("Not support '{}'".format(p))

# check linux version
ver = platform.release()
if tuple(map(int, ver.split('.'))) < (2,6,33):
    raise OSError("Upgrade kernel after 2.6.33")

# check input arguments
if len(sys.argv) != 3:
    print("Usage: sendfile.py f1 f2", file=sys.stderr)
    exit(1)

libc = CDLL('libc.so.6', use_errno=True)
sendfile = libc.sendfile

src = sys.argv[1]
dst = sys.argv[2]
src_size = os.stat(src).st_size

# clean destination first
try:
    os.remove(dst)
except OSError as e:
    if e.errno != errno.ENOENT: raise

offset = c_int64(0)

with open(src, 'r') as f1:
    with open(dst, 'w') as f2:
        src_fd = c_int(f1.fileno())
        dst_fd = c_int(f2.fileno())
        ret = sendfile(dst_fd, src_fd, byref(offset), src_size)
        if ret < 0:
            errno_ = get_errno()
            errmsg = "sendfile failed. {}".format(os.strerror(errno_))
            raise OSError(errno_, errmsg)

output:

$ python3 sendfile.py /etc/resolv.conf resolve.conf; cat resolve.conf
nameserver  192.168.1.1

PyObject header

// ref: python source code
// Python/Include/object.c
#define _PyObject_HEAD_EXTRA \
    struct _object *_ob_next;\
    struct _object *_ob_prev;

#define PyObject_HEAD    \
    _PyObject_HEAD_EXTRA \
    Py_ssize_t ob_refcnt;\
    struct _typeobject *ob_type;

Python C API Template

C API source

#include <Python.h>

typedef struct {
    PyObject_HEAD
} spamObj;

static PyTypeObject spamType = {
    PyObject_HEAD_INIT(&PyType_Type)
    0,                  //ob_size
    "spam.Spam",        //tp_name
    sizeof(spamObj),    //tp_basicsize
    0,                  //tp_itemsize
    0,                  //tp_dealloc
    0,                  //tp_print
    0,                  //tp_getattr
    0,                  //tp_setattr
    0,                  //tp_compare
    0,                  //tp_repr
    0,                  //tp_as_number
    0,                  //tp_as_sequence
    0,                  //tp_as_mapping
    0,                  //tp_hash
    0,                  //tp_call
    0,                  //tp_str
    0,                  //tp_getattro
    0,                  //tp_setattro
    0,                  //tp_as_buffer
    Py_TPFLAGS_DEFAULT, //tp_flags
    "spam objects",     //tp_doc
};

static PyMethodDef spam_methods[] = {
    {NULL}  /* Sentinel */
};

/* declarations for DLL import */
#ifndef PyMODINIT_FUNC
#define PyMODINIT_FUNC void
#endif

PyMODINIT_FUNC
initspam(void)
{
    PyObject *m;
    spamType.tp_new = PyType_GenericNew;
    if (PyType_Ready(&spamType) < 0) {
        goto END;
    }
    m = Py_InitModule3("spam", spam_methods, "Example of Module");
    Py_INCREF(&spamType);
    PyModule_AddObject(m, "spam", (PyObject *)&spamType);
END:
    return;
}

Prepare setup.py

from distutils.core import setup
from distutils.core import Extension

setup(name="spam",
      version="1.0",
      ext_modules=[Extension("spam", ["spam.c"])])

Build C API source

$ python setup.py build
$ python setup.py install

Run the C module

>>> import spam
>>> spam.__doc__
'Example of Module'
>>> spam.spam
<type 'spam.Spam'>

PyObject with Member and Methods

C API source

#include <Python.h>
#include <structmember.h>

typedef struct {
    PyObject_HEAD
    PyObject *hello;
    PyObject *world;
    int spam_id;
} spamObj;

static void
spamdealloc(spamObj *self)
{
    Py_XDECREF(self->hello);
    Py_XDECREF(self->world);
    self->ob_type
        ->tp_free((PyObject*)self);
}

/* __new__ */
static PyObject *
spamNew(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
    spamObj *self = NULL;

    self = (spamObj *)
           type->tp_alloc(type, 0);
    if (self == NULL) {
        goto END;
    }
    /* alloc str to hello */
    self->hello =
        PyString_FromString("");
    if (self->hello == NULL)
    {
        Py_XDECREF(self);
        self = NULL;
        goto END;
    }
    /* alloc str to world */
    self->world =
        PyString_FromString("");
    if (self->world == NULL)
    {
        Py_XDECREF(self);
        self = NULL;
        goto END;
    }
    self->spam_id = 0;
END:
    return (PyObject *)self;
}

/* __init__ */
static int
spamInit(spamObj *self, PyObject *args, PyObject *kwds)
{
    int ret = -1;
    PyObject *hello=NULL,
             *world=NULL,
             *tmp=NULL;

    static char *kwlist[] = {
        "hello",
        "world",
        "spam_id", NULL};

    /* parse input arguments */
    if (! PyArg_ParseTupleAndKeywords(
          args, kwds,
          "|OOi",
          kwlist,
          &hello, &world,
          &self->spam_id)) {
        goto END;
    }
    /* set attr hello */
    if (hello) {
        tmp = self->hello;
        Py_INCREF(hello);
        self->hello = hello;
        Py_XDECREF(tmp);
    }
    /* set attr world */
    if (world) {
        tmp = self->world;
        Py_INCREF(world);
        self->world = world;
        Py_XDECREF(tmp);
    }
    ret = 0;
END:
    return ret;
}

static long
fib(long n) {
    if (n<=2) {
        return 1;
    }
    return fib(n-1)+fib(n-2);
}

static PyObject *
spamFib(spamObj *self, PyObject *args)
{
    PyObject  *ret = NULL;
    long arg = 0;

    if (!PyArg_ParseTuple(args, "i", &arg)) {
        goto END;
    }
    ret = PyInt_FromLong(fib(arg));
END:
    return ret;
}

//ref: python doc
static PyMemberDef spam_members[] = {
    /* spameObj.hello*/
    {"hello",                   //name
     T_OBJECT_EX,               //type
     offsetof(spamObj, hello),  //offset
     0,                         //flags
     "spam hello"},             //doc
    /* spamObj.world*/
    {"world",
     T_OBJECT_EX,
     offsetof(spamObj, world),
     0,
     "spam world"},
    /* spamObj.spam_id*/
    {"spam_id",
     T_INT,
     offsetof(spamObj, spam_id),
     0,
     "spam id"},
    /* Sentiel */
    {NULL}
};

static PyMethodDef spam_methods[] = {
    /* fib */
    {"spam_fib",
     (PyCFunction)spamFib,
     METH_VARARGS,
     "Calculate fib number"},
    /* Sentiel */
    {NULL}
};

static PyMethodDef module_methods[] = {
    {NULL}  /* Sentinel */
};

static PyTypeObject spamKlass = {
    PyObject_HEAD_INIT(NULL)
    0,                               //ob_size
    "spam.spamKlass",                //tp_name
    sizeof(spamObj),                 //tp_basicsize
    0,                               //tp_itemsize
    (destructor) spamdealloc,        //tp_dealloc
    0,                               //tp_print
    0,                               //tp_getattr
    0,                               //tp_setattr
    0,                               //tp_compare
    0,                               //tp_repr
    0,                               //tp_as_number
    0,                               //tp_as_sequence
    0,                               //tp_as_mapping
    0,                               //tp_hash
    0,                               //tp_call
    0,                               //tp_str
    0,                               //tp_getattro
    0,                               //tp_setattro
    0,                               //tp_as_buffer
    Py_TPFLAGS_DEFAULT |
    Py_TPFLAGS_BASETYPE,             //tp_flags
    "spamKlass objects",             //tp_doc
    0,                               //tp_traverse
    0,                               //tp_clear
    0,                               //tp_richcompare
    0,                               //tp_weaklistoffset
    0,                               //tp_iter
    0,                               //tp_iternext
    spam_methods,                    //tp_methods
    spam_members,                    //tp_members
    0,                               //tp_getset
    0,                               //tp_base
    0,                               //tp_dict
    0,                               //tp_descr_get
    0,                               //tp_descr_set
    0,                               //tp_dictoffset
    (initproc)spamInit,              //tp_init
    0,                               //tp_alloc
    spamNew,                         //tp_new
};

/* declarations for DLL import */
#ifndef PyMODINIT_FUNC
#define PyMODINIT_FUNC void
#endif

PyMODINIT_FUNC
initspam(void)
{
    PyObject* m;

    if (PyType_Ready(&spamKlass) < 0) {
        goto END;
    }

    m = Py_InitModule3(
      "spam",         // Mod name
      module_methods, // Mod methods
      "Spam Module"); // Mod doc

    if (m == NULL) {
        goto END;
    }
    Py_INCREF(&spamKlass);
    PyModule_AddObject(
      m,                           // Module
      "SpamKlass",                 // Class Name
      (PyObject *) &spamKlass);    // Class
END:
    return;
}

Compare performance with pure Python

>>> import spam
>>> o = spam.SpamKlass()
>>> def profile(func):
...     def wrapper(*args, **kwargs):
...         s = time.time()
...         ret = func(*args, **kwargs)
...         e = time.time()
...         print(e-s)
...     return wrapper
...
>>> def fib(n):
...     if n <= 2:
...         return n
...     return fib(n-1)+fib(n-2)
...
>>> @profile
... def cfib(n):
...     o.spam_fib(n)
...
>>> @profile
... def pyfib(n):
...     fib(n)
...
>>> cfib(30)
0.0106310844421
>>> pyfib(30)
0.399799108505