Kivy A to Z -- 通过绑定进程运行CPU提高Python程序在多核CPU平台上的性能
1. 从Python GIL系列文章中我们已经对Python的GIL有了一个比较清醒的认识
2. 要提高Python程序在多核CPU情况下的性能，除了使用进程替代线程外，一个更为实用的方法就是绑定Python进程运行于指定CPU。
3. 接下来看下如何在Kivy中做到这一点
4. 修改src/jni/application/python/start.c
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#ifndef Py_PYTHON_H
#error Python headers needed to compile C extensions, please install development version of Python.
#else
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <jni.h>
#include "SDL.h"
#include "android/log.h"
#include "jniwrapperstuff.h"
#define LOG(x) __android_log_write(ANDROID_LOG_INFO, "python", (x))
#define _GNU_SOURCE
#include <sched.h>
#include <linux/unistd.h>
typedef unsigned int cpu_set_t;
static PyObject *androidembed_log(PyObject *self, PyObject *args) {
char *logstr = NULL;
if (!PyArg_ParseTuple(args, "s", &logstr)) {
return NULL;
}
LOG(logstr);
Py_RETURN_NONE;
}
static int
sched_setaffinity(pid_t pid, size_t len, cpu_set_t const * cpusetp)
{
return syscall(__NR_sched_setaffinity, pid, len, cpusetp);
}
static int
sched_getaffinity(pid_t pid, size_t len, cpu_set_t const * cpusetp)
{
return syscall(__NR_sched_getaffinity, pid, len, cpusetp);
}
static PyObject *
get_process_affinity_mask(PyObject *self, PyObject *args)
{
unsigned long cur_mask;
unsigned int len = sizeof(cur_mask);
pid_t pid;
if (!PyArg_ParseTuple(args, "i:get_process_affinity_mask", &pid))
return NULL;
if (sched_getaffinity(pid, len,
(cpu_set_t *)&cur_mask) < 0) {
PyErr_SetFromErrno(PyExc_ValueError);
return NULL;
}
return Py_BuildValue("l", cur_mask);
}
static PyObject *
set_process_affinity_mask(PyObject *self, PyObject *args)
{
unsigned long new_mask;
unsigned long cur_mask;
unsigned int len = sizeof(new_mask);
pid_t pid;
if (!PyArg_ParseTuple(args, "il:set_process_affinity_mask", &pid, &new_mask))
return NULL;
if (sched_getaffinity(pid, len,
(cpu_set_t *)&cur_mask) < 0) {
PyErr_SetFromErrno(PyExc_ValueError);
return NULL;
}
if (sched_setaffinity(pid, len, (cpu_set_t *)&new_mask)) {
PyErr_SetFromErrno(PyExc_ValueError);
return NULL;
}
return Py_BuildValue("l", cur_mask);
}
static PyMethodDef AndroidEmbedMethods[] = {
{"log", androidembed_log, METH_VARARGS,"Log on android platform"},
{"get_process_affinity_mask", get_process_affinity_mask, METH_VARARGS,"get_process_affinity_mask"},
{"set_process_affinity_mask", set_process_affinity_mask, METH_VARARGS,"set_process_affinity_mask"},
{NULL, NULL, 0, NULL}
};
PyMODINIT_FUNC initandroidembed(void) {
(void) Py_InitModule("androidembed", AndroidEmbedMethods);
}
int file_exists(const char * filename)
{
FILE *file;
if (file = fopen(filename, "r")) {
fclose(file);
return 1;
}
return 0;
}
int main(int argc, char **argv) {
char *env_argument = NULL;
int ret = 0;
FILE *fd;
LOG("Initialize Python for Android");
env_argument = getenv("ANDROID_ARGUMENT");
setenv("ANDROID_APP_PATH", env_argument, 1);
//setenv("PYTHONVERBOSE", "2", 1);
Py_SetProgramName(argv[0]);
Py_Initialize();
PySys_SetArgv(argc, argv);
/* ensure threads will work.
*/
PyEval_InitThreads();
/* our logging module for android
*/
initandroidembed();
/* inject our bootstrap code to redirect python stdin/stdout
* replace sys.path with our path
*/
PyRun_SimpleString(
"import sys, posix\n" \
"private = posix.environ['ANDROID_PRIVATE']\n" \
"argument = posix.environ['ANDROID_ARGUMENT']\n" \
"sys.path[:] = [ \n" \
" private + '/lib/python27.zip', \n" \
" private + '/lib/python2.7/', \n" \
" private + '/lib/python2.7/lib-dynload/', \n" \
" private + '/lib/python2.7/site-packages/', \n" \
" argument ]\n" \
"import androidembed\n" \
"class LogFile(object):\n" \
" def __init__(self):\n" \
" self.buffer = ''\n" \
" def write(self, s):\n" \
" s = self.buffer + s\n" \
" lines = s.split(\"\\n\")\n" \
" for l in lines[:-1]:\n" \
" androidembed.log(l)\n" \
" self.buffer = lines[-1]\n" \
" def flush(self):\n" \
" return\n" \
"sys.stdout = sys.stderr = LogFile()\n" \
"import site; print site.getsitepackages()\n"\
"print 'Android path', sys.path\n" \
"print 'Android kivy bootstrap done. __name__ is', __name__");
/* run it !
*/
LOG("Run user program, change dir and execute main.py");
chdir(env_argument);
/* search the initial main.py
*/
char *main_py = "main.pyo";
if ( file_exists(main_py) == 0 ) {
if ( file_exists("main.py") )
main_py = "main.py";
else
main_py = NULL;
}
if ( main_py == NULL ) {
LOG("No main.pyo / main.py found.");
return -1;
}
fd = fopen(main_py, "r");
if ( fd == NULL ) {
LOG("Open the main.py(o) failed");
return -1;
}
/* run python !
*/
ret = PyRun_SimpleFile(fd, main_py);
if (PyErr_Occurred() != NULL) {
ret = 1;
PyErr_Print(); /* This exits with the right code if SystemExit. */
if (Py_FlushLine())
PyErr_Clear();
}
/* close everything
*/
Py_Finalize();
fclose(fd);
LOG("Python for android ended.");
return ret;
}
JNIEXPORT void JNICALL JAVA_EXPORT_NAME(PythonService_nativeStart) ( JNIEnv* env, jobject thiz,
jstring j_android_private,
jstring j_android_argument,
jstring j_python_home,
jstring j_python_path,
jstring j_arg )
{
jboolean iscopy;
const char *android_private = (*env)->GetStringUTFChars(env, j_android_private, &iscopy);
const char *android_argument = (*env)->GetStringUTFChars(env, j_android_argument, &iscopy);
const char *python_home = (*env)->GetStringUTFChars(env, j_python_home, &iscopy);
const char *python_path = (*env)->GetStringUTFChars(env, j_python_path, &iscopy);
const char *arg = (*env)->GetStringUTFChars(env, j_arg, &iscopy);
setenv("ANDROID_PRIVATE", android_private, 1);
setenv("ANDROID_ARGUMENT", android_argument, 1);
setenv("PYTHONOPTIMIZE", "2", 1);
setenv("PYTHONHOME", python_home, 1);
setenv("PYTHONPATH", python_path, 1);
setenv("PYTHON_SERVICE_ARGUMENT", arg, 1);
char *argv[] = { "service" };
/* ANDROID_ARGUMENT points to service subdir,
* so main() will run main.py from this dir
*/
main(1, argv);
}
#endif
相比原来的代码添加了两个python接口：
get_process_affinity_mask
set_process_affinity_mask
还有由于sched_setaffinity，sched_getaffinity这两个函数在 在ndk中没有定义，所以参考网上的资料自己实现了下。
注意unistd.h不要包含出错了，ndk中有好几个unistd.h，应该是linux下的unistd.h，要不然传统编译出错。
5. 编译,具体的可参考《Kivy的编译环境的搭建以及编译和运行》一文
5.1先设置环境变量，
export ANDROIDSDK="/path/to/android/android-sdk-linux_86" export ANDROIDNDK="/path/to/android/android-ndk-r8c" export ANDROIDNDKVER=r8c export ANDROIDAPI=14 export PATH=$PATH:/mnt/develop/android_dev/kivy/apache-ant-1.9.3/bin:$ANDROIDNDK
5.2 在python-for-android目录执行：
./distribute.sh -m 'openssl pyjnius pil kivy'
6. 使用方法：在代码初始化时加上
import androidembed;androidembed.set_process_affinity_mask(0,1)
7. 打包python程序到apk
../../build/hostpython/Python-2.7.2/hostpython build.py --package org.test.touchtracer --name touchtracer --version 1.0 --dir ../../build/kivy/kivy-stable/examples/demo/touchtracer debug
8. 但是在实际的手机上测试时，Python在多线程情况下即使不绑定CPU，也不会造成性能上的的损失，看来在arm平台上和intel平台上还真不一样，但是总之有备无患吧。
9. 参考资料
https://code.google.com/p/android/issues/detail?id=19851