Integrate last part of Python code from Vasiliy Chernov

2 files changed, 142 insertions, 11 deletions

diff --git a/pcilib/CMakeLists.txt b/pcilib/CMakeLists.txt
index 27543e4..b24c7b3 100644
--- a/pcilib/CMakeLists.txt
+++ b/pcilib/CMakeLists.txt
@@ -21,6 +21,10 @@ install(FILES pcilib.h
     DESTINATION include
 )
 
-install(FILES bar.h kmem.h locking.h lock.h bank.h register.h xml.h dma.h event.h model.h error.h debug.h env.h tools.h timing.h cpu.h datacpy.h pagecpy.h memcpy.h export.h version.h view.h unit.h
+install(FILES bar.h kmem.h locking.h lock.h bank.h register.h xml.h dma.h event.h model.h error.h debug.h env.h tools.h timing.h cpu.h datacpy.h pagecpy.h memcpy.h export.h view.h unit.h
+    DESTINATION include/pcilib
+)
+
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.h
     DESTINATION include/pcilib
 )
diff --git a/pcilib/py.c b/pcilib/py.c
index 833532e..03e9d8d 100644
--- a/pcilib/py.c
+++ b/pcilib/py.c
@@ -2,6 +2,10 @@
 
 #ifdef HAVE_PYTHON
 # include <Python.h>
+
+# if PY_MAJOR_VERSION >= 3
+#  include <pthread.h>
+# endif /* PY_MAJOR_VERSION >= 3 */
 #endif /* HAVE_PYTHON */
 
 #include <stdio.h>
@@ -17,7 +21,7 @@
 #include "error.h"
 
 #ifdef HAVE_PYTHON
-#define PCILIB_PYTHON_WRAPPER "pcipywrap"
+# define PCILIB_PYTHON_WRAPPER "pcipywrap"
 
 typedef struct pcilib_script_s pcilib_script_t;
 
@@ -33,6 +37,12 @@ struct pcilib_py_s {
     PyObject *global_dict;		/**< Dictionary of main interpreter */
     PyObject *pcilib_pywrap;		/**< pcilib wrapper module */
     pcilib_script_t *script_hash;	/**< Hash with loaded scripts */
+    
+# if PY_MAJOR_VERSION >= 3
+    pthread_t pth;
+    pthread_cond_t cond;
+    pthread_mutex_t lock;
+# endif /* PY_MAJOR_VERSION > 3 */
 };
 #endif /* HAVE_PYTHON */
 
@@ -115,38 +125,114 @@ void pcilib_log_python_error(const char *file, int line, pcilib_log_flags_t flag
 #endif /* HAVE_PYTHON */
 }
 
-
+#ifdef HAVE_PYTHON
+# if PY_MAJOR_VERSION >= 3
+/**
+ * Python3 specially treats the main thread intializing Python. It crashes if
+ * the Lock is released and any Python code is executed under the GIL compaling
+ * that GIL is not locked. Python3 assumes that the main thread most of the time
+ * holds the Lock, only shortly giving it away to other threads and re-obtaining 
+ * it hereafter. This is not possible to do with GILs, but instead (probably)
+ * PyEval_{Save,Restore}Thread() should be used. On other hand, the other threads
+ * are working fine with GILs. This makes things complicated as we need to know
+ * if we are running in main thread or not.
+ * To simplify matters, during initalization we start a new thread which will
+ * performa actual initialization of Python and, hence, act as main thread.
+ * We only intialize here. No python code is executed afterwards. So we don't
+ * need to care about special locking mechanisms in main thread. Instead all
+ * our user threads can use GILs normally. 
+ * See more details here:
+ * http://stackoverflow.com/questions/24499393/cpython-locking-the-gil-in-the-main-thread
+ * http://stackoverflow.com/questions/15470367/pyeval-initthreads-in-python-3-how-when-to-call-it-the-saga-continues-ad-naus
+ */
+static void *pcilib_py_run_init_thread(void *arg) {
+    pcilib_py_t *py = (pcilib_py_t*)(arg);
+
+    Py_Initialize();
+    PyEval_InitThreads();
+    PyEval_ReleaseLock();
+
+	// Ensure that main thread waiting for our signal
+    pthread_lock(&(py->lock));
+   
+	// Inform the parent thread that initialization is finished
+    pthread_cond_signal(&(py->cond));
+
+	// Wait untill cleanup is requested
+    pthread_cond_wait(&(py->cond), &(py->lock));
+    pthread_unlock(&(py->lock)));
+    
+    Py_Finalize();
+    
+    return NULL;
+}
+# endif /* PY_MAJOR_VERSION < 3 */
+#endif /* HAVE_PYTHON */
 
 int pcilib_init_py(pcilib_t *ctx) {
 #ifdef HAVE_PYTHON
     ctx->py = (pcilib_py_t*)malloc(sizeof(pcilib_py_t));
     if (!ctx->py) return PCILIB_ERROR_MEMORY;
-
     memset(ctx->py, 0, sizeof(pcilib_py_t));
 
-    if(!Py_IsInitialized()) {
+    if (!Py_IsInitialized()) {
+# if PY_MAJOR_VERSION < 3
         Py_Initialize();
-        
     	    // Since python is being initializing from c programm, it needs to initialize threads to work properly with c threads
         PyEval_InitThreads();
         PyEval_ReleaseLock();
+# else /* PY_MAJOR_VERSION < 3 */
+	err = pthread_mutex_init(&(ctx->py.lock));
+	if (err) return PCILIB_ERROR_FAILED;
+
+	err = pthread_cond_init(&(ctx->py.cond));
+	if (err) {
+	    pthread_mutex_destroy(&(ctx->py.lock));
+	    return PCILIB_ERROR_FAILED;
+	}
+
+	err = pthread_mutex_lock(&(ctx->py.lock));
+	if (err) {
+	    pthread_cond_destroy(&(ctx->py.lock));
+	    pthread_mutex_destroy(&(ctx->py.lock));
+	    return PCILIB_ERROR_FAILED;
+	}
+
+	    // Create initalizer thread and wait until it releases the Lock
+        err = pthread_create(&(ctx->py.pth), NULL, pcilib_py_run_init_thread, &(ctx->py));
+	if (err) {
+	    pthread_mutex_unlock(&(ctx->py.lock));
+	    pthread_cond_destroy(&(ctx->py.cond));
+	    pthread_mutex_destroy(&(ctx->py.lock));
+	    return PCILIB_ERROR_FAILED;
+	}
+
+    	    // Wait until initialized and keep the lock afterwards until free executed
+	pthread_cond_wait(&(ctx->py.cond), (ctx->py.lock));
+# endif /* PY_MAJOR_VERSION < 3 */
 	ctx->py->finalyze = 1;
     }
-		
+
+	
+    PyGILState_STATE gstate = PyGILState_Ensure();
+
     ctx->py->main_module = PyImport_AddModule("__parser__");
     if (!ctx->py->main_module) {
+	PyGILState_Release(gstate);
 	pcilib_python_warning("Error importing python parser");
         return PCILIB_ERROR_FAILED;
     }
 
     ctx->py->global_dict = PyModule_GetDict(ctx->py->main_module);
     if (!ctx->py->global_dict) {
+	PyGILState_Release(gstate);
 	pcilib_python_warning("Error locating global python dictionary");
         return PCILIB_ERROR_FAILED;
     }
 
     PyObject *pywrap = PyImport_ImportModule(PCILIB_PYTHON_WRAPPER);
     if (!pywrap) {
+	PyGILState_Release(gstate);
 	pcilib_python_warning("Error importing pcilib python wrapper");
 	return PCILIB_ERROR_FAILED;
     }
@@ -158,9 +244,12 @@ int pcilib_init_py(pcilib_t *ctx) {
     Py_XDECREF(mod_name);
 	
     if (!ctx->py->pcilib_pywrap) {
+	PyGILState_Release(gstate);
 	pcilib_python_warning("Error initializing python wrapper");
         return PCILIB_ERROR_FAILED;
     }
+    
+    PyGILState_Release(gstate);
 #endif /* HAVE_PYTHON */
 
     return 0;
@@ -187,20 +276,24 @@ int pcilib_py_add_script_dir(pcilib_t *ctx, const char *dir) {
 	if (!script_dir) return PCILIB_ERROR_MEMORY;
 	sprintf(script_dir, "%s/%s", model_dir, dir);
     }
+    
+    PyGILState_STATE gstate = PyGILState_Ensure();
 
     pypath = PySys_GetObject("path");
     if (!pypath) {
+	PyGILState_Release(gstate);
 	pcilib_python_warning("Can't get python path");
 	return PCILIB_ERROR_FAILED;
     }
 
     pynewdir = PyUnicode_FromString(script_dir);
     if (!pynewdir) {
+	PyGILState_Release(gstate);
 	pcilib_python_warning("Can't create python string");
 	return PCILIB_ERROR_MEMORY;
     }
     
-	// Checking if the directory already in the path?
+    // Checking if the directory already in the path?
     pydict = PyDict_New();
     if (pydict) {
 	pystr = PyUnicode_FromString("cur");
@@ -225,6 +318,8 @@ int pcilib_py_add_script_dir(pcilib_t *ctx, const char *dir) {
     if (pyret) Py_DECREF(pyret);
     Py_DECREF(pynewdir);
 
+    PyGILState_Release(gstate);
+
     if (err) {
 	pcilib_python_warning("Can't add directory (%s) to python path", script_dir);
 	return err;
@@ -237,9 +332,13 @@ int pcilib_py_add_script_dir(pcilib_t *ctx, const char *dir) {
 void pcilib_free_py(pcilib_t *ctx) {
 #ifdef HAVE_PYTHON
     int finalyze = 0;
-	
+    
     if (ctx->py) {		
+	PyGILState_STATE gstate;
+
 	if (ctx->py->finalyze) finalyze = 1;
+	
+	gstate = PyGILState_Ensure();
 
 	if (ctx->py->script_hash) {
 	    pcilib_script_t *script, *script_tmp;
@@ -254,13 +353,31 @@ void pcilib_free_py(pcilib_t *ctx) {
 
 	if (ctx->py->pcilib_pywrap)
 	    Py_DECREF(ctx->py->pcilib_pywrap);
+
+	PyGILState_Release(gstate);
+    
 	
         free(ctx->py);
         ctx->py = NULL;
     }
     
-    if (finalyze)
-	Py_Finalize();
+    
+    if (finalyze) {
+#if PY_MAJOR_VERSION >= 3
+          // singal python init thread to stop and wait it to finish
+       pthread_cond_signal(&(ctx->py.cond));
+       pthread_mutex_unlock(&(ctx->py.lock));
+       pthread_join(ctx->py.pth, NULL);
+       
+          // destroy synchronization primitives
+       pthread_cond_destroy(&(ctx->py.cond));
+       pthread_mutex_destroy(&(ctx->py.lock));
+#else /* PY_MAJOR_VERSION < 3 */
+       Py_Finalize();
+#endif /* PY_MAJOR_VERSION < 3 */
+    }
+    
+
 #endif /* HAVE_PYTHON */
 }
 
@@ -268,6 +385,7 @@ int pcilib_py_load_script(pcilib_t *ctx, const char *script_name) {
 #ifdef HAVE_PYTHON
     PyObject* pymodule;
     pcilib_script_t *module = NULL;
+    PyGILState_STATE gstate;
 
     if (!ctx->py) return 0;
 
@@ -284,11 +402,14 @@ int pcilib_py_load_script(pcilib_t *ctx, const char *script_name) {
     HASH_FIND_STR(ctx->py->script_hash, script_name, module);
     if (module) return 0;
 
+    gstate = PyGILState_Ensure();
     pymodule = PyImport_ImportModule(module_name);
     if (!pymodule) {
+	PyGILState_Release(gstate);
 	pcilib_python_error("Error importing script (%s)", script_name);
 	return PCILIB_ERROR_FAILED;
     }
+    PyGILState_Release(gstate);
 
     module = (pcilib_script_t*)malloc(sizeof(pcilib_script_t));
     if (!module) return PCILIB_ERROR_MEMORY;
@@ -307,6 +428,7 @@ int pcilib_py_get_transform_script_properties(pcilib_t *ctx, const char *script_
     PyObject *dict;
     PyObject *pystr;
     pcilib_script_t *module;
+    PyGILState_STATE gstate;
 
     if (!ctx->py) {
 	if (mode_ret) *mode_ret = mode;
@@ -319,9 +441,12 @@ int pcilib_py_get_transform_script_properties(pcilib_t *ctx, const char *script_
 	pcilib_error("Script (%s) is not loaded yet", script_name);
 	return PCILIB_ERROR_NOTFOUND;
     }
+    
+    gstate = PyGILState_Ensure();
 	
     dict = PyModule_GetDict(module->module);
     if (!dict) {
+	PyGILState_Release(gstate);
 	pcilib_python_error("Error getting dictionary for script (%s)", script_name);
 	return PCILIB_ERROR_FAILED;
     }
@@ -337,6 +462,8 @@ int pcilib_py_get_transform_script_properties(pcilib_t *ctx, const char *script_
 	if (PyDict_Contains(dict, pystr)) mode |= PCILIB_ACCESS_W;
 	Py_DECREF(pystr);
     }
+    
+    PyGILState_Release(gstate);
 #endif /* HAVE_PYTHON */
 
     if (mode_ret) *mode_ret = mode;