Merge branch 'collectd-4.10' into collectd-5.0
[collectd.git] / src / python.c
index 25681b6..0fb49d4 100644 (file)
@@ -217,7 +217,7 @@ static int do_interactive = 0;
 
 static PyThreadState *state;
 
-static PyObject *cpy_format_exception;
+static PyObject *sys_path, *cpy_format_exception;
 
 static cpy_callback_t *cpy_config_callbacks;
 static cpy_callback_t *cpy_init_callbacks;
@@ -259,7 +259,7 @@ static void cpy_build_name(char *buf, size_t size, PyObject *callback, const cha
        PyErr_Clear();
 }
 
-static void cpy_log_exception(const char *context) {
+void cpy_log_exception(const char *context) {
        int l = 0, i;
        const char *typename = NULL, *message = NULL;
        PyObject *type, *value, *traceback, *tn, *m, *list;
@@ -282,17 +282,13 @@ static void cpy_log_exception(const char *context) {
        Py_END_ALLOW_THREADS
        Py_XDECREF(tn);
        Py_XDECREF(m);
-       if (!cpy_format_exception) {
+       if (!cpy_format_exception || !traceback) {
                PyErr_Clear();
-               Py_XDECREF(type);
+               Py_DECREF(type);
                Py_XDECREF(value);
                Py_XDECREF(traceback);
                return;
        }
-       if (!traceback) {
-               PyErr_Clear();
-               return;
-       }
        list = PyObject_CallFunction(cpy_format_exception, "NNN", type, value, traceback); /* New reference. */
        if (list)
                l = PyObject_Length(list);
@@ -313,6 +309,9 @@ static void cpy_log_exception(const char *context) {
        }
        Py_XDECREF(list);
        PyErr_Clear();
+       Py_DECREF(type);
+       Py_XDECREF(value);
+       Py_XDECREF(traceback);
 }
 
 static int cpy_read_callback(user_data_t *data) {
@@ -335,7 +334,7 @@ static int cpy_read_callback(user_data_t *data) {
 static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_list, user_data_t *data) {
        int i;
        cpy_callback_t *c = data->data;
-       PyObject *ret, *list;
+       PyObject *ret, *list, *temp, *dict = NULL;
        Values *v;
 
        CPY_LOCK_THREADS
@@ -345,45 +344,103 @@ static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_li
                        CPY_RETURN_FROM_THREADS 0;
                }
                for (i = 0; i < value_list->values_len; ++i) {
-                       if (ds->ds->type == DS_TYPE_COUNTER) {
+                       if (ds->ds[i].type == DS_TYPE_COUNTER) {
                                if ((long) value_list->values[i].counter == value_list->values[i].counter)
                                        PyList_SetItem(list, i, PyInt_FromLong(value_list->values[i].counter));
                                else
                                        PyList_SetItem(list, i, PyLong_FromUnsignedLongLong(value_list->values[i].counter));
-                       } else if (ds->ds->type == DS_TYPE_GAUGE) {
+                       } else if (ds->ds[i].type == DS_TYPE_GAUGE) {
                                PyList_SetItem(list, i, PyFloat_FromDouble(value_list->values[i].gauge));
-                       } else if (ds->ds->type == DS_TYPE_DERIVE) {
+                       } else if (ds->ds[i].type == DS_TYPE_DERIVE) {
                                if ((long) value_list->values[i].derive == value_list->values[i].derive)
                                        PyList_SetItem(list, i, PyInt_FromLong(value_list->values[i].derive));
                                else
                                        PyList_SetItem(list, i, PyLong_FromLongLong(value_list->values[i].derive));
-                       } else if (ds->ds->type == DS_TYPE_ABSOLUTE) {
+                       } else if (ds->ds[i].type == DS_TYPE_ABSOLUTE) {
                                if ((long) value_list->values[i].absolute == value_list->values[i].absolute)
                                        PyList_SetItem(list, i, PyInt_FromLong(value_list->values[i].absolute));
                                else
                                        PyList_SetItem(list, i, PyLong_FromUnsignedLongLong(value_list->values[i].absolute));
                        } else {
                                Py_BEGIN_ALLOW_THREADS
-                               ERROR("cpy_write_callback: Unknown value type %d.", ds->ds->type);
+                               ERROR("cpy_write_callback: Unknown value type %d.", ds->ds[i].type);
                                Py_END_ALLOW_THREADS
                                Py_DECREF(list);
                                CPY_RETURN_FROM_THREADS 0;
                        }
                        if (PyErr_Occurred() != NULL) {
                                cpy_log_exception("value building for write callback");
+                               Py_DECREF(list);
                                CPY_RETURN_FROM_THREADS 0;
                        }
                }
-               v = PyObject_New(Values, (void *) &ValuesType);
+               dict = PyDict_New();  /* New reference. */
+               if (value_list->meta) {
+                       int i, num;
+                       char **table;
+                       meta_data_t *meta = value_list->meta;
+
+                       num = meta_data_toc(meta, &table);
+                       for (i = 0; i < num; ++i) {
+                               int type;
+                               char *string;
+                               int64_t si;
+                               uint64_t ui;
+                               double d;
+                               _Bool b;
+                               
+                               type = meta_data_type(meta, table[i]);
+                               if (type == MD_TYPE_STRING) {
+                                       if (meta_data_get_string(meta, table[i], &string))
+                                               continue;
+                                       temp = cpy_string_to_unicode_or_bytes(string);  /* New reference. */
+                                       free(string);
+                                       PyDict_SetItemString(dict, table[i], temp);
+                                       Py_XDECREF(temp);
+                               } else if (type == MD_TYPE_SIGNED_INT) {
+                                       if (meta_data_get_signed_int(meta, table[i], &si))
+                                               continue;
+                                       temp = PyObject_CallFunctionObjArgs((void *) &SignedType, PyLong_FromLongLong(si), (void *) 0);  /* New reference. */
+                                       PyDict_SetItemString(dict, table[i], temp);
+                                       Py_XDECREF(temp);
+                               } else if (type == MD_TYPE_UNSIGNED_INT) {
+                                       if (meta_data_get_unsigned_int(meta, table[i], &ui))
+                                               continue;
+                                       temp = PyObject_CallFunctionObjArgs((void *) &UnsignedType, PyLong_FromUnsignedLongLong(ui), (void *) 0);  /* New reference. */
+                                       PyDict_SetItemString(dict, table[i], temp);
+                                       Py_XDECREF(temp);
+                               } else if (type == MD_TYPE_DOUBLE) {
+                                       if (meta_data_get_double(meta, table[i], &d))
+                                               continue;
+                                       temp = PyFloat_FromDouble(d);  /* New reference. */
+                                       PyDict_SetItemString(dict, table[i], temp);
+                                       Py_XDECREF(temp);
+                               } else if (type == MD_TYPE_BOOLEAN) {
+                                       if (meta_data_get_boolean(meta, table[i], &b))
+                                               continue;
+                                       if (b)
+                                               PyDict_SetItemString(dict, table[i], Py_True);
+                                       else
+                                               PyDict_SetItemString(dict, table[i], Py_False);
+                               }
+                               free(table[i]);
+                       }
+                       free(table);
+               }
+               v = (Values *) Values_New(); /* New reference. */
                sstrncpy(v->data.host, value_list->host, sizeof(v->data.host));
                sstrncpy(v->data.type, value_list->type, sizeof(v->data.type));
                sstrncpy(v->data.type_instance, value_list->type_instance, sizeof(v->data.type_instance));
                sstrncpy(v->data.plugin, value_list->plugin, sizeof(v->data.plugin));
                sstrncpy(v->data.plugin_instance, value_list->plugin_instance, sizeof(v->data.plugin_instance));
-               v->data.time = value_list->time;
-               v->interval = value_list->interval;
+               v->data.time = CDTIME_T_TO_DOUBLE(value_list->time);
+               v->interval = CDTIME_T_TO_DOUBLE(value_list->interval);
+               Py_CLEAR(v->values);
                v->values = list;
+               Py_CLEAR(v->meta);
+               v->meta = dict;  /* Steals a reference. */
                ret = PyObject_CallFunctionObjArgs(c->callback, v, c->data, (void *) 0); /* New reference. */
+               Py_XDECREF(v);
                if (ret == NULL) {
                        cpy_log_exception("write callback");
                } else {
@@ -395,20 +452,22 @@ static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_li
 
 static int cpy_notification_callback(const notification_t *notification, user_data_t *data) {
        cpy_callback_t *c = data->data;
-       PyObject *ret;
+       PyObject *ret, *notify;
        Notification *n;
 
        CPY_LOCK_THREADS
-               n = PyObject_New(Notification, (void *) &NotificationType);
+               notify = Notification_New(); /* New reference. */
+               n = (Notification *) notify;
                sstrncpy(n->data.host, notification->host, sizeof(n->data.host));
                sstrncpy(n->data.type, notification->type, sizeof(n->data.type));
                sstrncpy(n->data.type_instance, notification->type_instance, sizeof(n->data.type_instance));
                sstrncpy(n->data.plugin, notification->plugin, sizeof(n->data.plugin));
                sstrncpy(n->data.plugin_instance, notification->plugin_instance, sizeof(n->data.plugin_instance));
-               n->data.time = notification->time;
+               n->data.time = CDTIME_T_TO_DOUBLE(notification->time);
                sstrncpy(n->message, notification->message, sizeof(n->message));
                n->severity = notification->severity;
                ret = PyObject_CallFunctionObjArgs(c->callback, n, c->data, (void *) 0); /* New reference. */
+               Py_XDECREF(notify);
                if (ret == NULL) {
                        cpy_log_exception("notification callback");
                } else {
@@ -423,11 +482,11 @@ static void cpy_log_callback(int severity, const char *message, user_data_t *dat
        PyObject *ret, *text;
 
        CPY_LOCK_THREADS
-       text = cpy_string_to_unicode_or_bytes(message);
+       text = cpy_string_to_unicode_or_bytes(message);  /* New reference. */
        if (c->data == NULL)
-               ret = PyObject_CallFunction(c->callback, "iN", severity, text); /* New reference. */
+               ret = PyObject_CallFunction(c->callback, "iN", severity, text); /* New reference. Steals a reference from "text". */
        else
-               ret = PyObject_CallFunction(c->callback, "iNO", severity, text, c->data); /* New reference. */
+               ret = PyObject_CallFunction(c->callback, "iNO", severity, text, c->data); /* New reference. Steals a reference from "text". */
 
        if (ret == NULL) {
                /* FIXME */
@@ -464,12 +523,13 @@ static void cpy_flush_callback(int timeout, const char *id, user_data_t *data) {
 static PyObject *cpy_register_generic(cpy_callback_t **list_head, PyObject *args, PyObject *kwds) {
        char buf[512];
        cpy_callback_t *c;
-       const char *name = NULL;
+       char *name = NULL;
        PyObject *callback = NULL, *data = NULL, *mod = NULL;
        static char *kwlist[] = {"callback", "data", "name", NULL};
        
        if (PyArg_ParseTupleAndKeywords(args, kwds, "O|Oet", kwlist, &callback, &data, NULL, &name) == 0) return NULL;
        if (PyCallable_Check(callback) == 0) {
+               PyMem_Free(name);
                PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
                return NULL;
        }
@@ -484,18 +544,21 @@ static PyObject *cpy_register_generic(cpy_callback_t **list_head, PyObject *args
        c->next = *list_head;
        *list_head = c;
        Py_XDECREF(mod);
+       PyMem_Free(name);
        return cpy_string_to_unicode_or_bytes(buf);
 }
 
 static PyObject *cpy_flush(cpy_callback_t **list_head, PyObject *args, PyObject *kwds) {
        int timeout = -1;
-       const char *plugin = NULL, *identifier = NULL;
+       char *plugin = NULL, *identifier = NULL;
        static char *kwlist[] = {"plugin", "timeout", "identifier", NULL};
        
        if (PyArg_ParseTupleAndKeywords(args, kwds, "|etiet", kwlist, NULL, &plugin, &timeout, NULL, &identifier) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_flush(plugin, timeout, identifier);
        Py_END_ALLOW_THREADS
+       PyMem_Free(plugin);
+       PyMem_Free(identifier);
        Py_RETURN_NONE;
 }
 
@@ -514,16 +577,18 @@ static PyObject *cpy_register_generic_userdata(void *reg, void *handler, PyObjec
        reg_function_t *register_function = (reg_function_t *) reg;
        cpy_callback_t *c = NULL;
        user_data_t *user_data = NULL;
-       const char *name = NULL;
+       char *name = NULL;
        PyObject *callback = NULL, *data = NULL;
        static char *kwlist[] = {"callback", "data", "name", NULL};
        
        if (PyArg_ParseTupleAndKeywords(args, kwds, "O|Oet", kwlist, &callback, &data, NULL, &name) == 0) return NULL;
        if (PyCallable_Check(callback) == 0) {
+               PyMem_Free(name);
                PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
                return NULL;
        }
        cpy_build_name(buf, sizeof(buf), callback, name);
+       PyMem_Free(name);
        
        Py_INCREF(callback);
        Py_XINCREF(data);
@@ -544,17 +609,19 @@ static PyObject *cpy_register_read(PyObject *self, PyObject *args, PyObject *kwd
        cpy_callback_t *c = NULL;
        user_data_t *user_data = NULL;
        double interval = 0;
-       const char *name = NULL;
+       char *name = NULL;
        PyObject *callback = NULL, *data = NULL;
        struct timespec ts;
        static char *kwlist[] = {"callback", "interval", "data", "name", NULL};
        
        if (PyArg_ParseTupleAndKeywords(args, kwds, "O|dOet", kwlist, &callback, &interval, &data, NULL, &name) == 0) return NULL;
        if (PyCallable_Check(callback) == 0) {
+               PyMem_Free(name);
                PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
                return NULL;
        }
        cpy_build_name(buf, sizeof(buf), callback, name);
+       PyMem_Free(name);
        
        Py_INCREF(callback);
        Py_XINCREF(data);
@@ -568,7 +635,8 @@ static PyObject *cpy_register_read(PyObject *self, PyObject *args, PyObject *kwd
        user_data->data = c;
        ts.tv_sec = interval;
        ts.tv_nsec = (interval - ts.tv_sec) * 1000000000;
-       plugin_register_complex_read(buf, cpy_read_callback, &ts, user_data);
+       plugin_register_complex_read(/* group = */ NULL, buf,
+                       cpy_read_callback, &ts, user_data);
        return cpy_string_to_unicode_or_bytes(buf);
 }
 
@@ -597,48 +665,53 @@ static PyObject *cpy_register_shutdown(PyObject *self, PyObject *args, PyObject
 }
 
 static PyObject *cpy_error(PyObject *self, PyObject *args) {
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_ERR, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
        Py_RETURN_NONE;
 }
 
 static PyObject *cpy_warning(PyObject *self, PyObject *args) {
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_WARNING, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
        Py_RETURN_NONE;
 }
 
 static PyObject *cpy_notice(PyObject *self, PyObject *args) {
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_NOTICE, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
        Py_RETURN_NONE;
 }
 
 static PyObject *cpy_info(PyObject *self, PyObject *args) {
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_INFO, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
        Py_RETURN_NONE;
 }
 
 static PyObject *cpy_debug(PyObject *self, PyObject *args) {
 #ifdef COLLECT_DEBUG
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_DEBUG, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
 #endif
        Py_RETURN_NONE;
 }
@@ -691,18 +764,18 @@ static PyObject *cpy_unregister_generic_userdata(cpy_unregister_function_t *unre
                PyErr_Clear();
                if (!PyCallable_Check(arg)) {
                        PyErr_SetString(PyExc_TypeError, "This function needs a string or a callable object as its only parameter.");
-                       Py_DECREF(&arg);
+                       Py_DECREF(arg);
                        return NULL;
                }
                cpy_build_name(buf, sizeof(buf), arg, NULL);
                name = buf;
        }
        if (unreg(name) == 0) {
-               Py_DECREF(&arg);
+               Py_DECREF(arg);
                Py_RETURN_NONE;
        }
        PyErr_Format(PyExc_RuntimeError, "Unable to unregister %s callback '%s'.", desc, name);
-       Py_DECREF(&arg);
+       Py_DECREF(arg);
        return NULL;
 }
 
@@ -841,6 +914,11 @@ static int cpy_init(void) {
        static pthread_t thread;
        sigset_t sigset;
        
+       if (!Py_IsInitialized()) {
+               WARNING("python: Plugin loaded but not configured.");
+               plugin_unregister_shutdown("python");
+               return 0;
+       }
        PyEval_InitThreads();
        /* Now it's finally OK to use python threads. */
        for (c = cpy_init_callbacks; c; c = c->next) {
@@ -895,6 +973,7 @@ static PyObject *cpy_oconfig_to_pyconfig(oconfig_item_t *ci, PyObject *parent) {
        return item;
 }
 
+#ifdef IS_PY3K
 static struct PyModuleDef collectdmodule = {
        PyModuleDef_HEAD_INIT,
        "collectd",   /* name of module */
@@ -906,19 +985,12 @@ static struct PyModuleDef collectdmodule = {
 PyMODINIT_FUNC PyInit_collectd(void) {
        return PyModule_Create(&collectdmodule);
 }
+#endif
 
-static int cpy_config(oconfig_item_t *ci) {
-       int i;
-       PyObject *sys, *tb;
-       PyObject *sys_path;
+static int cpy_init_python() {
+       char *argv = "";
+       PyObject *sys;
        PyObject *module;
-       
-       /* Ok in theory we shouldn't do initialization at this point
-        * but we have to. In order to give python scripts a chance
-        * to register a config callback we need to be able to execute
-        * python code during the config callback so we have to start
-        * the interpreter here. */
-       /* Do *not* use the python "thread" module at this point! */
 
 #ifdef IS_PY3K
        /* Add a builtin module, before Py_Initialize */
@@ -933,6 +1005,10 @@ static int cpy_config(oconfig_item_t *ci) {
        PyType_Ready(&ValuesType);
        NotificationType.tp_base = &PluginDataType;
        PyType_Ready(&NotificationType);
+       SignedType.tp_base = &PyLong_Type;
+       PyType_Ready(&SignedType);
+       UnsignedType.tp_base = &PyLong_Type;
+       PyType_Ready(&UnsignedType);
        sys = PyImport_ImportModule("sys"); /* New reference. */
        if (sys == NULL) {
                cpy_log_exception("python initialization");
@@ -944,6 +1020,9 @@ static int cpy_config(oconfig_item_t *ci) {
                cpy_log_exception("python initialization");
                return 1;
        }
+       PySys_SetArgv(1, &argv);
+       PyList_SetSlice(sys_path, 0, 1, NULL);
+
 #ifdef IS_PY3K
        module = PyImport_ImportModule("collectd");
 #else
@@ -952,6 +1031,8 @@ static int cpy_config(oconfig_item_t *ci) {
        PyModule_AddObject(module, "Config", (void *) &ConfigType); /* Steals a reference. */
        PyModule_AddObject(module, "Values", (void *) &ValuesType); /* Steals a reference. */
        PyModule_AddObject(module, "Notification", (void *) &NotificationType); /* Steals a reference. */
+       PyModule_AddObject(module, "Signed", (void *) &SignedType); /* Steals a reference. */
+       PyModule_AddObject(module, "Unsigned", (void *) &UnsignedType); /* Steals a reference. */
        PyModule_AddIntConstant(module, "LOG_DEBUG", LOG_DEBUG);
        PyModule_AddIntConstant(module, "LOG_INFO", LOG_INFO);
        PyModule_AddIntConstant(module, "LOG_NOTICE", LOG_NOTICE);
@@ -960,6 +1041,22 @@ static int cpy_config(oconfig_item_t *ci) {
        PyModule_AddIntConstant(module, "NOTIF_FAILURE", NOTIF_FAILURE);
        PyModule_AddIntConstant(module, "NOTIF_WARNING", NOTIF_WARNING);
        PyModule_AddIntConstant(module, "NOTIF_OKAY", NOTIF_OKAY);
+       return 0;
+}
+
+static int cpy_config(oconfig_item_t *ci) {
+       int i;
+       PyObject *tb;
+
+       /* Ok in theory we shouldn't do initialization at this point
+        * but we have to. In order to give python scripts a chance
+        * to register a config callback we need to be able to execute
+        * python code during the config callback so we have to start
+        * the interpreter here. */
+       /* Do *not* use the python "thread" module at this point! */
+
+       if (!Py_IsInitialized() && cpy_init_python()) return 1;
+
        for (i = 0; i < ci->children_num; ++i) {
                oconfig_item_t *item = ci->children + i;
                
@@ -1059,7 +1156,6 @@ static int cpy_config(oconfig_item_t *ci) {
                        WARNING("python plugin: Ignoring unknown config key \"%s\".", item->key);
                }
        }
-       Py_DECREF(sys_path);
        return 0;
 }