Merge branch 'collectd-5.4'
[collectd.git] / src / python.c
index ee67388..8b378a2 100644 (file)
@@ -46,6 +46,19 @@ typedef struct cpy_callback_s {
 
 static char log_doc[] = "This function sends a string to all logging plugins.";
 
+static char get_ds_doc[] = "get_dataset(name) -> definition\n"
+               "\n"
+               "Returns the definition of a dataset specified by name.\n"
+               "\n"
+               "'name' is a string specifying the dataset to query.\n"
+               "'definition' is a list of 4-tuples. Every tuple represents a \n"
+               "    data source within the data set and its 4 values are the \n"
+               "    name, type, min and max value.\n"
+               "    'name' is a string.\n"
+               "    'type' is a string that is equal to either DS_TYPE_COUNTER,\n"
+               "        DS_TYPE_GAUGE, DS_TYPE_DERIVE or DS_TYPE_ABSOLUTE.\n"
+               "    'min' and 'max' are either a float or None.";
+
 static char flush_doc[] = "flush([plugin][, timeout][, identifier]) -> None\n"
                "\n"
                "Flushes the cache of another plugin.";
@@ -217,7 +230,7 @@ static int do_interactive = 0;
 
 static PyThreadState *state;
 
-static PyObject *cpy_format_exception;
+static PyObject *sys_path, *cpy_format_exception;
 
 static cpy_callback_t *cpy_config_callbacks;
 static cpy_callback_t *cpy_init_callbacks;
@@ -282,17 +295,13 @@ void cpy_log_exception(const char *context) {
        Py_END_ALLOW_THREADS
        Py_XDECREF(tn);
        Py_XDECREF(m);
-       if (!cpy_format_exception) {
+       if (!cpy_format_exception || !traceback) {
                PyErr_Clear();
-               Py_XDECREF(type);
+               Py_DECREF(type);
                Py_XDECREF(value);
                Py_XDECREF(traceback);
                return;
        }
-       if (!traceback) {
-               PyErr_Clear();
-               return;
-       }
        list = PyObject_CallFunction(cpy_format_exception, "NNN", type, value, traceback); /* New reference. */
        if (list)
                l = PyObject_Length(list);
@@ -313,6 +322,9 @@ void cpy_log_exception(const char *context) {
        }
        Py_XDECREF(list);
        PyErr_Clear();
+       Py_DECREF(type);
+       Py_XDECREF(value);
+       Py_XDECREF(traceback);
 }
 
 static int cpy_read_callback(user_data_t *data) {
@@ -335,7 +347,7 @@ static int cpy_read_callback(user_data_t *data) {
 static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_list, user_data_t *data) {
        int i;
        cpy_callback_t *c = data->data;
-       PyObject *ret, *list, *temp, *dict = NULL, *val;
+       PyObject *ret, *list, *temp, *dict = NULL;
        Values *v;
 
        CPY_LOCK_THREADS
@@ -375,7 +387,7 @@ static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_li
                                CPY_RETURN_FROM_THREADS 0;
                        }
                }
-               dict = PyDict_New();
+               dict = PyDict_New();  /* New reference. */
                if (value_list->meta) {
                        int i, num;
                        char **table;
@@ -394,26 +406,26 @@ static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_li
                                if (type == MD_TYPE_STRING) {
                                        if (meta_data_get_string(meta, table[i], &string))
                                                continue;
-                                       temp = cpy_string_to_unicode_or_bytes(string);
+                                       temp = cpy_string_to_unicode_or_bytes(string);  /* New reference. */
                                        free(string);
                                        PyDict_SetItemString(dict, table[i], temp);
                                        Py_XDECREF(temp);
                                } else if (type == MD_TYPE_SIGNED_INT) {
                                        if (meta_data_get_signed_int(meta, table[i], &si))
                                                continue;
-                                       temp = PyObject_CallFunctionObjArgs((void *) &SignedType, PyLong_FromLongLong(si), (void *) 0);
+                                       temp = PyObject_CallFunctionObjArgs((void *) &SignedType, PyLong_FromLongLong(si), (void *) 0);  /* New reference. */
                                        PyDict_SetItemString(dict, table[i], temp);
                                        Py_XDECREF(temp);
                                } else if (type == MD_TYPE_UNSIGNED_INT) {
                                        if (meta_data_get_unsigned_int(meta, table[i], &ui))
                                                continue;
-                                       temp = PyObject_CallFunctionObjArgs((void *) &UnsignedType, PyLong_FromUnsignedLongLong(ui), (void *) 0);
+                                       temp = PyObject_CallFunctionObjArgs((void *) &UnsignedType, PyLong_FromUnsignedLongLong(ui), (void *) 0);  /* New reference. */
                                        PyDict_SetItemString(dict, table[i], temp);
                                        Py_XDECREF(temp);
                                } else if (type == MD_TYPE_DOUBLE) {
                                        if (meta_data_get_double(meta, table[i], &d))
                                                continue;
-                                       temp = PyFloat_FromDouble(d);
+                                       temp = PyFloat_FromDouble(d);  /* New reference. */
                                        PyDict_SetItemString(dict, table[i], temp);
                                        Py_XDECREF(temp);
                                } else if (type == MD_TYPE_BOOLEAN) {
@@ -428,8 +440,7 @@ static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_li
                        }
                        free(table);
                }
-               val = Values_New(); /* New reference. */
-               v = (Values *) val; 
+               v = (Values *) Values_New(); /* New reference. */
                sstrncpy(v->data.host, value_list->host, sizeof(v->data.host));
                sstrncpy(v->data.type, value_list->type, sizeof(v->data.type));
                sstrncpy(v->data.type_instance, value_list->type_instance, sizeof(v->data.type_instance));
@@ -440,9 +451,9 @@ static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_li
                Py_CLEAR(v->values);
                v->values = list;
                Py_CLEAR(v->meta);
-               v->meta = dict;
+               v->meta = dict;  /* Steals a reference. */
                ret = PyObject_CallFunctionObjArgs(c->callback, v, c->data, (void *) 0); /* New reference. */
-               Py_XDECREF(val);
+               Py_XDECREF(v);
                if (ret == NULL) {
                        cpy_log_exception("write callback");
                } else {
@@ -484,11 +495,11 @@ static void cpy_log_callback(int severity, const char *message, user_data_t *dat
        PyObject *ret, *text;
 
        CPY_LOCK_THREADS
-       text = cpy_string_to_unicode_or_bytes(message);
+       text = cpy_string_to_unicode_or_bytes(message);  /* New reference. */
        if (c->data == NULL)
-               ret = PyObject_CallFunction(c->callback, "iN", severity, text); /* New reference. */
+               ret = PyObject_CallFunction(c->callback, "iN", severity, text); /* New reference. Steals a reference from "text". */
        else
-               ret = PyObject_CallFunction(c->callback, "iNO", severity, text, c->data); /* New reference. */
+               ret = PyObject_CallFunction(c->callback, "iNO", severity, text, c->data); /* New reference. Steals a reference from "text". */
 
        if (ret == NULL) {
                /* FIXME */
@@ -525,12 +536,13 @@ static void cpy_flush_callback(int timeout, const char *id, user_data_t *data) {
 static PyObject *cpy_register_generic(cpy_callback_t **list_head, PyObject *args, PyObject *kwds) {
        char buf[512];
        cpy_callback_t *c;
-       const char *name = NULL;
+       char *name = NULL;
        PyObject *callback = NULL, *data = NULL, *mod = NULL;
        static char *kwlist[] = {"callback", "data", "name", NULL};
        
        if (PyArg_ParseTupleAndKeywords(args, kwds, "O|Oet", kwlist, &callback, &data, NULL, &name) == 0) return NULL;
        if (PyCallable_Check(callback) == 0) {
+               PyMem_Free(name);
                PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
                return NULL;
        }
@@ -545,18 +557,53 @@ static PyObject *cpy_register_generic(cpy_callback_t **list_head, PyObject *args
        c->next = *list_head;
        *list_head = c;
        Py_XDECREF(mod);
+       PyMem_Free(name);
        return cpy_string_to_unicode_or_bytes(buf);
 }
 
-static PyObject *cpy_flush(cpy_callback_t **list_head, PyObject *args, PyObject *kwds) {
+static PyObject *float_or_none(float number) {
+       if (isnan(number)) {
+               Py_RETURN_NONE;
+       }
+       return PyFloat_FromDouble(number);
+}
+
+static PyObject *cpy_get_dataset(PyObject *self, PyObject *args) {
+       int i;
+       char *name;
+       const data_set_t *ds;
+       PyObject *list, *tuple;
+
+       if (PyArg_ParseTuple(args, "et", NULL, &name) == 0) return NULL;
+       ds = plugin_get_ds(name);
+       PyMem_Free(name);
+       if (ds == NULL) {
+               PyErr_Format(PyExc_TypeError, "Dataset %s not found", name);
+               return NULL;
+       }
+       list = PyList_New(ds->ds_num); /* New reference. */
+       for (i = 0; i < ds->ds_num; ++i) {
+               tuple = PyTuple_New(4);
+               PyTuple_SET_ITEM(tuple, 0, cpy_string_to_unicode_or_bytes(ds->ds[i].name));
+               PyTuple_SET_ITEM(tuple, 1, cpy_string_to_unicode_or_bytes(DS_TYPE_TO_STRING(ds->ds[i].type)));
+               PyTuple_SET_ITEM(tuple, 2, float_or_none(ds->ds[i].min));
+               PyTuple_SET_ITEM(tuple, 3, float_or_none(ds->ds[i].max));
+               PyList_SET_ITEM(list, i, tuple);
+       }
+       return list;
+}
+
+static PyObject *cpy_flush(PyObject *self, PyObject *args, PyObject *kwds) {
        int timeout = -1;
-       const char *plugin = NULL, *identifier = NULL;
+       char *plugin = NULL, *identifier = NULL;
        static char *kwlist[] = {"plugin", "timeout", "identifier", NULL};
        
        if (PyArg_ParseTupleAndKeywords(args, kwds, "|etiet", kwlist, NULL, &plugin, &timeout, NULL, &identifier) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_flush(plugin, timeout, identifier);
        Py_END_ALLOW_THREADS
+       PyMem_Free(plugin);
+       PyMem_Free(identifier);
        Py_RETURN_NONE;
 }
 
@@ -575,16 +622,18 @@ static PyObject *cpy_register_generic_userdata(void *reg, void *handler, PyObjec
        reg_function_t *register_function = (reg_function_t *) reg;
        cpy_callback_t *c = NULL;
        user_data_t *user_data = NULL;
-       const char *name = NULL;
+       char *name = NULL;
        PyObject *callback = NULL, *data = NULL;
        static char *kwlist[] = {"callback", "data", "name", NULL};
        
        if (PyArg_ParseTupleAndKeywords(args, kwds, "O|Oet", kwlist, &callback, &data, NULL, &name) == 0) return NULL;
        if (PyCallable_Check(callback) == 0) {
+               PyMem_Free(name);
                PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
                return NULL;
        }
        cpy_build_name(buf, sizeof(buf), callback, name);
+       PyMem_Free(name);
        
        Py_INCREF(callback);
        Py_XINCREF(data);
@@ -605,17 +654,19 @@ static PyObject *cpy_register_read(PyObject *self, PyObject *args, PyObject *kwd
        cpy_callback_t *c = NULL;
        user_data_t *user_data = NULL;
        double interval = 0;
-       const char *name = NULL;
+       char *name = NULL;
        PyObject *callback = NULL, *data = NULL;
        struct timespec ts;
        static char *kwlist[] = {"callback", "interval", "data", "name", NULL};
        
        if (PyArg_ParseTupleAndKeywords(args, kwds, "O|dOet", kwlist, &callback, &interval, &data, NULL, &name) == 0) return NULL;
        if (PyCallable_Check(callback) == 0) {
+               PyMem_Free(name);
                PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
                return NULL;
        }
        cpy_build_name(buf, sizeof(buf), callback, name);
+       PyMem_Free(name);
        
        Py_INCREF(callback);
        Py_XINCREF(data);
@@ -659,48 +710,53 @@ static PyObject *cpy_register_shutdown(PyObject *self, PyObject *args, PyObject
 }
 
 static PyObject *cpy_error(PyObject *self, PyObject *args) {
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_ERR, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
        Py_RETURN_NONE;
 }
 
 static PyObject *cpy_warning(PyObject *self, PyObject *args) {
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_WARNING, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
        Py_RETURN_NONE;
 }
 
 static PyObject *cpy_notice(PyObject *self, PyObject *args) {
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_NOTICE, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
        Py_RETURN_NONE;
 }
 
 static PyObject *cpy_info(PyObject *self, PyObject *args) {
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_INFO, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
        Py_RETURN_NONE;
 }
 
 static PyObject *cpy_debug(PyObject *self, PyObject *args) {
 #ifdef COLLECT_DEBUG
-       const char *text;
+       char *text;
        if (PyArg_ParseTuple(args, "et", NULL, &text) == 0) return NULL;
        Py_BEGIN_ALLOW_THREADS
        plugin_log(LOG_DEBUG, "%s", text);
        Py_END_ALLOW_THREADS
+       PyMem_Free(text);
 #endif
        Py_RETURN_NONE;
 }
@@ -806,6 +862,7 @@ static PyMethodDef cpy_methods[] = {
        {"notice", cpy_notice, METH_VARARGS, log_doc},
        {"warning", cpy_warning, METH_VARARGS, log_doc},
        {"error", cpy_error, METH_VARARGS, log_doc},
+       {"get_dataset", (PyCFunction) cpy_get_dataset, METH_VARARGS, get_ds_doc},
        {"flush", (PyCFunction) cpy_flush, METH_VARARGS | METH_KEYWORDS, flush_doc},
        {"register_log", (PyCFunction) cpy_register_log, METH_VARARGS | METH_KEYWORDS, reg_log_doc},
        {"register_init", (PyCFunction) cpy_register_init, METH_VARARGS | METH_KEYWORDS, reg_init_doc},
@@ -922,7 +979,7 @@ static int cpy_init(void) {
        pthread_sigmask(SIG_BLOCK, &sigset, NULL);
        state = PyEval_SaveThread();
        if (do_interactive) {
-               if (pthread_create(&thread, NULL, cpy_interactive, NULL)) {
+               if (plugin_thread_create(&thread, NULL, cpy_interactive, NULL)) {
                        ERROR("python: Error creating thread for interactive interpreter.");
                }
        }
@@ -976,23 +1033,16 @@ PyMODINIT_FUNC PyInit_collectd(void) {
 }
 #endif
 
-static int cpy_config(oconfig_item_t *ci) {
-       int i;
-       char *argv = "";
-       PyObject *sys, *tb;
-       PyObject *sys_path;
+static int cpy_init_python() {
+       PyObject *sys;
        PyObject *module;
-       
-       /* Ok in theory we shouldn't do initialization at this point
-        * but we have to. In order to give python scripts a chance
-        * to register a config callback we need to be able to execute
-        * python code during the config callback so we have to start
-        * the interpreter here. */
-       /* Do *not* use the python "thread" module at this point! */
 
 #ifdef IS_PY3K
+       wchar_t *argv = L"";
        /* Add a builtin module, before Py_Initialize */
        PyImport_AppendInittab("collectd", PyInit_collectd);
+#else
+       char *argv = "";
 #endif
        
        Py_Initialize();
@@ -1039,6 +1089,26 @@ static int cpy_config(oconfig_item_t *ci) {
        PyModule_AddIntConstant(module, "NOTIF_FAILURE", NOTIF_FAILURE);
        PyModule_AddIntConstant(module, "NOTIF_WARNING", NOTIF_WARNING);
        PyModule_AddIntConstant(module, "NOTIF_OKAY", NOTIF_OKAY);
+       PyModule_AddStringConstant(module, "DS_TYPE_COUNTER", DS_TYPE_TO_STRING(DS_TYPE_COUNTER));
+       PyModule_AddStringConstant(module, "DS_TYPE_GAUGE", DS_TYPE_TO_STRING(DS_TYPE_GAUGE));
+       PyModule_AddStringConstant(module, "DS_TYPE_DERIVE", DS_TYPE_TO_STRING(DS_TYPE_DERIVE));
+       PyModule_AddStringConstant(module, "DS_TYPE_ABSOLUTE", DS_TYPE_TO_STRING(DS_TYPE_ABSOLUTE));
+       return 0;
+}
+
+static int cpy_config(oconfig_item_t *ci) {
+       int i;
+       PyObject *tb;
+
+       /* Ok in theory we shouldn't do initialization at this point
+        * but we have to. In order to give python scripts a chance
+        * to register a config callback we need to be able to execute
+        * python code during the config callback so we have to start
+        * the interpreter here. */
+       /* Do *not* use the python "thread" module at this point! */
+
+       if (!Py_IsInitialized() && cpy_init_python()) return 1;
+
        for (i = 0; i < ci->children_num; ++i) {
                oconfig_item_t *item = ci->children + i;
                
@@ -1049,9 +1119,13 @@ static int cpy_config(oconfig_item_t *ci) {
                } else if (strcasecmp(item->key, "Encoding") == 0) {
                        if (item->values_num != 1 || item->values[0].type != OCONFIG_TYPE_STRING)
                                continue;
+#ifdef IS_PY3K
+                       NOTICE("python: \"Encoding\" was used in the config file but Python3 was used, which does not support changing encodings. Ignoring this.");
+#else
                        /* Why is this even necessary? And undocumented? */
                        if (PyUnicode_SetDefaultEncoding(item->values[0].value.string))
                                cpy_log_exception("setting default encoding");
+#endif
                } else if (strcasecmp(item->key, "LogTraces") == 0) {
                        if (item->values_num != 1 || item->values[0].type != OCONFIG_TYPE_BOOLEAN)
                                continue;
@@ -1085,8 +1159,8 @@ static int cpy_config(oconfig_item_t *ci) {
                                cpy_log_exception("python initialization");
                                continue;
                        }
-                       if (PyList_Append(sys_path, dir_object) != 0) {
-                               ERROR("python plugin: Unable to append \"%s\" to "
+                       if (PyList_Insert(sys_path, 0, dir_object) != 0) {
+                               ERROR("python plugin: Unable to prepend \"%s\" to "
                                      "python module path.", dir);
                                cpy_log_exception("python initialization");
                        }
@@ -1138,7 +1212,6 @@ static int cpy_config(oconfig_item_t *ci) {
                        WARNING("python plugin: Ignoring unknown config key \"%s\".", item->key);
                }
        }
-       Py_DECREF(sys_path);
        return 0;
 }