src/python.c

   1 #include <Python.h>
   2 #include <structmember.h>
   3
   4 #include "collectd.h"
   5 #include "common.h"
   6
   7 #include "cpython.h"
   8
   9 typedef struct cpy_callback_s {
  10         char *name;
  11         PyObject *callback;
  12         PyObject *data;
  13         struct cpy_callback_s *next;
  14 } cpy_callback_t;
  15
  16 /* This is our global thread state. Python saves some stuff in thread-local
  17  * storage. So if we allow the interpreter to run in the background
  18  * (the scriptwriters might have created some threads from python), we have
  19  * to save the state so we can resume it later after shutdown. */
  20
  21 static PyThreadState *state;
  22
  23 static cpy_callback_t *cpy_config_callbacks;
  24 static cpy_callback_t *cpy_init_callbacks;
  25 static cpy_callback_t *cpy_shutdown_callbacks;
  26
  27 static void cpy_destroy_user_data(void *data) {
  28         cpy_callback_t *c = data;
  29         free(c->name);
  30         Py_DECREF(c->callback);
  31         Py_XDECREF(c->data);
  32         free(c);
  33 }
  34
  35 /* You must hold the GIL to call this function!
  36  * But if you managed to extract the callback parameter then you probably already do. */
  37
  38 static void cpy_build_name(char *buf, size_t size, PyObject *callback, const char *name) {
  39         const char *module;
  40         PyObject *mod = NULL, *n = NULL;
  41
  42         if (name != NULL && strchr(name, '.') != NULL) {
  43                 snprintf(buf, size, "python.%s", name);
  44                 return;
  45         }
  46
  47         mod = PyObject_GetAttrString(callback, "__module__"); /* New reference. */
  48         if (mod != NULL)
  49                 module = PyString_AsString(mod);
  50         else
  51                 module = "collectd";
  52         if (name != NULL) {
  53                 snprintf(buf, size, "python.%s.%s", module, name);
  54                 Py_XDECREF(mod);
  55                 return;
  56         }
  57
  58         n = PyObject_GetAttrString(callback, "__name__"); /* New reference. */
  59         if (n != NULL)
  60                 name = PyString_AsString(n);
  61
  62         if (name != NULL)
  63                 snprintf(buf, size, "python.%s.%s", module, name);
  64         else
  65                 snprintf(buf, size, "python.%s.%p", module, callback);
  66         Py_XDECREF(mod);
  67         Py_XDECREF(n);
  68 }
  69
  70 static int cpy_read_callback(user_data_t *data) {
  71         cpy_callback_t *c = data->data;
  72         PyObject *ret;
  73
  74         CPY_LOCK_THREADS
  75                 if (c->data == NULL)
  76                         ret = PyObject_CallFunctionObjArgs(c->callback, (void *) 0);
  77                 else
  78                         ret = PyObject_CallFunctionObjArgs(c->callback, c->data, (void *) 0);
  79                 if (ret == NULL) {
  80                         /* FIXME */
  81                         PyErr_Print();
  82                 } else {
  83                         Py_DECREF(ret);
  84                 }
  85         CPY_RELEASE_THREADS
  86         return 0;
  87 }
  88
  89 static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_list, user_data_t *data) {
  90         int i;
  91         cpy_callback_t *c = data->data;
  92         PyObject *ret, *v, *list;
  93
  94         CPY_LOCK_THREADS
  95                 list = PyList_New(value_list->values_len); /* New reference. */
  96                 if (list == NULL) {
  97                         PyErr_Print();
  98                         CPY_RETURN_FROM_THREADS 0;
  99                 }
 100                 for (i = 0; i < value_list->values_len; ++i) {
 101                         if (ds->ds->type == DS_TYPE_COUNTER) {
 102                                 if ((long) value_list->values[i].counter == value_list->values[i].counter)
 103                                         PyList_SetItem(list, i, PyInt_FromLong(value_list->values[i].counter));
 104                                 else
 105                                         PyList_SetItem(list, i, PyLong_FromUnsignedLongLong(value_list->values[i].counter));
 106                         } else if (ds->ds->type == DS_TYPE_GAUGE) {
 107                                 PyList_SetItem(list, i, PyFloat_FromDouble(value_list->values[i].gauge));
 108                         } else if (ds->ds->type == DS_TYPE_DERIVE) {
 109                                 if ((long) value_list->values[i].derive == value_list->values[i].derive)
 110                                         PyList_SetItem(list, i, PyInt_FromLong(value_list->values[i].derive));
 111                                 else
 112                                         PyList_SetItem(list, i, PyLong_FromLongLong(value_list->values[i].derive));
 113                         } else if (ds->ds->type == DS_TYPE_ABSOLUTE) {
 114                                 if ((long) value_list->values[i].absolute == value_list->values[i].absolute)
 115                                         PyList_SetItem(list, i, PyInt_FromLong(value_list->values[i].absolute));
 116                                 else
 117                                         PyList_SetItem(list, i, PyLong_FromUnsignedLongLong(value_list->values[i].absolute));
 118                         } else {
 119                                 ERROR("cpy_write_callback: Unknown value type %d.", ds->ds->type);
 120                                 Py_DECREF(list);
 121                                 CPY_RETURN_FROM_THREADS 0;
 122                         }
 123                         if (PyErr_Occurred() != NULL) {
 124                                 PyErr_Print();
 125                                 CPY_RETURN_FROM_THREADS 0;
 126                         }
 127                 }
 128                 v = PyObject_CallFunction((PyObject *) &ValuesType, "sOssssdi", value_list->type, list,
 129                                 value_list->plugin_instance, value_list->type_instance, value_list->plugin,
 130                                 value_list->host, (double) value_list->time, value_list->interval);
 131                 Py_DECREF(list);
 132                 ret = PyObject_CallFunctionObjArgs(c->callback, v, (void *) 0);
 133                 if (ret == NULL) {
 134                         /* FIXME */
 135                         PyErr_Print();
 136                 } else {
 137                         Py_DECREF(ret);
 138                 }
 139         CPY_RELEASE_THREADS
 140         return 0;
 141 }
 142
 143 static void cpy_log_callback(int severity, const char *message, user_data_t *data) {
 144         cpy_callback_t * c = data->data;
 145         PyObject *ret;
 146
 147         CPY_LOCK_THREADS
 148         if (c->data == NULL)
 149                 ret = PyObject_CallFunction(c->callback, "is", severity, message); /* New reference. */
 150         else
 151                 ret = PyObject_CallFunction(c->callback, "isO", severity, message, c->data); /* New reference. */
 152
 153         if (ret == NULL) {
 154                 /* FIXME */
 155                 PyErr_Print();
 156         } else {
 157                 Py_DECREF(ret);
 158         }
 159         CPY_RELEASE_THREADS
 160 }
 161
 162 static PyObject *cpy_register_generic(cpy_callback_t **list_head, PyObject *args, PyObject *kwds) {
 163         cpy_callback_t *c;
 164         const char *name = NULL;
 165         PyObject *callback = NULL, *data = NULL, *mod = NULL;
 166         static char *kwlist[] = {"callback", "data", "name", NULL};
 167
 168         if (PyArg_ParseTupleAndKeywords(args, kwds, "O|Oz", kwlist, &callback, &data, &name) == 0) return NULL;
 169         if (PyCallable_Check(callback) == 0) {
 170                 PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
 171                 return NULL;
 172         }
 173         if (name == NULL) {
 174                 mod = PyObject_GetAttrString(callback, "__module__"); /* New reference. */
 175                 if (mod != NULL) name = PyString_AsString(mod);
 176                 if (name == NULL) {
 177                         Py_XDECREF(mod);
 178                         PyErr_SetString(PyExc_ValueError, "No module name specified and "
 179                                 "callback function does not have a \"__module__\" attribute.");
 180                         return NULL;
 181                 }
 182         }
 183         Py_INCREF(callback);
 184         Py_XINCREF(data);
 185         c = malloc(sizeof(*c));
 186         c->name = strdup(name);
 187         c->callback = callback;
 188         c->data = data;
 189         c->next = *list_head;
 190         *list_head = c;
 191         Py_XDECREF(mod);
 192         Py_RETURN_NONE;
 193 }
 194
 195 static PyObject *cpy_register_config(PyObject *self, PyObject *args, PyObject *kwds) {
 196         return cpy_register_generic(&cpy_config_callbacks, args, kwds);
 197 }
 198
 199 static PyObject *cpy_register_init(PyObject *self, PyObject *args, PyObject *kwds) {
 200         return cpy_register_generic(&cpy_init_callbacks, args, kwds);
 201 }
 202
 203 typedef int reg_function_t(const char *name, void *callback, void *data);
 204
 205 static PyObject *cpy_register_generic_userdata(void *reg, void *handler, PyObject *args, PyObject *kwds) {
 206         char buf[512];
 207         reg_function_t *register_function = (reg_function_t *) reg;
 208         cpy_callback_t *c = NULL;
 209         user_data_t *user_data = NULL;
 210         const char *name = NULL;
 211         PyObject *callback = NULL, *data = NULL;
 212         static char *kwlist[] = {"callback", "data", "name", NULL};
 213
 214         if (PyArg_ParseTupleAndKeywords(args, kwds, "O|Oz", kwlist, &callback, &data, &name) == 0) return NULL;
 215         if (PyCallable_Check(callback) == 0) {
 216                 PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
 217                 return NULL;
 218         }
 219         cpy_build_name(buf, sizeof(buf), callback, name);
 220
 221         Py_INCREF(callback);
 222         Py_XINCREF(data);
 223         c = malloc(sizeof(*c));
 224         c->name = strdup(buf);
 225         c->callback = callback;
 226         c->data = data;
 227         c->next = NULL;
 228         user_data = malloc(sizeof(*user_data));
 229         user_data->free_func = cpy_destroy_user_data;
 230         user_data->data = c;
 231         register_function(buf, handler, user_data);
 232         return PyString_FromString(buf);
 233 }
 234
 235 static PyObject *cpy_register_read(PyObject *self, PyObject *args, PyObject *kwds) {
 236         char buf[512];
 237         cpy_callback_t *c = NULL;
 238         user_data_t *user_data = NULL;
 239         double interval = 0;
 240         const char *name = NULL;
 241         PyObject *callback = NULL, *data = NULL;
 242         struct timespec ts;
 243         static char *kwlist[] = {"callback", "interval", "data", "name", NULL};
 244
 245         if (PyArg_ParseTupleAndKeywords(args, kwds, "O|dOz", kwlist, &callback, &interval, &data, &name) == 0) return NULL;
 246         if (PyCallable_Check(callback) == 0) {
 247                 PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
 248                 return NULL;
 249         }
 250         cpy_build_name(buf, sizeof(buf), callback, name);
 251
 252         Py_INCREF(callback);
 253         Py_XINCREF(data);
 254         c = malloc(sizeof(*c));
 255         c->name = strdup(buf);
 256         c->callback = callback;
 257         c->data = data;
 258         c->next = NULL;
 259         user_data = malloc(sizeof(*user_data));
 260         user_data->free_func = cpy_destroy_user_data;
 261         user_data->data = c;
 262         ts.tv_sec = interval;
 263         ts.tv_nsec = (interval - ts.tv_sec) * 1000000000;
 264         plugin_register_complex_read(buf, cpy_read_callback, &ts, user_data);
 265         return PyString_FromString(buf);
 266 }
 267
 268 static PyObject *cpy_register_log(PyObject *self, PyObject *args, PyObject *kwds) {
 269         return cpy_register_generic_userdata(plugin_register_log, cpy_log_callback, args, kwds);
 270 }
 271
 272 static PyObject *cpy_register_write(PyObject *self, PyObject *args, PyObject *kwds) {
 273         return cpy_register_generic_userdata(plugin_register_write, cpy_write_callback, args, kwds);
 274 }
 275
 276 static PyObject *cpy_register_shutdown(PyObject *self, PyObject *args, PyObject *kwds) {
 277         return cpy_register_generic(&cpy_shutdown_callbacks, args, kwds);
 278 }
 279
 280 static PyObject *cpy_Error(PyObject *self, PyObject *args) {
 281         const char *text;
 282         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 283         Py_BEGIN_ALLOW_THREADS
 284         plugin_log(LOG_ERR, "%s", text);
 285         Py_END_ALLOW_THREADS
 286         Py_RETURN_NONE;
 287 }
 288
 289 static PyObject *cpy_Warning(PyObject *self, PyObject *args) {
 290         const char *text;
 291         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 292         Py_BEGIN_ALLOW_THREADS
 293         plugin_log(LOG_WARNING, "%s", text);
 294         Py_END_ALLOW_THREADS
 295         Py_RETURN_NONE;
 296 }
 297
 298 static PyObject *cpy_Notice(PyObject *self, PyObject *args) {
 299         const char *text;
 300         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 301         Py_BEGIN_ALLOW_THREADS
 302         plugin_log(LOG_NOTICE, "%s", text);
 303         Py_END_ALLOW_THREADS
 304         Py_RETURN_NONE;
 305 }
 306
 307 static PyObject *cpy_Info(PyObject *self, PyObject *args) {
 308         const char *text;
 309         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 310         Py_BEGIN_ALLOW_THREADS
 311         plugin_log(LOG_INFO, "%s", text);
 312         Py_END_ALLOW_THREADS
 313         Py_RETURN_NONE;
 314 }
 315
 316 static PyObject *cpy_Debug(PyObject *self, PyObject *args) {
 317 #ifdef COLLECT_DEBUG
 318         const char *text;
 319         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 320         plugin_log(LOG_DEBUG, "%s", text);
 321 #endif
 322         Py_RETURN_NONE;
 323 }
 324
 325 static PyMethodDef cpy_methods[] = {
 326         {"Debug", cpy_Debug, METH_VARARGS, "This is an unhelpful text."},
 327         {"Info", cpy_Info, METH_VARARGS, "This is an unhelpful text."},
 328         {"Notice", cpy_Notice, METH_VARARGS, "This is an unhelpful text."},
 329         {"Warning", cpy_Warning, METH_VARARGS, "This is an unhelpful text."},
 330         {"Error", cpy_Error, METH_VARARGS, "This is an unhelpful text."},
 331         {"register_log", (PyCFunction) cpy_register_log, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 332         {"register_init", (PyCFunction) cpy_register_init, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 333         {"register_config", (PyCFunction) cpy_register_config, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 334         {"register_read", (PyCFunction) cpy_register_read, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 335         {"register_write", (PyCFunction) cpy_register_write, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 336         {"register_shutdown", (PyCFunction) cpy_register_shutdown, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 337         {0, 0, 0, 0}
 338 };
 339
 340 static int cpy_shutdown(void) {
 341         cpy_callback_t *c;
 342         PyObject *ret;
 343
 344         /* This can happen if the module was loaded but not configured. */
 345         if (state != NULL)
 346                 PyEval_RestoreThread(state);
 347
 348         for (c = cpy_shutdown_callbacks; c; c = c->next) {
 349                 if (c->data == NULL)
 350                         ret = PyObject_CallObject(c->callback, NULL); /* New reference. */
 351                 else
 352                         ret = PyObject_CallFunctionObjArgs(c->callback, c->data, (void *) 0); /* New reference. */
 353                 if (ret == NULL)
 354                         PyErr_Print(); /* FIXME */
 355                 else
 356                         Py_DECREF(ret);
 357         }
 358         Py_Finalize();
 359         return 0;
 360 }
 361
 362 static int cpy_init(void) {
 363         cpy_callback_t *c;
 364         PyObject *ret;
 365
 366         PyEval_InitThreads();
 367         /* Now it's finally OK to use python threads. */
 368         for (c = cpy_init_callbacks; c; c = c->next) {
 369                 if (c->data == NULL)
 370                         ret = PyObject_CallObject(c->callback, NULL); /* New reference. */
 371                 else
 372                         ret = PyObject_CallFunctionObjArgs(c->callback, c->data, (void *) 0); /* New reference. */
 373                 if (ret == NULL)
 374                         PyErr_Print(); /* FIXME */
 375                 else
 376                         Py_DECREF(ret);
 377         }
 378         state = PyEval_SaveThread();
 379         return 0;
 380 }
 381
 382 static PyObject *cpy_oconfig_to_pyconfig(oconfig_item_t *ci, PyObject *parent) {
 383         int i;
 384         PyObject *item, *values, *children, *tmp;
 385
 386         if (parent == NULL)
 387                 parent = Py_None;
 388
 389         values = PyTuple_New(ci->values_num); /* New reference. */
 390         for (i = 0; i < ci->values_num; ++i) {
 391                 if (ci->values[i].type == OCONFIG_TYPE_STRING) {
 392                         PyTuple_SET_ITEM(values, i, PyString_FromString(ci->values[i].value.string));
 393                 } else if (ci->values[i].type == OCONFIG_TYPE_NUMBER) {
 394                         PyTuple_SET_ITEM(values, i, PyFloat_FromDouble(ci->values[i].value.number));
 395                 } else if (ci->values[i].type == OCONFIG_TYPE_BOOLEAN) {
 396                         PyTuple_SET_ITEM(values, i, PyBool_FromLong(ci->values[i].value.boolean));
 397                 }
 398         }
 399
 400         item = PyObject_CallFunction((PyObject *) &ConfigType, "sONO", ci->key, parent, values, Py_None);
 401         if (item == NULL)
 402                 return NULL;
 403         children = PyTuple_New(ci->children_num); /* New reference. */
 404         for (i = 0; i < ci->children_num; ++i) {
 405                         PyTuple_SET_ITEM(children, i, cpy_oconfig_to_pyconfig(ci->children + i, item));
 406         }
 407         tmp = ((Config *) item)->children;
 408         ((Config *) item)->children = children;
 409         Py_XDECREF(tmp);
 410         return item;
 411 }
 412
 413 static int cpy_config(oconfig_item_t *ci) {
 414         int i;
 415         PyObject *sys;
 416         PyObject *sys_path;
 417         PyObject *module;
 418
 419         /* Ok in theory we shouldn't do initialization at this point
 420          * but we have to. In order to give python scripts a chance
 421          * to register a config callback we need to be able to execute
 422          * python code during the config callback so we have to start
 423          * the interpreter here. */
 424         /* Do *not* use the python "thread" module at this point! */
 425         Py_Initialize();
 426
 427         PyType_Ready(&ConfigType);
 428         PyType_Ready(&ValuesType);
 429         sys = PyImport_ImportModule("sys"); /* New reference. */
 430         if (sys == NULL) {
 431                 ERROR("python module: Unable to import \"sys\" module.");
 432                 /* Just print the default python exception text to stderr. */
 433                 PyErr_Print();
 434                 return 1;
 435         }
 436         sys_path = PyObject_GetAttrString(sys, "path"); /* New reference. */
 437         Py_DECREF(sys);
 438         if (sys_path == NULL) {
 439                 ERROR("python module: Unable to read \"sys.path\".");
 440                 PyErr_Print();
 441                 return 1;
 442         }
 443         module = Py_InitModule("collectd", cpy_methods); /* Borrowed reference. */
 444         PyModule_AddObject(module, "Config", (PyObject *) &ConfigType); /* Steals a reference. */
 445         PyModule_AddObject(module, "Values", (PyObject *) &ValuesType); /* Steals a reference. */
 446         PyModule_AddIntConstant(module, "LOG_DEBUG", LOG_DEBUG);
 447         PyModule_AddIntConstant(module, "LOG_INFO", LOG_INFO);
 448         PyModule_AddIntConstant(module, "LOG_NOTICE", LOG_NOTICE);
 449         PyModule_AddIntConstant(module, "LOG_WARNING", LOG_WARNING);
 450         PyModule_AddIntConstant(module, "LOG_ERROR", LOG_ERR);
 451         for (i = 0; i < ci->children_num; ++i) {
 452                 oconfig_item_t *item = ci->children + i;
 453
 454                 if (strcasecmp(item->key, "ModulePath") == 0) {
 455                         char *dir = NULL;
 456                         PyObject *dir_object;
 457
 458                         if (cf_util_get_string(item, &dir) != 0)
 459                                 continue;
 460                         dir_object = PyString_FromString(dir); /* New reference. */
 461                         if (dir_object == NULL) {
 462                                 ERROR("python plugin: Unable to convert \"%s\" to "
 463                                       "a python object.", dir);
 464                                 free(dir);
 465                                 PyErr_Print();
 466                                 continue;
 467                         }
 468                         if (PyList_Append(sys_path, dir_object) != 0) {
 469                                 ERROR("python plugin: Unable to append \"%s\" to "
 470                                       "python module path.", dir);
 471                                 PyErr_Print();
 472                         }
 473                         Py_DECREF(dir_object);
 474                         free(dir);
 475                 } else if (strcasecmp(item->key, "Import") == 0) {
 476                         char *module_name = NULL;
 477                         PyObject *module;
 478
 479                         if (cf_util_get_string(item, &module_name) != 0)
 480                                 continue;
 481                         module = PyImport_ImportModule(module_name); /* New reference. */
 482                         if (module == NULL) {
 483                                 ERROR("python plugin: Error importing module \"%s\".", module_name);
 484                                 PyErr_Print();
 485                         }
 486                         free(module_name);
 487                         Py_XDECREF(module);
 488                 } else if (strcasecmp(item->key, "Module") == 0) {
 489                         char *name = NULL;
 490                         cpy_callback_t *c;
 491                         PyObject *ret;
 492
 493                         if (cf_util_get_string(item, &name) != 0)
 494                                 continue;
 495                         for (c = cpy_config_callbacks; c; c = c->next) {
 496                                 if (strcasecmp(c->name, name) == 0)
 497                                         break;
 498                         }
 499                         if (c == NULL) {
 500                                 WARNING("python plugin: Found a configuration for the \"%s\" plugin, "
 501                                         "but the plugin isn't loaded or didn't register "
 502                                         "a configuration callback.", name);
 503                                 free(name);
 504                                 continue;
 505                         }
 506                         free(name);
 507                         if (c->data == NULL)
 508                                 ret = PyObject_CallFunction(c->callback, "N",
 509                                         cpy_oconfig_to_pyconfig(item, NULL)); /* New reference. */
 510                         else
 511                                 ret = PyObject_CallFunction(c->callback, "NO",
 512                                         cpy_oconfig_to_pyconfig(item, NULL), c->data); /* New reference. */
 513                         if (ret == NULL)
 514                                 PyErr_Print();
 515                         else
 516                                 Py_DECREF(ret);
 517                 } else {
 518                         WARNING("python plugin: Ignoring unknown config key \"%s\".", item->key);
 519                 }
 520         }
 521         Py_DECREF(sys_path);
 522         return 0;
 523 }
 524
 525 void module_register(void) {
 526         plugin_register_complex_config("python", cpy_config);
 527         plugin_register_init("python", cpy_init);
 528 //      plugin_register_read("python", cna_read);
 529         plugin_register_shutdown("python", cpy_shutdown);
 530 }