src/python.c

   1 #include <Python.h>
   2 #include <structmember.h>
   3
   4 #if HAVE_PTHREAD_H
   5 # include <pthread.h>
   6 #endif
   7
   8 #include "collectd.h"
   9 #include "common.h"
  10
  11 #include "cpython.h"
  12
  13 typedef struct cpy_callback_s {
  14         char *name;
  15         PyObject *callback;
  16         PyObject *data;
  17         struct cpy_callback_s *next;
  18 } cpy_callback_t;
  19
  20 static int do_interactive = 0;
  21
  22 /* This is our global thread state. Python saves some stuff in thread-local
  23  * storage. So if we allow the interpreter to run in the background
  24  * (the scriptwriters might have created some threads from python), we have
  25  * to save the state so we can resume it later after shutdown. */
  26
  27 static PyThreadState *state;
  28
  29 static cpy_callback_t *cpy_config_callbacks;
  30 static cpy_callback_t *cpy_init_callbacks;
  31 static cpy_callback_t *cpy_shutdown_callbacks;
  32
  33 static void cpy_destroy_user_data(void *data) {
  34         cpy_callback_t *c = data;
  35         free(c->name);
  36         Py_DECREF(c->callback);
  37         Py_XDECREF(c->data);
  38         free(c);
  39 }
  40
  41 /* You must hold the GIL to call this function!
  42  * But if you managed to extract the callback parameter then you probably already do. */
  43
  44 static void cpy_build_name(char *buf, size_t size, PyObject *callback, const char *name) {
  45         const char *module;
  46         PyObject *mod = NULL, *n = NULL;
  47
  48         if (name != NULL && strchr(name, '.') != NULL) {
  49                 snprintf(buf, size, "python.%s", name);
  50                 return;
  51         }
  52
  53         mod = PyObject_GetAttrString(callback, "__module__"); /* New reference. */
  54         if (mod != NULL)
  55                 module = PyString_AsString(mod);
  56         else
  57                 module = "collectd";
  58         if (name != NULL) {
  59                 snprintf(buf, size, "python.%s.%s", module, name);
  60                 Py_XDECREF(mod);
  61                 return;
  62         }
  63
  64         n = PyObject_GetAttrString(callback, "__name__"); /* New reference. */
  65         if (n != NULL)
  66                 name = PyString_AsString(n);
  67
  68         if (name != NULL)
  69                 snprintf(buf, size, "python.%s.%s", module, name);
  70         else
  71                 snprintf(buf, size, "python.%s.%p", module, callback);
  72         Py_XDECREF(mod);
  73         Py_XDECREF(n);
  74 }
  75
  76 static int cpy_read_callback(user_data_t *data) {
  77         cpy_callback_t *c = data->data;
  78         PyObject *ret;
  79
  80         CPY_LOCK_THREADS
  81                 ret = PyObject_CallFunctionObjArgs(c->callback, c->data, (void *) 0); /* New reference. */
  82                 if (ret == NULL) {
  83                         /* FIXME */
  84                         PyErr_Print();
  85                 } else {
  86                         Py_DECREF(ret);
  87                 }
  88         CPY_RELEASE_THREADS
  89         return 0;
  90 }
  91
  92 static int cpy_write_callback(const data_set_t *ds, const value_list_t *value_list, user_data_t *data) {
  93         int i;
  94         cpy_callback_t *c = data->data;
  95         PyObject *ret, *v, *list;
  96
  97         CPY_LOCK_THREADS
  98                 list = PyList_New(value_list->values_len); /* New reference. */
  99                 if (list == NULL) {
 100                         PyErr_Print();
 101                         CPY_RETURN_FROM_THREADS 0;
 102                 }
 103                 for (i = 0; i < value_list->values_len; ++i) {
 104                         if (ds->ds->type == DS_TYPE_COUNTER) {
 105                                 if ((long) value_list->values[i].counter == value_list->values[i].counter)
 106                                         PyList_SetItem(list, i, PyInt_FromLong(value_list->values[i].counter));
 107                                 else
 108                                         PyList_SetItem(list, i, PyLong_FromUnsignedLongLong(value_list->values[i].counter));
 109                         } else if (ds->ds->type == DS_TYPE_GAUGE) {
 110                                 PyList_SetItem(list, i, PyFloat_FromDouble(value_list->values[i].gauge));
 111                         } else if (ds->ds->type == DS_TYPE_DERIVE) {
 112                                 if ((long) value_list->values[i].derive == value_list->values[i].derive)
 113                                         PyList_SetItem(list, i, PyInt_FromLong(value_list->values[i].derive));
 114                                 else
 115                                         PyList_SetItem(list, i, PyLong_FromLongLong(value_list->values[i].derive));
 116                         } else if (ds->ds->type == DS_TYPE_ABSOLUTE) {
 117                                 if ((long) value_list->values[i].absolute == value_list->values[i].absolute)
 118                                         PyList_SetItem(list, i, PyInt_FromLong(value_list->values[i].absolute));
 119                                 else
 120                                         PyList_SetItem(list, i, PyLong_FromUnsignedLongLong(value_list->values[i].absolute));
 121                         } else {
 122                                 ERROR("cpy_write_callback: Unknown value type %d.", ds->ds->type);
 123                                 Py_DECREF(list);
 124                                 CPY_RETURN_FROM_THREADS 0;
 125                         }
 126                         if (PyErr_Occurred() != NULL) {
 127                                 PyErr_Print();
 128                                 CPY_RETURN_FROM_THREADS 0;
 129                         }
 130                 }
 131                 v = PyObject_CallFunction((PyObject *) &ValuesType, "sOssssdi", value_list->type, list,
 132                                 value_list->plugin_instance, value_list->type_instance, value_list->plugin,
 133                                 value_list->host, (double) value_list->time, value_list->interval);
 134                 Py_DECREF(list);
 135                 ret = PyObject_CallFunctionObjArgs(c->callback, v, c->data, (void *) 0); /* New reference. */
 136                 if (ret == NULL) {
 137                         /* FIXME */
 138                         PyErr_Print();
 139                 } else {
 140                         Py_DECREF(ret);
 141                 }
 142         CPY_RELEASE_THREADS
 143         return 0;
 144 }
 145
 146 static void cpy_log_callback(int severity, const char *message, user_data_t *data) {
 147         cpy_callback_t * c = data->data;
 148         PyObject *ret;
 149
 150         CPY_LOCK_THREADS
 151         if (c->data == NULL)
 152                 ret = PyObject_CallFunction(c->callback, "is", severity, message); /* New reference. */
 153         else
 154                 ret = PyObject_CallFunction(c->callback, "isO", severity, message, c->data); /* New reference. */
 155
 156         if (ret == NULL) {
 157                 /* FIXME */
 158                 PyErr_Print();
 159         } else {
 160                 Py_DECREF(ret);
 161         }
 162         CPY_RELEASE_THREADS
 163 }
 164
 165 static PyObject *cpy_register_generic(cpy_callback_t **list_head, PyObject *args, PyObject *kwds) {
 166         cpy_callback_t *c;
 167         const char *name = NULL;
 168         PyObject *callback = NULL, *data = NULL, *mod = NULL;
 169         static char *kwlist[] = {"callback", "data", "name", NULL};
 170
 171         if (PyArg_ParseTupleAndKeywords(args, kwds, "O|Oz", kwlist, &callback, &data, &name) == 0) return NULL;
 172         if (PyCallable_Check(callback) == 0) {
 173                 PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
 174                 return NULL;
 175         }
 176         if (name == NULL) {
 177                 mod = PyObject_GetAttrString(callback, "__module__"); /* New reference. */
 178                 if (mod != NULL) name = PyString_AsString(mod);
 179                 if (name == NULL) {
 180                         Py_XDECREF(mod);
 181                         PyErr_SetString(PyExc_ValueError, "No module name specified and "
 182                                 "callback function does not have a \"__module__\" attribute.");
 183                         return NULL;
 184                 }
 185         }
 186         Py_INCREF(callback);
 187         Py_XINCREF(data);
 188         c = malloc(sizeof(*c));
 189         c->name = strdup(name);
 190         c->callback = callback;
 191         c->data = data;
 192         c->next = *list_head;
 193         *list_head = c;
 194         Py_XDECREF(mod);
 195         Py_RETURN_NONE;
 196 }
 197
 198 static PyObject *cpy_register_config(PyObject *self, PyObject *args, PyObject *kwds) {
 199         return cpy_register_generic(&cpy_config_callbacks, args, kwds);
 200 }
 201
 202 static PyObject *cpy_register_init(PyObject *self, PyObject *args, PyObject *kwds) {
 203         return cpy_register_generic(&cpy_init_callbacks, args, kwds);
 204 }
 205
 206 typedef int reg_function_t(const char *name, void *callback, void *data);
 207
 208 static PyObject *cpy_register_generic_userdata(void *reg, void *handler, PyObject *args, PyObject *kwds) {
 209         char buf[512];
 210         reg_function_t *register_function = (reg_function_t *) reg;
 211         cpy_callback_t *c = NULL;
 212         user_data_t *user_data = NULL;
 213         const char *name = NULL;
 214         PyObject *callback = NULL, *data = NULL;
 215         static char *kwlist[] = {"callback", "data", "name", NULL};
 216
 217         if (PyArg_ParseTupleAndKeywords(args, kwds, "O|Oz", kwlist, &callback, &data, &name) == 0) return NULL;
 218         if (PyCallable_Check(callback) == 0) {
 219                 PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
 220                 return NULL;
 221         }
 222         cpy_build_name(buf, sizeof(buf), callback, name);
 223
 224         Py_INCREF(callback);
 225         Py_XINCREF(data);
 226         c = malloc(sizeof(*c));
 227         c->name = strdup(buf);
 228         c->callback = callback;
 229         c->data = data;
 230         c->next = NULL;
 231         user_data = malloc(sizeof(*user_data));
 232         user_data->free_func = cpy_destroy_user_data;
 233         user_data->data = c;
 234         register_function(buf, handler, user_data);
 235         return PyString_FromString(buf);
 236 }
 237
 238 static PyObject *cpy_register_read(PyObject *self, PyObject *args, PyObject *kwds) {
 239         char buf[512];
 240         cpy_callback_t *c = NULL;
 241         user_data_t *user_data = NULL;
 242         double interval = 0;
 243         const char *name = NULL;
 244         PyObject *callback = NULL, *data = NULL;
 245         struct timespec ts;
 246         static char *kwlist[] = {"callback", "interval", "data", "name", NULL};
 247
 248         if (PyArg_ParseTupleAndKeywords(args, kwds, "O|dOz", kwlist, &callback, &interval, &data, &name) == 0) return NULL;
 249         if (PyCallable_Check(callback) == 0) {
 250                 PyErr_SetString(PyExc_TypeError, "callback needs a be a callable object.");
 251                 return NULL;
 252         }
 253         cpy_build_name(buf, sizeof(buf), callback, name);
 254
 255         Py_INCREF(callback);
 256         Py_XINCREF(data);
 257         c = malloc(sizeof(*c));
 258         c->name = strdup(buf);
 259         c->callback = callback;
 260         c->data = data;
 261         c->next = NULL;
 262         user_data = malloc(sizeof(*user_data));
 263         user_data->free_func = cpy_destroy_user_data;
 264         user_data->data = c;
 265         ts.tv_sec = interval;
 266         ts.tv_nsec = (interval - ts.tv_sec) * 1000000000;
 267         plugin_register_complex_read(buf, cpy_read_callback, &ts, user_data);
 268         return PyString_FromString(buf);
 269 }
 270
 271 static PyObject *cpy_register_log(PyObject *self, PyObject *args, PyObject *kwds) {
 272         return cpy_register_generic_userdata(plugin_register_log, cpy_log_callback, args, kwds);
 273 }
 274
 275 static PyObject *cpy_register_write(PyObject *self, PyObject *args, PyObject *kwds) {
 276         return cpy_register_generic_userdata(plugin_register_write, cpy_write_callback, args, kwds);
 277 }
 278
 279 static PyObject *cpy_register_shutdown(PyObject *self, PyObject *args, PyObject *kwds) {
 280         return cpy_register_generic(&cpy_shutdown_callbacks, args, kwds);
 281 }
 282
 283 static PyObject *cpy_Error(PyObject *self, PyObject *args) {
 284         const char *text;
 285         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 286         Py_BEGIN_ALLOW_THREADS
 287         plugin_log(LOG_ERR, "%s", text);
 288         Py_END_ALLOW_THREADS
 289         Py_RETURN_NONE;
 290 }
 291
 292 static PyObject *cpy_Warning(PyObject *self, PyObject *args) {
 293         const char *text;
 294         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 295         Py_BEGIN_ALLOW_THREADS
 296         plugin_log(LOG_WARNING, "%s", text);
 297         Py_END_ALLOW_THREADS
 298         Py_RETURN_NONE;
 299 }
 300
 301 static PyObject *cpy_Notice(PyObject *self, PyObject *args) {
 302         const char *text;
 303         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 304         Py_BEGIN_ALLOW_THREADS
 305         plugin_log(LOG_NOTICE, "%s", text);
 306         Py_END_ALLOW_THREADS
 307         Py_RETURN_NONE;
 308 }
 309
 310 static PyObject *cpy_Info(PyObject *self, PyObject *args) {
 311         const char *text;
 312         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 313         Py_BEGIN_ALLOW_THREADS
 314         plugin_log(LOG_INFO, "%s", text);
 315         Py_END_ALLOW_THREADS
 316         Py_RETURN_NONE;
 317 }
 318
 319 static PyObject *cpy_Debug(PyObject *self, PyObject *args) {
 320 #ifdef COLLECT_DEBUG
 321         const char *text;
 322         if (PyArg_ParseTuple(args, "s", &text) == 0) return NULL;
 323         plugin_log(LOG_DEBUG, "%s", text);
 324 #endif
 325         Py_RETURN_NONE;
 326 }
 327
 328 static PyMethodDef cpy_methods[] = {
 329         {"debug", cpy_Debug, METH_VARARGS, "This is an unhelpful text."},
 330         {"info", cpy_Info, METH_VARARGS, "This is an unhelpful text."},
 331         {"notice", cpy_Notice, METH_VARARGS, "This is an unhelpful text."},
 332         {"warning", cpy_Warning, METH_VARARGS, "This is an unhelpful text."},
 333         {"error", cpy_Error, METH_VARARGS, "This is an unhelpful text."},
 334         {"register_log", (PyCFunction) cpy_register_log, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 335         {"register_init", (PyCFunction) cpy_register_init, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 336         {"register_config", (PyCFunction) cpy_register_config, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 337         {"register_read", (PyCFunction) cpy_register_read, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 338         {"register_write", (PyCFunction) cpy_register_write, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 339         {"register_shutdown", (PyCFunction) cpy_register_shutdown, METH_VARARGS | METH_KEYWORDS, "This is an unhelpful text."},
 340         {0, 0, 0, 0}
 341 };
 342
 343 static int cpy_shutdown(void) {
 344         cpy_callback_t *c;
 345         PyObject *ret;
 346
 347         /* This can happen if the module was loaded but not configured. */
 348         if (state != NULL)
 349                 PyEval_RestoreThread(state);
 350
 351         for (c = cpy_shutdown_callbacks; c; c = c->next) {
 352                 ret = PyObject_CallFunctionObjArgs(c->callback, c->data, (void *) 0); /* New reference. */
 353                 if (ret == NULL)
 354                         PyErr_Print(); /* FIXME */
 355                 else
 356                         Py_DECREF(ret);
 357         }
 358         Py_Finalize();
 359         return 0;
 360 }
 361
 362 static void *cpy_interactive(void *data) {
 363         CPY_LOCK_THREADS
 364                 if (PyImport_ImportModule("readline") == NULL) {
 365                         /* This interactive session will suck. */
 366                         PyErr_Print(); /* FIXME */
 367                 }
 368                 PyRun_InteractiveLoop(stdin, "<stdin>");
 369         CPY_RELEASE_THREADS
 370         NOTICE("python: Interactive interpreter exited, stopping collectd ...");
 371         raise(SIGINT);
 372         return NULL;
 373 }
 374
 375 static int cpy_init(void) {
 376         cpy_callback_t *c;
 377         PyObject *ret;
 378         static pthread_t thread;
 379
 380         PyEval_InitThreads();
 381         /* Now it's finally OK to use python threads. */
 382         for (c = cpy_init_callbacks; c; c = c->next) {
 383                 ret = PyObject_CallFunctionObjArgs(c->callback, c->data, (void *) 0); /* New reference. */
 384                 if (ret == NULL)
 385                         PyErr_Print(); /* FIXME */
 386                 else
 387                         Py_DECREF(ret);
 388         }
 389         state = PyEval_SaveThread();
 390         if (do_interactive) {
 391                 if (pthread_create(&thread, NULL, cpy_interactive, NULL)) {
 392                         ERROR("python: Error creating thread for interactive interpreter.");
 393                 }
 394         }
 395
 396         return 0;
 397 }
 398
 399 static PyObject *cpy_oconfig_to_pyconfig(oconfig_item_t *ci, PyObject *parent) {
 400         int i;
 401         PyObject *item, *values, *children, *tmp;
 402
 403         if (parent == NULL)
 404                 parent = Py_None;
 405
 406         values = PyTuple_New(ci->values_num); /* New reference. */
 407         for (i = 0; i < ci->values_num; ++i) {
 408                 if (ci->values[i].type == OCONFIG_TYPE_STRING) {
 409                         PyTuple_SET_ITEM(values, i, PyString_FromString(ci->values[i].value.string));
 410                 } else if (ci->values[i].type == OCONFIG_TYPE_NUMBER) {
 411                         PyTuple_SET_ITEM(values, i, PyFloat_FromDouble(ci->values[i].value.number));
 412                 } else if (ci->values[i].type == OCONFIG_TYPE_BOOLEAN) {
 413                         PyTuple_SET_ITEM(values, i, PyBool_FromLong(ci->values[i].value.boolean));
 414                 }
 415         }
 416
 417         item = PyObject_CallFunction((PyObject *) &ConfigType, "sONO", ci->key, parent, values, Py_None);
 418         if (item == NULL)
 419                 return NULL;
 420         children = PyTuple_New(ci->children_num); /* New reference. */
 421         for (i = 0; i < ci->children_num; ++i) {
 422                         PyTuple_SET_ITEM(children, i, cpy_oconfig_to_pyconfig(ci->children + i, item));
 423         }
 424         tmp = ((Config *) item)->children;
 425         ((Config *) item)->children = children;
 426         Py_XDECREF(tmp);
 427         return item;
 428 }
 429
 430 static int cpy_config(oconfig_item_t *ci) {
 431         int i;
 432         PyObject *sys;
 433         PyObject *sys_path;
 434         PyObject *module;
 435
 436         /* Ok in theory we shouldn't do initialization at this point
 437          * but we have to. In order to give python scripts a chance
 438          * to register a config callback we need to be able to execute
 439          * python code during the config callback so we have to start
 440          * the interpreter here. */
 441         /* Do *not* use the python "thread" module at this point! */
 442         Py_Initialize();
 443
 444         PyType_Ready(&ConfigType);
 445         PyType_Ready(&ValuesType);
 446         sys = PyImport_ImportModule("sys"); /* New reference. */
 447         if (sys == NULL) {
 448                 ERROR("python module: Unable to import \"sys\" module.");
 449                 /* Just print the default python exception text to stderr. */
 450                 PyErr_Print();
 451                 return 1;
 452         }
 453         sys_path = PyObject_GetAttrString(sys, "path"); /* New reference. */
 454         Py_DECREF(sys);
 455         if (sys_path == NULL) {
 456                 ERROR("python module: Unable to read \"sys.path\".");
 457                 PyErr_Print();
 458                 return 1;
 459         }
 460         module = Py_InitModule("collectd", cpy_methods); /* Borrowed reference. */
 461         PyModule_AddObject(module, "Config", (PyObject *) &ConfigType); /* Steals a reference. */
 462         PyModule_AddObject(module, "Values", (PyObject *) &ValuesType); /* Steals a reference. */
 463         PyModule_AddIntConstant(module, "LOG_DEBUG", LOG_DEBUG);
 464         PyModule_AddIntConstant(module, "LOG_INFO", LOG_INFO);
 465         PyModule_AddIntConstant(module, "LOG_NOTICE", LOG_NOTICE);
 466         PyModule_AddIntConstant(module, "LOG_WARNING", LOG_WARNING);
 467         PyModule_AddIntConstant(module, "LOG_ERROR", LOG_ERR);
 468         for (i = 0; i < ci->children_num; ++i) {
 469                 oconfig_item_t *item = ci->children + i;
 470
 471                 if (strcasecmp(item->key, "Interactive") == 0) {
 472                         if (item->values_num != 1 || item->values[0].type != OCONFIG_TYPE_BOOLEAN ||
 473                                         !item->values[0].value.boolean)
 474                                 continue;
 475                         do_interactive = 1;
 476                 } else if (strcasecmp(item->key, "ModulePath") == 0) {
 477                         char *dir = NULL;
 478                         PyObject *dir_object;
 479
 480                         if (cf_util_get_string(item, &dir) != 0)
 481                                 continue;
 482                         dir_object = PyString_FromString(dir); /* New reference. */
 483                         if (dir_object == NULL) {
 484                                 ERROR("python plugin: Unable to convert \"%s\" to "
 485                                       "a python object.", dir);
 486                                 free(dir);
 487                                 PyErr_Print();
 488                                 continue;
 489                         }
 490                         if (PyList_Append(sys_path, dir_object) != 0) {
 491                                 ERROR("python plugin: Unable to append \"%s\" to "
 492                                       "python module path.", dir);
 493                                 PyErr_Print();
 494                         }
 495                         Py_DECREF(dir_object);
 496                         free(dir);
 497                 } else if (strcasecmp(item->key, "Import") == 0) {
 498                         char *module_name = NULL;
 499                         PyObject *module;
 500
 501                         if (cf_util_get_string(item, &module_name) != 0)
 502                                 continue;
 503                         module = PyImport_ImportModule(module_name); /* New reference. */
 504                         if (module == NULL) {
 505                                 ERROR("python plugin: Error importing module \"%s\".", module_name);
 506                                 PyErr_Print();
 507                         }
 508                         free(module_name);
 509                         Py_XDECREF(module);
 510                 } else if (strcasecmp(item->key, "Module") == 0) {
 511                         char *name = NULL;
 512                         cpy_callback_t *c;
 513                         PyObject *ret;
 514
 515                         if (cf_util_get_string(item, &name) != 0)
 516                                 continue;
 517                         for (c = cpy_config_callbacks; c; c = c->next) {
 518                                 if (strcasecmp(c->name, name) == 0)
 519                                         break;
 520                         }
 521                         if (c == NULL) {
 522                                 WARNING("python plugin: Found a configuration for the \"%s\" plugin, "
 523                                         "but the plugin isn't loaded or didn't register "
 524                                         "a configuration callback.", name);
 525                                 free(name);
 526                                 continue;
 527                         }
 528                         free(name);
 529                         if (c->data == NULL)
 530                                 ret = PyObject_CallFunction(c->callback, "N",
 531                                         cpy_oconfig_to_pyconfig(item, NULL)); /* New reference. */
 532                         else
 533                                 ret = PyObject_CallFunction(c->callback, "NO",
 534                                         cpy_oconfig_to_pyconfig(item, NULL), c->data); /* New reference. */
 535                         if (ret == NULL)
 536                                 PyErr_Print();
 537                         else
 538                                 Py_DECREF(ret);
 539                 } else {
 540                         WARNING("python plugin: Ignoring unknown config key \"%s\".", item->key);
 541                 }
 542         }
 543         Py_DECREF(sys_path);
 544         return 0;
 545 }
 546
 547 void module_register(void) {
 548         plugin_register_complex_config("python", cpy_config);
 549         plugin_register_init("python", cpy_init);
 550 //      plugin_register_read("python", cna_read);
 551         plugin_register_shutdown("python", cpy_shutdown);
 552 }