From a5a33953ba9c8ca6a17531df5fa17181d59467c5 Mon Sep 17 00:00:00 2001 From: Sebastian Harl Date: Sat, 18 Apr 2015 19:36:08 +0200 Subject: [PATCH] Add request specific statistics to all CURL-based plugins. All metrics supported by curl_easy_getinfo can be retrieved and dispatched for each request done through CURL. All statistics are optional and disabled by default. --- src/Makefile.am | 9 +- src/curl.c | 18 +++- src/curl_json.c | 30 ++++-- src/curl_xml.c | 22 ++++- src/utils_curl_stats.c | 245 +++++++++++++++++++++++++++++++++++++++++++++++++ src/utils_curl_stats.h | 59 ++++++++++++ 6 files changed, 366 insertions(+), 17 deletions(-) create mode 100644 src/utils_curl_stats.c create mode 100644 src/utils_curl_stats.h diff --git a/src/Makefile.am b/src/Makefile.am index c7a60471..641e2fae 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -287,7 +287,8 @@ endif if BUILD_PLUGIN_CURL pkglib_LTLIBRARIES += curl.la -curl_la_SOURCES = curl.c +curl_la_SOURCES = curl.c \ + utils_curl_stats.c utils_curl_stats.h curl_la_LDFLAGS = $(PLUGIN_LDFLAGS) curl_la_CFLAGS = $(AM_CFLAGS) $(BUILD_WITH_LIBCURL_CFLAGS) curl_la_LIBADD = $(BUILD_WITH_LIBCURL_LIBS) @@ -295,7 +296,8 @@ endif if BUILD_PLUGIN_CURL_JSON pkglib_LTLIBRARIES += curl_json.la -curl_json_la_SOURCES = curl_json.c +curl_json_la_SOURCES = curl_json.c \ + utils_curl_stats.c utils_curl_stats.h curl_json_la_CFLAGS = $(AM_CFLAGS) $(BUILD_WITH_LIBCURL_CFLAGS) curl_json_la_CPPFLAGS = $(AM_CPPFLAGS) $(BUILD_WITH_LIBYAJL_CPPFLAGS) curl_json_la_LDFLAGS = $(PLUGIN_LDFLAGS) $(BUILD_WITH_LIBYAJL_LDFLAGS) @@ -304,7 +306,8 @@ endif if BUILD_PLUGIN_CURL_XML pkglib_LTLIBRARIES += curl_xml.la -curl_xml_la_SOURCES = curl_xml.c +curl_xml_la_SOURCES = curl_xml.c \ + utils_curl_stats.c utils_curl_stats.h curl_xml_la_LDFLAGS = $(PLUGIN_LDFLAGS) curl_xml_la_CFLAGS = $(AM_CFLAGS) \ $(BUILD_WITH_LIBCURL_CFLAGS) $(BUILD_WITH_LIBXML2_CFLAGS) diff --git a/src/curl.c b/src/curl.c index 7f298a26..6377780c 100644 --- a/src/curl.c +++ b/src/curl.c @@ -25,6 +25,7 @@ #include "common.h" #include "plugin.h" #include "configfile.h" +#include "utils_curl_stats.h" #include "utils_match.h" #include "utils_time.h" @@ -67,6 +68,7 @@ struct web_page_s /* {{{ */ _Bool response_time; _Bool response_code; int timeout; + curl_stats_t *stats; CURL *curl; char curl_errbuf[CURL_ERROR_SIZE]; @@ -156,6 +158,7 @@ static void cc_web_page_free (web_page_t *wp) /* {{{ */ sfree (wp->cacert); sfree (wp->post_body); curl_slist_free_all (wp->headers); + curl_stats_destroy (wp->stats); sfree (wp->buffer); @@ -453,6 +456,7 @@ static int cc_config_add_page (oconfig_item_t *ci) /* {{{ */ page->response_time = 0; page->response_code = 0; page->timeout = -1; + page->stats = NULL; page->instance = strdup (ci->values[0].value.string); if (page->instance == NULL) @@ -495,6 +499,11 @@ static int cc_config_add_page (oconfig_item_t *ci) /* {{{ */ status = cf_util_get_string (child, &page->post_body); else if (strcasecmp ("Timeout", child->key) == 0) status = cf_util_get_int (child, &page->timeout); + else if (strcasecmp ("Statistics", child->key) == 0) { + page->stats = curl_stats_from_config (child); + if (page->stats == NULL) + status = -1; + } else { WARNING ("curl plugin: Option `%s' not allowed here.", child->key); @@ -514,12 +523,13 @@ static int cc_config_add_page (oconfig_item_t *ci) /* {{{ */ status = -1; } - if (page->matches == NULL && !page->response_time && !page->response_code) + if (page->matches == NULL && page->stats == NULL + && !page->response_time && !page->response_code) { assert (page->instance != NULL); WARNING ("curl plugin: No (valid) `Match' block " - "or MeasureResponseTime or MeasureResponseCode within " - "`Page' block `%s'.", page->instance); + "or Statistics or MeasureResponseTime or MeasureResponseCode " + "within `Page' block `%s'.", page->instance); status = -1; } @@ -675,6 +685,8 @@ static int cc_read_page (web_page_t *wp) /* {{{ */ if (wp->response_time) cc_submit_response_time (wp, cdtime() - start); + if (wp->stats != NULL) + curl_stats_dispatch (wp->stats, wp->curl, hostname_g, "curl", wp->instance, NULL); if(wp->response_code) { diff --git a/src/curl_json.c b/src/curl_json.c index b19730b8..1cf6381a 100644 --- a/src/curl_json.c +++ b/src/curl_json.c @@ -27,6 +27,7 @@ #include "configfile.h" #include "utils_avltree.h" #include "utils_complain.h" +#include "utils_curl_stats.h" #include #include @@ -78,6 +79,7 @@ struct cj_s /* {{{ */ char *post_body; cdtime_t interval; int timeout; + curl_stats_t *stats; CURL *curl; char curl_errbuf[CURL_ERROR_SIZE]; @@ -432,6 +434,7 @@ static void cj_free (void *arg) /* {{{ */ sfree (db->cacert); sfree (db->post_body); curl_slist_free_all (db->headers); + curl_stats_destroy (db->stats); sfree (db); } /* }}} void cj_free */ @@ -726,6 +729,12 @@ static int cj_config_add_url (oconfig_item_t *ci) /* {{{ */ status = cf_util_get_cdtime(child, &db->interval); else if (strcasecmp ("Timeout", child->key) == 0) status = cf_util_get_int (child, &db->timeout); + else if (strcasecmp ("Statistics", child->key) == 0) + { + db->stats = curl_stats_from_config (child); + if (db->stats == NULL) + status = -1; + } else { WARNING ("curl_json plugin: Option `%s' not allowed here.", child->key); @@ -823,21 +832,22 @@ static int cj_config (oconfig_item_t *ci) /* {{{ */ /* }}} End of configuration handling functions */ +static const char *cj_host (cj_t *db) /* {{{ */ +{ + if ((db->host == NULL) + || (strcmp ("", db->host) == 0) + || (strcmp (CJ_DEFAULT_HOST, db->host) == 0)) + return hostname_g; + return db->host; +} /* }}} cj_host */ + static void cj_submit (cj_t *db, cj_key_t *key, value_t *value) /* {{{ */ { value_list_t vl = VALUE_LIST_INIT; - char *host; vl.values = value; vl.values_len = 1; - if ((db->host == NULL) - || (strcmp ("", db->host) == 0) - || (strcmp (CJ_DEFAULT_HOST, db->host) == 0)) - host = hostname_g; - else - host = db->host; - if (key->instance == NULL) { int i, len = 0; @@ -848,7 +858,7 @@ static void cj_submit (cj_t *db, cj_key_t *key, value_t *value) /* {{{ */ else sstrncpy (vl.type_instance, key->instance, sizeof (vl.type_instance)); - sstrncpy (vl.host, host, sizeof (vl.host)); + sstrncpy (vl.host, cj_host (db), sizeof (vl.host)); sstrncpy (vl.plugin, "curl_json", sizeof (vl.plugin)); sstrncpy (vl.plugin_instance, db->instance, sizeof (vl.plugin_instance)); sstrncpy (vl.type, key->type, sizeof (vl.type)); @@ -911,6 +921,8 @@ static int cj_curl_perform(cj_t *db) /* {{{ */ status, db->curl_errbuf, url); return (-1); } + if (db->stats != NULL) + curl_stats_dispatch (db->stats, db->curl, cj_host (db), "curl_json", db->instance, NULL); curl_easy_getinfo(db->curl, CURLINFO_EFFECTIVE_URL, &url); curl_easy_getinfo(db->curl, CURLINFO_RESPONSE_CODE, &rc); diff --git a/src/curl_xml.c b/src/curl_xml.c index 0f2b92b0..21e09252 100644 --- a/src/curl_xml.c +++ b/src/curl_xml.c @@ -23,6 +23,7 @@ #include "common.h" #include "plugin.h" #include "configfile.h" +#include "utils_curl_stats.h" #include "utils_llist.h" #include @@ -83,6 +84,7 @@ struct cx_s /* {{{ */ char *post_body; int timeout; struct curl_slist *headers; + curl_stats_t *stats; cx_namespace_t *namespaces; size_t namespaces_num; @@ -202,6 +204,7 @@ static void cx_free (void *arg) /* {{{ */ sfree (db->cacert); sfree (db->post_body); curl_slist_free_all (db->headers); + curl_stats_destroy (db->stats); for (i = 0; i < db->namespaces_num; i++) { @@ -213,6 +216,13 @@ static void cx_free (void *arg) /* {{{ */ sfree (db); } /* }}} void cx_free */ +static const char *cx_host (cx_t *db) /* {{{ */ +{ + if (db->host == NULL) + return hostname_g; + return db->host; +} /* }}} cx_host */ + static int cx_config_append_string (const char *name, struct curl_slist **dest, /* {{{ */ oconfig_item_t *ci) { @@ -515,7 +525,7 @@ static int cx_handle_base_xpath (char const *plugin_instance, /* {{{ */ vl.values_len = ds->ds_num; sstrncpy (vl.type, xpath->type, sizeof (vl.type)); sstrncpy (vl.plugin, "curl_xml", sizeof (vl.plugin)); - sstrncpy (vl.host, (host != NULL) ? host : hostname_g, sizeof (vl.host)); + sstrncpy (vl.host, host, sizeof (vl.host)); if (plugin_instance != NULL) sstrncpy (vl.plugin_instance, plugin_instance, sizeof (vl.plugin_instance)); @@ -558,7 +568,7 @@ static int cx_handle_parsed_xml(xmlDocPtr doc, /* {{{ */ ds = plugin_get_ds (xpath->type); if ( (cx_check_type(ds, xpath) == 0) && - (cx_handle_base_xpath(db->instance, db->host, + (cx_handle_base_xpath(db->instance, cx_host (db), xpath_ctx, ds, le->key, xpath) == 0) ) status = 0; /* we got atleast one success */ @@ -630,6 +640,8 @@ static int cx_curl_perform (cx_t *db, CURL *curl) /* {{{ */ status, db->curl_errbuf, url); return (-1); } + if (db->stats != NULL) + curl_stats_dispatch (db->stats, db->curl, cx_host (db), "curl_xml", db->instance, NULL); curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &rc); @@ -981,6 +993,12 @@ static int cx_config_add_url (oconfig_item_t *ci) /* {{{ */ status = cx_config_add_namespace (db, child); else if (strcasecmp ("Timeout", child->key) == 0) status = cf_util_get_int (child, &db->timeout); + else if (strcasecmp ("Statistics", child->key) == 0) + { + db->stats = curl_stats_from_config (child); + if (db->stats == NULL) + status = -1; + } else { WARNING ("curl_xml plugin: Option `%s' not allowed here.", child->key); diff --git a/src/utils_curl_stats.c b/src/utils_curl_stats.c new file mode 100644 index 00000000..6e98bcfd --- /dev/null +++ b/src/utils_curl_stats.c @@ -0,0 +1,245 @@ +/** + * collectd - src/utils_curl_stats.c + * Copyright (C) 2015 Sebastian 'tokkee' Harl + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Sebastian Harl + **/ + +#include "collectd.h" +#include "common.h" +#include "utils_curl_stats.h" + +#include +#include + +struct curl_stats_s +{ + bool total_time; + bool namelookup_time; + bool connect_time; + bool pretransfer_time; + bool size_upload; + bool size_download; + bool speed_download; + bool speed_upload; + bool header_size; + bool request_size; + bool content_length_download; + bool content_length_upload; + bool starttransfer_time; + bool redirect_time; + bool redirect_count; + bool num_connects; + bool appconnect_time; +}; + +/* + * Private functions + */ + +static int dispatch_gauge (CURL *curl, CURLINFO info, value_list_t *vl) +{ + CURLcode code; + value_t v; + + code = curl_easy_getinfo (curl, info, &v.gauge); + if (code != CURLE_OK) + return -1; + + vl->values = &v; + vl->values_len = 1; + + return plugin_dispatch_values (vl); +} /* dispatch_gauge */ + +/* dispatch a speed, in bytes/second */ +static int dispatch_speed (CURL *curl, CURLINFO info, value_list_t *vl) +{ + CURLcode code; + value_t v; + + code = curl_easy_getinfo (curl, info, &v.gauge); + if (code != CURLE_OK) + return -1; + + v.gauge *= 8; + + vl->values = &v; + vl->values_len = 1; + + return plugin_dispatch_values (vl); +} /* dispatch_speed */ + +/* dispatch a size/count, reported as a long value */ +static int dispatch_size (CURL *curl, CURLINFO info, value_list_t *vl) +{ + CURLcode code; + value_t v; + long raw; + + code = curl_easy_getinfo (curl, info, &raw); + if (code != CURLE_OK) + return -1; + + v.gauge = (double)raw; + + vl->values = &v; + vl->values_len = 1; + + return plugin_dispatch_values (vl); +} /* dispatch_size */ + +static struct { + const char *name; + size_t offset; + + int (*dispatcher)(CURL *, CURLINFO, value_list_t *); + const char *type; + CURLINFO info; +} field_specs[] = { +#define SPEC(name, dispatcher, type, info) \ + { #name, offsetof (curl_stats_t, name), dispatcher, type, info } + + SPEC (total_time, dispatch_gauge, "duration", CURLINFO_TOTAL_TIME), + SPEC (namelookup_time, dispatch_gauge, "duration", CURLINFO_NAMELOOKUP_TIME), + SPEC (connect_time, dispatch_gauge, "duration", CURLINFO_CONNECT_TIME), + SPEC (pretransfer_time, dispatch_gauge, "duration", CURLINFO_PRETRANSFER_TIME), + SPEC (size_upload, dispatch_gauge, "bytes", CURLINFO_SIZE_UPLOAD), + SPEC (size_download, dispatch_gauge, "bytes", CURLINFO_SIZE_DOWNLOAD), + SPEC (speed_download, dispatch_speed, "bitrate", CURLINFO_SPEED_DOWNLOAD), + SPEC (speed_upload, dispatch_speed, "bitrate", CURLINFO_SPEED_UPLOAD), + SPEC (header_size, dispatch_size, "bytes", CURLINFO_HEADER_SIZE), + SPEC (request_size, dispatch_size, "bytes", CURLINFO_REQUEST_SIZE), + SPEC (content_length_download, dispatch_gauge, "bytes", CURLINFO_CONTENT_LENGTH_DOWNLOAD), + SPEC (content_length_upload, dispatch_gauge, "bytes", CURLINFO_CONTENT_LENGTH_UPLOAD), + SPEC (starttransfer_time, dispatch_gauge, "duration", CURLINFO_STARTTRANSFER_TIME), + SPEC (redirect_time, dispatch_gauge, "duration", CURLINFO_REDIRECT_TIME), + SPEC (redirect_count, dispatch_size, "count", CURLINFO_REDIRECT_COUNT), + SPEC (num_connects, dispatch_size, "count", CURLINFO_NUM_CONNECTS), + SPEC (appconnect_time, dispatch_gauge, "duration", CURLINFO_APPCONNECT_TIME), + +#undef SPEC +}; + +static void enable_field (curl_stats_t *s, size_t offset) +{ + *(bool *)((char *)s + offset) = true; +} /* enable_field */ + +static bool field_enabled (curl_stats_t *s, size_t offset) +{ + return *(bool *)((char *)s + offset); +} /* field_enabled */ + +/* + * Public API + */ +curl_stats_t *curl_stats_from_config (oconfig_item_t *ci) +{ + curl_stats_t *s; + int i; + + if (ci == NULL) + return NULL; + + s = calloc (sizeof (*s), 1); + if (s == NULL) + return NULL; + + for (i = 0; i < ci->children_num; ++i) + { + oconfig_item_t *c = ci->children + i; + size_t field; + + for (field = 0; field < STATIC_ARRAY_SIZE (field_specs); ++field) + if (! strcasecmp (c->key, field_specs[field].name)) + break; + if (field >= STATIC_ARRAY_SIZE (field_specs)) + { + ERROR ("curl stats: Unknown field name %s", c->key); + free (s); + return NULL; + } + + if ((c->values_num != 1) + || ((c->values[0].type != OCONFIG_TYPE_STRING) + && (c->values[0].type != OCONFIG_TYPE_BOOLEAN))) { + ERROR ("curl stats: `%s' expects a single boolean argument", c->key); + free (s); + return NULL; + } + + if (((c->values[0].type == OCONFIG_TYPE_STRING) + && IS_TRUE (c->values[0].value.string)) + || ((c->values[0].type == OCONFIG_TYPE_BOOLEAN) + && c->values[0].value.boolean)) + enable_field (s, field_specs[field].offset); + } + + return s; +} /* curl_stats_from_config */ + +void curl_stats_destroy (curl_stats_t *s) +{ + if (s != NULL) + free (s); +} /* curl_stats_destroy */ + +int curl_stats_dispatch (curl_stats_t *s, CURL *curl, + const char *hostname, const char *plugin, const char *plugin_instance, + const char *instance_prefix) +{ + value_list_t vl = VALUE_LIST_INIT; + size_t field; + + if (s == NULL) + return 0; + if (curl == NULL) + return -1; + + if (hostname != NULL) + sstrncpy (vl.host, hostname, sizeof (vl.host)); + if (plugin != NULL) + sstrncpy (vl.plugin, plugin, sizeof (vl.plugin)); + if (plugin_instance != NULL) + sstrncpy (vl.plugin_instance, plugin_instance, sizeof (vl.plugin_instance)); + + for (field = 0; field < STATIC_ARRAY_SIZE (field_specs); ++field) + { + int status; + + if (! field_enabled (s, field_specs[field].offset)) + continue; + + sstrncpy (vl.type, field_specs[field].type, sizeof (vl.type)); + ssnprintf (vl.type_instance, sizeof (vl.type_instance), "%s%s", + instance_prefix ? instance_prefix : "", field_specs[field].name); + + vl.values = NULL; + vl.values_len = 0; + status = field_specs[field].dispatcher (curl, field_specs[field].info, &vl); + if (status < 0) + return status; + } + + return 0; +} /* curl_stats_dispatch */ diff --git a/src/utils_curl_stats.h b/src/utils_curl_stats.h new file mode 100644 index 00000000..3a0a26eb --- /dev/null +++ b/src/utils_curl_stats.h @@ -0,0 +1,59 @@ +/** + * collectd - src/utils_curl_stats.h + * Copyright (C) 2015 Sebastian 'tokkee' Harl + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Sebastian Harl + **/ + +#ifndef UTILS_CURL_STATS_H +#define UTILS_CURL_STATS_H 1 + +#include "configfile.h" +#include "plugin.h" + +#include + +struct curl_stats_s; +typedef struct curl_stats_s curl_stats_t; + +/* + * curl_stats_from_config allocates and constructs a CURL statistics object + * from the specified configuration which is expected to be a single block of + * boolean options named after CURL information fields. The boolean value + * indicates whether to collect the respective information. + * + * See http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html + */ +__attribute__((nonnull(1))) +curl_stats_t *curl_stats_from_config (oconfig_item_t *ci); + +void curl_stats_destroy (curl_stats_t *s); + +/* + * curl_stats_dispatch dispatches performance values from the the specified + * CURL session to the daemon. + */ +int curl_stats_dispatch (curl_stats_t *s, CURL *curl, + const char *hostname, const char *plugin, const char *plugin_instance, + const char *instance_prefix); + +#endif /* UTILS_CURL_STATS_H */ -- 2.11.0