From 25ac639c505394e4ae9600ee62f5d5aeea97c6d4 Mon Sep 17 00:00:00 2001 From: Mariusz Gronczewski Date: Wed, 1 Jul 2009 09:14:29 +0200 Subject: [PATCH] Introduce the DERIVE and ABSOLUTE data source types. Hi, i've updated my patch to 4.7.0, most of "data input" plugins (curl, java, exec, perl, tail, couchdb) should work with derive. In case of couchdb and curl, if u use absolute DS you can only "Set", no "Inc" or "Add" coz obviously that wouldn't make much sense with it. Other plugins can be "enabled" globally to use derive by changing "COUNTER" to "DERIVE" in types.db but that way is ugly (but makes sense in some cases, like when u have lot of tunnels or ppp interfaces) and either needs converting or recreating rrd files. Regards Mariusz --- Hi, ive been running my patch with 4.7.1, found a minor bug, but after repairing that i didnt had any problems with it on my servers, im including patch (against 4.7.1 from webpage), Regards, XANi --- src/common.c | 4 ++++ src/csv.c | 54 ++++++++++++++++++++++++++++++++++++++++-- src/curl.c | 23 ++++++++++++++++++ src/java.c | 4 ++++ src/libcollectdclient/client.c | 5 ++++ src/libcollectdclient/client.h | 6 +++++ src/perl.c | 16 ++++++++++--- src/plugin.h | 6 +++++ src/rrdcached.c | 15 ++++++++++-- src/rrdtool.c | 12 ++++++++-- src/tail.c | 20 ++++++++++++++++ src/types.db | 2 ++ src/types_list.c | 6 ++++- src/utils_match.c | 53 +++++++++++++++++++++++++++++++++++++++++ src/utils_match.h | 10 ++++++++ src/utils_rrdcreate.c | 4 ++++ 16 files changed, 230 insertions(+), 10 deletions(-) diff --git a/src/common.c b/src/common.c index d42453e4..8de871bd 100644 --- a/src/common.c +++ b/src/common.c @@ -845,6 +845,10 @@ int parse_value (const char *value, value_t *ret_value, const data_source_t ds) ret_value->counter = (counter_t)strtoll (value, &endptr, 0); else if (DS_TYPE_GAUGE == ds.type) ret_value->gauge = (gauge_t)strtod (value, &endptr); + else if (DS_TYPE_DERIVE == ds.type) + ret_value->counter = (derive_t)strtoll (value, &endptr, 0); + else if (DS_TYPE_ABSOLUTE == ds.type) + ret_value->counter = (absolute_t)strtoll (value, &endptr, 0); else { ERROR ("parse_value: Invalid data source \"%s\" " "(type = %i).", ds.name, ds.type); diff --git a/src/csv.c b/src/csv.c index 5c43b8a8..f38599dc 100644 --- a/src/csv.c +++ b/src/csv.c @@ -61,7 +61,9 @@ static int value_list_to_string (char *buffer, int buffer_len, for (i = 0; i < ds->ds_num; i++) { if ((ds->ds[i].type != DS_TYPE_COUNTER) - && (ds->ds[i].type != DS_TYPE_GAUGE)) + && (ds->ds[i].type != DS_TYPE_GAUGE) + && (ds->ds[i].type != DS_TYPE_DERIVE) + && (ds->ds[i].type != DS_TYPE_ABSOLUTE)) return (-1); if (ds->ds[i].type == DS_TYPE_COUNTER) @@ -88,10 +90,58 @@ static int value_list_to_string (char *buffer, int buffer_len, ",%lf", rates[i]); } } - else /* if (ds->ds[i].type == DS_TYPE_GAUGE) */ + else if (ds->ds[i].type == DS_TYPE_GAUGE) { status = ssnprintf (buffer + offset, buffer_len - offset, ",%lf", vl->values[i].gauge); + } + else if (ds->ds[i].type == DS_TYPE_DERIVE) + { + if (store_rates == 0) + { + status = ssnprintf (buffer + offset, + buffer_len - offset, + ",%llu", + vl->values[i].derive); + } + else /* if (store_rates == 1) */ + { + if (rates == NULL) + rates = uc_get_rate (ds, vl); + if (rates == NULL) + { + WARNING ("csv plugin: " + "uc_get_rate failed."); + return (-1); + } + status = ssnprintf (buffer + offset, + buffer_len - offset, + ",%lf", rates[i]); + } + } + else if (ds->ds[i].type == DS_TYPE_ABSOLUTE) + { + if (store_rates == 0) + { + status = ssnprintf (buffer + offset, + buffer_len - offset, + ",%llu", + vl->values[i].absolute); + } + else /* if (store_rates == 1) */ + { + if (rates == NULL) + rates = uc_get_rate (ds, vl); + if (rates == NULL) + { + WARNING ("csv plugin: " + "uc_get_rate failed."); + return (-1); + } + status = ssnprintf (buffer + offset, + buffer_len - offset, + ",%lf", rates[i]); + } } if ((status < 1) || (status >= (buffer_len - offset))) diff --git a/src/curl.c b/src/curl.c index 88b5496d..41f11f61 100644 --- a/src/curl.c +++ b/src/curl.c @@ -222,6 +222,29 @@ static int cc_config_add_match_dstype (int *dstype_ret, /* {{{ */ else dstype = 0; } +else if (strncasecmp ("Derive", ci->values[0].value.string, + strlen ("Derive")) == 0) + { + dstype = UTILS_MATCH_DS_TYPE_DERIVE; + if (strcasecmp ("DeriveSet", ci->values[0].value.string) == 0) + dstype |= UTILS_MATCH_CF_DERIVE_SET; + else if (strcasecmp ("DeriveAdd", ci->values[0].value.string) == 0) + dstype |= UTILS_MATCH_CF_DERIVE_ADD; + else if (strcasecmp ("DeriveInc", ci->values[0].value.string) == 0) + dstype |= UTILS_MATCH_CF_DERIVE_INC; + else + dstype = 0; + } +else if (strncasecmp ("Absolute", ci->values[0].value.string, + strlen ("Absolute")) == 0) + { + dstype = UTILS_MATCH_DS_TYPE_ABSOLUTE; + if (strcasecmp ("AbsoluteSet", ci->values[0].value.string) == 0) /* Absolute DS is reset-on-read so no sense doin anything else but set */ + dstype |= UTILS_MATCH_CF_ABSOLUTE_SET; + else + dstype = 0; + } + else { dstype = 0; diff --git a/src/java.c b/src/java.c index 20523f90..64e51d31 100644 --- a/src/java.c +++ b/src/java.c @@ -284,6 +284,10 @@ static jobject ctoj_value_to_number (JNIEnv *jvm_env, /* {{{ */ return (ctoj_jlong_to_number (jvm_env, (jlong) value.counter)); else if (ds_type == DS_TYPE_GAUGE) return (ctoj_jdouble_to_number (jvm_env, (jdouble) value.gauge)); + if (ds_type == DS_TYPE_DERIVE) + return (ctoj_jlong_to_number (jvm_env, (jlong) value.derive)); + if (ds_type == DS_TYPE_ABSOLUTE) + return (ctoj_jlong_to_number (jvm_env, (jlong) value.absolute)); else return (NULL); } /* }}} jobject ctoj_value_to_number */ diff --git a/src/libcollectdclient/client.c b/src/libcollectdclient/client.c index 501c729c..bb7ee956 100644 --- a/src/libcollectdclient/client.c +++ b/src/libcollectdclient/client.c @@ -792,6 +792,11 @@ int lcc_putval (lcc_connection_t *c, const lcc_value_list_t *vl) /* {{{ */ else SSTRCATF (command, ":%g", vl->values[i].gauge); } + else if (vl->values_types[i] == LCC_TYPE_DERIVE) + SSTRCATF (command, ":%"PRIu64, vl->values[i].derive); + else if (vl->values_types[i] == LCC_TYPE_ABSOLUTE) + SSTRCATF (command, ":%"PRIu64, vl->values[i].absolute); + } /* for (i = 0; i < vl->values_len; i++) */ status = lcc_sendreceive (c, command, &res); diff --git a/src/libcollectdclient/client.h b/src/libcollectdclient/client.h index d5371fb2..b0d092d0 100644 --- a/src/libcollectdclient/client.h +++ b/src/libcollectdclient/client.h @@ -42,16 +42,22 @@ */ #define LCC_TYPE_COUNTER 0 #define LCC_TYPE_GAUGE 1 +#define LCC_TYPE_DERIVE 2 +#define LCC_TYPE_ABSOLUTE 3 LCC_BEGIN_DECLS typedef uint64_t counter_t; typedef double gauge_t; +typedef uint64_t derive_t; +typedef uint64_t absolute_t; union value_u { counter_t counter; gauge_t gauge; + derive_t derive; + absolute_t absolute; }; typedef union value_u value_t; diff --git a/src/perl.c b/src/perl.c index b6e7b22d..dd82ed94 100644 --- a/src/perl.c +++ b/src/perl.c @@ -194,6 +194,8 @@ struct { { "Collectd::TYPE_DATASET", PLUGIN_DATASET }, { "Collectd::DS_TYPE_COUNTER", DS_TYPE_COUNTER }, { "Collectd::DS_TYPE_GAUGE", DS_TYPE_GAUGE }, + { "Collectd::DS_TYPE_DERIVE", DS_TYPE_DERIVE }, + { "Collectd::DS_TYPE_ABSOLUTE", DS_TYPE_ABSOLUTE }, { "Collectd::LOG_ERR", LOG_ERR }, { "Collectd::LOG_WARNING", LOG_WARNING }, { "Collectd::LOG_NOTICE", LOG_NOTICE }, @@ -267,7 +269,7 @@ static int hv2data_source (pTHX_ HV *hash, data_source_t *ds) if (NULL != (tmp = hv_fetch (hash, "type", 4, 0))) { ds->type = SvIV (*tmp); - if ((DS_TYPE_COUNTER != ds->type) && (DS_TYPE_GAUGE != ds->type)) { + if ((DS_TYPE_COUNTER != ds->type) && (DS_TYPE_GAUGE != ds->type) && (DS_TYPE_DERIVE != ds->type) && (DS_TYPE_ABSOLUTE != ds->type)) { log_err ("hv2data_source: Invalid DS type."); return -1; } @@ -320,8 +322,12 @@ static int av2value (pTHX_ char *name, AV *array, value_t *value, int len) if (NULL != tmp) { if (DS_TYPE_COUNTER == ds->ds[i].type) value[i].counter = SvIV (*tmp); - else + else if (DS_TYPE_GAUGE == ds->ds[i].type) value[i].gauge = SvNV (*tmp); + else if (DS_TYPE_DERIVE == ds->ds[i].type) + value[i].derive = SvIV (*tmp); + else if (DS_TYPE_ABSOLUTE == ds->ds[i].type) + value[i].absolute = SvIV (*tmp); } else { return -1; @@ -637,8 +643,12 @@ static int value_list2hv (pTHX_ value_list_t *vl, data_set_t *ds, HV *hash) if (DS_TYPE_COUNTER == ds->ds[i].type) val = newSViv (vl->values[i].counter); - else + else if (DS_TYPE_GAUGE == ds->ds[i].type) val = newSVnv (vl->values[i].gauge); + else if (DS_TYPE_DERIVE == ds->ds[i].type) + val = newSVnv (vl->values[i].derive); + else if (DS_TYPE_ABSOLUTE == ds->ds[i].type) + val = newSVnv (vl->values[i].absolute); if (NULL == av_store (values, i, val)) { av_undef (values); diff --git a/src/plugin.h b/src/plugin.h index e8e49567..fd741501 100644 --- a/src/plugin.h +++ b/src/plugin.h @@ -30,6 +30,8 @@ #define DS_TYPE_COUNTER 0 #define DS_TYPE_GAUGE 1 +#define DS_TYPE_DERIVE 2 +#define DS_TYPE_ABSOLUTE 3 #ifndef LOG_ERR # define LOG_ERR 3 @@ -58,11 +60,15 @@ */ typedef unsigned long long counter_t; typedef double gauge_t; +typedef unsigned long long derive_t; +typedef unsigned long long absolute_t; union value_u { counter_t counter; gauge_t gauge; + derive_t derive; + absolute_t absolute; }; typedef union value_u value_t; diff --git a/src/rrdcached.c b/src/rrdcached.c index 326a8898..6295a53f 100644 --- a/src/rrdcached.c +++ b/src/rrdcached.c @@ -75,7 +75,9 @@ static int value_list_to_string (char *buffer, int buffer_len, for (i = 0; i < ds->ds_num; i++) { if ((ds->ds[i].type != DS_TYPE_COUNTER) - && (ds->ds[i].type != DS_TYPE_GAUGE)) + && (ds->ds[i].type != DS_TYPE_GAUGE) + && (ds->ds[i].type != DS_TYPE_DERIVE) + && (ds->ds[i].type != DS_TYPE_ABSOLUTE) return (-1); if (ds->ds[i].type == DS_TYPE_COUNTER) @@ -83,11 +85,20 @@ static int value_list_to_string (char *buffer, int buffer_len, status = ssnprintf (buffer + offset, buffer_len - offset, ":%llu", vl->values[i].counter); } - else /* if (ds->ds[i].type == DS_TYPE_GAUGE) */ + else if (ds->ds[i].type == DS_TYPE_GAUGE) { status = ssnprintf (buffer + offset, buffer_len - offset, ":%lf", vl->values[i].gauge); } + else if (ds->ds[i].type == DS_TYPE_DERIVE) { + status = ssnprintf (buffer + offset, buffer_len - offset, + ":%llu", vl->values[i].derive); + } + else /* if (ds->ds[i].type == DS_TYPE_ABSOLUTE) */ { + status = ssnprintf (buffer + offset, buffer_len - offset, + ":%llu", vl->values[i].absolute); + + } if ((status < 1) || (status >= (buffer_len - offset))) return (-1); diff --git a/src/rrdtool.c b/src/rrdtool.c index 243a8c8f..a361501d 100644 --- a/src/rrdtool.c +++ b/src/rrdtool.c @@ -204,15 +204,23 @@ static int value_list_to_string (char *buffer, int buffer_len, for (i = 0; i < ds->ds_num; i++) { if ((ds->ds[i].type != DS_TYPE_COUNTER) - && (ds->ds[i].type != DS_TYPE_GAUGE)) + && (ds->ds[i].type != DS_TYPE_GAUGE) + && (ds->ds[i].type != DS_TYPE_DERIVE) + && (ds->ds[i].type != DS_TYPE_ABSOLUTE)) return (-1); if (ds->ds[i].type == DS_TYPE_COUNTER) status = ssnprintf (buffer + offset, buffer_len - offset, ":%llu", vl->values[i].counter); - else + else if (ds->ds[i].type == DS_TYPE_GAUGE) status = ssnprintf (buffer + offset, buffer_len - offset, ":%lf", vl->values[i].gauge); + else if (ds->ds[i].type == DS_TYPE_DERIVE) + status = ssnprintf (buffer + offset, buffer_len - offset, + ":%llu", vl->values[i].derive); + else /*if (ds->ds[i].type == DS_TYPE_ABSOLUTE) */ + status = ssnprintf (buffer + offset, buffer_len - offset, + ":%llu", vl->values[i].absolute); if ((status < 1) || (status >= (buffer_len - offset))) return (-1); diff --git a/src/tail.c b/src/tail.c index 02afd791..8becc05b 100644 --- a/src/tail.c +++ b/src/tail.c @@ -101,6 +101,26 @@ static int ctail_config_add_match_dstype (ctail_config_match_t *cm, else cm->flags = 0; } + else if (strncasecmp ("Derive", ci->values[0].value.string, strlen ("Derive")) == 0) + { + cm->flags = UTILS_MATCH_DS_TYPE_DERIVE; + if (strcasecmp ("DeriveSet", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_DERIVE_SET; + else if (strcasecmp ("DeriveAdd", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_DERIVE_ADD; + else if (strcasecmp ("DeriveInc", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_DERIVE_INC; + else + cm->flags = 0; + } + else if (strncasecmp ("Absolute", ci->values[0].value.string, strlen ("Absolute")) == 0) + { + cm->flags = UTILS_MATCH_DS_TYPE_ABSOLUTE; + if (strcasecmp ("AbsoluteSet", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_ABSOLUTE_SET; + else + cm->flags = 0; + } else { cm->flags = 0; diff --git a/src/types.db b/src/types.db index 7028fe7f..83e7007d 100644 --- a/src/types.db +++ b/src/types.db @@ -1,3 +1,4 @@ +absolute count:ABSOLUTE:0:U apache_bytes count:COUNTER:0:134217728 apache_connections count:GAUGE:0:65535 apache_requests count:COUNTER:0:134217728 @@ -15,6 +16,7 @@ cpufreq value:GAUGE:0:U cpu value:COUNTER:0:4294967295 current value:GAUGE:U:U delay seconds:GAUGE:-1000000:1000000 +derive value:DERIVE:0:U df used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623 disk_merged read:COUNTER:0:4294967295, write:COUNTER:0:4294967295 disk_octets read:COUNTER:0:17179869183, write:COUNTER:0:17179869183 diff --git a/src/types_list.c b/src/types_list.c index ed832fee..49714176 100644 --- a/src/types_list.c +++ b/src/types_list.c @@ -68,9 +68,13 @@ static int parse_ds (data_source_t *dsrc, char *buf, size_t buf_len) dsrc->type = DS_TYPE_GAUGE; else if (strcasecmp (fields[1], "COUNTER") == 0) dsrc->type = DS_TYPE_COUNTER; + else if (strcasecmp (fields[1], "DERIVE") == 0) + dsrc->type = DS_TYPE_DERIVE; + else if (strcasecmp (fields[1], "ABSOLUTE") == 0) + dsrc->type = DS_TYPE_ABSOLUTE; else { - ERROR ("(fields[1] = %s) != (GAUGE || COUNTER)", fields[1]); + ERROR ("(fields[1] = %s) != (GAUGE || COUNTER || DERIVE || ABSOLUTE)", fields[1]); return (-1); } diff --git a/src/utils_match.c b/src/utils_match.c index c19c5ffc..bdbad3f2 100644 --- a/src/utils_match.c +++ b/src/utils_match.c @@ -145,6 +145,59 @@ static int default_callback (const char __attribute__((unused)) *str, data->values_num++; } + else if (data->ds_type & UTILS_MATCH_DS_TYPE_DERIVE) + { + derive_t value; + char *endptr = NULL; + + if (data->ds_type & UTILS_MATCH_CF_DERIVE_INC) + { + data->value.counter++; + data->values_num++; + return (0); + } + + if (matches_num < 2) + return (-1); + + value = strtoll (matches[1], &endptr, 0); + if (matches[1] == endptr) + return (-1); + + if (data->ds_type & UTILS_MATCH_CF_DERIVE_SET) + data->value.derive = value; + else if (data->ds_type & UTILS_MATCH_CF_DERIVE_ADD) + data->value.derive += value; + else + { + ERROR ("utils_match: default_callback: obj->ds_type is invalid!"); + return (-1); + } + + data->values_num++; + } + else if (data->ds_type & UTILS_MATCH_DS_TYPE_ABSOLUTE) + { + absolute_t value; + char *endptr = NULL; + + if (matches_num < 2) + return (-1); + + value = strtoll (matches[1], &endptr, 0); + if (matches[1] == endptr) + return (-1); + + if (data->ds_type & UTILS_MATCH_CF_ABSOLUTE_SET) + data->value.absolute = value; + else + { + ERROR ("utils_match: default_callback: obj->ds_type is invalid!"); + return (-1); + } + + data->values_num++; + } else { ERROR ("utils_match: default_callback: obj->ds_type is invalid!"); diff --git a/src/utils_match.h b/src/utils_match.h index a32a6fe0..5a0c0337 100644 --- a/src/utils_match.h +++ b/src/utils_match.h @@ -30,6 +30,8 @@ */ #define UTILS_MATCH_DS_TYPE_GAUGE 0x10 #define UTILS_MATCH_DS_TYPE_COUNTER 0x20 +#define UTILS_MATCH_DS_TYPE_DERIVE 0x30 +#define UTILS_MATCH_DS_TYPE_ABSOLUTE 0x40 #define UTILS_MATCH_CF_GAUGE_AVERAGE 0x01 #define UTILS_MATCH_CF_GAUGE_MIN 0x02 @@ -40,6 +42,14 @@ #define UTILS_MATCH_CF_COUNTER_ADD 0x02 #define UTILS_MATCH_CF_COUNTER_INC 0x04 +#define UTILS_MATCH_CF_DERIVE_SET 0x01 +#define UTILS_MATCH_CF_DERIVE_ADD 0x02 +#define UTILS_MATCH_CF_DERIVE_INC 0x04 + +#define UTILS_MATCH_CF_ABSOLUTE_SET 0x01 +#define UTILS_MATCH_CF_ABSOLUTE_ADD 0x02 +#define UTILS_MATCH_CF_ABSOLUTE_INC 0x04 + /* * Data types */ diff --git a/src/utils_rrdcreate.c b/src/utils_rrdcreate.c index c4e9d8ba..0abbb40b 100644 --- a/src/utils_rrdcreate.c +++ b/src/utils_rrdcreate.c @@ -213,6 +213,10 @@ static int ds_get (char ***ret, /* {{{ */ type = "COUNTER"; else if (d->type == DS_TYPE_GAUGE) type = "GAUGE"; + else if (d->type == DS_TYPE_DERIVE) + type = "DERIVE"; + else if (d->type == DS_TYPE_ABSOLUTE) + type = "ABSOLUTE"; else { ERROR ("rrdtool plugin: Unknown DS type: %i", -- 2.11.0