Introduce the DERIVE and ABSOLUTE data source types.

Hi,

i've updated my patch to 4.7.0, most of "data input" plugins (curl, java, exec,
perl, tail, couchdb) should work with derive. In case of couchdb and curl, if u
use absolute DS you can only "Set", no "Inc" or "Add" coz obviously that
wouldn't make much sense with it. Other plugins can be "enabled" globally to
use derive by changing "COUNTER" to "DERIVE" in types.db but that way is ugly
(but makes sense in some cases, like when u have lot of tunnels or ppp
interfaces) and either needs converting or recreating rrd files.

Regards
Mariusz

---

Hi,

ive been running my patch with 4.7.1, found a minor bug, but after repairing
that i didnt had any problems with it on my servers, im including patch
(against 4.7.1 from webpage),

Regards,
XANi

diff --git a/src/common.c b/src/common.c
index d42453e..8de871b 100644 (file)
--- a/src/common.c
+++ b/src/common.c
@@ -845,6 +845,10 @@ int parse_value (const char *value, value_t *ret_value, const data_source_t ds)
                ret_value->counter = (counter_t)strtoll (value, &endptr, 0);
        else if (DS_TYPE_GAUGE == ds.type)
                ret_value->gauge = (gauge_t)strtod (value, &endptr);
+       else if (DS_TYPE_DERIVE == ds.type)
+               ret_value->counter = (derive_t)strtoll (value, &endptr, 0);
+       else if (DS_TYPE_ABSOLUTE == ds.type)
+               ret_value->counter = (absolute_t)strtoll (value, &endptr, 0);
        else {
                ERROR ("parse_value: Invalid data source \"%s\" "
                                "(type = %i).", ds.name, ds.type);

diff --git a/src/csv.c b/src/csv.c
index 5c43b8a..f38599d 100644 (file)
--- a/src/csv.c
+++ b/src/csv.c
@@ -61,7 +61,9 @@ static int value_list_to_string (char *buffer, int buffer_len,
        for (i = 0; i < ds->ds_num; i++)
        {
                if ((ds->ds[i].type != DS_TYPE_COUNTER)
-                               && (ds->ds[i].type != DS_TYPE_GAUGE))
+                               && (ds->ds[i].type != DS_TYPE_GAUGE)
+                               && (ds->ds[i].type != DS_TYPE_DERIVE)
+                               && (ds->ds[i].type != DS_TYPE_ABSOLUTE))
                        return (-1);
 
                if (ds->ds[i].type == DS_TYPE_COUNTER)
@@ -88,10 +90,58 @@ static int value_list_to_string (char *buffer, int buffer_len,
                                                ",%lf", rates[i]);
                        }
                }
-               else /* if (ds->ds[i].type == DS_TYPE_GAUGE) */
+               else  if (ds->ds[i].type == DS_TYPE_GAUGE) 
                {
                        status = ssnprintf (buffer + offset, buffer_len - offset,
                                        ",%lf", vl->values[i].gauge);
+               } 
+               else if (ds->ds[i].type == DS_TYPE_DERIVE)
+               {
+                       if (store_rates == 0)
+                       {
+                               status = ssnprintf (buffer + offset,
+                                               buffer_len - offset,
+                                               ",%llu",
+                                               vl->values[i].derive);
+                       }
+                       else /* if (store_rates == 1) */
+                       {
+                               if (rates == NULL)
+                                       rates = uc_get_rate (ds, vl);
+                               if (rates == NULL)
+                               {
+                                       WARNING ("csv plugin: "
+                                                       "uc_get_rate failed.");
+                                       return (-1);
+                               }
+                               status = ssnprintf (buffer + offset,
+                                               buffer_len - offset,
+                                               ",%lf", rates[i]);
+                       }
+               }
+               else if (ds->ds[i].type == DS_TYPE_ABSOLUTE)
+               {
+                       if (store_rates == 0)
+                       {
+                               status = ssnprintf (buffer + offset,
+                                               buffer_len - offset,
+                                               ",%llu",
+                                               vl->values[i].absolute);
+                       }
+                       else /* if (store_rates == 1) */
+                       {
+                               if (rates == NULL)
+                                       rates = uc_get_rate (ds, vl);
+                               if (rates == NULL)
+                               {
+                                       WARNING ("csv plugin: "
+                                                       "uc_get_rate failed.");
+                                       return (-1);
+                               }
+                               status = ssnprintf (buffer + offset,
+                                               buffer_len - offset,
+                                               ",%lf", rates[i]);
+                       }
                }
 
                if ((status < 1) || (status >= (buffer_len - offset)))

diff --git a/src/curl.c b/src/curl.c
index 88b5496..41f11f6 100644 (file)
--- a/src/curl.c
+++ b/src/curl.c
@@ -222,6 +222,29 @@ static int cc_config_add_match_dstype (int *dstype_ret, /* {{{ */
     else
       dstype = 0;
   }
+else if (strncasecmp ("Derive", ci->values[0].value.string,
+        strlen ("Derive")) == 0)
+  {
+    dstype = UTILS_MATCH_DS_TYPE_DERIVE;
+    if (strcasecmp ("DeriveSet", ci->values[0].value.string) == 0)
+      dstype |= UTILS_MATCH_CF_DERIVE_SET;
+    else if (strcasecmp ("DeriveAdd", ci->values[0].value.string) == 0)
+      dstype |= UTILS_MATCH_CF_DERIVE_ADD;
+    else if (strcasecmp ("DeriveInc", ci->values[0].value.string) == 0)
+      dstype |= UTILS_MATCH_CF_DERIVE_INC;
+    else
+      dstype = 0;
+  }
+else if (strncasecmp ("Absolute", ci->values[0].value.string,
+        strlen ("Absolute")) == 0)
+  {
+    dstype = UTILS_MATCH_DS_TYPE_ABSOLUTE;
+    if (strcasecmp ("AbsoluteSet", ci->values[0].value.string) == 0) /* Absolute DS is reset-on-read so no sense doin anything else but set */
+      dstype |= UTILS_MATCH_CF_ABSOLUTE_SET;
+    else
+      dstype = 0;
+  }
+
   else
   {
     dstype = 0;

diff --git a/src/java.c b/src/java.c
index 20523f9..64e51d3 100644 (file)
--- a/src/java.c
+++ b/src/java.c
@@ -284,6 +284,10 @@ static jobject ctoj_value_to_number (JNIEnv *jvm_env, /* {{{ */
     return (ctoj_jlong_to_number (jvm_env, (jlong) value.counter));
   else if (ds_type == DS_TYPE_GAUGE)
     return (ctoj_jdouble_to_number (jvm_env, (jdouble) value.gauge));
+  if (ds_type == DS_TYPE_DERIVE)
+    return (ctoj_jlong_to_number (jvm_env, (jlong) value.derive));
+  if (ds_type == DS_TYPE_ABSOLUTE)
+    return (ctoj_jlong_to_number (jvm_env, (jlong) value.absolute));
   else
     return (NULL);
 } /* }}} jobject ctoj_value_to_number */

diff --git a/src/libcollectdclient/client.c b/src/libcollectdclient/client.c
index 501c729..bb7ee95 100644 (file)
--- a/src/libcollectdclient/client.c
+++ b/src/libcollectdclient/client.c
@@ -792,6 +792,11 @@ int lcc_putval (lcc_connection_t *c, const lcc_value_list_t *vl) /* {{{ */
       else
         SSTRCATF (command, ":%g", vl->values[i].gauge);
     }
+    else if (vl->values_types[i] == LCC_TYPE_DERIVE)
+       SSTRCATF (command, ":%"PRIu64, vl->values[i].derive);
+    else if (vl->values_types[i] == LCC_TYPE_ABSOLUTE)
+       SSTRCATF (command, ":%"PRIu64, vl->values[i].absolute);
+
   } /* for (i = 0; i < vl->values_len; i++) */
 
   status = lcc_sendreceive (c, command, &res);

diff --git a/src/libcollectdclient/client.h b/src/libcollectdclient/client.h
index d5371fb..b0d092d 100644 (file)
--- a/src/libcollectdclient/client.h
+++ b/src/libcollectdclient/client.h
@@ -42,16 +42,22 @@
  */
 #define LCC_TYPE_COUNTER 0
 #define LCC_TYPE_GAUGE   1
+#define LCC_TYPE_DERIVE   2
+#define LCC_TYPE_ABSOLUTE   3
 
 LCC_BEGIN_DECLS
 
 typedef uint64_t counter_t;
 typedef double gauge_t;
+typedef uint64_t derive_t;
+typedef uint64_t absolute_t;
 
 union value_u
 {
   counter_t counter;
   gauge_t   gauge;
+  derive_t  derive;
+  absolute_t absolute;
 };
 typedef union value_u value_t;
 

diff --git a/src/perl.c b/src/perl.c
index b6e7b22..dd82ed9 100644 (file)
--- a/src/perl.c
+++ b/src/perl.c
@@ -194,6 +194,8 @@ struct {
        { "Collectd::TYPE_DATASET",       PLUGIN_DATASET },
        { "Collectd::DS_TYPE_COUNTER",    DS_TYPE_COUNTER },
        { "Collectd::DS_TYPE_GAUGE",      DS_TYPE_GAUGE },
+       { "Collectd::DS_TYPE_DERIVE",     DS_TYPE_DERIVE },
+       { "Collectd::DS_TYPE_ABSOLUTE",   DS_TYPE_ABSOLUTE },
        { "Collectd::LOG_ERR",            LOG_ERR },
        { "Collectd::LOG_WARNING",        LOG_WARNING },
        { "Collectd::LOG_NOTICE",         LOG_NOTICE },
@@ -267,7 +269,7 @@ static int hv2data_source (pTHX_ HV *hash, data_source_t *ds)
        if (NULL != (tmp = hv_fetch (hash, "type", 4, 0))) {
                ds->type = SvIV (*tmp);
 
-               if ((DS_TYPE_COUNTER != ds->type) && (DS_TYPE_GAUGE != ds->type)) {
+               if ((DS_TYPE_COUNTER != ds->type) && (DS_TYPE_GAUGE != ds->type) && (DS_TYPE_DERIVE != ds->type) && (DS_TYPE_ABSOLUTE != ds->type)) {
                        log_err ("hv2data_source: Invalid DS type.");
                        return -1;
                }
@@ -320,8 +322,12 @@ static int av2value (pTHX_ char *name, AV *array, value_t *value, int len)
                if (NULL != tmp) {
                        if (DS_TYPE_COUNTER == ds->ds[i].type)
                                value[i].counter = SvIV (*tmp);
-                       else
+                       else if (DS_TYPE_GAUGE == ds->ds[i].type)
                                value[i].gauge = SvNV (*tmp);
+                       else if (DS_TYPE_DERIVE == ds->ds[i].type)
+                               value[i].derive = SvIV (*tmp);
+                       else if (DS_TYPE_ABSOLUTE == ds->ds[i].type)
+                               value[i].absolute = SvIV (*tmp);
                }
                else {
                        return -1;
@@ -637,8 +643,12 @@ static int value_list2hv (pTHX_ value_list_t *vl, data_set_t *ds, HV *hash)
 
                if (DS_TYPE_COUNTER == ds->ds[i].type)
                        val = newSViv (vl->values[i].counter);
-               else
+               else if (DS_TYPE_GAUGE == ds->ds[i].type)
                        val = newSVnv (vl->values[i].gauge);
+               else if (DS_TYPE_DERIVE == ds->ds[i].type)
+                       val = newSVnv (vl->values[i].derive);
+               else if (DS_TYPE_ABSOLUTE == ds->ds[i].type)
+                       val = newSVnv (vl->values[i].absolute);
 
                if (NULL == av_store (values, i, val)) {
                        av_undef (values);

diff --git a/src/plugin.h b/src/plugin.h
index e8e4956..fd74150 100644 (file)
--- a/src/plugin.h
+++ b/src/plugin.h
@@ -30,6 +30,8 @@
 
 #define DS_TYPE_COUNTER 0
 #define DS_TYPE_GAUGE   1
+#define DS_TYPE_DERIVE   2
+#define DS_TYPE_ABSOLUTE   3
 
 #ifndef LOG_ERR
 # define LOG_ERR 3
@@ -58,11 +60,15 @@
  */
 typedef unsigned long long counter_t;
 typedef double gauge_t;
+typedef unsigned long long derive_t;
+typedef unsigned long long absolute_t;
 
 union value_u
 {
        counter_t counter;
        gauge_t   gauge;
+       derive_t   derive;
+       absolute_t   absolute;
 };
 typedef union value_u value_t;
 

diff --git a/src/rrdcached.c b/src/rrdcached.c
index 326a889..6295a53 100644 (file)
--- a/src/rrdcached.c
+++ b/src/rrdcached.c
@@ -75,7 +75,9 @@ static int value_list_to_string (char *buffer, int buffer_len,
   for (i = 0; i < ds->ds_num; i++)
   {
     if ((ds->ds[i].type != DS_TYPE_COUNTER)
-        && (ds->ds[i].type != DS_TYPE_GAUGE))
+        && (ds->ds[i].type != DS_TYPE_GAUGE)
+       && (ds->ds[i].type != DS_TYPE_DERIVE)
+       && (ds->ds[i].type != DS_TYPE_ABSOLUTE)
       return (-1);
 
     if (ds->ds[i].type == DS_TYPE_COUNTER)
@@ -83,11 +85,20 @@ static int value_list_to_string (char *buffer, int buffer_len,
       status = ssnprintf (buffer + offset, buffer_len - offset,
           ":%llu", vl->values[i].counter);
     }
-    else /* if (ds->ds[i].type == DS_TYPE_GAUGE) */
+    else if (ds->ds[i].type == DS_TYPE_GAUGE) 
     {
       status = ssnprintf (buffer + offset, buffer_len - offset,
           ":%lf", vl->values[i].gauge);
     }
+    else if (ds->ds[i].type == DS_TYPE_DERIVE) {
+      status = ssnprintf (buffer + offset, buffer_len - offset,
+         ":%llu", vl->values[i].derive);
+    }
+    else /* if (ds->ds[i].type == DS_TYPE_ABSOLUTE) */ {
+      status = ssnprintf (buffer + offset, buffer_len - offset,
+         ":%llu", vl->values[i].absolute);
+ 
+    }
 
     if ((status < 1) || (status >= (buffer_len - offset)))
       return (-1);

diff --git a/src/rrdtool.c b/src/rrdtool.c
index 243a8c8..a361501 100644 (file)
--- a/src/rrdtool.c
+++ b/src/rrdtool.c
@@ -204,15 +204,23 @@ static int value_list_to_string (char *buffer, int buffer_len,
        for (i = 0; i < ds->ds_num; i++)
        {
                if ((ds->ds[i].type != DS_TYPE_COUNTER)
-                               && (ds->ds[i].type != DS_TYPE_GAUGE))
+                               && (ds->ds[i].type != DS_TYPE_GAUGE)
+                               && (ds->ds[i].type != DS_TYPE_DERIVE)
+                               && (ds->ds[i].type != DS_TYPE_ABSOLUTE))
                        return (-1);
 
                if (ds->ds[i].type == DS_TYPE_COUNTER)
                        status = ssnprintf (buffer + offset, buffer_len - offset,
                                        ":%llu", vl->values[i].counter);
-               else
+               else if (ds->ds[i].type == DS_TYPE_GAUGE)
                        status = ssnprintf (buffer + offset, buffer_len - offset,
                                        ":%lf", vl->values[i].gauge);
+               else if (ds->ds[i].type == DS_TYPE_DERIVE)
+                       status = ssnprintf (buffer + offset, buffer_len - offset,
+                                       ":%llu", vl->values[i].derive);
+               else /*if (ds->ds[i].type == DS_TYPE_ABSOLUTE) */
+                       status = ssnprintf (buffer + offset, buffer_len - offset,
+                                       ":%llu", vl->values[i].absolute);
 
                if ((status < 1) || (status >= (buffer_len - offset)))
                        return (-1);

diff --git a/src/tail.c b/src/tail.c
index 02afd79..8becc05 100644 (file)
--- a/src/tail.c
+++ b/src/tail.c
@@ -101,6 +101,26 @@ static int ctail_config_add_match_dstype (ctail_config_match_t *cm,
     else
       cm->flags = 0;
   }
+  else if (strncasecmp ("Derive", ci->values[0].value.string, strlen ("Derive")) == 0)
+  {
+    cm->flags = UTILS_MATCH_DS_TYPE_DERIVE;
+    if (strcasecmp ("DeriveSet", ci->values[0].value.string) == 0)
+      cm->flags |= UTILS_MATCH_CF_DERIVE_SET;
+    else if (strcasecmp ("DeriveAdd", ci->values[0].value.string) == 0)
+      cm->flags |= UTILS_MATCH_CF_DERIVE_ADD;
+    else if (strcasecmp ("DeriveInc", ci->values[0].value.string) == 0)
+      cm->flags |= UTILS_MATCH_CF_DERIVE_INC;
+    else
+      cm->flags = 0;
+  }
+  else if (strncasecmp ("Absolute", ci->values[0].value.string, strlen ("Absolute")) == 0)
+  {
+    cm->flags = UTILS_MATCH_DS_TYPE_ABSOLUTE;
+    if (strcasecmp ("AbsoluteSet", ci->values[0].value.string) == 0)
+      cm->flags |= UTILS_MATCH_CF_ABSOLUTE_SET;
+    else
+      cm->flags = 0;
+  }
   else
   {
     cm->flags = 0;

diff --git a/src/types.db b/src/types.db
index 7028fe7..83e7007 100644 (file)
--- a/src/types.db
+++ b/src/types.db
@@ -1,3 +1,4 @@
+absolute               count:ABSOLUTE:0:U
 apache_bytes           count:COUNTER:0:134217728
 apache_connections     count:GAUGE:0:65535
 apache_requests                count:COUNTER:0:134217728
@@ -15,6 +16,7 @@ cpufreq                       value:GAUGE:0:U
 cpu                    value:COUNTER:0:4294967295
 current                        value:GAUGE:U:U
 delay                  seconds:GAUGE:-1000000:1000000
+derive                 value:DERIVE:0:U
 df                     used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623
 disk_merged            read:COUNTER:0:4294967295, write:COUNTER:0:4294967295
 disk_octets            read:COUNTER:0:17179869183, write:COUNTER:0:17179869183

diff --git a/src/types_list.c b/src/types_list.c
index ed832fe..4971417 100644 (file)
--- a/src/types_list.c
+++ b/src/types_list.c
@@ -68,9 +68,13 @@ static int parse_ds (data_source_t *dsrc, char *buf, size_t buf_len)
     dsrc->type = DS_TYPE_GAUGE;
   else if (strcasecmp (fields[1], "COUNTER") == 0)
     dsrc->type = DS_TYPE_COUNTER;
+  else if (strcasecmp (fields[1], "DERIVE") == 0)
+    dsrc->type = DS_TYPE_DERIVE;
+  else if (strcasecmp (fields[1], "ABSOLUTE") == 0)
+    dsrc->type = DS_TYPE_ABSOLUTE;
   else
   {
-    ERROR ("(fields[1] = %s) != (GAUGE || COUNTER)", fields[1]);
+    ERROR ("(fields[1] = %s) != (GAUGE || COUNTER || DERIVE || ABSOLUTE)", fields[1]);
     return (-1);
   }
 

diff --git a/src/utils_match.c b/src/utils_match.c
index c19c5ff..bdbad3f 100644 (file)
--- a/src/utils_match.c
+++ b/src/utils_match.c
@@ -145,6 +145,59 @@ static int default_callback (const char __attribute__((unused)) *str,
 
     data->values_num++;
   }
+  else if (data->ds_type & UTILS_MATCH_DS_TYPE_DERIVE)
+  {
+    derive_t value;
+    char *endptr = NULL;
+
+    if (data->ds_type & UTILS_MATCH_CF_DERIVE_INC)
+    {
+      data->value.counter++;
+      data->values_num++;
+      return (0);
+    }
+
+    if (matches_num < 2)
+      return (-1);
+
+    value = strtoll (matches[1], &endptr, 0);
+    if (matches[1] == endptr)
+      return (-1);
+
+    if (data->ds_type & UTILS_MATCH_CF_DERIVE_SET)
+      data->value.derive = value;
+    else if (data->ds_type & UTILS_MATCH_CF_DERIVE_ADD)
+      data->value.derive += value;
+    else
+    {
+      ERROR ("utils_match: default_callback: obj->ds_type is invalid!");
+      return (-1);
+    }
+
+    data->values_num++;
+  }
+  else if (data->ds_type & UTILS_MATCH_DS_TYPE_ABSOLUTE)
+  {
+    absolute_t value;
+    char *endptr = NULL;
+
+    if (matches_num < 2)
+      return (-1);
+
+    value = strtoll (matches[1], &endptr, 0);
+    if (matches[1] == endptr)
+      return (-1);
+
+    if (data->ds_type & UTILS_MATCH_CF_ABSOLUTE_SET)
+      data->value.absolute = value;
+    else
+    {
+      ERROR ("utils_match: default_callback: obj->ds_type is invalid!");
+      return (-1);
+    }
+
+    data->values_num++;
+  }
   else
   {
     ERROR ("utils_match: default_callback: obj->ds_type is invalid!");

diff --git a/src/utils_match.h b/src/utils_match.h
index a32a6fe..5a0c033 100644 (file)
--- a/src/utils_match.h
+++ b/src/utils_match.h
@@ -30,6 +30,8 @@
  */
 #define UTILS_MATCH_DS_TYPE_GAUGE   0x10
 #define UTILS_MATCH_DS_TYPE_COUNTER 0x20
+#define UTILS_MATCH_DS_TYPE_DERIVE 0x30
+#define UTILS_MATCH_DS_TYPE_ABSOLUTE 0x40
 
 #define UTILS_MATCH_CF_GAUGE_AVERAGE 0x01
 #define UTILS_MATCH_CF_GAUGE_MIN     0x02
@@ -40,6 +42,14 @@
 #define UTILS_MATCH_CF_COUNTER_ADD   0x02
 #define UTILS_MATCH_CF_COUNTER_INC   0x04
 
+#define UTILS_MATCH_CF_DERIVE_SET   0x01
+#define UTILS_MATCH_CF_DERIVE_ADD   0x02
+#define UTILS_MATCH_CF_DERIVE_INC   0x04
+
+#define UTILS_MATCH_CF_ABSOLUTE_SET   0x01
+#define UTILS_MATCH_CF_ABSOLUTE_ADD   0x02
+#define UTILS_MATCH_CF_ABSOLUTE_INC   0x04
+
 /*
  * Data types
  */

diff --git a/src/utils_rrdcreate.c b/src/utils_rrdcreate.c
index c4e9d8b..0abbb40 100644 (file)
--- a/src/utils_rrdcreate.c
+++ b/src/utils_rrdcreate.c
@@ -213,6 +213,10 @@ static int ds_get (char ***ret, /* {{{ */
       type = "COUNTER";
     else if (d->type == DS_TYPE_GAUGE)
       type = "GAUGE";
+    else if (d->type == DS_TYPE_DERIVE)
+      type = "DERIVE";
+    else if (d->type == DS_TYPE_ABSOLUTE)
+      type = "ABSOLUTE";
     else
     {
       ERROR ("rrdtool plugin: Unknown DS type: %i",