X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=src%2Fstatsd.c;h=7336d37ac2a413479329636fec5ae313755c0032;hb=f8379dd45f4a43595f4027992696ee8d02908bff;hp=9443fed40f55485f797bb54a0b538ad1c6a50f09;hpb=d53b119c992b580e7e11cf8d17480b91ab101111;p=collectd.git

diff --git a/src/statsd.c b/src/statsd.c
index 9443fed4..7336d37a 100644
--- a/src/statsd.c
+++ b/src/statsd.c
@@ -25,6 +25,7 @@
 #include "configfile.h"
 #include "utils_avltree.h"
 #include "utils_complain.h"
+#include "utils_latency.h"
 
 #include <pthread.h>
 
@@ -33,6 +34,11 @@
 #include <netdb.h>
 #include <poll.h>
 
+/* AIX doesn't have MSG_DONTWAIT */
+#ifndef MSG_DONTWAIT
+#  define MSG_DONTWAIT MSG_NONBLOCK
+#endif
+
 #ifndef STATSD_DEFAULT_NODE
 # define STATSD_DEFAULT_NODE NULL
 #endif
@@ -53,7 +59,8 @@ typedef enum metric_type_e metric_type_t;
 struct statsd_metric_s
 {
   metric_type_t type;
-  int64_t value;
+  double value;
+  latency_counter_t *latency;
   c_avl_tree_t *set;
   unsigned long updates_num;
 };
@@ -74,92 +81,159 @@ static _Bool conf_delete_timers   = 0;
 static _Bool conf_delete_gauges   = 0;
 static _Bool conf_delete_sets     = 0;
 
+static double *conf_timer_percentile = NULL;
+static size_t  conf_timer_percentile_num = 0;
+
+static _Bool conf_timer_lower     = 0;
+static _Bool conf_timer_upper     = 0;
+static _Bool conf_timer_sum       = 0;
+static _Bool conf_timer_count     = 0;
+
 /* Must hold metrics_lock when calling this function. */
-static int statsd_metric_set_unsafe (char const *name, int64_t value, /* {{{ */
+static statsd_metric_t *statsd_metric_lookup_unsafe (char const *name, /* {{{ */
     metric_type_t type)
 {
+  char key[DATA_MAX_NAME_LEN + 2];
+  char *key_copy;
   statsd_metric_t *metric;
-  char *key;
   int status;
 
-  status = c_avl_get (metrics_tree, name, (void *) &metric);
-  if (status == 0)
+  switch (type)
   {
-    metric->value = value;
-    metric->updates_num++;
+    case STATSD_COUNTER: key[0] = 'c'; break;
+    case STATSD_TIMER:   key[0] = 't'; break;
+    case STATSD_GAUGE:   key[0] = 'g'; break;
+    case STATSD_SET:     key[0] = 's'; break;
+    default: return (NULL);
+  }
 
-    return (0);
+  key[1] = ':';
+  sstrncpy (&key[2], name, sizeof (key) - 2);
+
+  status = c_avl_get (metrics_tree, key, (void *) &metric);
+  if (status == 0)
+    return (metric);
+
+  key_copy = strdup (key);
+  if (key_copy == NULL)
+  {
+    ERROR ("statsd plugin: strdup failed.");
+    return (NULL);
   }
 
-  DEBUG ("stats plugin: Adding new metric \"%s\".", name);
-  key = strdup (name);
-  metric = calloc (1, sizeof (*metric));
-  if ((key == NULL) || (metric == NULL))
+  metric = malloc (sizeof (*metric));
+  if (metric == NULL)
   {
-    sfree (key);
-    sfree (metric);
-    return (-1);
+    ERROR ("statsd plugin: malloc failed.");
+    sfree (key_copy);
+    return (NULL);
   }
+  memset (metric, 0, sizeof (*metric));
 
   metric->type = type;
-  metric->value = value;
-  metric->updates_num = 1;
+  metric->latency = NULL;
+  metric->set = NULL;
 
-  status = c_avl_insert (metrics_tree, key, metric);
+  status = c_avl_insert (metrics_tree, key_copy, metric);
   if (status != 0)
   {
-    sfree (key);
+    ERROR ("statsd plugin: c_avl_insert failed.");
+    sfree (key_copy);
     sfree (metric);
-
-    return (-1);
+    return (NULL);
   }
 
-  return (0);
-} /* }}} int statsd_metric_set_unsafe */
+  return (metric);
+} /* }}} statsd_metric_lookup_unsafe */
 
-static int statsd_metric_set (char const *name, int64_t value, /* {{{ */
+static int statsd_metric_set (char const *name, double value, /* {{{ */
     metric_type_t type)
 {
-  int status;
+  statsd_metric_t *metric;
 
   pthread_mutex_lock (&metrics_lock);
-  status = statsd_metric_set_unsafe (name, value, type);
+
+  metric = statsd_metric_lookup_unsafe (name, type);
+  if (metric == NULL)
+  {
+    pthread_mutex_unlock (&metrics_lock);
+    return (-1);
+  }
+
+  metric->value = value;
+  metric->updates_num++;
+
   pthread_mutex_unlock (&metrics_lock);
 
-  return (status);
+  return (0);
 } /* }}} int statsd_metric_set */
 
-static int statsd_metric_add (char const *name, int64_t delta, /* {{{ */
+static int statsd_metric_add (char const *name, double delta, /* {{{ */
     metric_type_t type)
 {
   statsd_metric_t *metric;
-  int status;
 
   pthread_mutex_lock (&metrics_lock);
 
-  status = c_avl_get (metrics_tree, name, (void *) &metric);
-  if (status == 0)
+  metric = statsd_metric_lookup_unsafe (name, type);
+  if (metric == NULL)
   {
-    metric->value += delta;
-    metric->updates_num++;
-
     pthread_mutex_unlock (&metrics_lock);
-    return (0);
+    return (-1);
+  }
+
+  metric->value += delta;
+  metric->updates_num++;
+
+  pthread_mutex_unlock (&metrics_lock);
+
+  return (0);
+} /* }}} int statsd_metric_add */
+
+static void statsd_metric_free (statsd_metric_t *metric) /* {{{ */
+{
+  if (metric == NULL)
+    return;
+
+  if (metric->latency != NULL)
+  {
+    latency_counter_destroy (metric->latency);
+    metric->latency = NULL;
   }
-  else /* no such value yet */
+
+  if (metric->set != NULL)
   {
-    status = statsd_metric_set_unsafe (name, delta, type);
+    void *key;
+    void *value;
 
-    pthread_mutex_unlock (&metrics_lock);
-    return (status);
+    while (c_avl_pick (metric->set, &key, &value) == 0)
+    {
+      sfree (key);
+      assert (value == NULL);
+    }
+
+    c_avl_destroy (metric->set);
+    metric->set = NULL;
   }
-} /* }}} int statsd_metric_add */
+
+  sfree (metric);
+} /* }}} void statsd_metric_free */
+
+static int statsd_parse_value (char const *str, value_t *ret_value) /* {{{ */
+{
+  char *endptr = NULL;
+
+  ret_value->gauge = (gauge_t) strtod (str, &endptr);
+  if ((str == endptr) || ((endptr != NULL) && (*endptr != 0)))
+    return (-1);
+
+  return (0);
+} /* }}} int statsd_parse_value */
 
 static int statsd_handle_counter (char const *name, /* {{{ */
     char const *value_str,
     char const *extra)
 {
-  char key[DATA_MAX_NAME_LEN + 2];
   value_t value;
   value_t scale;
   int status;
@@ -170,7 +244,7 @@ static int statsd_handle_counter (char const *name, /* {{{ */
   scale.gauge = 1.0;
   if (extra != NULL)
   {
-    status = parse_value (extra + 1, &scale, DS_TYPE_GAUGE);
+    status = statsd_parse_value (extra + 1, &scale);
     if (status != 0)
       return (status);
 
@@ -178,107 +252,102 @@ static int statsd_handle_counter (char const *name, /* {{{ */
       return (-1);
   }
 
-  value.derive = 1;
-  status = parse_value (value_str, &value, DS_TYPE_DERIVE);
+  value.gauge = 1.0;
+  status = statsd_parse_value (value_str, &value);
   if (status != 0)
     return (status);
 
-  if (value.derive < 1)
-    return (-1);
-
-  ssnprintf (key, sizeof (key), "c:%s", name);
-
-  return (statsd_metric_add (key,
-        (int64_t) (((gauge_t) value.derive) / scale.gauge),
+  return (statsd_metric_add (name, (double) (value.gauge / scale.gauge),
         STATSD_COUNTER));
 } /* }}} int statsd_handle_counter */
 
 static int statsd_handle_gauge (char const *name, /* {{{ */
     char const *value_str)
 {
-  char key[DATA_MAX_NAME_LEN + 2];
   value_t value;
   int status;
 
-  value.derive = 0;
-  status = parse_value (value_str, &value, DS_TYPE_DERIVE);
+  value.gauge = 0;
+  status = statsd_parse_value (value_str, &value);
   if (status != 0)
     return (status);
 
-  ssnprintf (key, sizeof (key), "g:%s", name);
-
   if ((value_str[0] == '+') || (value_str[0] == '-'))
-    return (statsd_metric_add (key, (int64_t) value.derive, STATSD_GAUGE));
+    return (statsd_metric_add (name, (double) value.gauge, STATSD_GAUGE));
   else
-    return (statsd_metric_set (key, (int64_t) value.derive, STATSD_GAUGE));
+    return (statsd_metric_set (name, (double) value.gauge, STATSD_GAUGE));
 } /* }}} int statsd_handle_gauge */
 
 static int statsd_handle_timer (char const *name, /* {{{ */
-    char const *value_str)
+    char const *value_str,
+    char const *extra)
 {
-  char key[DATA_MAX_NAME_LEN + 2];
-  value_t value;
+  statsd_metric_t *metric;
+  value_t value_ms;
+  value_t scale;
+  cdtime_t value;
   int status;
 
-  value.derive = 0;
-  status = parse_value (value_str, &value, DS_TYPE_DERIVE);
+  if ((extra != NULL) && (extra[0] != '@'))
+    return (-1);
+
+  scale.gauge = 1.0;
+  if (extra != NULL)
+  {
+    status = statsd_parse_value (extra + 1, &scale);
+    if (status != 0)
+      return (status);
+
+    if (!isfinite (scale.gauge) || (scale.gauge <= 0.0) || (scale.gauge > 1.0))
+      return (-1);
+  }
+
+  value_ms.derive = 0;
+  status = statsd_parse_value (value_str, &value_ms);
   if (status != 0)
     return (status);
 
-  ssnprintf (key, sizeof (key), "t:%s", name);
+  value = MS_TO_CDTIME_T (value_ms.gauge / scale.gauge);
+
+  pthread_mutex_lock (&metrics_lock);
+
+  metric = statsd_metric_lookup_unsafe (name, STATSD_TIMER);
+  if (metric == NULL)
+  {
+    pthread_mutex_unlock (&metrics_lock);
+    return (-1);
+  }
+
+  if (metric->latency == NULL)
+    metric->latency = latency_counter_create ();
+  if (metric->latency == NULL)
+  {
+    pthread_mutex_unlock (&metrics_lock);
+    return (-1);
+  }
+
+  latency_counter_add (metric->latency, value);
+  metric->updates_num++;
 
-  return (statsd_metric_add (key, (int64_t) value.derive, STATSD_TIMER));
+  pthread_mutex_unlock (&metrics_lock);
+  return (0);
 } /* }}} int statsd_handle_timer */
 
-static int statsd_handle_set (char const *key_orig, /* {{{ */
-    char const *name_orig)
+static int statsd_handle_set (char const *name, /* {{{ */
+    char const *set_key_orig)
 {
-  char key[DATA_MAX_NAME_LEN + 2];
-  char *name;
   statsd_metric_t *metric = NULL;
+  char *set_key;
   int status;
 
-  ssnprintf (key, sizeof (key), "s:%s", key_orig);
-
   pthread_mutex_lock (&metrics_lock);
 
-  status = c_avl_get (metrics_tree, key, (void *) &metric);
-  if (status != 0) /* Create a new metric */
+  metric = statsd_metric_lookup_unsafe (name, STATSD_SET);
+  if (metric == NULL)
   {
-    char *key_copy;
-
-    DEBUG ("stats plugin: Adding new metric \"%s\".", key);
-    key_copy = strdup (key);
-    if (key_copy == NULL)
-    {
-      pthread_mutex_unlock (&metrics_lock);
-      ERROR ("statsd plugin: strdup failed.");
-      return (-1);
-    }
-
-    metric = calloc (1, sizeof (*metric));
-    if (metric == NULL)
-    {
-      pthread_mutex_unlock (&metrics_lock);
-      ERROR ("statsd plugin: calloc failed.");
-      sfree (key_copy);
-      return (-1);
-    }
-    metric->type = STATSD_SET;
-    metric->set = NULL;
-
-    status = c_avl_insert (metrics_tree, key_copy, metric);
-    if (status != 0)
-    {
-      pthread_mutex_unlock (&metrics_lock);
-      ERROR ("statsd plugin: c_avl_insert (\"%s\") failed with status %i.",
-          key_copy, status);
-      sfree (key_copy);
-      sfree (metric);
-      return (-1);
-    }
+    pthread_mutex_unlock (&metrics_lock);
+    return (-1);
   }
-  assert (metric != NULL);
 
   /* Make sure metric->set exists. */
   if (metric->set == NULL)
@@ -291,27 +360,27 @@ static int statsd_handle_set (char const *key_orig, /* {{{ */
     return (-1);
   }
 
-  name = strdup (name_orig);
-  if (name == NULL)
+  set_key = strdup (set_key_orig);
+  if (set_key == NULL)
   {
     pthread_mutex_unlock (&metrics_lock);
     ERROR ("statsd plugin: strdup failed.");
     return (-1);
   }
 
-  status = c_avl_insert (metric->set, name, /* value = */ NULL);
+  status = c_avl_insert (metric->set, set_key, /* value = */ NULL);
   if (status < 0)
   {
     pthread_mutex_unlock (&metrics_lock);
     if (status < 0)
       ERROR ("statsd plugin: c_avl_insert (\"%s\") failed with status %i.",
-          name, status);
-    sfree (name);
+          set_key, status);
+    sfree (set_key);
     return (-1);
   }
   else if (status > 0) /* key already exists */
   {
-    sfree (name);
+    sfree (set_key);
   }
 
   metric->updates_num++;
@@ -348,15 +417,15 @@ static int statsd_parse_line (char *buffer) /* {{{ */
 
   if (strcmp ("c", type) == 0)
     return (statsd_handle_counter (name, value, extra));
+  else if (strcmp ("ms", type) == 0)
+    return (statsd_handle_timer (name, value, extra));
 
-  /* extra is only valid for counters */
+  /* extra is only valid for counters and timers */
   if (extra != NULL)
     return (-1);
 
   if (strcmp ("g", type) == 0)
     return (statsd_handle_gauge (name, value));
-  else if (strcmp ("ms", type) == 0)
-    return (statsd_handle_timer (name, value));
   else if (strcmp ("s", type) == 0)
     return (statsd_handle_set (name, value));
   else
@@ -365,22 +434,32 @@ static int statsd_parse_line (char *buffer) /* {{{ */
 
 static void statsd_parse_buffer (char *buffer) /* {{{ */
 {
-  char *dummy;
-  char *saveptr = NULL;
-  char *ptr;
-
-  for (dummy = buffer;
-      (ptr = strtok_r (dummy, "\r\n", &saveptr)) != NULL;
-      dummy = NULL)
+  while (buffer != NULL)
   {
-    char *line_orig = sstrdup (ptr);
+    char orig[64];
+    char *next;
     int status;
 
-    status = statsd_parse_line (ptr);
+    next = strchr (buffer, '\n');
+    if (next != NULL)
+    {
+      *next = 0;
+      next++;
+    }
+
+    if (*buffer == 0)
+    {
+      buffer = next;
+      continue;
+    }
+
+    sstrncpy (orig, buffer, sizeof (orig));
+
+    status = statsd_parse_line (buffer);
     if (status != 0)
-      ERROR ("statsd plugin: Unable to parse line: \"%s\"", line_orig);
+      ERROR ("statsd plugin: Unable to parse line: \"%s\"", orig);
 
-    sfree (line_orig);
+    buffer = next;
   }
 } /* }}} void statsd_parse_buffer */
 
@@ -478,6 +557,7 @@ static int statsd_network_init (struct pollfd **ret_fds, /* {{{ */
     if (tmp == NULL)
     {
       ERROR ("statsd plugin: realloc failed.");
+      close (fd);
       continue;
     }
     fds = tmp;
@@ -550,6 +630,37 @@ static void *statsd_network_thread (void *args) /* {{{ */
   return ((void *) 0);
 } /* }}} void *statsd_network_thread */
 
+static int statsd_config_timer_percentile (oconfig_item_t *ci) /* {{{ */
+{
+  double percent = NAN;
+  double *tmp;
+  int status;
+
+  status = cf_util_get_double (ci, &percent);
+  if (status != 0)
+    return (status);
+
+  if ((percent <= 0.0) || (percent >= 100))
+  {
+    ERROR ("statsd plugin: The value for \"%s\" must be between 0 and 100, "
+        "exclusively.", ci->key);
+    return (ERANGE);
+  }
+
+  tmp = realloc (conf_timer_percentile,
+      sizeof (*conf_timer_percentile) * (conf_timer_percentile_num + 1));
+  if (tmp == NULL)
+  {
+    ERROR ("statsd plugin: realloc failed.");
+    return (ENOMEM);
+  }
+  conf_timer_percentile = tmp;
+  conf_timer_percentile[conf_timer_percentile_num] = percent;
+  conf_timer_percentile_num++;
+
+  return (0);
+} /* }}} int statsd_config_timer_percentile */
+
 static int statsd_config (oconfig_item_t *ci) /* {{{ */
 {
   int i;
@@ -570,6 +681,16 @@ static int statsd_config (oconfig_item_t *ci) /* {{{ */
       cf_util_get_boolean (child, &conf_delete_gauges);
     else if (strcasecmp ("DeleteSets", child->key) == 0)
       cf_util_get_boolean (child, &conf_delete_sets);
+    else if (strcasecmp ("TimerLower", child->key) == 0)
+      cf_util_get_boolean (child, &conf_timer_lower);
+    else if (strcasecmp ("TimerUpper", child->key) == 0)
+      cf_util_get_boolean (child, &conf_timer_upper);
+    else if (strcasecmp ("TimerSum", child->key) == 0)
+      cf_util_get_boolean (child, &conf_timer_sum);
+    else if (strcasecmp ("TimerCount", child->key) == 0)
+      cf_util_get_boolean (child, &conf_timer_count);
+    else if (strcasecmp ("TimerPercentile", child->key) == 0)
+      statsd_config_timer_percentile (child);
     else
       ERROR ("statsd plugin: The \"%s\" config option is not valid.",
           child->key);
@@ -582,7 +703,7 @@ static int statsd_init (void) /* {{{ */
 {
   pthread_mutex_lock (&metrics_lock);
   if (metrics_tree == NULL)
-    metrics_tree = c_avl_create ((void *) strcasecmp);
+    metrics_tree = c_avl_create ((void *) strcmp);
 
   if (!network_thread_running)
   {
@@ -636,26 +757,6 @@ static int statsd_metric_submit_unsafe (char const *name, /* {{{ */
   value_t values[1];
   value_list_t vl = VALUE_LIST_INIT;
 
-  if (metric->type == STATSD_GAUGE)
-    values[0].gauge = (gauge_t) metric->value;
-  else if (metric->type == STATSD_TIMER)
-  {
-    if (metric->updates_num == 0)
-      values[0].gauge = NAN;
-    else
-      values[0].gauge =
-        ((gauge_t) metric->value) / ((gauge_t) metric->updates_num);
-  }
-  else if (metric->type == STATSD_SET)
-  {
-    if (metric->set == NULL)
-      values[0].gauge = 0.0;
-    else
-      values[0].gauge = (gauge_t) c_avl_size (metric->set);
-  }
-  else
-    values[0].derive = (derive_t) metric->value;
-
   vl.values = values;
   vl.values_len = 1;
   sstrncpy (vl.host, hostname_g, sizeof (vl.host));
@@ -672,6 +773,94 @@ static int statsd_metric_submit_unsafe (char const *name, /* {{{ */
 
   sstrncpy (vl.type_instance, name, sizeof (vl.type_instance));
 
+  if (metric->type == STATSD_GAUGE)
+    values[0].gauge = (gauge_t) metric->value;
+  else if (metric->type == STATSD_TIMER)
+  {
+    size_t i;
+    _Bool have_events = (metric->updates_num > 0);
+
+    /* Make sure all timer metrics share the *same* timestamp. */
+    vl.time = cdtime ();
+
+    ssnprintf (vl.type_instance, sizeof (vl.type_instance),
+        "%s-average", name);
+    values[0].gauge = have_events
+      ? CDTIME_T_TO_DOUBLE (latency_counter_get_average (metric->latency))
+      : NAN;
+    plugin_dispatch_values (&vl);
+
+    if (conf_timer_lower) {
+      ssnprintf (vl.type_instance, sizeof (vl.type_instance),
+          "%s-lower", name);
+      values[0].gauge = have_events
+        ? CDTIME_T_TO_DOUBLE (latency_counter_get_min (metric->latency))
+        : NAN;
+      plugin_dispatch_values (&vl);
+    }
+
+    if (conf_timer_upper) {
+      ssnprintf (vl.type_instance, sizeof (vl.type_instance),
+          "%s-upper", name);
+      values[0].gauge = have_events
+        ? CDTIME_T_TO_DOUBLE (latency_counter_get_max (metric->latency))
+        : NAN;
+      plugin_dispatch_values (&vl);
+    }
+
+    if (conf_timer_sum) {
+      ssnprintf (vl.type_instance, sizeof (vl.type_instance),
+          "%s-sum", name);
+      values[0].gauge = have_events
+        ? CDTIME_T_TO_DOUBLE (latency_counter_get_sum (metric->latency))
+        : NAN;
+      plugin_dispatch_values (&vl);
+    }
+
+    for (i = 0; i < conf_timer_percentile_num; i++)
+    {
+      ssnprintf (vl.type_instance, sizeof (vl.type_instance),
+          "%s-percentile-%.0f", name, conf_timer_percentile[i]);
+      values[0].gauge = have_events
+        ? CDTIME_T_TO_DOUBLE (latency_counter_get_percentile (metric->latency, conf_timer_percentile[i]))
+        : NAN;
+      plugin_dispatch_values (&vl);
+    }
+
+    /* Keep this at the end, since vl.type is set to "gauge" here. The
+     * vl.type's above are implicitly set to "latency". */
+    if (conf_timer_count) {
+      sstrncpy (vl.type, "gauge", sizeof (vl.type));
+      ssnprintf (vl.type_instance, sizeof (vl.type_instance),
+          "%s-count", name);
+      values[0].gauge = latency_counter_get_num (metric->latency);
+      plugin_dispatch_values (&vl);
+    }
+
+    latency_counter_reset (metric->latency);
+    return (0);
+  }
+  else if (metric->type == STATSD_SET)
+  {
+    if (metric->set == NULL)
+      values[0].gauge = 0.0;
+    else
+      values[0].gauge = (gauge_t) c_avl_size (metric->set);
+  }
+  else { /* STATSD_COUNTER */
+      /*
+       * Expand a single value to two metrics:
+       *
+       * - The absolute counter, as a gauge
+       * - A derived rate for this counter
+       */
+      values[0].derive = (derive_t) metric->value;
+      plugin_dispatch_values(&vl);
+
+      sstrncpy(vl.type, "gauge", sizeof (vl.type));
+      values[0].gauge = (gauge_t) metric->value;
+  }
+
   return (plugin_dispatch_values (&vl));
 } /* }}} int statsd_metric_submit_unsafe */
 
@@ -732,7 +921,7 @@ static int statsd_read (void) /* {{{ */
     }
 
     sfree (name);
-    sfree (metric);
+    statsd_metric_free (metric);
   }
 
   pthread_mutex_unlock (&metrics_lock);
@@ -760,7 +949,7 @@ static int statsd_shutdown (void) /* {{{ */
   while (c_avl_pick (metrics_tree, &key, &value) == 0)
   {
     sfree (key);
-    sfree (value);
+    statsd_metric_free (value);
   }
   c_avl_destroy (metrics_tree);
   metrics_tree = NULL;