2 * collectd - src/threshold.c
3 * Copyright (C) 2007-2010 Florian Forster
4 * Copyright (C) 2008-2009 Sebastian Harl
5 * Copyright (C) 2009 Andrés J. Díaz
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; only version 2 of the License is applicable.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 * Florian octo Forster <octo at collectd.org>
22 * Sebastian Harl <sh at tokkee.org>
23 * Andrés J. Díaz <ajdiaz at connectical.com>
29 #include "utils_avltree.h"
30 #include "utils_cache.h"
31 #include "utils_threshold.h"
37 * Threshold management
38 * ====================
39 * The following functions add, delete, search, etc. configured thresholds to
40 * the underlying AVL trees.
44 * int ut_threshold_add
46 * Adds a threshold configuration to the list of thresholds. The threshold_t
47 * structure is copied and may be destroyed after this call. Returns zero on
48 * success, non-zero otherwise.
50 static int ut_threshold_add (const threshold_t *th)
52 char name[6 * DATA_MAX_NAME_LEN];
58 if (format_name (name, sizeof (name), th->host,
59 th->plugin, th->plugin_instance,
60 th->type, th->type_instance) != 0)
62 ERROR ("ut_threshold_add: format_name failed.");
66 name_copy = strdup (name);
67 if (name_copy == NULL)
69 ERROR ("ut_threshold_add: strdup failed.");
73 th_copy = malloc (sizeof (*th_copy));
77 ERROR ("ut_threshold_add: malloc failed.");
80 memcpy (th_copy, th, sizeof (threshold_t));
82 DEBUG ("ut_threshold_add: Adding entry `%s'", name);
84 pthread_mutex_lock (&threshold_lock);
86 th_ptr = threshold_get (th->host, th->plugin, th->plugin_instance,
87 th->type, th->type_instance);
89 while ((th_ptr != NULL) && (th_ptr->next != NULL))
90 th_ptr = th_ptr->next;
92 if (th_ptr == NULL) /* no such threshold yet */
94 status = c_avl_insert (threshold_tree, name_copy, th_copy);
96 else /* th_ptr points to the last threshold in the list */
98 th_ptr->next = th_copy;
99 /* name_copy isn't needed */
103 pthread_mutex_unlock (&threshold_lock);
107 ERROR ("ut_threshold_add: c_avl_insert (%s) failed.", name);
113 } /* }}} int ut_threshold_add */
118 * The following approximately two hundred functions are used to handle the
119 * configuration and fill the threshold list.
121 static int ut_config_type_datasource (threshold_t *th, oconfig_item_t *ci)
123 if ((ci->values_num != 1)
124 || (ci->values[0].type != OCONFIG_TYPE_STRING))
126 WARNING ("threshold values: The `DataSource' option needs exactly one "
131 sstrncpy (th->data_source, ci->values[0].value.string,
132 sizeof (th->data_source));
135 } /* int ut_config_type_datasource */
137 static int ut_config_type_instance (threshold_t *th, oconfig_item_t *ci)
139 if ((ci->values_num != 1)
140 || (ci->values[0].type != OCONFIG_TYPE_STRING))
142 WARNING ("threshold values: The `Instance' option needs exactly one "
147 sstrncpy (th->type_instance, ci->values[0].value.string,
148 sizeof (th->type_instance));
151 } /* int ut_config_type_instance */
153 static int ut_config_type_max (threshold_t *th, oconfig_item_t *ci)
155 if ((ci->values_num != 1)
156 || (ci->values[0].type != OCONFIG_TYPE_NUMBER))
158 WARNING ("threshold values: The `%s' option needs exactly one "
159 "number argument.", ci->key);
163 if (strcasecmp (ci->key, "WarningMax") == 0)
164 th->warning_max = ci->values[0].value.number;
166 th->failure_max = ci->values[0].value.number;
169 } /* int ut_config_type_max */
171 static int ut_config_type_min (threshold_t *th, oconfig_item_t *ci)
173 if ((ci->values_num != 1)
174 || (ci->values[0].type != OCONFIG_TYPE_NUMBER))
176 WARNING ("threshold values: The `%s' option needs exactly one "
177 "number argument.", ci->key);
181 if (strcasecmp (ci->key, "WarningMin") == 0)
182 th->warning_min = ci->values[0].value.number;
184 th->failure_min = ci->values[0].value.number;
187 } /* int ut_config_type_min */
189 static int ut_config_type_hits (threshold_t *th, oconfig_item_t *ci)
191 if ((ci->values_num != 1)
192 || (ci->values[0].type != OCONFIG_TYPE_NUMBER))
194 WARNING ("threshold values: The `%s' option needs exactly one "
195 "number argument.", ci->key);
199 th->hits = ci->values[0].value.number;
202 } /* int ut_config_type_hits */
204 static int ut_config_type_hysteresis (threshold_t *th, oconfig_item_t *ci)
206 if ((ci->values_num != 1)
207 || (ci->values[0].type != OCONFIG_TYPE_NUMBER))
209 WARNING ("threshold values: The `%s' option needs exactly one "
210 "number argument.", ci->key);
214 th->hysteresis = ci->values[0].value.number;
217 } /* int ut_config_type_hysteresis */
219 static int ut_config_type (const threshold_t *th_orig, oconfig_item_t *ci)
225 if ((ci->values_num != 1)
226 || (ci->values[0].type != OCONFIG_TYPE_STRING))
228 WARNING ("threshold values: The `Type' block needs exactly one string "
233 if (ci->children_num < 1)
235 WARNING ("threshold values: The `Type' block needs at least one option.");
239 memcpy (&th, th_orig, sizeof (th));
240 sstrncpy (th.type, ci->values[0].value.string, sizeof (th.type));
242 th.warning_min = NAN;
243 th.warning_max = NAN;
244 th.failure_min = NAN;
245 th.failure_max = NAN;
248 th.flags = UT_FLAG_INTERESTING; /* interesting by default */
250 for (i = 0; i < ci->children_num; i++)
252 oconfig_item_t *option = ci->children + i;
254 if (strcasecmp ("Instance", option->key) == 0)
255 status = ut_config_type_instance (&th, option);
256 else if (strcasecmp ("DataSource", option->key) == 0)
257 status = ut_config_type_datasource (&th, option);
258 else if ((strcasecmp ("WarningMax", option->key) == 0)
259 || (strcasecmp ("FailureMax", option->key) == 0))
260 status = ut_config_type_max (&th, option);
261 else if ((strcasecmp ("WarningMin", option->key) == 0)
262 || (strcasecmp ("FailureMin", option->key) == 0))
263 status = ut_config_type_min (&th, option);
264 else if (strcasecmp ("Interesting", option->key) == 0)
265 status = cf_util_get_flag (option, &th.flags, UT_FLAG_INTERESTING);
266 else if (strcasecmp ("Invert", option->key) == 0)
267 status = cf_util_get_flag (option, &th.flags, UT_FLAG_INVERT);
268 else if (strcasecmp ("Persist", option->key) == 0)
269 status = cf_util_get_flag (option, &th.flags, UT_FLAG_PERSIST);
270 else if (strcasecmp ("PersistOK", option->key) == 0)
271 status = cf_util_get_flag (option, &th.flags, UT_FLAG_PERSIST_OK);
272 else if (strcasecmp ("Percentage", option->key) == 0)
273 status = cf_util_get_flag (option, &th.flags, UT_FLAG_PERCENTAGE);
274 else if (strcasecmp ("Hits", option->key) == 0)
275 status = ut_config_type_hits (&th, option);
276 else if (strcasecmp ("Hysteresis", option->key) == 0)
277 status = ut_config_type_hysteresis (&th, option);
280 WARNING ("threshold values: Option `%s' not allowed inside a `Type' "
281 "block.", option->key);
291 status = ut_threshold_add (&th);
295 } /* int ut_config_type */
297 static int ut_config_plugin_instance (threshold_t *th, oconfig_item_t *ci)
299 if ((ci->values_num != 1)
300 || (ci->values[0].type != OCONFIG_TYPE_STRING))
302 WARNING ("threshold values: The `Instance' option needs exactly one "
307 sstrncpy (th->plugin_instance, ci->values[0].value.string,
308 sizeof (th->plugin_instance));
311 } /* int ut_config_plugin_instance */
313 static int ut_config_plugin (const threshold_t *th_orig, oconfig_item_t *ci)
319 if ((ci->values_num != 1)
320 || (ci->values[0].type != OCONFIG_TYPE_STRING))
322 WARNING ("threshold values: The `Plugin' block needs exactly one string "
327 if (ci->children_num < 1)
329 WARNING ("threshold values: The `Plugin' block needs at least one nested "
334 memcpy (&th, th_orig, sizeof (th));
335 sstrncpy (th.plugin, ci->values[0].value.string, sizeof (th.plugin));
337 for (i = 0; i < ci->children_num; i++)
339 oconfig_item_t *option = ci->children + i;
341 if (strcasecmp ("Type", option->key) == 0)
342 status = ut_config_type (&th, option);
343 else if (strcasecmp ("Instance", option->key) == 0)
344 status = ut_config_plugin_instance (&th, option);
347 WARNING ("threshold values: Option `%s' not allowed inside a `Plugin' "
348 "block.", option->key);
357 } /* int ut_config_plugin */
359 static int ut_config_host (const threshold_t *th_orig, oconfig_item_t *ci)
365 if ((ci->values_num != 1)
366 || (ci->values[0].type != OCONFIG_TYPE_STRING))
368 WARNING ("threshold values: The `Host' block needs exactly one string "
373 if (ci->children_num < 1)
375 WARNING ("threshold values: The `Host' block needs at least one nested "
380 memcpy (&th, th_orig, sizeof (th));
381 sstrncpy (th.host, ci->values[0].value.string, sizeof (th.host));
383 for (i = 0; i < ci->children_num; i++)
385 oconfig_item_t *option = ci->children + i;
387 if (strcasecmp ("Type", option->key) == 0)
388 status = ut_config_type (&th, option);
389 else if (strcasecmp ("Plugin", option->key) == 0)
390 status = ut_config_plugin (&th, option);
393 WARNING ("threshold values: Option `%s' not allowed inside a `Host' "
394 "block.", option->key);
403 } /* int ut_config_host */
405 * End of the functions used to configure threshold values.
410 * int ut_report_state
412 * Checks if the `state' differs from the old state and creates a notification
416 static int ut_report_state (const data_set_t *ds,
417 const value_list_t *vl,
418 const threshold_t *th,
419 const gauge_t *values,
431 /* Check if hits matched */
432 if ( (th->hits != 0) )
434 int hits = uc_get_hits(ds,vl);
435 /* STATE_OKAY resets hits unless PERSIST_OK flag is set. Hits resets if
436 * threshold is hit. */
437 if ( ( (state == STATE_OKAY) && ((th->flags & UT_FLAG_PERSIST_OK) == 0) ) || (hits > th->hits) )
439 DEBUG("ut_report_state: reset uc_get_hits = 0");
440 uc_set_hits(ds,vl,0); /* reset hit counter and notify */
442 DEBUG("ut_report_state: th->hits = %d, uc_get_hits = %d",th->hits,uc_get_hits(ds,vl));
443 (void) uc_inc_hits(ds,vl,1); /* increase hit counter */
446 } /* end check hits */
448 state_old = uc_get_state (ds, vl);
450 /* If the state didn't change, report if `persistent' is specified. If the
451 * state is `okay', then only report if `persist_ok` flag is set. */
452 if (state == state_old)
454 if ((th->flags & UT_FLAG_PERSIST) == 0)
456 else if ( (state == STATE_OKAY) && ((th->flags & UT_FLAG_PERSIST_OK) == 0) )
460 if (state != state_old)
461 uc_set_state (ds, vl, state);
463 NOTIFICATION_INIT_VL (&n, vl);
466 bufsize = sizeof (n.message);
468 if (state == STATE_OKAY)
469 n.severity = NOTIF_OKAY;
470 else if (state == STATE_WARNING)
471 n.severity = NOTIF_WARNING;
473 n.severity = NOTIF_FAILURE;
477 status = ssnprintf (buf, bufsize, "Host %s, plugin %s",
478 vl->host, vl->plugin);
482 if (vl->plugin_instance[0] != '\0')
484 status = ssnprintf (buf, bufsize, " (instance %s)",
485 vl->plugin_instance);
490 status = ssnprintf (buf, bufsize, " type %s", vl->type);
494 if (vl->type_instance[0] != '\0')
496 status = ssnprintf (buf, bufsize, " (instance %s)",
502 plugin_notification_meta_add_string (&n, "DataSource",
503 ds->ds[ds_index].name);
504 plugin_notification_meta_add_double (&n, "CurrentValue", values[ds_index]);
505 plugin_notification_meta_add_double (&n, "WarningMin", th->warning_min);
506 plugin_notification_meta_add_double (&n, "WarningMax", th->warning_max);
507 plugin_notification_meta_add_double (&n, "FailureMin", th->failure_min);
508 plugin_notification_meta_add_double (&n, "FailureMax", th->failure_max);
510 /* Send an okay notification */
511 if (state == STATE_OKAY)
513 if (state_old == STATE_MISSING)
514 ssnprintf (buf, bufsize, ": Value is no longer missing.");
516 ssnprintf (buf, bufsize,
517 ": All data sources are within range again. "
518 "Current value of \"%s\" is %f.",
519 ds->ds[ds_index].name, values[ds_index]);
526 min = (state == STATE_ERROR) ? th->failure_min : th->warning_min;
527 max = (state == STATE_ERROR) ? th->failure_max : th->warning_max;
529 if (th->flags & UT_FLAG_INVERT)
531 if (!isnan (min) && !isnan (max))
533 ssnprintf (buf, bufsize, ": Data source \"%s\" is currently "
534 "%f. That is within the %s region of %f%s and %f%s.",
535 ds->ds[ds_index].name, values[ds_index],
536 (state == STATE_ERROR) ? "failure" : "warning",
537 min, ((th->flags & UT_FLAG_PERCENTAGE) != 0) ? "%" : "",
538 max, ((th->flags & UT_FLAG_PERCENTAGE) != 0) ? "%" : "");
542 ssnprintf (buf, bufsize, ": Data source \"%s\" is currently "
543 "%f. That is %s the %s threshold of %f%s.",
544 ds->ds[ds_index].name, values[ds_index],
545 isnan (min) ? "below" : "above",
546 (state == STATE_ERROR) ? "failure" : "warning",
547 isnan (min) ? max : min,
548 ((th->flags & UT_FLAG_PERCENTAGE) != 0) ? "%" : "");
551 else if (th->flags & UT_FLAG_PERCENTAGE)
558 for (i = 0; i < vl->values_len; i++)
560 if (isnan (values[i]))
569 value = 100.0 * values[ds_index] / sum;
571 ssnprintf (buf, bufsize, ": Data source \"%s\" is currently "
572 "%g (%.2f%%). That is %s the %s threshold of %.2f%%.",
573 ds->ds[ds_index].name, values[ds_index], value,
574 (value < min) ? "below" : "above",
575 (state == STATE_ERROR) ? "failure" : "warning",
576 (value < min) ? min : max);
578 else /* is not inverted */
580 ssnprintf (buf, bufsize, ": Data source \"%s\" is currently "
581 "%f. That is %s the %s threshold of %f.",
582 ds->ds[ds_index].name, values[ds_index],
583 (values[ds_index] < min) ? "below" : "above",
584 (state == STATE_ERROR) ? "failure" : "warning",
585 (values[ds_index] < min) ? min : max);
589 plugin_dispatch_notification (&n);
591 plugin_notification_meta_free (n.meta);
593 } /* }}} int ut_report_state */
596 * int ut_check_one_data_source
598 * Checks one data source against the given threshold configuration. If the
599 * `DataSource' option is set in the threshold, and the name does NOT match,
600 * `okay' is returned. If the threshold does match, its failure and warning
601 * min and max values are checked and `failure' or `warning' is returned if
605 static int ut_check_one_data_source (const data_set_t *ds,
606 const value_list_t __attribute__((unused)) *vl,
607 const threshold_t *th,
608 const gauge_t *values,
614 int prev_state = STATE_OKAY;
616 /* check if this threshold applies to this data source */
619 ds_name = ds->ds[ds_index].name;
620 if ((th->data_source[0] != 0)
621 && (strcmp (ds_name, th->data_source) != 0))
625 if ((th->flags & UT_FLAG_INVERT) != 0)
631 /* XXX: This is an experimental code, not optimized, not fast, not reliable,
632 * and probably, do not work as you expect. Enjoy! :D */
633 if (th->hysteresis > 0)
635 prev_state = uc_get_state(ds,vl);
636 /* The purpose of hysteresis is elliminating flapping state when the value
637 * oscilates around the thresholds. In other words, what is important is
638 * the previous state; if the new value would trigger a transition, make
639 * sure that we artificially widen the range which is considered to apply
640 * for the previous state, and only trigger the notification if the value
641 * is outside of this expanded range.
643 * There is no hysteresis for the OKAY state.
645 gauge_t hysteresis_for_warning = 0, hysteresis_for_failure = 0;
649 hysteresis_for_failure = th->hysteresis;
652 hysteresis_for_warning = th->hysteresis;
655 /* do nothing -- the hysteresis only applies to the non-normal states */
659 if ((!isnan (th->failure_min) && (th->failure_min + hysteresis_for_failure > values[ds_index]))
660 || (!isnan (th->failure_max) && (th->failure_max - hysteresis_for_failure < values[ds_index])))
663 if ((!isnan (th->warning_min) && (th->warning_min + hysteresis_for_warning > values[ds_index]))
664 || (!isnan (th->warning_max) && (th->warning_max - hysteresis_for_warning < values[ds_index])))
668 else { /* no hysteresis */
669 if ((!isnan (th->failure_min) && (th->failure_min > values[ds_index]))
670 || (!isnan (th->failure_max) && (th->failure_max < values[ds_index])))
673 if ((!isnan (th->warning_min) && (th->warning_min > values[ds_index]))
674 || (!isnan (th->warning_max) && (th->warning_max < values[ds_index])))
679 return (STATE_ERROR);
682 return (STATE_WARNING);
685 } /* }}} int ut_check_one_data_source */
688 * int ut_check_one_threshold
690 * Checks all data sources of a value list against the given threshold, using
691 * the ut_check_one_data_source function above. Returns the worst status,
692 * which is `okay' if nothing has failed.
693 * Returns less than zero if the data set doesn't have any data sources.
695 static int ut_check_one_threshold (const data_set_t *ds,
696 const value_list_t *vl,
697 const threshold_t *th,
698 const gauge_t *values,
704 gauge_t values_copy[ds->ds_num];
706 memcpy (values_copy, values, sizeof (values_copy));
708 if ((th->flags & UT_FLAG_PERCENTAGE) != 0)
715 WARNING ("ut_check_one_threshold: The %s type has only one data "
716 "source, but you have configured to check this as a percentage. "
717 "That doesn't make much sense, because the percentage will always "
718 "be 100%%!", ds->type);
721 /* Prepare `sum' and `num'. */
722 for (i = 0; i < ds->ds_num; i++)
723 if (!isnan (values[i]))
729 if ((num == 0) /* All data sources are undefined. */
730 || (sum == 0.0)) /* Sum is zero, cannot calculate percentage. */
732 for (i = 0; i < ds->ds_num; i++)
733 values_copy[i] = NAN;
735 else /* We can actually calculate the percentage. */
737 for (i = 0; i < ds->ds_num; i++)
738 values_copy[i] = 100.0 * values[i] / sum;
740 } /* if (UT_FLAG_PERCENTAGE) */
742 for (i = 0; i < ds->ds_num; i++)
746 status = ut_check_one_data_source (ds, vl, th, values_copy, i);
752 } /* for (ds->ds_num) */
754 if (ret_ds_index != NULL)
755 *ret_ds_index = ds_index;
758 } /* }}} int ut_check_one_threshold */
761 * int ut_check_threshold
763 * Gets a list of matching thresholds and searches for the worst status by one
764 * of the thresholds. Then reports that status using the ut_report_state
766 * Returns zero on success and if no threshold has been configured. Returns
767 * less than zero on failure.
769 static int ut_check_threshold (const data_set_t *ds, const value_list_t *vl,
770 __attribute__((unused)) user_data_t *ud)
776 int worst_state = -1;
777 threshold_t *worst_th = NULL;
778 int worst_ds_index = -1;
780 if (threshold_tree == NULL)
783 /* Is this lock really necessary? So far, thresholds are only inserted at
785 pthread_mutex_lock (&threshold_lock);
786 th = threshold_search (vl);
787 pthread_mutex_unlock (&threshold_lock);
791 DEBUG ("ut_check_threshold: Found matching threshold(s)");
793 values = uc_get_rate (ds, vl);
801 status = ut_check_one_threshold (ds, vl, th, values, &ds_index);
804 ERROR ("ut_check_threshold: ut_check_one_threshold failed.");
809 if (worst_state < status)
811 worst_state = status;
813 worst_ds_index = ds_index;
819 status = ut_report_state (ds, vl, worst_th, values,
820 worst_ds_index, worst_state);
823 ERROR ("ut_check_threshold: ut_report_state failed.");
831 } /* }}} int ut_check_threshold */
836 * This function is called whenever a value goes "missing".
838 static int ut_missing (const value_list_t *vl,
839 __attribute__((unused)) user_data_t *ud)
842 cdtime_t missing_time;
843 char identifier[6 * DATA_MAX_NAME_LEN];
847 if (threshold_tree == NULL)
850 th = threshold_search (vl);
851 /* dispatch notifications for "interesting" values only */
852 if ((th == NULL) || ((th->flags & UT_FLAG_INTERESTING) == 0))
856 missing_time = now - vl->time;
857 FORMAT_VL (identifier, sizeof (identifier), vl);
859 NOTIFICATION_INIT_VL (&n, vl);
860 ssnprintf (n.message, sizeof (n.message),
861 "%s has not been updated for %.3f seconds.",
862 identifier, CDTIME_T_TO_DOUBLE (missing_time));
865 plugin_dispatch_notification (&n);
868 } /* }}} int ut_missing */
870 static int ut_config (oconfig_item_t *ci)
874 int old_size = c_avl_size (threshold_tree);
878 if (threshold_tree == NULL)
880 threshold_tree = c_avl_create ((void *) strcmp);
881 if (threshold_tree == NULL)
883 ERROR ("ut_config: c_avl_create failed.");
888 memset (&th, '\0', sizeof (th));
889 th.warning_min = NAN;
890 th.warning_max = NAN;
891 th.failure_min = NAN;
892 th.failure_max = NAN;
896 th.flags = UT_FLAG_INTERESTING; /* interesting by default */
898 for (i = 0; i < ci->children_num; i++)
900 oconfig_item_t *option = ci->children + i;
902 if (strcasecmp ("Type", option->key) == 0)
903 status = ut_config_type (&th, option);
904 else if (strcasecmp ("Plugin", option->key) == 0)
905 status = ut_config_plugin (&th, option);
906 else if (strcasecmp ("Host", option->key) == 0)
907 status = ut_config_host (&th, option);
910 WARNING ("threshold values: Option `%s' not allowed here.", option->key);
918 /* register callbacks if this is the first time we see a valid config */
919 if ((old_size == 0) && (c_avl_size (threshold_tree) > 0))
921 plugin_register_missing ("threshold", ut_missing,
922 /* user data = */ NULL);
923 plugin_register_write ("threshold", ut_check_threshold,
924 /* user data = */ NULL);
928 } /* }}} int um_config */
930 void module_register (void)
932 plugin_register_complex_config ("threshold", ut_config);
935 /* vim: set sw=2 ts=8 sts=2 tw=78 et fdm=marker : */