From: Pierre-Yves Ritschard Date: Fri, 25 Jul 2014 14:32:11 +0000 (+0200) Subject: Merge pull request #360 from pyr/feature-log-logstash X-Git-Tag: collectd-5.5.0~288 X-Git-Url: https://git.octo.it/?a=commitdiff_plain;h=082be7a56f25e0561be6ce87e5662e01b1e91975;hp=70160d8f57c9a767ae2ca14e31c8687ecbb10db3;p=collectd.git Merge pull request #360 from pyr/feature-log-logstash add log_logstash plugin to emit logstash json_event messages. --- diff --git a/configure.ac b/configure.ac index f09887e4..b95e3229 100644 --- a/configure.ac +++ b/configure.ac @@ -4910,6 +4910,7 @@ plugin_ipvs="no" plugin_irq="no" plugin_libvirt="no" plugin_load="no" +plugin_log_logstash="no" plugin_memory="no" plugin_multimeter="no" plugin_nfs="no" @@ -5140,6 +5141,11 @@ then plugin_load="yes" fi +if test "x$with_libyajl" = "xyes" +then + plugin_log_logstash="yes" +fi + if test "x$c_cv_have_libperl$c_cv_have_perl_ithreads" = "xyesyes" then plugin_perl="yes" @@ -5250,6 +5256,7 @@ AC_PLUGIN([java], [$with_java], [Embed the Java Virtual Machine]) AC_PLUGIN([libvirt], [$plugin_libvirt], [Virtual machine statistics]) AC_PLUGIN([load], [$plugin_load], [System load]) AC_PLUGIN([logfile], [yes], [File logging plugin]) +AC_PLUGIN([log_logstash], [$plugin_log_logstash], [Logstash json_event compatible logging]) AC_PLUGIN([lpar], [$with_perfstat], [AIX logical partitions statistics]) AC_PLUGIN([lvm], [$with_liblvm2app], [LVM statistics]) AC_PLUGIN([madwifi], [$have_linux_wireless_h], [Madwifi wireless statistics]) @@ -5599,6 +5606,7 @@ Configuration: load . . . . . . . . $enable_load logfile . . . . . . . $enable_logfile lpar . . . . . . . . $enable_lpar + log_logstash ........ $enable_log_logstash lvm . . . . . . . . . $enable_lvm madwifi . . . . . . . $enable_madwifi match_empty_counter . $enable_match_empty_counter diff --git a/src/Makefile.am b/src/Makefile.am index 5e8f76ae..86995265 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -582,6 +582,17 @@ collectd_LDADD += "-dlopen" logfile.la collectd_DEPENDENCIES += logfile.la endif +if BUILD_PLUGIN_LOG_LOGSTASH +pkglib_LTLIBRARIES += log_logstash.la +log_logstash_la_SOURCES = log_logstash.c +log_logstash_la_CFLAGS = $(AM_CFLAGS) +log_logstash_la_LDFLAGS = -module -avoid-version $(BUILD_WITH_LIBYAJL_LDFLAGS) +log_logstash_la_CPPFLAGS = $(AM_CPPFLAGS) $(BUILD_WITH_LIBYAJL_CPPFLAGS) +log_logstash_la_LIBADD = $(BUILD_WITH_LIBYAJL_LIBS) +collectd_LDADD += "-dlopen" log_logstash.la +collectd_DEPENDENCIES += log_logstash.la +endif + if BUILD_PLUGIN_LPAR pkglib_LTLIBRARIES += lpar.la lpar_la_SOURCES = lpar.c diff --git a/src/collectd.conf.in b/src/collectd.conf.in index 667cd527..c689aaff 100644 --- a/src/collectd.conf.in +++ b/src/collectd.conf.in @@ -52,6 +52,7 @@ @LOAD_PLUGIN_SYSLOG@LoadPlugin syslog @LOAD_PLUGIN_LOGFILE@LoadPlugin logfile +@LOAD_PLUGIN_LOG_LOGSTASH@LoadPlugin log_logstash # # LogLevel @DEFAULT_LOG_LEVEL@ @@ -60,6 +61,11 @@ # PrintSeverity false # +# +# LogLevel @DEFAULT_LOG_LEVEL@ +# File "@localstatedir@/log/@PACKAGE_NAME@.json.log" +# + # # LogLevel @DEFAULT_LOG_LEVEL@ # diff --git a/src/collectd.conf.pod b/src/collectd.conf.pod index 39dea7e5..70ae78b6 100644 --- a/src/collectd.conf.pod +++ b/src/collectd.conf.pod @@ -2305,6 +2305,34 @@ B: There is no need to notify the daemon after moving or removing the log file (e.Eg. when rotating the logs). The plugin reopens the file for each line it writes. +=head2 Plugin C + +The I behaves like the logfile plugin but formats +messages as JSON events for logstash to parse and input. + +=over 4 + +=item B B + +Sets the log-level. If, for example, set to B, then all events with +severity B, B, or B will be written to the logfile. + +Please note that B is only available if collectd has been compiled with +debugging support. + +=item B I + +Sets the file to write log messages to. The special strings B and +B can be used to write to the standard output and standard error +channels, respectively. This, of course, only makes much sense when I +is running in foreground- or non-daemon-mode. + +=back + +B: There is no need to notify the daemon after moving or removing the +log file (e.Eg. when rotating the logs). The plugin reopens the file +for each line it writes. + =head2 Plugin C The I reads CPU statistics of I, a diff --git a/src/log_logstash.c b/src/log_logstash.c new file mode 100644 index 00000000..6cb5b0a1 --- /dev/null +++ b/src/log_logstash.c @@ -0,0 +1,389 @@ +/** + * collectd - src/log_logstash.c + * Copyright (C) 2013 Pierre-Yves Ritschard + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; only version 2 of the License is applicable. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Pierre-Yves Ritschard + * Acknowledgements: + * This file is largely inspired by logfile.c + **/ + +#include "collectd.h" +#include "common.h" +#include "plugin.h" + +#include +#include +#include +#include +#if HAVE_YAJL_YAJL_VERSION_H +# include +#endif +#if defined(YAJL_MAJOR) && (YAJL_MAJOR > 1) +# define HAVE_YAJL_V2 1 +#endif + +#define DEFAULT_LOGFILE LOCALSTATEDIR"/log/"PACKAGE_NAME".json.log" + +#if COLLECT_DEBUG +static int log_level = LOG_DEBUG; +#else +static int log_level = LOG_INFO; +#endif /* COLLECT_DEBUG */ + +static pthread_mutex_t file_lock = PTHREAD_MUTEX_INITIALIZER; + +static char *log_file = NULL; + +static const char *config_keys[] = +{ + "LogLevel", + "File" +}; +static int config_keys_num = STATIC_ARRAY_SIZE (config_keys); + +static int log_logstash_config (const char *key, const char *value) +{ + + if (0 == strcasecmp (key, "LogLevel")) { + log_level = parse_log_severity(value); + if (log_level < 0) { + log_level = LOG_INFO; + ERROR("log_logstash: invalid loglevel [%s] defaulting to 'info'", + value); + return 1; + } + } + else if (0 == strcasecmp (key, "File")) { + sfree (log_file); + log_file = strdup (value); + } + else { + return -1; + } + return 0; +} /* int log_logstash_config (const char *, const char *) */ + +static void log_logstash_print (yajl_gen g, int severity, + cdtime_t timestamp_time) +{ + FILE *fh; + _Bool do_close = 0; + struct tm timestamp_tm; + char timestamp_str[64]; + const unsigned char *buf; + time_t tt; +#if HAVE_YAJL_V2 + size_t len; +#else + unsigned int len; +#endif + + if (yajl_gen_string(g, (u_char *)"@level", strlen("@level")) != + yajl_gen_status_ok) + goto err; + + switch (severity) { + case LOG_ERR: + if (yajl_gen_string(g, (u_char *)"error", strlen("error")) != + yajl_gen_status_ok) + goto err; + break; + case LOG_WARNING: + if (yajl_gen_string(g, (u_char *)"warning", + strlen("warning")) != + yajl_gen_status_ok) + goto err; + break; + case LOG_NOTICE: + if (yajl_gen_string(g, (u_char *)"notice", strlen("notice")) != + yajl_gen_status_ok) + goto err; + break; + case LOG_INFO: + if (yajl_gen_string(g, (u_char *)"info", strlen("info")) != + yajl_gen_status_ok) + goto err; + break; + case LOG_DEBUG: + if (yajl_gen_string(g, (u_char *)"debug", strlen("debug")) != + yajl_gen_status_ok) + goto err; + break; + default: + if (yajl_gen_string(g, (u_char *)"unknown", + strlen("unknown")) != + yajl_gen_status_ok) + goto err; + break; + } + + if (yajl_gen_string(g, (u_char *)"@timestamp", strlen("@timestamp")) != + yajl_gen_status_ok) + goto err; + + tt = CDTIME_T_TO_TIME_T (timestamp_time); + gmtime_r (&tt, ×tamp_tm); + + /* + * format time as a UTC ISO 8601 compliant string + */ + strftime (timestamp_str, sizeof (timestamp_str), + "%Y-%m-%d %H:%M:%SZ", ×tamp_tm); + timestamp_str[sizeof (timestamp_str) - 1] = '\0'; + + if (yajl_gen_string(g, (u_char *)timestamp_str, + strlen(timestamp_str)) != + yajl_gen_status_ok) + goto err; + + if (yajl_gen_map_close(g) != yajl_gen_status_ok) + goto err; + + if (yajl_gen_get_buf(g, &buf, &len) != yajl_gen_status_ok) + goto err; + pthread_mutex_lock (&file_lock); + + if (log_file == NULL) + { + fh = fopen (DEFAULT_LOGFILE, "a"); + do_close = 1; + } else if (strcasecmp(log_file, "stdout") == 0) { + fh = stdout; + do_close = 0; + } else if (strcasecmp(log_file, "stderr") == 0) { + fh = stderr; + do_close = 0; + } else { + fh = fopen (log_file, "a"); + do_close = 1; + } + + if (fh == NULL) + { + char errbuf[1024]; + fprintf (stderr, "log_logstash plugin: fopen (%s) failed: %s\n", + (log_file == NULL) ? DEFAULT_LOGFILE : log_file, + sstrerror (errno, errbuf, sizeof (errbuf))); + } + else + { + fprintf(fh, "%s\n", buf); + if (do_close) { + fclose (fh); + } else { + fflush(fh); + } + } + pthread_mutex_unlock (&file_lock); + yajl_gen_free(g); + return; + + err: + yajl_gen_free(g); + fprintf(stderr, "Could not correctly generate JSON message\n"); + return; +} /* void log_logstash_print */ + +static void log_logstash_log (int severity, const char *msg, + user_data_t __attribute__((unused)) *user_data) +{ + yajl_gen g; +#if !defined(HAVE_YAJL_V2) + yajl_gen_config conf; + + conf.beautify = 0; +#endif + + if (severity > log_level) + return; + +#if HAVE_YAJL_V2 + g = yajl_gen_alloc(NULL); +#else + g = yajl_gen_alloc(&conf, NULL); +#endif + + if (g == NULL) { + fprintf(stderr, "Could not allocate JSON generator.\n"); + return; + } + + if (yajl_gen_map_open(g) != yajl_gen_status_ok) + goto err; + if (yajl_gen_string(g, (u_char *)"@message", strlen("@message")) != + yajl_gen_status_ok) + goto err; + if (yajl_gen_string(g, (u_char *)msg, strlen(msg)) != + yajl_gen_status_ok) + goto err; + + log_logstash_print (g, severity, cdtime ()); + return; + err: + yajl_gen_free(g); + fprintf(stderr, "Could not generate JSON message preamble\n"); + return; + +} /* void log_logstash_log (int, const char *) */ + +static int log_logstash_notification (const notification_t *n, + user_data_t __attribute__((unused)) *user_data) +{ + yajl_gen g; +#if HAVE_YAJL_V2 + g = yajl_gen_alloc(NULL); +#else + yajl_gen_config conf; + + conf.beautify = 0; + g = yajl_gen_alloc(&conf, NULL); +#endif + + if (g == NULL) { + fprintf(stderr, "Could not allocate JSON generator.\n"); + return (0); + } + + if (yajl_gen_map_open(g) != yajl_gen_status_ok) + goto err; + if (yajl_gen_string(g, (u_char *)"@message", strlen("@message")) != + yajl_gen_status_ok) + goto err; + if (strlen(n->message) > 0) { + if (yajl_gen_string(g, (u_char *)n->message, + strlen(n->message)) != + yajl_gen_status_ok) + goto err; + } else { + if (yajl_gen_string(g, (u_char *)"notification without a message", + strlen("notification without a message")) != + yajl_gen_status_ok) + goto err; + } + + + if (yajl_gen_string(g, (u_char *)"@fields", strlen("@fields")) != + yajl_gen_status_ok) + goto err; + if (yajl_gen_map_open(g) != + yajl_gen_status_ok) + goto err; + + if (strlen(n->host) > 0) { + if (yajl_gen_string(g, (u_char *)"host", strlen("host")) != + yajl_gen_status_ok) + goto err; + if (yajl_gen_string(g, (u_char *)n->host, strlen(n->host)) != + yajl_gen_status_ok) + goto err; + + } + if (strlen(n->plugin) > 0) { + if (yajl_gen_string(g, (u_char *)"plugin", strlen("plugin")) != + yajl_gen_status_ok) + goto err; + if (yajl_gen_string(g, (u_char *)n->plugin, strlen(n->plugin)) != + yajl_gen_status_ok) + goto err; + } + if (strlen(n->plugin_instance) > 0) { + if (yajl_gen_string(g, (u_char *)"plugin_instance", + strlen("plugin_instance")) != + yajl_gen_status_ok) + goto err; + if (yajl_gen_string(g, (u_char *)n->plugin_instance, + strlen(n->plugin_instance)) != + yajl_gen_status_ok) + goto err; + } + if (strlen(n->type) > 0) { + if (yajl_gen_string(g, (u_char *)"type", strlen("type")) != + yajl_gen_status_ok) + goto err; + if (yajl_gen_string(g, (u_char *)n->type, strlen(n->type)) != + yajl_gen_status_ok) + goto err; + } + if (strlen(n->type_instance) > 0) { + if (yajl_gen_string(g, (u_char *)"type_instance", + strlen("type_instance")) != + yajl_gen_status_ok) + goto err; + if (yajl_gen_string(g, (u_char *)n->type_instance, + strlen(n->type_instance)) != + yajl_gen_status_ok) + goto err; + } + + if (yajl_gen_string(g, (u_char *)"severity", + strlen("severity")) != + yajl_gen_status_ok) + goto err; + + switch (n->severity) { + case NOTIF_FAILURE: + if (yajl_gen_string(g, (u_char *)"failure", + strlen("failure")) != + yajl_gen_status_ok) + goto err; + break; + case NOTIF_WARNING: + if (yajl_gen_string(g, (u_char *)"warning", + strlen("warning")) != + yajl_gen_status_ok) + goto err; + break; + case NOTIF_OKAY: + if (yajl_gen_string(g, (u_char *)"ok", + strlen("ok")) != + yajl_gen_status_ok) + goto err; + break; + default: + if (yajl_gen_string(g, (u_char *)"unknown", + strlen("unknown")) != + yajl_gen_status_ok) + goto err; + break; + } + if (yajl_gen_map_close(g) != yajl_gen_status_ok) + goto err; + + log_logstash_print (g, LOG_INFO, (n->time != 0) ? n->time : cdtime ()); + return (0); + + err: + yajl_gen_free(g); + fprintf(stderr, "Could not correctly generate JSON notification\n"); + return (0); +} /* int log_logstash_notification */ + +void module_register (void) +{ + plugin_register_config ("log_logstash", + log_logstash_config, + config_keys, + config_keys_num); + plugin_register_log ("log_logstash", + log_logstash_log, + /* user_data = */ NULL); + plugin_register_notification ("log_logstash", + log_logstash_notification, + /* user_data = */ NULL); +} /* void module_register (void) */ + +/* vim: set sw=4 ts=4 tw=78 noexpandtab : */