Merge pull request #360 from pyr/feature-log-logstash
authorPierre-Yves Ritschard <pyr@spootnik.org>
Fri, 25 Jul 2014 14:32:11 +0000 (16:32 +0200)
committerPierre-Yves Ritschard <pyr@spootnik.org>
Fri, 25 Jul 2014 14:32:11 +0000 (16:32 +0200)
add log_logstash plugin to emit logstash json_event messages.

configure.ac
src/Makefile.am
src/collectd.conf.in
src/collectd.conf.pod
src/log_logstash.c [new file with mode: 0644]

index f09887e..b95e322 100644 (file)
@@ -4910,6 +4910,7 @@ plugin_ipvs="no"
 plugin_irq="no"
 plugin_libvirt="no"
 plugin_load="no"
+plugin_log_logstash="no"
 plugin_memory="no"
 plugin_multimeter="no"
 plugin_nfs="no"
@@ -5140,6 +5141,11 @@ then
        plugin_load="yes"
 fi
 
+if test "x$with_libyajl" = "xyes"
+then
+       plugin_log_logstash="yes"
+fi
+
 if test "x$c_cv_have_libperl$c_cv_have_perl_ithreads" = "xyesyes"
 then
        plugin_perl="yes"
@@ -5250,6 +5256,7 @@ AC_PLUGIN([java],        [$with_java],         [Embed the Java Virtual Machine])
 AC_PLUGIN([libvirt],     [$plugin_libvirt],    [Virtual machine statistics])
 AC_PLUGIN([load],        [$plugin_load],       [System load])
 AC_PLUGIN([logfile],     [yes],                [File logging plugin])
+AC_PLUGIN([log_logstash], [$plugin_log_logstash], [Logstash json_event compatible logging])
 AC_PLUGIN([lpar],        [$with_perfstat],     [AIX logical partitions statistics])
 AC_PLUGIN([lvm],         [$with_liblvm2app],   [LVM statistics])
 AC_PLUGIN([madwifi],     [$have_linux_wireless_h], [Madwifi wireless statistics])
@@ -5599,6 +5606,7 @@ Configuration:
     load  . . . . . . . . $enable_load
     logfile . . . . . . . $enable_logfile
     lpar  . . . . . . . . $enable_lpar
+    log_logstash ........ $enable_log_logstash
     lvm . . . . . . . . . $enable_lvm
     madwifi . . . . . . . $enable_madwifi
     match_empty_counter . $enable_match_empty_counter
index 5e8f76a..8699526 100644 (file)
@@ -582,6 +582,17 @@ collectd_LDADD += "-dlopen" logfile.la
 collectd_DEPENDENCIES += logfile.la
 endif
 
+if BUILD_PLUGIN_LOG_LOGSTASH
+pkglib_LTLIBRARIES += log_logstash.la
+log_logstash_la_SOURCES = log_logstash.c
+log_logstash_la_CFLAGS = $(AM_CFLAGS)
+log_logstash_la_LDFLAGS = -module -avoid-version $(BUILD_WITH_LIBYAJL_LDFLAGS)
+log_logstash_la_CPPFLAGS = $(AM_CPPFLAGS) $(BUILD_WITH_LIBYAJL_CPPFLAGS)
+log_logstash_la_LIBADD = $(BUILD_WITH_LIBYAJL_LIBS)
+collectd_LDADD += "-dlopen" log_logstash.la
+collectd_DEPENDENCIES += log_logstash.la
+endif
+
 if BUILD_PLUGIN_LPAR
 pkglib_LTLIBRARIES += lpar.la
 lpar_la_SOURCES = lpar.c
index 667cd52..c689aaf 100644 (file)
@@ -52,6 +52,7 @@
 
 @LOAD_PLUGIN_SYSLOG@LoadPlugin syslog
 @LOAD_PLUGIN_LOGFILE@LoadPlugin logfile
+@LOAD_PLUGIN_LOG_LOGSTASH@LoadPlugin log_logstash
 
 #<Plugin logfile>
 #      LogLevel @DEFAULT_LOG_LEVEL@
 #      PrintSeverity false
 #</Plugin>
 
+#<Plugin log_logstash>
+#      LogLevel @DEFAULT_LOG_LEVEL@
+#      File "@localstatedir@/log/@PACKAGE_NAME@.json.log"
+#</Plugin>
+
 #<Plugin syslog>
 #      LogLevel @DEFAULT_LOG_LEVEL@
 #</Plugin>
index 39dea7e..70ae78b 100644 (file)
@@ -2305,6 +2305,34 @@ B<Note>: There is no need to notify the daemon after moving or removing the
 log file (e.E<nbsp>g. when rotating the logs). The plugin reopens the file
 for each line it writes.
 
+=head2 Plugin C<log_logstash>
+
+The I<log logstash plugin> behaves like the logfile plugin but formats
+messages as JSON events for logstash to parse and input.
+
+=over 4
+
+=item B<LogLevel> B<debug|info|notice|warning|err>
+
+Sets the log-level. If, for example, set to B<notice>, then all events with
+severity B<notice>, B<warning>, or B<err> will be written to the logfile.
+
+Please note that B<debug> is only available if collectd has been compiled with
+debugging support.
+
+=item B<File> I<File>
+
+Sets the file to write log messages to. The special strings B<stdout> and
+B<stderr> can be used to write to the standard output and standard error
+channels, respectively. This, of course, only makes much sense when I<collectd>
+is running in foreground- or non-daemon-mode.
+
+=back
+
+B<Note>: There is no need to notify the daemon after moving or removing the
+log file (e.E<nbsp>g. when rotating the logs). The plugin reopens the file
+for each line it writes.
+
 =head2 Plugin C<lpar>
 
 The I<LPAR plugin> reads CPU statistics of I<Logical Partitions>, a
diff --git a/src/log_logstash.c b/src/log_logstash.c
new file mode 100644 (file)
index 0000000..6cb5b0a
--- /dev/null
@@ -0,0 +1,389 @@
+/**
+ * collectd - src/log_logstash.c
+ * Copyright (C) 2013 Pierre-Yves Ritschard
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; only version 2 of the License is applicable.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ * Authors:
+ *   Pierre-Yves Ritschard <pyr at spootnik.org>
+ * Acknowledgements:
+ *   This file is largely inspired by logfile.c
+ **/
+
+#include "collectd.h"
+#include "common.h"
+#include "plugin.h"
+
+#include <sys/types.h>
+#include <pthread.h>
+#include <yajl/yajl_common.h>
+#include <yajl/yajl_gen.h>
+#if HAVE_YAJL_YAJL_VERSION_H
+# include <yajl/yajl_version.h>
+#endif
+#if defined(YAJL_MAJOR) && (YAJL_MAJOR > 1)
+# define HAVE_YAJL_V2 1
+#endif
+
+#define DEFAULT_LOGFILE LOCALSTATEDIR"/log/"PACKAGE_NAME".json.log"
+
+#if COLLECT_DEBUG
+static int log_level = LOG_DEBUG;
+#else
+static int log_level = LOG_INFO;
+#endif /* COLLECT_DEBUG */
+
+static pthread_mutex_t file_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static char *log_file = NULL;
+
+static const char *config_keys[] =
+{
+       "LogLevel",
+       "File"
+};
+static int config_keys_num = STATIC_ARRAY_SIZE (config_keys);
+
+static int log_logstash_config (const char *key, const char *value)
+{
+
+       if (0 == strcasecmp (key, "LogLevel")) {
+               log_level = parse_log_severity(value);
+        if (log_level < 0) {
+            log_level = LOG_INFO;
+            ERROR("log_logstash: invalid loglevel [%s] defaulting to 'info'",
+                  value);
+            return 1;
+        }
+       }
+       else if (0 == strcasecmp (key, "File")) {
+               sfree (log_file);
+               log_file = strdup (value);
+       }
+       else {
+               return -1;
+       }
+       return 0;
+} /* int log_logstash_config (const char *, const char *) */
+
+static void log_logstash_print (yajl_gen g, int severity,
+               cdtime_t timestamp_time)
+{
+       FILE *fh;
+       _Bool do_close = 0;
+       struct tm timestamp_tm;
+       char timestamp_str[64];
+       const unsigned char *buf;
+       time_t tt;
+#if HAVE_YAJL_V2
+       size_t len;
+#else
+       unsigned int len;
+#endif
+
+       if (yajl_gen_string(g, (u_char *)"@level", strlen("@level")) !=
+           yajl_gen_status_ok)
+               goto err;
+
+       switch (severity) {
+       case LOG_ERR:
+               if (yajl_gen_string(g, (u_char *)"error", strlen("error")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       case LOG_WARNING:
+               if (yajl_gen_string(g, (u_char *)"warning",
+                                   strlen("warning")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       case LOG_NOTICE:
+               if (yajl_gen_string(g, (u_char *)"notice", strlen("notice")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       case LOG_INFO:
+               if (yajl_gen_string(g, (u_char *)"info", strlen("info")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       case LOG_DEBUG:
+               if (yajl_gen_string(g, (u_char *)"debug", strlen("debug")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       default:
+               if (yajl_gen_string(g, (u_char *)"unknown",
+                                   strlen("unknown")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       }
+
+       if (yajl_gen_string(g, (u_char *)"@timestamp", strlen("@timestamp")) !=
+           yajl_gen_status_ok)
+               goto err;
+
+       tt = CDTIME_T_TO_TIME_T (timestamp_time);
+       gmtime_r (&tt, &timestamp_tm);
+
+       /*
+        * format time as a UTC ISO 8601 compliant string
+        */
+       strftime (timestamp_str, sizeof (timestamp_str),
+                 "%Y-%m-%d %H:%M:%SZ", &timestamp_tm);
+       timestamp_str[sizeof (timestamp_str) - 1] = '\0';
+
+       if (yajl_gen_string(g, (u_char *)timestamp_str,
+                           strlen(timestamp_str)) !=
+           yajl_gen_status_ok)
+               goto err;
+
+       if (yajl_gen_map_close(g) != yajl_gen_status_ok)
+               goto err;
+
+       if (yajl_gen_get_buf(g, &buf, &len) != yajl_gen_status_ok)
+               goto err;
+       pthread_mutex_lock (&file_lock);
+
+       if (log_file == NULL)
+       {
+               fh = fopen (DEFAULT_LOGFILE, "a");
+               do_close = 1;
+       } else if (strcasecmp(log_file, "stdout") == 0) {
+        fh = stdout;
+        do_close = 0;
+       } else if (strcasecmp(log_file, "stderr") == 0) {
+        fh = stderr;
+        do_close = 0;
+       } else {
+               fh = fopen (log_file, "a");
+               do_close = 1;
+       }
+
+       if (fh == NULL)
+       {
+                       char errbuf[1024];
+                       fprintf (stderr, "log_logstash plugin: fopen (%s) failed: %s\n",
+                                       (log_file == NULL) ? DEFAULT_LOGFILE : log_file,
+                                       sstrerror (errno, errbuf, sizeof (errbuf)));
+       }
+       else
+       {
+               fprintf(fh, "%s\n", buf);
+               if (do_close) {
+                       fclose (fh);
+               } else {
+                       fflush(fh);
+               }
+       }
+       pthread_mutex_unlock (&file_lock);
+       yajl_gen_free(g);
+       return;
+
+ err:
+       yajl_gen_free(g);
+       fprintf(stderr, "Could not correctly generate JSON message\n");
+       return;
+} /* void log_logstash_print */
+
+static void log_logstash_log (int severity, const char *msg,
+               user_data_t __attribute__((unused)) *user_data)
+{
+       yajl_gen g;
+#if !defined(HAVE_YAJL_V2)
+       yajl_gen_config conf;
+
+       conf.beautify = 0;
+#endif
+
+       if (severity > log_level)
+               return;
+
+#if HAVE_YAJL_V2
+       g = yajl_gen_alloc(NULL);
+#else
+       g = yajl_gen_alloc(&conf, NULL);
+#endif
+
+       if (g == NULL) {
+               fprintf(stderr, "Could not allocate JSON generator.\n");
+               return;
+       }
+
+       if (yajl_gen_map_open(g) != yajl_gen_status_ok)
+               goto err;
+       if (yajl_gen_string(g, (u_char *)"@message", strlen("@message")) !=
+           yajl_gen_status_ok)
+               goto err;
+       if (yajl_gen_string(g, (u_char *)msg, strlen(msg)) !=
+           yajl_gen_status_ok)
+               goto err;
+
+       log_logstash_print (g, severity, cdtime ());
+       return;
+ err:
+       yajl_gen_free(g);
+       fprintf(stderr, "Could not generate JSON message preamble\n");
+       return;
+
+} /* void log_logstash_log (int, const char *) */
+
+static int log_logstash_notification (const notification_t *n,
+               user_data_t __attribute__((unused)) *user_data)
+{
+       yajl_gen g;
+#if HAVE_YAJL_V2
+       g = yajl_gen_alloc(NULL);
+#else
+       yajl_gen_config conf;
+
+       conf.beautify = 0;
+       g = yajl_gen_alloc(&conf, NULL);
+#endif
+
+       if (g == NULL) {
+               fprintf(stderr, "Could not allocate JSON generator.\n");
+               return (0);
+       }
+
+       if (yajl_gen_map_open(g) != yajl_gen_status_ok)
+               goto err;
+       if (yajl_gen_string(g, (u_char *)"@message", strlen("@message")) !=
+           yajl_gen_status_ok)
+               goto err;
+       if (strlen(n->message) > 0) {
+               if (yajl_gen_string(g, (u_char *)n->message,
+                                   strlen(n->message)) !=
+                   yajl_gen_status_ok)
+                       goto err;
+       } else {
+               if (yajl_gen_string(g, (u_char *)"notification without a message",
+                                   strlen("notification without a message")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+       }
+
+
+       if (yajl_gen_string(g, (u_char *)"@fields", strlen("@fields")) !=
+           yajl_gen_status_ok)
+               goto err;
+       if (yajl_gen_map_open(g) !=
+           yajl_gen_status_ok)
+               goto err;
+
+       if (strlen(n->host) > 0) {
+               if (yajl_gen_string(g, (u_char *)"host", strlen("host")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               if (yajl_gen_string(g, (u_char *)n->host, strlen(n->host)) !=
+                   yajl_gen_status_ok)
+                       goto err;
+
+       }
+       if (strlen(n->plugin) > 0) {
+               if (yajl_gen_string(g, (u_char *)"plugin", strlen("plugin")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               if (yajl_gen_string(g, (u_char *)n->plugin, strlen(n->plugin)) !=
+                   yajl_gen_status_ok)
+                       goto err;
+       }
+       if (strlen(n->plugin_instance) > 0) {
+               if (yajl_gen_string(g, (u_char *)"plugin_instance",
+                                   strlen("plugin_instance")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               if (yajl_gen_string(g, (u_char *)n->plugin_instance,
+                                   strlen(n->plugin_instance)) !=
+                   yajl_gen_status_ok)
+                       goto err;
+       }
+       if (strlen(n->type) > 0) {
+               if (yajl_gen_string(g, (u_char *)"type", strlen("type")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               if (yajl_gen_string(g, (u_char *)n->type, strlen(n->type)) !=
+                   yajl_gen_status_ok)
+                       goto err;
+       }
+       if (strlen(n->type_instance) > 0) {
+               if (yajl_gen_string(g, (u_char *)"type_instance",
+                                   strlen("type_instance")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               if (yajl_gen_string(g, (u_char *)n->type_instance,
+                                   strlen(n->type_instance)) !=
+                   yajl_gen_status_ok)
+                       goto err;
+       }
+
+       if (yajl_gen_string(g, (u_char *)"severity",
+                           strlen("severity")) !=
+           yajl_gen_status_ok)
+               goto err;
+
+       switch (n->severity) {
+       case NOTIF_FAILURE:
+               if (yajl_gen_string(g, (u_char *)"failure",
+                                   strlen("failure")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       case NOTIF_WARNING:
+               if (yajl_gen_string(g, (u_char *)"warning",
+                                   strlen("warning")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       case NOTIF_OKAY:
+               if (yajl_gen_string(g, (u_char *)"ok",
+                                   strlen("ok")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       default:
+               if (yajl_gen_string(g, (u_char *)"unknown",
+                                   strlen("unknown")) !=
+                   yajl_gen_status_ok)
+                       goto err;
+               break;
+       }
+       if (yajl_gen_map_close(g) != yajl_gen_status_ok)
+               goto err;
+
+       log_logstash_print (g, LOG_INFO, (n->time != 0) ? n->time : cdtime ());
+       return (0);
+
+ err:
+       yajl_gen_free(g);
+       fprintf(stderr, "Could not correctly generate JSON notification\n");
+       return (0);
+} /* int log_logstash_notification */
+
+void module_register (void)
+{
+       plugin_register_config ("log_logstash",
+                               log_logstash_config,
+                               config_keys,
+                               config_keys_num);
+       plugin_register_log ("log_logstash",
+                            log_logstash_log,
+                            /* user_data = */ NULL);
+       plugin_register_notification ("log_logstash",
+                                     log_logstash_notification,
+                                     /* user_data = */ NULL);
+} /* void module_register (void) */
+
+/* vim: set sw=4 ts=4 tw=78 noexpandtab : */