Merge remote-tracking branch 'github/pr/2469'

diff --git a/src/collectd.conf.in b/src/collectd.conf.in
index 6bddd8c..826b486 100644 (file)
--- a/src/collectd.conf.in
+++ b/src/collectd.conf.in
@@ -1560,6 +1560,7 @@
 #      CollectManagement false    # Varnish 4 only
 #      CollectSMF false           # Varnish 4 only
 #      CollectVBE false           # Varnish 4 only
+#      CollectMSE false           # Varnish-Plus 4 only
 #   </Instance>
 #</Plugin>
 

diff --git a/src/collectd.conf.pod b/src/collectd.conf.pod
index 14222c6..7b037bc 100644 (file)
--- a/src/collectd.conf.pod
+++ b/src/collectd.conf.pod
@@ -8438,6 +8438,7 @@ Synopsis:
      CollectManagement  false
      CollectSMF         false
      CollectVBE         false
+     CollectMSE         false
    </Instance>
  </Plugin>
 
@@ -8513,11 +8514,9 @@ log messages which is flushed to disk when full. True by default.
 =item B<CollectSMA> B<true>|B<false>
 
 malloc or umem (umem_alloc(3MALLOC) based) storage statistics. The umem storage
-component is Solaris specific.
-Note: SMA and SMF share counters, enable only the one used by the Varnish
-instance.
-Only available with Varnish 2.x. False by
-default.
+component is Solaris specific. Note: SMA, SMF and MSE share counters, enable
+only the one used by the Varnish instance. Only available with Varnish 2.x.
+False by default.
 
 =item B<CollectSMS> B<true>|B<false>
 
@@ -8565,9 +8564,8 @@ Backend counters. Only available with Varnish 4.x. False by default.
 =item B<CollectSMF> B<true>|B<false>
 
 file (memory mapped file) storage statistics. Only available with Varnish 4.x.
-Note: SMA and SMF share counters, enable only the one used by the Varnish
-instance.
-Used to be called SM in Varnish 2.x. False by default.
+Note: SMA, SMF and MSE share counters, enable only the one used by the Varnish
+instance. Used to be called SM in Varnish 2.x. False by default.
 
 =item B<CollectManagement> B<true>|B<false>
 
@@ -8581,6 +8579,13 @@ Lock counters. Only available with Varnish 4.x. False by default.
 
 Memory pool counters. Only available with Varnish 4.x. False by default.
 
+=item B<CollectMSE> B<true>|B<false>
+
+Varnish Massive Storage Engine 2.0 (MSE2) is an improved storage backend for
+Varnish, replacing the traditional malloc and file storages. Only available
+with Varnish-Plus 4.x. Note: SMA, SMF and MSE share counters, enable only the
+one used by the Varnish instance. False by default.
+
 =back
 
 =head2 Plugin C<virt>

diff --git a/src/intel_pmu.c b/src/intel_pmu.c
index e5f19ce..fd2bd6f 100644 (file)
--- a/src/intel_pmu.c
+++ b/src/intel_pmu.c
@@ -67,7 +67,7 @@ struct intel_pmu_ctx_s {
   _Bool hw_cache_events;
   _Bool kernel_pmu_events;
   _Bool sw_events;
-  char  event_list_fn[PATH_MAX];
+  char event_list_fn[PATH_MAX];
   char **hw_events;
   size_t hw_events_count;
   struct eventlist *event_list;
@@ -265,7 +265,8 @@ static int pmu_config(oconfig_item_t *ci) {
   return 0;
 }
 
-static void pmu_submit_counter(int cpu, char *event, counter_t value) {
+static void pmu_submit_counter(int cpu, char *event, counter_t value,
+                               meta_data_t *meta) {
   value_list_t vl = VALUE_LIST_INIT;
 
   vl.values = &(value_t){.counter = value};
@@ -275,6 +276,7 @@ static void pmu_submit_counter(int cpu, char *event, counter_t value) {
   if (cpu == -1) {
     snprintf(vl.plugin_instance, sizeof(vl.plugin_instance), "all");
   } else {
+    vl.meta = meta;
     snprintf(vl.plugin_instance, sizeof(vl.plugin_instance), "%d", cpu);
   }
   sstrncpy(vl.type, "counter", sizeof(vl.type));
@@ -283,6 +285,27 @@ static void pmu_submit_counter(int cpu, char *event, counter_t value) {
   plugin_dispatch_values(&vl);
 }
 
+meta_data_t *pmu_meta_data_create(const struct efd *efd) {
+  meta_data_t *meta = NULL;
+
+  /* create meta data only if value was scaled */
+  if (efd->val[1] == efd->val[2] || !efd->val[2]) {
+    return NULL;
+  }
+
+  meta = meta_data_create();
+  if (meta == NULL) {
+    ERROR(PMU_PLUGIN ": meta_data_create failed.");
+    return NULL;
+  }
+
+  meta_data_add_unsigned_int(meta, "intel_pmu:raw_count", efd->val[0]);
+  meta_data_add_unsigned_int(meta, "intel_pmu:time_enabled", efd->val[1]);
+  meta_data_add_unsigned_int(meta, "intel_pmu:time_running", efd->val[2]);
+
+  return meta;
+}
+
 static void pmu_dispatch_data(void) {
 
   struct event *e;
@@ -297,17 +320,27 @@ static void pmu_dispatch_data(void) {
 
       event_enabled++;
 
+      /* If there are more events than counters, the kernel uses time
+       * multiplexing. With multiplexing, at the end of the run,
+       * the counter is scaled basing on total time enabled vs time running.
+       * final_count = raw_count * time_enabled/time_running
+       */
       uint64_t value = event_scaled_value(e, i);
       all_value += value;
 
+      /* get meta data with information about scaling */
+      meta_data_t *meta = pmu_meta_data_create(&e->efd[i]);
+
       /* dispatch per CPU value */
-      pmu_submit_counter(i, e->event, value);
+      pmu_submit_counter(i, e->event, value, meta);
+
+      meta_data_destroy(meta);
     }
 
     if (event_enabled > 0) {
       DEBUG(PMU_PLUGIN ": %-20s %'10lu", e->event, all_value);
       /* dispatch all CPU value */
-      pmu_submit_counter(-1, e->event, all_value);
+      pmu_submit_counter(-1, e->event, all_value, NULL);
     }
   }
 }
@@ -539,7 +572,6 @@ init_error:
   sfree(g_ctx.hw_events);
   g_ctx.hw_events_count = 0;
 
-
   return ret;
 }
 

diff --git a/src/varnish.c b/src/varnish.c
index d57413c..e4daf4b 100644 (file)
--- a/src/varnish.c
+++ b/src/varnish.c
@@ -88,6 +88,7 @@ struct user_config_s {
   _Bool collect_mgt;
   _Bool collect_smf;
   _Bool collect_vbe;
+  _Bool collect_mse;
 #endif
 };
 typedef struct user_config_s user_config_t; /* }}} */
@@ -227,6 +228,9 @@ static int varnish_monitor(void *priv,
     else if (strcmp(name, "esi_warnings") == 0)
       return varnish_submit_derive(conf->instance, "esi", "total_operations",
                                    "warning", val);
+    else if (strcmp(name, "esi_maxdepth") == 0)
+      return varnish_submit_derive(conf->instance, "esi", "total_operations",
+                                   "max_depth", val);
   }
 
   if (conf->collect_backend) {
@@ -588,12 +592,18 @@ static int varnish_monitor(void *priv,
     else if (strcmp(name, "s_req_bodybytes") == 0)
       return varnish_submit_derive(conf->instance, "totals", "total_bytes",
                                    "req_body", val);
+    else if (strcmp(name, "s_req_protobytes") == 0)
+      return varnish_submit_derive(conf->instance, "totals", "total_bytes",
+                                   "req_proto", val);
     else if (strcmp(name, "s_resp_hdrbytes") == 0)
       return varnish_submit_derive(conf->instance, "totals", "total_bytes",
                                    "resp_header", val);
     else if (strcmp(name, "s_resp_bodybytes") == 0)
       return varnish_submit_derive(conf->instance, "totals", "total_bytes",
                                    "resp_body", val);
+    else if (strcmp(name, "s_resp_protobytes") == 0)
+      return varnish_submit_derive(conf->instance, "totals", "total_bytes",
+                                   "resp_proto", val);
     else if (strcmp(name, "s_pipe_hdrbytes") == 0)
       return varnish_submit_derive(conf->instance, "totals", "total_bytes",
                                    "pipe_header", val);
@@ -694,7 +704,6 @@ static int varnish_monitor(void *priv,
     else if (strcmp(name, "busy_killed") == 0)
       return varnish_submit_derive(conf->instance, "workers", "http_requests",
                                    "busy_killed", val);
-
 #endif
   }
 
@@ -727,12 +736,18 @@ static int varnish_monitor(void *priv,
     else if (strcmp(name, "bereq_bodybytes") == 0)
       return varnish_submit_derive(conf->instance, "vbe",
                                    "total_bytes", "bereq_bodybytes", val);
+    else if (strcmp(name, "bereq_protobytes") == 0)
+      return varnish_submit_derive(conf->instance, "vbe",
+                                   "total_bytes", "bereq_protobytes", val);
     else if (strcmp(name, "beresp_hdrbytes") == 0)
       return varnish_submit_derive(conf->instance, "vbe",
                                    "total_bytes", "beresp_hdrbytes", val);
     else if (strcmp(name, "beresp_bodybytes") == 0)
       return varnish_submit_derive(conf->instance, "vbe",
                                    "total_bytes", "beresp_bodybytes", val);
+    else if (strcmp(name, "beresp_protobytes") == 0)
+      return varnish_submit_derive(conf->instance, "vbe",
+                                   "total_bytes", "beresp_protobytes", val);
     else if (strcmp(name, "pipe_hdrbytes") == 0)
       return varnish_submit_derive(conf->instance, "vbe",
                                    "total_bytes", "pipe_hdrbytes", val);
@@ -751,13 +766,15 @@ static int varnish_monitor(void *priv,
   }
 
   /* All Stevedores support these counters */
-  if (conf->collect_sma || conf->collect_smf) {
+  if (conf->collect_sma || conf->collect_smf || conf->collect_mse) {
 
     char category[4];
     if (conf->collect_sma)
       strncpy(category, "sma", 4);
-    else
+    else if (conf->collect_smf)
       strncpy(category, "smf", 4);
+    else
+      strncpy(category, "mse", 4);
 
     if (strcmp(name, "c_req") == 0)
       return varnish_submit_derive(conf->instance, category,
@@ -867,6 +884,99 @@ static int varnish_monitor(void *priv,
       return varnish_submit_gauge(conf->instance, "mempool",
                                    "objects", "ran_dry", val);
   }
+
+  if (conf->collect_mse) {
+    if (strcmp(name, "c_full") == 0)
+      return varnish_submit_derive(conf->instance, "mse",
+                                  "total_operations", "full_allocs", val);
+    else if (strcmp(name, "c_truncated") == 0)
+      return varnish_submit_derive(conf->instance, "mse",
+                                  "total_operations", "truncated_allocs", val);
+    else if (strcmp(name, "c_expanded") == 0)
+      return varnish_submit_derive(conf->instance, "mse",
+                                  "total_operations", "expanded_allocs", val);
+    else if (strcmp(name, "c_failed") == 0)
+      return varnish_submit_derive(conf->instance, "mse",
+                                  "total_operations", "failed_allocs", val);
+    else if (strcmp(name, "c_bytes") == 0)
+      return varnish_submit_derive(conf->instance, "mse",
+                                  "total_bytes", "bytes_allocated", val);
+    else if (strcmp(name, "c_freed") == 0)
+      return varnish_submit_derive(conf->instance, "mse",
+                                  "total_bytes", "bytes_freed", val);
+    else if (strcmp(name, "g_fo_alloc") == 0)
+      return varnish_submit_derive(conf->instance, "mse",
+                                  "total_operations", "fo_allocs_outstanding", val);
+    else if (strcmp(name, "g_fo_bytes") == 0)
+      return varnish_submit_gauge(conf->instance, "mse",
+                                  "bytes", "fo_bytes_outstanding", val);
+    else if (strcmp(name, "g_membuf_alloc") == 0)
+      return varnish_submit_gauge(conf->instance, "mse",
+                                  "objects", "membufs_allocated", val);
+    else if (strcmp(name, "g_membuf_inuse") == 0)
+      return varnish_submit_gauge(conf->instance, "mse",
+                                  "objects", "membufs_inuse", val);
+    else if (strcmp(name, "g_bans_bytes") == 0)
+      return varnish_submit_gauge(conf->instance, "mse",
+                                  "bytes", "persisted_banspace_used", val);
+    else if (strcmp(name, "g_bans_space") == 0)
+      return varnish_submit_gauge(conf->instance, "mse",
+                                  "bytes", "persisted_banspace_available", val);
+    else if (strcmp(name, "g_bans_persisted") == 0)
+      return varnish_submit_derive(conf->instance, "mse",
+                                  "total_operations", "bans_persisted", val);
+    else if (strcmp(name, "g_bans_lost") == 0)
+      return varnish_submit_derive(conf->instance, "mse",
+                                  "total_operations", "bans_lost", val);
+
+     /* mse seg */
+    else if (strcmp(name, "g_journal_bytes") == 0)
+      return varnish_submit_gauge(conf->instance, "mse_reg",
+                                  "bytes", "journal_bytes_used", val);
+    else if (strcmp(name, "g_journal_space") == 0)
+      return varnish_submit_gauge(conf->instance, "mse_reg",
+                                  "bytes", "journal_bytes_free", val);
+
+    /* mse segagg */
+    else if (strcmp(name, "g_bigspace") == 0)
+      return varnish_submit_gauge(conf->instance, "mse_segagg",
+                                  "bytes", "big_extents_bytes_available", val);
+    else if (strcmp(name, "g_extfree") == 0)
+      return varnish_submit_gauge(conf->instance, "mse_segagg",
+                                  "objects", "free_extents", val);
+    else if (strcmp(name, "g_sparenode") == 0)
+      return varnish_submit_gauge(conf->instance, "mse_segagg",
+                                  "objects", "spare_nodes_available", val);
+    else if (strcmp(name, "g_objnode") == 0)
+      return varnish_submit_gauge(conf->instance, "mse_segagg",
+                                  "objects", "object_nodes_in_use", val);
+    else if (strcmp(name, "g_extnode") == 0)
+      return varnish_submit_gauge(conf->instance, "mse_segagg",
+                                  "objects", "extent_nodes_in_use", val);
+    else if (strcmp(name, "g_bigextfree") == 0)
+      return varnish_submit_gauge(conf->instance, "mse_segagg",
+                                  "objects", "free_big_extents", val);
+    else if (strcmp(name, "c_pruneloop") == 0)
+      return varnish_submit_derive(conf->instance, "mse_segagg",
+                                  "total_operations", "prune_loops", val);
+    else if (strcmp(name, "c_pruned") == 0)
+      return varnish_submit_derive(conf->instance, "mse_segagg",
+                                  "total_objects", "pruned_objects", val);
+    else if (strcmp(name, "c_spared") == 0)
+      return varnish_submit_derive(conf->instance, "mse_segagg",
+                                  "total_operations", "spared_objects", val);
+    else if (strcmp(name, "c_skipped") == 0)
+      return varnish_submit_derive(conf->instance, "mse_segagg",
+                                  "total_operations", "missed_objects", val);
+    else if (strcmp(name, "c_nuked") == 0)
+      return varnish_submit_derive(conf->instance, "mse_segagg",
+                                  "total_operations", "nuked_objects", val);
+    else if (strcmp(name, "c_sniped") == 0)
+      return varnish_submit_derive(conf->instance, "mse_segagg",
+                                  "total_operations", "sniped_objects", val);
+
+  }
+
 #endif
 
   return 0;
@@ -1355,6 +1465,7 @@ static int varnish_config_apply_default(user_config_t *conf) /* {{{ */
   conf->collect_mgt = 0;
   conf->collect_smf = 0;
   conf->collect_vbe = 0;
+  conf->collect_mse = 0;
 #endif
 
   return 0;
@@ -1543,6 +1654,13 @@ static int varnish_config_instance(const oconfig_item_t *ci) /* {{{ */
       WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.",
               child->key, "v4");
 #endif
+    else if (strcasecmp("CollectMSE", child->key) == 0)
+#if HAVE_VARNISH_V4
+      cf_util_get_boolean(child, &conf->collect_mse);
+#else
+      WARNING("Varnish plugin: \"%s\" is available for Varnish %s only.",
+              child->key, "Plus v4");
+#endif
     else {
       WARNING("Varnish plugin: Ignoring unknown "
               "configuration option: \"%s\". Did "
@@ -1578,6 +1696,7 @@ static int varnish_config_instance(const oconfig_item_t *ci) /* {{{ */
 #if HAVE_VARNISH_V4
       && !conf->collect_vsm && !conf->collect_vbe && !conf->collect_smf
       && !conf->collect_mgt && !conf->collect_lck && !conf->collect_mempool
+      && !conf->collect_mse
 #endif
       ) {
     WARNING("Varnish plugin: No metric has been configured for "