940854200b83ba4e4c58c641bb7751fcb5ea0ceb
[collectd.git] / src / varnish.c
1 /**
2  * collectd - src/varnish.c
3  * Copyright (C) 2010 Jerome Renard
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms of the GNU General Public License as published by the
7  * Free Software Foundation; only version 2 of the License is applicable.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License along
15  * with this program; if not, write to the Free Software Foundation, Inc.,
16  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
17  *
18  * Authors:
19  *   Florian octo Forster <octo at verplant.org>
20  *   Jerome Renard <jerome.renard@gmail.com>
21  **/
22
23 /**
24  * Current list of what is monitored and what is not monitored (yet)
25  * {{{
26  * Field name           Description                           Monitored
27  * ----------           -----------                           ---------
28  * uptime               Child uptime                              N
29  * client_conn          Client connections accepted               Y
30  * client_drop          Connection dropped, no sess               Y
31  * client_req           Client requests received                  Y
32  * cache_hit            Cache hits                                Y
33  * cache_hitpass        Cache hits for pass                       Y
34  * cache_miss           Cache misses                              Y
35  * backend_conn         Backend conn. success                     Y
36  * backend_unhealthy    Backend conn. not attempted               Y
37  * backend_busy         Backend conn. too many                    Y
38  * backend_fail         Backend conn. failures                    Y
39  * backend_reuse        Backend conn. reuses                      Y
40  * backend_toolate      Backend conn. was closed                  Y
41  * backend_recycle      Backend conn. recycles                    Y
42  * backend_unused       Backend conn. unused                      Y
43  * fetch_head           Fetch head                                Y
44  * fetch_length         Fetch with Length                         Y
45  * fetch_chunked        Fetch chunked                             Y
46  * fetch_eof            Fetch EOF                                 Y
47  * fetch_bad            Fetch had bad headers                     Y
48  * fetch_close          Fetch wanted close                        Y
49  * fetch_oldhttp        Fetch pre HTTP/1.1 closed                 Y
50  * fetch_zero           Fetch zero len                            Y
51  * fetch_failed         Fetch failed                              Y
52  * n_sess_mem           N struct sess_mem                         N
53  * n_sess               N struct sess                             N
54  * n_object             N struct object                           N
55  * n_vampireobject      N unresurrected objects                   N
56  * n_objectcore         N struct objectcore                       N
57  * n_objecthead         N struct objecthead                       N
58  * n_smf                N struct smf                              N
59  * n_smf_frag           N small free smf                          N
60  * n_smf_large          N large free smf                          N
61  * n_vbe_conn           N struct vbe_conn                         N
62  * n_wrk                N worker threads                          N
63  * n_wrk_create         N worker threads created                  N
64  * n_wrk_failed         N worker threads not created              N
65  * n_wrk_max            N worker threads limited                  N
66  * n_wrk_queue          N queued work requests                    N
67  * n_wrk_overflow       N overflowed work requests                N
68  * n_wrk_drop           N dropped work requests                   N
69  * n_backend            N backends                                N
70  * n_expired            N expired objects                         N
71  * n_lru_nuked          N LRU nuked objects                       N
72  * n_lru_saved          N LRU saved objects                       N
73  * n_lru_moved          N LRU moved objects                       N
74  * n_deathrow           N objects on deathrow                     N
75  * losthdr              HTTP header overflows                     N
76  * n_objsendfile        Objects sent with sendfile                N
77  * n_objwrite           Objects sent with write                   N
78  * n_objoverflow        Objects overflowing workspace             N
79  * s_sess               Total Sessions                            Y
80  * s_req                Total Requests                            Y
81  * s_pipe               Total pipe                                Y
82  * s_pass               Total pass                                Y
83  * s_fetch              Total fetch                               Y
84  * s_hdrbytes           Total header bytes                        Y
85  * s_bodybytes          Total body bytes                          Y
86  * sess_closed          Session Closed                            N
87  * sess_pipeline        Session Pipeline                          N
88  * sess_readahead       Session Read Ahead                        N
89  * sess_linger          Session Linger                            N
90  * sess_herd            Session herd                              N
91  * shm_records          SHM records                               Y
92  * shm_writes           SHM writes                                Y
93  * shm_flushes          SHM flushes due to overflow               Y
94  * shm_cont             SHM MTX contention                        Y
95  * shm_cycles           SHM cycles through buffer                 Y
96  * sm_nreq              allocator requests                        Y
97  * sm_nobj              outstanding allocations                   Y
98  * sm_balloc            bytes allocated                           Y
99  * sm_bfree             bytes free                                Y
100  * sma_nreq             SMA allocator requests                    Y
101  * sma_nobj             SMA outstanding allocations               Y
102  * sma_nbytes           SMA outstanding bytes                     Y
103  * sma_balloc           SMA bytes allocated                       Y
104  * sma_bfree            SMA bytes free                            Y
105  * sms_nreq             SMS allocator requests                    Y
106  * sms_nobj             SMS outstanding allocations               Y
107  * sms_nbytes           SMS outstanding bytes                     Y
108  * sms_balloc           SMS bytes allocated                       Y
109  * sms_bfree            SMS bytes freed                           Y
110  * backend_req          Backend requests made                     N
111  * n_vcl                N vcl total                               N
112  * n_vcl_avail          N vcl available                           N
113  * n_vcl_discard        N vcl discarded                           N
114  * n_purge              N total active purges                     N
115  * n_purge_add          N new purges added                        N
116  * n_purge_retire       N old purges deleted                      N
117  * n_purge_obj_test     N objects tested                          N
118  * n_purge_re_test      N regexps tested against                  N
119  * n_purge_dups         N duplicate purges removed                N
120  * hcb_nolock           HCB Lookups without lock                  Y
121  * hcb_lock             HCB Lookups with lock                     Y
122  * hcb_insert           HCB Inserts                               Y
123  * esi_parse            Objects ESI parsed (unlock)               Y
124  * esi_errors           ESI parse errors (unlock)                 Y
125  * }}}
126  */
127 #include "collectd.h"
128 #include "common.h"
129 #include "plugin.h"
130 #include "configfile.h"
131
132 #include <varnish/varnishapi.h>
133
134 /* {{{ user_config_s */
135 struct user_config_s {
136         char *instance;
137
138         _Bool collect_cache;
139         _Bool collect_connections;
140         _Bool collect_esi;
141         _Bool collect_backend;
142         _Bool collect_fetch;
143         _Bool collect_hcb;
144         _Bool collect_shm;
145         _Bool collect_sma;
146         _Bool collect_sms;
147         _Bool collect_sm;
148         _Bool collect_totals;
149 };
150 typedef struct user_config_s user_config_t; /* }}} */
151
152 static _Bool have_instance = 0;
153
154 static int varnish_submit (const char *plugin_instance, /* {{{ */
155                 const char *type, const char *type_instance, value_t value)
156 {
157         value_list_t vl = VALUE_LIST_INIT;
158
159         vl.values = &value;
160         vl.values_len = 1;
161
162         sstrncpy(vl.host, hostname_g, sizeof(vl.host));
163
164         sstrncpy(vl.plugin, "varnish", sizeof(vl.plugin));
165         if (plugin_instance != NULL)
166                 sstrncpy (vl.plugin_instance, plugin_instance,
167                                 sizeof (vl.plugin_instance));
168
169         sstrncpy(vl.type, type, sizeof(vl.type));
170         if (type_instance != NULL)
171                 sstrncpy(vl.type_instance, type_instance,
172                                 sizeof(vl.type_instance));
173
174         return (plugin_dispatch_values(&vl));
175 } /* }}} int varnish_submit */
176
177 static int varnish_submit_gauge (const char *plugin_instance, /* {{{ */
178                 const char *type, const char *type_instance,
179                 uint64_t gauge_value)
180 {
181         value_t value;
182
183         value.gauge = (gauge_t) gauge_value;
184
185         return (varnish_submit (plugin_instance, type, type_instance, value));
186 } /* }}} int varnish_submit_gauge */
187
188 static int varnish_submit_derive (const char *plugin_instance, /* {{{ */
189                 const char *type, const char *type_instance,
190                 uint64_t derive_value)
191 {
192         value_t value;
193
194         value.derive = (derive_t) derive_value;
195
196         return (varnish_submit (plugin_instance, type, type_instance, value));
197 } /* }}} int varnish_submit_derive */
198
199 static void varnish_monitor(const user_config_t *conf, struct varnish_stats *VSL_stats) /* {{{ */
200 {
201         if(conf->collect_cache)
202         {
203                 /* Cache hits */
204                 varnish_submit_derive (conf->instance, "cache_result", "hit"    , VSL_stats->cache_hit);
205                 /* Cache misses */
206                 varnish_submit_derive (conf->instance, "cache_result", "miss"   , VSL_stats->cache_miss);
207                 /* Cache hits for pass */
208                 varnish_submit_derive (conf->instance, "cache_result", "hitpass", VSL_stats->cache_hitpass);
209         }
210
211         if(conf->collect_connections)
212         {
213                 /* Client connections accepted */
214                 varnish_submit_derive (conf->instance, "connections", "client-accepted", VSL_stats->client_conn);
215                 /* Connection dropped, no sess */
216                 varnish_submit_derive (conf->instance, "connections", "client-dropped" , VSL_stats->client_drop);
217                 /* Client requests received    */
218                 varnish_submit_derive (conf->instance, "connections", "client-received", VSL_stats->client_req);
219         }
220
221         if(conf->collect_esi)
222         {
223                 /* Objects ESI parsed (unlock) */
224                 varnish_submit_derive (conf->instance, "total_operations", "esi-parsed", VSL_stats->esi_parse);
225                 /* ESI parse errors (unlock)   */
226                 varnish_submit_derive (conf->instance, "total_operations", "esi-error", VSL_stats->esi_errors);
227         }
228
229         if(conf->collect_backend)
230         {
231                 /* Backend conn. success       */
232                 varnish_submit_derive (conf->instance, "connections", "backend-success"      , VSL_stats->backend_conn);
233                 /* Backend conn. not attempted */
234                 varnish_submit_derive (conf->instance, "connections", "backend-not-attempted", VSL_stats->backend_unhealthy);
235                 /* Backend conn. too many      */
236                 varnish_submit_derive (conf->instance, "connections", "backend-too-many"     , VSL_stats->backend_busy);
237                 /* Backend conn. failures      */
238                 varnish_submit_derive (conf->instance, "connections", "backend-failures"     , VSL_stats->backend_fail);
239                 /* Backend conn. reuses        */
240                 varnish_submit_derive (conf->instance, "connections", "backend-reuses"       , VSL_stats->backend_reuse);
241                 /* Backend conn. was closed    */
242                 varnish_submit_derive (conf->instance, "connections", "backend-was-closed"   , VSL_stats->backend_toolate);
243                 /* Backend conn. recycles      */
244                 varnish_submit_derive (conf->instance, "connections", "backend-recycled"     , VSL_stats->backend_recycle);
245                 /* Backend conn. unused        */
246                 varnish_submit_derive (conf->instance, "connections", "backend-unused"       , VSL_stats->backend_unused);
247         }
248
249         if(conf->collect_fetch)
250         {
251                 /* Fetch head                */
252                 varnish_submit_derive (conf->instance, "http_requests", "fetch-head"       , VSL_stats->fetch_head);
253                 /* Fetch with length         */
254                 varnish_submit_derive (conf->instance, "http_requests", "fetch-length"     , VSL_stats->fetch_length);
255                 /* Fetch chunked             */
256                 varnish_submit_derive (conf->instance, "http_requests", "fetch-chunked"    , VSL_stats->fetch_chunked);
257                 /* Fetch EOF                 */
258                 varnish_submit_derive (conf->instance, "http_requests", "fetch-eof"        , VSL_stats->fetch_eof);
259                 /* Fetch bad headers         */
260                 varnish_submit_derive (conf->instance, "http_requests", "fetch-bad_headers", VSL_stats->fetch_bad);
261                 /* Fetch wanted close        */
262                 varnish_submit_derive (conf->instance, "http_requests", "fetch-close"      , VSL_stats->fetch_close);
263                 /* Fetch pre HTTP/1.1 closed */
264                 varnish_submit_derive (conf->instance, "http_requests", "fetch-oldhttp"    , VSL_stats->fetch_oldhttp);
265                 /* Fetch zero len            */
266                 varnish_submit_derive (conf->instance, "http_requests", "fetch-zero"       , VSL_stats->fetch_zero);
267                 /* Fetch failed              */
268                 varnish_submit_derive (conf->instance, "http_requests", "fetch-failed"     , VSL_stats->fetch_failed);
269         }
270
271         if(conf->collect_hcb)
272         {
273                 /* HCB Lookups without lock */
274                 varnish_submit_derive (conf->instance, "cache_operation", "lookup_nolock", VSL_stats->hcb_nolock);
275                 /* HCB Lookups with lock    */
276                 varnish_submit_derive (conf->instance, "cache_operation", "lookup_lock",   VSL_stats->hcb_lock);
277                 /* HCB Inserts              */
278                 varnish_submit_derive (conf->instance, "cache_operation", "insert",        VSL_stats->hcb_insert);
279         }
280
281         if(conf->collect_shm)
282         {
283                 /* SHM records                 */
284                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-records"   , VSL_stats->shm_records);
285                 /* SHM writes                  */
286                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-writes"    , VSL_stats->shm_writes);
287                 /* SHM flushes due to overflow */
288                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-flushes"   , VSL_stats->shm_flushes);
289                 /* SHM MTX contention          */
290                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-contention", VSL_stats->shm_cont);
291                 /* SHM cycles through buffer   */
292                 varnish_submit_derive (conf->instance, "total_operations", "shmlog-cycles"    , VSL_stats->shm_cycles);
293         }
294
295         if(conf->collect_sm)
296         {
297                 /* allocator requests */
298                 varnish_submit_derive (conf->instance, "total_requests", "storage-file",      VSL_stats->sm_nreq);
299                 /* outstanding allocations */
300                 varnish_submit_gauge (conf->instance, "requests", "storage-file-outstanding", VSL_stats->sm_nobj);
301                 /* bytes allocated */
302                 varnish_submit_gauge (conf->instance, "bytes", "storage-file-allocated",      VSL_stats->sm_balloc);
303                 /* bytes free */
304                 varnish_submit_gauge (conf->instance, "bytes", "storage-file-free",           VSL_stats->sm_bfree);
305         }
306
307         if(conf->collect_sma)
308         {
309                 /* SMA allocator requests */
310                 varnish_submit_derive (conf->instance, "total_requests", "storage-mem",      VSL_stats->sma_nreq);
311                 /* SMA outstanding allocations */
312                 varnish_submit_gauge (conf->instance, "requests", "storage-mem-outstanding", VSL_stats->sma_nobj);
313                 /* SMA outstanding bytes */
314                 varnish_submit_gauge (conf->instance, "bytes", "storage-mem-outstanding",    VSL_stats->sma_nbytes);
315                 /* SMA bytes allocated */
316                 varnish_submit_gauge (conf->instance, "bytes", "storage-mem-allocated",      VSL_stats->sma_balloc);
317                 /* SMA bytes free */
318                 varnish_submit_gauge (conf->instance, "bytes", "storage-mem-free" ,          VSL_stats->sma_bfree);
319         }
320
321         if(conf->collect_sms)
322         {
323                 /* SMS allocator requests */
324                 varnish_submit_derive (conf->instance, "total_requests", "storage-synth",      VSL_stats->sms_nreq);
325                 /* SMS outstanding allocations */
326                 varnish_submit_gauge (conf->instance, "requests", "storage-synth-outstanding", VSL_stats->sms_nobj);
327                 /* SMS outstanding bytes */
328                 varnish_submit_gauge (conf->instance, "bytes", "storage-synth-outstanding",    VSL_stats->sms_nbytes);
329                 /* SMS bytes allocated */
330                 varnish_submit_gauge (conf->instance, "bytes", "storage-synth-allocated",      VSL_stats->sms_balloc);
331                 /* SMS bytes freed */
332                 varnish_submit_gauge (conf->instance, "bytes", "storage-synth-free",           VSL_stats->sms_bfree);
333         }
334
335         if(conf->collect_totals)
336         {
337                 /* Total Sessions */
338                 varnish_submit_derive (conf->instance, "total_counters", "sessions", VSL_stats->s_sess);
339                 /* Total Requests */
340                 varnish_submit_derive (conf->instance, "total_requests", "requests", VSL_stats->s_req);
341                 /* Total pipe */
342                 varnish_submit_derive (conf->instance, "total_operations", "pipe", VSL_stats->s_pipe);
343                 /* Total pass */
344                 varnish_submit_derive (conf->instance, "total_operations", "pass", VSL_stats->s_pass);
345                 /* Total fetch */
346                 varnish_submit_derive (conf->instance, "total_operations", "fetches", VSL_stats->s_fetch);
347                 /* Total header bytes */
348                 varnish_submit_derive (conf->instance, "total_bytes", "header-bytes", VSL_stats->s_hdrbytes);
349                 /* Total body byte */
350                 varnish_submit_derive (conf->instance, "total_bytes", "body-bytes", VSL_stats->s_bodybytes);
351         }
352 } /* }}} void varnish_monitor */
353
354 static int varnish_read(user_data_t *ud) /* {{{ */
355 {
356         struct varnish_stats *VSL_stats;
357         user_config_t *conf;
358
359         if ((ud == NULL) || (ud->data == NULL))
360                 return (EINVAL);
361
362         conf = ud->data;
363
364         VSL_stats = VSL_OpenStats(conf->instance);
365         if (VSL_stats == NULL)
366         {
367                 ERROR("Varnish plugin : unable to load statistics");
368
369                 return (-1);
370         }
371
372         varnish_monitor(conf, VSL_stats);
373
374     return (0);
375 } /* }}} */
376
377 static void varnish_config_free (void *ptr) /* {{{ */
378 {
379         user_config_t *conf = ptr;
380
381         if (conf == NULL)
382                 return;
383
384         sfree (conf->instance);
385         sfree (conf);
386 } /* }}} */
387
388 static int varnish_init (void) /* {{{ */
389 {
390         user_config_t *conf;
391         user_data_t ud;
392
393         if (have_instance)
394                 return (0);
395
396         conf = malloc (sizeof (*conf));
397         if (conf == NULL)
398                 return (ENOMEM);
399         memset (conf, 0, sizeof (*conf));
400
401         /* Default settings: */
402         conf->instance = NULL;
403         conf->collect_cache = 1;
404         conf->collect_backend = 1;
405         conf->collect_connections = 1;
406         conf->collect_shm = 1;
407
408         ud.data = conf;
409         ud.free_func = varnish_config_free;
410
411         plugin_register_complex_read (/* group = */ "varnish",
412                         /* name      = */ "varnish/localhost",
413                         /* callback  = */ varnish_read,
414                         /* interval  = */ NULL,
415                         /* user data = */ &ud);
416
417         return (0);
418 } /* }}} int varnish_init */
419
420 static int varnish_config_instance (const oconfig_item_t *ci) /* {{{ */
421 {
422         user_config_t *conf;
423         user_data_t ud;
424         char callback_name[DATA_MAX_NAME_LEN];
425         int i;
426
427         conf = malloc (sizeof (*conf));
428         if (conf == NULL)
429                 return (ENOMEM);
430         memset (conf, 0, sizeof (*conf));
431         conf->instance = NULL;
432
433         if (ci->values_num == 1)
434         {
435                 int status;
436
437                 status = cf_util_get_string (ci, &conf->instance);
438                 if (status != 0)
439                 {
440                         sfree (conf);
441                         return (status);
442                 }
443                 assert (conf->instance != NULL);
444
445                 if (strcmp ("localhost", conf->instance) == 0)
446                 {
447                         sfree (conf->instance);
448                         conf->instance = NULL;
449                 }
450         }
451         else if (ci->values_num > 1)
452         {
453                 WARNING ("Varnish plugin: \"Instance\" blocks accept only "
454                                 "one argument.");
455                 return (EINVAL);
456         }
457
458         for (i = 0; i < ci->children_num; i++)
459         {
460                 oconfig_item_t *child = ci->children + i;
461
462                 if (strcasecmp ("CollectCache", child->key) == 0)
463                         cf_util_get_boolean (child, &conf->collect_cache);
464                 else if (strcasecmp ("CollectConnections", child->key) == 0)
465                         cf_util_get_boolean (child, &conf->collect_connections);
466                 else if (strcasecmp ("CollectESI", child->key) == 0)
467                         cf_util_get_boolean (child, &conf->collect_esi);
468                 else if (strcasecmp ("CollectBackend", child->key) == 0)
469                         cf_util_get_boolean (child, &conf->collect_backend);
470                 else if (strcasecmp ("CollectFetch", child->key) == 0)
471                         cf_util_get_boolean (child, &conf->collect_fetch);
472                 else if (strcasecmp ("CollectHCB", child->key) == 0)
473                         cf_util_get_boolean (child, &conf->collect_hcb);
474                 else if (strcasecmp ("CollectSHM", child->key) == 0)
475                         cf_util_get_boolean (child, &conf->collect_shm);
476                 else if (strcasecmp ("CollectSMA", child->key) == 0)
477                         cf_util_get_boolean (child, &conf->collect_sma);
478                 else if (strcasecmp ("CollectSMS", child->key) == 0)
479                         cf_util_get_boolean (child, &conf->collect_sms);
480                 else if (strcasecmp ("CollectSM", child->key) == 0)
481                         cf_util_get_boolean (child, &conf->collect_sm);
482                 else if (strcasecmp ("CollectTotals", child->key) == 0)
483                         cf_util_get_boolean (child, &conf->collect_totals);
484                 else
485                 {
486                         WARNING ("Varnish plugin: Ignoring unknown "
487                                         "configuration option: \"%s\"",
488                                         child->key);
489                 }
490         }
491
492         if (!conf->collect_cache
493                         && !conf->collect_connections
494                         && !conf->collect_esi
495                         && !conf->collect_backend
496                         && !conf->collect_fetch
497                         && !conf->collect_hcb
498                         && !conf->collect_shm
499                         && !conf->collect_sma
500                         && !conf->collect_sms
501                         && !conf->collect_sm
502                         && !conf->collect_totals)
503         {
504                 WARNING ("Varnish plugin: No metric has been configured for "
505                                 "instance \"%s\". Disabling this instance.",
506                                 (conf->instance == NULL) ? "localhost" : conf->instance);
507                 return (EINVAL);
508         }
509
510         ssnprintf (callback_name, sizeof (callback_name), "varnish/%s",
511                         (conf->instance == NULL) ? "localhost" : conf->instance);
512
513         ud.data = conf;
514         ud.free_func = varnish_config_free;
515
516         plugin_register_complex_read (/* group = */ "varnish",
517                         /* name      = */ callback_name,
518                         /* callback  = */ varnish_read,
519                         /* interval  = */ NULL,
520                         /* user data = */ &ud);
521
522         have_instance = 1;
523
524         return (0);
525 } /* }}} int varnish_config_instance */
526
527 static int varnish_config (oconfig_item_t *ci) /* {{{ */
528 {
529         int i;
530
531         for (i = 0; i < ci->children_num; i++)
532         {
533                 oconfig_item_t *child = ci->children + i;
534
535                 if (strcasecmp ("Instance", child->key) == 0)
536                         varnish_config_instance (child);
537                 else
538                 {
539                         WARNING ("Varnish plugin: Ignoring unknown "
540                                         "configuration option: \"%s\"",
541                                         child->key);
542                 }
543         }
544
545         return (0);
546 } /* }}} int varnish_config */
547
548 void module_register (void) /* {{{ */
549 {
550         plugin_register_complex_config("varnish", varnish_config);
551         plugin_register_init ("varnish", varnish_init);
552 } /* }}} */
553
554 /* vim: set sw=8 noet fdm=marker : */