3 # collectd - contrib/rrd_filter.px
4 # Copyright (C) 2007-2008 Florian octo Forster
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by the
8 # Free Software Foundation; only version 2 of the License is applicable.
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # General Public License for more details.
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc.,
17 # 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 # Florian octo Forster <octo at verplant.org>
27 rrd_filter.px - Perform same advanced non-standard operations on an RRD file.
31 rrd_filter.px -i input.rrd -o output.rrd [options]
35 rrd_filter.px requires the RRDTool binary, Perl and the included
36 L<Getopt::Long> module.
40 use Getopt::Long ('GetOptions');
58 The following options can be passed on the command line:
62 =item B<--infile> I<file>
66 Reads from I<file>. If I<file> ends in C<.rrd>, then C<rrdtool dump> is invoked
67 to create an XML dump of the RRD file. Otherwise the XML dump is expected
68 directly. The special filename C<-> can be used to read from STDIN.
70 =item B<--outfile> I<file>
74 Writes output to I<file>. If I<file> ends in C<.rrd>, then C<rrdtool restore>
75 is invoked to create a binary RRD file. Otherwise an XML output is written. The
76 special filename C<-> can be used to write to STDOUT.
78 =item B<--map> I<in_ds>:I<out_ds>
80 =item B<-m> I<in_ds>:I<out_ds>
82 Writes the datasource I<in_ds> to the output and renames it to I<out_ds>. This
83 is useful to extract one DS from an RRD file.
85 =item B<--step> I<seconds>
87 =item B<-s> I<seconds>
89 Changes the step of the output RRD file to be I<seconds>. The new stepsize must
90 be a multiple of the old stepsize of the other way around. When increasing the
91 stepsize the number of PDPs in each RRA must be dividable by the factor by
92 which the stepsize is increased. The length of CDPs and the absolute length of
93 RRAs (and thus the data itself) is not altered.
97 step = 10, rra_steps = 12 => step = 60, rra_steps = 2
98 step = 300, rra_steps = 1 => step = 10, rra_steps = 30
100 =item B<--rra> B<RRA>:I<CF>:I<XFF>:I<steps>:I<rows>
102 =item B<-a> B<RRA>:I<CF>:I<XFF>:I<steps>:I<rows>
104 Inserts a new RRA in the generated RRD file. This is done B<after> the step has
105 been adjusted, take that into account when specifying I<steps> and I<rows>. For
106 an explanation of the format please see L<rrdcreate(1)>.
108 =item B<--scale> I<factor>
110 Scales the values by the factor I<factor>, i.E<nbsp>e. all values are
111 multiplied by I<factor>.
113 =item B<--shift> I<offset>
115 Shifts all values by I<offset>, i.E<nbsp>e. I<offset> is added to all values.
121 GetOptions ("infile|i=s" => \$InFile,
122 "outfile|o=s" => \$OutFile,
125 my ($in_ds, $out_ds) = split (':', $_[1]);
126 if (!defined ($in_ds) || !defined ($out_ds))
128 print STDERR "Argument for `map' incorrect! The format is `--map in_ds:out_ds'\n";
131 push (@$InDS, $in_ds);
132 push (@$OutDS, $out_ds);
134 'step|s=i' => \$Step,
137 my ($rra, $cf, $xff, $steps, $rows) = split (':', $_[1]);
138 if (($rra ne 'RRA') || !defined ($rows))
140 print STDERR "Please use the standard RRDTool syntax when adding RRAs. I. e. RRA:<cf><xff>:<steps>:<rows>.\n";
143 push (@$NewRRAs, {cf => $cf, xff => $xff, steps => $steps, rows => $rows});
145 'scale=f' => \$Scale,
149 if (!$InFile || !$OutFile)
151 print STDERR "Usage: $0 -i <infile> -m <in_ds>:<out_ds> -s <step>\n";
154 if ((1 + @$InDS) != (1 + @$OutDS))
156 print STDERR "You need the same amount of in- and out-DSes\n";
160 main ($InFile, $OutFile);
166 # state 0 == searching for DS index
167 # state 1 == parse RRA header
168 # state 2 == parse values
171 sub handle_line_dsmap
177 if ((@$InDS == 0) || (@$OutDS == 0))
179 post_line ($line, $index + 1);
183 if (!defined ($state))
189 # $ds_index->[new_index] = old_index
191 for (my $i = 0; $i < @$InDS; $i++)
193 print STDOUT "DS map $i: $InDS->[$i] -> $OutDS->[$i]\n" if ($Debug);
194 $ds_index->[$i] = -1;
200 if ($line =~ m/<ds>/)
203 $out_cache->[$current_index] = $line;
205 elsif ($line =~ m#<name>\s*([^<\s]+)\s*</name>#)
207 # old_index == $current_index
209 for (my $i = 0; $i < @$InDS; $i++)
211 next if ($ds_index->[$i] >= 0);
213 if ($1 eq $InDS->[$i])
215 $line =~ s#<name>\s*([^<\s]+)\s*</name>#<name> $OutDS->[$i] </name>#;
216 $ds_index->[$i] = $current_index;
221 $out_cache->[$current_index] .= $line;
223 elsif ($line =~ m#<last_ds>\s*([^\s>]+)\s*</last_ds>#i)
225 $out_cache->[$current_index] .= "\t\t<last_ds> NaN </last_ds>\n";
227 elsif ($line =~ m#<value>\s*([^\s>]+)\s*</value>#i)
229 $out_cache->[$current_index] .= "\t\t<value> NaN </value>\n";
231 elsif ($line =~ m#</ds>#)
233 $out_cache->[$current_index] .= $line;
235 elsif ($line =~ m#<rra>#)
237 # Print out all the DS definitions we need
238 for (my $new_index = 0; $new_index < @$InDS; $new_index++)
240 my $old_index = $ds_index->[$new_index];
241 while ($out_cache->[$old_index] =~ m/^(.*)$/gm)
243 post_line ("$1\n", $index + 1);
247 # Clear the cache - it's used in state1, too.
248 for (my $i = 0; $i <= $current_index; $i++)
250 $out_cache->[$i] = '';
257 elsif ($current_index == -1)
259 # Print all the lines before the first DS definition
264 # Something belonging to a DS-definition
265 $out_cache->[$current_index] .= $line;
270 if ($line =~ m#<ds>#)
273 $out_cache->[$current_index] .= $line;
275 elsif ($line =~ m#<value>\s*([^\s>]+)\s*</value>#i)
277 $out_cache->[$current_index] .= "\t\t\t<value> NaN </value>\n";
279 elsif ($line =~ m#</cdp_prep>#)
281 # Print out all the DS definitions we need
282 for (my $new_index = 0; $new_index < @$InDS; $new_index++)
284 my $old_index = $ds_index->[$new_index];
285 while ($out_cache->[$old_index] =~ m/^(.*)$/gm)
287 post_line ("$1\n", $index + 1);
292 for (my $i = 0; $i <= $current_index; $i++)
294 $out_cache->[$i] = '';
300 elsif ($line =~ m#<database>#)
305 elsif ($current_index == -1)
307 # Print all the lines before the first DS definition
313 # Something belonging to a DS-definition
314 $out_cache->[$current_index] .= $line;
319 if ($line =~ m#</database>#)
332 if ($line =~ m#(<!-- .*? -->)#)
339 while ($line =~ m#<v>\s*([^<\s]+)\s*</v>#g)
345 for (my $new_index = 0; $new_index < @$InDS; $new_index++)
347 my $old_index = $ds_index->[$new_index];
348 $ret .= '<v> ' . $values[$old_index] . ' </v> ';
360 post_line ($ret, $index + 1);
362 }} # handle_line_dsmap
369 my $step_factor_down;
377 post_line ($line, $index + 1);
381 if ($Debug && !defined ($step_factor_up))
383 print STDOUT "New step: $Step\n";
386 $step_factor_up ||= 0;
387 $step_factor_down ||= 0;
389 if (($step_factor_up == 0) && ($step_factor_down == 0))
391 if ($line =~ m#<step>\s*(\d+)\s*</step>#i)
393 my $old_step = 0 + $1;
394 if ($Step < $old_step)
396 $step_factor_down = int ($old_step / $Step);
397 if (($step_factor_down * $Step) != $old_step)
399 print STDERR "The old step ($old_step seconds) "
400 . "is not a multiple of the new step "
401 . "($Step seconds).\n";
404 $line = "<step> $Step </step>\n";
406 elsif ($Step > $old_step)
408 $step_factor_up = int ($Step / $old_step);
409 if (($step_factor_up * $old_step) != $Step)
411 print STDERR "The new step ($Step seconds) "
412 . "is not a multiple of the old step "
413 . "($old_step seconds).\n";
416 $line = "<step> $Step </step>\n";
424 elsif ($line =~ m#<pdp_per_row>\s*(\d+)\s*</pdp_per_row>#i)
426 my $old_val = 0 + $1;
430 $new_val = int ($old_val / $step_factor_up);
431 if (($new_val * $step_factor_up) != $old_val)
433 print STDERR "Can't divide number of PDPs per row ($old_val) by step-factor ($step_factor_up).\n";
439 $new_val = $step_factor_down * $old_val;
441 $line = "<pdp_per_row> $new_val </pdp_per_row>\n";
444 post_line ($line, $index + 1);
445 }} # handle_line_step
448 # The _add RRA_ handler
453 sub handle_line_add_rra
458 my $post = sub { for (@_) { post_line ($_, $index + 1); } };
462 if (!@$NewRRAs || $add_rra_done)
468 if ($line =~ m#<ds>#i)
472 elsif ($line =~ m#<rra>#i)
474 for (my $i = 0; $i < @$NewRRAs; $i++)
476 my $rra = $NewRRAs->[$i];
481 print STDOUT "Adding RRA: CF = $rra->{'cf'}, xff = $rra->{'xff'}, steps = $rra->{'steps'}, rows = $rra->{'rows'}, num_ds = $num_ds\n";
485 "\t\t<cf> $rra->{'cf'} </cf>\n",
486 "\t\t<pdp_per_row> $rra->{'steps'} </pdp_per_row>\n",
488 "\t\t\t<xff> $rra->{'xff'} </xff>\n",
492 for (my $j = 0; $j < $num_ds; $j++)
494 $post->("\t\t\t<ds>\n",
495 "\t\t\t\t<primary_value> NaN </primary_value>\n",
496 "\t\t\t\t<secondary_value> NaN </secondary_value>\n",
497 "\t\t\t\t<value> NaN </value>\n",
498 "\t\t\t\t<unknown_datapoints> 0 </unknown_datapoints>\n",
502 $post->("\t\t</cdp_prep>\n", "\t\t<database>\n");
503 $temp = "\t\t\t<row>" . join ('', map { "<v> NaN </v>" } (1 .. $num_ds)) . "</row>\n";
504 for (my $j = 0; $j < $rra->{'rows'}; $j++)
508 $post->("\t\t</database>\n", "\t</rra>\n");
515 }} # handle_line_add_rra
518 # The _scale/shift_ handler
520 sub calculate_scale_shift
527 if (lc ("$value") eq 'nan')
530 return ("<$tag> NaN </$tag>");
533 $value = ($scale * (0.0 + $value)) + $shift;
534 return (sprintf ("<%s> %1.10e </%s>", $tag, $value, $tag));
537 sub handle_line_scale_shift
542 if (($Scale != 1.0) || ($Shift != 0.0))
544 $line =~ s#<(min|max|last_ds|value|primary_value|secondary_value|v)>\s*([^\s<]+)\s*</[^>]+>#calculate_scale_shift ($2, $1, $Scale, $Shift)#eg;
547 post_line ($line, $index + 1);
551 # The _output_ handler
553 # This filter is unfinished!
564 my $previous_differences;
566 sub handle_line_peak_detect
571 if (!$previous_values)
573 $previous_values = [];
574 $previous_differences = [];
577 if ($line =~ m#</database>#i)
579 $previous_values = [];
580 $previous_differences = [];
581 print STDERR "==============================================================================\n";
583 elsif ($line =~ m#<pdp_per_row>\s*([1-9][0-9]*)\s*</pdp_per_row>#)
585 $pdp_per_row = int ($1);
586 print STDERR "pdp_per_row = $pdp_per_row;\n";
588 elsif ($line =~ m#<row>#)
591 while ($line =~ m#<v>\s*([^\s>]+)\s*</v>#ig)
595 push (@values, undef);
599 push (@values, 0.0 + $1);
603 for (my $i = 0; $i < @values; $i++)
605 if (!defined ($values[$i]))
607 $previous_values->[$i] = undef;
609 elsif (!defined ($previous_values->[$i]))
611 $previous_values->[$i] = $values[$i];
613 elsif (!defined ($previous_differences->[$i]))
615 $previous_differences->[$i] = abs ($previous_values->[$i] - $values[$i]);
619 my $divisor = ($previous_differences->[$i] < 1.0) ? 1.0 : $previous_differences->[$i];
620 my $difference = abs ($previous_values->[$i] - $values[$i]);
621 my $change = $pdp_per_row * $difference / $divisor;
622 if (($divisor > 10.0) && ($change > 10e5))
624 print STDERR "i = $i; average difference = " . $previous_differences->[$i]. "; current difference = " . $difference. "; change = $change;\n";
626 $previous_values->[$i] = $values[$i];
627 $previous_differences->[$i] = (0.95 * $previous_differences->[$i]) + (0.05 * $difference);
632 post_line ($line, $index + 1);
633 }} # handle_line_peak_detect
635 sub handle_line_output
642 post_line ($line, $index + 1);
647 }} # handle_line_output
658 die unless (ref ($handler) eq 'CODE');
659 push (@handlers, $handler);
671 print "DEBUG: post_line ($copy, $index);\n";
674 if ($index > $#handlers)
678 $handlers[$index]->($line, $index);
686 set_output ($out_fh);
690 add_handler (\&handle_line_dsmap);
695 add_handler (\&handle_line_step);
698 if (($Scale != 1.0) || ($Shift != 0.0))
700 add_handler (\&handle_line_scale_shift);
703 #add_handler (\&handle_line_peak_detect);
707 add_handler (\&handle_line_add_rra);
710 add_handler (\&handle_line_output);
712 while (my $line = <$in_fh>)
714 post_line ($line, 0);
721 my $out_file = shift;
726 my $in_needs_close = 1;
727 my $out_needs_close = 1;
729 if ($in_file =~ m/\.rrd$/i)
731 open ($in_fh, '-|', 'rrdtool', 'dump', $in_file) or die ("open (rrdtool): $!");
733 elsif ($in_file eq '-')
740 open ($in_fh, '<', $in_file) or die ("open ($in_file): $!");
743 if ($out_file =~ m/\.rrd$/i)
745 open ($out_fh, '|-', 'rrdtool', 'restore', '-', $out_file) or die ("open (rrdtool): $!");
747 elsif ($out_file eq '-')
750 $out_needs_close = 0;
754 open ($out_fh, '>', $out_file) or die ("open ($out_file): $!");
757 handle_fh ($in_fh, $out_fh);
763 if ($out_needs_close)
771 This script is licensed under the GNU general public license, versionE<nbsp>2
776 Florian octo Forster E<lt>octo at verplant.orgE<gt>