S<[B<--data-source-type>|B<-d> I<ds-name>:I<DST>]>
S<[B<--data-source-rename>|B<-r> I<old-name>:I<new-name>]>
S<[B<--deltapos> I<scale-value>]>
-S<[B<--deltaneg> I<scale value>]>
+S<[B<--deltaneg> I<scale-value>]>
S<[B<--failure-threshold> I<failure-threshold>]>
S<[B<--window-length> I<window-length>]>
S<[B<--alpha> I<adaption-parameter>]>
S<[B<--beta> I<adaption-parameter>]>
S<[B<--gamma> I<adaption-parameter>]>
+S<[B<--gamma-deviation> I<adaption-parameter>]>
+S<[B<--aberrant-reset> I<ds-name>]>
=head1 DESCRIPTION
The tune option allows you to alter some of the basic configuration
values stored in the header area of a Round Robin Database (B<RRD>).
-All these tunable parameters together decide when data fed into an
-B<RRD> is to be regarded as invalid. Invalid data is entered into the
-database as *UNKNOWN*.
One application of the B<tune> function is to relax the
validation rules on an B<RRD>. This allows to fill a new B<RRD> with
used internally to calculate violations for the FAILURES B<RRA>. The default
value is 2. Note that this parameter is not related to graphing confidence
bounds, that scale factor must be specified as a CDEF argument to generate
-a graph with confidence bounds. It need not agree with the value used internally
-by the FAILURES B<RRA> (although common sense dictates it should).
+a graph with confidence bounds. The graph scale factor need not agree with the
+value used internally by the FAILURES B<RRA>.
=item S<B<--deltaneg> I<scale-value>>
=item S<B<--gamma> I<adaption-parameter>>
-Alter the seasonal coefficient and deviation adaptation parameters the SEASONAL and
-DEVSEAONAL B<RRAs>. Must be between 0 and 1.
+Alter the seasonal coefficient adaptation parameter for the SEASONAL
+B<RRA>. Must be between 0 and 1.
+
+=item S<B<--gamma-deviation> I<adaption-parameter>>
+
+Alter the seasonal deviation adaptation parameter for the DEVSEASONAL
+B<RRA>. Must be between 0 and 1.
+
+=item S<B<--aberrant-reset> I<ds-name>>
+
+This option causes the aberrant behavior detection algorithm to reset
+for the specified data source; that is, forget all it is has learn.
+Specifically, for the HWPREDICT B<RRA>, it sets the intercept and slope
+coefficients to unknown. For the SEASONAL B<RRA>, it sets all seasonal
+coefficients to unknown. For the DEVSEASONL B<RRA>, it sets all seasonal
+deviation coefficients to unknown. For the FAILURES B<RRA>, it erases
+the violation history. Note that reset does not erase past predictions
+(the values of the HWPREDICT B<RRA>), predicted deviations (the values of the
+DEVPREDICT B<RRA>), or failure history (the values of the FAILURES B<RRA>).
+This option will function even if not all the listed B<RRAs> are present.
+
+Due to the implementation of this option, there is an indirect impact on
+other data sources in the RRD. A smoothing algorithm is applied to
+SEASONAL and DEVSEASONAL values on a periodic basis. During bootstrap
+initialization this smoothing is deferred. For efficiency, the implementation
+of smoothing is not data source specific. This means that utilizing
+reset for one data source will delay running the smoothing algorithm
+for all data sources in the file. This is unlikely to have serious
+consequences, unless the data being collected for the non-reset data soures
+is unusually volatile during the reinitialization period of the reset
+data source.
+
+Use of this tuning option is advised when the behavior of the data source
+time series changes in a drastic and permanent manner.
=back
#include "rrd_tool.h"
#include "rrd_rpncalc.h"
+#include "rrd_hw.h"
unsigned long FnvHash(char *str);
int create_hw_contingent_rras(rrd_t *rrd, unsigned short period, unsigned long hashed_name);
switch (cf_conv(rrd->rra_def[i].cf_nam))
{
case CF_HWPREDICT:
- rrd->cdp_prep->scratch[CDP_hw_intercept].u_val = DNAN;
- rrd->cdp_prep->scratch[CDP_hw_last_intercept].u_val = DNAN;
- rrd->cdp_prep->scratch[CDP_hw_slope].u_val = DNAN;
- rrd->cdp_prep->scratch[CDP_hw_last_slope].u_val = DNAN;
- rrd->cdp_prep->scratch[CDP_null_count].u_cnt = 1;
- rrd->cdp_prep->scratch[CDP_last_null_count].u_cnt = 1;
+ init_hwpredict_cdp(rrd->cdp_prep);
break;
case CF_SEASONAL:
case CF_DEVSEASONAL:
- rrd->cdp_prep->scratch[CDP_hw_seasonal].u_val = DNAN;
- rrd->cdp_prep->scratch[CDP_hw_last_seasonal].u_val = DNAN;
- rrd->cdp_prep->scratch[CDP_init_seasonal].u_cnt = 1;
+ init_seasonal_cdp(rrd->cdp_prep);
break;
case CF_FAILURES:
/* initialize violation history to 0 */
fprintf(stderr,"erase_violations called for non-FAILURES RRA: %s\n",
rrd -> rra_def[rra_idx].cf);
#endif
- return;
+ return;
}
#ifdef DEBUG
fprintf(stderr,"\n");
#endif
- /* WARNING: this cast makes XML files non-portable across platforms,
- * because an array of longs on disk is treated as an array of chars
- * in memory. */
+ /* WARNING: an array of longs on disk is treated as an array of chars
+ * in memory. */
violations_array = (char *) ((void *) rrd -> cdp_prep[cdp_idx].scratch);
/* erase everything in the part of the CDP scratch array that will be
- * used to store violations for the current window */
+ * used to store violations for the current window */
for (i = rrd -> rra_def[rra_idx].par[RRA_window_len].u_cnt; i > 0; i--)
{
violations_array[i-1] = 0;
unsigned long offset;
FIFOqueue **buffers;
rrd_value_t *working_average;
+ rrd_value_t *baseline;
offset = floor(0.025*row_count);
if (offset == 0) return 0; /* no smoothing */
}
/* need working average initialized to 0 */
working_average = (rrd_value_t *) calloc(row_length,sizeof(rrd_value_t));
+ baseline = (rrd_value_t *) calloc(row_length,sizeof(rrd_value_t));
/* compute sums of the first 2*offset terms */
for (i = 0; i < 2*offset; ++i)
/* overwrite rdd_values entry, the old value is already
* saved in buffers */
rrd_values[k*row_length + j] = working_average[j]/(2*offset + 1);
+ baseline[j] += rrd_values[k*row_length + j];
/* remove a term from the sum */
working_average[j] -= queue_pop(buffers[j]);
}
}
-
+
for (i = 0; i < row_length; ++i)
{
queue_dealloc(buffers[i]);
+ baseline[i] /= row_count;
}
free(buffers);
free(working_average);
+ if (cf_conv(rrd->rra_def[rra_idx].cf_nam) == CF_SEASONAL) {
+ for (j = 0; j < row_length; ++j)
+ {
+ for (i = 0; i < row_count; ++i)
+ {
+ rrd_values[i*row_length + j] -= baseline[j];
+ }
+ /* update the baseline coefficient,
+ * first, compute the cdp_index. */
+ offset = (rrd->rra_def[rra_idx].par[RRA_dependent_rra_idx].u_cnt)
+ * row_length + j;
+ (rrd->cdp_prep[offset]).scratch[CDP_hw_intercept].u_val += baseline[j];
+ }
+ /* flush cdp to disk */
+ fflush(rrd_file);
+ if (fseek(rrd_file,sizeof(stat_head_t) +
+ rrd->stat_head->ds_cnt * sizeof(ds_def_t) +
+ rrd->stat_head->rra_cnt * sizeof(rra_def_t) +
+ sizeof(live_head_t) +
+ rrd->stat_head->ds_cnt * sizeof(pdp_prep_t),SEEK_SET))
+ {
+ rrd_set_error("apply_smoother: seek to cdp_prep failed");
+ free(rrd_values);
+ return -1;
+ }
+ if (fwrite( rrd -> cdp_prep,
+ sizeof(cdp_prep_t),
+ (rrd->stat_head->rra_cnt) * rrd->stat_head->ds_cnt, rrd_file)
+ != (rrd->stat_head->rra_cnt) * (rrd->stat_head->ds_cnt) )
+ {
+ rrd_set_error("apply_smoother: cdp_prep write failed");
+ free(rrd_values);
+ return -1;
+ }
+ } /* endif CF_SEASONAL */
+
/* flush updated values to disk */
fflush(rrd_file);
if (fseek(rrd_file,rra_start,SEEK_SET))
}
/* write as a single block */
if (fwrite(rrd_values,sizeof(rrd_value_t),row_length*row_count,rrd_file)
- != row_length*row_count)
+ != row_length*row_count)
{
rrd_set_error("apply_smoother: write failed to %lu",rra_start);
free(rrd_values);
return 0;
}
+/* Reset aberrant behavior model coefficients, including intercept, slope,
+ * seasonal, and seasonal deviation for the specified data source. */
+void
+reset_aberrant_coefficients(rrd_t *rrd, FILE *rrd_file, unsigned long ds_idx)
+{
+ unsigned long cdp_idx, rra_idx, i;
+ unsigned long cdp_start, rra_start;
+ rrd_value_t nan_buffer = DNAN;
+
+ /* compute the offset for the cdp area */
+ cdp_start = sizeof(stat_head_t) +
+ rrd->stat_head->ds_cnt * sizeof(ds_def_t) +
+ rrd->stat_head->rra_cnt * sizeof(rra_def_t) +
+ sizeof(live_head_t) +
+ rrd->stat_head->ds_cnt * sizeof(pdp_prep_t);
+ /* compute the offset for the first rra */
+ rra_start = cdp_start +
+ (rrd->stat_head->ds_cnt) * (rrd->stat_head->rra_cnt) * sizeof(cdp_prep_t) +
+ rrd->stat_head->rra_cnt * sizeof(rra_ptr_t);
+
+ /* loop over the RRAs */
+ for (rra_idx = 0; rra_idx < rrd -> stat_head -> rra_cnt; rra_idx++)
+ {
+ cdp_idx = rra_idx * (rrd-> stat_head-> ds_cnt) + ds_idx;
+ switch (cf_conv(rrd -> rra_def[rra_idx].cf_nam))
+ {
+ case CF_HWPREDICT:
+ init_hwpredict_cdp(&(rrd -> cdp_prep[cdp_idx]));
+ break;
+ case CF_SEASONAL:
+ case CF_DEVSEASONAL:
+ /* don't use init_seasonal because it will reset burn-in, which
+ * means different data sources will be calling for the smoother
+ * at different times. */
+ rrd->cdp_prep[cdp_idx].scratch[CDP_hw_seasonal].u_val = DNAN;
+ rrd->cdp_prep[cdp_idx].scratch[CDP_hw_last_seasonal].u_val = DNAN;
+ /* move to first entry of data source for this rra */
+ fseek(rrd_file,rra_start + ds_idx * sizeof(rrd_value_t),SEEK_SET);
+ /* entries for the same data source are not contiguous,
+ * temporal entries are contiguous */
+ for (i = 0; i < rrd->rra_def[rra_idx].row_cnt; ++i)
+ {
+ if (fwrite(&nan_buffer,sizeof(rrd_value_t),1,rrd_file) != 1)
+ {
+ rrd_set_error(
+ "reset_aberrant_coefficients: write failed data source %lu rra %s",
+ ds_idx,rrd->rra_def[rra_idx].cf_nam);
+ return;
+ }
+ fseek(rrd_file,(rrd->stat_head->ds_cnt - 1) *
+ sizeof(rrd_value_t),SEEK_CUR);
+ }
+ break;
+ case CF_FAILURES:
+ erase_violations(rrd,cdp_idx,rra_idx);
+ break;
+ default:
+ break;
+ }
+ /* move offset to the next rra */
+ rra_start += rrd->rra_def[rra_idx].row_cnt * rrd->stat_head->ds_cnt *
+ sizeof(rrd_value_t);
+ }
+ fseek(rrd_file,cdp_start,SEEK_SET);
+ if (fwrite( rrd -> cdp_prep,
+ sizeof(cdp_prep_t),
+ (rrd->stat_head->rra_cnt) * rrd->stat_head->ds_cnt, rrd_file)
+ != (rrd->stat_head->rra_cnt) * (rrd->stat_head->ds_cnt) )
+ {
+ rrd_set_error("reset_aberrant_coefficients: cdp_prep write failed");
+ return;
+ }
+}
+
+void init_hwpredict_cdp(cdp_prep_t *cdp)
+{
+ cdp->scratch[CDP_hw_intercept].u_val = DNAN;
+ cdp->scratch[CDP_hw_last_intercept].u_val = DNAN;
+ cdp->scratch[CDP_hw_slope].u_val = DNAN;
+ cdp->scratch[CDP_hw_last_slope].u_val = DNAN;
+ cdp->scratch[CDP_null_count].u_cnt = 1;
+ cdp->scratch[CDP_last_null_count].u_cnt = 1;
+}
+
+void init_seasonal_cdp(cdp_prep_t *cdp)
+{
+ cdp->scratch[CDP_hw_seasonal].u_val = DNAN;
+ cdp->scratch[CDP_hw_last_seasonal].u_val = DNAN;
+ cdp->scratch[CDP_init_seasonal].u_cnt = 1;
+}
+
int
update_aberrant_CF(rrd_t *rrd, rrd_value_t pdp_val, enum cf_en current_cf,
unsigned long cdp_idx, unsigned long rra_idx, unsigned long ds_idx,
void erase_violations(rrd_t *rrd, unsigned long cdp_idx, unsigned long rra_idx);
int apply_smoother(rrd_t *rrd, unsigned long rra_idx, unsigned long rra_start,
FILE *rrd_file);
+void reset_aberrant_coefficients(rrd_t *rrd, FILE *rrd_file, unsigned long ds_idx);
+void init_hwpredict_cdp(cdp_prep_t *);
+void init_seasonal_cdp(cdp_prep_t *);
#define BURNIN_CYCLES 3
/* a standard fixed-capacity FIFO queue implementation */
* into a CDEF string. This function is used by rrd_dump.
* arguments:
* rpnc: an array of compact RPN operator nodes
- * rrd: a pointer an rrd header (only the ds_cnt and ds_def elements need
- * to be valid) for lookup of data source names by index
+ * ds_def: a pointer to the data source definition section of an RRD header
+ * for lookup of data source names by index
* str: out string, memory is allocated by the function, must be freed by the
* the caller */
void rpn_compact2str(rpn_cdefds_t *rpnc,ds_def_t *ds_def,char **str)
"* create - create a new RRD\n\n"
"\trrdtool create filename [--start|-b start time]\n"
"\t\t[--step|-s step]\n"
- "\t\t[DS:ds-name:DST:heartbeat:min:max] [RRA:CF:xff:steps:rows]\n\n";
+ "\t\t[DS:ds-name:DST:dst arguments]\n"
+ "\t\t[RRA:CF:cf arguments]\n\n";
char help_dump[] =
"* dump - dump an RRD to XML\n\n"
"\trrdtool dump filename.rrd >filename.xml\n\n";
char help_info[] =
- "* info - returns the configuration and status of the\n\n"
+ "* info - returns the configuration and status of the RRD\n\n"
"\trrdtool info filename.rrd\n\n";
char help_restore[] =
" * tune - Modify some basic properties of an RRD\n\n"
"\trrdtool tune filename\n"
"\t\t[--heartbeat|-h ds-name:heartbeat]\n"
- "\t\t[--data-source-type|-d ds-name:DST\n"
- "\t\t[--data-source-rename|-r old-name:new-name\n"
- "\t\t[--minimum|-i ds-name:min] [--maximum|-a ds-name:max]\n\n";
+ "\t\t[--data-source-type|-d ds-name:DST]\n"
+ "\t\t[--data-source-rename|-r old-name:new-name]\n"
+ "\t\t[--minimum|-i ds-name:min] [--maximum|-a ds-name:max]\n"
+ "\t\t[--deltapos scale-value] [--deltaneg scale-value]\n"
+ "\t\t[--failure-threshold integer]\n"
+ "\t\t[--window-length integer]\n"
+ "\t\t[--alpha adaptation-parameter]\n"
+ "\t\t[--beta adaptation-parameter]\n"
+ "\t\t[--gamma adaptation-parameter]\n"
+ "\t\t[--gamma-deviation adaptation-parameter]\n"
+ "\t\t[--aberrant-reset ds-name]\n\n";
char help_resize[] =
" * resize - alter the lenght of one of the RRAs in an RRD\n\n"
*****************************************************************************
* $Id$
* $Log$
+ * Revision 1.4 2001/08/22 22:29:07 jake
+ * Contents of this patch:
+ * (1) Adds/revises documentation for rrd tune in rrd_tool.c and pod files.
+ * (2) Moves some initialization code from rrd_create.c to rrd_hw.c.
+ * (3) Adds another pass to smoothing for SEASONAL and DEVSEASONAL RRAs.
+ * This pass computes the coefficients as deviations from an average; the
+ * average is added the baseline coefficient of HWPREDICT. Statistical texts
+ * suggest this to preserve algorithm stability. It will not invalidate
+ * RRD files created and smoothed with the old code.
+ * (4) Adds the aberrant-reset flag to rrd tune. This operation, which is
+ * specified for a single data source, causes the holt-winters algorithm to
+ * forget everthing it has learned and start over.
+ * (5) Fixes a few out-of-date code comments.
+ *
* Revision 1.3 2001/03/07 21:21:54 oetiker
* complete rewrite of rrdgraph documentation. This also includs info
* on upcomming/planned changes to the rrdgraph interface and functionality
*****************************************************************************/
#include "rrd_tool.h"
+#include "rrd_rpncalc.h"
+#include "rrd_hw.h"
int set_hwarg(rrd_t *rrd,enum cf_en cf,enum rra_par_en rra_par,char *arg);
int set_deltaarg(rrd_t *rrd,enum rra_par_en rra_par,char *arg);
{"alpha",required_argument,0,'x'},
{"beta",required_argument,0,'y'},
{"gamma",required_argument,0,'z'},
+ {"gamma-deviation",required_argument,0,'v'},
+ {"aberrant-reset",required_argument,0,'b'},
{0,0,0,0}
};
int option_index = 0;
int opt;
- opt = getopt_long(argc, argv, "h:i:a:d:r:p:n:w:f:x:y:z:",
+ opt = getopt_long(argc, argv, "h:i:a:d:r:p:n:w:f:x:y:z:v:b:",
long_options, &option_index);
if (opt == EOF)
break;
return -1;
}
break;
+ case 'v':
+ if (set_hwarg(&rrd,CF_DEVSEASONAL,RRA_seasonal_gamma,optarg)) {
+ rrd_free(&rrd);
+ return -1;
+ }
+ break;
+ case 'b':
+ if (sscanf(optarg,DS_NAM_FMT,ds_nam) != 1){
+ rrd_set_error("invalid argument for aberrant-reset");
+ rrd_free(&rrd);
+ fclose(rrd_file);
+ return -1;
+ }
+ if ((ds=ds_match(&rrd,ds_nam))==-1){
+ /* ds_match handles it own errors */
+ rrd_free(&rrd);
+ fclose(rrd_file);
+ return -1;
+ }
+ reset_aberrant_coefficients(&rrd,rrd_file,(unsigned long) ds);
+ if (rrd_test_error()) {
+ rrd_free(&rrd);
+ fclose(rrd_file);
+ return -1;
+ }
+ break;
case '?':
if (optopt != 0)
rrd_set_error("unknown option '%c'", optopt);
return -1;
}
}
- if(optcnt>0){
+ if(optcnt>0){
fseek(rrd_file,0,SEEK_SET);
fwrite(rrd.stat_head,
} else {
int i;
for(i=0;i< rrd.stat_head->ds_cnt;i++)
+ if (dst_conv(rrd.ds_def[i].dst) != DST_CDEF) {
printf("DS[%s] typ: %s\thbt: %ld\tmin: %1.4f\tmax: %1.4f\n",
rrd.ds_def[i].ds_nam,
rrd.ds_def[i].dst,
rrd.ds_def[i].par[DS_mrhb_cnt].u_cnt,
rrd.ds_def[i].par[DS_min_val].u_val,
rrd.ds_def[i].par[DS_max_val].u_val);
+ } else {
+ char *buffer;
+ rpn_compact2str((rpn_cdefds_t *) &(rrd.ds_def[i].par[DS_cdef]),rrd.ds_def,&buffer);
+ printf("DS[%s] typ: %s\tcdef: %s\n", rrd.ds_def[i].ds_nam,rrd.ds_def[i].dst,buffer);
+ free(buffer);
+ }
}
fclose(rrd_file);
rrd_free(&rrd);
{
double param;
unsigned long i;
- signed short rra_idx = -1, devseasonal_idx = -1;
+ signed short rra_idx = -1;
/* read the value */
param = atof(arg);
if (param <= 0.0 || param >= 1.0)
if (cf_conv(rrd -> rra_def[i].cf_nam) == cf)
{
rra_idx = i;
- }
- /* find the DEVSEASONAL index separately, as it is optional */
- if (cf_conv(rrd -> rra_def[i].cf_nam) == CF_DEVSEASONAL)
- {
- devseasonal_idx = i;
+ break;
}
}
if (rra_idx == -1)
/* set the value */
rrd -> rra_def[rra_idx].par[rra_par].u_val = param;
- if (devseasonal_idx > -1)
- rrd -> rra_def[devseasonal_idx].par[rra_par].u_val = param;
return 0;
}
signed short rra_idx = -1;
/* read the value */
param = atoi(arg);
- /* there are 4 chars to a long, reserve on CDP entry for future
- * use. */
if (param < 1 || param > MAX_FAILURES_WINDOW_LEN)
{
rrd_set_error("Parameter must be between %d and %d",
- 1, MAX_CDP_PAR_EN - 1);
+ 1, MAX_FAILURES_WINDOW_LEN);
return -1;
}
/* does the appropriate RRA exist? */
projects / rrdtool.git / commitdiff