src/rrd_format.h

   1 /*****************************************************************************
   2  * RRDtool 1.0.33  Copyright Tobias Oetiker, 1997, 1998, 1999
   3  *****************************************************************************
   4  * rrd_format.h  RRD Database Format header
   5  *****************************************************************************/
   6
   7 #ifndef _RRD_FORMAT_H
   8 #define _RRD_FORMAT_H
   9
  10 #include "rrd.h"
  11
  12 /*****************************************************************************
  13  * put this in your /usr/lib/magic file (/etc/magic on HPUX)
  14  *
  15  *  # rrd database format
  16  *  0       string          RRD\0           rrd file
  17  *  >5      string          >\0             version '%s'
  18  *
  19  *****************************************************************************/
  20
  21 #define RRD_COOKIE    "RRD"
  22 #define RRD_VERSION   "0002"
  23 #define FLOAT_COOKIE  8.642135E130
  24
  25 #include "rrd_nan_inf.h"
  26
  27 typedef union unival {
  28     unsigned long u_cnt;
  29     rrd_value_t   u_val;
  30 } unival;
  31
  32
  33 /****************************************************************************
  34  * The RRD Database Structure
  35  * ---------------------------
  36  *
  37  * In oder to properly describe the database structure lets define a few
  38  * new words:
  39  *
  40  * ds - Data Source (ds) providing input to the database. A Data Source (ds)
  41  *       can be a traffic counter, a temperature, the number of users logged
  42  *       into a system. The rrd database format can handle the input of
  43  *       several Data Sources (ds) in a singe database.
  44  *
  45  * dst - Data Source Type (dst). The Data Source Type (dst) defines the rules
  46  *       applied to Build Primary Data Points from the input provided by the
  47  *       data sources (ds).
  48  *
  49  * pdp - Primary Data Point (pdp). After the database has accepted the
  50  *       input from the data sources (ds). It starts building Primary
  51  *       Data Points (pdp) from the data. Primary Data Points (pdp)
  52  *       are evenly spaced along the time axis (pdp_step). The values
  53  *       of the Primary Data Points are calculated from the values of
  54  *       the data source (ds) and the exact time these values were
  55  *       provided by the data source (ds).
  56  *
  57  * pdp_st - PDP Start (pdp_st). The moments (pdp_st) in time where
  58  *       these steps occur are defined by the moments where the
  59  *       number of seconds since 1970-jan-1 modulo pdp_step equals
  60  *       zero (pdp_st).
  61  *
  62  * cf -  Consolidation Function (cf). An arbitrary Consolidation Function (cf)
  63  *       (averaging, min, max) is applied to the primary data points (pdp) to
  64  *       calculate the consolidated data point.
  65  *
  66  * cdp - Consolidated Data Point (cdp) is the long term storage format for data
  67  *       in the rrd database. Consolidated Data Points represent one or
  68  *       several primary data points collected along the time axis. The
  69  *       Consolidated Data Points (cdp) are stored in Round Robin Archives
  70  *       (rra).
  71  *
  72  * rra - Round Robin Archive (rra). This is the place where the
  73  *       consolidated data points (cdp) get stored. The data is
  74  *       organized in rows (row) and columns (col). The Round Robin
  75  *       Archive got its name from the method data is stored in
  76  *       there. An RRD database can contain several Round Robin
  77  *       Archives. Each Round Robin Archive can have a different row
  78  *       spacing along the time axis (pdp_cnt) and a different
  79  *       consolidation function (cf) used to build its consolidated
  80  *       data points (cdp).
  81  *
  82  * rra_st - RRA Start (rra_st). The moments (rra_st) in time where
  83  *       Consolidated Data Points (cdp) are added to an rra are
  84  *       defined by the moments where the number of seconds since
  85  *       1970-jan-1 modulo pdp_cnt*pdp_step equals zero (rra_st).
  86  *
  87  * row - Row (row). A row represent all consolidated data points (cdp)
  88  *       in a round robin archive who are of the same age.
  89  *
  90  * col - Column (col). A column (col) represent all consolidated
  91  *       data points (cdp) in a round robin archive (rra) who
  92  *       originated from the same data source (ds).
  93  *
  94  */
  95
  96 /****************************************************************************
  97  * POS 1: stat_head_t                           static header of the database
  98  ****************************************************************************/
  99
 100 typedef struct stat_head_t {
 101
 102     /* Data Base Identification Section ***/
 103     char             cookie[4];          /* RRD */
 104     char             version[5];         /* version of the format */
 105     double           float_cookie;       /* is it the correct double
 106                                           * representation ?  */
 107
 108     /* Data Base Structure Definition *****/
 109     unsigned long    ds_cnt;             /* how many different ds provide
 110                                           * input to the rrd */
 111     unsigned long    rra_cnt;            /* how many rras will be maintained
 112                                           * in the rrd */
 113     unsigned long    pdp_step;           /* pdp interval in seconds */
 114
 115     unival           par[10];            /* global parameters ... unused
 116                                             at the moment */
 117 } stat_head_t;
 118
 119
 120 /****************************************************************************
 121  * POS 2: ds_def_t  (* ds_cnt)                        Data Source definitions
 122  ****************************************************************************/
 123
 124 enum dst_en          { DST_COUNTER=0,     /* data source types available */
 125                        DST_ABSOLUTE,
 126                        DST_GAUGE,
 127                        DST_DERIVE,
 128                                            DST_CDEF};
 129
 130 enum ds_param_en {   DS_mrhb_cnt=0,       /* minimum required heartbeat. A
 131                                            * data source must provide input at
 132                                            * least every ds_mrhb seconds,
 133                                            * otherwise it is regarded dead and
 134                                            * will be set to UNKNOWN */
 135                      DS_min_val,          /* the processed input of a ds must */
 136                      DS_max_val,      /* be between max_val and min_val
 137                                            * both can be set to UNKNOWN if you
 138                                            * do not care. Data outside the limits
 139                                            * set to UNKNOWN */
 140                      DS_cdef = DS_mrhb_cnt}; /* pointer to encoded rpn
 141                                            * expression only applies to DST_CDEF */
 142
 143 /* The magic number here is one less than DS_NAM_SIZE */
 144 #define DS_NAM_FMT    "%19[a-zA-Z0-9_-]"
 145 #define DS_NAM_SIZE   20
 146
 147 #define DST_FMT    "%19[A-Z]"
 148 #define DST_SIZE   20
 149
 150 typedef struct ds_def_t {
 151     char             ds_nam[DS_NAM_SIZE]; /* Name of the data source (null terminated)*/
 152     char             dst[DST_SIZE];       /* Type of data source (null terminated)*/
 153     unival           par[10];             /* index of this array see ds_param_en */
 154 } ds_def_t;
 155
 156 /****************************************************************************
 157  * POS 3: rra_def_t ( *  rra_cnt)         one for each store to be maintained
 158  ****************************************************************************/
 159 enum cf_en           { CF_AVERAGE=0,     /* data consolidation functions */
 160                        CF_MINIMUM,
 161                        CF_MAXIMUM,
 162                        CF_LAST,
 163                        CF_HWPREDICT,
 164                                            /* An array of predictions using the seasonal
 165                                                 * Holt-Winters algorithm. Requires an RRA of type
 166                                                 * CF_SEASONAL for this data source. */
 167                                            CF_SEASONAL,
 168                                            /* An array of seasonal effects. Requires an RRA of
 169                                                 * type CF_HWPREDICT for this data source. */
 170                                            CF_DEVPREDICT,
 171                                            /* An array of deviation predictions based upon
 172                                                 * smoothed seasonal deviations. Requires an RRA of
 173                                                 * type CF_DEVSEASONAL for this data source. */
 174                                            CF_DEVSEASONAL,
 175                                            /* An array of smoothed seasonal deviations. Requires
 176                                                 * an RRA of type CF_HWPREDICT for this data source.
 177                                                 * */
 178                                            CF_FAILURES};
 179                                            /* A binary array of failure indicators: 1 indicates
 180                                                 * that the number of violations in the prescribed
 181                                                 * window exceeded the prescribed threshold. */
 182
 183 #define MAX_RRA_PAR_EN 10
 184 enum rra_par_en {   RRA_cdp_xff_val=0,  /* what part of the consolidated
 185                      * datapoint must be known, to produce a
 186                                          * valid entry in the rra */
 187                                         RRA_hw_alpha,
 188                                         /* exponential smoothing parameter for the intercept in
 189                                          * the Holt-Winters prediction algorithm. */
 190                                         RRA_hw_beta,
 191                                         /* exponential smoothing parameter for the slope in
 192                                          * the Holt-Winters prediction algorithm. */
 193                                         RRA_dependent_rra_idx,
 194                                         /* For CF_HWPREDICT: index of the RRA with the seasonal
 195                                          * effects of the Holt-Winters algorithm (of type
 196                                          * CF_SEASONAL).
 197                                          * For CF_DEVPREDICT: index of the RRA with the seasonal
 198                                          * deviation predictions (of type CF_DEVSEASONAL).
 199                                          * For CF_SEASONAL: index of the RRA with the Holt-Winters
 200                                          * intercept and slope coefficient (of type CF_HWPREDICT).
 201                                          * For CF_DEVSEASONAL: index of the RRA with the
 202                                          * Holt-Winters prediction (of type CF_HWPREDICT).
 203                                          * For CF_FAILURES: index of the CF_DEVSEASONAL array.
 204                                          * */
 205                                         RRA_seasonal_smooth_idx,
 206                                         /* For CF_SEASONAL and CF_DEVSEASONAL:
 207                                          * an integer between 0 and row_count - 1 which
 208                                          * is index in the seasonal cycle for applying
 209                                          * the period smoother. */
 210                                     RRA_failure_threshold,
 211                                         /* For CF_FAILURES, number of violations within the last
 212                                          * window required to mark a failure. */
 213                     RRA_seasonal_gamma = RRA_hw_alpha,
 214                                         /* exponential smoothing parameter for seasonal effects.
 215                                          * */
 216                     RRA_delta_pos = RRA_hw_alpha,
 217                     RRA_delta_neg = RRA_hw_beta,
 218                                         /* confidence bound scaling parameters for the
 219                                          * the FAILURES RRA. */
 220                     RRA_window_len = RRA_seasonal_smooth_idx};
 221                                         /* For CF_FAILURES, the length of the window for measuring
 222                                          * failures. */
 223
 224 #define CF_NAM_FMT    "%19[A-Z]"
 225 #define CF_NAM_SIZE   20
 226
 227 typedef struct rra_def_t {
 228     char             cf_nam[CF_NAM_SIZE];/* consolidation function (null term) */
 229     unsigned long    row_cnt;            /* number of entries in the store */
 230     unsigned long    pdp_cnt;            /* how many primary data points are
 231                                           * required for a consolidated data
 232                                           * point?*/
 233     unival           par[MAX_RRA_PAR_EN];            /* index see rra_param_en */
 234
 235 } rra_def_t;
 236
 237
 238 /****************************************************************************
 239  ****************************************************************************
 240  ****************************************************************************
 241  * LIVE PART OF THE HEADER. THIS WILL BE WRITTEN ON EVERY UPDATE         *
 242  ****************************************************************************
 243  ****************************************************************************
 244  ****************************************************************************/
 245 /****************************************************************************
 246  * POS 4: live_head_t
 247  ****************************************************************************/
 248
 249 typedef struct live_head_t {
 250     time_t           last_up;            /* when was rrd last updated */
 251 } live_head_t;
 252
 253
 254 /****************************************************************************
 255  * POS 5: pdp_prep_t  (* ds_cnt)                     here we prepare the pdps
 256  ****************************************************************************/
 257 #define LAST_DS_LEN 30 /* DO NOT CHANGE THIS ... */
 258
 259 enum pdp_par_en {   PDP_unkn_sec_cnt=0,  /* how many seconds of the current
 260                                           * pdp value is unknown data? */
 261
 262                     PDP_val};            /* current value of the pdp.
 263                                             this depends on dst */
 264
 265 typedef struct pdp_prep_t{
 266     char last_ds[LAST_DS_LEN];           /* the last reading from the data
 267                                           * source.  this is stored in ASCII
 268                                           * to cater for very large counters
 269                                           * we might encounter in connection
 270                                           * with SNMP. */
 271     unival          scratch[10];         /* contents according to pdp_par_en */
 272 } pdp_prep_t;
 273
 274 /* data is passed from pdp to cdp when seconds since epoch modulo pdp_step == 0
 275    obviously the updates do not occur at these times only. Especially does the
 276    format allow for updates to occur at different times for each data source.
 277    The rules which makes this work is as follows:
 278
 279    * DS updates may only occur at ever increasing points in time
 280    * When any DS update arrives after a cdp update time, the *previous*
 281      update cycle gets executed. All pdps are transfered to cdps and the
 282      cdps feed the rras where necessary. Only then the new DS value
 283      is loaded into the PDP.                                                   */
 284
 285
 286 /****************************************************************************
 287  * POS 6: cdp_prep_t (* rra_cnt * ds_cnt )      data prep area for cdp values
 288  ****************************************************************************/
 289 #define MAX_CDP_PAR_EN 10
 290 #define MAX_CDP_FAILURES_IDX 8
 291 /* max CDP scratch entries avail to record violations for a FAILURES RRA */
 292 #define MAX_FAILURES_WINDOW_LEN 28
 293 enum cdp_par_en {  CDP_val=0,
 294                    /* the base_interval is always an
 295                                         * average */
 296                            CDP_unkn_pdp_cnt,
 297                                    /* how many unknown pdp were
 298                     * integrated. This and the cdp_xff
 299                                         * will decide if this is going to
 300                                         * be a UNKNOWN or a valid value */
 301                                    CDP_hw_intercept,
 302                                    /* Current intercept coefficient for the Holt-Winters
 303                                         * prediction algorithm. */
 304                                    CDP_hw_last_intercept,
 305                                    /* Last iteration intercept coefficient for the Holt-Winters
 306                                         * prediction algorihtm. */
 307                                    CDP_hw_slope,
 308                                    /* Current slope coefficient for the Holt-Winters
 309                                         * prediction algorithm. */
 310                                    CDP_hw_last_slope,
 311                                    /* Last iteration slope coeffient. */
 312                                    CDP_null_count,
 313                                    /* Number of sequential Unknown (DNAN) values + 1 preceding
 314                                     * the current prediction.
 315                                         * */
 316                                    CDP_last_null_count,
 317                                    /* Last iteration count of Unknown (DNAN) values. */
 318                                    CDP_primary_val = 8,
 319                                    /* optimization for bulk updates: the value of the first CDP
 320                                         * value to be written in the bulk update. */
 321                                    CDP_secondary_val = 9,
 322                                    /* optimization for bulk updates: the value of subsequent
 323                                         * CDP values to be written in the bulk update. */
 324                    CDP_hw_seasonal = CDP_hw_intercept,
 325                    /* Current seasonal coefficient for the Holt-Winters
 326                     * prediction algorithm. This is stored in CDP prep to avoid
 327                     * redundant seek operations. */
 328                    CDP_hw_last_seasonal = CDP_hw_last_intercept,
 329                    /* Last iteration seasonal coeffient. */
 330                    CDP_seasonal_deviation = CDP_hw_intercept,
 331                    CDP_last_seasonal_deviation = CDP_hw_last_intercept,
 332                    CDP_init_seasonal = CDP_null_count};
 333                    /* init_seasonal is a flag which when > 0, forces smoothing updates
 334                     * to occur when rra_ptr.cur_row == 0 */
 335
 336 typedef struct cdp_prep_t{
 337     unival         scratch[MAX_CDP_PAR_EN];
 338                                                                                  /* contents according to cdp_par_en *
 339                                           * init state should be NAN */
 340
 341 } cdp_prep_t;
 342
 343 /****************************************************************************
 344  * POS 7: rra_ptr_t (* rra_cnt)       pointers to the current row in each rra
 345  ****************************************************************************/
 346
 347 typedef struct rra_ptr_t {
 348     unsigned long    cur_row;            /* current row in the rra*/
 349 } rra_ptr_t;
 350
 351
 352 /****************************************************************************
 353  ****************************************************************************
 354  * One single struct to hold all the others. For convenience.
 355  ****************************************************************************
 356  ****************************************************************************/
 357 typedef struct rrd_t {
 358     stat_head_t      *stat_head;          /* the static header */
 359     ds_def_t         *ds_def;             /* list of data source definitions */
 360     rra_def_t        *rra_def;            /* list of round robin archive def */
 361     live_head_t      *live_head;
 362     pdp_prep_t       *pdp_prep;           /* pdp data prep area */
 363     cdp_prep_t       *cdp_prep;           /* cdp prep area */
 364     rra_ptr_t        *rra_ptr;            /* list of rra pointers */
 365     rrd_value_t      *rrd_value;          /* list of rrd values */
 366 } rrd_t;
 367
 368 /****************************************************************************
 369  ****************************************************************************
 370  * AFTER the header section we have the DATA STORAGE AREA it is made up from
 371  * Consolidated Data Points organized in Round Robin Archives.
 372  ****************************************************************************
 373  ****************************************************************************
 374
 375  *RRA 0
 376  (0,0) .................... ( ds_cnt -1 , 0)
 377  .
 378  .
 379  .
 380  (0, row_cnt -1) ... (ds_cnt -1, row_cnt -1)
 381
 382  *RRA 1
 383  *RRA 2
 384
 385  *RRA rra_cnt -1
 386
 387  ****************************************************************************/
 388
 389
 390 #endif
 391
 392
 393
 394