src/rrd_format.h

   1 /*****************************************************************************
   2  * RRDtool 1.2rc6  Copyright by Tobi Oetiker, 1997-2005
   3  *****************************************************************************
   4  * rrd_format.h  RRD Database Format header
   5  *****************************************************************************/
   6
   7 #ifndef _RRD_FORMAT_H
   8 #define _RRD_FORMAT_H
   9
  10 #include "rrd.h"
  11
  12 /*****************************************************************************
  13  * put this in your /usr/lib/magic file (/etc/magic on HPUX)
  14  *
  15  *  # rrd database format
  16  *  0       string          RRD\0           rrd file
  17  *  >5      string          >\0             version '%s'
  18  *
  19  *****************************************************************************/
  20
  21 #define RRD_COOKIE    "RRD"
  22 /* #define RRD_VERSION   "0002" */
  23 /* changed because microsecond precision requires another field */
  24 #define RRD_VERSION   "0003"
  25 #define FLOAT_COOKIE  8.642135E130
  26
  27 #include "rrd_nan_inf.h"
  28
  29 typedef union unival {
  30     unsigned long u_cnt;
  31     rrd_value_t   u_val;
  32 } unival;
  33
  34
  35 /****************************************************************************
  36  * The RRD Database Structure
  37  * ---------------------------
  38  *
  39  * In oder to properly describe the database structure lets define a few
  40  * new words:
  41  *
  42  * ds - Data Source (ds) providing input to the database. A Data Source (ds)
  43  *       can be a traffic counter, a temperature, the number of users logged
  44  *       into a system. The rrd database format can handle the input of
  45  *       several Data Sources (ds) in a singe database.
  46  *
  47  * dst - Data Source Type (dst). The Data Source Type (dst) defines the rules
  48  *       applied to Build Primary Data Points from the input provided by the
  49  *       data sources (ds).
  50  *
  51  * pdp - Primary Data Point (pdp). After the database has accepted the
  52  *       input from the data sources (ds). It starts building Primary
  53  *       Data Points (pdp) from the data. Primary Data Points (pdp)
  54  *       are evenly spaced along the time axis (pdp_step). The values
  55  *       of the Primary Data Points are calculated from the values of
  56  *       the data source (ds) and the exact time these values were
  57  *       provided by the data source (ds).
  58  *
  59  * pdp_st - PDP Start (pdp_st). The moments (pdp_st) in time where
  60  *       these steps occur are defined by the moments where the
  61  *       number of seconds since 1970-jan-1 modulo pdp_step equals
  62  *       zero (pdp_st).
  63  *
  64  * cf -  Consolidation Function (cf). An arbitrary Consolidation Function (cf)
  65  *       (averaging, min, max) is applied to the primary data points (pdp) to
  66  *       calculate the consolidated data point.
  67  *
  68  * cdp - Consolidated Data Point (cdp) is the long term storage format for data
  69  *       in the rrd database. Consolidated Data Points represent one or
  70  *       several primary data points collected along the time axis. The
  71  *       Consolidated Data Points (cdp) are stored in Round Robin Archives
  72  *       (rra).
  73  *
  74  * rra - Round Robin Archive (rra). This is the place where the
  75  *       consolidated data points (cdp) get stored. The data is
  76  *       organized in rows (row) and columns (col). The Round Robin
  77  *       Archive got its name from the method data is stored in
  78  *       there. An RRD database can contain several Round Robin
  79  *       Archives. Each Round Robin Archive can have a different row
  80  *       spacing along the time axis (pdp_cnt) and a different
  81  *       consolidation function (cf) used to build its consolidated
  82  *       data points (cdp).
  83  *
  84  * rra_st - RRA Start (rra_st). The moments (rra_st) in time where
  85  *       Consolidated Data Points (cdp) are added to an rra are
  86  *       defined by the moments where the number of seconds since
  87  *       1970-jan-1 modulo pdp_cnt*pdp_step equals zero (rra_st).
  88  *
  89  * row - Row (row). A row represent all consolidated data points (cdp)
  90  *       in a round robin archive who are of the same age.
  91  *
  92  * col - Column (col). A column (col) represent all consolidated
  93  *       data points (cdp) in a round robin archive (rra) who
  94  *       originated from the same data source (ds).
  95  *
  96  */
  97
  98 /****************************************************************************
  99  * POS 1: stat_head_t                           static header of the database
 100  ****************************************************************************/
 101
 102 typedef struct stat_head_t {
 103
 104     /* Data Base Identification Section ***/
 105     char             cookie[4];          /* RRD */
 106     char             version[5];         /* version of the format */
 107     double           float_cookie;       /* is it the correct double
 108                                           * representation ?  */
 109
 110     /* Data Base Structure Definition *****/
 111     unsigned long    ds_cnt;             /* how many different ds provide
 112                                           * input to the rrd */
 113     unsigned long    rra_cnt;            /* how many rras will be maintained
 114                                           * in the rrd */
 115     unsigned long    pdp_step;           /* pdp interval in seconds */
 116
 117     unival           par[10];            /* global parameters ... unused
 118                                             at the moment */
 119 } stat_head_t;
 120
 121
 122 /****************************************************************************
 123  * POS 2: ds_def_t  (* ds_cnt)                        Data Source definitions
 124  ****************************************************************************/
 125
 126 enum dst_en          { DST_COUNTER=0,     /* data source types available */
 127                        DST_ABSOLUTE,
 128                        DST_GAUGE,
 129                        DST_DERIVE,
 130                                            DST_CDEF};
 131
 132 enum ds_param_en {   DS_mrhb_cnt=0,       /* minimum required heartbeat. A
 133                                            * data source must provide input at
 134                                            * least every ds_mrhb seconds,
 135                                            * otherwise it is regarded dead and
 136                                            * will be set to UNKNOWN */
 137                      DS_min_val,          /* the processed input of a ds must */
 138                      DS_max_val,      /* be between max_val and min_val
 139                                            * both can be set to UNKNOWN if you
 140                                            * do not care. Data outside the limits
 141                                            * set to UNKNOWN */
 142                      DS_cdef = DS_mrhb_cnt}; /* pointer to encoded rpn
 143                                            * expression only applies to DST_CDEF */
 144
 145 /* The magic number here is one less than DS_NAM_SIZE */
 146 #define DS_NAM_FMT    "%19[a-zA-Z0-9_-]"
 147 #define DS_NAM_SIZE   20
 148
 149 #define DST_FMT    "%19[A-Z]"
 150 #define DST_SIZE   20
 151
 152 typedef struct ds_def_t {
 153     char             ds_nam[DS_NAM_SIZE]; /* Name of the data source (null terminated)*/
 154     char             dst[DST_SIZE];       /* Type of data source (null terminated)*/
 155     unival           par[10];             /* index of this array see ds_param_en */
 156 } ds_def_t;
 157
 158 /****************************************************************************
 159  * POS 3: rra_def_t ( *  rra_cnt)         one for each store to be maintained
 160  ****************************************************************************/
 161 enum cf_en           { CF_AVERAGE=0,     /* data consolidation functions */
 162                        CF_MINIMUM,
 163                        CF_MAXIMUM,
 164                        CF_LAST,
 165                        CF_HWPREDICT,
 166                                            /* An array of predictions using the seasonal
 167                                                 * Holt-Winters algorithm. Requires an RRA of type
 168                                                 * CF_SEASONAL for this data source. */
 169                                            CF_SEASONAL,
 170                                            /* An array of seasonal effects. Requires an RRA of
 171                                                 * type CF_HWPREDICT for this data source. */
 172                                            CF_DEVPREDICT,
 173                                            /* An array of deviation predictions based upon
 174                                                 * smoothed seasonal deviations. Requires an RRA of
 175                                                 * type CF_DEVSEASONAL for this data source. */
 176                                            CF_DEVSEASONAL,
 177                                            /* An array of smoothed seasonal deviations. Requires
 178                                                 * an RRA of type CF_HWPREDICT for this data source.
 179                                                 * */
 180                                            CF_FAILURES};
 181                                            /* A binary array of failure indicators: 1 indicates
 182                                                 * that the number of violations in the prescribed
 183                                                 * window exceeded the prescribed threshold. */
 184
 185 #define MAX_RRA_PAR_EN 10
 186 enum rra_par_en {   RRA_cdp_xff_val=0,  /* what part of the consolidated
 187                      * datapoint must be known, to produce a
 188                                          * valid entry in the rra */
 189                                         RRA_hw_alpha,
 190                                         /* exponential smoothing parameter for the intercept in
 191                                          * the Holt-Winters prediction algorithm. */
 192                                         RRA_hw_beta,
 193                                         /* exponential smoothing parameter for the slope in
 194                                          * the Holt-Winters prediction algorithm. */
 195                                         RRA_dependent_rra_idx,
 196                                         /* For CF_HWPREDICT: index of the RRA with the seasonal
 197                                          * effects of the Holt-Winters algorithm (of type
 198                                          * CF_SEASONAL).
 199                                          * For CF_DEVPREDICT: index of the RRA with the seasonal
 200                                          * deviation predictions (of type CF_DEVSEASONAL).
 201                                          * For CF_SEASONAL: index of the RRA with the Holt-Winters
 202                                          * intercept and slope coefficient (of type CF_HWPREDICT).
 203                                          * For CF_DEVSEASONAL: index of the RRA with the
 204                                          * Holt-Winters prediction (of type CF_HWPREDICT).
 205                                          * For CF_FAILURES: index of the CF_DEVSEASONAL array.
 206                                          * */
 207                                         RRA_seasonal_smooth_idx,
 208                                         /* For CF_SEASONAL and CF_DEVSEASONAL:
 209                                          * an integer between 0 and row_count - 1 which
 210                                          * is index in the seasonal cycle for applying
 211                                          * the period smoother. */
 212                                     RRA_failure_threshold,
 213                                         /* For CF_FAILURES, number of violations within the last
 214                                          * window required to mark a failure. */
 215                     RRA_seasonal_gamma = RRA_hw_alpha,
 216                                         /* exponential smoothing parameter for seasonal effects.
 217                                          * */
 218                     RRA_delta_pos = RRA_hw_alpha,
 219                     RRA_delta_neg = RRA_hw_beta,
 220                                         /* confidence bound scaling parameters for the
 221                                          * the FAILURES RRA. */
 222                     RRA_window_len = RRA_seasonal_smooth_idx};
 223                                         /* For CF_FAILURES, the length of the window for measuring
 224                                          * failures. */
 225
 226 #define CF_NAM_FMT    "%19[A-Z]"
 227 #define CF_NAM_SIZE   20
 228
 229 typedef struct rra_def_t {
 230     char             cf_nam[CF_NAM_SIZE];/* consolidation function (null term) */
 231     unsigned long    row_cnt;            /* number of entries in the store */
 232     unsigned long    pdp_cnt;            /* how many primary data points are
 233                                           * required for a consolidated data
 234                                           * point?*/
 235     unival           par[MAX_RRA_PAR_EN];            /* index see rra_param_en */
 236
 237 } rra_def_t;
 238
 239
 240 /****************************************************************************
 241  ****************************************************************************
 242  ****************************************************************************
 243  * LIVE PART OF THE HEADER. THIS WILL BE WRITTEN ON EVERY UPDATE         *
 244  ****************************************************************************
 245  ****************************************************************************
 246  ****************************************************************************/
 247 /****************************************************************************
 248  * POS 4: live_head_t
 249  ****************************************************************************/
 250
 251 typedef struct live_head_t {
 252     time_t           last_up;            /* when was rrd last updated */
 253     long             last_up_usec;       /* micro seconds part of the
 254                                             update timestamp. Always >= 0 */
 255 } live_head_t;
 256
 257
 258 /****************************************************************************
 259  * POS 5: pdp_prep_t  (* ds_cnt)                     here we prepare the pdps
 260  ****************************************************************************/
 261 #define LAST_DS_LEN 30 /* DO NOT CHANGE THIS ... */
 262
 263 enum pdp_par_en {   PDP_unkn_sec_cnt=0,  /* how many seconds of the current
 264                                           * pdp value is unknown data? */
 265
 266                     PDP_val};            /* current value of the pdp.
 267                                             this depends on dst */
 268
 269 typedef struct pdp_prep_t{
 270     char last_ds[LAST_DS_LEN];           /* the last reading from the data
 271                                           * source.  this is stored in ASCII
 272                                           * to cater for very large counters
 273                                           * we might encounter in connection
 274                                           * with SNMP. */
 275     unival          scratch[10];         /* contents according to pdp_par_en */
 276 } pdp_prep_t;
 277
 278 /* data is passed from pdp to cdp when seconds since epoch modulo pdp_step == 0
 279    obviously the updates do not occur at these times only. Especially does the
 280    format allow for updates to occur at different times for each data source.
 281    The rules which makes this work is as follows:
 282
 283    * DS updates may only occur at ever increasing points in time
 284    * When any DS update arrives after a cdp update time, the *previous*
 285      update cycle gets executed. All pdps are transfered to cdps and the
 286      cdps feed the rras where necessary. Only then the new DS value
 287      is loaded into the PDP.                                                   */
 288
 289
 290 /****************************************************************************
 291  * POS 6: cdp_prep_t (* rra_cnt * ds_cnt )      data prep area for cdp values
 292  ****************************************************************************/
 293 #define MAX_CDP_PAR_EN 10
 294 #define MAX_CDP_FAILURES_IDX 8
 295 /* max CDP scratch entries avail to record violations for a FAILURES RRA */
 296 #define MAX_FAILURES_WINDOW_LEN 28
 297 enum cdp_par_en {  CDP_val=0,
 298                    /* the base_interval is always an
 299                                         * average */
 300                            CDP_unkn_pdp_cnt,
 301                                    /* how many unknown pdp were
 302                     * integrated. This and the cdp_xff
 303                                         * will decide if this is going to
 304                                         * be a UNKNOWN or a valid value */
 305                                    CDP_hw_intercept,
 306                                    /* Current intercept coefficient for the Holt-Winters
 307                                         * prediction algorithm. */
 308                                    CDP_hw_last_intercept,
 309                                    /* Last iteration intercept coefficient for the Holt-Winters
 310                                         * prediction algorihtm. */
 311                                    CDP_hw_slope,
 312                                    /* Current slope coefficient for the Holt-Winters
 313                                         * prediction algorithm. */
 314                                    CDP_hw_last_slope,
 315                                    /* Last iteration slope coeffient. */
 316                                    CDP_null_count,
 317                                    /* Number of sequential Unknown (DNAN) values + 1 preceding
 318                                     * the current prediction.
 319                                         * */
 320                                    CDP_last_null_count,
 321                                    /* Last iteration count of Unknown (DNAN) values. */
 322                                    CDP_primary_val = 8,
 323                                    /* optimization for bulk updates: the value of the first CDP
 324                                         * value to be written in the bulk update. */
 325                                    CDP_secondary_val = 9,
 326                                    /* optimization for bulk updates: the value of subsequent
 327                                         * CDP values to be written in the bulk update. */
 328                    CDP_hw_seasonal = CDP_hw_intercept,
 329                    /* Current seasonal coefficient for the Holt-Winters
 330                     * prediction algorithm. This is stored in CDP prep to avoid
 331                     * redundant seek operations. */
 332                    CDP_hw_last_seasonal = CDP_hw_last_intercept,
 333                    /* Last iteration seasonal coeffient. */
 334                    CDP_seasonal_deviation = CDP_hw_intercept,
 335                    CDP_last_seasonal_deviation = CDP_hw_last_intercept,
 336                    CDP_init_seasonal = CDP_null_count};
 337                    /* init_seasonal is a flag which when > 0, forces smoothing updates
 338                     * to occur when rra_ptr.cur_row == 0 */
 339
 340 typedef struct cdp_prep_t{
 341     unival         scratch[MAX_CDP_PAR_EN];
 342                                                                                  /* contents according to cdp_par_en *
 343                                           * init state should be NAN */
 344
 345 } cdp_prep_t;
 346
 347 /****************************************************************************
 348  * POS 7: rra_ptr_t (* rra_cnt)       pointers to the current row in each rra
 349  ****************************************************************************/
 350
 351 typedef struct rra_ptr_t {
 352     unsigned long    cur_row;            /* current row in the rra*/
 353 } rra_ptr_t;
 354
 355
 356 /****************************************************************************
 357  ****************************************************************************
 358  * One single struct to hold all the others. For convenience.
 359  ****************************************************************************
 360  ****************************************************************************/
 361 typedef struct rrd_t {
 362     stat_head_t      *stat_head;          /* the static header */
 363     ds_def_t         *ds_def;             /* list of data source definitions */
 364     rra_def_t        *rra_def;            /* list of round robin archive def */
 365     live_head_t      *live_head;
 366     pdp_prep_t       *pdp_prep;           /* pdp data prep area */
 367     cdp_prep_t       *cdp_prep;           /* cdp prep area */
 368     rra_ptr_t        *rra_ptr;            /* list of rra pointers */
 369     rrd_value_t      *rrd_value;          /* list of rrd values */
 370 } rrd_t;
 371
 372 /****************************************************************************
 373  ****************************************************************************
 374  * AFTER the header section we have the DATA STORAGE AREA it is made up from
 375  * Consolidated Data Points organized in Round Robin Archives.
 376  ****************************************************************************
 377  ****************************************************************************
 378
 379  *RRA 0
 380  (0,0) .................... ( ds_cnt -1 , 0)
 381  .
 382  .
 383  .
 384  (0, row_cnt -1) ... (ds_cnt -1, row_cnt -1)
 385
 386  *RRA 1
 387  *RRA 2
 388
 389  *RRA rra_cnt -1
 390
 391  ****************************************************************************/
 392
 393
 394 #endif
 395
 396
 397
 398