1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_create.c creates new rrds
5 *****************************************************************************/
8 #include "rrd_rpncalc.h"
11 #include "rrd_is_thread_safe.h"
13 unsigned long FnvHash(
15 int create_hw_contingent_rras(
17 unsigned short period,
18 unsigned long hashed_name);
28 time_t last_up = time(NULL) - 10;
29 unsigned long pdp_step = 300;
30 struct rrd_time_value last_up_tv;
31 char *parsetime_error = NULL;
36 opterr = 0; /* initialize getopt */
39 static struct option long_options[] = {
40 {"start", required_argument, 0, 'b'},
41 {"step", required_argument, 0, 's'},
47 opt = getopt_long(argc, argv, "b:s:", long_options, &option_index);
54 if ((parsetime_error = parsetime(optarg, &last_up_tv))) {
55 rrd_set_error("start time: %s", parsetime_error);
58 if (last_up_tv.type == RELATIVE_TO_END_TIME ||
59 last_up_tv.type == RELATIVE_TO_START_TIME) {
60 rrd_set_error("specifying time relative to the 'start' "
61 "or 'end' makes no sense here");
65 last_up = mktime(&last_up_tv.tm) + last_up_tv.offset;
67 if (last_up < 3600 * 24 * 365 * 10) {
69 ("the first entry to the RRD should be after 1980");
75 long_tmp = atol(optarg);
77 rrd_set_error("step size should be no less than one second");
85 rrd_set_error("unknown option '%c'", optopt);
87 rrd_set_error("unknown option '%s'", argv[optind - 1]);
92 rrd_set_error("what is the name of the rrd file you want to create?");
95 rc = rrd_create_r(argv[optind],
97 argc - optind - 1, (const char **) (argv + optind + 1));
104 const char *filename,
105 unsigned long pdp_step,
114 char dummychar1[2], dummychar2[2];
115 unsigned short token_idx, error_flag, period = 0;
116 unsigned long hashed_name;
121 if ((rrd.stat_head = calloc(1, sizeof(stat_head_t))) == NULL) {
122 rrd_set_error("allocating rrd.stat_head");
128 if ((rrd.live_head = calloc(1, sizeof(live_head_t))) == NULL) {
129 rrd_set_error("allocating rrd.live_head");
134 /* set some defaults */
135 strcpy(rrd.stat_head->cookie, RRD_COOKIE);
136 strcpy(rrd.stat_head->version, RRD_VERSION);
137 rrd.stat_head->float_cookie = FLOAT_COOKIE;
138 rrd.stat_head->ds_cnt = 0; /* this will be adjusted later */
139 rrd.stat_head->rra_cnt = 0; /* ditto */
140 rrd.stat_head->pdp_step = pdp_step; /* 5 minute default */
142 /* a default value */
146 rrd.live_head->last_up = last_up;
148 /* optind points to the first non-option command line arg,
149 * in this case, the file name. */
150 /* Compute the FNV hash value (used by SEASONAL and DEVSEASONAL
152 hashed_name = FnvHash(filename);
153 for (i = 0; i < argc; i++) {
156 if (strncmp(argv[i], "DS:", 3) == 0) {
157 size_t old_size = sizeof(ds_def_t) * (rrd.stat_head->ds_cnt);
159 if ((rrd.ds_def = rrd_realloc(rrd.ds_def,
160 old_size + sizeof(ds_def_t))) ==
162 rrd_set_error("allocating rrd.ds_def");
166 memset(&rrd.ds_def[rrd.stat_head->ds_cnt], 0, sizeof(ds_def_t));
167 /* extract the name and type */
168 switch (sscanf(&argv[i][3],
169 DS_NAM_FMT "%1[:]" DST_FMT "%1[:]%n",
170 rrd.ds_def[rrd.stat_head->ds_cnt].ds_nam,
172 rrd.ds_def[rrd.stat_head->ds_cnt].dst,
173 dummychar2, &offset)) {
176 rrd_set_error("Invalid DS name");
180 rrd_set_error("Invalid DS type");
182 case 4: /* (%n may or may not be counted) */
183 case 5: /* check for duplicate datasource names */
184 for (ii = 0; ii < rrd.stat_head->ds_cnt; ii++)
185 if (strcmp(rrd.ds_def[rrd.stat_head->ds_cnt].ds_nam,
186 rrd.ds_def[ii].ds_nam) == 0)
187 rrd_set_error("Duplicate DS name: %s",
188 rrd.ds_def[ii].ds_nam);
189 /* DS_type may be valid or not. Checked later */
192 rrd_set_error("invalid DS format");
194 if (rrd_test_error()) {
199 /* parse the remainder of the arguments */
200 switch (dst_conv(rrd.ds_def[rrd.stat_head->ds_cnt].dst)) {
205 parseGENERIC_DS(&argv[i][offset + 3], &rrd,
206 rrd.stat_head->ds_cnt);
209 parseCDEF_DS(&argv[i][offset + 3], &rrd,
210 rrd.stat_head->ds_cnt);
213 rrd_set_error("invalid DS type specified");
217 if (rrd_test_error()) {
221 rrd.stat_head->ds_cnt++;
222 } else if (strncmp(argv[i], "RRA:", 4) == 0) {
225 size_t old_size = sizeof(rra_def_t) * (rrd.stat_head->rra_cnt);
227 if ((rrd.rra_def = rrd_realloc(rrd.rra_def,
228 old_size + sizeof(rra_def_t))) ==
230 rrd_set_error("allocating rrd.rra_def");
234 memset(&rrd.rra_def[rrd.stat_head->rra_cnt], 0,
237 argvcopy = strdup(argv[i]);
238 token = strtok_r(&argvcopy[4], ":", &tokptr);
239 token_idx = error_flag = 0;
240 while (token != NULL) {
243 if (sscanf(token, CF_NAM_FMT,
244 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam) !=
246 rrd_set_error("Failed to parse CF name");
248 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
250 /* initialize some parameters */
251 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_alpha].
253 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_beta].
255 rrd.rra_def[rrd.stat_head->rra_cnt].
256 par[RRA_dependent_rra_idx].u_cnt =
257 rrd.stat_head->rra_cnt;
261 /* initialize some parameters */
262 rrd.rra_def[rrd.stat_head->rra_cnt].
263 par[RRA_seasonal_gamma].u_val = 0.1;
266 rrd.rra_def[rrd.stat_head->rra_cnt].
267 par[RRA_dependent_rra_idx].u_cnt = -1;
270 rrd.rra_def[rrd.stat_head->rra_cnt].
271 par[RRA_delta_pos].u_val = 2.0;
272 rrd.rra_def[rrd.stat_head->rra_cnt].
273 par[RRA_delta_neg].u_val = 2.0;
274 rrd.rra_def[rrd.stat_head->rra_cnt].
275 par[RRA_window_len].u_cnt = 3;
276 rrd.rra_def[rrd.stat_head->rra_cnt].
277 par[RRA_failure_threshold].u_cnt = 2;
278 rrd.rra_def[rrd.stat_head->rra_cnt].
279 par[RRA_dependent_rra_idx].u_cnt = -1;
281 /* invalid consolidation function */
284 ("Unrecognized consolidation function %s",
285 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam);
289 /* default: 1 pdp per cdp */
290 rrd.rra_def[rrd.stat_head->rra_cnt].pdp_cnt = 1;
294 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
300 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt =
304 rrd.rra_def[rrd.stat_head->rra_cnt].
305 par[RRA_cdp_xff_val].u_val = atof(token);
306 if (rrd.rra_def[rrd.stat_head->rra_cnt].
307 par[RRA_cdp_xff_val].u_val < 0.0
308 || rrd.rra_def[rrd.stat_head->rra_cnt].
309 par[RRA_cdp_xff_val].u_val >= 1.0)
311 ("Invalid xff: must be between 0 and 1");
317 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
319 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_alpha].
321 if (atof(token) <= 0.0 || atof(token) >= 1.0)
323 ("Invalid alpha: must be between 0 and 1");
327 rrd.rra_def[rrd.stat_head->rra_cnt].
328 par[RRA_seasonal_gamma].u_val = atof(token);
329 if (atof(token) <= 0.0 || atof(token) >= 1.0)
331 ("Invalid gamma: must be between 0 and 1");
332 rrd.rra_def[rrd.stat_head->rra_cnt].
333 par[RRA_seasonal_smooth_idx].u_cnt =
335 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt;
338 /* specifies the # of violations that constitutes the failure threshold */
339 rrd.rra_def[rrd.stat_head->rra_cnt].
340 par[RRA_failure_threshold].u_cnt = atoi(token);
342 || atoi(token) > MAX_FAILURES_WINDOW_LEN)
344 ("Failure threshold is out of range %d, %d",
345 1, MAX_FAILURES_WINDOW_LEN);
348 /* specifies the index (1-based) of CF_DEVSEASONAL array
349 * associated with this CF_DEVPREDICT array. */
350 rrd.rra_def[rrd.stat_head->rra_cnt].
351 par[RRA_dependent_rra_idx].u_cnt =
355 rrd.rra_def[rrd.stat_head->rra_cnt].pdp_cnt =
362 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
364 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_beta].
366 if (atof(token) < 0.0 || atof(token) > 1.0)
368 ("Invalid beta: must be between 0 and 1");
372 /* specifies the index (1-based) of CF_HWPREDICT array
373 * associated with this CF_DEVSEASONAL or CF_SEASONAL array.
375 rrd.rra_def[rrd.stat_head->rra_cnt].
376 par[RRA_dependent_rra_idx].u_cnt =
380 /* specifies the window length */
381 rrd.rra_def[rrd.stat_head->rra_cnt].
382 par[RRA_window_len].u_cnt = atoi(token);
384 || atoi(token) > MAX_FAILURES_WINDOW_LEN)
386 ("Window length is out of range %d, %d", 1,
387 MAX_FAILURES_WINDOW_LEN);
388 /* verify that window length exceeds the failure threshold */
389 if (rrd.rra_def[rrd.stat_head->rra_cnt].
390 par[RRA_window_len].u_cnt <
391 rrd.rra_def[rrd.stat_head->rra_cnt].
392 par[RRA_failure_threshold].u_cnt)
394 ("Window length is shorter than the failure threshold");
397 /* shouldn't be any more arguments */
399 ("Unexpected extra argument for consolidation function DEVPREDICT");
402 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt =
409 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
411 /* specifies the index (1-based) of CF_DEVSEASONAL array
412 * associated with this CF_DEVFAILURES array. */
413 rrd.rra_def[rrd.stat_head->rra_cnt].
414 par[RRA_dependent_rra_idx].u_cnt =
418 /* length of the associated CF_SEASONAL and CF_DEVSEASONAL arrays. */
419 period = atoi(token);
421 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt)
423 ("Length of seasonal cycle exceeds length of HW prediction array");
426 /* shouldn't be any more arguments */
428 ("Unexpected extra argument for consolidation function %s",
429 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam);
434 /* If we are here, this must be a CF_HWPREDICT RRA.
435 * Specifies the index (1-based) of CF_SEASONAL array
436 * associated with this CF_HWPREDICT array. If this argument
437 * is missing, then the CF_SEASONAL, CF_DEVSEASONAL, CF_DEVPREDICT,
439 * arrays are created automatically. */
440 rrd.rra_def[rrd.stat_head->rra_cnt].
441 par[RRA_dependent_rra_idx].u_cnt = atoi(token) - 1;
444 /* should never get here */
445 rrd_set_error("Unknown error");
448 if (rrd_test_error()) {
449 /* all errors are unrecoverable */
454 token = strtok_r(NULL, ":", &tokptr);
460 "Creating RRA CF: %s, dep idx %lu, current idx %lu\n",
461 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam,
462 rrd.rra_def[rrd.stat_head->rra_cnt].
463 par[RRA_dependent_rra_idx].u_cnt, rrd.stat_head->rra_cnt);
465 /* should we create CF_SEASONAL, CF_DEVSEASONAL, and CF_DEVPREDICT? */
466 if (cf_conv(rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam) ==
468 && rrd.rra_def[rrd.stat_head->rra_cnt].
469 par[RRA_dependent_rra_idx].u_cnt == rrd.stat_head->rra_cnt) {
471 fprintf(stderr, "Creating HW contingent RRAs\n");
473 if (create_hw_contingent_rras(&rrd, period, hashed_name) ==
475 rrd_set_error("creating contingent RRA");
480 rrd.stat_head->rra_cnt++;
482 rrd_set_error("can't parse argument '%s'", argv[i]);
489 if (rrd.stat_head->rra_cnt < 1) {
490 rrd_set_error("you must define at least one Round Robin Archive");
495 if (rrd.stat_head->ds_cnt < 1) {
496 rrd_set_error("you must define at least one Data Source");
500 return rrd_create_fn(filename, &rrd);
503 void parseGENERIC_DS(
508 char minstr[DS_NAM_SIZE], maxstr[DS_NAM_SIZE];
513 temp = sscanf(def,"%lu:%18[^:]:%18[^:]",
514 &(rrd -> ds_def[ds_idx].par[DS_mrhb_cnt].u_cnt),
517 if (sscanf(def, "%lu:%18[^:]:%18[^:]",
518 &(rrd->ds_def[ds_idx].par[DS_mrhb_cnt].u_cnt),
519 minstr, maxstr) == 3) {
520 if (minstr[0] == 'U' && minstr[1] == 0)
521 rrd->ds_def[ds_idx].par[DS_min_val].u_val = DNAN;
523 rrd->ds_def[ds_idx].par[DS_min_val].u_val = atof(minstr);
525 if (maxstr[0] == 'U' && maxstr[1] == 0)
526 rrd->ds_def[ds_idx].par[DS_max_val].u_val = DNAN;
528 rrd->ds_def[ds_idx].par[DS_max_val].u_val = atof(maxstr);
530 if (!isnan(rrd->ds_def[ds_idx].par[DS_min_val].u_val) &&
531 !isnan(rrd->ds_def[ds_idx].par[DS_max_val].u_val) &&
532 rrd->ds_def[ds_idx].par[DS_min_val].u_val
533 >= rrd->ds_def[ds_idx].par[DS_max_val].u_val) {
534 rrd_set_error("min must be less than max in DS definition");
538 rrd_set_error("failed to parse data source %s", def);
542 /* Create the CF_DEVPREDICT, CF_DEVSEASONAL, CF_SEASONAL, and CF_FAILURES RRAs
543 * associated with a CF_HWPREDICT RRA. */
544 int create_hw_contingent_rras(
546 unsigned short period,
547 unsigned long hashed_name)
550 rra_def_t *current_rra;
552 /* save index to CF_HWPREDICT */
553 unsigned long hw_index = rrd->stat_head->rra_cnt;
555 /* advance the pointer */
556 (rrd->stat_head->rra_cnt)++;
557 /* allocate the memory for the 4 contingent RRAs */
558 old_size = sizeof(rra_def_t) * (rrd->stat_head->rra_cnt);
559 if ((rrd->rra_def = rrd_realloc(rrd->rra_def,
560 old_size + 4 * sizeof(rra_def_t))) ==
562 rrd_set_error("allocating rrd.rra_def");
566 memset(&(rrd->rra_def[rrd->stat_head->rra_cnt]), 0,
567 4 * sizeof(rra_def_t));
569 /* create the CF_SEASONAL RRA */
570 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
571 strcpy(current_rra->cf_nam, "SEASONAL");
572 current_rra->row_cnt = period;
573 current_rra->par[RRA_seasonal_smooth_idx].u_cnt = hashed_name % period;
574 current_rra->pdp_cnt = 1;
575 current_rra->par[RRA_seasonal_gamma].u_val =
576 rrd->rra_def[hw_index].par[RRA_hw_alpha].u_val;
577 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index;
578 rrd->rra_def[hw_index].par[RRA_dependent_rra_idx].u_cnt =
579 rrd->stat_head->rra_cnt;
581 /* create the CF_DEVSEASONAL RRA */
582 (rrd->stat_head->rra_cnt)++;
583 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
584 strcpy(current_rra->cf_nam, "DEVSEASONAL");
585 current_rra->row_cnt = period;
586 current_rra->par[RRA_seasonal_smooth_idx].u_cnt = hashed_name % period;
587 current_rra->pdp_cnt = 1;
588 current_rra->par[RRA_seasonal_gamma].u_val =
589 rrd->rra_def[hw_index].par[RRA_hw_alpha].u_val;
590 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index;
592 /* create the CF_DEVPREDICT RRA */
593 (rrd->stat_head->rra_cnt)++;
594 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
595 strcpy(current_rra->cf_nam, "DEVPREDICT");
596 current_rra->row_cnt = (rrd->rra_def[hw_index]).row_cnt;
597 current_rra->pdp_cnt = 1;
598 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index + 2; /* DEVSEASONAL */
600 /* create the CF_FAILURES RRA */
601 (rrd->stat_head->rra_cnt)++;
602 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
603 strcpy(current_rra->cf_nam, "FAILURES");
604 current_rra->row_cnt = period;
605 current_rra->pdp_cnt = 1;
606 current_rra->par[RRA_delta_pos].u_val = 2.0;
607 current_rra->par[RRA_delta_neg].u_val = 2.0;
608 current_rra->par[RRA_failure_threshold].u_cnt = 7;
609 current_rra->par[RRA_window_len].u_cnt = 9;
610 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index + 2; /* DEVSEASONAL */
614 /* create and empty rrd file according to the specs given */
617 const char *file_name,
622 rrd_value_t *unknown;
627 if ((rrd_file = fopen(file_name, "wb")) == NULL) {
628 rrd_set_error("creating '%s': %s", file_name, rrd_strerror(errno));
629 free(rrd->stat_head);
630 rrd->stat_head = NULL;
638 fwrite(rrd->stat_head, sizeof(stat_head_t), 1, rrd_file);
640 fwrite(rrd->ds_def, sizeof(ds_def_t), rrd->stat_head->ds_cnt, rrd_file);
643 sizeof(rra_def_t), rrd->stat_head->rra_cnt, rrd_file);
645 fwrite(rrd->live_head, sizeof(live_head_t), 1, rrd_file);
647 if ((rrd->pdp_prep = calloc(1, sizeof(pdp_prep_t))) == NULL) {
648 rrd_set_error("allocating pdp_prep");
654 strcpy(rrd->pdp_prep->last_ds, "UNKN");
656 rrd->pdp_prep->scratch[PDP_val].u_val = 0.0;
657 rrd->pdp_prep->scratch[PDP_unkn_sec_cnt].u_cnt =
658 rrd->live_head->last_up % rrd->stat_head->pdp_step;
660 for (i = 0; i < rrd->stat_head->ds_cnt; i++)
661 fwrite(rrd->pdp_prep, sizeof(pdp_prep_t), 1, rrd_file);
663 if ((rrd->cdp_prep = calloc(1, sizeof(cdp_prep_t))) == NULL) {
664 rrd_set_error("allocating cdp_prep");
671 for (i = 0; i < rrd->stat_head->rra_cnt; i++) {
672 switch (cf_conv(rrd->rra_def[i].cf_nam)) {
674 init_hwpredict_cdp(rrd->cdp_prep);
678 init_seasonal_cdp(rrd->cdp_prep);
681 /* initialize violation history to 0 */
682 for (ii = 0; ii < MAX_CDP_PAR_EN; ii++) {
683 /* We can zero everything out, by setting u_val to the
684 * NULL address. Each array entry in scratch is 8 bytes
685 * (a double), but u_cnt only accessed 4 bytes (long) */
686 rrd->cdp_prep->scratch[ii].u_val = 0.0;
690 /* can not be zero because we don't know anything ... */
691 rrd->cdp_prep->scratch[CDP_val].u_val = DNAN;
692 /* startup missing pdp count */
693 rrd->cdp_prep->scratch[CDP_unkn_pdp_cnt].u_cnt =
694 ((rrd->live_head->last_up -
695 rrd->pdp_prep->scratch[PDP_unkn_sec_cnt].u_cnt)
696 % (rrd->stat_head->pdp_step
697 * rrd->rra_def[i].pdp_cnt)) / rrd->stat_head->pdp_step;
701 for (ii = 0; ii < rrd->stat_head->ds_cnt; ii++) {
702 fwrite(rrd->cdp_prep, sizeof(cdp_prep_t), 1, rrd_file);
706 /* now, we must make sure that the rest of the rrd
707 struct is properly initialized */
709 if ((rrd->rra_ptr = calloc(1, sizeof(rra_ptr_t))) == NULL) {
710 rrd_set_error("allocating rra_ptr");
716 /* changed this initialization to be consistent with
717 * rrd_restore. With the old value (0), the first update
718 * would occur for cur_row = 1 because rrd_update increments
719 * the pointer a priori. */
720 for (i = 0; i < rrd->stat_head->rra_cnt; i++) {
721 rrd->rra_ptr->cur_row = rrd->rra_def[i].row_cnt - 1;
722 fwrite(rrd->rra_ptr, sizeof(rra_ptr_t), 1, rrd_file);
724 rrd_head_size = ftell(rrd_file);
726 /* write the empty data area */
727 if ((unknown = (rrd_value_t *) malloc(512 * sizeof(rrd_value_t))) == NULL) {
728 rrd_set_error("allocating unknown");
733 for (i = 0; i < 512; ++i)
737 for (i = 0; i < rrd->stat_head->rra_cnt; i++)
738 unkn_cnt += rrd->stat_head->ds_cnt * rrd->rra_def[i].row_cnt;
740 while (unkn_cnt > 0) {
741 fwrite(unknown, sizeof(rrd_value_t), min(unkn_cnt, 512), rrd_file);
746 /* lets see if we had an error */
747 if (ferror(rrd_file)) {
748 rrd_set_error("a file error occurred while creating '%s'", file_name);
753 #ifdef HAVE_POSIX_FADVISE
754 /* this file is not going to be read again any time
755 soon, so we drop everything except the header portion from
756 the buffer cache. for this to work, we have to fdsync the file
757 first though. This will not be all that fast, but 'good' data
758 like other rrdfiles headers will stay in cache. Now this only works if creating
759 a single rrd file is not too large, but I assume this should not be the case
760 in general. Otherwhise we would have to sync and release while writing all
763 fdatasync(fileno(rrd_file));
765 posix_fadvise(fileno(rrd_file), rrd_head_size, 0,
766 POSIX_FADV_DONTNEED)) {
767 rrd_set_error("setting POSIX_FADV_DONTNEED on '%s': %s", file_name,
768 rrd_strerror(errno));