1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_create.c creates new rrds
5 *****************************************************************************/
8 #include "rrd_rpncalc.h"
11 #include "rrd_is_thread_safe.h"
13 unsigned long FnvHash(
15 int create_hw_contingent_rras(
17 unsigned short period,
18 unsigned long hashed_name);
28 time_t last_up = time(NULL) - 10;
29 unsigned long pdp_step = 300;
30 struct rrd_time_value last_up_tv;
31 char *parsetime_error = NULL;
36 opterr = 0; /* initialize getopt */
39 static struct option long_options[] = {
40 {"start", required_argument, 0, 'b'},
41 {"step", required_argument, 0, 's'},
47 opt = getopt_long(argc, argv, "b:s:", long_options, &option_index);
54 if ((parsetime_error = parsetime(optarg, &last_up_tv))) {
55 rrd_set_error("start time: %s", parsetime_error);
58 if (last_up_tv.type == RELATIVE_TO_END_TIME ||
59 last_up_tv.type == RELATIVE_TO_START_TIME) {
60 rrd_set_error("specifying time relative to the 'start' "
61 "or 'end' makes no sense here");
65 last_up = mktime(&last_up_tv.tm) +last_up_tv.offset;
67 if (last_up < 3600 * 24 * 365 * 10) {
69 ("the first entry to the RRD should be after 1980");
75 long_tmp = atol(optarg);
77 rrd_set_error("step size should be no less than one second");
85 rrd_set_error("unknown option '%c'", optopt);
87 rrd_set_error("unknown option '%s'", argv[optind - 1]);
92 rrd_set_error("need name of an rrd file to create");
95 rc = rrd_create_r(argv[optind],
97 argc - optind - 1, (const char **) (argv + optind + 1));
104 const char *filename,
105 unsigned long pdp_step,
114 char dummychar1[2], dummychar2[2];
115 unsigned short token_idx, error_flag, period = 0;
116 unsigned long hashed_name;
121 if ((rrd.stat_head = calloc(1, sizeof(stat_head_t))) == NULL) {
122 rrd_set_error("allocating rrd.stat_head");
128 if ((rrd.live_head = calloc(1, sizeof(live_head_t))) == NULL) {
129 rrd_set_error("allocating rrd.live_head");
134 /* set some defaults */
135 strcpy(rrd.stat_head->cookie, RRD_COOKIE);
136 strcpy(rrd.stat_head->version, RRD_VERSION);
137 rrd.stat_head->float_cookie = FLOAT_COOKIE;
138 rrd.stat_head->ds_cnt = 0; /* this will be adjusted later */
139 rrd.stat_head->rra_cnt = 0; /* ditto */
140 rrd.stat_head->pdp_step = pdp_step; /* 5 minute default */
142 /* a default value */
146 rrd.live_head->last_up = last_up;
148 /* optind points to the first non-option command line arg,
149 * in this case, the file name. */
150 /* Compute the FNV hash value (used by SEASONAL and DEVSEASONAL
152 hashed_name = FnvHash(filename);
153 for (i = 0; i < argc; i++) {
156 if (strncmp(argv[i], "DS:", 3) == 0) {
157 size_t old_size = sizeof(ds_def_t) * (rrd.stat_head->ds_cnt);
159 if ((rrd.ds_def = rrd_realloc(rrd.ds_def,
160 old_size + sizeof(ds_def_t))) ==
162 rrd_set_error("allocating rrd.ds_def");
166 memset(&rrd.ds_def[rrd.stat_head->ds_cnt], 0, sizeof(ds_def_t));
167 /* extract the name and type */
168 switch (sscanf(&argv[i][3],
169 DS_NAM_FMT "%1[:]" DST_FMT "%1[:]%n",
170 rrd.ds_def[rrd.stat_head->ds_cnt].ds_nam,
172 rrd.ds_def[rrd.stat_head->ds_cnt].dst,
173 dummychar2, &offset)) {
176 rrd_set_error("Invalid DS name");
180 rrd_set_error("Invalid DS type");
182 case 4: /* (%n may or may not be counted) */
183 case 5: /* check for duplicate datasource names */
184 for (ii = 0; ii < rrd.stat_head->ds_cnt; ii++)
185 if (strcmp(rrd.ds_def[rrd.stat_head->ds_cnt].ds_nam,
186 rrd.ds_def[ii].ds_nam) == 0)
187 rrd_set_error("Duplicate DS name: %s",
188 rrd.ds_def[ii].ds_nam);
189 /* DS_type may be valid or not. Checked later */
192 rrd_set_error("invalid DS format");
194 if (rrd_test_error()) {
199 /* parse the remainder of the arguments */
200 switch (dst_conv(rrd.ds_def[rrd.stat_head->ds_cnt].dst)) {
205 parseGENERIC_DS(&argv[i][offset + 3], &rrd,
206 rrd.stat_head->ds_cnt);
209 parseCDEF_DS(&argv[i][offset + 3], &rrd,
210 rrd.stat_head->ds_cnt);
213 rrd_set_error("invalid DS type specified");
217 if (rrd_test_error()) {
221 rrd.stat_head->ds_cnt++;
222 } else if (strncmp(argv[i], "RRA:", 4) == 0) {
225 size_t old_size = sizeof(rra_def_t) * (rrd.stat_head->rra_cnt);
227 if ((rrd.rra_def = rrd_realloc(rrd.rra_def,
228 old_size + sizeof(rra_def_t))) ==
230 rrd_set_error("allocating rrd.rra_def");
234 memset(&rrd.rra_def[rrd.stat_head->rra_cnt], 0,
237 argvcopy = strdup(argv[i]);
238 token = strtok_r(&argvcopy[4], ":", &tokptr);
239 token_idx = error_flag = 0;
240 while (token != NULL) {
243 if (sscanf(token, CF_NAM_FMT,
244 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam) !=
246 rrd_set_error("Failed to parse CF name");
248 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
251 /* initialize some parameters */
252 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_alpha].
254 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_beta].
256 rrd.rra_def[rrd.stat_head->rra_cnt].
257 par[RRA_dependent_rra_idx].u_cnt =
258 rrd.stat_head->rra_cnt;
262 /* initialize some parameters */
263 rrd.rra_def[rrd.stat_head->rra_cnt].
264 par[RRA_seasonal_gamma].u_val = 0.1;
267 rrd.rra_def[rrd.stat_head->rra_cnt].
268 par[RRA_dependent_rra_idx].u_cnt = -1;
271 rrd.rra_def[rrd.stat_head->rra_cnt].
272 par[RRA_delta_pos].u_val = 2.0;
273 rrd.rra_def[rrd.stat_head->rra_cnt].
274 par[RRA_delta_neg].u_val = 2.0;
275 rrd.rra_def[rrd.stat_head->rra_cnt].
276 par[RRA_window_len].u_cnt = 3;
277 rrd.rra_def[rrd.stat_head->rra_cnt].
278 par[RRA_failure_threshold].u_cnt = 2;
279 rrd.rra_def[rrd.stat_head->rra_cnt].
280 par[RRA_dependent_rra_idx].u_cnt = -1;
282 /* invalid consolidation function */
285 ("Unrecognized consolidation function %s",
286 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam);
290 /* default: 1 pdp per cdp */
291 rrd.rra_def[rrd.stat_head->rra_cnt].pdp_cnt = 1;
295 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
302 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt =
306 rrd.rra_def[rrd.stat_head->rra_cnt].
307 par[RRA_cdp_xff_val].u_val = atof(token);
308 if (rrd.rra_def[rrd.stat_head->rra_cnt].
309 par[RRA_cdp_xff_val].u_val < 0.0
310 || rrd.rra_def[rrd.stat_head->rra_cnt].
311 par[RRA_cdp_xff_val].u_val >= 1.0)
313 ("Invalid xff: must be between 0 and 1");
319 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
322 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_alpha].
324 if (atof(token) <= 0.0 || atof(token) >= 1.0)
326 ("Invalid alpha: must be between 0 and 1");
330 rrd.rra_def[rrd.stat_head->rra_cnt].
331 par[RRA_seasonal_gamma].u_val = atof(token);
332 if (atof(token) <= 0.0 || atof(token) >= 1.0)
334 ("Invalid gamma: must be between 0 and 1");
335 rrd.rra_def[rrd.stat_head->rra_cnt].
336 par[RRA_seasonal_smooth_idx].u_cnt =
338 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt;
341 /* specifies the # of violations that constitutes the failure threshold */
342 rrd.rra_def[rrd.stat_head->rra_cnt].
343 par[RRA_failure_threshold].u_cnt = atoi(token);
345 || atoi(token) > MAX_FAILURES_WINDOW_LEN)
347 ("Failure threshold is out of range %d, %d",
348 1, MAX_FAILURES_WINDOW_LEN);
351 /* specifies the index (1-based) of CF_DEVSEASONAL array
352 * associated with this CF_DEVPREDICT array. */
353 rrd.rra_def[rrd.stat_head->rra_cnt].
354 par[RRA_dependent_rra_idx].u_cnt =
358 rrd.rra_def[rrd.stat_head->rra_cnt].pdp_cnt =
365 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
368 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_beta].
370 if (atof(token) < 0.0 || atof(token) > 1.0)
372 ("Invalid beta: must be between 0 and 1");
376 /* specifies the index (1-based) of CF_HWPREDICT array
377 * associated with this CF_DEVSEASONAL or CF_SEASONAL array.
379 rrd.rra_def[rrd.stat_head->rra_cnt].
380 par[RRA_dependent_rra_idx].u_cnt =
384 /* specifies the window length */
385 rrd.rra_def[rrd.stat_head->rra_cnt].
386 par[RRA_window_len].u_cnt = atoi(token);
388 || atoi(token) > MAX_FAILURES_WINDOW_LEN)
390 ("Window length is out of range %d, %d", 1,
391 MAX_FAILURES_WINDOW_LEN);
392 /* verify that window length exceeds the failure threshold */
393 if (rrd.rra_def[rrd.stat_head->rra_cnt].
394 par[RRA_window_len].u_cnt <
395 rrd.rra_def[rrd.stat_head->rra_cnt].
396 par[RRA_failure_threshold].u_cnt)
398 ("Window length is shorter than the failure threshold");
401 /* shouldn't be any more arguments */
403 ("Unexpected extra argument for consolidation function DEVPREDICT");
406 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt =
413 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
415 /* specifies the index (1-based) of CF_DEVSEASONAL array
416 * associated with this CF_DEVFAILURES array. */
417 rrd.rra_def[rrd.stat_head->rra_cnt].
418 par[RRA_dependent_rra_idx].u_cnt =
423 /* length of the associated CF_SEASONAL and CF_DEVSEASONAL arrays. */
424 period = atoi(token);
426 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt)
428 ("Length of seasonal cycle exceeds length of HW prediction array");
431 /* shouldn't be any more arguments */
433 ("Unexpected extra argument for consolidation function %s",
434 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam);
439 /* If we are here, this must be a CF_HWPREDICT RRA.
440 * Specifies the index (1-based) of CF_SEASONAL array
441 * associated with this CF_HWPREDICT array. If this argument
442 * is missing, then the CF_SEASONAL, CF_DEVSEASONAL, CF_DEVPREDICT,
444 * arrays are created automatically. */
445 rrd.rra_def[rrd.stat_head->rra_cnt].
446 par[RRA_dependent_rra_idx].u_cnt = atoi(token) - 1;
449 /* should never get here */
450 rrd_set_error("Unknown error");
453 if (rrd_test_error()) {
454 /* all errors are unrecoverable */
459 token = strtok_r(NULL, ":", &tokptr);
465 "Creating RRA CF: %s, dep idx %lu, current idx %lu\n",
466 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam,
467 rrd.rra_def[rrd.stat_head->rra_cnt].
468 par[RRA_dependent_rra_idx].u_cnt, rrd.stat_head->rra_cnt);
470 /* should we create CF_SEASONAL, CF_DEVSEASONAL, and CF_DEVPREDICT? */
471 if ((cf_conv(rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam) ==
473 || cf_conv(rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam) ==
475 && rrd.rra_def[rrd.stat_head->rra_cnt].
476 par[RRA_dependent_rra_idx].u_cnt == rrd.stat_head->rra_cnt) {
478 fprintf(stderr, "Creating HW contingent RRAs\n");
480 if (create_hw_contingent_rras(&rrd, period, hashed_name) ==
482 rrd_set_error("creating contingent RRA");
487 rrd.stat_head->rra_cnt++;
489 rrd_set_error("can't parse argument '%s'", argv[i]);
496 if (rrd.stat_head->rra_cnt < 1) {
497 rrd_set_error("you must define at least one Round Robin Archive");
502 if (rrd.stat_head->ds_cnt < 1) {
503 rrd_set_error("you must define at least one Data Source");
507 return rrd_create_fn(filename, &rrd);
510 void parseGENERIC_DS(
515 char minstr[DS_NAM_SIZE], maxstr[DS_NAM_SIZE];
520 temp = sscanf(def,"%lu:%18[^:]:%18[^:]",
521 &(rrd -> ds_def[ds_idx].par[DS_mrhb_cnt].u_cnt),
524 if (sscanf(def, "%lu:%18[^:]:%18[^:]",
525 &(rrd->ds_def[ds_idx].par[DS_mrhb_cnt].u_cnt),
526 minstr, maxstr) == 3) {
527 if (minstr[0] == 'U' && minstr[1] == 0)
528 rrd->ds_def[ds_idx].par[DS_min_val].u_val = DNAN;
530 rrd->ds_def[ds_idx].par[DS_min_val].u_val = atof(minstr);
532 if (maxstr[0] == 'U' && maxstr[1] == 0)
533 rrd->ds_def[ds_idx].par[DS_max_val].u_val = DNAN;
535 rrd->ds_def[ds_idx].par[DS_max_val].u_val = atof(maxstr);
537 if (!isnan(rrd->ds_def[ds_idx].par[DS_min_val].u_val) &&
538 !isnan(rrd->ds_def[ds_idx].par[DS_max_val].u_val) &&
539 rrd->ds_def[ds_idx].par[DS_min_val].u_val
540 >= rrd->ds_def[ds_idx].par[DS_max_val].u_val) {
541 rrd_set_error("min must be less than max in DS definition");
545 rrd_set_error("failed to parse data source %s", def);
549 /* Create the CF_DEVPREDICT, CF_DEVSEASONAL, CF_SEASONAL, and CF_FAILURES RRAs
550 * associated with a CF_HWPREDICT RRA. */
551 int create_hw_contingent_rras(
553 unsigned short period,
554 unsigned long hashed_name)
557 rra_def_t *current_rra;
559 /* save index to CF_HWPREDICT */
560 unsigned long hw_index = rrd->stat_head->rra_cnt;
562 /* advance the pointer */
563 (rrd->stat_head->rra_cnt)++;
564 /* allocate the memory for the 4 contingent RRAs */
565 old_size = sizeof(rra_def_t) * (rrd->stat_head->rra_cnt);
566 if ((rrd->rra_def = rrd_realloc(rrd->rra_def,
567 old_size + 4 * sizeof(rra_def_t))) ==
569 rrd_set_error("allocating rrd.rra_def");
573 memset(&(rrd->rra_def[rrd->stat_head->rra_cnt]), 0,
574 4 * sizeof(rra_def_t));
576 /* create the CF_SEASONAL RRA */
577 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
578 strcpy(current_rra->cf_nam, "SEASONAL");
579 current_rra->row_cnt = period;
580 current_rra->par[RRA_seasonal_smooth_idx].u_cnt = hashed_name % period;
581 current_rra->pdp_cnt = 1;
582 current_rra->par[RRA_seasonal_gamma].u_val =
583 rrd->rra_def[hw_index].par[RRA_hw_alpha].u_val;
584 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index;
585 rrd->rra_def[hw_index].par[RRA_dependent_rra_idx].u_cnt =
586 rrd->stat_head->rra_cnt;
588 /* create the CF_DEVSEASONAL RRA */
589 (rrd->stat_head->rra_cnt)++;
590 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
591 strcpy(current_rra->cf_nam, "DEVSEASONAL");
592 current_rra->row_cnt = period;
593 current_rra->par[RRA_seasonal_smooth_idx].u_cnt = hashed_name % period;
594 current_rra->pdp_cnt = 1;
595 current_rra->par[RRA_seasonal_gamma].u_val =
596 rrd->rra_def[hw_index].par[RRA_hw_alpha].u_val;
597 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index;
599 /* create the CF_DEVPREDICT RRA */
600 (rrd->stat_head->rra_cnt)++;
601 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
602 strcpy(current_rra->cf_nam, "DEVPREDICT");
603 current_rra->row_cnt = (rrd->rra_def[hw_index]).row_cnt;
604 current_rra->pdp_cnt = 1;
605 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index + 2; /* DEVSEASONAL */
607 /* create the CF_FAILURES RRA */
608 (rrd->stat_head->rra_cnt)++;
609 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
610 strcpy(current_rra->cf_nam, "FAILURES");
611 current_rra->row_cnt = period;
612 current_rra->pdp_cnt = 1;
613 current_rra->par[RRA_delta_pos].u_val = 2.0;
614 current_rra->par[RRA_delta_neg].u_val = 2.0;
615 current_rra->par[RRA_failure_threshold].u_cnt = 7;
616 current_rra->par[RRA_window_len].u_cnt = 9;
617 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index + 2; /* DEVSEASONAL */
621 /* create and empty rrd file according to the specs given */
624 const char *file_name,
629 rrd_value_t *unknown;
634 if ((rrd_file = fopen(file_name, "wb")) == NULL) {
635 rrd_set_error("creating '%s': %s", file_name, rrd_strerror(errno));
636 free(rrd->stat_head);
637 rrd->stat_head = NULL;
645 fwrite(rrd->stat_head, sizeof(stat_head_t), 1, rrd_file);
647 fwrite(rrd->ds_def, sizeof(ds_def_t), rrd->stat_head->ds_cnt, rrd_file);
650 sizeof(rra_def_t), rrd->stat_head->rra_cnt, rrd_file);
652 fwrite(rrd->live_head, sizeof(live_head_t), 1, rrd_file);
654 if ((rrd->pdp_prep = calloc(1, sizeof(pdp_prep_t))) == NULL) {
655 rrd_set_error("allocating pdp_prep");
661 strcpy(rrd->pdp_prep->last_ds, "UNKN");
663 rrd->pdp_prep->scratch[PDP_val].u_val = 0.0;
664 rrd->pdp_prep->scratch[PDP_unkn_sec_cnt].u_cnt =
665 rrd->live_head->last_up % rrd->stat_head->pdp_step;
667 for (i = 0; i < rrd->stat_head->ds_cnt; i++)
668 fwrite(rrd->pdp_prep, sizeof(pdp_prep_t), 1, rrd_file);
670 if ((rrd->cdp_prep = calloc(1, sizeof(cdp_prep_t))) == NULL) {
671 rrd_set_error("allocating cdp_prep");
678 for (i = 0; i < rrd->stat_head->rra_cnt; i++) {
679 switch (cf_conv(rrd->rra_def[i].cf_nam)) {
682 init_hwpredict_cdp(rrd->cdp_prep);
686 init_seasonal_cdp(rrd->cdp_prep);
689 /* initialize violation history to 0 */
690 for (ii = 0; ii < MAX_CDP_PAR_EN; ii++) {
691 /* We can zero everything out, by setting u_val to the
692 * NULL address. Each array entry in scratch is 8 bytes
693 * (a double), but u_cnt only accessed 4 bytes (long) */
694 rrd->cdp_prep->scratch[ii].u_val = 0.0;
698 /* can not be zero because we don't know anything ... */
699 rrd->cdp_prep->scratch[CDP_val].u_val = DNAN;
700 /* startup missing pdp count */
701 rrd->cdp_prep->scratch[CDP_unkn_pdp_cnt].u_cnt =
702 ((rrd->live_head->last_up -
703 rrd->pdp_prep->scratch[PDP_unkn_sec_cnt].u_cnt)
704 % (rrd->stat_head->pdp_step
705 * rrd->rra_def[i].pdp_cnt)) / rrd->stat_head->pdp_step;
709 for (ii = 0; ii < rrd->stat_head->ds_cnt; ii++) {
710 fwrite(rrd->cdp_prep, sizeof(cdp_prep_t), 1, rrd_file);
714 /* now, we must make sure that the rest of the rrd
715 struct is properly initialized */
717 if ((rrd->rra_ptr = calloc(1, sizeof(rra_ptr_t))) == NULL) {
718 rrd_set_error("allocating rra_ptr");
724 /* changed this initialization to be consistent with
725 * rrd_restore. With the old value (0), the first update
726 * would occur for cur_row = 1 because rrd_update increments
727 * the pointer a priori. */
728 for (i = 0; i < rrd->stat_head->rra_cnt; i++) {
729 rrd->rra_ptr->cur_row = rrd->rra_def[i].row_cnt - 1;
730 fwrite(rrd->rra_ptr, sizeof(rra_ptr_t), 1, rrd_file);
732 rrd_head_size = ftell(rrd_file);
734 /* write the empty data area */
735 if ((unknown = (rrd_value_t *) malloc(512 * sizeof(rrd_value_t))) == NULL) {
736 rrd_set_error("allocating unknown");
741 for (i = 0; i < 512; ++i)
745 for (i = 0; i < rrd->stat_head->rra_cnt; i++)
746 unkn_cnt += rrd->stat_head->ds_cnt * rrd->rra_def[i].row_cnt;
748 while (unkn_cnt > 0) {
749 fwrite(unknown, sizeof(rrd_value_t), min(unkn_cnt, 512), rrd_file);
754 /* lets see if we had an error */
755 if (ferror(rrd_file)) {
756 rrd_set_error("a file error occurred while creating '%s'", file_name);
761 #ifdef HAVE_POSIX_FADVISE
762 /* this file is not going to be read again any time
763 soon, so we drop everything except the header portion from
764 the buffer cache. for this to work, we have to fdsync the file
765 first though. This will not be all that fast, but 'good' data
766 like other rrdfiles headers will stay in cache. Now this only works if creating
767 a single rrd file is not too large, but I assume this should not be the case
768 in general. Otherwhise we would have to sync and release while writing all
771 fdatasync(fileno(rrd_file));
773 posix_fadvise(fileno(rrd_file), rrd_head_size, 0,
774 POSIX_FADV_DONTNEED)) {
775 rrd_set_error("setting POSIX_FADV_DONTNEED on '%s': %s", file_name,
776 rrd_strerror(errno));