1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_open.c Open an RRD File
5 *****************************************************************************
7 *****************************************************************************/
13 /* DEBUG 2 prints information obtained via mincore(2) */
15 /* do not calculate exact madvise hints but assume 1 page for headers and
16 * set DONTNEED for the rest, which is assumed to be data */
18 /* Avoid calling madvise on areas that were already hinted. May be benefical if
19 * your syscalls are very slow */
20 #define CHECK_MADVISE_OVERLAPS 1
23 /* the cast to void* is there to avoid this warning seen on ia64 with certain
24 versions of gcc: 'cast increases required alignment of target type'
26 #define __rrd_read(dst, dst_t, cnt) \
27 (dst) = (dst_t*)(void*) (data + offset); \
28 offset += sizeof(dst_t) * (cnt)
30 #define __rrd_read(dst, dst_t, cnt) \
31 if ((dst = malloc(sizeof(dst_t)*(cnt))) == NULL) { \
32 rrd_set_error(#dst " malloc"); \
33 goto out_nullify_head; \
35 offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt))
38 /* next page-aligned (i.e. page-align up) */
40 #define PAGE_ALIGN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
42 /* previous page-aligned (i.e. page-align down) */
43 #ifndef PAGE_ALIGN_DOWN
44 #define PAGE_ALIGN_DOWN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
48 /* vector of last madvise hint */
49 typedef struct _madvise_vec_t {
53 _madvise_vec_t _madv_vec = { NULL, 0 };
56 #if defined CHECK_MADVISE_OVERLAPS
57 #define _madvise(_start, _off, _hint) \
58 if ((_start) != _madv_vec.start && (ssize_t)(_off) != _madv_vec.length) { \
59 _madv_vec.start = (_start) ; _madv_vec.length = (_off); \
60 madvise((_start), (_off), (_hint)); \
63 #define _madvise(_start, _off, _hint) \
64 madvise((_start), (_off), (_hint))
67 /* Open a database file, return its header and an open filehandle,
68 * positioned to the first cdp in the first rra.
69 * In the error path of rrd_open, only rrd_free(&rrd) has to be called
70 * before returning an error. Do not call rrd_close upon failure of rrd_open.
74 const char *const file_name,
79 mode_t mode = S_IRUSR;
83 ssize_t _page_size = sysconf(_SC_PAGESIZE);
84 int mm_prot = PROT_READ, mm_flags = 0;
89 rrd_file_t *rrd_file = NULL;
90 off_t newfile_size = 0;
92 if (rdwr & RRD_CREAT) {
93 /* yes bad inline signaling alert, we are using the
94 floatcookie to pass the size in ... only used in resize */
95 newfile_size = (off_t) rrd->stat_head->float_cookie;
99 rrd_file = malloc(sizeof(rrd_file_t));
100 if (rrd_file == NULL) {
101 rrd_set_error("allocating rrd_file descriptor for '%s'", file_name);
104 memset(rrd_file, 0, sizeof(rrd_file_t));
107 if ((rdwr & (RRD_READONLY | RRD_READWRITE)) ==
108 (RRD_READONLY | RRD_READWRITE)) {
109 /* Both READONLY and READWRITE were given, which is invalid. */
110 rrd_set_error("in read/write request mask");
114 if (rdwr & RRD_READONLY) {
117 mm_flags = MAP_PRIVATE;
118 # ifdef MAP_NORESERVE
119 mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */
123 if (rdwr & RRD_READWRITE) {
127 mm_flags = MAP_SHARED;
128 mm_prot |= PROT_WRITE;
131 if (rdwr & RRD_CREAT) {
132 flags |= (O_CREAT | O_TRUNC);
135 if (rdwr & RRD_READAHEAD) {
137 mm_flags |= MAP_POPULATE; /* populate ptes and data */
139 #if defined MAP_NONBLOCK
140 mm_flags |= MAP_NONBLOCK; /* just populate ptes */
151 if ((rrd_file->fd = open(file_name, flags, mode)) < 0) {
152 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
156 /* Better try to avoid seeks as much as possible. stat may be heavy but
157 * many concurrent seeks are even worse. */
158 if (newfile_size == 0 && ((fstat(rrd_file->fd, &statb)) < 0)) {
159 rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
162 if (newfile_size == 0) {
163 rrd_file->file_len = statb.st_size;
165 rrd_file->file_len = newfile_size;
166 lseek(rrd_file->fd, newfile_size - 1, SEEK_SET);
167 write(rrd_file->fd, "\0", 1); /* poke */
168 lseek(rrd_file->fd, 0, SEEK_SET);
170 #ifdef HAVE_POSIX_FADVISE
171 /* In general we need no read-ahead when dealing with rrd_files.
172 When we stop reading, it is highly unlikely that we start up again.
173 In this manner we actually save time and diskaccess (and buffer cache).
174 Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
175 if (0 != posix_fadvise(rrd_file->fd, 0, 0, POSIX_FADV_RANDOM)) {
176 rrd_set_error("setting POSIX_FADV_RANDOM on '%s': %s", file_name,
177 rrd_strerror(errno));
183 if (rdwr & RRD_READWRITE)
185 if (setvbuf((rrd_file->fd),NULL,_IONBF,2)) {
186 rrd_set_error("failed to disable the stream buffer\n");
192 data = mmap(0, rrd_file->file_len, mm_prot, mm_flags,
193 rrd_file->fd, offset);
195 /* lets see if the first read worked */
196 if (data == MAP_FAILED) {
197 rrd_set_error("mmaping file '%s': %s", file_name,
198 rrd_strerror(errno));
201 rrd_file->file_start = data;
202 if (rdwr & RRD_CREAT) {
203 memset(data, DNAN, newfile_size - 1);
207 if (rdwr & RRD_CREAT)
210 if (rdwr & RRD_COPY) {
211 /* We will read everything in a moment (copying) */
212 _madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
215 /* We do not need to read anything in for the moment */
216 _madvise(data, rrd_file->file_len, MADV_DONTNEED);
217 /* the stat_head will be needed soonish, so hint accordingly */
218 _madvise(data + PAGE_ALIGN_DOWN(offset),
219 PAGE_ALIGN(sizeof(stat_head_t)),
220 MADV_WILLNEED | MADV_RANDOM);
223 /* alternatively: keep 1 page worth of data, likely headers,
224 * don't need the rest. */
225 _madvise(data, _page_size, MADV_WILLNEED | MADV_SEQUENTIAL);
226 _madvise(data + _page_size, (rrd_file->file_len >= _page_size)
227 ? rrd_file->file_len - _page_size : 0, MADV_DONTNEED);
232 __rrd_read(rrd->stat_head, stat_head_t,
235 /* lets do some test if we are on track ... */
236 if (memcmp(rrd->stat_head->cookie, RRD_COOKIE, sizeof(RRD_COOKIE)) != 0) {
237 rrd_set_error("'%s' is not an RRD file", file_name);
238 goto out_nullify_head;
241 if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
242 rrd_set_error("This RRD was created on another architecture");
243 goto out_nullify_head;
246 version = atoi(rrd->stat_head->version);
248 if (version > atoi(RRD_VERSION)) {
249 rrd_set_error("can't handle RRD file version %s",
250 rrd->stat_head->version);
251 goto out_nullify_head;
253 #if defined USE_MADVISE && !defined ONE_PAGE
254 /* the ds_def will be needed soonish, so hint accordingly */
255 _madvise(data + PAGE_ALIGN_DOWN(offset),
256 PAGE_ALIGN(sizeof(ds_def_t) * rrd->stat_head->ds_cnt),
259 __rrd_read(rrd->ds_def, ds_def_t,
260 rrd->stat_head->ds_cnt);
262 #if defined USE_MADVISE && !defined ONE_PAGE
263 /* the rra_def will be needed soonish, so hint accordingly */
264 _madvise(data + PAGE_ALIGN_DOWN(offset),
265 PAGE_ALIGN(sizeof(rra_def_t) * rrd->stat_head->rra_cnt),
268 __rrd_read(rrd->rra_def, rra_def_t,
269 rrd->stat_head->rra_cnt);
271 /* handle different format for the live_head */
273 rrd->live_head = (live_head_t *) malloc(sizeof(live_head_t));
274 if (rrd->live_head == NULL) {
275 rrd_set_error("live_head_t malloc");
279 memmove(&rrd->live_head->last_up, data + offset, sizeof(long));
280 offset += sizeof(long);
282 offset += read(rrd_file->fd, &rrd->live_head->last_up, sizeof(long));
284 rrd->live_head->last_up_usec = 0;
286 #if defined USE_MADVISE && !defined ONE_PAGE
287 /* the live_head will be needed soonish, so hint accordingly */
288 _madvise(data + PAGE_ALIGN_DOWN(offset),
289 PAGE_ALIGN(sizeof(live_head_t)), MADV_WILLNEED);
291 __rrd_read(rrd->live_head, live_head_t,
294 //XXX: This doesn't look like it needs madvise
295 __rrd_read(rrd->pdp_prep, pdp_prep_t,
296 rrd->stat_head->ds_cnt);
298 //XXX: This could benefit from madvise()ing
299 __rrd_read(rrd->cdp_prep, cdp_prep_t,
300 rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt);
302 //XXX: This could benefit from madvise()ing
303 __rrd_read(rrd->rra_ptr, rra_ptr_t,
304 rrd->stat_head->rra_cnt);
306 rrd_file->header_len = offset;
307 rrd_file->pos = offset;
311 rrd->stat_head = NULL;
320 /* Close a reference to an rrd_file. */
323 rrd_file_t *rrd_file)
327 #if defined HAVE_MMAP || defined DEBUG
328 ssize_t _page_size = sysconf(_SC_PAGESIZE);
330 #if defined DEBUG && DEBUG > 1
331 /* pretty print blocks in core */
335 off = rrd_file->file_len +
336 ((rrd_file->file_len + _page_size - 1) / _page_size);
340 if (mincore(rrd_file->file_start, rrd_file->file_len, vec) == 0) {
342 unsigned is_in = 0, was_in = 0;
344 for (off = 0, prev = 0; off < rrd_file->file_len; ++off) {
345 is_in = vec[off] & 1; /* if lsb set then is core resident */
348 if (was_in != is_in) {
349 fprintf(stderr, "%sin core: %p len %ld\n",
350 was_in ? "" : "not ", vec + prev, off - prev);
356 "%sin core: %p len %ld\n",
357 was_in ? "" : "not ", vec + prev, off - prev);
359 fprintf(stderr, "mincore: %s", rrd_strerror(errno));
365 /* Keep headers around, round up to next page boundary. */
367 PAGE_ALIGN(rrd_file->header_len % _page_size + rrd_file->header_len);
368 if (rrd_file->file_len > ret)
369 _madvise(rrd_file->file_start + ret,
370 rrd_file->file_len - ret, MADV_DONTNEED);
372 /* ignoring errors from RRDs that are smaller then the file_len+rounding */
373 _madvise(rrd_file->file_start + PAGE_ALIGN_DOWN(rrd_file->header_len),
374 rrd_file->file_len - PAGE_ALIGN(rrd_file->header_len),
379 ret = munmap(rrd_file->file_start, rrd_file->file_len);
381 rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno));
383 ret = close(rrd_file->fd);
385 rrd_set_error("closing file: %s", rrd_strerror(errno));
392 /* Set position of rrd_file. */
395 rrd_file_t *rrd_file,
402 if (whence == SEEK_SET)
404 else if (whence == SEEK_CUR)
405 rrd_file->pos += off;
406 else if (whence == SEEK_END)
407 rrd_file->pos = rrd_file->file_len + off;
409 ret = lseek(rrd_file->fd, off, whence);
411 rrd_set_error("lseek: %s", rrd_strerror(errno));
414 //XXX: mimic fseek, which returns 0 upon success
415 return ret == -1; //XXX: or just ret to mimic lseek
419 /* Get current position in rrd_file. */
421 inline off_t rrd_tell(
422 rrd_file_t *rrd_file)
424 return rrd_file->pos;
428 /* read count bytes into buffer buf, starting at rrd_file->pos.
429 * Returns the number of bytes read or <0 on error. */
431 inline ssize_t rrd_read(
432 rrd_file_t *rrd_file,
438 ssize_t _surplus = rrd_file->pos + _cnt - rrd_file->file_len;
440 if (_surplus > 0) { /* short read */
445 buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, _cnt);
447 rrd_file->pos += _cnt; /* mimmic read() semantics */
452 ret = read(rrd_file->fd, buf, count);
454 rrd_file->pos += ret; /* mimmic read() semantics */
460 /* write count bytes from buffer buf to the current position
461 * rrd_file->pos of rrd_file->fd.
462 * Returns the number of bytes written. */
464 inline ssize_t rrd_write(
465 rrd_file_t *rrd_file,
470 memcpy(rrd_file->file_start + rrd_file->pos, buf, count);
471 rrd_file->pos += count;
472 return count; /* mimmic write() semantics */
474 ssize_t _sz = write(rrd_file->fd, buf, count);
477 rrd_file->pos += _sz;
483 /* flush all data pending to be written to FD. */
485 inline void rrd_flush(
486 rrd_file_t *rrd_file)
488 if (fdatasync(rrd_file->fd) != 0) {
489 rrd_set_error("flushing fd %d: %s", rrd_file->fd,
490 rrd_strerror(errno));
495 /* Initialize RRD header. */
500 rrd->stat_head = NULL;
503 rrd->live_head = NULL;
505 rrd->pdp_prep = NULL;
506 rrd->cdp_prep = NULL;
507 rrd->rrd_value = NULL;
511 /* free RRD header data. */
514 inline void rrd_free(
522 free(rrd->live_head);
523 free(rrd->stat_head);
529 free(rrd->rrd_value);
534 /* routine used by external libraries to free memory allocated by
544 /* XXX: FIXME: missing documentation. */
545 /*XXX: FIXME should be renamed to rrd_readfile or _rrd_readfile */
547 int /*_rrd_*/ readfile(
548 const char *file_name,
552 long writecnt = 0, totalcnt = MEMBLK;
557 if ((strcmp("-", file_name) == 0)) {
560 if ((input = fopen(file_name, "rb")) == NULL) {
561 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
569 } while (c != '\n' && !feof(input));
571 if (strcmp("-", file_name)) {
572 fseek(input, 0, SEEK_END);
573 /* have extra space for detecting EOF without realloc */
574 totalcnt = (ftell(input) + 1) / sizeof(char) - offset;
575 if (totalcnt < MEMBLK)
576 totalcnt = MEMBLK; /* sanitize */
577 fseek(input, offset * sizeof(char), SEEK_SET);
579 if (((*buffer) = (char *) malloc((totalcnt + 4) * sizeof(char))) == NULL) {
580 perror("Allocate Buffer:");
585 fread((*buffer) + writecnt, 1,
586 (totalcnt - writecnt) * sizeof(char), input);
587 if (writecnt >= totalcnt) {
590 rrd_realloc((*buffer),
591 (totalcnt + 4) * sizeof(char))) == NULL) {
592 perror("Realloc Buffer:");
596 } while (!feof(input));
597 (*buffer)[writecnt] = '\0';
598 if (strcmp("-", file_name) != 0) {