/*****************************************************************************
- * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
+ * RRDtool 1.2.99907080300 Copyright by Tobi Oetiker, 1997-2007
*****************************************************************************
* rrd_open.c Open an RRD File
*****************************************************************************
* $Id$
- * $Log$
- * Revision 1.10 2004/05/26 22:11:12 oetiker
- * reduce compiler warnings. Many small fixes. -- Mike Slifcak <slif@bellsouth.net>
- *
- * Revision 1.9 2003/04/29 21:56:49 oetiker
- * readline in rrd_open.c reads the file in 8 KB blocks, and calls realloc for
- * each block. realloc is very slow in Mac OS X for huge blocks, e.g. when
- * restoring databases from huge xml files. This patch finds the size of the
- * file, and starts out with malloc'ing the full size.
- * -- Peter Speck <speck@ruc.dk>
- *
- * Revision 1.8 2003/04/11 19:43:44 oetiker
- * New special value COUNT which allows calculations based on the position of a
- * value within a data set. Bug fix in rrd_rpncalc.c. PREV returned erroneus
- * value for the second value. Bug fix in rrd_restore.c. Bug causing seek error
- * when accesing an RRD restored from an xml that holds an RRD version <3.
- * -- Ruben Justo <ruben@ainek.com>
- *
- * Revision 1.7 2003/03/31 21:22:12 oetiker
- * enables RRDtool updates with microsecond or in case of windows millisecond
- * precision. This is needed to reduce time measurement error when archive step
- * is small. (<30s) -- Sasha Mikheev <sasha@avalon-net.co.il>
- *
- * Revision 1.6 2003/02/13 07:05:27 oetiker
- * Find attached the patch I promised to send to you. Please note that there
- * are three new source files (src/rrd_is_thread_safe.h, src/rrd_thread_safe.c
- * and src/rrd_not_thread_safe.c) and the introduction of librrd_th. This
- * library is identical to librrd, but it contains support code for per-thread
- * global variables currently used for error information only. This is similar
- * to how errno per-thread variables are implemented. librrd_th must be linked
- * alongside of libpthred
- *
- * There is also a new file "THREADS", holding some documentation.
- *
- * -- Peter Stamfest <peter@stamfest.at>
- *
- * Revision 1.5 2002/06/20 00:21:03 jake
- * More Win32 build changes; thanks to Kerry Calvert.
- *
- * Revision 1.4 2002/02/01 20:34:49 oetiker
- * fixed version number and date/time
- *
- * Revision 1.3 2001/03/04 13:01:55 oetiker
- * Aberrant Behavior Detection support. A brief overview added to rrdtool.pod.
- * Major updates to rrd_update.c, rrd_create.c. Minor update to other core files.
- * This is backwards compatible! But new files using the Aberrant stuff are not readable
- * by old rrdtool versions. See http://cricket.sourceforge.net/aberrant/rrd_hw.htm
- * -- Jake Brutlag <jakeb@corp.webtv.net>
- *
- * Revision 1.2 2001/03/04 10:29:20 oetiker
- * fixed filedescriptor leak
- * -- Mike Franusich <mike@franusich.com>
- *
- * Revision 1.1.1.1 2001/02/25 22:25:05 oetiker
- * checkin
- *
*****************************************************************************/
#include "rrd_tool.h"
#define MEMBLK 8192
/* DEBUG 2 prints information obtained via mincore(2) */
-// #define DEBUG 2
+#define DEBUG 1
/* do not calculate exact madvise hints but assume 1 page for headers and
* set DONTNEED for the rest, which is assumed to be data */
-//#define ONE_PAGE 1
/* Avoid calling madvise on areas that were already hinted. May be benefical if
* your syscalls are very slow */
-//#define CHECK_MADVISE_OVERLAPS 1
#ifdef HAVE_MMAP
+/* the cast to void* is there to avoid this warning seen on ia64 with certain
+ versions of gcc: 'cast increases required alignment of target type'
+*/
#define __rrd_read(dst, dst_t, cnt) \
- (dst) = (dst_t*) (data + offset); \
+ (dst) = (dst_t*)(void*) (data + offset); \
offset += sizeof(dst_t) * (cnt)
#else
#define __rrd_read(dst, dst_t, cnt) \
offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt))
#endif
-/* next page-aligned (i.e. page-align up) */
-#ifndef PAGE_ALIGN
-#define PAGE_ALIGN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
-#endif
-/* previous page-aligned (i.e. page-align down) */
-#ifndef PAGE_ALIGN_DOWN
-#define PAGE_ALIGN_DOWN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
-#endif
-
-#ifdef HAVE_MMAP
-/* vector of last madvise hint */
-typedef struct _madvise_vec_t {
- void *start;
- ssize_t length;
-} _madvise_vec_t;
-_madvise_vec_t _madv_vec = { NULL, 0 };
+/* get the address of the start of this page */
+#ifndef PAGE_START
+#define PAGE_START(addr) ((addr)&(~(_page_size-1)))
#endif
-#if defined CHECK_MADVISE_OVERLAPS
-#define _madvise(_start, _off, _hint) \
- if ((_start) != _madv_vec.start && (ssize_t)(_off) != _madv_vec.length) { \
- _madv_vec.start = (_start) ; _madv_vec.length = (_off); \
- madvise((_start), (_off), (_hint)); \
- }
-#else
-#define _madvise(_start, _off, _hint) \
- madvise((_start), (_off), (_hint))
-#endif
/* Open a database file, return its header and an open filehandle,
* positioned to the first cdp in the first rra.
off_t offset = 0;
struct stat statb;
rrd_file_t *rrd_file = NULL;
+ off_t newfile_size = 0;
+ if (rdwr & RRD_CREAT) {
+ /* yes bad inline signaling alert, we are using the
+ floatcookie to pass the size in ... only used in resize */
+ newfile_size = (off_t) rrd->stat_head->float_cookie;
+ free(rrd->stat_head);
+ }
rrd_init(rrd);
rrd_file = malloc(sizeof(rrd_file_t));
if (rrd_file == NULL) {
#if defined MAP_NONBLOCK
mm_flags |= MAP_NONBLOCK; /* just populate ptes */
#endif
-#ifdef USE_DIRECT_IO
- } else {
- flags |= O_DIRECT;
-#endif
}
-#ifdef O_NONBLOCK
- flags |= O_NONBLOCK;
-#endif
if ((rrd_file->fd = open(file_name, flags, mode)) < 0) {
rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
- return NULL;
+ goto out_free;
}
/* Better try to avoid seeks as much as possible. stat may be heavy but
* many concurrent seeks are even worse. */
- if ((fstat(rrd_file->fd, &statb)) < 0) {
+ if (newfile_size == 0 && ((fstat(rrd_file->fd, &statb)) < 0)) {
rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
goto out_close;
}
- rrd_file->file_len = statb.st_size;
-
+ if (newfile_size == 0) {
+ rrd_file->file_len = statb.st_size;
+ } else {
+ rrd_file->file_len = newfile_size;
+ lseek(rrd_file->fd, newfile_size - 1, SEEK_SET);
+ write(rrd_file->fd, "\0", 1); /* poke */
+ lseek(rrd_file->fd, 0, SEEK_SET);
+ }
#ifdef HAVE_POSIX_FADVISE
/* In general we need no read-ahead when dealing with rrd_files.
When we stop reading, it is highly unlikely that we start up again.
/* lets see if the first read worked */
if (data == MAP_FAILED) {
- rrd_set_error("error mmaping file '%s': %s", file_name,
+ rrd_set_error("mmaping file '%s': %s", file_name,
rrd_strerror(errno));
goto out_close;
}
rrd_file->file_start = data;
+ if (rdwr & RRD_CREAT) {
+ memset(data, DNAN, newfile_size - 1);
+ goto out_done;
+ }
#endif
+ if (rdwr & RRD_CREAT)
+ goto out_done;
#ifdef USE_MADVISE
if (rdwr & RRD_COPY) {
/* We will read everything in a moment (copying) */
- _madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
- goto out_done;
+ madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
+ } else {
+ /* We do not need to read anything in for the moment */
+ madvise(data, rrd_file->file_len, MADV_RANDOM);
+ /* the stat_head will be needed soonish, so hint accordingly */
+ madvise(data, sizeof(stat_head_t), MADV_WILLNEED | MADV_RANDOM);
}
- /* We do not need to read anything in for the moment */
-#ifndef ONE_PAGE
- _madvise(data, rrd_file->file_len, MADV_DONTNEED);
-// _madvise(data, rrd_file->file_len, MADV_RANDOM);
-#else
-/* alternatively: keep 2 pages worth of data, likely headers,
- * don't need the rest. */
- _madvise(data, _page_size, MADV_WILLNEED | MADV_SEQUENTIAL);
- _madvise(data + _page_size, (rrd_file->file_len >= _page_size)
- ? rrd_file->file_len - _page_size : 0, MADV_DONTNEED);
-#endif
-#endif
-
-#if defined USE_MADVISE && !defined ONE_PAGE
- /* the stat_head will be needed soonish, so hint accordingly */
-// too finegrained to calc the individual sizes, just keep 2 pages worth of hdr
- _madvise(data + PAGE_ALIGN_DOWN(offset), PAGE_ALIGN(sizeof(stat_head_t)),
- MADV_WILLNEED);
-
#endif
__rrd_read(rrd->stat_head, stat_head_t,
}
if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
- rrd_set_error("This RRD was created on other architecture");
+ rrd_set_error("This RRD was created on another architecture");
goto out_nullify_head;
}
rrd->stat_head->version);
goto out_nullify_head;
}
-#if defined USE_MADVISE && !defined ONE_PAGE
+#if defined USE_MADVISE
/* the ds_def will be needed soonish, so hint accordingly */
- _madvise(data + PAGE_ALIGN_DOWN(offset),
- PAGE_ALIGN(sizeof(ds_def_t) * rrd->stat_head->ds_cnt),
- MADV_WILLNEED);
+ madvise(data + PAGE_START(offset),
+ sizeof(ds_def_t) * rrd->stat_head->ds_cnt, MADV_WILLNEED);
#endif
__rrd_read(rrd->ds_def, ds_def_t,
rrd->stat_head->ds_cnt);
-#if defined USE_MADVISE && !defined ONE_PAGE
+#if defined USE_MADVISE
/* the rra_def will be needed soonish, so hint accordingly */
- _madvise(data + PAGE_ALIGN_DOWN(offset),
- PAGE_ALIGN(sizeof(rra_def_t) * rrd->stat_head->rra_cnt),
- MADV_WILLNEED);
+ madvise(data + PAGE_START(offset),
+ sizeof(rra_def_t) * rrd->stat_head->rra_cnt, MADV_WILLNEED);
#endif
__rrd_read(rrd->rra_def, rra_def_t,
rrd->stat_head->rra_cnt);
#endif
rrd->live_head->last_up_usec = 0;
} else {
-#if defined USE_MADVISE && !defined ONE_PAGE
+#if defined USE_MADVISE
/* the live_head will be needed soonish, so hint accordingly */
- _madvise(data + PAGE_ALIGN_DOWN(offset),
- PAGE_ALIGN(sizeof(live_head_t)), MADV_WILLNEED);
+ madvise(data + PAGE_START(offset),
+ sizeof(live_head_t), MADV_WILLNEED);
#endif
__rrd_read(rrd->live_head, live_head_t,
1);
__rrd_read(rrd->rra_ptr, rra_ptr_t,
rrd->stat_head->rra_cnt);
-#ifdef USE_MADVISE
- out_done:
-#endif
rrd_file->header_len = offset;
rrd_file->pos = offset;
-
+ out_done:
return (rrd_file);
out_nullify_head:
rrd->stat_head = NULL;
out_close:
close(rrd_file->fd);
+ out_free:
+ free(rrd_file);
return NULL;
}
/* Close a reference to an rrd_file. */
-
-int rrd_close(
- rrd_file_t *rrd_file)
+static
+void mincore_print(
+ rrd_file_t *rrd_file,
+ char *mark)
{
- int ret;
-
-#if defined HAVE_MMAP || defined DEBUG
- ssize_t _page_size = sysconf(_SC_PAGESIZE);
-#endif
-#if defined DEBUG && DEBUG > 1
+#ifdef HAVE_MMAP
/* pretty print blocks in core */
off_t off;
unsigned char *vec;
+ ssize_t _page_size = sysconf(_SC_PAGESIZE);
off = rrd_file->file_len +
((rrd_file->file_len + _page_size - 1) / _page_size);
if (off == 0)
was_in = is_in;
if (was_in != is_in) {
- fprintf(stderr, "%sin core: %p len %ld\n",
+ fprintf(stderr, "%s: %sin core: %p len %ld\n", mark,
was_in ? "" : "not ", vec + prev, off - prev);
was_in = is_in;
prev = off;
}
}
fprintf(stderr,
- "%sin core: %p len %ld\n",
+ "%s: %sin core: %p len %ld\n", mark,
was_in ? "" : "not ", vec + prev, off - prev);
} else
fprintf(stderr, "mincore: %s", rrd_strerror(errno));
}
-#endif /* DEBUG */
+#else
+ fprintf(stderr, "sorry mincore only works with mmap");
+#endif
+}
+
+
+/* drop cache except for the header and the active pages */
+void rrd_dontneed(
+ rrd_file_t *rrd_file,
+ rrd_t *rrd)
+{
+ unsigned long dontneed_start;
+ unsigned long rra_start;
+ unsigned long active_block;
+ unsigned long i;
+ ssize_t _page_size = sysconf(_SC_PAGESIZE);
+
+#if defined DEBUG && DEBUG > 1
+ mincore_print(rrd_file, "before");
+#endif
-#ifdef USE_MADVISE
-# ifdef ONE_PAGE
- /* Keep headers around, round up to next page boundary. */
- ret =
- PAGE_ALIGN(rrd_file->header_len % _page_size + rrd_file->header_len);
- if (rrd_file->file_len > ret)
- _madvise(rrd_file->file_start + ret,
- rrd_file->file_len - ret, MADV_DONTNEED);
-# else
/* ignoring errors from RRDs that are smaller then the file_len+rounding */
- _madvise(rrd_file->file_start + PAGE_ALIGN_DOWN(rrd_file->header_len),
- rrd_file->file_len - PAGE_ALIGN(rrd_file->header_len),
- MADV_DONTNEED);
-# endif
+ rra_start = rrd_file->header_len;
+ dontneed_start = PAGE_START(rra_start) + _page_size;
+ for (i = 0; i < rrd->stat_head->rra_cnt; ++i) {
+ active_block =
+ PAGE_START(rra_start
+ + rrd->rra_ptr[i].cur_row
+ * rrd->stat_head->ds_cnt * sizeof(rrd_value_t));
+ if (active_block > dontneed_start) {
+#ifdef USE_MADVISE
+ madvise(rrd_file->file_start + dontneed_start,
+ active_block - dontneed_start - 1, MADV_DONTNEED);
+#endif
+/* in linux at least only fadvise DONTNEED seems to purge pages from cache */
+#ifdef HAVE_POSIX_FADVISE
+ posix_fadvise(rrd_file->fd, dontneed_start,
+ active_block - dontneed_start - 1,
+ POSIX_FADV_DONTNEED);
#endif
+ }
+ dontneed_start = active_block;
+ /* do not relase 'hot' block if update for this RAA will occure within 10 minutes */
+ if (rrd->stat_head->pdp_step * rrd->rra_def[i].pdp_cnt -
+ rrd->live_head->last_up % (rrd->stat_head->pdp_step *
+ rrd->rra_def[i].pdp_cnt) < 10 * 60) {
+ dontneed_start += _page_size;
+ }
+ rra_start +=
+ rrd->rra_def[i].row_cnt * rrd->stat_head->ds_cnt *
+ sizeof(rrd_value_t);
+ }
+#ifdef USE_MADVISE
+ madvise(rrd_file->file_start + dontneed_start,
+ rrd_file->file_len - dontneed_start, MADV_DONTNEED);
+#endif
+#ifdef HAVE_POSIX_FADVISE
+ posix_fadvise(rrd_file->fd, dontneed_start,
+ rrd_file->file_len - dontneed_start, POSIX_FADV_DONTNEED);
+#endif
+#if defined DEBUG && DEBUG > 1
+ mincore_print(rrd_file, "after");
+#endif
+}
+
+int rrd_close(
+ rrd_file_t *rrd_file)
+{
+ int ret;
+
#ifdef HAVE_MMAP
ret = munmap(rrd_file->file_start, rrd_file->file_len);
if (ret != 0)
/* read count bytes into buffer buf, starting at rrd_file->pos.
- * Returns the number of bytes read. */
+ * Returns the number of bytes read or <0 on error. */
inline ssize_t rrd_read(
rrd_file_t *rrd_file,
size_t count)
{
#ifdef HAVE_MMAP
- buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, count);
- rrd_file->pos += count; /* mimmic read() semantics */
- return count;
+ size_t _cnt = count;
+ ssize_t _surplus;
+
+ if (rrd_file->pos > rrd_file->file_len || _cnt == 0) /* EOF */
+ return 0;
+ if (buf == NULL)
+ return -1; /* EINVAL */
+ _surplus = rrd_file->pos + _cnt - rrd_file->file_len;
+ if (_surplus > 0) { /* short read */
+ _cnt -= _surplus;
+ }
+ if (_cnt == 0)
+ return 0; /* EOF */
+ buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, _cnt);
+
+ rrd_file->pos += _cnt; /* mimmic read() semantics */
+ return _cnt;
#else
ssize_t ret;
ret = read(rrd_file->fd, buf, count);
- //XXX: eventually add generic rrd_set_error(""); here
- rrd_file->pos += count; /* mimmic read() semantics */
+ if (ret > 0)
+ rrd_file->pos += ret; /* mimmic read() semantics */
return ret;
#endif
}
/* write count bytes from buffer buf to the current position
* rrd_file->pos of rrd_file->fd.
- * Returns the number of bytes written. */
+ * Returns the number of bytes written or <0 on error. */
inline ssize_t rrd_write(
rrd_file_t *rrd_file,
size_t count)
{
#ifdef HAVE_MMAP
+ if (count == 0)
+ return 0;
+ if (buf == NULL)
+ return -1 /* EINVAL */
memcpy(rrd_file->file_start + rrd_file->pos, buf, count);
rrd_file->pos += count;
return count; /* mimmic write() semantics */
#else
- ssize_t _sz = write(rrd_file->fd, buf, count);
+ ssize_t _sz = write(rrd_file->fd, buf, count);
+
if (_sz > 0)
rrd_file->pos += _sz;
return _sz;
void rrd_free(
rrd_t *rrd)
{
- if (atoi(rrd->stat_head->version) < 3)
- free(rrd->live_head);
+ free(rrd->live_head);
free(rrd->stat_head);
free(rrd->ds_def);
free(rrd->rra_def);
projects / rrdtool.git / blobdiff