X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=src%2Frrd_open.c;h=c6a2d7d9d49f09a65eaa01f263e23975c97a4064;hb=f972da18c85ebeb8591f585e7480fae9c1c5c380;hp=baeb00355308de85f3a83e6a54c25947ff08fde9;hpb=8a21b9cb276aa9b7401533474019f4ee1251a377;p=rrdtool.git diff --git a/src/rrd_open.c b/src/rrd_open.c index baeb003..c6a2d7d 100644 --- a/src/rrd_open.c +++ b/src/rrd_open.c @@ -1,74 +1,28 @@ /***************************************************************************** - * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007 + * RRDtool 1.2.99907080300 Copyright by Tobi Oetiker, 1997-2007 ***************************************************************************** * rrd_open.c Open an RRD File ***************************************************************************** * $Id$ - * $Log$ - * Revision 1.10 2004/05/26 22:11:12 oetiker - * reduce compiler warnings. Many small fixes. -- Mike Slifcak - * - * Revision 1.9 2003/04/29 21:56:49 oetiker - * readline in rrd_open.c reads the file in 8 KB blocks, and calls realloc for - * each block. realloc is very slow in Mac OS X for huge blocks, e.g. when - * restoring databases from huge xml files. This patch finds the size of the - * file, and starts out with malloc'ing the full size. - * -- Peter Speck - * - * Revision 1.8 2003/04/11 19:43:44 oetiker - * New special value COUNT which allows calculations based on the position of a - * value within a data set. Bug fix in rrd_rpncalc.c. PREV returned erroneus - * value for the second value. Bug fix in rrd_restore.c. Bug causing seek error - * when accesing an RRD restored from an xml that holds an RRD version <3. - * -- Ruben Justo - * - * Revision 1.7 2003/03/31 21:22:12 oetiker - * enables RRDtool updates with microsecond or in case of windows millisecond - * precision. This is needed to reduce time measurement error when archive step - * is small. (<30s) -- Sasha Mikheev - * - * Revision 1.6 2003/02/13 07:05:27 oetiker - * Find attached the patch I promised to send to you. Please note that there - * are three new source files (src/rrd_is_thread_safe.h, src/rrd_thread_safe.c - * and src/rrd_not_thread_safe.c) and the introduction of librrd_th. This - * library is identical to librrd, but it contains support code for per-thread - * global variables currently used for error information only. This is similar - * to how errno per-thread variables are implemented. librrd_th must be linked - * alongside of libpthred - * - * There is also a new file "THREADS", holding some documentation. - * - * -- Peter Stamfest - * - * Revision 1.5 2002/06/20 00:21:03 jake - * More Win32 build changes; thanks to Kerry Calvert. - * - * Revision 1.4 2002/02/01 20:34:49 oetiker - * fixed version number and date/time - * - * Revision 1.3 2001/03/04 13:01:55 oetiker - * Aberrant Behavior Detection support. A brief overview added to rrdtool.pod. - * Major updates to rrd_update.c, rrd_create.c. Minor update to other core files. - * This is backwards compatible! But new files using the Aberrant stuff are not readable - * by old rrdtool versions. See http://cricket.sourceforge.net/aberrant/rrd_hw.htm - * -- Jake Brutlag - * - * Revision 1.2 2001/03/04 10:29:20 oetiker - * fixed filedescriptor leak - * -- Mike Franusich - * - * Revision 1.1.1.1 2001/02/25 22:25:05 oetiker - * checkin - * *****************************************************************************/ #include "rrd_tool.h" #include "unused.h" #define MEMBLK 8192 +/* DEBUG 2 prints information obtained via mincore(2) */ +#define DEBUG 1 +/* do not calculate exact madvise hints but assume 1 page for headers and + * set DONTNEED for the rest, which is assumed to be data */ +/* Avoid calling madvise on areas that were already hinted. May be benefical if + * your syscalls are very slow */ + #ifdef HAVE_MMAP +/* the cast to void* is there to avoid this warning seen on ia64 with certain + versions of gcc: 'cast increases required alignment of target type' +*/ #define __rrd_read(dst, dst_t, cnt) \ - (dst) = (dst_t*) (data + offset); \ + (dst) = (dst_t*)(void*) (data + offset); \ offset += sizeof(dst_t) * (cnt) #else #define __rrd_read(dst, dst_t, cnt) \ @@ -79,8 +33,17 @@ offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt)) #endif -/* open a database file, return its header and an open filehandle */ -/* positioned to the first cdp in the first rra */ +/* get the address of the start of this page */ +#ifndef PAGE_START +#define PAGE_START(addr) ((addr)&(~(_page_size-1))) +#endif + + +/* Open a database file, return its header and an open filehandle, + * positioned to the first cdp in the first rra. + * In the error path of rrd_open, only rrd_free(&rrd) has to be called + * before returning an error. Do not call rrd_close upon failure of rrd_open. + */ rrd_file_t *rrd_open( const char *const file_name, @@ -92,19 +55,29 @@ rrd_file_t *rrd_open( int version; #ifdef HAVE_MMAP + ssize_t _page_size = sysconf(_SC_PAGESIZE); int mm_prot = PROT_READ, mm_flags = 0; char *data; #endif off_t offset = 0; struct stat statb; - rrd_file_t *rrd_file = malloc(sizeof(rrd_file_t)); - + rrd_file_t *rrd_file = NULL; + off_t newfile_size = 0; + + if (rdwr & RRD_CREAT) { + /* yes bad inline signaling alert, we are using the + floatcookie to pass the size in ... only used in resize */ + newfile_size = (off_t) rrd->stat_head->float_cookie; + free(rrd->stat_head); + } + rrd_init(rrd); + rrd_file = malloc(sizeof(rrd_file_t)); if (rrd_file == NULL) { rrd_set_error("allocating rrd_file descriptor for '%s'", file_name); return NULL; } memset(rrd_file, 0, sizeof(rrd_file_t)); - rrd_init(rrd); + #ifdef DEBUG if ((rdwr & (RRD_READONLY | RRD_READWRITE)) == (RRD_READONLY | RRD_READWRITE)) { @@ -118,9 +91,8 @@ rrd_file_t *rrd_open( #ifdef HAVE_MMAP mm_flags = MAP_PRIVATE; # ifdef MAP_NORESERVE - mm_flags |= MAP_NORESERVE; + mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */ # endif - mm_flags |= MAP_PRIVATE; #endif } else { if (rdwr & RRD_READWRITE) { @@ -137,33 +109,32 @@ rrd_file_t *rrd_open( } if (rdwr & RRD_READAHEAD) { #ifdef MAP_POPULATE - mm_flags |= MAP_POPULATE; -#endif -#if defined MAP_NONBLOCK && !defined USE_DIRECT_IO - mm_flags |= MAP_NONBLOCK; /* just populage ptes */ + mm_flags |= MAP_POPULATE; /* populate ptes and data */ #endif - } else { -#ifdef USE_DIRECT_IO - flags |= O_DIRECT; -#endif -#if 0 //def O_NONBLOCK - flags |= O_NONBLOCK; +#if defined MAP_NONBLOCK + mm_flags |= MAP_NONBLOCK; /* just populate ptes */ #endif } if ((rrd_file->fd = open(file_name, flags, mode)) < 0) { rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno)); - return NULL; + goto out_free; } /* Better try to avoid seeks as much as possible. stat may be heavy but * many concurrent seeks are even worse. */ - if ((fstat(rrd_file->fd, &statb)) < 0) { + if (newfile_size == 0 && ((fstat(rrd_file->fd, &statb)) < 0)) { rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno)); goto out_close; } - rrd_file->file_len = statb.st_size; - + if (newfile_size == 0) { + rrd_file->file_len = statb.st_size; + } else { + rrd_file->file_len = newfile_size; + lseek(rrd_file->fd, newfile_size - 1, SEEK_SET); + write(rrd_file->fd, "\0", 1); /* poke */ + lseek(rrd_file->fd, 0, SEEK_SET); + } #ifdef HAVE_POSIX_FADVISE /* In general we need no read-ahead when dealing with rrd_files. When we stop reading, it is highly unlikely that we start up again. @@ -191,26 +162,28 @@ rrd_file_t *rrd_open( /* lets see if the first read worked */ if (data == MAP_FAILED) { - rrd_set_error("error mmaping file '%s': %s", file_name, + rrd_set_error("mmaping file '%s': %s", file_name, rrd_strerror(errno)); goto out_close; } rrd_file->file_start = data; -#else + if (rdwr & RRD_CREAT) { + memset(data, DNAN, newfile_size - 1); + goto out_done; + } #endif + if (rdwr & RRD_CREAT) + goto out_done; #ifdef USE_MADVISE - if (rdwr & RRD_COPY) { /*XXX: currently not used! */ + if (rdwr & RRD_COPY) { /* We will read everything in a moment (copying) */ madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL); - goto out_done; + } else { + /* We do not need to read anything in for the moment */ + madvise(data, rrd_file->file_len, MADV_RANDOM); + /* the stat_head will be needed soonish, so hint accordingly */ + madvise(data, sizeof(stat_head_t), MADV_WILLNEED | MADV_RANDOM); } - /* We do not need to read anything in for the moment */ - madvise(data, rrd_file->file_len, MADV_DONTNEED); -#endif - -#ifdef USE_MADVISE - /* the stat_head will be needed soonish, so hint accordingly */ - madvise(data + offset, sizeof(stat_head_t), MADV_WILLNEED); #endif __rrd_read(rrd->stat_head, stat_head_t, @@ -223,7 +196,7 @@ rrd_file_t *rrd_open( } if (rrd->stat_head->float_cookie != FLOAT_COOKIE) { - rrd_set_error("This RRD was created on other architecture"); + rrd_set_error("This RRD was created on another architecture"); goto out_nullify_head; } @@ -234,18 +207,18 @@ rrd_file_t *rrd_open( rrd->stat_head->version); goto out_nullify_head; } -#ifdef USE_MADVISE +#if defined USE_MADVISE /* the ds_def will be needed soonish, so hint accordingly */ - madvise(data + offset, sizeof(ds_def_t) * rrd->stat_head->ds_cnt, - MADV_WILLNEED); + madvise(data + PAGE_START(offset), + sizeof(ds_def_t) * rrd->stat_head->ds_cnt, MADV_WILLNEED); #endif __rrd_read(rrd->ds_def, ds_def_t, rrd->stat_head->ds_cnt); -#ifdef USE_MADVISE +#if defined USE_MADVISE /* the rra_def will be needed soonish, so hint accordingly */ - madvise(data + offset, sizeof(rra_def_t) * rrd->stat_head->rra_cnt, - MADV_WILLNEED); + madvise(data + PAGE_START(offset), + sizeof(rra_def_t) * rrd->stat_head->rra_cnt, MADV_WILLNEED); #endif __rrd_read(rrd->rra_def, rra_def_t, rrd->stat_head->rra_cnt); @@ -265,9 +238,10 @@ rrd_file_t *rrd_open( #endif rrd->live_head->last_up_usec = 0; } else { -#ifdef USE_MADVISE +#if defined USE_MADVISE /* the live_head will be needed soonish, so hint accordingly */ - madvise(data + offset, sizeof(live_head_t), MADV_WILLNEED); + madvise(data + PAGE_START(offset), + sizeof(live_head_t), MADV_WILLNEED); #endif __rrd_read(rrd->live_head, live_head_t, 1); @@ -284,21 +258,123 @@ rrd_file_t *rrd_open( __rrd_read(rrd->rra_ptr, rra_ptr_t, rrd->stat_head->rra_cnt); -#ifdef USE_MADVISE - out_done: -#endif rrd_file->header_len = offset; rrd_file->pos = offset; -/* we could close(rrd_file->fd); here, the mapping is still valid anyway */ + out_done: return (rrd_file); out_nullify_head: rrd->stat_head = NULL; out_close: close(rrd_file->fd); + out_free: + free(rrd_file); return NULL; } + /* Close a reference to an rrd_file. */ +static +void mincore_print( + rrd_file_t *rrd_file, + char *mark) +{ +#ifdef HAVE_MMAP + /* pretty print blocks in core */ + off_t off; + unsigned char *vec; + ssize_t _page_size = sysconf(_SC_PAGESIZE); + + off = rrd_file->file_len + + ((rrd_file->file_len + _page_size - 1) / _page_size); + vec = malloc(off); + if (vec != NULL) { + memset(vec, 0, off); + if (mincore(rrd_file->file_start, rrd_file->file_len, vec) == 0) { + int prev; + unsigned is_in = 0, was_in = 0; + + for (off = 0, prev = 0; off < rrd_file->file_len; ++off) { + is_in = vec[off] & 1; /* if lsb set then is core resident */ + if (off == 0) + was_in = is_in; + if (was_in != is_in) { + fprintf(stderr, "%s: %sin core: %p len %ld\n", mark, + was_in ? "" : "not ", vec + prev, off - prev); + was_in = is_in; + prev = off; + } + } + fprintf(stderr, + "%s: %sin core: %p len %ld\n", mark, + was_in ? "" : "not ", vec + prev, off - prev); + } else + fprintf(stderr, "mincore: %s", rrd_strerror(errno)); + } +#else + fprintf(stderr, "sorry mincore only works with mmap"); +#endif +} + + +/* drop cache except for the header and the active pages */ +void rrd_dontneed( + rrd_file_t *rrd_file, + rrd_t *rrd) +{ + unsigned long dontneed_start; + unsigned long rra_start; + unsigned long active_block; + unsigned long i; + ssize_t _page_size = sysconf(_SC_PAGESIZE); + +#if defined DEBUG && DEBUG > 1 + mincore_print(rrd_file, "before"); +#endif + + /* ignoring errors from RRDs that are smaller then the file_len+rounding */ + rra_start = rrd_file->header_len; + dontneed_start = PAGE_START(rra_start) + _page_size; + for (i = 0; i < rrd->stat_head->rra_cnt; ++i) { + active_block = + PAGE_START(rra_start + + rrd->rra_ptr[i].cur_row + * rrd->stat_head->ds_cnt * sizeof(rrd_value_t)); + if (active_block > dontneed_start) { +#ifdef USE_MADVISE + madvise(rrd_file->file_start + dontneed_start, + active_block - dontneed_start - 1, MADV_DONTNEED); +#endif +/* in linux at least only fadvise DONTNEED seems to purge pages from cache */ +#ifdef HAVE_POSIX_FADVISE + posix_fadvise(rrd_file->fd, dontneed_start, + active_block - dontneed_start - 1, + POSIX_FADV_DONTNEED); +#endif + } + dontneed_start = active_block; + /* do not relase 'hot' block if update for this RAA will occure within 10 minutes */ + if (rrd->stat_head->pdp_step * rrd->rra_def[i].pdp_cnt - + rrd->live_head->last_up % (rrd->stat_head->pdp_step * + rrd->rra_def[i].pdp_cnt) < 10 * 60) { + dontneed_start += _page_size; + } + rra_start += + rrd->rra_def[i].row_cnt * rrd->stat_head->ds_cnt * + sizeof(rrd_value_t); + } +#ifdef USE_MADVISE + madvise(rrd_file->file_start + dontneed_start, + rrd_file->file_len - dontneed_start, MADV_DONTNEED); +#endif +#ifdef HAVE_POSIX_FADVISE + posix_fadvise(rrd_file->fd, dontneed_start, + rrd_file->file_len - dontneed_start, POSIX_FADV_DONTNEED); +#endif +#if defined DEBUG && DEBUG > 1 + mincore_print(rrd_file, "after"); +#endif +} + int rrd_close( rrd_file_t *rrd_file) { @@ -317,7 +393,9 @@ int rrd_close( return ret; } + /* Set position of rrd_file. */ + off_t rrd_seek( rrd_file_t *rrd_file, off_t off, @@ -342,53 +420,78 @@ off_t rrd_seek( return ret == -1; //XXX: or just ret to mimic lseek } + /* Get current position in rrd_file. */ + inline off_t rrd_tell( rrd_file_t *rrd_file) { return rrd_file->pos; } + /* read count bytes into buffer buf, starting at rrd_file->pos. - * Returns the number of bytes read. */ -ssize_t rrd_read( + * Returns the number of bytes read or <0 on error. */ + +inline ssize_t rrd_read( rrd_file_t *rrd_file, void *buf, size_t count) { #ifdef HAVE_MMAP - char *pos = rrd_file->file_start + rrd_file->pos; + size_t _cnt = count; + ssize_t _surplus; + + if (rrd_file->pos > rrd_file->file_len || _cnt == 0) /* EOF */ + return 0; + if (buf == NULL) + return -1; /* EINVAL */ + _surplus = rrd_file->pos + _cnt - rrd_file->file_len; + if (_surplus > 0) { /* short read */ + _cnt -= _surplus; + } + if (_cnt == 0) + return 0; /* EOF */ + buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, _cnt); - buf = memmove(buf, pos, count); - rrd_file->pos += count; /* mimmic read() semantics */ - return count; + rrd_file->pos += _cnt; /* mimmic read() semantics */ + return _cnt; #else ssize_t ret; ret = read(rrd_file->fd, buf, count); - //XXX: eventually add generic rrd_set_error(""); here - rrd_file->pos += count; /* mimmic read() semantics */ + if (ret > 0) + rrd_file->pos += ret; /* mimmic read() semantics */ return ret; #endif } + /* write count bytes from buffer buf to the current position * rrd_file->pos of rrd_file->fd. * Returns the number of bytes written. */ -ssize_t rrd_write( + +inline ssize_t rrd_write( rrd_file_t *rrd_file, const void *buf, size_t count) { #ifdef HAVE_MMAP - memmove(rrd_file->file_start + rrd_file->pos, buf, count); + memcpy(rrd_file->file_start + rrd_file->pos, buf, count); + rrd_file->pos += count; return count; /* mimmic write() semantics */ #else - return write(rrd_file->fd, buf, count); + ssize_t _sz = write(rrd_file->fd, buf, count); + + if (_sz > 0) + rrd_file->pos += _sz; + return _sz; #endif } + /* flush all data pending to be written to FD. */ + inline void rrd_flush( rrd_file_t *rrd_file) { @@ -398,6 +501,9 @@ inline void rrd_flush( } } + +/* Initialize RRD header. */ + void rrd_init( rrd_t *rrd) { @@ -411,12 +517,19 @@ void rrd_init( rrd->rrd_value = NULL; } -void rrd_free( + +/* free RRD header data. */ + +#ifdef HAVE_MMAP +inline void rrd_free( rrd_t UNUSED(*rrd)) { -#ifndef HAVE_MMAP - if (atoi(rrd->stat_head->version) < 3) - free(rrd->live_head); +} +#else +void rrd_free( + rrd_t *rrd) +{ + free(rrd->live_head); free(rrd->stat_head); free(rrd->ds_def); free(rrd->rra_def); @@ -424,19 +537,24 @@ void rrd_free( free(rrd->pdp_prep); free(rrd->cdp_prep); free(rrd->rrd_value); -//XXX: ? rrd_init(rrd); -#endif } +#endif + /* routine used by external libraries to free memory allocated by * rrd library */ + void rrd_freemem( void *mem) { free(mem); } -int readfile( + +/* XXX: FIXME: missing documentation. */ +/*XXX: FIXME should be renamed to rrd_readfile or _rrd_readfile */ + +int /*_rrd_*/ readfile( const char *file_name, char **buffer, int skipfirst)