X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=src%2Frrd_open.c;h=4306eff957708e47badafa12fc2b0d6f703ec5d1;hb=aeeff12d6f81965685019e9d416f0ec71a1ce0ab;hp=7812da183b5dd5e61a7da9e34bdeab0dc3bc8291;hpb=e7094fb9bb30ba35b2a7e4262f747297bc428172;p=rrdtool.git diff --git a/src/rrd_open.c b/src/rrd_open.c index 7812da1..4306eff 100644 --- a/src/rrd_open.c +++ b/src/rrd_open.c @@ -4,62 +4,6 @@ * rrd_open.c Open an RRD File ***************************************************************************** * $Id$ - * $Log$ - * Revision 1.10 2004/05/26 22:11:12 oetiker - * reduce compiler warnings. Many small fixes. -- Mike Slifcak - * - * Revision 1.9 2003/04/29 21:56:49 oetiker - * readline in rrd_open.c reads the file in 8 KB blocks, and calls realloc for - * each block. realloc is very slow in Mac OS X for huge blocks, e.g. when - * restoring databases from huge xml files. This patch finds the size of the - * file, and starts out with malloc'ing the full size. - * -- Peter Speck - * - * Revision 1.8 2003/04/11 19:43:44 oetiker - * New special value COUNT which allows calculations based on the position of a - * value within a data set. Bug fix in rrd_rpncalc.c. PREV returned erroneus - * value for the second value. Bug fix in rrd_restore.c. Bug causing seek error - * when accesing an RRD restored from an xml that holds an RRD version <3. - * -- Ruben Justo - * - * Revision 1.7 2003/03/31 21:22:12 oetiker - * enables RRDtool updates with microsecond or in case of windows millisecond - * precision. This is needed to reduce time measurement error when archive step - * is small. (<30s) -- Sasha Mikheev - * - * Revision 1.6 2003/02/13 07:05:27 oetiker - * Find attached the patch I promised to send to you. Please note that there - * are three new source files (src/rrd_is_thread_safe.h, src/rrd_thread_safe.c - * and src/rrd_not_thread_safe.c) and the introduction of librrd_th. This - * library is identical to librrd, but it contains support code for per-thread - * global variables currently used for error information only. This is similar - * to how errno per-thread variables are implemented. librrd_th must be linked - * alongside of libpthred - * - * There is also a new file "THREADS", holding some documentation. - * - * -- Peter Stamfest - * - * Revision 1.5 2002/06/20 00:21:03 jake - * More Win32 build changes; thanks to Kerry Calvert. - * - * Revision 1.4 2002/02/01 20:34:49 oetiker - * fixed version number and date/time - * - * Revision 1.3 2001/03/04 13:01:55 oetiker - * Aberrant Behavior Detection support. A brief overview added to rrdtool.pod. - * Major updates to rrd_update.c, rrd_create.c. Minor update to other core files. - * This is backwards compatible! But new files using the Aberrant stuff are not readable - * by old rrdtool versions. See http://cricket.sourceforge.net/aberrant/rrd_hw.htm - * -- Jake Brutlag - * - * Revision 1.2 2001/03/04 10:29:20 oetiker - * fixed filedescriptor leak - * -- Mike Franusich - * - * Revision 1.1.1.1 2001/02/25 22:25:05 oetiker - * checkin - * *****************************************************************************/ #include "rrd_tool.h" @@ -67,13 +11,11 @@ #define MEMBLK 8192 /* DEBUG 2 prints information obtained via mincore(2) */ -// #define DEBUG 2 +#define DEBUG 1 /* do not calculate exact madvise hints but assume 1 page for headers and * set DONTNEED for the rest, which is assumed to be data */ -//#define ONE_PAGE 1 /* Avoid calling madvise on areas that were already hinted. May be benefical if * your syscalls are very slow */ -#define CHECK_MADVISE_OVERLAPS 1 #ifdef HAVE_MMAP /* the cast to void* is there to avoid this warning seen on ia64 with certain @@ -91,34 +33,11 @@ offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt)) #endif -/* next page-aligned (i.e. page-align up) */ -#ifndef PAGE_ALIGN -#define PAGE_ALIGN(addr) (((addr)+_page_size-1)&(~(_page_size-1))) -#endif -/* previous page-aligned (i.e. page-align down) */ -#ifndef PAGE_ALIGN_DOWN -#define PAGE_ALIGN_DOWN(addr) (((addr)+_page_size-1)&(~(_page_size-1))) -#endif - -#ifdef HAVE_MMAP -/* vector of last madvise hint */ -typedef struct _madvise_vec_t { - void *start; - ssize_t length; -} _madvise_vec_t; -_madvise_vec_t _madv_vec = { NULL, 0 }; +/* get the address of the start of this page */ +#ifndef PAGE_START +#define PAGE_START(addr) ((addr)&(~(_page_size-1))) #endif -#if defined CHECK_MADVISE_OVERLAPS -#define _madvise(_start, _off, _hint) \ - if ((_start) != _madv_vec.start && (ssize_t)(_off) != _madv_vec.length) { \ - _madv_vec.start = (_start) ; _madv_vec.length = (_off); \ - madvise((_start), (_off), (_hint)); \ - } -#else -#define _madvise(_start, _off, _hint) \ - madvise((_start), (_off), (_hint)) -#endif /* Open a database file, return its header and an open filehandle, * positioned to the first cdp in the first rra. @@ -145,8 +64,12 @@ rrd_file_t *rrd_open( rrd_file_t *rrd_file = NULL; off_t newfile_size = 0; - if (rdwr & RRD_CREAT) - newfile_size = (off_t) rrd->stat_head; + if (rdwr & RRD_CREAT) { + /* yes bad inline signaling alert, we are using the + floatcookie to pass the size in ... only used in resize */ + newfile_size = (off_t) rrd->stat_head->float_cookie; + free(rrd->stat_head); + } rrd_init(rrd); rrd_file = malloc(sizeof(rrd_file_t)); if (rrd_file == NULL) { @@ -189,17 +112,9 @@ rrd_file_t *rrd_open( mm_flags |= MAP_POPULATE; /* populate ptes and data */ #endif #if defined MAP_NONBLOCK -// if (!(rdwr & RRD_COPY)) mm_flags |= MAP_NONBLOCK; /* just populate ptes */ #endif -#ifdef USE_DIRECT_IO - } else { - flags |= O_DIRECT; -#endif } -#ifdef O_NONBLOCK - flags |= O_NONBLOCK; -#endif if ((rrd_file->fd = open(file_name, flags, mode)) < 0) { rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno)); @@ -217,7 +132,7 @@ rrd_file_t *rrd_open( } else { rrd_file->file_len = newfile_size; lseek(rrd_file->fd, newfile_size - 1, SEEK_SET); - write(rrd_file->fd, "\0", 1); /* poke */ + write(rrd_file->fd, "\0", 1); /* poke */ lseek(rrd_file->fd, 0, SEEK_SET); } #ifdef HAVE_POSIX_FADVISE @@ -253,29 +168,21 @@ rrd_file_t *rrd_open( } rrd_file->file_start = data; if (rdwr & RRD_CREAT) { + memset(data, DNAN, newfile_size - 1); goto out_done; } #endif + if (rdwr & RRD_CREAT) + goto out_done; #ifdef USE_MADVISE if (rdwr & RRD_COPY) { /* We will read everything in a moment (copying) */ - _madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL); + madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL); } else { -# ifndef ONE_PAGE /* We do not need to read anything in for the moment */ - _madvise(data, rrd_file->file_len, MADV_DONTNEED); + madvise(data, rrd_file->file_len, MADV_RANDOM); /* the stat_head will be needed soonish, so hint accordingly */ - _madvise(data + PAGE_ALIGN_DOWN(offset), - PAGE_ALIGN(sizeof(stat_head_t)), - MADV_WILLNEED | MADV_RANDOM); - -# else -/* alternatively: keep 1 page worth of data, likely headers, - * don't need the rest. */ - _madvise(data, _page_size, MADV_WILLNEED | MADV_SEQUENTIAL); - _madvise(data + _page_size, (rrd_file->file_len >= _page_size) - ? rrd_file->file_len - _page_size : 0, MADV_DONTNEED); -# endif + madvise(data, sizeof(stat_head_t), MADV_WILLNEED | MADV_RANDOM); } #endif @@ -300,19 +207,19 @@ rrd_file_t *rrd_open( rrd->stat_head->version); goto out_nullify_head; } -#if defined USE_MADVISE && !defined ONE_PAGE +#if defined USE_MADVISE /* the ds_def will be needed soonish, so hint accordingly */ - _madvise(data + PAGE_ALIGN_DOWN(offset), - PAGE_ALIGN(sizeof(ds_def_t) * rrd->stat_head->ds_cnt), + madvise(data + PAGE_START(offset), + sizeof(ds_def_t) * rrd->stat_head->ds_cnt, MADV_WILLNEED); #endif __rrd_read(rrd->ds_def, ds_def_t, rrd->stat_head->ds_cnt); -#if defined USE_MADVISE && !defined ONE_PAGE +#if defined USE_MADVISE /* the rra_def will be needed soonish, so hint accordingly */ - _madvise(data + PAGE_ALIGN_DOWN(offset), - PAGE_ALIGN(sizeof(rra_def_t) * rrd->stat_head->rra_cnt), + madvise(data + PAGE_START(offset), + sizeof(rra_def_t) * rrd->stat_head->rra_cnt, MADV_WILLNEED); #endif __rrd_read(rrd->rra_def, rra_def_t, @@ -333,10 +240,10 @@ rrd_file_t *rrd_open( #endif rrd->live_head->last_up_usec = 0; } else { -#if defined USE_MADVISE && !defined ONE_PAGE +#if defined USE_MADVISE /* the live_head will be needed soonish, so hint accordingly */ - _madvise(data + PAGE_ALIGN_DOWN(offset), - PAGE_ALIGN(sizeof(live_head_t)), MADV_WILLNEED); + madvise(data + PAGE_START(offset), + sizeof(live_head_t), MADV_WILLNEED); #endif __rrd_read(rrd->live_head, live_head_t, 1); @@ -355,9 +262,7 @@ rrd_file_t *rrd_open( rrd_file->header_len = offset; rrd_file->pos = offset; -#ifdef USE_MADVISE out_done: -#endif return (rrd_file); out_nullify_head: rrd->stat_head = NULL; @@ -370,19 +275,13 @@ rrd_file_t *rrd_open( /* Close a reference to an rrd_file. */ - -int rrd_close( - rrd_file_t *rrd_file) -{ - int ret; - -#if defined HAVE_MMAP || defined DEBUG - ssize_t _page_size = sysconf(_SC_PAGESIZE); -#endif -#if defined DEBUG && DEBUG > 1 +static +void mincore_print(rrd_file_t *rrd_file,char * mark){ +#ifdef HAVE_MMAP /* pretty print blocks in core */ off_t off; unsigned char *vec; + ssize_t _page_size = sysconf(_SC_PAGESIZE); off = rrd_file->file_len + ((rrd_file->file_len + _page_size - 1) / _page_size); @@ -398,35 +297,84 @@ int rrd_close( if (off == 0) was_in = is_in; if (was_in != is_in) { - fprintf(stderr, "%sin core: %p len %ld\n", + fprintf(stderr, "%s: %sin core: %p len %ld\n",mark, was_in ? "" : "not ", vec + prev, off - prev); was_in = is_in; prev = off; } } fprintf(stderr, - "%sin core: %p len %ld\n", + "%s: %sin core: %p len %ld\n", mark, was_in ? "" : "not ", vec + prev, off - prev); } else fprintf(stderr, "mincore: %s", rrd_strerror(errno)); } -#endif /* DEBUG */ +#else + fprintf(stderr, "sorry mincore only works with mmap"); +#endif +} + + +/* drop cache except for the header and the active pages */ +void +rrd_dontneed ( + rrd_file_t *rrd_file, + rrd_t *rrd){ + unsigned long dontneed_start; + unsigned long rra_start; + unsigned long active_block; + unsigned long i; + ssize_t _page_size = sysconf(_SC_PAGESIZE); + +#if defined DEBUG && DEBUG > 1 + mincore_print(rrd_file,"before"); +#endif -#ifdef USE_MADVISE -# ifdef ONE_PAGE - /* Keep headers around, round up to next page boundary. */ - ret = - PAGE_ALIGN(rrd_file->header_len % _page_size + rrd_file->header_len); - if (rrd_file->file_len > ret) - _madvise(rrd_file->file_start + ret, - rrd_file->file_len - ret, MADV_DONTNEED); -# else /* ignoring errors from RRDs that are smaller then the file_len+rounding */ - _madvise(rrd_file->file_start + PAGE_ALIGN_DOWN(rrd_file->header_len), - rrd_file->file_len - PAGE_ALIGN(rrd_file->header_len), + rra_start = rrd_file->header_len; + dontneed_start = PAGE_START(rra_start)+_page_size; + for (i = 0; i < rrd->stat_head->rra_cnt; ++i) { + active_block = + PAGE_START(rra_start + + rrd->rra_ptr[i].cur_row + * rrd->stat_head->ds_cnt + * sizeof(rrd_value_t)); + if (active_block > dontneed_start) { +#ifdef USE_MADVISE + madvise(rrd_file->file_start + dontneed_start, + active_block-dontneed_start-1, + MADV_DONTNEED); +#endif +/* in linux at least only fadvise DONTNEED seems to purge pages from cache */ +#ifdef HAVE_POSIX_FADVISE + posix_fadvise(rrd_file->fd, dontneed_start, active_block-dontneed_start-1, POSIX_FADV_DONTNEED); +#endif + } + dontneed_start = active_block; + /* do not relase 'hot' block if update for this RAA will occure within 10 minutes */ + if ( rrd->stat_head->pdp_step * rrd->rra_def[i].pdp_cnt - + rrd->live_head->last_up % (rrd->stat_head->pdp_step * rrd->rra_def[i].pdp_cnt) < 10*60 ){ + dontneed_start += _page_size; + } + rra_start += rrd->rra_def[i].row_cnt * rrd->stat_head->ds_cnt * sizeof(rrd_value_t); + } +#ifdef USE_MADVISE + madvise(rrd_file->file_start + dontneed_start, + rrd_file->file_len - dontneed_start, MADV_DONTNEED); -# endif #endif +#ifdef HAVE_POSIX_FADVISE + posix_fadvise(rrd_file->fd, dontneed_start, rrd_file->file_len-dontneed_start, POSIX_FADV_DONTNEED); +#endif +#if defined DEBUG && DEBUG > 1 + mincore_print(rrd_file,"after"); +#endif +} + +int rrd_close( + rrd_file_t *rrd_file) +{ + int ret; #ifdef HAVE_MMAP ret = munmap(rrd_file->file_start, rrd_file->file_len); if (ret != 0) @@ -486,23 +434,24 @@ inline ssize_t rrd_read( size_t count) { #ifdef HAVE_MMAP - size_t _cnt = count; - ssize_t _surplus = rrd_file->pos + _cnt - rrd_file->file_len; + size_t _cnt = count; + ssize_t _surplus = rrd_file->pos + _cnt - rrd_file->file_len; + if (_surplus > 0) { /* short read */ _cnt -= _surplus; } if (_cnt == 0) - return 0; /* EOF */ + return 0; /* EOF */ buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, _cnt); - rrd_file->pos += _cnt; /* mimmic read() semantics */ + rrd_file->pos += _cnt; /* mimmic read() semantics */ return _cnt; #else ssize_t ret; ret = read(rrd_file->fd, buf, count); if (ret > 0) - rrd_file->pos += ret; /* mimmic read() semantics */ + rrd_file->pos += ret; /* mimmic read() semantics */ return ret; #endif } @@ -570,8 +519,7 @@ inline void rrd_free( void rrd_free( rrd_t *rrd) { - if (atoi(rrd->stat_head->version) < 3) - free(rrd->live_head); + free(rrd->live_head); free(rrd->stat_head); free(rrd->ds_def); free(rrd->rra_def);