X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=src%2Frrd_open.c;h=e83918454cad9e3bee9ac37b819ca6e5ce401cdc;hb=be7a55473565ee9d2bd83f5be70d08dbb9550eb3;hp=c986f750beca7bec9fb3f0cba5c5a488cfda17d2;hpb=009ac7ac9dde1ba7c450b230b9118cff2cedc9f6;p=rrdtool.git diff --git a/src/rrd_open.c b/src/rrd_open.c index c986f75..e839184 100644 --- a/src/rrd_open.c +++ b/src/rrd_open.c @@ -68,16 +68,19 @@ /* DEBUG 2 prints information obtained via mincore(2) */ // #define DEBUG 2 -/* do not calculate exact madvise hints but assume 2 pages for headers and +/* do not calculate exact madvise hints but assume 1 page for headers and * set DONTNEED for the rest, which is assumed to be data */ -//#define TWO_PAGES 1 +//#define ONE_PAGE 1 /* Avoid calling madvise on areas that were already hinted. May be benefical if * your syscalls are very slow */ -//#define CHECK_MADVISE_OVERLAPS 1 +#define CHECK_MADVISE_OVERLAPS 1 #ifdef HAVE_MMAP +/* the cast to void* is there to avoid this warning seen on ia64 with certain + versions of gcc: 'cast increases required alignment of target type' +*/ #define __rrd_read(dst, dst_t, cnt) \ - (dst) = (dst_t*) (data + offset); \ + (dst) = (dst_t*)(void*) (data + offset); \ offset += sizeof(dst_t) * (cnt) #else #define __rrd_read(dst, dst_t, cnt) \ @@ -117,8 +120,11 @@ _madvise_vec_t _madv_vec = { NULL, 0 }; madvise((_start), (_off), (_hint)) #endif -/* open a database file, return its header and an open filehandle */ -/* positioned to the first cdp in the first rra */ +/* Open a database file, return its header and an open filehandle, + * positioned to the first cdp in the first rra. + * In the error path of rrd_open, only rrd_free(&rrd) has to be called + * before returning an error. Do not call rrd_close upon failure of rrd_open. + */ rrd_file_t *rrd_open( const char *const file_name, @@ -136,14 +142,19 @@ rrd_file_t *rrd_open( #endif off_t offset = 0; struct stat statb; - rrd_file_t *rrd_file = malloc(sizeof(rrd_file_t)); + rrd_file_t *rrd_file = NULL; + off_t newfile_size = 0; + if (rdwr & RRD_CREAT) + newfile_size = (off_t) rrd->stat_head; + rrd_init(rrd); + rrd_file = malloc(sizeof(rrd_file_t)); if (rrd_file == NULL) { rrd_set_error("allocating rrd_file descriptor for '%s'", file_name); return NULL; } memset(rrd_file, 0, sizeof(rrd_file_t)); - rrd_init(rrd); + #ifdef DEBUG if ((rdwr & (RRD_READONLY | RRD_READWRITE)) == (RRD_READONLY | RRD_READWRITE)) { @@ -178,6 +189,7 @@ rrd_file_t *rrd_open( mm_flags |= MAP_POPULATE; /* populate ptes and data */ #endif #if defined MAP_NONBLOCK +// if (!(rdwr & RRD_COPY)) mm_flags |= MAP_NONBLOCK; /* just populate ptes */ #endif #ifdef USE_DIRECT_IO @@ -191,17 +203,23 @@ rrd_file_t *rrd_open( if ((rrd_file->fd = open(file_name, flags, mode)) < 0) { rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno)); - return NULL; + goto out_free; } /* Better try to avoid seeks as much as possible. stat may be heavy but * many concurrent seeks are even worse. */ - if ((fstat(rrd_file->fd, &statb)) < 0) { + if (newfile_size == 0 && ((fstat(rrd_file->fd, &statb)) < 0)) { rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno)); goto out_close; } - rrd_file->file_len = statb.st_size; - + if (newfile_size == 0) { + rrd_file->file_len = statb.st_size; + } else { + rrd_file->file_len = newfile_size; + lseek(rrd_file->fd, newfile_size - 1, SEEK_SET); + write(rrd_file->fd, "\0", 1); /* poke */ + lseek(rrd_file->fd, 0, SEEK_SET); + } #ifdef HAVE_POSIX_FADVISE /* In general we need no read-ahead when dealing with rrd_files. When we stop reading, it is highly unlikely that we start up again. @@ -229,37 +247,37 @@ rrd_file_t *rrd_open( /* lets see if the first read worked */ if (data == MAP_FAILED) { - rrd_set_error("error mmaping file '%s': %s", file_name, + rrd_set_error("mmaping file '%s': %s", file_name, rrd_strerror(errno)); goto out_close; } rrd_file->file_start = data; + if (rdwr & RRD_CREAT) { + memset(data, DNAN, newfile_size - 1); + goto out_done; + } #endif #ifdef USE_MADVISE if (rdwr & RRD_COPY) { /* We will read everything in a moment (copying) */ _madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL); - goto out_done; - } - /* We do not need to read anything in for the moment */ -#ifndef TWO_PAGES - _madvise(data, rrd_file->file_len, MADV_DONTNEED); -// _madvise(data, rrd_file->file_len, MADV_RANDOM); -#else -/* alternatively: keep 2 pages worth of data, likely headers, - * don't need the rest. */ - _madvise(data, _page_size * 2, MADV_WILLNEED | MADV_SEQUENTIAL); - _madvise(data + _page_size * 2, (rrd_file->file_len >= _page_size * 2) - ? rrd_file->file_len - _page_size * 2 : 0, MADV_DONTNEED); -#endif -#endif - -#if defined USE_MADVISE && !defined TWO_PAGES - /* the stat_head will be needed soonish, so hint accordingly */ -// too finegrained to calc the individual sizes, just keep 2 pages worth of hdr - _madvise(data + PAGE_ALIGN_DOWN(offset), PAGE_ALIGN(sizeof(stat_head_t)), - MADV_WILLNEED); + } else { +# ifndef ONE_PAGE + /* We do not need to read anything in for the moment */ + _madvise(data, rrd_file->file_len, MADV_DONTNEED); + /* the stat_head will be needed soonish, so hint accordingly */ + _madvise(data + PAGE_ALIGN_DOWN(offset), + PAGE_ALIGN(sizeof(stat_head_t)), + MADV_WILLNEED | MADV_RANDOM); +# else +/* alternatively: keep 1 page worth of data, likely headers, + * don't need the rest. */ + _madvise(data, _page_size, MADV_WILLNEED | MADV_SEQUENTIAL); + _madvise(data + _page_size, (rrd_file->file_len >= _page_size) + ? rrd_file->file_len - _page_size : 0, MADV_DONTNEED); +# endif + } #endif __rrd_read(rrd->stat_head, stat_head_t, @@ -272,7 +290,7 @@ rrd_file_t *rrd_open( } if (rrd->stat_head->float_cookie != FLOAT_COOKIE) { - rrd_set_error("This RRD was created on other architecture"); + rrd_set_error("This RRD was created on another architecture"); goto out_nullify_head; } @@ -283,7 +301,7 @@ rrd_file_t *rrd_open( rrd->stat_head->version); goto out_nullify_head; } -#if defined USE_MADVISE && !defined TWO_PAGES +#if defined USE_MADVISE && !defined ONE_PAGE /* the ds_def will be needed soonish, so hint accordingly */ _madvise(data + PAGE_ALIGN_DOWN(offset), PAGE_ALIGN(sizeof(ds_def_t) * rrd->stat_head->ds_cnt), @@ -292,7 +310,7 @@ rrd_file_t *rrd_open( __rrd_read(rrd->ds_def, ds_def_t, rrd->stat_head->ds_cnt); -#if defined USE_MADVISE && !defined TWO_PAGES +#if defined USE_MADVISE && !defined ONE_PAGE /* the rra_def will be needed soonish, so hint accordingly */ _madvise(data + PAGE_ALIGN_DOWN(offset), PAGE_ALIGN(sizeof(rra_def_t) * rrd->stat_head->rra_cnt), @@ -316,7 +334,7 @@ rrd_file_t *rrd_open( #endif rrd->live_head->last_up_usec = 0; } else { -#if defined USE_MADVISE && !defined TWO_PAGES +#if defined USE_MADVISE && !defined ONE_PAGE /* the live_head will be needed soonish, so hint accordingly */ _madvise(data + PAGE_ALIGN_DOWN(offset), PAGE_ALIGN(sizeof(live_head_t)), MADV_WILLNEED); @@ -336,17 +354,18 @@ rrd_file_t *rrd_open( __rrd_read(rrd->rra_ptr, rra_ptr_t, rrd->stat_head->rra_cnt); + rrd_file->header_len = offset; + rrd_file->pos = offset; #ifdef USE_MADVISE out_done: #endif - rrd_file->header_len = offset; - rrd_file->pos = offset; - return (rrd_file); out_nullify_head: rrd->stat_head = NULL; out_close: close(rrd_file->fd); + out_free: + free(rrd_file); return NULL; } @@ -358,7 +377,7 @@ int rrd_close( { int ret; -#if defined HAVE_MMAP +#if defined HAVE_MMAP || defined DEBUG ssize_t _page_size = sysconf(_SC_PAGESIZE); #endif #if defined DEBUG && DEBUG > 1 @@ -366,10 +385,8 @@ int rrd_close( off_t off; unsigned char *vec; - off = - rrd_file->file_len + - ((rrd_file->file_len + sysconf(_SC_PAGESIZE) - - 1) / sysconf(_SC_PAGESIZE)); + off = rrd_file->file_len + + ((rrd_file->file_len + _page_size - 1) / _page_size); vec = malloc(off); if (vec != NULL) { memset(vec, 0, off); @@ -397,31 +414,28 @@ int rrd_close( #endif /* DEBUG */ #ifdef USE_MADVISE -#ifdef TWO_PAGES -//XXX: ? - /* Keep 2 pages worth of headers around, round up to next page boundary. */ +# ifdef ONE_PAGE + /* Keep headers around, round up to next page boundary. */ ret = PAGE_ALIGN(rrd_file->header_len % _page_size + rrd_file->header_len); if (rrd_file->file_len > ret) _madvise(rrd_file->file_start + ret, rrd_file->file_len - ret, MADV_DONTNEED); -#else +# else /* ignoring errors from RRDs that are smaller then the file_len+rounding */ _madvise(rrd_file->file_start + PAGE_ALIGN_DOWN(rrd_file->header_len), rrd_file->file_len - PAGE_ALIGN(rrd_file->header_len), MADV_DONTNEED); -#endif +# endif #endif #ifdef HAVE_MMAP ret = munmap(rrd_file->file_start, rrd_file->file_len); if (ret != 0) rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno)); -#else - ret = 0; #endif -// ret = close(rrd_file->fd); -// if (ret != 0) -// rrd_set_error("closing file: %s", rrd_strerror(errno)); + ret = close(rrd_file->fd); + if (ret != 0) + rrd_set_error("closing file: %s", rrd_strerror(errno)); free(rrd_file); rrd_file = NULL; return ret; @@ -457,30 +471,39 @@ off_t rrd_seek( /* Get current position in rrd_file. */ -inline off_t rrd_tell(rrd_file_t *rrd_file) +inline off_t rrd_tell( + rrd_file_t *rrd_file) { return rrd_file->pos; } /* read count bytes into buffer buf, starting at rrd_file->pos. - * Returns the number of bytes read. */ + * Returns the number of bytes read or <0 on error. */ -ssize_t rrd_read( +inline ssize_t rrd_read( rrd_file_t *rrd_file, void *buf, size_t count) { #ifdef HAVE_MMAP - buf = memmove(buf, rrd_file->file_start + rrd_file->pos, count); - rrd_file->pos += count; /* mimmic read() semantics */ - return count; + size_t _cnt = count; + ssize_t _surplus = rrd_file->pos + _cnt - rrd_file->file_len; + if (_surplus > 0) { /* short read */ + _cnt -= _surplus; + } + if (_cnt == 0) + return 0; /* EOF */ + buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, _cnt); + + rrd_file->pos += _cnt; /* mimmic read() semantics */ + return _cnt; #else ssize_t ret; ret = read(rrd_file->fd, buf, count); - //XXX: eventually add generic rrd_set_error(""); here - rrd_file->pos += count; /* mimmic read() semantics */ + if (ret > 0) + rrd_file->pos += ret; /* mimmic read() semantics */ return ret; #endif } @@ -490,16 +513,21 @@ ssize_t rrd_read( * rrd_file->pos of rrd_file->fd. * Returns the number of bytes written. */ -ssize_t rrd_write( +inline ssize_t rrd_write( rrd_file_t *rrd_file, const void *buf, size_t count) { #ifdef HAVE_MMAP - memmove(rrd_file->file_start + rrd_file->pos, buf, count); + memcpy(rrd_file->file_start + rrd_file->pos, buf, count); + rrd_file->pos += count; return count; /* mimmic write() semantics */ #else - return write(rrd_file->fd, buf, count); + ssize_t _sz = write(rrd_file->fd, buf, count); + + if (_sz > 0) + rrd_file->pos += _sz; + return _sz; #endif } @@ -534,10 +562,15 @@ void rrd_init( /* free RRD header data. */ -void rrd_free( +#ifdef HAVE_MMAP +inline void rrd_free( rrd_t UNUSED(*rrd)) { -#ifndef HAVE_MMAP +} +#else +void rrd_free( + rrd_t *rrd) +{ if (atoi(rrd->stat_head->version) < 3) free(rrd->live_head); free(rrd->stat_head); @@ -547,8 +580,8 @@ void rrd_free( free(rrd->pdp_prep); free(rrd->cdp_prep); free(rrd->rrd_value); -#endif } +#endif /* routine used by external libraries to free memory allocated by