X-Git-Url: https://git.octo.it/?p=rrdtool.git;a=blobdiff_plain;f=src%2Frrd_open.c;h=5f3723f30cef086d44c6bab4a3c434b1d74e6507;hp=f262413befce5f8e9f561b4d32eef0bfaef8a87f;hb=9d7aa8ddae065b774fe4eb1df76a3242b9db2d53;hpb=b375d439fffdc6985be67b9ca6cdf4d072c704b1 diff --git a/src/rrd_open.c b/src/rrd_open.c index f262413..5f3723f 100644 --- a/src/rrd_open.c +++ b/src/rrd_open.c @@ -1,5 +1,5 @@ /***************************************************************************** - * RRDtool 1.3.2 Copyright by Tobi Oetiker, 1997-2008 + * RRDtool 1.4.3 Copyright by Tobi Oetiker, 1997-2010 ***************************************************************************** * rrd_open.c Open an RRD File ***************************************************************************** @@ -8,8 +8,36 @@ #include "rrd_tool.h" #include "unused.h" + +#ifdef WIN32 +#include +#include +#include +#endif + + +#ifdef HAVE_BROKEN_MS_ASYNC +#include +#include +#endif + #define MEMBLK 8192 +#ifdef WIN32 +#define _LK_UNLCK 0 /* Unlock */ +#define _LK_LOCK 1 /* Lock */ +#define _LK_NBLCK 2 /* Non-blocking lock */ +#define _LK_RLCK 3 /* Lock for read only */ +#define _LK_NBRLCK 4 /* Non-blocking lock for read only */ + + +#define LK_UNLCK _LK_UNLCK +#define LK_LOCK _LK_LOCK +#define LK_NBLCK _LK_NBLCK +#define LK_RLCK _LK_RLCK +#define LK_NBRLCK _LK_NBRLCK +#endif + /* DEBUG 2 prints information obtained via mincore(2) */ #define DEBUG 1 /* do not calculate exact madvise hints but assume 1 page for headers and @@ -34,11 +62,11 @@ #define __rrd_read(dst, dst_t, cnt) { \ size_t wanted = sizeof(dst_t)*(cnt); \ size_t got; \ - if ((dst = malloc(wanted)) == NULL) { \ + if ((dst = (dst_t*)malloc(wanted)) == NULL) { \ rrd_set_error(#dst " malloc"); \ goto out_nullify_head; \ } \ - got = read (rrd_file->fd, dst, wanted); \ + got = read (rrd_simple_file->fd, dst, wanted); \ if (got != wanted) { \ rrd_set_error("short read while reading header " #dst); \ goto out_nullify_head; \ @@ -58,6 +86,10 @@ * positioned to the first cdp in the first rra. * In the error path of rrd_open, only rrd_free(&rrd) has to be called * before returning an error. Do not call rrd_close upon failure of rrd_open. + * If creating a new file, the parameter rrd must be initialised with + * details of the file content. + * If opening an existing file, then use rrd must be initialised by + * rrd_init(rrd) prior to invoking rrd_open */ rrd_file_t *rrd_open( @@ -65,34 +97,50 @@ rrd_file_t *rrd_open( rrd_t *rrd, unsigned rdwr) { + unsigned long ui; int flags = 0; - mode_t mode = S_IRUSR; int version; #ifdef HAVE_MMAP ssize_t _page_size = sysconf(_SC_PAGESIZE); - int mm_prot = PROT_READ, mm_flags = 0; char *data = MAP_FAILED; #endif off_t offset = 0; struct stat statb; rrd_file_t *rrd_file = NULL; - off_t newfile_size = 0; + rrd_simple_file_t *rrd_simple_file = NULL; + size_t newfile_size = 0; + size_t header_len, value_cnt, data_len; - if (rdwr & RRD_CREAT) { - /* yes bad inline signaling alert, we are using the - floatcookie to pass the size in ... only used in resize */ - newfile_size = (off_t) rrd->stat_head->float_cookie; - free(rrd->stat_head); + /* Are we creating a new file? */ + if((rdwr & RRD_CREAT) && (rrd->stat_head != NULL)) + { + header_len = rrd_get_header_size(rrd); + + value_cnt = 0; + for (ui = 0; ui < rrd->stat_head->rra_cnt; ui++) + value_cnt += rrd->stat_head->ds_cnt * rrd->rra_def[ui].row_cnt; + + data_len = sizeof(rrd_value_t) * value_cnt; + + newfile_size = header_len + data_len; } - rrd_init(rrd); - rrd_file = malloc(sizeof(rrd_file_t)); + + rrd_file = (rrd_file_t*)malloc(sizeof(rrd_file_t)); if (rrd_file == NULL) { rrd_set_error("allocating rrd_file descriptor for '%s'", file_name); return NULL; } memset(rrd_file, 0, sizeof(rrd_file_t)); + rrd_file->pvt = malloc(sizeof(rrd_simple_file_t)); + if(rrd_file->pvt == NULL) { + rrd_set_error("allocating rrd_simple_file for '%s'", file_name); + return NULL; + } + memset(rrd_file->pvt, 0, sizeof(rrd_simple_file_t)); + rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt; + #ifdef DEBUG if ((rdwr & (RRD_READONLY | RRD_READWRITE)) == (RRD_READONLY | RRD_READWRITE)) { @@ -101,47 +149,72 @@ rrd_file_t *rrd_open( exit(-1); } #endif + +#ifdef HAVE_MMAP + rrd_simple_file->mm_prot = PROT_READ; + rrd_simple_file->mm_flags = 0; +#endif + if (rdwr & RRD_READONLY) { flags |= O_RDONLY; #ifdef HAVE_MMAP - mm_flags = MAP_PRIVATE; +# if !defined(AIX) + rrd_simple_file->mm_flags = MAP_PRIVATE; +# endif # ifdef MAP_NORESERVE - mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */ + rrd_simple_file->mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */ # endif #endif } else { if (rdwr & RRD_READWRITE) { - mode |= S_IWUSR; flags |= O_RDWR; -#ifdef HAVE_MMAP - mm_flags = MAP_SHARED; - mm_prot |= PROT_WRITE; -#endif +#ifdef HAVE_MMAP + rrd_simple_file->mm_flags = MAP_SHARED; + rrd_simple_file->mm_prot |= PROT_WRITE; +#endif } if (rdwr & RRD_CREAT) { flags |= (O_CREAT | O_TRUNC); } + if (rdwr & RRD_EXCL) { + flags |= O_EXCL; + } } if (rdwr & RRD_READAHEAD) { #ifdef MAP_POPULATE - mm_flags |= MAP_POPULATE; /* populate ptes and data */ + rrd_simple_file->mm_flags |= MAP_POPULATE; /* populate ptes and data */ #endif #if defined MAP_NONBLOCK - mm_flags |= MAP_NONBLOCK; /* just populate ptes */ + rrd_simple_file->mm_flags |= MAP_NONBLOCK; /* just populate ptes */ #endif } #if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__) flags |= O_BINARY; #endif - if ((rrd_file->fd = open(file_name, flags, mode)) < 0) { + if ((rrd_simple_file->fd = open(file_name, flags, 0666)) < 0) { rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno)); goto out_free; } +#ifdef HAVE_MMAP +#ifdef HAVE_BROKEN_MS_ASYNC + if (rdwr & RRD_READWRITE) { + /* some unices, the files mtime does not get update + on msync MS_ASYNC, in order to help them, + we update the the timestamp at this point. + The thing happens pretty 'close' to the open + call so the chances of a race should be minimal. + + Maybe ask your vendor to fix your OS ... */ + utime(file_name,NULL); + } +#endif +#endif + /* Better try to avoid seeks as much as possible. stat may be heavy but * many concurrent seeks are even worse. */ - if (newfile_size == 0 && ((fstat(rrd_file->fd, &statb)) < 0)) { + if (newfile_size == 0 && ((fstat(rrd_simple_file->fd, &statb)) < 0)) { rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno)); goto out_close; } @@ -149,30 +222,35 @@ rrd_file_t *rrd_open( rrd_file->file_len = statb.st_size; } else { rrd_file->file_len = newfile_size; - lseek(rrd_file->fd, newfile_size - 1, SEEK_SET); - write(rrd_file->fd, "\0", 1); /* poke */ - lseek(rrd_file->fd, 0, SEEK_SET); + lseek(rrd_simple_file->fd, newfile_size - 1, SEEK_SET); + if ( write(rrd_simple_file->fd, "\0", 1) == -1){ /* poke */ + rrd_set_error("write '%s': %s", file_name, rrd_strerror(errno)); + goto out_close; + } + lseek(rrd_simple_file->fd, 0, SEEK_SET); } #ifdef HAVE_POSIX_FADVISE /* In general we need no read-ahead when dealing with rrd_files. When we stop reading, it is highly unlikely that we start up again. In this manner we actually save time and diskaccess (and buffer cache). Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */ - posix_fadvise(rrd_file->fd, 0, 0, POSIX_FADV_RANDOM); + posix_fadvise(rrd_simple_file->fd, 0, 0, POSIX_FADV_RANDOM); #endif /* if (rdwr & RRD_READWRITE) { - if (setvbuf((rrd_file->fd),NULL,_IONBF,2)) { + if (setvbuf((rrd_simple_file->fd),NULL,_IONBF,2)) { rrd_set_error("failed to disable the stream buffer\n"); return (-1); } } */ + #ifdef HAVE_MMAP - data = mmap(0, rrd_file->file_len, mm_prot, mm_flags, - rrd_file->fd, offset); + data = mmap(0, rrd_file->file_len, + rrd_simple_file->mm_prot, rrd_simple_file->mm_flags, + rrd_simple_file->fd, offset); /* lets see if the first read worked */ if (data == MAP_FAILED) { @@ -180,7 +258,7 @@ rrd_file_t *rrd_open( rrd_strerror(errno)); goto out_close; } - rrd_file->file_start = data; + rrd_simple_file->file_start = data; if (rdwr & RRD_CREAT) { memset(data, DNAN, newfile_size - 1); goto out_done; @@ -191,12 +269,14 @@ rrd_file_t *rrd_open( #ifdef USE_MADVISE if (rdwr & RRD_COPY) { /* We will read everything in a moment (copying) */ - madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL); + madvise(data, rrd_file->file_len, MADV_WILLNEED ); + madvise(data, rrd_file->file_len, MADV_SEQUENTIAL ); } else { /* We do not need to read anything in for the moment */ madvise(data, rrd_file->file_len, MADV_RANDOM); /* the stat_head will be needed soonish, so hint accordingly */ - madvise(data, sizeof(stat_head_t), MADV_WILLNEED | MADV_RANDOM); + madvise(data, sizeof(stat_head_t), MADV_WILLNEED); + madvise(data, sizeof(stat_head_t), MADV_RANDOM); } #endif @@ -274,12 +354,11 @@ rrd_file_t *rrd_open( { unsigned long row_cnt = 0; - unsigned long i; - for (i=0; istat_head->rra_cnt; i++) - row_cnt += rrd->rra_def[i].row_cnt; + for (ui=0; uistat_head->rra_cnt; ui++) + row_cnt += rrd->rra_def[ui].row_cnt; - off_t correct_len = rrd_file->header_len + + size_t correct_len = rrd_file->header_len + sizeof(rrd_value_t) * row_cnt * rrd->stat_head->ds_cnt; if (correct_len > rrd_file->file_len) @@ -299,8 +378,10 @@ rrd_file_t *rrd_open( if (data != MAP_FAILED) munmap(data, rrd_file->file_len); #endif - close(rrd_file->fd); + + close(rrd_simple_file->fd); out_free: + free(rrd_file->pvt); free(rrd_file); return NULL; } @@ -313,9 +394,11 @@ void mincore_print( rrd_file_t *rrd_file, char *mark) { + rrd_simple_file_t *rrd_simple_file; + rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt; #ifdef HAVE_MMAP /* pretty print blocks in core */ - off_t off; + size_t off; unsigned char *vec; ssize_t _page_size = sysconf(_SC_PAGESIZE); @@ -324,7 +407,7 @@ void mincore_print( vec = malloc(off); if (vec != NULL) { memset(vec, 0, off); - if (mincore(rrd_file->file_start, rrd_file->file_len, vec) == 0) { + if (mincore(rrd_simple_file->file_start, rrd_file->file_len, vec) == 0) { int prev; unsigned is_in = 0, was_in = 0; @@ -351,17 +434,55 @@ void mincore_print( } #endif /* defined DEBUG && DEBUG > 1 */ +/* + * get exclusive lock to whole file. + * lock gets removed when we close the file + * + * returns 0 on success + */ +int rrd_lock( + rrd_file_t *rrd_file) +{ + int rcstat; + rrd_simple_file_t *rrd_simple_file; + rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt; + + { +#if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__) + struct _stat st; + + if (_fstat(rrd_simple_file->fd, &st) == 0) { + rcstat = _locking(rrd_simple_file->fd, _LK_NBLCK, st.st_size); + } else { + rcstat = -1; + } +#else + struct flock lock; + + lock.l_type = F_WRLCK; /* exclusive write lock */ + lock.l_len = 0; /* whole file */ + lock.l_start = 0; /* start of file */ + lock.l_whence = SEEK_SET; /* end of file */ + + rcstat = fcntl(rrd_simple_file->fd, F_SETLK, &lock); +#endif + } + + return (rcstat); +} + /* drop cache except for the header and the active pages */ void rrd_dontneed( rrd_file_t *rrd_file, rrd_t *rrd) { + rrd_simple_file_t *rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt; #if defined USE_MADVISE || defined HAVE_POSIX_FADVISE - off_t dontneed_start; - off_t rra_start; - off_t active_block; - unsigned long i; + size_t dontneed_start; + size_t rra_start; + size_t active_block; + size_t i; ssize_t _page_size = sysconf(_SC_PAGESIZE); if (rrd_file == NULL) { @@ -385,12 +506,12 @@ void rrd_dontneed( * rrd->stat_head->ds_cnt * sizeof(rrd_value_t)); if (active_block > dontneed_start) { #ifdef USE_MADVISE - madvise(rrd_file->file_start + dontneed_start, + madvise(rrd_simple_file->file_start + dontneed_start, active_block - dontneed_start - 1, MADV_DONTNEED); #endif /* in linux at least only fadvise DONTNEED seems to purge pages from cache */ #ifdef HAVE_POSIX_FADVISE - posix_fadvise(rrd_file->fd, dontneed_start, + posix_fadvise(rrd_simple_file->fd, dontneed_start, active_block - dontneed_start - 1, POSIX_FADV_DONTNEED); #endif @@ -410,11 +531,11 @@ void rrd_dontneed( if (dontneed_start < rrd_file->file_len) { #ifdef USE_MADVISE - madvise(rrd_file->file_start + dontneed_start, + madvise(rrd_simple_file->file_start + dontneed_start, rrd_file->file_len - dontneed_start, MADV_DONTNEED); #endif #ifdef HAVE_POSIX_FADVISE - posix_fadvise(rrd_file->fd, dontneed_start, + posix_fadvise(rrd_simple_file->fd, dontneed_start, rrd_file->file_len - dontneed_start, POSIX_FADV_DONTNEED); #endif @@ -423,7 +544,7 @@ void rrd_dontneed( #if defined DEBUG && DEBUG > 1 mincore_print(rrd_file, "after"); #endif -#endif /* without madvise and posix_fadvise ist does not make much sense todo anything */ +#endif /* without madvise and posix_fadvise it does not make much sense todo anything */ } @@ -433,19 +554,22 @@ void rrd_dontneed( int rrd_close( rrd_file_t *rrd_file) { + rrd_simple_file_t *rrd_simple_file; + rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt; int ret; #ifdef HAVE_MMAP - ret = msync(rrd_file->file_start, rrd_file->file_len, MS_ASYNC); + ret = msync(rrd_simple_file->file_start, rrd_file->file_len, MS_ASYNC); if (ret != 0) rrd_set_error("msync rrd_file: %s", rrd_strerror(errno)); - ret = munmap(rrd_file->file_start, rrd_file->file_len); + ret = munmap(rrd_simple_file->file_start, rrd_file->file_len); if (ret != 0) rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno)); #endif - ret = close(rrd_file->fd); + ret = close(rrd_simple_file->fd); if (ret != 0) rrd_set_error("closing file: %s", rrd_strerror(errno)); + free(rrd_file->pvt); free(rrd_file); rrd_file = NULL; return ret; @@ -460,6 +584,8 @@ off_t rrd_seek( int whence) { off_t ret = 0; + rrd_simple_file_t *rrd_simple_file; + rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt; #ifdef HAVE_MMAP if (whence == SEEK_SET) @@ -469,7 +595,7 @@ off_t rrd_seek( else if (whence == SEEK_END) rrd_file->pos = rrd_file->file_len + off; #else - ret = lseek(rrd_file->fd, off, whence); + ret = lseek(rrd_simple_file->fd, off, whence); if (ret < 0) rrd_set_error("lseek: %s", rrd_strerror(errno)); rrd_file->pos = ret; @@ -496,6 +622,7 @@ ssize_t rrd_read( void *buf, size_t count) { + rrd_simple_file_t *rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt; #ifdef HAVE_MMAP size_t _cnt = count; ssize_t _surplus; @@ -510,14 +637,14 @@ ssize_t rrd_read( } if (_cnt == 0) return 0; /* EOF */ - buf = memcpy(buf, rrd_file->file_start + rrd_file->pos, _cnt); + buf = memcpy(buf, rrd_simple_file->file_start + rrd_file->pos, _cnt); rrd_file->pos += _cnt; /* mimmic read() semantics */ return _cnt; #else ssize_t ret; - ret = read(rrd_file->fd, buf, count); + ret = read(rrd_simple_file->fd, buf, count); if (ret > 0) rrd_file->pos += ret; /* mimmic read() semantics */ return ret; @@ -526,7 +653,7 @@ ssize_t rrd_read( /* Write count bytes from buffer buf to the current position - * rrd_file->pos of rrd_file->fd. + * rrd_file->pos of rrd_simple_file->fd. * Returns the number of bytes written or <0 on error. */ ssize_t rrd_write( @@ -534,16 +661,24 @@ ssize_t rrd_write( const void *buf, size_t count) { + rrd_simple_file_t *rrd_simple_file = (rrd_simple_file_t *)rrd_file->pvt; #ifdef HAVE_MMAP + size_t old_size = rrd_file->file_len; if (count == 0) return 0; if (buf == NULL) return -1; /* EINVAL */ - memcpy(rrd_file->file_start + rrd_file->pos, buf, count); + + if((rrd_file->pos + count) > old_size) + { + rrd_set_error("attempting to write beyond end of file"); + return -1; + } + memcpy(rrd_simple_file->file_start + rrd_file->pos, buf, count); rrd_file->pos += count; return count; /* mimmic write() semantics */ #else - ssize_t _sz = write(rrd_file->fd, buf, count); + ssize_t _sz = write(rrd_simple_file->fd, buf, count); if (_sz > 0) rrd_file->pos += _sz; @@ -552,18 +687,13 @@ ssize_t rrd_write( } -/* flush all data pending to be written to FD. */ - +/* this is a leftover from the old days, it serves no purpose + and is therefore turned into a no-op */ void rrd_flush( - rrd_file_t *rrd_file) + rrd_file_t UNUSED(*rrd_file)) { - if (fdatasync(rrd_file->fd) != 0) { - rrd_set_error("flushing fd %d: %s", rrd_file->fd, - rrd_strerror(errno)); - } } - /* Initialize RRD header. */ void rrd_init( @@ -615,3 +745,32 @@ void rrd_freemem( { free(mem); } + +/* + * rra_update informs us about the RRAs being updated + * The low level storage API may use this information for + * aligning RRAs within stripes, or other performance enhancements + */ +void rrd_notify_row( + rrd_file_t UNUSED(*rrd_file), + int UNUSED(rra_idx), + unsigned long UNUSED(rra_row), + time_t UNUSED(rra_time)) +{ +} + +/* + * This function is called when creating a new RRD + * The storage implementation can use this opportunity to select + * a sensible starting row within the file. + * The default implementation is random, to ensure that all RRAs + * don't change to a new disk block at the same time + */ +unsigned long rrd_select_initial_row( + rrd_file_t UNUSED(*rrd_file), + int UNUSED(rra_idx), + rra_def_t *rra + ) +{ + return rrd_random() % rra->row_cnt; +}