* Add utility functions to allocate pointers in variable size chunks.

* Introduce "-m" argument to reduce calls to realloc(). -- kevin

git-svn-id: svn://svn.oetiker.ch/rrdtool/trunk/program@2053 a5681a0c-68f1-0310-ab6d-d61299d08faa

diff --git a/doc/librrd.pod b/doc/librrd.pod
index dfe6f29..0d05f6e 100644 (file)
--- a/doc/librrd.pod
+++ b/doc/librrd.pod
@@ -81,6 +81,16 @@ end of the new C<dest>.  Returns 1 on success, 0 on failure.
     if (!rrd_add_ptr(&arr, &arr_size, elem))
         handle_failure();
 
     if (!rrd_add_ptr(&arr, &arr_size, elem))
         handle_failure();
 

+=item B<rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, size_t *alloc, size_t chunk)>
+
+Like C<rrd_add_ptr>, except the destination is allocated in chunks of
+C<chunk>.  C<alloc> points to the number of entries allocated, whereas
+C<dest_size> points to the number of valid pointers.  If more pointers are
+needed, C<chunk> pointers are allocated and C<alloc> is increased
+accordingly.  C<alloc> must be E<gt>= C<dest_size>.
+
+This method improves performance on hosts with expensive C<realloc()>.
+

 =item B<rrd_add_strdup(char ***dest, size_t *dest_size, char *src)>
 
 Like C<rrd_add_ptr>, except adds a C<strdup> of the source string.
 =item B<rrd_add_strdup(char ***dest, size_t *dest_size, char *src)>
 
 Like C<rrd_add_ptr>, except adds a C<strdup> of the source string.


@@ -91,6 +101,14 @@ Like C<rrd_add_ptr>, except adds a C<strdup> of the source string.
     if (!rrd_add_strdup(&arr, &arr_size, str))
         handle_failure();
 
     if (!rrd_add_strdup(&arr, &arr_size, str))
         handle_failure();
 

+=item B<rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, size_t *alloc, size_t chunk)>
+
+Like C<rrd_add_strdup>, except the destination is allocated in chunks of
+C<chunk>.  C<alloc> points to the number of entries allocated, whereas
+C<dest_size> points to the number of valid pointers.  If more pointers are
+needed, C<chunk> pointers are allocated and C<alloc> is increased
+accordingly.  C<alloc> must be E<gt>= C<dest_size>.
+

 =item B<rrd_free_ptrs(void ***src, size_t *cnt)>
 
 Free an array of pointers allocated by C<rrd_add_ptr> or
 =item B<rrd_free_ptrs(void ***src, size_t *cnt)>
 
 Free an array of pointers allocated by C<rrd_add_ptr> or

diff --git a/doc/rrdcached.pod b/doc/rrdcached.pod
index d6bfec3..57eb655 100644 (file)
--- a/doc/rrdcached.pod
+++ b/doc/rrdcached.pod
@@ -19,6 +19,7 @@ B<rrdcached>
 [-F]
 [-g]
 [B<-b>E<nbsp>I<base_dir>E<nbsp>[B<-B>]]
 [-F]
 [-g]
 [B<-b>E<nbsp>I<base_dir>E<nbsp>[B<-B>]]

+[B<-m>E<nbsp>I<alloc_size>]

 
 =head1 DESCRIPTION
 
 
 =head1 DESCRIPTION
 


@@ -233,6 +234,14 @@ Only permit writes into the base directory specified in B<-b> (and any
 sub-directories).  This does B<NOT> detect symbolic links.  Paths
 containing C<../> will also be blocked.
 
 sub-directories).  This does B<NOT> detect symbolic links.  Paths
 containing C<../> will also be blocked.
 

+=item B<-m> I<alloc_size>
+
+Allocate value pointers in chunks of I<alloc_size>.  This may improve CPU
+utilization on machines with slow C<realloc()> implementations, in
+exchange for slightly higher memory utilization.  The default isE<nbsp>1.
+Do not set this more than the B<-w> value divided by your average RRD step
+size.
+

 =back
 
 =head1 AFFECTED RRDTOOL COMMANDS
 =back
 
 =head1 AFFECTED RRDTOOL COMMANDS

diff --git a/src/rrd.h b/src/rrd.h
index 31148c8..28a50d7 100644 (file)
--- a/src/rrd.h
+++ b/src/rrd.h
@@ -333,8 +333,12 @@ int       rrd_proc_start_end(
 
     long rrd_random(void);
 
 
     long rrd_random(void);
 

+    int rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src,
+                          size_t *alloc, size_t chunk);

     int rrd_add_ptr(void ***dest, size_t *dest_size, void *src);
     int rrd_add_strdup(char ***dest, size_t *dest_size, char *src);
     int rrd_add_ptr(void ***dest, size_t *dest_size, void *src);
     int rrd_add_strdup(char ***dest, size_t *dest_size, char *src);

+    int rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src,
+                             size_t *alloc, size_t chunk);

     void rrd_free_ptrs(void ***src, size_t *cnt);
 
     int rrd_mkdir_p(const char *pathname, mode_t mode);
     void rrd_free_ptrs(void ***src, size_t *cnt);
 
     int rrd_mkdir_p(const char *pathname, mode_t mode);

diff --git a/src/rrd_daemon.c b/src/rrd_daemon.c
index 128f3c4..9b8d9ee 100644 (file)
--- a/src/rrd_daemon.c
+++ b/src/rrd_daemon.c
@@ -183,7 +183,8 @@ struct cache_item_s
 {
   char *file;
   char **values;
 {
   char *file;
   char **values;

-  size_t values_num;
+  size_t values_num;           /* number of valid pointers */
+  size_t values_alloc;         /* number of allocated pointers */

   time_t last_flush_time;
   time_t last_update_stamp;
 #define CI_FLAGS_IN_TREE  (1<<0)
   time_t last_flush_time;
   time_t last_update_stamp;
 #define CI_FLAGS_IN_TREE  (1<<0)


@@ -260,6 +261,7 @@ static char *config_pid_file = NULL;
 static char *config_base_dir = NULL;
 static size_t _config_base_dir_len = 0;
 static int config_write_base_only = 0;
 static char *config_base_dir = NULL;
 static size_t _config_base_dir_len = 0;
 static int config_write_base_only = 0;

+static size_t config_alloc_chunk = 1;

 
 static listen_socket_t **config_listen_address_list = NULL;
 static size_t config_listen_address_list_len = 0;
 
 static listen_socket_t **config_listen_address_list = NULL;
 static size_t config_listen_address_list_len = 0;


@@ -647,6 +649,7 @@ static void wipe_ci_values(cache_item_t *ci, time_t when)
 {
   ci->values = NULL;
   ci->values_num = 0;
 {
   ci->values = NULL;
   ci->values_num = 0;

+  ci->values_alloc = 0;

 
   ci->last_flush_time = when;
   if (config_write_jitter > 0)
 
   ci->last_flush_time = when;
   if (config_write_jitter > 0)


@@ -1444,7 +1447,8 @@ static int handle_request_update (HANDLER_PROTO) /* {{{ */
     else
       ci->last_update_stamp = stamp;
 
     else
       ci->last_update_stamp = stamp;
 

-    if (!rrd_add_strdup(&ci->values, &ci->values_num, value))
+    if (!rrd_add_strdup_chunk(&ci->values, &ci->values_num, value,
+                              &ci->values_alloc, config_alloc_chunk))

     {
       RRDD_LOG (LOG_ERR, "handle_request_update: rrd_add_strdup failed.");
       continue;
     {
       RRDD_LOG (LOG_ERR, "handle_request_update: rrd_add_strdup failed.");
       continue;


@@ -2776,7 +2780,7 @@ static int read_options (int argc, char **argv) /* {{{ */
   gid_t  socket_group = (gid_t)-1;
   mode_t socket_permissions = (mode_t)-1;
 
   gid_t  socket_group = (gid_t)-1;
   mode_t socket_permissions = (mode_t)-1;
 

-  while ((option = getopt(argc, argv, "gl:s:m:P:f:w:z:t:Bb:p:Fj:h?")) != -1)
+  while ((option = getopt(argc, argv, "gl:s:m:P:f:w:z:t:Bb:p:Fj:m:h?")) != -1)

   {
     switch (option)
     {
   {
     switch (option)
     {


@@ -3084,6 +3088,19 @@ static int read_options (int argc, char **argv) /* {{{ */
       }
       break;
 
       }
       break;
 

+      case 'm':
+      {
+        int temp = atoi(optarg);
+        if (temp > 0)
+          config_alloc_chunk = temp;
+        else
+        {
+          fprintf(stderr, "Invalid allocation size: %s\n", optarg);
+          status = 10;
+        }
+      }
+      break;
+

       case 'h':
       case '?':
         printf ("RRDCacheD %s\n"
       case 'h':
       case '?':
         printf ("RRDCacheD %s\n"

diff --git a/src/rrd_utils.c b/src/rrd_utils.c
index 3936cff..6853c66 100644 (file)
--- a/src/rrd_utils.c
+++ b/src/rrd_utils.c
@@ -50,29 +50,53 @@ long rrd_random(void)
     return random();
 }
 
     return random();
 }
 

-/* rrd_add_ptr: add a pointer to a dynamically sized array of pointers,
- * realloc as necessary.  returns 1 on success, 0 on failure.
+/* rrd_add_ptr_chunk: add a pointer to a dynamically sized array of
+ * pointers, realloc as necessary in multiples of "chunk".
+ *
+ * "alloc" is the number of pointers allocated
+ * "dest_size" is the number of valid pointers
+ *
+ * returns 1 on success, 0 on failure.

  */
 
  */
 

-int rrd_add_ptr(void ***dest, size_t *dest_size, void *src)
+int rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src,
+                      size_t *alloc, size_t chunk)

 {
     void **temp;
 
     assert(dest != NULL);
 {
     void **temp;
 
     assert(dest != NULL);

+    assert(alloc != NULL);
+    assert(*alloc >= *dest_size);

 
 

-    temp = (void **) rrd_realloc(*dest, (*dest_size+1) * sizeof(*dest));
-    if (!temp)
-        return 0;
+    if (*alloc == *dest_size)
+    {
+        temp = (void **) rrd_realloc(*dest, (*alloc+chunk) * sizeof(*dest));
+        if (!temp)
+            return 0;

 
 

-    *dest = temp;
-    temp[*dest_size] = src;
+        *dest = temp;
+        *alloc += chunk;
+    }
+
+    (*dest)[*dest_size] = src;

     (*dest_size)++;
 
     return 1;
 }
 
     (*dest_size)++;
 
     return 1;
 }
 

-/* like rrd_add_ptr, but calls strdup() on a string first. */
-int rrd_add_strdup(char ***dest, size_t *dest_size, char *src)
+/* rrd_add_ptr: add a pointer to a dynamically sized array of pointers,
+ * realloc as necessary.  returns 1 on success, 0 on failure.
+ */
+int rrd_add_ptr(void ***dest, size_t *dest_size, void *src)
+{
+    size_t alloc = *dest_size;
+
+    return rrd_add_ptr_chunk(dest, dest_size, src, &alloc, 1);
+}
+
+/* like rrd_add_ptr_chunk, but calls strdup() on a string first. */
+int rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src,
+                         size_t *alloc, size_t chunk)

 {
     char *dup_src;
     int add_ok;
 {
     char *dup_src;
     int add_ok;


@@ -84,13 +108,20 @@ int rrd_add_strdup(char ***dest, size_t *dest_size, char *src)
     if (!dup_src)
         return 0;
 
     if (!dup_src)
         return 0;
 

-    add_ok = rrd_add_ptr((void ***)dest, dest_size, (void *)dup_src);
+    add_ok = rrd_add_ptr_chunk((void ***)dest, dest_size, (void *)dup_src, alloc, chunk);

     if (!add_ok)
         free(dup_src);
 
     return add_ok;
 }
 
     if (!add_ok)
         free(dup_src);
 
     return add_ok;
 }
 

+int rrd_add_strdup(char ***dest, size_t *dest_size, char *src)
+{
+    size_t alloc = *dest_size;
+
+    return rrd_add_strdup_chunk(dest, dest_size, src, &alloc, 1);
+}
+

 void rrd_free_ptrs(void ***src, size_t *cnt)
 {
     void **sp;
 void rrd_free_ptrs(void ***src, size_t *cnt)
 {
     void **sp;