Turbostat: Re-order code and add comments
[collectd.git] / src / turbostat.c
index 8c7af14..f7b0b1d 100644 (file)
 
 #define PLUGIN_NAME "turbostat"
 
-static const char *proc_stat = "/proc/stat";
-static unsigned int skip_c0;
-static unsigned int skip_c1;
+/*
+ * This tool uses the Model-Specific Registers (MSRs) present on Intel processors.
+ * The general description each of these registers, depending on the architecture,
+ * can be found in the IntelĀ® 64 and IA-32 Architectures Software Developer Manual,
+ * Volume 3 Chapter 35.
+ */
+
+/*
+ * If set, aperf_mperf_unstable disables a/mperf based stats.
+ * This includes: C0 & C1 states, frequency
+ *
+ * This value is automatically set if mperf or aperf go backward
+ */
+static _Bool aperf_mperf_unstable;
+
+/*
+ * Bitmask of the list of core C states supported by the processor.
+ * Currently supported C-states (by this plugin): 3, 6, 7
+ */
 static unsigned int do_core_cstate;
+
+/*
+ * Bitmask of the list of pacages C states supported by the processor.
+ * Currently supported C-states (by this plugin): 2, 3, 6, 7, 8, 9, 10
+ */
 static unsigned int do_pkg_cstate;
-static unsigned int do_rapl;
-static unsigned int do_dts;
-static unsigned int do_ptm;
+
+/*
+ * Boolean indicating if the processor supports 'Digital temperature sensor'
+ * This feature enables the monitoring of the temperature of each core
+ *
+ * This feature has two limitations:
+ *  - if MSR_IA32_TEMPERATURE_TARGET is not supported, the absolute temperature might be wrong
+ *  - Temperatures above the tcc_activation_temp are not recorded
+ */
+static _Bool do_dts;
+
+/*
+ * Boolean indicating if the processor supports 'Package thermal management'
+ * This feature allows the monitoring of the temperature of each package
+ *
+ * This feature has two limitations:
+ *  - if MSR_IA32_TEMPERATURE_TARGET is not supported, the absolute temperature might be wrong
+ *  - Temperatures above the tcc_activation_temp are not recorded
+ */
+static _Bool do_ptm;
+
+/*
+ * Thermal Control Circuit Activation Temperature as configured by the user.
+ * This override the automated detection via MSR_IA32_TEMPERATURE_TARGET
+ * and should only be used if the automated detection fails.
+ */
 static unsigned int tcc_activation_temp;
-static unsigned int tcc_activation_temp_override;
+
+static unsigned int do_rapl;
 static double rapl_energy_units;
 
 #define RAPL_PKG               (1 << 0)
@@ -100,11 +145,8 @@ static double rapl_energy_units;
                                        /* 0x642 MSR_PP1_POLICY */
 #define        TJMAX_DEFAULT   100
 
-int aperf_mperf_unstable;
-int backwards_count;
-
-cpu_set_t *cpu_present_set, *cpu_affinity_set;
-size_t cpu_present_setsize, cpu_affinity_setsize;
+cpu_set_t *cpu_present_set, *cpu_affinity_set, *cpu_saved_affinity_set;
+size_t cpu_present_setsize, cpu_affinity_setsize, cpu_saved_affinity_setsize;
 
 struct thread_data {
        unsigned long long tsc;
@@ -116,7 +158,7 @@ struct thread_data {
        unsigned int flags;
 #define CPU_IS_FIRST_THREAD_IN_CORE    0x2
 #define CPU_IS_FIRST_CORE_IN_PACKAGE   0x4
-} *thread_even, *thread_odd;
+} *thread_delta, *thread_even, *thread_odd;
 
 struct core_data {
        unsigned long long c3;
@@ -124,7 +166,7 @@ struct core_data {
        unsigned long long c7;
        unsigned int core_temp_c;
        unsigned int core_id;
-} *core_even, *core_odd;
+} *core_delta, *core_even, *core_odd;
 
 struct pkg_data {
        unsigned long long pc2;
@@ -141,10 +183,11 @@ struct pkg_data {
        unsigned int energy_gfx;        /* MSR_PP1_ENERGY_STATUS */
        unsigned int rapl_pkg_perf_status;      /* MSR_PKG_PERF_STATUS */
        unsigned int rapl_dram_perf_status;     /* MSR_DRAM_PERF_STATUS */
+       unsigned int tcc_activation_temp;
        unsigned int pkg_temp_c;
+} *package_delta, *package_even, *package_odd;
 
-} *package_even, *package_odd;
-
+#define DELTA_COUNTERS thread_delta, core_delta, package_delta
 #define ODD_COUNTERS thread_odd, core_odd, package_odd
 #define EVEN_COUNTERS thread_even, core_even, package_even
 static _Bool is_even = 1;
@@ -174,6 +217,7 @@ struct timeval tv_even, tv_odd, tv_delta;
 enum return_values {
        OK = 0,
        ERR_CPU_MIGRATE,
+       ERR_CPU_SAVE_SCHED_AFFINITY,
        ERR_MSR_IA32_APERF,
        ERR_MSR_IA32_MPERF,
        ERR_MSR_SMI_COUNT,
@@ -211,70 +255,35 @@ enum return_values {
        UNSUPPORTED_CPU,
 };
 
-static int setup_all_buffers(void);
 
-static int
-cpu_is_not_present(int cpu)
-{
-       return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set);
-}
+/*****************************
+ *  MSR Manipulation helpers *
+ *****************************/
+
 /*
- * run func(thread, core, package) in topology order
- * skip non-present cpus
+ * Open a MSR device for reading
+ * Can change the scheduling affinity of the current process if multiple_read is 1
  */
-
 static int __attribute__((warn_unused_result))
-for_all_cpus(int (func)(struct thread_data *, struct core_data *, struct pkg_data *),
-       struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base)
-{
-       int retval, pkg_no, core_no, thread_no;
-
-       for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
-               for (core_no = 0; core_no < topo.num_cores_per_pkg; ++core_no) {
-                       for (thread_no = 0; thread_no <
-                               topo.num_threads_per_core; ++thread_no) {
-                               struct thread_data *t;
-                               struct core_data *c;
-                               struct pkg_data *p;
-
-                               t = GET_THREAD(thread_base, thread_no, core_no, pkg_no);
-
-                               if (cpu_is_not_present(t->cpu_id))
-                                       continue;
-
-                               c = GET_CORE(core_base, core_no, pkg_no);
-                               p = GET_PKG(pkg_base, pkg_no);
-
-                               retval = func(t, c, p);
-                               if (retval)
-                                       return retval;
-                       }
-               }
-       }
-       return 0;
-}
-
-static int __attribute__((warn_unused_result))
-cpu_migrate(int cpu)
-{
-       CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
-       CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set);
-       if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1)
-               return -ERR_CPU_MIGRATE;
-       else
-               return 0;
-}
-
-static int __attribute__((warn_unused_result))
-open_msr(int cpu)
+open_msr(int cpu, _Bool multiple_read)
 {
        char pathname[32];
        int fd;
 
-       /* FIXME: Do we really need this, why? */
-       if (cpu_migrate(cpu)) {
-               ERROR("Could not migrate to CPU %d", cpu);
-               return -ERR_CPU_MIGRATE;
+       /*
+        * If we need to do multiple read, let's migrate to the CPU
+        * Otherwise, we would lose time calling functions on another CPU
+        *
+        * If we are not yet initialized (cpu_affinity_setsize = 0),
+        * we need to skip this optimisation.
+        */
+       if (multiple_read && cpu_affinity_setsize) {
+               CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
+               CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set);
+               if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1) {
+                       ERROR("Could not migrate to CPU %d", cpu);
+                       return -ERR_CPU_MIGRATE;
+               }
        }
 
        ssnprintf(pathname, sizeof(pathname), "/dev/cpu/%d/msr", cpu);
@@ -284,6 +293,9 @@ open_msr(int cpu)
        return fd;
 }
 
+/*
+ * Read a single MSR from an open file descriptor
+ */
 static int __attribute__((warn_unused_result))
 read_msr(int fd, off_t offset, unsigned long long *msr)
 {
@@ -298,13 +310,17 @@ read_msr(int fd, off_t offset, unsigned long long *msr)
        return 0;
 }
 
+/*
+ * Open a MSR device for reading, read the value asked for and close it.
+ * This call will not affect the scheduling affinity of this thread.
+ */
 static int __attribute__((warn_unused_result))
 get_msr(int cpu, off_t offset, unsigned long long *msr)
 {
        ssize_t retval;
        int fd;
 
-       fd = open_msr(cpu);
+       fd = open_msr(cpu, 0);
        if (fd < 0)
                return fd;
        retval = read_msr(fd, offset, msr);
@@ -312,66 +328,196 @@ get_msr(int cpu, off_t offset, unsigned long long *msr)
        return retval;
 }
 
-#define DELTA_WRAP32(new, old)                 \
-       if (new > old) {                        \
-               old = new - old;                \
-       } else {                                \
-               old = 0x100000000 + new - old;  \
+
+/********************************
+ * Raw data acquisition (1 CPU) *
+ ********************************/
+
+/*
+ * Read every data avalaible for a single CPU
+ *
+ * Core data is shared for all threads in one core: extracted only for the first thread
+ * Package data is shared for all core in one package: extracted only for the first thread of the first core
+ *
+ * Side effect: migrates to the targeted CPU
+ */
+static int __attribute__((warn_unused_result))
+get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p)
+{
+       int cpu = t->cpu_id;
+       unsigned long long msr;
+       int msr_fd;
+       int retval = 0;
+
+       msr_fd = open_msr(cpu, 1);
+       if (msr_fd < 0)
+               return msr_fd;
+
+#define READ_MSR(msr, dst)                     \
+do {                                           \
+       if (read_msr(msr_fd, msr, dst)) {       \
+               retval = -ERR_##msr;            \
+               goto out;                       \
+       }                                       \
+} while (0)
+
+       READ_MSR(MSR_IA32_TSC, &t->tsc);
+
+       READ_MSR(MSR_IA32_APERF, &t->aperf);
+       READ_MSR(MSR_IA32_MPERF, &t->mperf);
+
+       READ_MSR(MSR_SMI_COUNT, &msr);
+       t->smi_count = msr & 0xFFFFFFFF;
+
+       /* collect core counters only for 1st thread in core */
+       if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE)) {
+               retval = 0;
+               goto out;
        }
 
-static void
-delta_package(struct pkg_data *new, struct pkg_data *old)
+       if (do_core_cstate & (1 << 3))
+               READ_MSR(MSR_CORE_C3_RESIDENCY, &c->c3);
+       if (do_core_cstate & (1 << 6))
+               READ_MSR(MSR_CORE_C6_RESIDENCY, &c->c6);
+       if (do_core_cstate & (1 << 7))
+               READ_MSR(MSR_CORE_C7_RESIDENCY, &c->c7);
+
+       if (do_dts) {
+               READ_MSR(MSR_IA32_THERM_STATUS, &msr);
+               c->core_temp_c = p->tcc_activation_temp - ((msr >> 16) & 0x7F);
+       }
+
+       /* collect package counters only for 1st core in package */
+       if (!(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) {
+               retval = 0;
+               goto out;
+       }
+
+        if (do_pkg_cstate & (1 << 2))
+                READ_MSR(MSR_PKG_C2_RESIDENCY, &p->pc2);
+        if (do_pkg_cstate & (1 << 3))
+                READ_MSR(MSR_PKG_C3_RESIDENCY, &p->pc3);
+        if (do_pkg_cstate & (1 << 6))
+                READ_MSR(MSR_PKG_C6_RESIDENCY, &p->pc6);
+        if (do_pkg_cstate & (1 << 7))
+                READ_MSR(MSR_PKG_C7_RESIDENCY, &p->pc7);
+        if (do_pkg_cstate & (1 << 8))
+                READ_MSR(MSR_PKG_C8_RESIDENCY, &p->pc8);
+        if (do_pkg_cstate & (1 << 9))
+                READ_MSR(MSR_PKG_C9_RESIDENCY, &p->pc9);
+        if (do_pkg_cstate & (1 << 10))
+                READ_MSR(MSR_PKG_C10_RESIDENCY, &p->pc10);
+
+       if (do_rapl & RAPL_PKG) {
+               READ_MSR(MSR_PKG_ENERGY_STATUS, &msr);
+               p->energy_pkg = msr & 0xFFFFFFFF;
+       }
+       if (do_rapl & RAPL_CORES) {
+               READ_MSR(MSR_PP0_ENERGY_STATUS, &msr);
+               p->energy_cores = msr & 0xFFFFFFFF;
+       }
+       if (do_rapl & RAPL_DRAM) {
+               READ_MSR(MSR_DRAM_ENERGY_STATUS, &msr);
+               p->energy_dram = msr & 0xFFFFFFFF;
+       }
+       if (do_rapl & RAPL_GFX) {
+               READ_MSR(MSR_PP1_ENERGY_STATUS, &msr);
+               p->energy_gfx = msr & 0xFFFFFFFF;
+       }
+       if (do_rapl & RAPL_PKG_PERF_STATUS) {
+               READ_MSR(MSR_PKG_PERF_STATUS, &msr);
+               p->rapl_pkg_perf_status = msr & 0xFFFFFFFF;
+       }
+       if (do_rapl & RAPL_DRAM_PERF_STATUS) {
+               READ_MSR(MSR_DRAM_PERF_STATUS, &msr);
+               p->rapl_dram_perf_status = msr & 0xFFFFFFFF;
+       }
+       if (do_ptm) {
+               READ_MSR(MSR_IA32_PACKAGE_THERM_STATUS, &msr);
+               p->pkg_temp_c = p->tcc_activation_temp - ((msr >> 16) & 0x7F);
+       }
+
+out:
+       close(msr_fd);
+       return retval;
+}
+
+
+/**********************************
+ * Evaluating the changes (1 CPU) *
+ **********************************/
+
+/*
+ * Do delta = new - old on 32bits cyclique intergers
+ */
+#define DELTA_WRAP32(delta, new, old)                  \
+       if (new > old) {                                \
+               delta = new - old;                      \
+       } else {                                        \
+               delta = 0x100000000 + new - old;        \
+       }
+
+/*
+ * Extract the evolution old->new in delta at a package level
+ * (some are not new-delta, e.g. temperature)
+ */
+static inline void
+delta_package(struct pkg_data *delta, const struct pkg_data *new, const struct pkg_data *old)
 {
-       old->pc2 = new->pc2 - old->pc2;
-       old->pc3 = new->pc3 - old->pc3;
-       old->pc6 = new->pc6 - old->pc6;
-       old->pc7 = new->pc7 - old->pc7;
-       old->pc8 = new->pc8 - old->pc8;
-       old->pc9 = new->pc9 - old->pc9;
-       old->pc10 = new->pc10 - old->pc10;
-       old->pkg_temp_c = new->pkg_temp_c;
-
-       DELTA_WRAP32(new->energy_pkg, old->energy_pkg);
-       DELTA_WRAP32(new->energy_cores, old->energy_cores);
-       DELTA_WRAP32(new->energy_gfx, old->energy_gfx);
-       DELTA_WRAP32(new->energy_dram, old->energy_dram);
-       DELTA_WRAP32(new->rapl_pkg_perf_status, old->rapl_pkg_perf_status);
-       DELTA_WRAP32(new->rapl_dram_perf_status, old->rapl_dram_perf_status);
+       delta->pc2 = new->pc2 - old->pc2;
+       delta->pc3 = new->pc3 - old->pc3;
+       delta->pc6 = new->pc6 - old->pc6;
+       delta->pc7 = new->pc7 - old->pc7;
+       delta->pc8 = new->pc8 - old->pc8;
+       delta->pc9 = new->pc9 - old->pc9;
+       delta->pc10 = new->pc10 - old->pc10;
+       delta->pkg_temp_c = new->pkg_temp_c;
+
+       DELTA_WRAP32(delta->energy_pkg, new->energy_pkg, old->energy_pkg);
+       DELTA_WRAP32(delta->energy_cores, new->energy_cores, old->energy_cores);
+       DELTA_WRAP32(delta->energy_gfx, new->energy_gfx, old->energy_gfx);
+       DELTA_WRAP32(delta->energy_dram, new->energy_dram, old->energy_dram);
+       DELTA_WRAP32(delta->rapl_pkg_perf_status, new->rapl_pkg_perf_status, old->rapl_pkg_perf_status);
+       DELTA_WRAP32(delta->rapl_dram_perf_status, new->rapl_dram_perf_status, old->rapl_dram_perf_status);
 }
 
-static void
-delta_core(struct core_data *new, struct core_data *old)
+/*
+ * Extract the evolution old->new in delta at a core level
+ * (some are not new-delta, e.g. temperature)
+ */
+static inline void
+delta_core(struct core_data *delta, const struct core_data *new, const struct core_data *old)
 {
-       old->c3 = new->c3 - old->c3;
-       old->c6 = new->c6 - old->c6;
-       old->c7 = new->c7 - old->c7;
-       old->core_temp_c = new->core_temp_c;
+       delta->c3 = new->c3 - old->c3;
+       delta->c6 = new->c6 - old->c6;
+       delta->c7 = new->c7 - old->c7;
+       delta->core_temp_c = new->core_temp_c;
 }
 
 /*
- * old = new - old
+ * Extract the evolution old->new in delta at a package level
+ * core_delta is required for c1 estimation (tsc - c0 - all core cstates)
  */
-static int __attribute__((warn_unused_result))
-delta_thread(struct thread_data *new, struct thread_data *old,
-       struct core_data *core_delta)
+static inline int __attribute__((warn_unused_result))
+delta_thread(struct thread_data *delta, const struct thread_data *new, const struct thread_data *old,
+       const struct core_data *core_delta)
 {
-       old->tsc = new->tsc - old->tsc;
+       delta->tsc = new->tsc - old->tsc;
 
        /* check for TSC < 1 Mcycles over interval */
-       if (old->tsc < (1000 * 1000)) {
+       if (delta->tsc < (1000 * 1000)) {
                WARNING("Insanely slow TSC rate, TSC stops in idle? ");
                WARNING("You can disable all c-states by booting with \"idle=poll\" ");
                WARNING("or just the deep ones with \"processor.max_cstate=1\"");
                return -1;
        }
 
-       old->c1 = new->c1 - old->c1;
+       delta->c1 = new->c1 - old->c1;
 
        if ((new->aperf > old->aperf) && (new->mperf > old->mperf)) {
-               old->aperf = new->aperf - old->aperf;
-               old->mperf = new->mperf - old->mperf;
+               delta->aperf = new->aperf - old->aperf;
+               delta->mperf = new->mperf - old->mperf;
        } else {
-
                if (!aperf_mperf_unstable) {
                        WARNING(" APERF or MPERF went backwards * ");
                        WARNING("* Frequency results do not cover entire interval *");
@@ -379,167 +525,259 @@ delta_thread(struct thread_data *new, struct thread_data *old,
 
                        aperf_mperf_unstable = 1;
                }
-               /*
-                * mperf delta is likely a huge "positive" number
-                * can not use it for calculating c0 time
-                */
-               skip_c0 = 1;
-               skip_c1 = 1;
        }
 
-
        /*
         * As counter collection is not atomic,
         * it is possible for mperf's non-halted cycles + idle states
         * to exceed TSC's all cycles: show c1 = 0% in that case.
         */
-       if ((old->mperf + core_delta->c3 + core_delta->c6 + core_delta->c7) > old->tsc)
-               old->c1 = 0;
+       if ((delta->mperf + core_delta->c3 + core_delta->c6 + core_delta->c7) > delta->tsc)
+               delta->c1 = 0;
        else {
                /* normal case, derive c1 */
-               old->c1 = old->tsc - old->mperf - core_delta->c3
+               delta->c1 = delta->tsc - delta->mperf - core_delta->c3
                        - core_delta->c6 - core_delta->c7;
        }
 
-       if (old->mperf == 0) {
+       if (delta->mperf == 0) {
                WARNING("cpu%d MPERF 0!", old->cpu_id);
-               old->mperf = 1; /* divide by 0 protection */
+               delta->mperf = 1;       /* divide by 0 protection */
        }
 
-       old->smi_count = new->smi_count - old->smi_count;
+       delta->smi_count = new->smi_count - old->smi_count;
+
+       return 0;
+}
+
+/**********************************
+ * Submitting the results (1 CPU) *
+ **********************************/
+
+/*
+ * Submit one gauge value
+ */
+static void
+turbostat_submit (const char *plugin_instance,
+       const char *type, const char *type_instance,
+       gauge_t value)
+{
+       value_list_t vl = VALUE_LIST_INIT;
+       value_t v;
+
+       v.gauge = value;
+       vl.values = &v;
+       vl.values_len = 1;
+       sstrncpy (vl.host, hostname_g, sizeof (vl.host));
+       sstrncpy (vl.plugin, PLUGIN_NAME, sizeof (vl.plugin));
+       if (plugin_instance != NULL)
+               sstrncpy (vl.plugin_instance, plugin_instance, sizeof (vl.plugin_instance));
+       sstrncpy (vl.type, type, sizeof (vl.type));
+       if (type_instance != NULL)
+               sstrncpy (vl.type_instance, type_instance, sizeof (vl.type_instance));
+
+       plugin_dispatch_values (&vl);
+}
+
+/*
+ * Submit every data for a single CPU
+ *
+ * Core data is shared for all threads in one core: submitted only for the first thread
+ * Package data is shared for all core in one package: submitted only for the first thread of the first core
+ */
+static int
+submit_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p)
+{
+       char name[12];
+       double interval_float;
+
+       interval_float = tv_delta.tv_sec + tv_delta.tv_usec/1000000.0;
+
+       ssnprintf(name, sizeof(name), "cpu%02d", t->cpu_id);
+
+       if (!aperf_mperf_unstable)
+               turbostat_submit(name, "percent", "c0", 100.0 * t->mperf/t->tsc);
+       if (!aperf_mperf_unstable)
+               turbostat_submit(name, "percent", "c1", 100.0 * t->c1/t->tsc);
+
+       /* GHz */
+       if ((!aperf_mperf_unstable) || (!(t->aperf > t->tsc || t->mperf > t->tsc)))
+               turbostat_submit(NULL, "frequency", name, 1.0 * t->tsc / 1000000000 * t->aperf / t->mperf / interval_float);
+
+       /* SMI */
+       turbostat_submit(NULL, "current", name, t->smi_count);
+
+       /* submit per-core data only for 1st thread in core */
+       if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE))
+               goto done;
+
+       ssnprintf(name, sizeof(name), "core%02d", c->core_id);
+
+       if (do_core_cstate & (1 << 3))
+               turbostat_submit(name, "percent", "c3", 100.0 * c->c3/t->tsc);
+       if (do_core_cstate & (1 << 6))
+               turbostat_submit(name, "percent", "c6", 100.0 * c->c6/t->tsc);
+       if (do_core_cstate & (1 << 7))
+               turbostat_submit(name, "percent", "c7", 100.0 * c->c7/t->tsc);
+
+       if (do_dts)
+               turbostat_submit(NULL, "temperature", name, c->core_temp_c);
+
+       /* submit per-package data only for 1st core in package */
+       if (!(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE))
+               goto done;
+
+       ssnprintf(name, sizeof(name), "pkg%02d", p->package_id);
+
+       if (do_ptm)
+               turbostat_submit(NULL, "temperature", name, p->pkg_temp_c);
 
+       if (do_pkg_cstate & (1 << 2))
+               turbostat_submit(name, "percent", "pc2", 100.0 * p->pc2/t->tsc);
+       if (do_pkg_cstate & (1 << 3))
+               turbostat_submit(name, "percent", "pc3", 100.0 * p->pc3/t->tsc);
+       if (do_pkg_cstate & (1 << 6))
+               turbostat_submit(name, "percent", "pc6", 100.0 * p->pc6/t->tsc);
+       if (do_pkg_cstate & (1 << 7))
+               turbostat_submit(name, "percent", "pc7", 100.0 * p->pc7/t->tsc);
+       if (do_pkg_cstate & (1 << 8))
+               turbostat_submit(name, "percent", "pc8", 100.0 * p->pc8/t->tsc);
+       if (do_pkg_cstate & (1 << 9))
+               turbostat_submit(name, "percent", "pc9", 100.0 * p->pc9/t->tsc);
+       if (do_pkg_cstate & (1 << 10))
+               turbostat_submit(name, "percent", "pc10", 100.0 * p->pc10/t->tsc);
+
+       if (do_rapl) {
+               if (do_rapl & RAPL_PKG)
+                       turbostat_submit(name, "power", "Pkg_W", p->energy_pkg * rapl_energy_units / interval_float);
+               if (do_rapl & RAPL_CORES)
+                       turbostat_submit(name, "power", "Cor_W", p->energy_cores * rapl_energy_units / interval_float);
+               if (do_rapl & RAPL_GFX)
+                       turbostat_submit(name, "power", "GFX_W", p->energy_gfx * rapl_energy_units / interval_float);
+               if (do_rapl & RAPL_DRAM)
+                       turbostat_submit(name, "power", "RAM_W", p->energy_dram * rapl_energy_units / interval_float);
+       }
+done:
        return 0;
 }
 
+/**********************************
+ * Looping function over all CPUs *
+ **********************************/
+
+/*
+ * Check if a given cpu id is in our compiled list of existing CPUs
+ *
+ * CHECKME: Why do we need this?
+ */
+static int
+cpu_is_not_present(int cpu)
+{
+       return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set);
+}
+
+/*
+ * Loop on all CPUs in topological order
+ *
+ * Skip 'non-present' cpus (CHECKME: Why do we need this?)
+ * Return the error code at the first error or 0
+ */
 static int __attribute__((warn_unused_result))
-delta_cpu(struct thread_data *t, struct core_data *c,
-       struct pkg_data *p, struct thread_data *t2,
-       struct core_data *c2, struct pkg_data *p2)
+for_all_cpus(int (func)(struct thread_data *, struct core_data *, struct pkg_data *),
+       struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base)
 {
-       int ret;
+       int retval, pkg_no, core_no, thread_no;
 
-       /* calculate core delta only for 1st thread in core */
-       if (t->flags & CPU_IS_FIRST_THREAD_IN_CORE)
-               delta_core(c, c2);
+       for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
+               for (core_no = 0; core_no < topo.num_cores_per_pkg; ++core_no) {
+                       for (thread_no = 0; thread_no <
+                               topo.num_threads_per_core; ++thread_no) {
+                               struct thread_data *t;
+                               struct core_data *c;
+                               struct pkg_data *p;
 
-       /* always calculate thread delta */
-       ret = delta_thread(t, t2, c2);  /* c2 is core delta */
-       if (ret != 0)
-               return ret;
+                               t = GET_THREAD(thread_base, thread_no, core_no, pkg_no);
 
-       /* calculate package delta only for 1st core in package */
-       if (t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)
-               delta_package(p, p2);
+                               if (cpu_is_not_present(t->cpu_id))
+                                       continue;
+
+                               c = GET_CORE(core_base, core_no, pkg_no);
+                               p = GET_PKG(pkg_base, pkg_no);
 
+                               retval = func(t, c, p);
+                               if (retval)
+                                       return retval;
+                       }
+               }
+       }
        return 0;
 }
 
 /*
- * get_counters(...)
- * migrate to cpu
- * acquire and record local counters for that cpu
+ * Dedicated loop: Extract every data evolution for all CPU
+ *
+ * Core data is shared for all threads in one core: extracted only for the first thread
+ * Package data is shared for all core in one package: extracted only for the first thread of the first core
  */
 static int __attribute__((warn_unused_result))
-get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p)
+for_all_cpus_delta(const struct thread_data *thread_new_base, const struct core_data *core_new_base, const struct pkg_data *pkg_new_base,
+                  const struct thread_data *thread_old_base, const struct core_data *core_old_base, const struct pkg_data *pkg_old_base)
 {
-       int cpu = t->cpu_id;
-       unsigned long long msr;
-       int msr_fd;
-       int retval = 0;
+       int retval, pkg_no, core_no, thread_no;
 
-       msr_fd = open_msr(cpu);
-       if (msr_fd < 0)
-               return msr_fd;
+       for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
+               for (core_no = 0; core_no < topo.num_cores_per_pkg; ++core_no) {
+                       for (thread_no = 0; thread_no <
+                               topo.num_threads_per_core; ++thread_no) {
+                               struct thread_data *t_delta;
+                               const struct thread_data *t_old, *t_new;
+                               struct core_data *c_delta;
 
-#define READ_MSR(msr, dst)                     \
-do {                                           \
-       if (read_msr(msr_fd, msr, dst)) {       \
-               retval = -ERR_##msr;            \
-               goto out;                       \
-       }                                       \
-} while (0)
+                               /* Get correct pointers for threads */
+                               t_delta = GET_THREAD(thread_delta, thread_no, core_no, pkg_no);
+                               t_new = GET_THREAD(thread_new_base, thread_no, core_no, pkg_no);
+                               t_old = GET_THREAD(thread_old_base, thread_no, core_no, pkg_no);
 
-       READ_MSR(MSR_IA32_TSC, &t->tsc);
+                               /* Skip threads that disappeared */
+                               if (cpu_is_not_present(t_delta->cpu_id))
+                                       continue;
 
-       READ_MSR(MSR_IA32_APERF, &t->aperf);
-       READ_MSR(MSR_IA32_MPERF, &t->mperf);
+                               /* c_delta is always required for delta_thread */
+                               c_delta = GET_CORE(core_delta, core_no, pkg_no);
 
-       READ_MSR(MSR_SMI_COUNT, &msr);
-       t->smi_count = msr & 0xFFFFFFFF;
+                               /* calculate core delta only for 1st thread in core */
+                               if (t_new->flags & CPU_IS_FIRST_THREAD_IN_CORE) {
+                                       const struct core_data *c_old, *c_new;
 
-       /* collect core counters only for 1st thread in core */
-       if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE)) {
-               retval = 0;
-               goto out;
-       }
+                                       c_new = GET_CORE(core_new_base, core_no, pkg_no);
+                                       c_old = GET_CORE(core_old_base, core_no, pkg_no);
 
-       if (do_core_cstate & (1 << 3))
-               READ_MSR(MSR_CORE_C3_RESIDENCY, &c->c3);
-       if (do_core_cstate & (1 << 6))
-               READ_MSR(MSR_CORE_C6_RESIDENCY, &c->c6);
-       if (do_core_cstate & (1 << 7))
-               READ_MSR(MSR_CORE_C7_RESIDENCY, &c->c7);
+                                       delta_core(c_delta, c_new, c_old);
+                               }
 
-       if (do_dts) {
-               READ_MSR(MSR_IA32_THERM_STATUS, &msr);
-               c->core_temp_c = tcc_activation_temp - ((msr >> 16) & 0x7F);
-       }
+                               /* Always calculate thread delta */
+                               retval = delta_thread(t_delta, t_new, t_old, c_delta);
+                               if (retval)
+                                       return retval;
 
-       /* collect package counters only for 1st core in package */
-       if (!(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) {
-               retval = 0;
-               goto out;
-       }
+                               /* calculate package delta only for 1st core in package */
+                               if (t_new->flags & CPU_IS_FIRST_CORE_IN_PACKAGE) {
+                                       struct pkg_data *p_delta;
+                                       const struct pkg_data *p_old, *p_new;
 
-        if (do_pkg_cstate & (1 << 2))
-                READ_MSR(MSR_PKG_C2_RESIDENCY, &p->pc2);
-        if (do_pkg_cstate & (1 << 3))
-                READ_MSR(MSR_PKG_C3_RESIDENCY, &p->pc3);
-        if (do_pkg_cstate & (1 << 6))
-                READ_MSR(MSR_PKG_C6_RESIDENCY, &p->pc6);
-        if (do_pkg_cstate & (1 << 7))
-                READ_MSR(MSR_PKG_C7_RESIDENCY, &p->pc7);
-        if (do_pkg_cstate & (1 << 8))
-                READ_MSR(MSR_PKG_C8_RESIDENCY, &p->pc8);
-        if (do_pkg_cstate & (1 << 9))
-                READ_MSR(MSR_PKG_C9_RESIDENCY, &p->pc9);
-        if (do_pkg_cstate & (1 << 10))
-                READ_MSR(MSR_PKG_C10_RESIDENCY, &p->pc10);
+                                       p_delta = GET_PKG(package_delta, pkg_no);
+                                       p_new = GET_PKG(pkg_new_base, pkg_no);
+                                       p_old = GET_PKG(pkg_old_base, pkg_no);
 
-       if (do_rapl & RAPL_PKG) {
-               READ_MSR(MSR_PKG_ENERGY_STATUS, &msr);
-               p->energy_pkg = msr & 0xFFFFFFFF;
-       }
-       if (do_rapl & RAPL_CORES) {
-               READ_MSR(MSR_PP0_ENERGY_STATUS, &msr);
-               p->energy_cores = msr & 0xFFFFFFFF;
-       }
-       if (do_rapl & RAPL_DRAM) {
-               READ_MSR(MSR_DRAM_ENERGY_STATUS, &msr);
-               p->energy_dram = msr & 0xFFFFFFFF;
-       }
-       if (do_rapl & RAPL_GFX) {
-               READ_MSR(MSR_PP1_ENERGY_STATUS, &msr);
-               p->energy_gfx = msr & 0xFFFFFFFF;
-       }
-       if (do_rapl & RAPL_PKG_PERF_STATUS) {
-               READ_MSR(MSR_PKG_PERF_STATUS, &msr);
-               p->rapl_pkg_perf_status = msr & 0xFFFFFFFF;
-       }
-       if (do_rapl & RAPL_DRAM_PERF_STATUS) {
-               READ_MSR(MSR_DRAM_PERF_STATUS, &msr);
-               p->rapl_dram_perf_status = msr & 0xFFFFFFFF;
-       }
-       if (do_ptm) {
-               READ_MSR(MSR_IA32_PACKAGE_THERM_STATUS, &msr);
-               p->pkg_temp_c = tcc_activation_temp - ((msr >> 16) & 0x7F);
+                                       delta_package(p_delta, p_new, p_old);
+                               }
+                       }
+               }
        }
-
-out:
-       close(msr_fd);
-       return retval;
+       return 0;
 }
 
+
 static void
 free_all_buffers(void)
 {
@@ -554,6 +792,10 @@ free_all_buffers(void)
        cpu_affinity_set = NULL;
        cpu_affinity_setsize = 0;
 
+       CPU_FREE(cpu_saved_affinity_set);
+       cpu_saved_affinity_set = NULL;
+       cpu_saved_affinity_setsize = 0;
+
        free(thread_even);
        free(core_even);
        free(package_even);
@@ -569,6 +811,14 @@ free_all_buffers(void)
        thread_odd = NULL;
        core_odd = NULL;
        package_odd = NULL;
+
+       free(thread_delta);
+       free(core_delta);
+       free(package_delta);
+
+       thread_delta = NULL;
+       core_delta = NULL;
+       package_delta = NULL;
 }
 
 /*
@@ -661,52 +911,6 @@ get_num_ht_siblings(int cpu)
 }
 
 /*
- * run func(thread, core, package) in topology order
- * skip non-present cpus
- */
-
-
-static int __attribute__((warn_unused_result))
-for_all_cpus_2(int (func)(struct thread_data *, struct core_data *,
-       struct pkg_data *, struct thread_data *, struct core_data *,
-       struct pkg_data *), struct thread_data *thread_base,
-       struct core_data *core_base, struct pkg_data *pkg_base,
-       struct thread_data *thread_base2, struct core_data *core_base2,
-       struct pkg_data *pkg_base2)
-{
-       int retval, pkg_no, core_no, thread_no;
-
-       for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
-               for (core_no = 0; core_no < topo.num_cores_per_pkg; ++core_no) {
-                       for (thread_no = 0; thread_no <
-                               topo.num_threads_per_core; ++thread_no) {
-                               struct thread_data *t, *t2;
-                               struct core_data *c, *c2;
-                               struct pkg_data *p, *p2;
-
-                               t = GET_THREAD(thread_base, thread_no, core_no, pkg_no);
-
-                               if (cpu_is_not_present(t->cpu_id))
-                                       continue;
-
-                               t2 = GET_THREAD(thread_base2, thread_no, core_no, pkg_no);
-
-                               c = GET_CORE(core_base, core_no, pkg_no);
-                               c2 = GET_CORE(core_base2, core_no, pkg_no);
-
-                               p = GET_PKG(pkg_base, pkg_no);
-                               p2 = GET_PKG(pkg_base2, pkg_no);
-
-                               retval = func(t, c, p, t2, c2, p2);
-                               if (retval)
-                                       return retval;
-                       }
-               }
-       }
-       return 0;
-}
-
-/*
  * run func(cpu) on every cpu in /proc/stat
  * return max_cpu number
  */
@@ -717,15 +921,16 @@ for_all_proc_cpus(int (func)(int))
        int cpu_num;
        int retval;
 
-       fp = fopen(proc_stat, "r");
+       fp = fopen("/proc/stat", "r");
         if (!fp) {
-                ERROR("%s: open failed", proc_stat);
+                ERROR("Failed to open /proc/stat");
                 return -ERR_CANT_OPEN_FILE;
         }
 
        retval = fscanf(fp, "cpu %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n");
        if (retval != 0) {
-               ERROR("%s: failed to parse format", proc_stat);
+               ERROR("Failed to parse /proc/stat");
+               fclose(fp);
                return -ERR_CANT_READ_PROC_STAT;
        }
 
@@ -765,121 +970,7 @@ mark_cpu_present(int cpu)
 }
 
 
-static void
-turbostat_submit (const char *plugin_instance,
-       const char *type, const char *type_instance,
-       gauge_t value)
-{
-       value_list_t vl = VALUE_LIST_INIT;
-       value_t v;
-
-       v.gauge = value;
-       vl.values = &v;
-       vl.values_len = 1;
-       sstrncpy (vl.host, hostname_g, sizeof (vl.host));
-       sstrncpy (vl.plugin, PLUGIN_NAME, sizeof (vl.plugin));
-       if (plugin_instance != NULL)
-               sstrncpy (vl.plugin_instance, plugin_instance, sizeof (vl.plugin_instance));
-       sstrncpy (vl.type, type, sizeof (vl.type));
-       if (type_instance != NULL)
-               sstrncpy (vl.type_instance, type_instance, sizeof (vl.type_instance));
-
-       plugin_dispatch_values (&vl);
-}
-
-/*
- * column formatting convention & formats
- * package: "pk" 2 columns %2d
- * core: "cor" 3 columns %3d
- * CPU: "CPU" 3 columns %3d
- * Pkg_W: %6.2
- * Cor_W: %6.2
- * GFX_W: %5.2
- * RAM_W: %5.2
- * GHz: "GHz" 3 columns %3.2
- * TSC: "TSC" 3 columns %3.2
- * SMI: "SMI" 4 columns %4d
- * percentage " %pc3" %6.2
- * Perf Status percentage: %5.2
- * "CTMP" 4 columns %4d
- */
-#define NAME_LEN 12
-static int
-submit_counters(struct thread_data *t, struct core_data *c,
-       struct pkg_data *p)
-{
-       char name[NAME_LEN];
-       double interval_float;
-
-       interval_float = tv_delta.tv_sec + tv_delta.tv_usec/1000000.0;
-
-       ssnprintf(name, sizeof(name), "cpu%02d", t->cpu_id);
-
-       if (!skip_c0)
-               turbostat_submit(name, "percent", "c0", 100.0 * t->mperf/t->tsc);
-       if (!skip_c1)
-               turbostat_submit(name, "percent", "c1", 100.0 * t->c1/t->tsc);
-
-       /* GHz */
-       if ((!aperf_mperf_unstable) || (!(t->aperf > t->tsc || t->mperf > t->tsc)))
-               turbostat_submit(NULL, "frequency", name, 1.0 * t->tsc / 1000000000 * t->aperf / t->mperf / interval_float);
-
-       /* SMI */
-       turbostat_submit(NULL, "current", name, t->smi_count);
-
-       /* print per-core data only for 1st thread in core */
-       if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE))
-               goto done;
-
-       ssnprintf(name, sizeof(name), "core%02d", c->core_id);
-
-       if (do_core_cstate & (1 << 3))
-               turbostat_submit(name, "percent", "c3", 100.0 * c->c3/t->tsc);
-       if (do_core_cstate & (1 << 6))
-               turbostat_submit(name, "percent", "c6", 100.0 * c->c6/t->tsc);
-       if (do_core_cstate & (1 << 7))
-               turbostat_submit(name, "percent", "c7", 100.0 * c->c7/t->tsc);
-
-       if (do_dts)
-               turbostat_submit(NULL, "temperature", name, c->core_temp_c);
-
-       /* print per-package data only for 1st core in package */
-       if (!(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE))
-               goto done;
-
-       ssnprintf(name, sizeof(name), "pkg%02d", p->package_id);
-
-       if (do_ptm)
-               turbostat_submit(NULL, "temperature", name, p->pkg_temp_c);
-
-       if (do_pkg_cstate & (1 << 2))
-               turbostat_submit(name, "percent", "pc2", 100.0 * p->pc2/t->tsc);
-       if (do_pkg_cstate & (1 << 3))
-               turbostat_submit(name, "percent", "pc3", 100.0 * p->pc3/t->tsc);
-       if (do_pkg_cstate & (1 << 6))
-               turbostat_submit(name, "percent", "pc6", 100.0 * p->pc6/t->tsc);
-       if (do_pkg_cstate & (1 << 7))
-               turbostat_submit(name, "percent", "pc7", 100.0 * p->pc7/t->tsc);
-       if (do_pkg_cstate & (1 << 8))
-               turbostat_submit(name, "percent", "pc8", 100.0 * p->pc8/t->tsc);
-       if (do_pkg_cstate & (1 << 9))
-               turbostat_submit(name, "percent", "pc9", 100.0 * p->pc9/t->tsc);
-       if (do_pkg_cstate & (1 << 10))
-               turbostat_submit(name, "percent", "pc10", 100.0 * p->pc10/t->tsc);
-
-       if (do_rapl) {
-               if (do_rapl & RAPL_PKG)
-                       turbostat_submit(name, "power", "Pkg_W", p->energy_pkg * rapl_energy_units / interval_float);
-               if (do_rapl & RAPL_CORES)
-                       turbostat_submit(name, "power", "Cor_W", p->energy_cores * rapl_energy_units / interval_float);
-               if (do_rapl & RAPL_GFX)
-                       turbostat_submit(name, "power", "GFX_W", p->energy_gfx * rapl_energy_units / interval_float);
-               if (do_rapl & RAPL_DRAM)
-                       turbostat_submit(name, "power", "RAM_W", p->energy_dram * rapl_energy_units / interval_float);
-       }
-done:
-       return 0;
-}
+static int setup_all_buffers(void);
 
 static int
 turbostat_read(user_data_t * not_used)
@@ -899,37 +990,49 @@ turbostat_read(user_data_t * not_used)
                        return -ERR_CPU_NOT_PRESENT;
        }
 
+       /* Saving the scheduling affinity, as it will be modified by get_counters */
+       if (sched_getaffinity(0, cpu_saved_affinity_setsize, cpu_saved_affinity_set) != 0)
+               return -ERR_CPU_SAVE_SCHED_AFFINITY;
+
        if (!initialized) {
                if ((ret = for_all_cpus(get_counters, EVEN_COUNTERS)) < 0)
-                       return ret;
+                       goto out;
                gettimeofday(&tv_even, (struct timezone *)NULL);
                is_even = 1;
                initialized = 1;
-               return 0;
+               ret = 0;
+               goto out;
        }
 
        if (is_even) {
                if ((ret = for_all_cpus(get_counters, ODD_COUNTERS)) < 0)
-                       return ret;
+                       goto out;
                gettimeofday(&tv_odd, (struct timezone *)NULL);
                is_even = 0;
                timersub(&tv_odd, &tv_even, &tv_delta);
-               if ((ret = for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) < 0)
-                       return ret;
-               if ((ret = for_all_cpus(submit_counters, EVEN_COUNTERS)) < 0)
-                       return ret;
+               if ((ret = for_all_cpus_delta(ODD_COUNTERS, EVEN_COUNTERS)) < 0)
+                       goto out;
+               if ((ret = for_all_cpus(submit_counters, DELTA_COUNTERS)) < 0)
+                       goto out;
        } else {
                if ((ret = for_all_cpus(get_counters, EVEN_COUNTERS)) < 0)
-                       return ret;
+                       goto out;
                gettimeofday(&tv_even, (struct timezone *)NULL);
                is_even = 1;
                timersub(&tv_even, &tv_odd, &tv_delta);
-               if ((ret = for_all_cpus_2(delta_cpu, EVEN_COUNTERS, ODD_COUNTERS)) < 0)
-                       return ret;
-               if ((ret = for_all_cpus(submit_counters, ODD_COUNTERS)) < 0)
-                       return ret;
+               if ((ret = for_all_cpus_delta(EVEN_COUNTERS, ODD_COUNTERS)) < 0)
+                       goto out;
+               if ((ret = for_all_cpus(submit_counters, DELTA_COUNTERS)) < 0)
+                       goto out;
        }
-       return 0;
+       ret = 0;
+out:
+       /*
+        * Let's restore the affinity
+        * This might fail if the number of CPU changed, but we can't do anything in that case..
+        */
+       (void)sched_setaffinity(0, cpu_saved_affinity_setsize, cpu_saved_affinity_set);
+       return ret;
 }
 
 static int __attribute__((warn_unused_result))
@@ -960,7 +1063,7 @@ check_super_user()
  * This is usually equal to tjMax.
  *
  * Older processors do not have this MSR, so there we guess,
- * but also allow cmdline over-ride with -T.
+ * but also allow conficuration over-ride with "TCCActivationTemp".
  *
  * Several MSR temperature values are in units of degrees-C
  * below this value, including the Digital Thermal Sensor (DTS),
@@ -980,29 +1083,27 @@ set_temperature_target(struct thread_data *t, struct core_data *c, struct pkg_da
        if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE) || !(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE))
                return 0;
 
-       if (tcc_activation_temp_override != 0) {
-               tcc_activation_temp = tcc_activation_temp_override;
-               ERROR("cpu%d: Using cmdline TCC Target (%d C)\n",
-                       cpu, tcc_activation_temp);
+       if (tcc_activation_temp != 0) {
+               p->tcc_activation_temp = tcc_activation_temp;
                return 0;
        }
 
        if (get_msr(t->cpu_id, MSR_IA32_TEMPERATURE_TARGET, &msr))
                goto guess;
 
-       target_c_local = (msr >> 16) & 0x7F;
+       target_c_local = (msr >> 16) & 0xFF;
 
-       if (target_c_local < 85 || target_c_local > 127)
+       if (!target_c_local)
                goto guess;
 
-       tcc_activation_temp = target_c_local;
+       p->tcc_activation_temp = target_c_local;
 
        return 0;
 
 guess:
-       tcc_activation_temp = TJMAX_DEFAULT;
-       WARNING("cpu%d: Guessing tjMax %d C, Please use -T to specify",
-               t->cpu_id, tcc_activation_temp);
+       p->tcc_activation_temp = TJMAX_DEFAULT;
+       WARNING("cpu%d: Guessing tjMax %d C, Please use TCCActivationTemp to specify",
+               t->cpu_id, p->tcc_activation_temp);
 
        return 0;
 }
@@ -1144,28 +1245,28 @@ probe_cpu()
                case 0x3A: /* IVB */
                case 0x3E: /* IVB Xeon */
                        do_core_cstate = (1 << 3) | (1 << 6) | (1 << 7);
-                       do_pkg_cstate = (1 << 3) | (1 << 6) | (1 << 7);
+                       do_pkg_cstate = (1 << 2) | (1 << 3) | (1 << 6) | (1 << 7);
                        break;
                /* Haswell Bridge */
                case 0x3C: /* HSW */
                case 0x3F: /* HSW */
                case 0x46: /* HSW */
                        do_core_cstate = (1 << 3) | (1 << 6) | (1 << 7);
-                       do_pkg_cstate = (1 << 3) | (1 << 6) | (1 << 7);
+                       do_pkg_cstate = (1 << 2) | (1 << 3) | (1 << 6) | (1 << 7);
                        break;
                case 0x45: /* HSW */
                        do_core_cstate = (1 << 3) | (1 << 6) | (1 << 7);
-                       do_pkg_cstate = (1 << 3) | (1 << 6) | (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10);
+                       do_pkg_cstate = (1 << 2) | (1 << 3) | (1 << 6) | (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10);
                        break;
                /* Broadwel */
                case 0x4F: /* BDW */
                case 0x56: /* BDX-DE */
                        do_core_cstate = (1 << 3) | (1 << 6) | (1 << 7);
-                       do_pkg_cstate = (1 << 3) | (1 << 6) | (1 << 7);
+                       do_pkg_cstate = (1 << 2) | (1 << 3) | (1 << 6) | (1 << 7);
                        break;
                case 0x3D: /* BDW */
                        do_core_cstate = (1 << 3) | (1 << 6) | (1 << 7);
-                       do_pkg_cstate = (1 << 3) | (1 << 6) | (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10);
+                       do_pkg_cstate = (1 << 2) | (1 << 3) | (1 << 6) | (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10);
                        break;
                default:
                        ERROR("Unsupported CPU");
@@ -1272,6 +1373,19 @@ topology_probe()
 
 
        /*
+        * Allocate and initialize cpu_saved_affinity_set
+        */
+       cpu_saved_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1));
+       if (cpu_saved_affinity_set == NULL) {
+               free(cpus);
+               ERROR("CPU_ALLOC");
+               return -ERR_CPU_ALLOC;
+       }
+       cpu_saved_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
+       CPU_ZERO_S(cpu_saved_affinity_setsize, cpu_saved_affinity_set);
+
+
+       /*
         * For online cpus
         * find max_core_id, max_package_id
         */
@@ -1418,6 +1532,9 @@ initialize_counters(int cpu_id)
        ret = init_counter(ODD_COUNTERS, my_thread_id, my_core_id, my_package_id, cpu_id);
        if (ret < 0)
                return ret;
+       ret = init_counter(DELTA_COUNTERS, my_thread_id, my_core_id, my_package_id, cpu_id);
+       if (ret < 0)
+               return ret;
        return 0;
 }
 
@@ -1435,7 +1552,10 @@ static int setup_all_buffers(void)
        DO_OR_GOTO_ERR(topology_probe());
        DO_OR_GOTO_ERR(allocate_counters(&thread_even, &core_even, &package_even));
        DO_OR_GOTO_ERR(allocate_counters(&thread_odd, &core_odd, &package_odd));
+       DO_OR_GOTO_ERR(allocate_counters(&thread_delta, &core_delta, &package_delta));
        DO_OR_GOTO_ERR(for_all_proc_cpus(initialize_counters));
+       DO_OR_GOTO_ERR(for_all_cpus(set_temperature_target, EVEN_COUNTERS));
+       DO_OR_GOTO_ERR(for_all_cpus(set_temperature_target, ODD_COUNTERS));
 
        allocated = 1;
        return 0;
@@ -1453,7 +1573,6 @@ turbostat_init(void)
        DO_OR_GOTO_ERR(probe_cpu());
        DO_OR_GOTO_ERR(check_dev_msr());
        DO_OR_GOTO_ERR(setup_all_buffers());
-       DO_OR_GOTO_ERR(for_all_cpus(set_temperature_target, EVEN_COUNTERS));
 
        plugin_register_complex_read(NULL, PLUGIN_NAME, turbostat_read, NULL, NULL);
 
@@ -1463,8 +1582,32 @@ err:
        return ret;
 }
 
+static const char *config_keys[] =
+{
+       "TCCActivationTemp",
+};
+static const int config_keys_num = STATIC_ARRAY_SIZE (config_keys);
+
+static int
+turbostat_config(const char *key, const char *value)
+{
+       long unsigned int tmp_val;
+       char *end;
+
+       if (strcasecmp("TCCActivationTemp", key) == 0) {
+               tmp_val = strtoul(value, &end, 0);
+               if (*end != '\0' || tmp_val > UINT_MAX)
+                       return -1;
+               tcc_activation_temp = (unsigned int) tmp_val;
+       } else {
+               return -1;
+       }
+       return 0;
+}
+
 void module_register(void);
 void module_register(void)
 {
        plugin_register_init(PLUGIN_NAME, turbostat_init);
+       plugin_register_config(PLUGIN_NAME, turbostat_config, config_keys, config_keys_num);
 }