X-Git-Url: http://git.lttng.org/?a=blobdiff_plain;f=wrapper%2Ftrace-clock.h;h=7f17ccd4f82fd9bcb51446bfda45e6c778c10079;hb=8f89e22da25c7cc8b82d4b6574631b0632fc37f3;hp=649c93f358d62c9bcede2cd33bd3f890ffa412ab;hpb=a9df1445c3d54e647d9aea248cc1b26bb6f15298;p=lttng-modules.git

diff --git a/wrapper/trace-clock.h b/wrapper/trace-clock.h
index 649c93f3..7f17ccd4 100644
--- a/wrapper/trace-clock.h
+++ b/wrapper/trace-clock.h
@@ -52,68 +52,43 @@ extern struct lttng_trace_clock *lttng_trace_clock;
  * CONFIG_DEBUG_TIMEKEEPING") introduces a buggy ktime_get_mono_fast_ns().
  * This is fixed by patch "timekeeping: Fix __ktime_get_fast_ns() regression".
  */
-#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0) \
-	&& !LTTNG_KERNEL_RANGE(4,8,0, 4,8,1) \
-	&& !LTTNG_KERNEL_RANGE(4,7,4, 4,7,7) \
-	&& !LTTNG_KERNEL_RANGE(4,4,20, 4,4,24) \
-	&& !LTTNG_KERNEL_RANGE(4,1,32, 4,1,34))
-
-DECLARE_PER_CPU(local_t, lttng_last_tsc);
+#if (LTTNG_KERNEL_RANGE(4,8,0, 4,8,2) \
+	|| LTTNG_KERNEL_RANGE(4,7,4, 4,7,8) \
+	|| LTTNG_KERNEL_RANGE(4,4,20, 4,4,25) \
+	|| LTTNG_KERNEL_RANGE(4,1,32, 4,1,35))
+#define LTTNG_CLOCK_NMI_SAFE_BROKEN
+#endif
 
-#if (BITS_PER_LONG == 32)
 /*
- * Fixup "src_now" using the 32 LSB from "last". We need to handle overflow and
- * underflow of the 32nd bit. "last" can be above, below or equal to the 32 LSB
- * of "src_now".
+ * We need clock values to be monotonically increasing per-cpu, which is
+ * not strictly guaranteed by ktime_get_mono_fast_ns(). It is
+ * straightforward to do on architectures with a 64-bit cmpxchg(), but
+ * not so on architectures without 64-bit cmpxchg. For now, only enable
+ * this feature on 64-bit architectures.
  */
-static inline u64 trace_clock_fixup(u64 src_now, u32 last)
-{
-	u64 now;
 
-	now = src_now & 0xFFFFFFFF00000000ULL;
-	now |= (u64) last;
-	/* Detect overflow or underflow between now and last. */
-	if ((src_now & 0x80000000U) && !(last & 0x80000000U)) {
-		/*
-		 * If 32nd bit transitions from 1 to 0, and we move forward in
-		 * time from "now" to "last", then we have an overflow.
-		 */
-		if (((s32) now - (s32) last) < 0)
-			now += 0x0000000100000000ULL;
-	} else if (!(src_now & 0x80000000U) && (last & 0x80000000U)) {
-		/*
-		 * If 32nd bit transitions from 0 to 1, and we move backward in
-		 * time from "now" to "last", then we have an underflow.
-		 */
-		if (((s32) now - (s32) last) > 0)
-			now -= 0x0000000100000000ULL;
-	}
-	return now;
-}
-#else /* #if (BITS_PER_LONG == 32) */
-/*
- * The fixup is pretty easy on 64-bit architectures: "last" is a 64-bit
- * value, so we can use last directly as current time.
- */
-static inline u64 trace_clock_fixup(u64 src_now, u64 last)
-{
-	return last;
-}
-#endif /* #else #if (BITS_PER_LONG == 32) */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0) \
+	&& BITS_PER_LONG == 64 \
+	&& !defined(LTTNG_CLOCK_NMI_SAFE_BROKEN))
+#define LTTNG_USE_NMI_SAFE_CLOCK
+#endif
+
+#ifdef LTTNG_USE_NMI_SAFE_CLOCK
+
+DECLARE_PER_CPU(u64, lttng_last_tsc);
 
 /*
  * Sometimes called with preemption enabled. Can be interrupted.
  */
 static inline u64 trace_clock_monotonic_wrapper(void)
 {
-	u64 now;
-	unsigned long last, result;
-	local_t *last_tsc;
+	u64 now, last, result;
+	u64 *last_tsc_ptr;
 
 	/* Use fast nmi-safe monotonic clock provided by the Linux kernel. */
 	preempt_disable();
-	last_tsc = lttng_this_cpu_ptr(&lttng_last_tsc);
-	last = local_read(last_tsc);
+	last_tsc_ptr = lttng_this_cpu_ptr(&lttng_last_tsc);
+	last = *last_tsc_ptr;
 	/*
 	 * Read "last" before "now". It is not strictly required, but it ensures
 	 * that an interrupt coming in won't artificially trigger a case where
@@ -122,9 +97,9 @@ static inline u64 trace_clock_monotonic_wrapper(void)
 	 */
 	barrier();
 	now = ktime_get_mono_fast_ns();
-	if (((long) now - (long) last) < 0)
-		now = trace_clock_fixup(now, last);
-	result = local_cmpxchg(last_tsc, last, (unsigned long) now);
+	if (U64_MAX / 2 < now - last)
+		now = last;
+	result = cmpxchg64_local(last_tsc_ptr, last, now);
 	preempt_enable();
 	if (result == last) {
 		/* Update done. */
@@ -135,11 +110,11 @@ static inline u64 trace_clock_monotonic_wrapper(void)
 		 * "result", since it has been sampled concurrently with our
 		 * time read, so it should not be far from "now".
 		 */
-		return trace_clock_fixup(now, result);
+		return result;
 	}
 }
 
-#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0)) */
+#else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 static inline u64 trace_clock_monotonic_wrapper(void)
 {
 	ktime_t ktime;
@@ -154,7 +129,7 @@ static inline u64 trace_clock_monotonic_wrapper(void)
 	ktime = ktime_get();
 	return ktime_to_ns(ktime);
 }
-#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0)) */
+#endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 
 static inline u64 trace_clock_read64_monotonic(void)
 {
@@ -181,19 +156,19 @@ static inline const char *trace_clock_description_monotonic(void)
 	return "Monotonic Clock";
 }
 
-#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0))
+#ifdef LTTNG_USE_NMI_SAFE_CLOCK
 static inline int get_trace_clock(void)
 {
 	printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic fast clock, which is NMI-safe.\n");
 	return 0;
 }
-#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0)) */
+#else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 static inline int get_trace_clock(void)
 {
 	printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic clock. NMIs will not be traced.\n");
 	return 0;
 }
-#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0)) */
+#endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 
 static inline void put_trace_clock(void)
 {