[lttng-modules.git] / include / counter / counter-api.h

/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
 *
 * counter/counter-api.h
 *
 * LTTng Counters API, requiring counter/config.h
 *
 * Copyright (C) 2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 */

#ifndef _LTTNG_COUNTER_API_H
#define _LTTNG_COUNTER_API_H

#include <linux/types.h>
#include <linux/percpu.h>
#include <linux/bitops.h>
#include <counter/counter.h>
#include <counter/counter-internal.h>
#include <wrapper/compiler_attributes.h>
#include <wrapper/limits.h>

/*
 * Using unsigned arithmetic because overflow is defined.
 */
static __always_inline int __lttng_counter_add(const struct lib_counter_config *config,
				       enum lib_counter_config_alloc alloc,
				       enum lib_counter_config_sync sync,
				       struct lib_counter *counter,
				       const size_t *dimension_indexes, int64_t v,
				       int64_t *remainder)
{
	size_t index;
	bool overflow = false, underflow = false;
	struct lib_counter_layout *layout;
	int64_t move_sum = 0;

	if (unlikely(lttng_counter_validate_indexes(config, counter, dimension_indexes)))
		return -EOVERFLOW;
	index = lttng_counter_get_index(config, counter, dimension_indexes);

	switch (alloc) {
	case COUNTER_ALLOC_PER_CPU:
		layout = per_cpu_ptr(counter->percpu_counters, smp_processor_id());
		break;
	case COUNTER_ALLOC_GLOBAL:
		layout = &counter->global_counters;
		break;
	default:
		return -EINVAL;
	}

	switch (config->counter_size) {
	case COUNTER_SIZE_8_BIT:
	{
		int8_t *int_p = (int8_t *) layout->counters + index;
		int8_t old, n, res;
		int8_t global_sum_step = counter->global_sum_step.s8;

		res = *int_p;
		switch (alloc) {
		case COUNTER_ALLOC_PER_CPU:
		{
			do {
				move_sum = 0;
				old = res;
				n = (int8_t) ((uint8_t) old + (uint8_t) v);
				if (unlikely(global_sum_step)) {
					if (unlikely(n > (int8_t) global_sum_step))
						move_sum = (int8_t) global_sum_step / 2;
					else if (unlikely(n < -(int8_t) global_sum_step))
						move_sum = -((int8_t) global_sum_step / 2);
					n -= move_sum;
				}
				res = cmpxchg_local(int_p, old, n);
			} while (old != res);
			break;
		}
		case COUNTER_ALLOC_GLOBAL:
		{
			do {
				old = res;
				n = (int8_t) ((uint8_t) old + (uint8_t) v);
				res = cmpxchg(int_p, old, n);
			} while (old != res);
			break;
		}
		default:
			return -EINVAL;
		}
		if (v > 0 && (v >= U8_MAX || n < old))
			overflow = true;
		else if (v < 0 && (v <= -(s64) U8_MAX || n > old))
			underflow = true;
		break;
	}
	case COUNTER_SIZE_16_BIT:
	{
		int16_t *int_p = (int16_t *) layout->counters + index;
		int16_t old, n, res;
		int16_t global_sum_step = counter->global_sum_step.s16;

		res = *int_p;
		switch (alloc) {
		case COUNTER_ALLOC_PER_CPU:
		{
			do {
				move_sum = 0;
				old = res;
				n = (int16_t) ((uint16_t) old + (uint16_t) v);
				if (unlikely(global_sum_step)) {
					if (unlikely(n > (int16_t) global_sum_step))
						move_sum = (int16_t) global_sum_step / 2;
					else if (unlikely(n < -(int16_t) global_sum_step))
						move_sum = -((int16_t) global_sum_step / 2);
					n -= move_sum;
				}
				res = cmpxchg_local(int_p, old, n);
			} while (old != res);
			break;
		}
		case COUNTER_ALLOC_GLOBAL:
		{
			do {
				old = res;
				n = (int16_t) ((uint16_t) old + (uint16_t) v);
				res = cmpxchg(int_p, old, n);
			} while (old != res);
			break;
		}
		default:
			return -EINVAL;
		}
		if (v > 0 && (v >= U16_MAX || n < old))
			overflow = true;
		else if (v < 0 && (v <= -(s64) U16_MAX || n > old))
			underflow = true;
		break;
	}
	case COUNTER_SIZE_32_BIT:
	{
		int32_t *int_p = (int32_t *) layout->counters + index;
		int32_t old, n, res;
		int32_t global_sum_step = counter->global_sum_step.s32;

		res = *int_p;
		switch (alloc) {
		case COUNTER_ALLOC_PER_CPU:
		{
			do {
				move_sum = 0;
				old = res;
				n = (int32_t) ((uint32_t) old + (uint32_t) v);
				if (unlikely(global_sum_step)) {
					if (unlikely(n > (int32_t) global_sum_step))
						move_sum = (int32_t) global_sum_step / 2;
					else if (unlikely(n < -(int32_t) global_sum_step))
						move_sum = -((int32_t) global_sum_step / 2);
					n -= move_sum;
				}
				res = cmpxchg_local(int_p, old, n);
			} while (old != res);
			break;
		}
		case COUNTER_ALLOC_GLOBAL:
		{
			do {
				old = res;
				n = (int32_t) ((uint32_t) old + (uint32_t) v);
				res = cmpxchg(int_p, old, n);
			} while (old != res);
			break;
		}
		default:
			return -EINVAL;
		}
		if (v > 0 && (v >= U32_MAX || n < old))
			overflow = true;
		else if (v < 0 && (v <= -(s64) U32_MAX || n > old))
			underflow = true;
		break;
	}
#if BITS_PER_LONG == 64
	case COUNTER_SIZE_64_BIT:
	{
		int64_t *int_p = (int64_t *) layout->counters + index;
		int64_t old, n, res;
		int64_t global_sum_step = counter->global_sum_step.s64;

		res = *int_p;
		switch (alloc) {
		case COUNTER_ALLOC_PER_CPU:
		{
			do {
				move_sum = 0;
				old = res;
				n = (int64_t) ((uint64_t) old + (uint64_t) v);
				if (unlikely(global_sum_step)) {
					if (unlikely(n > (int64_t) global_sum_step))
						move_sum = (int64_t) global_sum_step / 2;
					else if (unlikely(n < -(int64_t) global_sum_step))
						move_sum = -((int64_t) global_sum_step / 2);
					n -= move_sum;
				}
				res = cmpxchg_local(int_p, old, n);
			} while (old != res);
			break;
		}
		case COUNTER_ALLOC_GLOBAL:
		{
			do {
				old = res;
				n = (int64_t) ((uint64_t) old + (uint64_t) v);
				res = cmpxchg(int_p, old, n);
			} while (old != res);
			break;
		}
		default:
			return -EINVAL;
		}
		if (v > 0 && n < old)
			overflow = true;
		else if (v < 0 && n > old)
			underflow = true;
		break;
	}
#endif
	default:
		return -EINVAL;
	}
	if (unlikely(overflow && !test_bit(index, layout->overflow_bitmap)))
		set_bit(index, layout->overflow_bitmap);
	else if (unlikely(underflow && !test_bit(index, layout->underflow_bitmap)))
		set_bit(index, layout->underflow_bitmap);
	if (remainder)
		*remainder = move_sum;
	return 0;
}

static __always_inline int __lttng_counter_add_percpu(const struct lib_counter_config *config,
				     struct lib_counter *counter,
				     const size_t *dimension_indexes, int64_t v)
{
	int64_t move_sum;
	int ret;

	ret = __lttng_counter_add(config, COUNTER_ALLOC_PER_CPU, config->sync,
				       counter, dimension_indexes, v, &move_sum);
	if (unlikely(ret))
		return ret;
	if (unlikely(move_sum))
		return __lttng_counter_add(config, COUNTER_ALLOC_GLOBAL, COUNTER_SYNC_GLOBAL,
					   counter, dimension_indexes, move_sum, NULL);
	return 0;
}

static __always_inline int __lttng_counter_add_global(const struct lib_counter_config *config,
				     struct lib_counter *counter,
				     const size_t *dimension_indexes, int64_t v)
{
	return __lttng_counter_add(config, COUNTER_ALLOC_GLOBAL, config->sync, counter,
				   dimension_indexes, v, NULL);
}

static __always_inline int lttng_counter_add(const struct lib_counter_config *config,
				    struct lib_counter *counter,
				    const size_t *dimension_indexes, int64_t v)
{
	switch (config->alloc) {
	case COUNTER_ALLOC_PER_CPU:
		lttng_fallthrough;
	case COUNTER_ALLOC_PER_CPU | COUNTER_ALLOC_GLOBAL:
		return __lttng_counter_add_percpu(config, counter, dimension_indexes, v);
	case COUNTER_ALLOC_GLOBAL:
		return __lttng_counter_add_global(config, counter, dimension_indexes, v);
	default:
		return -EINVAL;
	}
}

static __always_inline int lttng_counter_inc(const struct lib_counter_config *config,
				     struct lib_counter *counter,
				     const size_t *dimension_indexes)
{
	return lttng_counter_add(config, counter, dimension_indexes, 1);
}

static __always_inline int lttng_counter_dec(const struct lib_counter_config *config,
				    struct lib_counter *counter,
				    const size_t *dimension_indexes)
{
	return lttng_counter_add(config, counter, dimension_indexes, -1);
}

#endif /* _LTTNG_COUNTER_API_H */
Commit	Line	Data
a101fa10 MD	1	/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
	2	*
	3	* counter/counter-api.h
	4	*
	5	* LTTng Counters API, requiring counter/config.h
	6	*
	7	* Copyright (C) 2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	8	*/
	9
	10	#ifndef _LTTNG_COUNTER_API_H
	11	#define _LTTNG_COUNTER_API_H
	12
	13	#include <linux/types.h>
	14	#include <linux/percpu.h>
	15	#include <linux/bitops.h>
	16	#include <counter/counter.h>
	17	#include <counter/counter-internal.h>
c190d76e	18	#include <wrapper/compiler_attributes.h>
236233f7	19	#include <wrapper/limits.h>
a101fa10 MD	20
	21	/*
	22	* Using unsigned arithmetic because overflow is defined.
	23	*/
2b8161a4	24	static __always_inline int __lttng_counter_add(const struct lib_counter_config *config,
a101fa10 MD	25	enum lib_counter_config_alloc alloc,
	26	enum lib_counter_config_sync sync,
	27	struct lib_counter *counter,
	28	const size_t *dimension_indexes, int64_t v,
	29	int64_t *remainder)
	30	{
	31	size_t index;
	32	bool overflow = false, underflow = false;
	33	struct lib_counter_layout *layout;
	34	int64_t move_sum = 0;
	35
	36	if (unlikely(lttng_counter_validate_indexes(config, counter, dimension_indexes)))
	37	return -EOVERFLOW;
	38	index = lttng_counter_get_index(config, counter, dimension_indexes);
	39
	40	switch (alloc) {
	41	case COUNTER_ALLOC_PER_CPU:
	42	layout = per_cpu_ptr(counter->percpu_counters, smp_processor_id());
	43	break;
	44	case COUNTER_ALLOC_GLOBAL:
	45	layout = &counter->global_counters;
	46	break;
	47	default:
	48	return -EINVAL;
	49	}
	50
	51	switch (config->counter_size) {
	52	case COUNTER_SIZE_8_BIT:
	53	{
	54	int8_t int_p = (int8_t ) layout->counters + index;
	55	int8_t old, n, res;
	56	int8_t global_sum_step = counter->global_sum_step.s8;
	57
	58	res = *int_p;
bc826918 MD	59	switch (alloc) {
bc826918 MD	60	case COUNTER_ALLOC_PER_CPU:
a101fa10 MD	61	{
	62	do {
	63	move_sum = 0;
	64	old = res;
	65	n = (int8_t) ((uint8_t) old + (uint8_t) v);
bf59e80c MD	66	if (unlikely(global_sum_step)) {
	67	if (unlikely(n > (int8_t) global_sum_step))
	68	move_sum = (int8_t) global_sum_step / 2;
	69	else if (unlikely(n < -(int8_t) global_sum_step))
	70	move_sum = -((int8_t) global_sum_step / 2);
	71	n -= move_sum;
	72	}
a101fa10 MD	73	res = cmpxchg_local(int_p, old, n);
	74	} while (old != res);
	75	break;
	76	}
bc826918	77	case COUNTER_ALLOC_GLOBAL:
a101fa10 MD	78	{
	79	do {
	80	old = res;
	81	n = (int8_t) ((uint8_t) old + (uint8_t) v);
	82	res = cmpxchg(int_p, old, n);
	83	} while (old != res);
	84	break;
	85	}
cf30b6d4 MD	86	default:
cf30b6d4 MD	87	return -EINVAL;
a101fa10 MD	88	}
	89	if (v > 0 && (v >= U8_MAX \|\| n < old))
	90	overflow = true;
f801323d	91	else if (v < 0 && (v <= -(s64) U8_MAX \|\| n > old))
a101fa10 MD	92	underflow = true;
	93	break;
	94	}
	95	case COUNTER_SIZE_16_BIT:
	96	{
	97	int16_t int_p = (int16_t ) layout->counters + index;
	98	int16_t old, n, res;
	99	int16_t global_sum_step = counter->global_sum_step.s16;
	100
	101	res = *int_p;
bc826918 MD	102	switch (alloc) {
bc826918 MD	103	case COUNTER_ALLOC_PER_CPU:
a101fa10 MD	104	{
	105	do {
	106	move_sum = 0;
	107	old = res;
	108	n = (int16_t) ((uint16_t) old + (uint16_t) v);
bf59e80c MD	109	if (unlikely(global_sum_step)) {
	110	if (unlikely(n > (int16_t) global_sum_step))
	111	move_sum = (int16_t) global_sum_step / 2;
	112	else if (unlikely(n < -(int16_t) global_sum_step))
	113	move_sum = -((int16_t) global_sum_step / 2);
	114	n -= move_sum;
	115	}
a101fa10 MD	116	res = cmpxchg_local(int_p, old, n);
	117	} while (old != res);
	118	break;
	119	}
bc826918	120	case COUNTER_ALLOC_GLOBAL:
a101fa10 MD	121	{
	122	do {
	123	old = res;
	124	n = (int16_t) ((uint16_t) old + (uint16_t) v);
	125	res = cmpxchg(int_p, old, n);
	126	} while (old != res);
	127	break;
	128	}
cf30b6d4 MD	129	default:
cf30b6d4 MD	130	return -EINVAL;
a101fa10 MD	131	}
	132	if (v > 0 && (v >= U16_MAX \|\| n < old))
	133	overflow = true;
f801323d	134	else if (v < 0 && (v <= -(s64) U16_MAX \|\| n > old))
a101fa10 MD	135	underflow = true;
	136	break;
	137	}
	138	case COUNTER_SIZE_32_BIT:
	139	{
	140	int32_t int_p = (int32_t ) layout->counters + index;
	141	int32_t old, n, res;
	142	int32_t global_sum_step = counter->global_sum_step.s32;
	143
	144	res = *int_p;
bc826918 MD	145	switch (alloc) {
bc826918 MD	146	case COUNTER_ALLOC_PER_CPU:
a101fa10 MD	147	{
	148	do {
	149	move_sum = 0;
	150	old = res;
	151	n = (int32_t) ((uint32_t) old + (uint32_t) v);
bf59e80c MD	152	if (unlikely(global_sum_step)) {
	153	if (unlikely(n > (int32_t) global_sum_step))
	154	move_sum = (int32_t) global_sum_step / 2;
	155	else if (unlikely(n < -(int32_t) global_sum_step))
	156	move_sum = -((int32_t) global_sum_step / 2);
	157	n -= move_sum;
	158	}
a101fa10 MD	159	res = cmpxchg_local(int_p, old, n);
	160	} while (old != res);
	161	break;
	162	}
bc826918	163	case COUNTER_ALLOC_GLOBAL:
a101fa10 MD	164	{
	165	do {
	166	old = res;
	167	n = (int32_t) ((uint32_t) old + (uint32_t) v);
	168	res = cmpxchg(int_p, old, n);
	169	} while (old != res);
	170	break;
	171	}
cf30b6d4 MD	172	default:
cf30b6d4 MD	173	return -EINVAL;
a101fa10 MD	174	}
	175	if (v > 0 && (v >= U32_MAX \|\| n < old))
	176	overflow = true;
f801323d	177	else if (v < 0 && (v <= -(s64) U32_MAX \|\| n > old))
a101fa10 MD	178	underflow = true;
	179	break;
	180	}
	181	#if BITS_PER_LONG == 64
	182	case COUNTER_SIZE_64_BIT:
	183	{
	184	int64_t int_p = (int64_t ) layout->counters + index;
	185	int64_t old, n, res;
	186	int64_t global_sum_step = counter->global_sum_step.s64;
	187
	188	res = *int_p;
bc826918 MD	189	switch (alloc) {
bc826918 MD	190	case COUNTER_ALLOC_PER_CPU:
a101fa10 MD	191	{
	192	do {
	193	move_sum = 0;
	194	old = res;
	195	n = (int64_t) ((uint64_t) old + (uint64_t) v);
bf59e80c MD	196	if (unlikely(global_sum_step)) {
	197	if (unlikely(n > (int64_t) global_sum_step))
	198	move_sum = (int64_t) global_sum_step / 2;
	199	else if (unlikely(n < -(int64_t) global_sum_step))
	200	move_sum = -((int64_t) global_sum_step / 2);
	201	n -= move_sum;
	202	}
a101fa10 MD	203	res = cmpxchg_local(int_p, old, n);
	204	} while (old != res);
	205	break;
	206	}
bc826918	207	case COUNTER_ALLOC_GLOBAL:
a101fa10 MD	208	{
	209	do {
	210	old = res;
	211	n = (int64_t) ((uint64_t) old + (uint64_t) v);
	212	res = cmpxchg(int_p, old, n);
	213	} while (old != res);
	214	break;
	215	}
cf30b6d4 MD	216	default:
cf30b6d4 MD	217	return -EINVAL;
a101fa10 MD	218	}
	219	if (v > 0 && n < old)
	220	overflow = true;
	221	else if (v < 0 && n > old)
	222	underflow = true;
	223	break;
	224	}
	225	#endif
	226	default:
	227	return -EINVAL;
	228	}
	229	if (unlikely(overflow && !test_bit(index, layout->overflow_bitmap)))
	230	set_bit(index, layout->overflow_bitmap);
	231	else if (unlikely(underflow && !test_bit(index, layout->underflow_bitmap)))
	232	set_bit(index, layout->underflow_bitmap);
	233	if (remainder)
	234	*remainder = move_sum;
	235	return 0;
	236	}
	237
2b8161a4	238	static __always_inline int __lttng_counter_add_percpu(const struct lib_counter_config *config,
a101fa10 MD	239	struct lib_counter *counter,
	240	const size_t *dimension_indexes, int64_t v)
	241	{
	242	int64_t move_sum;
	243	int ret;
	244
	245	ret = __lttng_counter_add(config, COUNTER_ALLOC_PER_CPU, config->sync,
	246	counter, dimension_indexes, v, &move_sum);
	247	if (unlikely(ret))
	248	return ret;
	249	if (unlikely(move_sum))
	250	return __lttng_counter_add(config, COUNTER_ALLOC_GLOBAL, COUNTER_SYNC_GLOBAL,
	251	counter, dimension_indexes, move_sum, NULL);
	252	return 0;
	253	}
	254
2b8161a4	255	static __always_inline int __lttng_counter_add_global(const struct lib_counter_config *config,
a101fa10 MD	256	struct lib_counter *counter,
	257	const size_t *dimension_indexes, int64_t v)
	258	{
	259	return __lttng_counter_add(config, COUNTER_ALLOC_GLOBAL, config->sync, counter,
	260	dimension_indexes, v, NULL);
	261	}
	262
2b8161a4	263	static __always_inline int lttng_counter_add(const struct lib_counter_config *config,
a101fa10 MD	264	struct lib_counter *counter,
	265	const size_t *dimension_indexes, int64_t v)
	266	{
	267	switch (config->alloc) {
c190d76e MJ	268	case COUNTER_ALLOC_PER_CPU:
c190d76e MJ	269	lttng_fallthrough;
a101fa10 MD	270	case COUNTER_ALLOC_PER_CPU \| COUNTER_ALLOC_GLOBAL:
	271	return __lttng_counter_add_percpu(config, counter, dimension_indexes, v);
	272	case COUNTER_ALLOC_GLOBAL:
	273	return __lttng_counter_add_global(config, counter, dimension_indexes, v);
	274	default:
	275	return -EINVAL;
	276	}
	277	}
	278
2b8161a4	279	static __always_inline int lttng_counter_inc(const struct lib_counter_config *config,
a101fa10 MD	280	struct lib_counter *counter,
	281	const size_t *dimension_indexes)
	282	{
	283	return lttng_counter_add(config, counter, dimension_indexes, 1);
	284	}
	285
2b8161a4	286	static __always_inline int lttng_counter_dec(const struct lib_counter_config *config,
a101fa10 MD	287	struct lib_counter *counter,
	288	const size_t *dimension_indexes)
	289	{
	290	return lttng_counter_add(config, counter, dimension_indexes, -1);
	291	}
	292
	293	#endif /* _LTTNG_COUNTER_API_H */