X-Git-Url: https://git.lttng.org/?a=blobdiff_plain;f=tests%2Futils%2Futils.sh;h=2680daae923e15b937f8263d203210efd44549c1;hb=729c1fec5976af17fded2f026725a08ef1924290;hp=88a4ddd20c30216d6f19ea07b4160df998cb3c9b;hpb=c8e000ef6be08c492f79b51b9752e5d8b9b26d59;p=lttng-tools.git

diff --git a/tests/utils/utils.sh b/tests/utils/utils.sh
index 88a4ddd20..2680daae9 100644
--- a/tests/utils/utils.sh
+++ b/tests/utils/utils.sh
@@ -12,9 +12,22 @@ CONSUMERD_MATCH=".*lttng-consumerd.*"
 RELAYD_BIN="lttng-relayd"
 RELAYD_MATCH=".*lttng-relayd.*"
 LTTNG_BIN="lttng"
-BABELTRACE_BIN="babeltrace"
+BABELTRACE_BIN="babeltrace2"
 OUTPUT_DEST=/dev/null
 ERROR_OUTPUT_DEST=/dev/null
+MI_XSD_MAJOR_VERSION=4
+MI_XSD_MINOR_VERSION=1
+MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
+MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
+
+XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
+XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
+XML_NODE_CHECK="${XML_EXTRACT} -e"
+
+# To match 20201127-175802
+date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
+# The size of a long on this system
+system_long_bit_size=$(getconf LONG_BIT)
 
 # Minimal kernel version supported for session daemon tests
 KERNEL_MAJOR_VERSION=2
@@ -40,25 +53,60 @@ if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
 	LTTNG_TEST_TEARDOWN_TIMEOUT=60
 fi
 
-function full_cleanup ()
+# Enable job monitor mode.
+# Here we are mostly interested in the following from the monitor mode:
+#    All processes run in a separate process group.
+# This allows us to ensure that all subprocesses from all background tasks are
+# cleaned up correctly using signal to process group id.
+set -m
+
+kill_background_jobs ()
+{
+	local pids
+	pids=$(jobs -p)
+
+	if [ -z "$pids" ]; then
+		# Empty
+		return 0
+	fi
+
+	while read -r pid;
+	do
+		# Use negative number to send the signal to the process group.
+		# This ensure that any subprocesses receive the signal.
+		# /dev/null is used since there is an acceptable race between
+		# the moments the pids are listed and the moment we send a
+		# signal.
+		kill -SIGTERM -- "-$pid" 2>/dev/null
+	done <<< "$pids"
+}
+
+function cleanup ()
 {
 	# Try to kill daemons gracefully
-	stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
-	stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
+	stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
+	stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
 
 	# If daemons are still present, forcibly kill them
-	stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
-	stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
-	stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
-
-	# Disable trap for SIGTERM since the following kill to the
-	# pidgroup will be SIGTERM. Otherwise it loops.
-	# The '-' before the pid number ($$) indicates 'kill' to signal the
-	# whole process group.
-	trap - SIGTERM && kill -- -$$
+	stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
+	stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
+	stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
+
+	kill_background_jobs
+}
+
+function full_cleanup ()
+{
+	cleanup
 	exit 1
 }
 
+function LTTNG_BAIL_OUT ()
+{
+	cleanup
+	BAIL_OUT "$@"
+}
+
 function null_pipes ()
 {
 	exec 0>/dev/null
@@ -74,6 +122,42 @@ trap full_cleanup SIGINT SIGTERM
 
 trap null_pipes SIGPIPE
 
+# Check pgrep from env, default to pgrep if none
+if [ -z "$PGREP" ]; then
+	PGREP=pgrep
+fi
+
+# Due to the renaming of threads we need to use the full command (pgrep -f) to
+# identify the pids for multiple lttng related processes. The problem with "pgrep
+# -f" is that it ends up also looking at the arguments. We use a two stage
+# lookup. The first one is using "pgrep -f" yielding potential candidate.
+# The second on perform grep on the basename of the first field of the
+# /proc/pid/cmdline of the previously identified pids. The first field
+# correspond to the actual command.
+function lttng_pgrep ()
+{
+	local pattern=$1
+	local possible_pids
+	local full_command_no_argument
+	local command_basename
+
+	possible_pids=$($PGREP -f "$pattern")
+	if [ -z "$possible_pids" ]; then
+		return 0
+	fi
+
+	while IFS= read -r pid ; do
+		# /proc/pid/cmdline is null separated.
+		if full_command_no_argument=$(cut -d '' -f 1 2>/dev/null < /proc/"$pid"/cmdline); then
+			command_basename=$(basename "$full_command_no_argument")
+			if grep -q "$pattern" <<< "$command_basename"; then
+				echo "$pid"
+			fi
+		fi
+	done <<< "$possible_pids"
+	return 0
+}
+
 function print_ok ()
 {
 	# Check if we are a terminal
@@ -139,10 +223,33 @@ function conf_proc_count()
 # Bail out on failure
 function validate_lttng_modules_present ()
 {
+	# Check for loadable modules.
 	modprobe -n lttng-tracer 2>/dev/null
-	if [ $? -ne 0  ]; then
-		BAIL_OUT "LTTng modules not detected."
+	if [ $? -eq 0 ]; then
+		return 0
+	fi
+
+	# Check for builtin modules.
+	ls /proc/lttng > /dev/null 2>&1
+	if [ $? -eq 0 ]; then
+		return 0
 	fi
+
+	LTTNG_BAIL_OUT "LTTng modules not detected."
+}
+
+# Run the lttng binary.
+#
+# The first two arguments are stdout and stderr redirect paths, respectively.
+# The rest of the arguments are forwarded to the lttng binary
+function _run_lttng_cmd
+{
+	local stdout_dest="$1"
+	local stderr_dest="$2"
+	shift 2
+
+	diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 }
 
 function enable_kernel_lttng_event
@@ -165,7 +272,8 @@ function enable_kernel_lttng_event
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event "$event_name" $chan -s $sess_name -k
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test $ret -ne "0"
@@ -220,7 +328,8 @@ function lttng_enable_kernel_syscall()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event --syscall "$syscall_name" $chan -s $sess_name -k
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test $ret -ne "0"
@@ -259,7 +368,8 @@ function lttng_disable_kernel_syscall()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		disable-event --syscall "$syscall_name" $chan -s $sess_name -k
 
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
@@ -280,6 +390,28 @@ function lttng_disable_kernel_syscall_fail()
 	lttng_disable_kernel_syscall 1 "$@"
 }
 
+function lttng_enable_kernel_function_event ()
+{
+	local expected_to_fail="$1"
+	local sess_name="$2"
+	local target="$3"
+	local event_name="$4"
+
+	"$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test $ret -ne "0"
+		ok $? "Enable kernel function event for session $sess_name failed as expected"
+	else
+		ok $ret "Enable kernel function event for session $sess_name"
+	fi
+}
+
+function lttng_enable_kernel_function_event_ok ()
+{
+	lttng_enable_kernel_function_event 0 "$@"
+}
+
 function lttng_enable_kernel_userspace_probe_event ()
 {
 	local expected_to_fail="$1"
@@ -323,7 +455,8 @@ function lttng_enable_kernel_channel()
 	local channel_name=$4
 	local opts="${@:5}"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-channel -k $channel_name -s $sess_name $opts
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -364,7 +497,8 @@ function lttng_disable_kernel_channel()
 	local sess_name=$2
 	local channel_name=$3
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		disable-channel -k $channel_name -s $sess_name
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -392,7 +526,7 @@ function start_lttng_relayd_opt()
 
 	DIR=$(readlink -f "$TESTDIR")
 
-	if [ -z $(pgrep $RELAYD_MATCH) ]; then
+	if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 		# shellcheck disable=SC2086
 		$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 		#$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
@@ -424,27 +558,32 @@ function start_lttng_relayd_notap()
 function stop_lttng_relayd_opt()
 {
 	local withtap=$1
-	local signal=$2
+	local is_cleanup=$2
+	local signal=$3
+	local timeout_s=$4
+	local dtimeleft_s=
+	local retval=0
+	local pids
 
 	if [ -z "$signal" ]; then
 		signal="SIGTERM"
 	fi
 
-	local timeout_s=$3
-	local dtimeleft_s=
 
 	# Multiply time by 2 to simplify integer arithmetic
 	if [ -n "$timeout_s" ]; then
 		dtimeleft_s=$((timeout_s * 2))
 	fi
 
-	local retval=0
-	local pids=
 
-	pids=$(pgrep "$RELAYD_MATCH")
+	pids=$(lttng_pgrep "$RELAYD_MATCH")
 	if [ -z "$pids" ]; then
-		if [ "$withtap" -eq "1" ]; then
-			pass "No relay daemon to kill"
+		if [ "$is_cleanup" -eq 1 ]; then
+			:
+		elif [ "$withtap" -eq "1" ]; then
+			fail "No relay daemon to kill"
+		else
+			LTTNG_BAIL_OUT "No relay daemon to kill"
 		fi
 		return 0
 	fi
@@ -460,7 +599,7 @@ function stop_lttng_relayd_opt()
 	else
 		out=1
 		while [ -n "$out" ]; do
-			out=$(pgrep "$RELAYD_MATCH")
+			out=$(lttng_pgrep "$RELAYD_MATCH")
 			if [ -n "$dtimeleft_s" ]; then
 				if [ $dtimeleft_s -lt 0 ]; then
 					out=
@@ -483,12 +622,17 @@ function stop_lttng_relayd_opt()
 
 function stop_lttng_relayd()
 {
-	stop_lttng_relayd_opt 1 "$@"
+	stop_lttng_relayd_opt 1 0 "$@"
 }
 
 function stop_lttng_relayd_notap()
 {
-	stop_lttng_relayd_opt 0 "$@"
+	stop_lttng_relayd_opt 0 0 "$@"
+}
+
+function stop_lttng_relayd_cleanup()
+{
+	stop_lttng_relayd_opt 0 1 "$@"
 }
 
 #First arg: show tap output
@@ -498,6 +642,9 @@ function start_lttng_sessiond_opt()
 	local withtap=$1
 	local load_path=$2
 
+	# The rest of the arguments will be passed directly to lttng-sessiond.
+	shift 2
+
 	local env_vars=""
 	local consumerd=""
 
@@ -532,20 +679,23 @@ function start_lttng_sessiond_opt()
 
 	if ! validate_kernel_version; then
 	    fail "Start session daemon"
-	    BAIL_OUT "*** Kernel too old for session daemon tests ***"
+	    LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 	fi
 
+	diag "export LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/config/"
 	: "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 	export LTTNG_SESSION_CONFIG_XSD_PATH
 
-	if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
+	if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 		# Have a load path ?
 		if [ -n "$load_path" ]; then
+			diag "env $env_vars --load $load_path --background $consumerd $@"
 			# shellcheck disable=SC2086
-			env $env_vars --load "$load_path" --background "$consumerd"
+			env $env_vars --load "$load_path" --background "$consumerd" "$@"
 		else
+			diag "env $env_vars --background $consumerd $@"
 			# shellcheck disable=SC2086
-			env $env_vars --background "$consumerd"
+			env $env_vars --background "$consumerd" "$@"
 		fi
 		#$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 		status=$?
@@ -568,15 +718,18 @@ function start_lttng_sessiond_notap()
 function stop_lttng_sessiond_opt()
 {
 	local withtap=$1
-	local signal=$2
+	local is_cleanup=$2
+	local signal=$3
+	local timeout_s=$4
+	local dtimeleft_s=
+	local retval=0
+	local runas_pids
+	local pids
 
 	if [ -z "$signal" ]; then
 		signal=SIGTERM
 	fi
 
-	local timeout_s=$3
-	local dtimeleft_s=
-
 	# Multiply time by 2 to simplify integer arithmetic
 	if [ -n "$timeout_s" ]; then
 		dtimeleft_s=$((timeout_s * 2))
@@ -587,21 +740,20 @@ function stop_lttng_sessiond_opt()
 		return 0
 	fi
 
-	local retval=0
-
-	local runas_pids=
-	runas_pids=$(pgrep "$RUNAS_MATCH")
-
-	local pids=
-	pids=$(pgrep "$SESSIOND_MATCH")
+	runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
+	pids=$(lttng_pgrep "$SESSIOND_MATCH")
 
 	if [ -n "$runas_pids" ]; then
 		pids="$pids $runas_pids"
 	fi
 
 	if [ -z "$pids" ]; then
-		if [ "$withtap" -eq "1" ]; then
-			pass "No session daemon to kill"
+		if [ "$is_cleanup" -eq 1 ]; then
+			:
+		elif [ "$withtap" -eq "1" ]; then
+			fail "No session daemon to kill"
+		else
+			LTTNG_BAIL_OUT "No session daemon to kill"
 		fi
 		return 0
 	fi
@@ -617,7 +769,7 @@ function stop_lttng_sessiond_opt()
 	else
 		out=1
 		while [ -n "$out" ]; do
-			out=$(pgrep "${SESSIOND_MATCH}")
+			out=$(lttng_pgrep "${SESSIOND_MATCH}")
 			if [ -n "$dtimeleft_s" ]; then
 				if [ $dtimeleft_s -lt 0 ]; then
 					out=
@@ -629,7 +781,7 @@ function stop_lttng_sessiond_opt()
 		done
 		out=1
 		while [ -n "$out" ]; do
-			out=$(pgrep "$CONSUMERD_MATCH")
+			out=$(lttng_pgrep "$CONSUMERD_MATCH")
 			if [ -n "$dtimeleft_s" ]; then
 				if [ $dtimeleft_s -lt 0 ]; then
 					out=
@@ -655,7 +807,7 @@ function stop_lttng_sessiond_opt()
 
 			if [ -n "$modules" ]; then
 				diag "Unloading all LTTng modules"
-				modprobe -r "$modules"
+				modprobe --remove "$modules"
 			fi
 		fi
 	fi
@@ -665,44 +817,50 @@ function stop_lttng_sessiond_opt()
 
 function stop_lttng_sessiond()
 {
-	stop_lttng_sessiond_opt 1 "$@"
+	stop_lttng_sessiond_opt 1 0 "$@"
 }
 
 function stop_lttng_sessiond_notap()
 {
-	stop_lttng_sessiond_opt 0 "$@"
+	stop_lttng_sessiond_opt 0 0 "$@"
+}
+
+function stop_lttng_sessiond_cleanup()
+{
+	stop_lttng_sessiond_opt 0 1 "$@"
 }
 
 function sigstop_lttng_sessiond_opt()
 {
 	local withtap=$1
 	local signal=SIGSTOP
+	local pids
 
 	if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 		# Env variable requested no session daemon
 		return
 	fi
 
-	PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
+	pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 
 	if [ "$withtap" -eq "1" ]; then
-		diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
+		diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 	fi
 
 	# shellcheck disable=SC2086
-	if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
+	if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 		if [ "$withtap" -eq "1" ]; then
 			fail "Sending SIGSTOP to session daemon"
 		fi
 	else
 		out=1
 		while [ $out -ne 0 ]; do
-			pid="$(pgrep "$SESSIOND_MATCH")"
+			pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 
 			# Wait until state becomes stopped for session
 			# daemon(s).
 			out=0
-			for sessiond_pid in $pid; do
+			for sessiond_pid in $pids; do
 				state="$(ps -p "$sessiond_pid" -o state= )"
 				if [[ -n "$state" && "$state" != "T" ]]; then
 					out=1
@@ -729,35 +887,39 @@ function sigstop_lttng_sessiond_notap()
 function stop_lttng_consumerd_opt()
 {
 	local withtap=$1
-	local signal=$2
+	local is_cleanup=$2
+	local signal=$3
+	local timeout_s=$4
+	local dtimeleft_s=
+	local retval=0
+	local pids
 
 	if [ -z "$signal" ]; then
 		signal=SIGTERM
 	fi
 
-	local timeout_s=$3
-	local dtimeleft_s=
-
 	# Multiply time by 2 to simplify integer arithmetic
 	if [ -n "$timeout_s" ]; then
 		dtimeleft_s=$((timeout_s * 2))
 	fi
 
-	local retval=0
-
-	PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
+	pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 
-	if [ -z "$PID_CONSUMERD" ]; then
-		if [ "$withtap" -eq "1" ]; then
-			pass "No consumer daemon to kill"
+	if [ -z "$pids" ]; then
+		if [ "$is_cleanup" -eq 1 ]; then
+			:
+		elif [ "$withtap" -eq "1" ]; then
+			fail "No consumerd daemon to kill"
+		else
+			LTTNG_BAIL_OUT "No consumerd daemon to kill"
 		fi
 		return 0
 	fi
 
-	diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
+	diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 
 	# shellcheck disable=SC2086
-	if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
+	if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 		retval=1
 		if [ "$withtap" -eq "1" ]; then
 			fail "Kill consumer daemon"
@@ -765,12 +927,12 @@ function stop_lttng_consumerd_opt()
 	else
 		out=1
 		while [ $out -ne 0 ]; do
-			pid="$(pgrep "$CONSUMERD_MATCH")"
+			pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 
 			# If consumerds are still present check their status.
 			# A zombie status qualifies the consumerd as *killed*
 			out=0
-			for consumer_pid in $pid; do
+			for consumer_pid in $pids; do
 				state="$(ps -p "$consumer_pid" -o state= )"
 				if [[ -n "$state" && "$state" != "Z" ]]; then
 					out=1
@@ -799,25 +961,31 @@ function stop_lttng_consumerd_opt()
 
 function stop_lttng_consumerd()
 {
-	stop_lttng_consumerd_opt 1 "$@"
+	stop_lttng_consumerd_opt 1 0 "$@"
 }
 
 function stop_lttng_consumerd_notap()
 {
-	stop_lttng_consumerd_opt 0 "$@"
+	stop_lttng_consumerd_opt 0 0 "$@"
+}
+
+function stop_lttng_consumerd_cleanup()
+{
+	stop_lttng_consumerd_opt 0 1 "$@"
 }
 
 function sigstop_lttng_consumerd_opt()
 {
 	local withtap=$1
 	local signal=SIGSTOP
+	local pids
 
-	PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
+	pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 
-	diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
+	diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 
 	# shellcheck disable=SC2086
-	kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	retval=$?
 
 	if [ $retval -eq 1 ]; then
@@ -828,12 +996,12 @@ function sigstop_lttng_consumerd_opt()
 	else
 		out=1
 		while [ $out -ne 0 ]; do
-			pid="$(pgrep "$CONSUMERD_MATCH")"
+			pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 
 			# Wait until state becomes stopped for all
 			# consumers.
 			out=0
-			for consumer_pid in $pid; do
+			for consumer_pid in $pids; do
 				state="$(ps -p "$consumer_pid" -o state= )"
 				if [[ -n "$state" && "$state" != "T" ]]; then
 					out=1
@@ -860,9 +1028,26 @@ function sigstop_lttng_consumerd_notap()
 
 function list_lttng_with_opts ()
 {
+	local ret
+	local withtap=$1
+	shift
 	local opts=$1
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
-	ok $? "Lttng-tool list command with option $opts"
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		list $opts
+	ret=$?
+	if [ $withtap -eq "1" ]; then
+		ok $ret "Lttng-tool list command with option $opts"
+	fi
+}
+
+function list_lttng_ok ()
+{
+	list_lttng_with_opts 1 "$@"
+}
+
+function list_lttng_notap ()
+{
+	list_lttng_with_opts 0 "$@"
 }
 
 function create_lttng_session_no_output ()
@@ -870,7 +1055,8 @@ function create_lttng_session_no_output ()
 	local sess_name=$1
 	local opts="${@:2}"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		create $sess_name --no-output $opts
 	ok $? "Create session $sess_name in no-output mode"
 }
 
@@ -879,7 +1065,8 @@ function create_lttng_session_uri () {
 	local uri=$2
 	local opts="${@:3}"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		create $sess_name -U $uri $opts
 	ok $? "Create session $sess_name with uri:$uri and opts: $opts"
 }
 
@@ -898,7 +1085,8 @@ function create_lttng_session ()
 		trace_path="-o $trace_path"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN create "$sess_name" $trace_path $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		create "$sess_name" $trace_path $opt
 	ret=$?
 	if [ $expected_to_fail -eq "1" ]; then
 		test "$ret" -ne "0"
@@ -938,7 +1126,8 @@ function enable_ust_lttng_channel ()
 	local channel_name=$4
 	local opts="${@:5}"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-channel -u $channel_name -s $sess_name $opts
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -974,7 +1163,8 @@ function disable_ust_lttng_channel()
 	local sess_name=$1
 	local channel_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		disable-channel -u $channel_name -s $sess_name
 	ok $? "Disable channel $channel_name for session $sess_name"
 }
 
@@ -983,7 +1173,8 @@ function enable_lttng_mmap_overwrite_kernel_channel()
 	local sess_name=$1
 	local channel_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-channel -s $sess_name $channel_name -k --output mmap --overwrite
 	ok $? "Enable channel $channel_name for session $sess_name"
 }
 
@@ -992,7 +1183,8 @@ function enable_lttng_mmap_discard_small_kernel_channel()
 	local sess_name=$1
 	local channel_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
 	ok $? "Enable small discard channel $channel_name for session $sess_name"
 }
 
@@ -1001,7 +1193,8 @@ function enable_lttng_mmap_overwrite_small_kernel_channel()
 	local sess_name=$1
 	local channel_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
 	ok $? "Enable small discard channel $channel_name for session $sess_name"
 }
 
@@ -1010,7 +1203,8 @@ function enable_lttng_mmap_overwrite_ust_channel()
 	local sess_name=$1
 	local channel_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-channel -s $sess_name $channel_name -u --output mmap --overwrite
 	ok $? "Enable channel $channel_name for session $sess_name"
 }
 
@@ -1029,7 +1223,8 @@ function enable_ust_lttng_event ()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event "$event_name" $chan -s "$sess_name" -u
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test $ret -ne "0"
@@ -1073,7 +1268,8 @@ function enable_jul_lttng_event()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event "$event_name" $chan -s $sess_name -j
 	ok $? "Enable JUL event $event_name for session $sess_name"
 }
 
@@ -1091,7 +1287,8 @@ function enable_jul_lttng_event_loglevel()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j
 	ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
 }
 
@@ -1108,7 +1305,8 @@ function enable_log4j_lttng_event()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event "$event_name" $chan -s $sess_name -l
 	ok $? "Enable LOG4J event $event_name for session $sess_name"
 }
 
@@ -1126,7 +1324,8 @@ function enable_log4j_lttng_event_loglevel()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l
 	ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
 }
 
@@ -1143,7 +1342,8 @@ function enable_python_lttng_event()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event "$event_name" $chan -s $sess_name -p
 	ok $? "Enable Python event $event_name for session $sess_name"
 }
 
@@ -1161,7 +1361,8 @@ function enable_python_lttng_event_loglevel()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p
 	ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
 }
 
@@ -1170,8 +1371,17 @@ function enable_ust_lttng_event_filter()
 	local sess_name="$1"
 	local event_name="$2"
 	local filter="$3"
+	local channel_name=$4
+
+	if [ -z $channel_name ]; then
+		# default channel if none specified
+		chan=""
+	else
+		chan="-c $channel_name"
+	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event $chan "$event_name" -s $sess_name -u --filter "$filter"
 	ok $? "Enable event $event_name with filtering for session $sess_name"
 }
 
@@ -1181,7 +1391,8 @@ function enable_ust_lttng_event_loglevel()
 	local event_name="$2"
 	local loglevel="$3"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event "$event_name" -s $sess_name -u --loglevel $loglevel
 	ok $? "Enable event $event_name with loglevel $loglevel"
 }
 
@@ -1191,7 +1402,8 @@ function enable_ust_lttng_event_loglevel_only()
 	local event_name="$2"
 	local loglevel="$3"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel
 	ok $? "Enable event $event_name with loglevel-only $loglevel"
 }
 
@@ -1208,7 +1420,8 @@ function disable_ust_lttng_event ()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		disable-event "$event_name" -s $sess_name $chan -u
 	ok $? "Disable event $event_name for session $sess_name"
 }
 
@@ -1235,7 +1448,8 @@ function disable_python_lttng_event ()
 	local sess_name="$1"
 	local event_name="$2"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		disable-event "$event_name" -s $sess_name -p
 	ok $? "Disable Python event $event_name for session $sess_name"
 }
 
@@ -1245,7 +1459,8 @@ function start_lttng_tracing_opt ()
 	local expected_to_fail=$2
 	local sess_name=$3
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		start $sess_name
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -1281,7 +1496,8 @@ function stop_lttng_tracing_opt ()
 	local expected_to_fail=$2
 	local sess_name=$3
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		stop $sess_name
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -1317,7 +1533,8 @@ function destroy_lttng_session ()
 	local expected_to_fail=$2
 	local sess_name=$3
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		destroy $sess_name
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -1350,7 +1567,8 @@ function destroy_lttng_session_notap ()
 
 function destroy_lttng_sessions ()
 {
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		destroy --all
 	ok $? "Destroy all lttng sessions"
 }
 
@@ -1361,7 +1579,8 @@ function lttng_snapshot_add_output ()
 	local trace_path=$3
 	local opts=$4
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		snapshot add-output -s $sess_name $trace_path $opts
 	ret=$?
 	if [[ $expected_to_fail -eq 1 ]]; then
 		test "$ret" -ne "0"
@@ -1387,7 +1606,8 @@ function lttng_snapshot_del_output ()
 	local sess_name=$2
 	local id=$3
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		snapshot del-output -s $sess_name $id
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -1410,15 +1630,18 @@ function lttng_snapshot_del_output_fail ()
 function lttng_snapshot_record ()
 {
 	local sess_name=$1
+	local trace_path=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		snapshot record -s "$sess_name" "$trace_path"
 	ok $? "Snapshot recorded"
 }
 
 function lttng_snapshot_list ()
 {
 	local sess_name=$1
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		snapshot list-output -s $sess_name
 	ok $? "Snapshot list"
 }
 
@@ -1427,7 +1650,8 @@ function lttng_save()
 	local sess_name=$1
 	local opts=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		save $sess_name $opts
 	ok $? "Session saved"
 }
 
@@ -1436,7 +1660,8 @@ function lttng_load()
 	local expected_to_fail=$1
 	local opts=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		load $opts
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test $ret -ne "0"
@@ -1561,7 +1786,8 @@ function lttng_untrack_kernel_fail()
 
 function lttng_add_context_list()
 {
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		add-context --list
 	ret=$?
 	ok $ret "Context listing"
 }
@@ -1574,7 +1800,8 @@ function add_context_lttng()
 	local channel_name="$4"
 	local type="$5"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		add-context -s $session_name -c $channel_name -t $type $domain 
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test $ret -ne "0"
@@ -1630,17 +1857,24 @@ function wait_live_viewer_connect ()
 	pass "Waiting for live viewers on url: $url"
 }
 
+function bail_out_if_no_babeltrace()
+{
+	which "$BABELTRACE_BIN" >/dev/null
+	if [ $? -ne 0 ]; then
+		LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
+	fi
+}
+
 function validate_metadata_event ()
 {
 	local event_name=$1
 	local nr_event_id=$2
 	local trace_path=$3
 
-	local metadata_file=$(find $trace_path | grep metadata)
+	local metadata_file=$(find $trace_path -name "metadata")
 	local metadata_path=$(dirname $metadata_file)
 
-	which $BABELTRACE_BIN >/dev/null
-	skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
+	bail_out_if_no_babeltrace
 
 	local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
 
@@ -1659,8 +1893,7 @@ function trace_matches ()
 	local nr_iter=$2
 	local trace_path=$3
 
-	which $BABELTRACE_BIN >/dev/null
-	skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
+	bail_out_if_no_babeltrace
 
 	local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
 
@@ -1678,8 +1911,9 @@ function trace_match_only()
 	local nr_iter=$2
 	local trace_path=$3
 
-	which $BABELTRACE_BIN >/dev/null
-	skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
+	bail_out_if_no_babeltrace
+	#which "$BABELTRACE_BIN" >/dev/null
+	#skip $? -ne 0 "\"$BABELTRACE_BIN\" binary not found. Skipping trace comparison"
 
 	local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
 	local total=$($BABELTRACE_BIN $trace_path | wc -l)
@@ -1697,10 +1931,7 @@ function validate_trace
 	local event_name=$1
 	local trace_path=$2
 
-	which $BABELTRACE_BIN >/dev/null
-	if [ $? -ne 0 ]; then
-	    skip 0 "Babeltrace binary not found. Skipping trace validation"
-	fi
+	bail_out_if_no_babeltrace
 
 	OLDIFS=$IFS
 	IFS=","
@@ -1710,7 +1941,7 @@ function validate_trace
 			pass "Validate trace for event $i, $traced events"
 		else
 			fail "Validate trace for event $i"
-			diag "Found $traced occurences of $i"
+			diag "Found $traced occurrences of $i"
 		fi
 	done
 	ret=$?
@@ -1724,10 +1955,7 @@ function validate_trace_count
 	local trace_path=$2
 	local expected_count=$3
 
-	which $BABELTRACE_BIN >/dev/null
-	if [ $? -ne 0 ]; then
-	    skip 0 "Babeltrace binary not found. Skipping trace validation"
-	fi
+	bail_out_if_no_babeltrace
 
 	cnt=0
 	OLDIFS=$IFS
@@ -1738,7 +1966,7 @@ function validate_trace_count
 			pass "Validate trace for event $i, $traced events"
 		else
 			fail "Validate trace for event $i"
-			diag "Found $traced occurences of $i"
+			diag "Found $traced occurrences of $i"
 		fi
 		cnt=$(($cnt + $traced))
 	done
@@ -1754,10 +1982,7 @@ function validate_trace_count_range_incl_min_excl_max
 	local expected_min=$3
 	local expected_max=$4
 
-	which $BABELTRACE_BIN >/dev/null
-	if [ $? -ne 0 ]; then
-	    skip 0 "Babeltrace binary not found. Skipping trace validation"
-	fi
+	bail_out_if_no_babeltrace
 
 	cnt=0
 	OLDIFS=$IFS
@@ -1768,7 +1993,7 @@ function validate_trace_count_range_incl_min_excl_max
 			pass "Validate trace for event $i, $traced events"
 		else
 			fail "Validate trace for event $i"
-			diag "Found $traced occurences of $i"
+			diag "Found $traced occurrences of $i"
 		fi
 		cnt=$(($cnt + $traced))
 	done
@@ -1781,11 +2006,6 @@ function trace_first_line
 {
 	local trace_path=$1
 
-	which $BABELTRACE_BIN >/dev/null
-	if [ $? -ne 0 ]; then
-	    skip 0 "Babeltrace binary not found. Skipping trace validation"
-	fi
-
 	$BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
 }
 
@@ -1794,15 +2014,14 @@ function validate_trace_exp()
 	local event_exp=$1
 	local trace_path=$2
 
-	which $BABELTRACE_BIN >/dev/null
-	skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
+	bail_out_if_no_babeltrace
 
 	traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
 	if [ "$traced" -ne 0 ]; then
 		pass "Validate trace for expression '${event_exp}', $traced events"
 	else
 		fail "Validate trace for expression '${event_exp}'"
-		diag "Found $traced occurences of '${event_exp}'"
+		diag "Found $traced occurrences of '${event_exp}'"
 	fi
 	ret=$?
 	return $ret
@@ -1813,8 +2032,7 @@ function validate_trace_only_exp()
 	local event_exp=$1
 	local trace_path=$2
 
-	which $BABELTRACE_BIN >/dev/null
-	skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
+	bail_out_if_no_babeltrace
 
 	local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
 	local total=$($BABELTRACE_BIN $trace_path | wc -l)
@@ -1833,10 +2051,7 @@ function validate_trace_empty()
 {
 	local trace_path=$1
 
-	which $BABELTRACE_BIN >/dev/null
-	if [ $? -ne 0 ]; then
-	    skip 0 "Babeltrace binary not found. Skipping trace validation"
-	fi
+	bail_out_if_no_babeltrace
 
 	events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
 	ret=$?
@@ -1888,7 +2103,8 @@ function regenerate_metadata ()
 	local expected_to_fail=$1
 	local sess_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		regenerate metadata -s $sess_name
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -1913,7 +2129,8 @@ function regenerate_statedump ()
 	local expected_to_fail=$1
 	local sess_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		regenerate statedump -s $sess_name
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -1938,7 +2155,8 @@ function rotate_session ()
 	local expected_to_fail=$1
 	local sess_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		rotate $sess_name
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -1960,7 +2178,7 @@ function rotate_session_fail ()
 
 function destructive_tests_enabled ()
 {
-	if [ ${LTTNG_ENABLE_DESTRUCTIVE_TESTS} = "will-break-my-system" ]; then
+	if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
 		return 0
 	else
 		return 1
@@ -1973,7 +2191,8 @@ function lttng_enable_rotation_timer ()
 	local sess_name=$2
 	local period=$3
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-rotation -s $sess_name --timer $period
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -1999,7 +2218,8 @@ function lttng_enable_rotation_size ()
 	local sess_name=$2
 	local size=$3
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		enable-rotation -s $sess_name --size $size
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -2024,7 +2244,8 @@ function lttng_clear_session ()
 	local expected_to_fail=$1
 	local sess_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN clear $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		clear $sess_name
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -2046,6 +2267,335 @@ function lttng_clear_session_fail ()
 
 function lttng_clear_all ()
 {
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN clear --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	_run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
+		clear --all
 	ok $? "Clear all lttng sessions"
 }
+
+function lttng_add_trigger()
+{
+	local expected_to_fail="$1"
+	local trigger_name="$2"
+	shift 2
+	local args=("$@")
+
+	diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test "$ret" -ne "0"
+		ok $? "Add trigger $trigger_name failed as expected"
+	else
+		ok $ret "Add trigger $trigger_name"
+	fi
+}
+
+function lttng_remove_trigger()
+{
+	local expected_to_fail="$1"
+	local trigger_name="$2"
+	shift 2
+
+	diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
+	"$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test "$ret" -ne "0"
+		ok $? "Remove trigger $trigger_name failed as expected"
+	else
+		ok $ret "Remove trigger $trigger_name"
+	fi
+}
+
+function lttng_add_trigger_ok()
+{
+	lttng_add_trigger 0 "$@"
+}
+
+function lttng_add_trigger_fail()
+{
+	lttng_add_trigger 1 "$@"
+}
+
+function lttng_remove_trigger_ok()
+{
+	lttng_remove_trigger 0 "$@"
+}
+
+function list_triggers_matches_ok ()
+{
+	local tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
+	local tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
+
+	local test_name="$1"
+	local expected_stdout_file="$2"
+
+	diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
+
+	"$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
+	ok $? "${test_name}: exit code is 0"
+
+	diff -u "${expected_stdout_file}" "${tmp_stdout}"
+	ok $? "${test_name}: expected stdout"
+
+	diff -u /dev/null "${tmp_stderr}"
+	ok $? "${test_name}: expected stderr"
+
+	rm -f "${tmp_stdout}"
+	rm -f "${tmp_stderr}"
+}
+
+function list_triggers_matches_mi_ok ()
+{
+	local tmp_stdout
+	local tmp_stdout_raw
+	local tmp_stderr
+
+	local test_name="$1"
+	local expected_stdout_file="$2"
+
+	tmp_stdout_raw=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
+	tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
+	tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
+
+	diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
+
+	"$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
+	ok $? "${test_name}: exit code is 0"
+
+	# Pretty-fy xml before further test.
+	$XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
+
+	$MI_VALIDATE "${tmp_stdout}"
+	ok $? "list-trigger mi is valid"
+
+	diff -u "${expected_stdout_file}" "${tmp_stdout}"
+	ok $? "${test_name}: expected stdout"
+
+	diff -u /dev/null "${tmp_stderr}"
+	ok $? "${test_name}: expected stderr"
+
+	rm -f "${tmp_stdout}"
+	rm -f "${tmp_stdout_raw}"
+	rm -f "${tmp_stderr}"
+}
+
+function validate_path_pattern ()
+{
+	local message=$1
+	local pattern=$2
+	# Base path is only used in error case and is used to list the content
+	# of the base path.
+	local base_path=$3
+
+
+	[ -f $pattern ]
+	ret=$?
+	ok $ret "$message"
+
+	if [ "$ret" -ne "0" ]; then
+		diag "Path pattern expected: $pattern"
+		# List the tracepath for more info. We use find as a recursive
+		# directory lister.
+		diag "The base path content:"
+		find "$base_path" -print
+	fi
+}
+
+function validate_trace_path_ust_uid ()
+{
+	local trace_path=$1
+	local session_name=$2
+	local uid=$UID
+	local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
+
+	validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_ust_uid_network ()
+{
+	local trace_path=$1
+	local session_name=$2
+	local base_path=$3
+	local uid=$UID
+	local hostname=$HOSTNAME
+	local pattern
+	local ret
+
+	# If the session was given a network base path (e.g
+	# 127.0.0.1/my/custom/path on creation, there is no session name
+	# component to the path on the relayd side. Caller can simply not pass a
+	# session name for this scenario.
+	if [ -n "$session_name" ]; then
+		session_name="$session_name-$date_time_pattern"
+		if [ -n "$base_path" ]; then
+			fail "Session name and base path are mutually exclusive"
+			return
+		fi
+	fi
+
+	pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
+
+	validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_ust_uid_snapshot_network ()
+{
+	local trace_path=$1
+	local session_name=$2
+	local snapshot_name=$3
+	local snapshot_number=$4
+	local base_path=$5
+	local hostname=$HOSTNAME
+	local uid=$UID
+	local pattern
+	local ret
+
+	# If the session/output was given a network base path (e.g
+	# 127.0.0.1/my/custom/path on creation, there is no session name
+	# component to the path on the relayd side. Caller can simply not pass a
+	# session name for this scenario.
+	if [ -n "$session_name" ]; then
+		session_name="$session_name-$date_time_pattern"
+		if [ -n "$base_path" ]; then
+			fail "Session name and base path are mutually exclusive"
+			return
+		fi
+	fi
+
+	pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
+
+	validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_ust_uid_snapshot ()
+{
+	local trace_path=$1
+	local session_name=$2
+	local snapshot_name=$3
+	local snapshot_number=$4
+	local base_path=$5
+	local uid=$UID
+	local pattern
+	local ret
+
+	# If the session/output was given a network base path (e.g
+	# 127.0.0.1/my/custom/path) on creation, there is no session name
+	# component to the path on the relayd side. Caller can simply not pass a
+	# session name for this scenario.
+	if [ -n "$session_name" ]; then
+		session_name="$session_name-$date_time_pattern"
+		if [ -n "$base_path" ]; then
+			fail "Session name and base path are mutually exclusive"
+			return
+		fi
+	fi
+
+	pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
+
+	validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_ust_pid ()
+{
+	local trace_path=$1
+	local session_name=$2
+	local app_string=$3
+	local pid=$4
+	local pattern
+	local ret
+
+	# If the session was given a trace path on creation, there is no session
+	# name component to the path. Caller can simply not pass a session name
+	# for this scenario.
+	if [ -n "$session_name" ]; then
+		session_name="$session_name-$date_time_pattern"
+	fi
+
+	pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
+
+	validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_kernel ()
+{
+	local trace_path=$1
+	local session_name=$2
+	local pattern
+
+	# If the session was given a trace path on creation, there is no session
+	# name component to the path. Caller can simply not pass a session name
+	# for this scenario.
+	if [ -n "$session_name" ]; then
+		session_name="$session_name-$date_time_pattern"
+	fi
+
+	pattern="$trace_path/$session_name/kernel/metadata"
+
+	validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_kernel_network ()
+{
+	local trace_path=$1
+	local session_name=$2
+	local hostname=$HOSTNAME
+	local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
+
+	validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_kernel_snapshot ()
+{
+	local trace_path=$1
+	local session_name=$2
+	local snapshot_name=$3
+	local snapshot_number=$4
+	local base_path=$5
+	local pattern
+	local ret
+
+	# If the session/output was given a network base path (e.g
+	# 127.0.0.1/my/custom/path on creation, there is no session name
+	# component to the path on the relayd side. Caller can simply not pass a
+	# session name for this scenario.
+	if [ -n "$session_name" ]; then
+		session_name="$session_name-$date_time_pattern"
+		if [ -n "$base_path" ]; then
+			fail "Session name and base path are mutually exclusive"
+			return
+		fi
+	fi
+
+	pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
+
+	validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
+}
+
+function validate_trace_path_kernel_snapshot_network ()
+{
+	local trace_path=$1
+	local session_name=$2
+	local snapshot_name=$3
+	local snapshot_number=$4
+	local base_path=$5
+	local hostname=$HOSTNAME
+	local pattern
+	local ret
+
+	# If the session/output was given a network base path (e.g
+	# 127.0.0.1/my/custom/path on creation, there is no session name
+	# component to the path on the relayd side. Caller can simply not pass a
+	# session name for this scenario.
+	if [ -n "$session_name" ]; then
+		session_name="$session_name-$date_time_pattern"
+		if [ -n "$base_path" ]; then
+			fail "Session name and base path are mutually exclusive"
+			return
+		fi
+	fi
+
+	pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
+
+	validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
+}