X-Git-Url: https://git.lttng.org/?p=lttng-tools.git;a=blobdiff_plain;f=tests%2Futils%2Futils.sh;h=be83e73a622637812feb51fe3ff35d99029fc48a;hp=71ddaa2cb5ea5a5c0e814eb307d201dcb2189817;hb=6ac56bb9dd3adec203a6c689e37ea43f6f3ac19b;hpb=529bb942d15366eacc37f14df5d4404dfcd64893

diff --git a/tests/utils/utils.sh b/tests/utils/utils.sh
index 71ddaa2cb..be83e73a6 100644
--- a/tests/utils/utils.sh
+++ b/tests/utils/utils.sh
@@ -1,17 +1,7 @@
-# Copyright (C) - 2012 David Goulet <dgoulet@efficios.com>
+# Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
 #
-# This library is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by the Free
-# Software Foundation; version 2.1 of the License.
+# SPDX-License-Identifier: LGPL-2.1-only
 #
-# This library is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this library; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 
 SESSIOND_BIN="lttng-sessiond"
 SESSIOND_MATCH=".*lttng-sess.*"
@@ -46,31 +36,115 @@ export LTTNG_SESSIOND_PATH="/bin/true"
 
 source $TESTDIR/utils/tap/tap.sh
 
-if [ -z $LTTNG_TEST_TEARDOWN_TIMEOUT ]; then
+if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
 	LTTNG_TEST_TEARDOWN_TIMEOUT=60
 fi
 
-function full_cleanup ()
+# Enable job monitor mode.
+# Here we are mostly interested in the following from the monitor mode:
+#    All processes run in a separate process group.
+# This allows us to ensure that all subprocesses from all background tasks are
+# cleaned up correctly using signal to process group id.
+set -m
+
+kill_background_jobs ()
+{
+	local pids
+	pids=$(jobs -p)
+
+	if [ -z "$pids" ]; then
+		# Empty
+		return 0
+	fi
+
+	while read -r pid;
+	do
+		# Use negative number to send the signal to the process group.
+		# This ensure that any subprocesses receive the signal.
+		# /dev/null is used since there is an acceptable race between
+		# the moments the pids are listed and the moment we send a
+		# signal.
+		kill -SIGTERM -- "-$pid" 2>/dev/null
+	done <<< "$pids"
+}
+
+function cleanup ()
 {
 	# Try to kill daemons gracefully
-	stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
-	stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
+	stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
+	stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
 
 	# If daemons are still present, forcibly kill them
-	stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
-	stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
-	stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
+	stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
+	stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
+	stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
 
-	# Disable trap for SIGTERM since the following kill to the
-	# pidgroup will be SIGTERM. Otherwise it loops.
-	# The '-' before the pid number ($$) indicates 'kill' to signal the
-	# whole process group.
-	trap - SIGTERM && kill -- -$$
+	kill_background_jobs
 }
 
+function full_cleanup ()
+{
+	cleanup
+	exit 1
+}
+
+function LTTNG_BAIL_OUT ()
+{
+	cleanup
+	BAIL_OUT "$@"
+}
+
+function null_pipes ()
+{
+	exec 0>/dev/null
+	exec 1>/dev/null
+	exec 2>/dev/null
+}
 
 trap full_cleanup SIGINT SIGTERM
 
+# perl prove closes its child pipes before giving it a chance to run its
+# signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
+# to allow those trap handlers to proceed.
+
+trap null_pipes SIGPIPE
+
+# Check pgrep from env, default to pgrep if none
+if [ -z "$PGREP" ]; then
+	PGREP=pgrep
+fi
+
+# Due to the renaming of threads we need to use the full command (pgrep -f) to
+# identify the pids for multiple lttng related processes. The problem with "pgrep
+# -f" is that it ends up also looking at the arguments. We use a two stage
+# lookup. The first one is using "pgrep -f" yielding potential candidate.
+# The second on perform grep on the basename of the first field of the
+# /proc/pid/cmdline of the previously identified pids. The first field
+# correspond to the actual command.
+function lttng_pgrep ()
+{
+	local pattern=$1
+	local possible_pids
+	local full_command_no_argument
+	local command_basename
+
+	possible_pids=$($PGREP -f "$pattern")
+	if [ -z "$possible_pids" ]; then
+		return 0
+	fi
+
+	while IFS= read -r pid ; do
+		# /proc/pid/cmdline is null separated.
+		if full_command_no_argument=$(cut -d '' -f 1 2>/dev/null < /proc/"$pid"/cmdline); then
+			command_basename=$(basename "$full_command_no_argument")
+			if grep -q "$pattern" <<< "$command_basename"; then
+				echo "$pid"
+			fi
+		fi
+	done <<< "$possible_pids"
+	return 0
+}
+
 function print_ok ()
 {
 	# Check if we are a terminal
@@ -136,10 +210,19 @@ function conf_proc_count()
 # Bail out on failure
 function validate_lttng_modules_present ()
 {
+	# Check for loadable modules.
 	modprobe -n lttng-tracer 2>/dev/null
-	if [ $? -ne 0  ]; then
-		BAIL_OUT "LTTng modules not detected."
+	if [ $? -eq 0 ]; then
+		return 0
+	fi
+
+	# Check for builtin modules.
+	ls /proc/lttng > /dev/null 2>&1
+	if [ $? -eq 0 ]; then
+		return 0
 	fi
+
+	LTTNG_BAIL_OUT "LTTng modules not detected."
 }
 
 function enable_kernel_lttng_event
@@ -277,6 +360,28 @@ function lttng_disable_kernel_syscall_fail()
 	lttng_disable_kernel_syscall 1 "$@"
 }
 
+function lttng_enable_kernel_function_event ()
+{
+	local expected_to_fail="$1"
+	local sess_name="$2"
+	local target="$3"
+	local event_name="$4"
+
+	"$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test $ret -ne "0"
+		ok $? "Enable kernel function event for session $sess_name failed as expected"
+	else
+		ok $ret "Enable kernel function event for session $sess_name"
+	fi
+}
+
+function lttng_enable_kernel_function_event_ok ()
+{
+	lttng_enable_kernel_function_event 0 "$@"
+}
+
 function lttng_enable_kernel_userspace_probe_event ()
 {
 	local expected_to_fail="$1"
@@ -318,9 +423,9 @@ function lttng_enable_kernel_channel()
 	local expected_to_fail=$2
 	local sess_name=$3
 	local channel_name=$4
-	local opt=$5
+	local opts="${@:5}"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -384,22 +489,23 @@ function lttng_disable_kernel_channel_fail()
 function start_lttng_relayd_opt()
 {
 	local withtap=$1
-	local opt=$2
+	local process_mode=$2
+	local opt=$3
 
 	DIR=$(readlink -f "$TESTDIR")
 
-	if [ -z "$(pgrep "$RELAYD_MATCH")" ]; then
+	if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 		# shellcheck disable=SC2086
-		"$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
-		#"$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" $opt -vvv >>/tmp/relayd.log 2>&1 &
+		$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+		#$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 		if [ $? -eq 1 ]; then
-			if [ "$withtap" -eq "1" ]; then
-				fail "Start lttng-relayd (opt: $opt)"
+			if [ $withtap -eq "1" ]; then
+				fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 			fi
 			return 1
 		else
-			if [ "$withtap" -eq "1" ]; then
-				pass "Start lttng-relayd (opt: $opt)"
+			if [ $withtap -eq "1" ]; then
+				pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 			fi
 		fi
 	else
@@ -409,38 +515,43 @@ function start_lttng_relayd_opt()
 
 function start_lttng_relayd()
 {
-	start_lttng_relayd_opt 1 "$@"
+	start_lttng_relayd_opt 1 "-b" "$@"
 }
 
 function start_lttng_relayd_notap()
 {
-	start_lttng_relayd_opt 0 "$@"
+	start_lttng_relayd_opt 0 "-b" "$@"
 }
 
 function stop_lttng_relayd_opt()
 {
 	local withtap=$1
-	local signal=$2
+	local is_cleanup=$2
+	local signal=$3
+	local timeout_s=$4
+	local dtimeleft_s=
+	local retval=0
+	local pids
 
 	if [ -z "$signal" ]; then
 		signal="SIGTERM"
 	fi
 
-	local timeout_s=$3
-	local dtimeleft_s=
 
 	# Multiply time by 2 to simplify integer arithmetic
 	if [ -n "$timeout_s" ]; then
 		dtimeleft_s=$((timeout_s * 2))
 	fi
 
-	local retval=0
-	local pids=
 
-	pids=$(pgrep "$RELAYD_MATCH")
+	pids=$(lttng_pgrep "$RELAYD_MATCH")
 	if [ -z "$pids" ]; then
-		if [ "$withtap" -eq "1" ]; then
-			pass "No relay daemon to kill"
+		if [ "$is_cleanup" -eq 1 ]; then
+			:
+		elif [ "$withtap" -eq "1" ]; then
+			fail "No relay daemon to kill"
+		else
+			LTTNG_BAIL_OUT "No relay daemon to kill"
 		fi
 		return 0
 	fi
@@ -456,7 +567,7 @@ function stop_lttng_relayd_opt()
 	else
 		out=1
 		while [ -n "$out" ]; do
-			out=$(pgrep "$RELAYD_MATCH")
+			out=$(lttng_pgrep "$RELAYD_MATCH")
 			if [ -n "$dtimeleft_s" ]; then
 				if [ $dtimeleft_s -lt 0 ]; then
 					out=
@@ -479,12 +590,17 @@ function stop_lttng_relayd_opt()
 
 function stop_lttng_relayd()
 {
-	stop_lttng_relayd_opt 1 "$@"
+	stop_lttng_relayd_opt 1 0 "$@"
 }
 
 function stop_lttng_relayd_notap()
 {
-	stop_lttng_relayd_opt 0 "$@"
+	stop_lttng_relayd_opt 0 0 "$@"
+}
+
+function stop_lttng_relayd_cleanup()
+{
+	stop_lttng_relayd_opt 0 1 "$@"
 }
 
 #First arg: show tap output
@@ -494,6 +610,9 @@ function start_lttng_sessiond_opt()
 	local withtap=$1
 	local load_path=$2
 
+	# The rest of the arguments will be passed directly to lttng-sessiond.
+	shift 2
+
 	local env_vars=""
 	local consumerd=""
 
@@ -528,20 +647,20 @@ function start_lttng_sessiond_opt()
 
 	if ! validate_kernel_version; then
 	    fail "Start session daemon"
-	    BAIL_OUT "*** Kernel too old for session daemon tests ***"
+	    LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 	fi
 
 	: "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 	export LTTNG_SESSION_CONFIG_XSD_PATH
 
-	if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
+	if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 		# Have a load path ?
 		if [ -n "$load_path" ]; then
 			# shellcheck disable=SC2086
-			env $env_vars --load "$load_path" --background "$consumerd"
+			env $env_vars --load "$load_path" --background "$consumerd" "$@"
 		else
 			# shellcheck disable=SC2086
-			env $env_vars --background "$consumerd"
+			env $env_vars --background "$consumerd" "$@"
 		fi
 		#$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 		status=$?
@@ -564,15 +683,18 @@ function start_lttng_sessiond_notap()
 function stop_lttng_sessiond_opt()
 {
 	local withtap=$1
-	local signal=$2
+	local is_cleanup=$2
+	local signal=$3
+	local timeout_s=$4
+	local dtimeleft_s=
+	local retval=0
+	local runas_pids
+	local pids
 
 	if [ -z "$signal" ]; then
 		signal=SIGTERM
 	fi
 
-	local timeout_s=$3
-	local dtimeleft_s=
-
 	# Multiply time by 2 to simplify integer arithmetic
 	if [ -n "$timeout_s" ]; then
 		dtimeleft_s=$((timeout_s * 2))
@@ -583,21 +705,20 @@ function stop_lttng_sessiond_opt()
 		return 0
 	fi
 
-	local retval=0
-
-	local runas_pids=
-	runas_pids=$(pgrep "$RUNAS_MATCH")
-
-	local pids=
-	pids=$(pgrep "$SESSIOND_MATCH")
+	runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
+	pids=$(lttng_pgrep "$SESSIOND_MATCH")
 
 	if [ -n "$runas_pids" ]; then
 		pids="$pids $runas_pids"
 	fi
 
 	if [ -z "$pids" ]; then
-		if [ "$withtap" -eq "1" ]; then
-			pass "No session daemon to kill"
+		if [ "$is_cleanup" -eq 1 ]; then 
+			:
+		elif [ "$withtap" -eq "1" ]; then
+			fail "No session daemon to kill"
+		else
+			LTTNG_BAIL_OUT "No session daemon to kill"
 		fi
 		return 0
 	fi
@@ -613,7 +734,7 @@ function stop_lttng_sessiond_opt()
 	else
 		out=1
 		while [ -n "$out" ]; do
-			out=$(pgrep "${SESSIOND_MATCH}")
+			out=$(lttng_pgrep "${SESSIOND_MATCH}")
 			if [ -n "$dtimeleft_s" ]; then
 				if [ $dtimeleft_s -lt 0 ]; then
 					out=
@@ -625,7 +746,7 @@ function stop_lttng_sessiond_opt()
 		done
 		out=1
 		while [ -n "$out" ]; do
-			out=$(pgrep "$CONSUMERD_MATCH")
+			out=$(lttng_pgrep "$CONSUMERD_MATCH")
 			if [ -n "$dtimeleft_s" ]; then
 				if [ $dtimeleft_s -lt 0 ]; then
 					out=
@@ -651,7 +772,7 @@ function stop_lttng_sessiond_opt()
 
 			if [ -n "$modules" ]; then
 				diag "Unloading all LTTng modules"
-				modprobe -r "$modules"
+				modprobe --remove "$modules"
 			fi
 		fi
 	fi
@@ -661,44 +782,50 @@ function stop_lttng_sessiond_opt()
 
 function stop_lttng_sessiond()
 {
-	stop_lttng_sessiond_opt 1 "$@"
+	stop_lttng_sessiond_opt 1 0 "$@"
 }
 
 function stop_lttng_sessiond_notap()
 {
-	stop_lttng_sessiond_opt 0 "$@"
+	stop_lttng_sessiond_opt 0 0 "$@"
+}
+
+function stop_lttng_sessiond_cleanup()
+{
+	stop_lttng_sessiond_opt 0 1 "$@"
 }
 
 function sigstop_lttng_sessiond_opt()
 {
 	local withtap=$1
 	local signal=SIGSTOP
+	local pids
 
 	if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 		# Env variable requested no session daemon
 		return
 	fi
 
-	PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
+	pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 
 	if [ "$withtap" -eq "1" ]; then
-		diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
+		diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 	fi
 
 	# shellcheck disable=SC2086
-	if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
+	if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 		if [ "$withtap" -eq "1" ]; then
 			fail "Sending SIGSTOP to session daemon"
 		fi
 	else
 		out=1
 		while [ $out -ne 0 ]; do
-			pid="$(pgrep "$SESSIOND_MATCH")"
+			pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 
 			# Wait until state becomes stopped for session
 			# daemon(s).
 			out=0
-			for sessiond_pid in $pid; do
+			for sessiond_pid in $pids; do
 				state="$(ps -p "$sessiond_pid" -o state= )"
 				if [[ -n "$state" && "$state" != "T" ]]; then
 					out=1
@@ -725,35 +852,39 @@ function sigstop_lttng_sessiond_notap()
 function stop_lttng_consumerd_opt()
 {
 	local withtap=$1
-	local signal=$2
+	local is_cleanup=$2
+	local signal=$3
+	local timeout_s=$4
+	local dtimeleft_s=
+	local retval=0
+	local pids
 
 	if [ -z "$signal" ]; then
 		signal=SIGTERM
 	fi
 
-	local timeout_s=$3
-	local dtimeleft_s=
-
 	# Multiply time by 2 to simplify integer arithmetic
 	if [ -n "$timeout_s" ]; then
 		dtimeleft_s=$((timeout_s * 2))
 	fi
 
-	local retval=0
-
-	PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
+	pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 
-	if [ -z "$PID_CONSUMERD" ]; then
-		if [ "$withtap" -eq "1" ]; then
-			pass "No consumer daemon to kill"
+	if [ -z "$pids" ]; then
+		if [ "$is_cleanup" -eq 1 ]; then
+			:
+		elif [ "$withtap" -eq "1" ]; then
+			fail "No consumerd daemon to kill"
+		else
+			LTTNG_BAIL_OUT "No consumerd daemon to kill"
 		fi
 		return 0
 	fi
 
-	diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
+	diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 
 	# shellcheck disable=SC2086
-	if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
+	if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 		retval=1
 		if [ "$withtap" -eq "1" ]; then
 			fail "Kill consumer daemon"
@@ -761,12 +892,12 @@ function stop_lttng_consumerd_opt()
 	else
 		out=1
 		while [ $out -ne 0 ]; do
-			pid="$(pgrep "$CONSUMERD_MATCH")"
+			pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 
 			# If consumerds are still present check their status.
 			# A zombie status qualifies the consumerd as *killed*
 			out=0
-			for consumer_pid in $pid; do
+			for consumer_pid in $pids; do
 				state="$(ps -p "$consumer_pid" -o state= )"
 				if [[ -n "$state" && "$state" != "Z" ]]; then
 					out=1
@@ -795,25 +926,31 @@ function stop_lttng_consumerd_opt()
 
 function stop_lttng_consumerd()
 {
-	stop_lttng_consumerd_opt 1 "$@"
+	stop_lttng_consumerd_opt 1 0 "$@"
 }
 
 function stop_lttng_consumerd_notap()
 {
-	stop_lttng_consumerd_opt 0 "$@"
+	stop_lttng_consumerd_opt 0 0 "$@"
+}
+
+function stop_lttng_consumerd_cleanup()
+{
+	stop_lttng_consumerd_opt 0 1 "$@"
 }
 
 function sigstop_lttng_consumerd_opt()
 {
 	local withtap=$1
 	local signal=SIGSTOP
+	local pids
 
-	PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
+	pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 
-	diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
+	diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 
 	# shellcheck disable=SC2086
-	kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	retval=$?
 
 	if [ $retval -eq 1 ]; then
@@ -824,12 +961,12 @@ function sigstop_lttng_consumerd_opt()
 	else
 		out=1
 		while [ $out -ne 0 ]; do
-			pid="$(pgrep "$CONSUMERD_MATCH")"
+			pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 
 			# Wait until state becomes stopped for all
 			# consumers.
 			out=0
-			for consumer_pid in $pid; do
+			for consumer_pid in $pids; do
 				state="$(ps -p "$consumer_pid" -o state= )"
 				if [[ -n "$state" && "$state" != "T" ]]; then
 					out=1
@@ -887,7 +1024,14 @@ function create_lttng_session ()
 	local trace_path=$4
 	local opt=$5
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path $opt > $OUTPUT_DEST
+	if [ -z "$trace_path" ]; then
+		# Use lttng-sessiond default output.
+		trace_path=""
+	else
+		trace_path="-o $trace_path"
+	fi
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN create "$sess_name" $trace_path $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
 	if [ $expected_to_fail -eq "1" ]; then
 		test "$ret" -ne "0"
@@ -925,9 +1069,9 @@ function enable_ust_lttng_channel ()
 	local expected_to_fail=$2
 	local sess_name=$3
 	local channel_name=$4
-	local opt=$5
+	local opts="${@:5}"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
@@ -1159,8 +1303,16 @@ function enable_ust_lttng_event_filter()
 	local sess_name="$1"
 	local event_name="$2"
 	local filter="$3"
+	local channel_name=$4
+
+	if [ -z $channel_name ]; then
+		# default channel if none specified
+		chan=""
+	else
+		chan="-c $channel_name"
+	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event $chan "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable event $event_name with filtering for session $sess_name"
 }
 
@@ -1348,14 +1500,15 @@ function lttng_snapshot_add_output ()
 	local expected_to_fail=$1
 	local sess_name=$2
 	local trace_path=$3
+	local opts=$4
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
 	if [[ $expected_to_fail -eq 1 ]]; then
 		test "$ret" -ne "0"
-		ok $? "Added snapshot output file://$trace_path failed as expected"
+		ok $? "Added snapshot output $trace_path failed as expected"
 	else
-		ok $ret "Added snapshot output file://$trace_path"
+		ok $ret "Added snapshot output $trace_path"
 	fi
 }
 
@@ -1400,7 +1553,7 @@ function lttng_snapshot_record ()
 	local sess_name=$1
 	local trace_path=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s "$sess_name" "$trace_path" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Snapshot recorded"
 }
 
@@ -1447,8 +1600,9 @@ function lttng_load_fail()
 
 function lttng_track()
 {
-	local expected_to_fail=$1
-	local opts=$2
+	local expected_to_fail="$1"
+	shift 1
+	local opts="$@"
 	$TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
@@ -1471,8 +1625,9 @@ function lttng_track_fail()
 
 function lttng_untrack()
 {
-	local expected_to_fail=$1
-	local opts=$2
+	local expected_to_fail="$1"
+	shift 1
+	local opts="$@"
 	$TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
@@ -1506,6 +1661,46 @@ function lttng_untrack_kernel_all_ok()
 	ok $? "Lttng untrack all pid on the kernel domain"
 }
 
+function lttng_track_ust_ok()
+{
+	lttng_track_ok -u "$@"
+}
+
+function lttng_track_ust_fail()
+{
+	lttng_track_fail -u "$@"
+}
+
+function lttng_track_kernel_ok()
+{
+	lttng_track_ok -k "$@"
+}
+
+function lttng_track_kernel_fail()
+{
+	lttng_track_fail -k "$@"
+}
+
+function lttng_untrack_ust_ok()
+{
+	lttng_untrack_ok -u "$@"
+}
+
+function lttng_untrack_ust_fail()
+{
+	lttng_untrack_fail -u "$@"
+}
+
+function lttng_untrack_kernel_ok()
+{
+	lttng_untrack_ok -k "$@"
+}
+
+function lttng_untrack_kernel_fail()
+{
+	lttng_untrack_fail -k "$@"
+}
+
 function lttng_add_context_list()
 {
 	$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
@@ -1551,6 +1746,32 @@ function add_context_kernel_fail()
 	add_context_lttng 1 -k "$@"
 }
 
+function wait_live_trace_ready ()
+{
+	local url=$1
+	local zero_client_match=0
+
+	diag "Waiting for live trace at url: $url"
+	while [ $zero_client_match -eq 0 ]; do
+		zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
+		sleep 0.5
+	done
+	pass "Waiting for live trace at url: $url"
+}
+
+function wait_live_viewer_connect ()
+{
+	local url=$1
+	local one_client_match=0
+
+	diag "Waiting for live viewers on url: $url"
+	while [ $one_client_match -eq 0 ]; do
+		one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
+		sleep 0.5
+	done
+	pass "Waiting for live viewers on url: $url"
+}
+
 function validate_metadata_event ()
 {
 	local event_name=$1
@@ -1668,6 +1889,36 @@ function validate_trace_count
 	ok $? "Read a total of $cnt events, expected $expected_count"
 }
 
+function validate_trace_count_range_incl_min_excl_max
+{
+	local event_name=$1
+	local trace_path=$2
+	local expected_min=$3
+	local expected_max=$4
+
+	which $BABELTRACE_BIN >/dev/null
+	if [ $? -ne 0 ]; then
+	    skip 0 "Babeltrace binary not found. Skipping trace validation"
+	fi
+
+	cnt=0
+	OLDIFS=$IFS
+	IFS=","
+	for i in $event_name; do
+		traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
+		if [ "$traced" -ge $expected_min ]; then
+			pass "Validate trace for event $i, $traced events"
+		else
+			fail "Validate trace for event $i"
+			diag "Found $traced occurences of $i"
+		fi
+		cnt=$(($cnt + $traced))
+	done
+	IFS=$OLDIFS
+	test $cnt -lt $expected_max
+	ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
+}
+
 function trace_first_line
 {
 	local trace_path=$1
@@ -1747,6 +1998,33 @@ function validate_trace_empty()
 	return $ret
 }
 
+function validate_directory_empty ()
+{
+	local trace_path="$1"
+
+	# Do not double quote `$trace_path` below as we want wildcards to be
+	# expanded.
+	files="$(ls -A $trace_path)"
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		fail "Failed to list content of directory \"$trace_path\""
+		return $ret
+	fi
+
+	nb_files="$(echo -n "$files" | wc -l)"
+	ok $nb_files "Directory \"$trace_path\" is empty"
+}
+
+function validate_trace_session_ust_empty()
+{
+	validate_directory_empty "$1"/ust
+}
+
+function validate_trace_session_kernel_empty()
+{
+	validate_trace_empty "$1"/kernel
+}
+
 function regenerate_metadata ()
 {
 	local expected_to_fail=$1
@@ -1783,7 +2061,7 @@ function regenerate_statedump ()
 		test "$ret" -ne "0"
 		ok $? "Expected fail on regenerate statedump $sess_name"
 	else
-		ok $ret "Metadata regenerate $sess_name"
+		ok $ret "Statedump regenerate $sess_name"
 	fi
 }
 
@@ -1882,3 +2160,80 @@ function lttng_enable_rotation_size_fail ()
 {
 	lttng_enable_rotation_size 1 $@
 }
+
+function lttng_clear_session ()
+{
+	local expected_to_fail=$1
+	local sess_name=$2
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN clear $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test "$ret" -ne "0"
+		ok $? "Expected fail on clear session $sess_name"
+	else
+		ok $ret "Clear session $sess_name"
+	fi
+}
+
+function lttng_clear_session_ok ()
+{
+	lttng_clear_session 0 $@
+}
+
+function lttng_clear_session_fail ()
+{
+	lttng_clear_session 1 $@
+}
+
+function lttng_clear_all ()
+{
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN clear --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ok $? "Clear all lttng sessions"
+}
+
+function lttng_add_trigger()
+{
+	local expected_to_fail="$1"
+	local trigger_name="$2"
+	shift 2
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --id "$trigger_name" "$@" 1> /dev/null 2> /dev/null
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test "$ret" -ne "0"
+		ok $? "Add trigger $trigger_name failed as expected"
+	else
+		ok $ret "Add trigger $trigger_name"
+	fi
+}
+
+function lttng_remove_trigger()
+{
+	local expected_to_fail="$1"
+	local trigger_name="$2"
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger "$trigger_name" 1> /dev/null 2> /dev/null
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test "$ret" -ne "0"
+		ok $? "Remove trigger $trigger_name failed as expected"
+	else
+		ok $ret "Remove trigger $trigger_name"
+	fi
+}
+
+function lttng_add_trigger_ok()
+{
+	lttng_add_trigger 0 "$@"
+}
+
+function lttng_add_trigger_fail()
+{
+	lttng_add_trigger 1 "$@"
+}
+
+function lttng_remove_trigger_ok()
+{
+	lttng_remove_trigger 0 "$@"
+}