X-Git-Url: https://git.lttng.org/?p=lttng-tools.git;a=blobdiff_plain;f=tests%2Futils%2Futils.sh;h=fe834cfaf47b1b2c3e394a20e880c987a4f15276;hp=2647769f27106087ecf4f72e0c92d7590b652fa5;hb=19356f3ab163d5ee161900db49247996f49dfc35;hpb=8cfcd41c7fac2e14d8f8527238d928bd4ec732fb

diff --git a/tests/utils/utils.sh b/tests/utils/utils.sh
index 2647769f2..fe834cfaf 100644
--- a/tests/utils/utils.sh
+++ b/tests/utils/utils.sh
@@ -16,17 +16,30 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 
 SESSIOND_BIN="lttng-sessiond"
+RUNAS_BIN="lttng-runas"
 CONSUMERD_BIN="lttng-consumerd"
 RELAYD_BIN="lttng-relayd"
 LTTNG_BIN="lttng"
 BABELTRACE_BIN="babeltrace"
-OUTPUT_DEST=/dev/null 2>&1
+OUTPUT_DEST=/dev/null
+ERROR_OUTPUT_DEST=/dev/null
 
 # Minimal kernel version supported for session daemon tests
 KERNEL_MAJOR_VERSION=2
 KERNEL_MINOR_VERSION=6
 KERNEL_PATCHLEVEL_VERSION=27
 
+# We set the default UST register timeout to "wait forever", so that
+# basic tests don't have to worry about hitting timeouts on busy
+# systems. Specialized tests should test those corner-cases.
+export LTTNG_UST_REGISTER_TIMEOUT=-1
+
+# We set the default lttng-sessiond path to /bin/true to prevent the spawning
+# of a daemonized sessiond. This is necessary since 'lttng create' will spawn
+# its own sessiond if none is running. It also ensures that 'lttng create'
+# fails when no sessiond is running.
+export LTTNG_SESSIOND_PATH="/bin/true"
+
 source $TESTDIR/utils/tap/tap.sh
 
 function print_ok ()
@@ -76,30 +89,63 @@ function validate_kernel_version ()
 function randstring()
 {
 	[ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
-	cat /dev/urandom | tr -cd "$CHAR" | head -c ${1:-16}
+	cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 	echo
 }
 
-function lttng_enable_kernel_event
+# Return the number of _configured_ CPUs.
+function conf_proc_count()
 {
-	local sess_name=$1
-	local event_name=$2
-	local channel_name=$3
+	getconf _NPROCESSORS_CONF
+	if [ $? -ne 0 ]; then
+		diag "Failed to get the number of configured CPUs"
+	fi
+	echo
+}
 
-	if [ -z $event_name ]; then
+function enable_kernel_lttng_event
+{
+	local expected_to_fail="$1"
+	local sess_name="$2"
+	local event_name="$3"
+	local channel_name="$4"
+
+	if [ -z "$event_name" ]; then
 		# Enable all event if no event name specified
 		event_name="-a"
 	fi
 
-	if [ -z $channel_name ]; then
+	if [ -z "$channel_name" ]; then
 		# default channel if none specified
 		chan=""
 	else
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k >$OUTPUT_DEST
-	ok $? "Enable kernel event $event_name for session $sess_name"
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test $ret -ne "0"
+		ok $? "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
+	else
+		ok $ret "Enable kernel event $event_name for session $sess_name"
+	fi
+}
+
+function enable_kernel_lttng_event_ok ()
+{
+	enable_kernel_lttng_event 0 "$@"
+}
+
+function enable_kernel_lttng_event_fail ()
+{
+	enable_kernel_lttng_event 1 "$@"
+}
+
+# Old interface
+function lttng_enable_kernel_event
+{
+	enable_kernel_lttng_event_ok "$@"
 }
 
 function lttng_enable_kernel_syscall()
@@ -121,7 +167,7 @@ function lttng_enable_kernel_syscall()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test $ret -ne "0"
@@ -133,12 +179,12 @@ function lttng_enable_kernel_syscall()
 
 function lttng_enable_kernel_syscall_ok()
 {
-	lttng_enable_kernel_syscall 0 ${*}
+	lttng_enable_kernel_syscall 0 "$@"
 }
 
 function lttng_enable_kernel_syscall_fail()
 {
-	lttng_enable_kernel_syscall 1 ${*}
+	lttng_enable_kernel_syscall 1 "$@"
 }
 
 function lttng_disable_kernel_syscall()
@@ -160,12 +206,12 @@ function lttng_disable_kernel_syscall()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 
 	ret=$?
 	if [[ $expected_to_fail -eq "1" ]]; then
 		test $ret -ne "0"
-		ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
+		ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 	else
 		ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 	fi
@@ -173,70 +219,148 @@ function lttng_disable_kernel_syscall()
 
 function lttng_disable_kernel_syscall_ok()
 {
-	lttng_disable_kernel_syscall 0 ${*}
+	lttng_disable_kernel_syscall 0 "$@"
 }
 
 function lttng_disable_kernel_syscall_fail()
 {
-	lttng_disable_kernel_syscall 1 ${*}
+	lttng_disable_kernel_syscall 1 "$@"
 }
 
-function start_lttng_relayd
+function lttng_enable_kernel_channel()
 {
-	local opt=$1
+	local expected_to_fail=$1
+	local sess_name=$2
+	local channel_name=$3
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test "$ret" -ne "0"
+		ok $? "Enable channel $channel_name for session $sess_name failed as expected"
+	else
+		ok $ret "Enable channel $channel_name for session $sess_name"
+	fi
+}
+
+function lttng_enable_kernel_channel_ok()
+{
+	lttng_enable_kernel_channel 0 "$@"
+}
+
+function lttng_enable_kernel_channel_fail()
+{
+	lttng_enable_kernel_channel 1 "$@"
+}
+
+function lttng_disable_kernel_channel()
+{
+	local expected_to_fail=$1
+	local sess_name=$2
+	local channel_name=$3
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test "$ret" -ne "0"
+		ok $? "Disable channel $channel_name for session $sess_name failed as expected"
+	else
+		ok $ret "Disable channel $channel_name for session $sess_name"
+	fi
+}
+
+function lttng_disable_kernel_channel_ok()
+{
+	lttng_disable_kernel_channel 0 "$@"
+}
+
+function lttng_disable_kernel_channel_fail()
+{
+	lttng_disable_kernel_channel 1 "$@"
+}
+
+function start_lttng_relayd_opt()
+{
+	local withtap=$1
+	local opt=$2
 
 	DIR=$(readlink -f $TESTDIR)
 
-	if [ -z $(pidof lt-$RELAYD_BIN) ]; then
-		$DIR/../src/bin/lttng-relayd/$RELAYD_BIN -b $opt >$OUTPUT_DEST
+	if [ -z $(pgrep --full lt-$RELAYD_BIN) ]; then
+		$DIR/../src/bin/lttng-relayd/$RELAYD_BIN -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 		#$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 		if [ $? -eq 1 ]; then
-			fail "Start lttng-relayd (opt: $opt)"
+			if [ $withtap -eq "1" ]; then
+				fail "Start lttng-relayd (opt: $opt)"
+			fi
 			return 1
 		else
-			pass "Start lttng-relayd (opt: $opt)"
+			if [ $withtap -eq "1" ]; then
+				pass "Start lttng-relayd (opt: $opt)"
+			fi
 		fi
 	else
 		pass "Start lttng-relayd (opt: $opt)"
 	fi
 }
 
-function stop_lttng_relayd_nocheck
+function start_lttng_relayd()
 {
-	PID_RELAYD=`pidof lt-$RELAYD_BIN`
+	start_lttng_relayd_opt 1 "$@"
+}
 
-	diag "Killing lttng-relayd (pid: $PID_RELAYD)"
-	kill $PID_RELAYD >$OUTPUT_DEST
+function start_lttng_relayd_notap()
+{
+	start_lttng_relayd_opt 0 "$@"
+}
+
+function stop_lttng_relayd_opt()
+{
+	local withtap=$1
+
+	PID_RELAYD=`pgrep --full lt-$RELAYD_BIN`
+
+	if [ $withtap -eq "1" ]; then
+		diag "Killing lttng-relayd (pid: $PID_RELAYD)"
+	fi
+	kill $PID_RELAYD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	retval=$?
 
-	if [ $retval -eq 1 ]; then
+	if [ $? -eq 1 ]; then
+		if [ $withtap -eq "1" ]; then
+			fail "Kill relay daemon"
+		fi
+		return 1
+	else
 		out=1
 		while [ -n "$out" ]; do
-			out=$(pidof lt-$RELAYD_BIN)
+			out=$(pgrep --full lt-$RELAYD_BIN)
 			sleep 0.5
 		done
+		if [ $withtap -eq "1" ]; then
+			pass "Kill relay daemon"
+		fi
 	fi
 	return $retval
 }
 
-function stop_lttng_relayd
+function stop_lttng_relayd()
 {
-	stop_lttng_relayd_nocheck
+	stop_lttng_relayd_opt 1 "$@"
+}
 
-	if [ $? -eq 1 ]; then
-		fail "Killed lttng-relayd (pid: $PID_RELAYD)"
-		return 1
-	else
-		pass "Killed lttng-relayd (pid: $PID_RELAYD)"
-		return 0
-	fi
+function stop_lttng_relayd_notap()
+{
+	stop_lttng_relayd_opt 0 "$@"
 }
 
-#First argument: load path for automatic loading
-function start_lttng_sessiond()
+#First arg: show tap output
+#Second argument: load path for automatic loading
+function start_lttng_sessiond_opt()
 {
+	local withtap=$1
+	local load_path=$2
 
-	local load_path="$1"
 	if [ -n $TEST_NO_SESSIOND ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 		# Env variable requested no session daemon
 		return
@@ -252,52 +376,255 @@ function start_lttng_sessiond()
 	: ${LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/config/}
 	export LTTNG_SESSION_CONFIG_XSD_PATH
 
-	if [ -z $(pidof lt-$SESSIOND_BIN) ]; then
+	if [ -z $(pgrep --full lt-$SESSIOND_BIN) ]; then
 		# Have a load path ?
-		if [ -n "$1" ]; then
-			$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --load "$1" --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
+		if [ -n "$load_path" ]; then
+			$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --load "$load_path" --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 		else
 			$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 		fi
-		#$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1 &
+		#$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 		status=$?
-		ok $status "Start session daemon"
+		if [ $withtap -eq "1" ]; then
+			ok $status "Start session daemon"
+		fi
 	fi
 }
 
-function stop_lttng_sessiond ()
+function start_lttng_sessiond()
 {
+	start_lttng_sessiond_opt 1 "$@"
+}
+
+function start_lttng_sessiond_notap()
+{
+	start_lttng_sessiond_opt 0 "$@"
+}
+
+function stop_lttng_sessiond_opt()
+{
+	local withtap=$1
+	local signal=$2
+	local kill_opt=""
+
 	if [ -n $TEST_NO_SESSIOND ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 		# Env variable requested no session daemon
 		return
 	fi
 
-	PID_SESSIOND=`pidof lt-$SESSIOND_BIN`
+	PID_SESSIOND="$(pgrep --full lt-$SESSIOND_BIN) $(pgrep --full $RUNAS_BIN)"
 
-	kill $PID_SESSIOND >$OUTPUT_DEST
+	if [ -n "$2" ]; then
+		kill_opt="$kill_opt -s $signal"
+	fi
+	if [ $withtap -eq "1" ]; then
+		diag "Killing lt-$SESSIOND_BIN pids: $(echo $PID_SESSIOND | tr '\n' ' ')"
+	fi
+	kill $kill_opt $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 
 	if [ $? -eq 1 ]; then
-		fail "Kill sessions daemon"
-		return 1
+		if [ $withtap -eq "1" ]; then
+			fail "Kill sessions daemon"
+		fi
 	else
 		out=1
 		while [ -n "$out" ]; do
-			out=$(pidof lt-$SESSIOND_BIN)
+			out=$(pgrep --full lt-$SESSIOND_BIN)
 			sleep 0.5
 		done
 		out=1
 		while [ -n "$out" ]; do
-			out=$(pidof $CONSUMERD_BIN)
+			out=$(pgrep --full $CONSUMERD_BIN)
+			sleep 0.5
+		done
+		if [ $withtap -eq "1" ]; then
+			pass "Kill session daemon"
+		fi
+	fi
+}
+
+function stop_lttng_sessiond()
+{
+	stop_lttng_sessiond_opt 1 "$@"
+}
+
+function stop_lttng_sessiond_notap()
+{
+	stop_lttng_sessiond_opt 0 "$@"
+}
+
+function sigstop_lttng_sessiond_opt()
+{
+	local withtap=$1
+	local signal=SIGSTOP
+	local kill_opt=""
+
+	if [ -n $TEST_NO_SESSIOND ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
+		# Env variable requested no session daemon
+		return
+	fi
+
+	PID_SESSIOND="$(pgrep --full lt-$SESSIOND_BIN) $(pgrep --full $RUNAS_BIN)"
+
+	kill_opt="$kill_opt -s $signal"
+
+	if [ $withtap -eq "1" ]; then
+		diag "Sending SIGSTOP to lt-$SESSIOND_BIN pids: $(echo $PID_SESSIOND | tr '\n' ' ')"
+	fi
+	kill $kill_opt $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+
+	if [ $? -eq 1 ]; then
+		if [ $withtap -eq "1" ]; then
+			fail "Sending SIGSTOP to session daemon"
+		fi
+	else
+		out=1
+		while [ $out -ne 0 ]; do
+			pid=$(pgrep --full lt-$SESSIOND_BIN)
+
+			# Wait until state becomes stopped for session
+			# daemon(s).
+			out=0
+			for sessiond_pid in $pid; do
+				state=$(ps -p $sessiond_pid -o state= )
+				if [[ -n "$state" && "$state" != "T" ]]; then
+					out=1
+				fi
+			done
+			sleep 0.5
+		done
+		if [ $withtap -eq "1" ]; then
+			pass "Sending SIGSTOP to session daemon"
+		fi
+	fi
+}
+
+function sigstop_lttng_sessiond()
+{
+	sigstop_lttng_sessiond_opt 1 "$@"
+}
+
+function sigstop_lttng_sessiond_notap()
+{
+	sigstop_lttng_sessiond_opt 0 "$@"
+}
+
+function stop_lttng_consumerd_opt()
+{
+	local withtap=$1
+	local signal=$2
+	local kill_opt=""
+
+	PID_CONSUMERD=`pgrep --full $CONSUMERD_BIN`
+
+	if [ -n "$2" ]; then
+		kill_opt="$kill_opt -s $signal"
+	fi
+
+	if [ $withtap -eq "1" ]; then
+		diag "Killing $CONSUMERD_BIN pids: $(echo $PID_CONSUMERD | tr '\n' ' ')"
+	fi
+	kill $kill_opt $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	retval=$?
+	set +x
+
+	if [ $? -eq 1 ]; then
+		if [ $withtap -eq "1" ]; then
+			fail "Kill consumer daemon"
+		fi
+		return 1
+	else
+		out=1
+		while [ $out -ne 0 ]; do
+			pid=$(pgrep --full $CONSUMERD_BIN)
+
+			# If consumerds are still present check their status.
+			# A zombie status qualifies the consumerd as *killed*
+			out=0
+			for consumer_pid in $pid; do
+				state=$(ps -p $consumer_pid -o state= )
+				if [[ -n "$state" && "$state" != "Z" ]]; then
+					out=1
+				fi
+			done
 			sleep 0.5
 		done
-		pass "Kill session daemon"
+		if [ $withtap -eq "1" ]; then
+			pass "Kill consumer daemon"
+		fi
 	fi
+	return $retval
+}
+
+function stop_lttng_consumerd()
+{
+	stop_lttng_consumerd_opt 1 "$@"
+}
+
+function stop_lttng_consumerd_notap()
+{
+	stop_lttng_consumerd_opt 0 "$@"
+}
+
+function sigstop_lttng_consumerd_opt()
+{
+	local withtap=$1
+	local signal=SIGSTOP
+	local kill_opt=""
+
+	PID_CONSUMERD=`pgrep --full $CONSUMERD_BIN`
+
+	kill_opt="$kill_opt -s $signal"
+
+	if [ $withtap -eq "1" ]; then
+		diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo $PID_CONSUMERD | tr '\n' ' ')"
+	fi
+	kill $kill_opt $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	retval=$?
+	set +x
+
+	if [ $? -eq 1 ]; then
+		if [ $withtap -eq "1" ]; then
+			fail "Sending SIGSTOP to consumer daemon"
+		fi
+		return 1
+	else
+		out=1
+		while [ $out -ne 0 ]; do
+			pid=$(pgrep --full $CONSUMERD_BIN)
+
+			# Wait until state becomes stopped for all
+			# consumers.
+			out=0
+			for consumer_pid in $pid; do
+				state=$(ps -p $consumer_pid -o state= )
+				if [[ -n "$state" && "$state" != "T" ]]; then
+					out=1
+				fi
+			done
+			sleep 0.5
+		done
+		if [ $withtap -eq "1" ]; then
+			pass "Sending SIGSTOP to consumer daemon"
+		fi
+	fi
+	return $retval
+}
+
+function sigstop_lttng_consumerd()
+{
+	sigstop_lttng_consumerd_opt 1 "$@"
+}
+
+function sigstop_lttng_consumerd_notap()
+{
+	sigstop_lttng_consumerd_opt 0 "$@"
 }
 
 function list_lttng_with_opts ()
 {
 	local opts=$1
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Lttng-tool list command with option $opts"
 }
 
@@ -305,48 +632,71 @@ function create_lttng_session_no_output ()
 {
 	local sess_name=$1
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Create session $sess_name in no-output mode"
 }
 
 function create_lttng_session ()
 {
-	local sess_name=$1
-	local trace_path=$2
-	local expected_to_fail=$3
+	local expected_to_fail=$1
+	local sess_name=$2
+	local trace_path=$3
+	local opt=$4
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path > $OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path $opt > $OUTPUT_DEST
 	ret=$?
-	if [[ $expected_to_fail ]]; then
+	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
-		ok $? "Expected fail on session creation $sess_name in $trace_path"
+		ok $? "Create session $sess_name in $trace_path failed as expected"
 	else
 		ok $ret "Create session $sess_name in $trace_path"
 	fi
 }
 
-function enable_ust_lttng_channel()
+function create_lttng_session_ok ()
 {
-	local sess_name=$1
-	local channel_name=$2
-	local expect_fail=$3
+	create_lttng_session 0 "$@"
+}
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name >$OUTPUT_DEST
+function create_lttng_session_fail ()
+{
+	create_lttng_session 1 "$@"
+}
+
+
+function enable_ust_lttng_channel ()
+{
+	local expected_to_fail=$1
+	local sess_name=$2
+	local channel_name=$3
+	local opt=$4
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
-	if [[ $expect_fail ]]; then
+	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
-		ok $? "Expected fail on ust channel creation $channel_name in $sess_name"
+		ok $? "Enable channel $channel_name for session $sess_name failed as expected"
 	else
 		ok $ret "Enable channel $channel_name for session $sess_name"
 	fi
 }
 
+function enable_ust_lttng_channel_ok ()
+{
+	enable_ust_lttng_channel 0 "$@"
+}
+
+function enable_ust_lttng_channel_fail ()
+{
+	enable_ust_lttng_channel 1 "$@"
+}
+
 function disable_ust_lttng_channel()
 {
 	local sess_name=$1
 	local channel_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Disable channel $channel_name for session $sess_name"
 }
 
@@ -355,7 +705,7 @@ function enable_lttng_mmap_overwrite_kernel_channel()
 	local sess_name=$1
 	local channel_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable channel $channel_name for session $sess_name"
 }
 
@@ -364,16 +714,16 @@ function enable_lttng_mmap_overwrite_ust_channel()
 	local sess_name=$1
 	local channel_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable channel $channel_name for session $sess_name"
 }
 
 function enable_ust_lttng_event ()
 {
-	local sess_name=$1
-	local event_name="$2"
-	local channel_name=$3
-	local expected_to_fail=$4
+	local expected_to_fail=$1
+	local sess_name=$2
+	local event_name="$3"
+	local channel_name=$4
 
 	if [ -z $channel_name ]; then
 		# default channel if none specified
@@ -382,16 +732,26 @@ function enable_ust_lttng_event ()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
-	if [[ $expected_to_fail ]]; then
+	if [[ $expected_to_fail -eq "1" ]]; then
 		test $ret -ne "0"
-		ok $? "Enable ust event $event_name for session $session_name on channel $channel_name failed as expected"
+		ok $? "Enable ust event $event_name for session $session_name failed as expected"
 	else
-		ok $ret "Enable event $event_name for session $sess_name"
+		ok $ret "Enable ust event $event_name for session $sess_name"
 	fi
 }
 
+function enable_ust_lttng_event_ok ()
+{
+	enable_ust_lttng_event 0 "$@"
+}
+
+function enable_ust_lttng_event_fail ()
+{
+	enable_ust_lttng_event 1 "$@"
+}
+
 function enable_jul_lttng_event()
 {
 	sess_name=$1
@@ -405,7 +765,7 @@ function enable_jul_lttng_event()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable JUL event $event_name for session $sess_name"
 }
 
@@ -423,7 +783,7 @@ function enable_jul_lttng_event_loglevel()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
 }
 
@@ -440,7 +800,7 @@ function enable_log4j_lttng_event()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable LOG4J event $event_name for session $sess_name"
 }
 
@@ -458,17 +818,52 @@ function enable_log4j_lttng_event_loglevel()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
 }
 
+function enable_python_lttng_event()
+{
+	sess_name=$1
+	event_name="$2"
+	channel_name=$3
+
+	if [ -z $channel_name ]; then
+		# default channel if none specified
+		chan=""
+	else
+		chan="-c $channel_name"
+	fi
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ok $? "Enable Python event $event_name for session $sess_name"
+}
+
+function enable_python_lttng_event_loglevel()
+{
+	local sess_name=$1
+	local event_name="$2"
+	local loglevel=$3
+	local channel_name=$4
+
+	if [ -z $channel_name ]; then
+		# default channel if none specified
+		chan=""
+	else
+		chan="-c $channel_name"
+	fi
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
+}
+
 function enable_ust_lttng_event_filter()
 {
 	local sess_name="$1"
 	local event_name="$2"
 	local filter="$3"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable event $event_name with filtering for session $sess_name"
 }
 
@@ -478,7 +873,7 @@ function enable_ust_lttng_event_loglevel()
 	local event_name="$2"
 	local loglevel="$3"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable event $event_name with loglevel $loglevel"
 }
 
@@ -488,7 +883,7 @@ function enable_ust_lttng_event_loglevel_only()
 	local event_name="$2"
 	local loglevel="$3"
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Enable event $event_name with loglevel-only $loglevel"
 }
 
@@ -505,7 +900,7 @@ function disable_ust_lttng_event ()
 		chan="-c $channel_name"
 	fi
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Disable event $event_name for session $sess_name"
 }
 
@@ -527,102 +922,163 @@ function disable_log4j_lttng_event ()
 	ok $? "Disable LOG4J event $event_name for session $sess_name"
 }
 
+function disable_python_lttng_event ()
+{
+	local sess_name="$1"
+	local event_name="$2"
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ok $? "Disable Python event $event_name for session $sess_name"
+}
+
 function start_lttng_tracing ()
 {
-	local sess_name=$1
-	local expected_to_fail=$2
+	local expected_to_fail=$1
+	local sess_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
-	if [[ $expected_to_fail ]]; then
+	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
-		ok $? "Expected fail on start tracing for session: $sess_name"
+		ok $? "Start tracing for session $sess_name failed as expected"
 	else
 		ok $ret "Start tracing for session $sess_name"
 	fi
 }
 
+function start_lttng_tracing_ok ()
+{
+	start_lttng_tracing 0 "$@"
+}
+
+function start_lttng_tracing_fail ()
+{
+	start_lttng_tracing 1 "$@"
+}
+
 function stop_lttng_tracing ()
 {
-	local sess_name=$1
-	local expected_to_fail=$2
+	local expected_to_fail=$1
+	local sess_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
-	if [[ $expected_to_fail ]]; then
+	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
-		ok $? "Expected fail on stop tracing for session: $sess_name"
+		ok $? "Stop lttng tracing for session $sess_name failed as expected"
 	else
 		ok $ret "Stop lttng tracing for session $sess_name"
 	fi
 }
 
+function stop_lttng_tracing_ok ()
+{
+	stop_lttng_tracing 0 "$@"
+}
+
+function stop_lttng_tracing_fail ()
+{
+	stop_lttng_tracing 1 "$@"
+}
+
 function destroy_lttng_session ()
 {
-	local sess_name=$1
-	local expected_to_fail=$2
+	local expected_to_fail=$1
+	local sess_name=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
-	if [[ $expected_to_fail ]]; then
+	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
-		ok $? "Expected fail on session deletion $sess_name"
+		ok $? "Destroy session $sess_name failed as expected"
 	else
 		ok $ret "Destroy session $sess_name"
 	fi
 }
 
+function destroy_lttng_session_ok ()
+{
+	destroy_lttng_session 0 "$@"
+
+}
+
+function destroy_lttng_session_fail ()
+{
+	destroy_lttng_session 1 "$@"
+}
+
+
 function destroy_lttng_sessions ()
 {
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Destroy all lttng sessions"
 }
 
 function lttng_snapshot_add_output ()
 {
-	local sess_name=$1
-	local trace_path=$2
-	local expected_to_fail=$3
+	local expected_to_fail=$1
+	local sess_name=$2
+	local trace_path=$3
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name file://$trace_path >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name file://$trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
-	if [[ $expected_to_fail ]]; then
+	if [[ $expected_to_fail -eq 1 ]]; then
 		test "$ret" -ne "0"
-		ok $? "Failed to add a  snapshot output file://$trace_path as expected"
+		ok $? "Added snapshot output file://$trace_path failed as expected"
 	else
 		ok $ret "Added snapshot output file://$trace_path"
 	fi
 }
 
+function lttng_snapshot_add_output_ok ()
+{
+	lttng_snapshot_add_output 0 "$@"
+}
+
+function lttng_snapshot_add_output_fail ()
+{
+	lttng_snapshot_add_output 1 "$@"
+}
+
 function lttng_snapshot_del_output ()
 {
-	local sess_name=$1
-	local id=$2
-	local expected_to_fail=$3
+	local expected_to_fail=$1
+	local sess_name=$2
+	local id=$3
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ret=$?
-	if [[ $expected_to_fail ]]; then
+	if [[ $expected_to_fail -eq "1" ]]; then
 		test "$ret" -ne "0"
-		ok $? "Expect fail on deletion of snapshot output id $id"
+		ok $? "Deleted snapshot output id $id failed as expected"
 	else
 		ok $ret "Deleted snapshot output id $id"
 	fi
 }
 
+function lttng_snapshot_del_output_ok ()
+{
+	lttng_snapshot_del_output 0 "$@"
+}
+
+function lttng_snapshot_del_output_fail ()
+{
+	lttng_snapshot_del_output 1 "$@"
+}
+
 function lttng_snapshot_record ()
 {
 	local sess_name=$1
 	local trace_path=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Snapshot recorded"
 }
 
 function lttng_snapshot_list ()
 {
 	local sess_name=$1
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name >$OUTPUT_DEST
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 	ok $? "Snapshot list"
 }
 
@@ -631,16 +1087,102 @@ function lttng_save()
 	local sess_name=$1
 	local opts=$2
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts >$OUTPUT_DEST
-	ok $? "Session successfully saved"
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ok $? "Session saved"
 }
 
 function lttng_load()
 {
 	local opts=$1
 
-	$TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts >$OUTPUT_DEST
-	ok $? "Load command successful"
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ok $? "Load command with opts: $opts"
+}
+
+function lttng_track()
+{
+	local expected_to_fail=$1
+	local opts=$2
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test $ret -ne "0"
+		ok $? "Track command failed as expected with opts: $opts"
+	else
+		ok $ret "Track command with opts: $opts"
+	fi
+}
+
+function lttng_track_ok()
+{
+	lttng_track 0 "$@"
+}
+
+function lttng_track_fail()
+{
+	lttng_track 1 "$@"
+}
+
+function lttng_untrack()
+{
+	local expected_to_fail=$1
+	local opts=$2
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test $ret -ne "0"
+		ok $? "Untrack command failed as expected with opts: $opts"
+	else
+		ok $ret "Untrack command with opts: $opts"
+	fi
+}
+
+function lttng_untrack_ok()
+{
+	lttng_untrack 0 "$@"
+}
+
+function lttng_untrack_fail()
+{
+	lttng_untrack 1 "$@"
+}
+
+function add_context_lttng()
+{
+	local expected_to_fail="$1"
+	local domain="$2"
+	local session_name="$3"
+	local channel_name="$4"
+	local type="$5"
+
+	$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+	ret=$?
+	if [[ $expected_to_fail -eq "1" ]]; then
+		test $ret -ne "0"
+		ok $? "Add context command failed as expected for type: $type"
+	else
+		ok $ret "Add context command for type: $type"
+	fi
+}
+
+function add_context_ust_ok()
+{
+	add_context_lttng 0 -u "$@"
+}
+
+function add_context_ust_fail()
+{
+	add_context_lttng 1 -u "$@"
+}
+
+function add_context_kernel_ok()
+{
+	add_context_lttng 0 -k "$@"
+}
+
+function add_context_kernel_fail()
+{
+	add_context_lttng 1 -k "$@"
 }
 
 function trace_matches ()