[lttng-tools.git] / tests / regression / tools / notification / test_notification_multi_app

#!/bin/bash
#
# Copyright (C) - 2017 Jonathan Rajotte <jonathan.rajotte-julien@efficiso.com>>
#
# This library is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation; version 2.1 of the License.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA

TEST_DESC="Notification"

CURDIR=$(dirname $0)/
TESTDIR=$CURDIR/../../../

TESTPOINT=$(readlink -f ${CURDIR}/.libs/libpause_consumer.so)

TESTAPP_PATH="$TESTDIR/utils/testapp"
TESTAPP_NAME="gen-ust-events"
TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"

NR_ITER=-1
NR_USEC_WAIT=5

SESSION_NAME="my_session"
UST_CHANNEL_NAME="my_ust_channel"
EVENT_NAME="tp:tptest"


TRACE_PATH=$(mktemp -d)

DIR=$(readlink -f $TESTDIR)

PAGE_SIZE=$(getconf PAGE_SIZE)

NUM_TESTS=62

source $TESTDIR/utils/utils.sh

consumerd_pipe=()
file_sync_after_first_event=$(mktemp -u)

# MUST set TESTDIR before calling those functions
plan_tests $NUM_TESTS

print_test_banner "$TEST_DESC"

app_pids=()

function start_client {
	local pid=-1
	local output_file=$1
	local session_name=$2
	local channel_name=$3
	local domain_type=$4
	local buffer_usage_type=$5
	local buffer_usage_threshold_type=$6
	local buffer_usage_threshold_value=$7
	local nr_expected_notification=$8

	${CURDIR}/base_client ${session_name} ${channel_name} ${domain_type} ${buffer_usage_type} ${buffer_usage_threshold_type} ${buffer_usage_threshold_value} ${nr_expected_notification} > ${output_file} &
	pid=$!

	app_pids+=("$pid")
}

function wait_for_message ()
{
	local file_pattern=$1
	local message=$2

	for file in $CURDIR/${file_pattern}*; do
		while(true); do
			# Check for "error" message
			grep -q "error:" ${file}
			app_error=$?
			if [ $app_error -eq "0" ] ; then
				# An error occurred
				fail "Waiting message: error logged see file content: ${message}, ${file}"
				return 1
			fi

			grep -q "${message}" ${file}
			if [[ "$?" -ne "0" ]]; then
				# Lookup failed restart loop
				diag "Lookup failed sleep and retry grep for: ${message}, ${file}"
				sleep 0.25
				continue
			fi
			break
		done
	done
	pass "Message received: ${message}"
	return 0
}

function print_errors ()
{
	local file_pattern=$1

	for file in $CURDIR/${file_pattern}*; do
		# Check for "error" message
		error_message=$(grep "error:" ${file})
		if [[ "${error_message}x" != "x" ]]; then
			diag "Errors for application ${file}:"
			diag "${error_message}"
		fi
	done
}

function comm_consumerd ()
{
	local message=$1
	local pipe=$2
	echo -ne "${message}" > "${pipe}"
	return $?
}

function stop_consumerd ()
{
	local pipe=$1
	comm_consumerd "1" "$pipe"
	ok $? "Stopping consumption consumerd"
}

function resume_consumerd ()
{
	local pipe=$1
	comm_consumerd "\0" "$pipe"
	ok $? "Resuming consumerd"
}

function test_multi_app ()
{
	local app_pids=()
	local low_output_file_pattern="low_app_output_file_"
	local high_output_file_pattern="high_app_output_file_"

	local testpoint_base_path=$(readlink -f "$CURDIR/lttng.t_p_n_multi_app")
	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")

	local nr_notification_expected=5
	local nr_client_app=50

	# Cleanup
	rm ${CURDIR}/${low_output_file_pattern}* 2> /dev/null
	rm ${CURDIR}/${high_output_file_pattern}* 2> /dev/null

	# Setup
	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	start_lttng_sessiond

	# Start app in infinite loop
	$TESTAPP_BIN $NR_ITER $NR_USEC_WAIT $file_sync_after_first_event &
	app_pid=$!
	# Pin to CPU zero to force specific sub buffer usage
	taskset -p -c 0 $app_pid > /dev/null 2>&1

	# Wait for sync with app
	while [ ! -f "${file_sync_after_first_event}" ]; do
		sleep 0.5
	done
	rm ${file_sync_after_first_event}

	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	enable_ust_lttng_channel_ok $SESSION_NAME $UST_CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	enable_ust_lttng_event_ok $SESSION_NAME $EVENT_NAME $UST_CHANNEL_NAME

	# Fetch consumerd testpoint pipe information
	# This is needed since the testpoint create a pipe with the consumer type suffixed
	for f in "$testpoint_base_path"*; do
		consumerd_pipe+=("$f")
	done

	for (( i = 0; i < $nr_client_app; i++ )); do
		low_app_output_file=$CURDIR/${low_output_file_pattern}${i}
		high_app_output_file=$CURDIR/${high_output_file_pattern}${i}
		start_client $low_app_output_file $SESSION_NAME $UST_CHANNEL_NAME LTTNG_DOMAIN_UST LOW RATIO 0.0 $nr_notification_expected
		start_client $high_app_output_file $SESSION_NAME $UST_CHANNEL_NAME LTTNG_DOMAIN_UST HIGH RATIO 0.420 $nr_notification_expected
	done

	wait_for_message "${low_output_file_pattern}" "sync: ready"
	wait_for_message "${high_output_file_pattern}" "sync: ready"

	# Test notification reception
	for (( i = 0; i < $nr_notification_expected; i++ )); do

		# Stop consumerd consumption to force high notification
		start_lttng_tracing_ok $SESSION_NAME
		for pipe in "${consumerd_pipe[@]}"; do
			stop_consumerd "${pipe}"
		done

		wait_for_message "${high_output_file_pattern}" "notification: high $i"

		# Resume consumerd
		for pipe in "${consumerd_pipe[@]}"; do
			resume_consumerd "${pipe}"
		done
		# Stop tracing forcing full buffer consumption
		stop_lttng_tracing $SESSION_NAME

		# Check for notifications reception
		wait_for_message "${low_output_file_pattern}" "notification: low $i"
		ret=$?
		ok $ret "Notifications $i received"
		if [[ $ret -ne "0" ]]; then
			# Error occurred bail out
			break;
		fi
	done

	wait_for_message "${low_output_file_pattern}" "exit: 0"
	ret=$?
	ok $ret "Application for low notification terminated normally"
	if [[ $ret -eq "0" ]]; then
		rm ${CURDIR}/${low_output_file_pattern}* 2> /dev/null
	else
		# Keep the file
		print_errors "${low_output_file_pattern}"
	fi

	wait_for_message "${high_output_file_pattern}" "exit: 0"
	ret=$?
	ok $ret "Application for high notification terminated normally"
	if [[ $ret -eq "0" ]]; then
		rm ${CURDIR}/${high_output_file_pattern}* 2> /dev/null
	else
		# Keep the file
		print_errors "${high_output_file_pattern}"
	fi

	kill -9 $app_pid
	wait $app_pid 2> /dev/null

	stop_lttng_sessiond
}

function test_on_register_evaluation ()
{
	local app_pids=()
	local high_output_file_pattern="high_app_output_file_on_register_evaluation"

	local testpoint_base_path=$(readlink -f "$CURDIR/lttng.t_p_n_register_evaluation")
	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")

	# Cleanup
	rm ${CURDIR}/${high_output_file_pattern}* 2> /dev/null

	# Setup
	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	start_lttng_sessiond

	# Start app in infinite loop
	$TESTAPP_BIN $NR_ITER $NR_USEC_WAIT $file_sync_after_first_event &
	app_pid=$!
	# Pin to CPU zero to force specific sub buffer usage
	taskset -p -c 0 $app_pid > /dev/null 2>&1

	# Wait for sync with app
	while [ ! -f "${file_sync_after_first_event}" ]; do
		sleep 0.5
	done
	rm ${file_sync_after_first_event}

	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	enable_ust_lttng_channel_ok $SESSION_NAME $UST_CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	enable_ust_lttng_event_ok $SESSION_NAME $EVENT_NAME $UST_CHANNEL_NAME

	# Fetch consumerd testpoint pipe information
	# This is needed since the testpoint create a pipe with the consumer type suffixed
	for f in "$testpoint_base_path"*; do
		consumerd_pipe+=("$f")
	done

	high_app_output_file=${high_output_file_pattern}.first_receiver
	high_app_output_path=$CURDIR/${high_app_output_file}
	start_client $high_app_output_path $SESSION_NAME $UST_CHANNEL_NAME LTTNG_DOMAIN_UST HIGH RATIO 0.420 1

	wait_for_message "${high_app_output_file}" "sync: ready"

	# Stop consumerd consumption to force high notification
	start_lttng_tracing_ok $SESSION_NAME

	for pipe in "${consumerd_pipe[@]}"; do
		stop_consumerd "${pipe}"
	done

	wait_for_message "${high_app_output_file}" "notification: high 0"

	# Start a second receiver, the receiver should receive a high
	# notification on subscription
	high_app_output_file=${high_output_file_pattern}.second_receiver
	high_app_output_path=$CURDIR/${high_app_output_file}
	start_client $high_app_output_path $SESSION_NAME $UST_CHANNEL_NAME LTTNG_DOMAIN_UST HIGH RATIO 0.420 1
	wait_for_message "${high_app_output_file}" "sync: ready"
	wait_for_message "${high_app_output_file}" "notification: high 0"

	# Resume consumerd
	for pipe in "${consumerd_pipe[@]}"; do
		resume_consumerd "${pipe}"
	done

	wait_for_message "${high_output_file_pattern}" "exit: 0"
	ret=$?
	ok $ret "Application for high notification terminated normally"
	if [[ $ret -eq "0" ]]; then
		rm ${CURDIR}/${high_output_file_pattern}* 2> /dev/null
	else
		# Keep the file
		print_errors "${high_output_file_pattern}"
	fi

	stop_lttng_sessiond

	kill -9 $app_pid
	wait $app_pid 2> /dev/null
}


TESTS=(
	test_multi_app
	test_on_register_evaluation
)

for fct_test in ${TESTS[@]};
do
	TRACE_PATH=$(mktemp -d)

	${fct_test}
	if [ $? -ne 0 ]; then
		break;
	fi

	# Only delete if successful
	rm -rf $TRACE_PATH
done
Commit	Line	Data
434f8068 JR	1	#!/bin/bash
	2	#
	3	# Copyright (C) - 2017 Jonathan Rajotte <jonathan.rajotte-julien@efficiso.com>>
	4	#
	5	# This library is free software; you can redistribute it and/or modify it under
	6	# the terms of the GNU Lesser General Public License as published by the Free
	7	# Software Foundation; version 2.1 of the License.
	8	#
	9	# This library is distributed in the hope that it will be useful, but WITHOUT
	10	# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
	11	# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
	12	# details.
	13	#
	14	# You should have received a copy of the GNU Lesser General Public License
	15	# along with this library; if not, write to the Free Software Foundation, Inc.,
	16	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	17
	18	TEST_DESC="Notification"
	19
	20	CURDIR=$(dirname $0)/
	21	TESTDIR=$CURDIR/../../../
	22
434f8068 JR	23	TESTPOINT=$(readlink -f ${CURDIR}/.libs/libpause_consumer.so)
	24
	25	TESTAPP_PATH="$TESTDIR/utils/testapp"
	26	TESTAPP_NAME="gen-ust-events"
	27	TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"
	28
	29	NR_ITER=-1
	30	NR_USEC_WAIT=5
	31
	32	SESSION_NAME="my_session"
	33	UST_CHANNEL_NAME="my_ust_channel"
	34	EVENT_NAME="tp:tptest"
	35
434f8068 JR	36
	37	TRACE_PATH=$(mktemp -d)
	38
	39	DIR=$(readlink -f $TESTDIR)
	40
1bd26228 JG	41	PAGE_SIZE=$(getconf PAGE_SIZE)
1bd26228 JG	42
867313e2	43	NUM_TESTS=62
434f8068 JR	44
	45	source $TESTDIR/utils/utils.sh
	46
	47	consumerd_pipe=()
	48	file_sync_after_first_event=$(mktemp -u)
	49
	50	# MUST set TESTDIR before calling those functions
	51	plan_tests $NUM_TESTS
	52
	53	print_test_banner "$TEST_DESC"
	54
	55	app_pids=()
867313e2	56
434f8068 JR	57	function start_client {
	58	local pid=-1
	59	local output_file=$1
	60	local session_name=$2
	61	local channel_name=$3
	62	local domain_type=$4
	63	local buffer_usage_type=$5
	64	local buffer_usage_threshold_type=$6
	65	local buffer_usage_threshold_value=$7
	66	local nr_expected_notification=$8
	67
	68	${CURDIR}/base_client ${session_name} ${channel_name} ${domain_type} ${buffer_usage_type} ${buffer_usage_threshold_type} ${buffer_usage_threshold_value} ${nr_expected_notification} > ${output_file} &
	69	pid=$!
	70
	71	app_pids+=("$pid")
	72	}
	73
	74	function wait_for_message ()
	75	{
	76	local file_pattern=$1
	77	local message=$2
	78
	79	for file in $CURDIR/${file_pattern}*; do
	80	while(true); do
	81	# Check for "error" message
	82	grep -q "error:" ${file}
	83	app_error=$?
	84	if [ $app_error -eq "0" ] ; then
	85	# An error occurred
	86	fail "Waiting message: error logged see file content: ${message}, ${file}"
	87	return 1
	88	fi
	89
	90	grep -q "${message}" ${file}
	91	if [[ "$?" -ne "0" ]]; then
	92	# Lookup failed restart loop
	93	diag "Lookup failed sleep and retry grep for: ${message}, ${file}"
	94	sleep 0.25
	95	continue
	96	fi
	97	break
	98	done
	99	done
	100	pass "Message received: ${message}"
	101	return 0
	102	}
	103
	104	function print_errors ()
	105	{
	106	local file_pattern=$1
	107
	108	for file in $CURDIR/${file_pattern}*; do
	109	# Check for "error" message
	110	error_message=$(grep "error:" ${file})
867313e2	111	if [[ "${error_message}x" != "x" ]]; then
434f8068 JR	112	diag "Errors for application ${file}:"
	113	diag "${error_message}"
	114	fi
	115	done
	116	}
	117
	118	function comm_consumerd ()
	119	{
	120	local message=$1
	121	local pipe=$2
	122	echo -ne "${message}" > "${pipe}"
	123	return $?
	124	}
	125
	126	function stop_consumerd ()
	127	{
	128	local pipe=$1
	129	comm_consumerd "1" "$pipe"
	130	ok $? "Stopping consumption consumerd"
	131	}
	132
	133	function resume_consumerd ()
	134	{
	135	local pipe=$1
	136	comm_consumerd "\0" "$pipe"
	137	ok $? "Resuming consumerd"
	138	}
	139
	140	function test_multi_app ()
	141	{
	142	local app_pids=()
	143	local low_output_file_pattern="low_app_output_file_"
	144	local high_output_file_pattern="high_app_output_file_"
	145
6a1eee92 JR	146	local testpoint_base_path=$(readlink -f "$CURDIR/lttng.t_p_n_multi_app")
	147	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
	148
	149	local nr_notification_expected=5
	150	local nr_client_app=50
	151
434f8068 JR	152	# Cleanup
	153	rm ${CURDIR}/${low_output_file_pattern}* 2> /dev/null
	154	rm ${CURDIR}/${high_output_file_pattern}* 2> /dev/null
	155
	156	# Setup
6a1eee92	157	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
434f8068 JR	158	start_lttng_sessiond
	159
	160	# Start app in infinite loop
	161	$TESTAPP_BIN $NR_ITER $NR_USEC_WAIT $file_sync_after_first_event &
	162	app_pid=$!
	163	# Pin to CPU zero to force specific sub buffer usage
	164	taskset -p -c 0 $app_pid > /dev/null 2>&1
	165
	166	# Wait for sync with app
	167	while [ ! -f "${file_sync_after_first_event}" ]; do
	168	sleep 0.5
	169	done
	170	rm ${file_sync_after_first_event}
	171
	172	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
1bd26228	173	enable_ust_lttng_channel_ok $SESSION_NAME $UST_CHANNEL_NAME --subbuf-size=$PAGE_SIZE
434f8068 JR	174	enable_ust_lttng_event_ok $SESSION_NAME $EVENT_NAME $UST_CHANNEL_NAME
	175
	176	# Fetch consumerd testpoint pipe information
	177	# This is needed since the testpoint create a pipe with the consumer type suffixed
6a1eee92	178	for f in "$testpoint_base_path"*; do
434f8068 JR	179	consumerd_pipe+=("$f")
	180	done
	181
6a1eee92	182	for (( i = 0; i < $nr_client_app; i++ )); do
434f8068 JR	183	low_app_output_file=$CURDIR/${low_output_file_pattern}${i}
434f8068 JR	184	high_app_output_file=$CURDIR/${high_output_file_pattern}${i}
6a1eee92 JR	185	start_client $low_app_output_file $SESSION_NAME $UST_CHANNEL_NAME LTTNG_DOMAIN_UST LOW RATIO 0.0 $nr_notification_expected
6a1eee92 JR	186	start_client $high_app_output_file $SESSION_NAME $UST_CHANNEL_NAME LTTNG_DOMAIN_UST HIGH RATIO 0.420 $nr_notification_expected
434f8068 JR	187	done
	188
	189	wait_for_message "${low_output_file_pattern}" "sync: ready"
	190	wait_for_message "${high_output_file_pattern}" "sync: ready"
	191
	192	# Test notification reception
6a1eee92	193	for (( i = 0; i < $nr_notification_expected; i++ )); do
434f8068 JR	194
	195	# Stop consumerd consumption to force high notification
	196	start_lttng_tracing_ok $SESSION_NAME
	197	for pipe in "${consumerd_pipe[@]}"; do
	198	stop_consumerd "${pipe}"
	199	done
	200
	201	wait_for_message "${high_output_file_pattern}" "notification: high $i"
	202
	203	# Resume consumerd
	204	for pipe in "${consumerd_pipe[@]}"; do
	205	resume_consumerd "${pipe}"
	206	done
	207	# Stop tracing forcing full buffer consumption
	208	stop_lttng_tracing $SESSION_NAME
	209
	210	# Check for notifications reception
	211	wait_for_message "${low_output_file_pattern}" "notification: low $i"
	212	ret=$?
	213	ok $ret "Notifications $i received"
	214	if [[ $ret -ne "0" ]]; then
	215	# Error occurred bail out
	216	break;
	217	fi
	218	done
	219
	220	wait_for_message "${low_output_file_pattern}" "exit: 0"
	221	ret=$?
	222	ok $ret "Application for low notification terminated normally"
	223	if [[ $ret -eq "0" ]]; then
	224	rm ${CURDIR}/${low_output_file_pattern}* 2> /dev/null
	225	else
	226	# Keep the file
	227	print_errors "${low_output_file_pattern}"
	228	fi
	229
	230	wait_for_message "${high_output_file_pattern}" "exit: 0"
	231	ret=$?
	232	ok $ret "Application for high notification terminated normally"
	233	if [[ $ret -eq "0" ]]; then
	234	rm ${CURDIR}/${high_output_file_pattern}* 2> /dev/null
	235	else
	236	# Keep the file
	237	print_errors "${high_output_file_pattern}"
	238	fi
	239
	240	kill -9 $app_pid
	241	wait $app_pid 2> /dev/null
	242
	243	stop_lttng_sessiond
	244	}
	245
867313e2 JR	246	function test_on_register_evaluation ()
	247	{
	248	local app_pids=()
	249	local high_output_file_pattern="high_app_output_file_on_register_evaluation"
	250
	251	local testpoint_base_path=$(readlink -f "$CURDIR/lttng.t_p_n_register_evaluation")
	252	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
	253
	254	# Cleanup
	255	rm ${CURDIR}/${high_output_file_pattern}* 2> /dev/null
	256
	257	# Setup
	258	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	259	start_lttng_sessiond
	260
	261	# Start app in infinite loop
	262	$TESTAPP_BIN $NR_ITER $NR_USEC_WAIT $file_sync_after_first_event &
	263	app_pid=$!
	264	# Pin to CPU zero to force specific sub buffer usage
	265	taskset -p -c 0 $app_pid > /dev/null 2>&1
	266
	267	# Wait for sync with app
	268	while [ ! -f "${file_sync_after_first_event}" ]; do
	269	sleep 0.5
	270	done
	271	rm ${file_sync_after_first_event}
	272
	273	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	274	enable_ust_lttng_channel_ok $SESSION_NAME $UST_CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	275	enable_ust_lttng_event_ok $SESSION_NAME $EVENT_NAME $UST_CHANNEL_NAME
	276
	277	# Fetch consumerd testpoint pipe information
	278	# This is needed since the testpoint create a pipe with the consumer type suffixed
	279	for f in "$testpoint_base_path"*; do
	280	consumerd_pipe+=("$f")
	281	done
	282
	283	high_app_output_file=${high_output_file_pattern}.first_receiver
	284	high_app_output_path=$CURDIR/${high_app_output_file}
	285	start_client $high_app_output_path $SESSION_NAME $UST_CHANNEL_NAME LTTNG_DOMAIN_UST HIGH RATIO 0.420 1
	286
	287	wait_for_message "${high_app_output_file}" "sync: ready"
	288
	289	# Stop consumerd consumption to force high notification
	290	start_lttng_tracing_ok $SESSION_NAME
	291
	292	for pipe in "${consumerd_pipe[@]}"; do
	293	stop_consumerd "${pipe}"
	294	done
	295
	296	wait_for_message "${high_app_output_file}" "notification: high 0"
	297
	298	# Start a second receiver, the receiver should receive a high
	299	# notification on subscription
	300	high_app_output_file=${high_output_file_pattern}.second_receiver
	301	high_app_output_path=$CURDIR/${high_app_output_file}
	302	start_client $high_app_output_path $SESSION_NAME $UST_CHANNEL_NAME LTTNG_DOMAIN_UST HIGH RATIO 0.420 1
	303	wait_for_message "${high_app_output_file}" "sync: ready"
	304	wait_for_message "${high_app_output_file}" "notification: high 0"
	305
	306	# Resume consumerd
	307	for pipe in "${consumerd_pipe[@]}"; do
	308	resume_consumerd "${pipe}"
	309	done
310
311	wait_for_message "${high_output_file_pattern}" "exit: 0"
312	ret=$?
313	ok $ret "Application for high notification terminated normally"
314	if [[ $ret -eq "0" ]]; then
315	rm ${CURDIR}/${high_output_file_pattern}* 2> /dev/null
316	else
317	# Keep the file
318	print_errors "${high_output_file_pattern}"
319	fi
320
321	stop_lttng_sessiond
322
323	kill -9 $app_pid
324	wait $app_pid 2> /dev/null
325	}
326
434f8068 JR	327
	328	TESTS=(
	329	test_multi_app
867313e2	330	test_on_register_evaluation
434f8068 JR	331	)
	332
	333	for fct_test in ${TESTS[@]};
	334	do
	335	TRACE_PATH=$(mktemp -d)
	336
	337	${fct_test}
	338	if [ $? -ne 0 ]; then
	339	break;
	340	fi
	341
	342	# Only delete if successful
	343	rm -rf $TRACE_PATH
	344	done
	345