tests/regression/tools/notification/test_notification_multi_app

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2017 Jonathan Rajotte <jonathan.rajotte-julien@efficiso.com>>
   4 #
   5 # SPDX-License-Identifier: LGPL-2.1-only
   6
   7 TEST_DESC="Notification"
   8
   9 CURDIR=$(dirname $0)/
  10 TESTDIR=$CURDIR/../../../
  11
  12 TESTPOINT=$(readlink -f ${CURDIR}/.libs/libpause_consumer.so)
  13
  14 TESTAPP_PATH="$TESTDIR/utils/testapp"
  15 TESTAPP_NAME="gen-ust-events"
  16 TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"
  17 TESTAPP_STATE_FILE="$(mktemp -u)"
  18
  19 NR_ITER=1000
  20 NR_USEC_WAIT=5
  21
  22 SESSION_NAME="my_session"
  23 CHANNEL_NAME="my_channel"
  24
  25
  26 DIR=$(readlink -f $TESTDIR)
  27
  28 PAGE_SIZE=$(getconf PAGE_SIZE)
  29
  30 NUM_TEST_UST=50
  31 NUM_TEST_KERNEL=50
  32 NUM_TESTS=$(($NUM_TEST_UST + $NUM_TEST_KERNEL))
  33
  34 source $TESTDIR/utils/utils.sh
  35
  36 consumerd_pipe=()
  37 file_sync_after_first_event=$(mktemp -u)
  38
  39 # MUST set TESTDIR before calling those functions
  40 plan_tests $NUM_TESTS
  41
  42 print_test_banner "$TEST_DESC"
  43
  44 app_pids=()
  45
  46 function kernel_event_generator_toggle_state
  47 {
  48         kernel_event_generator_suspended=$((kernel_event_generator_suspended==0))
  49
  50 }
  51 function kernel_event_generator
  52 {
  53         state_file=$1
  54         kernel_event_generator_suspended=0
  55         trap kernel_event_generator_toggle_state SIGUSR1
  56
  57         while (true); do
  58                 if [[ $kernel_event_generator_suspended -eq "1" ]]; then
  59                         touch $state_file
  60                         sleep 0.5
  61                 else
  62                         if [[ -f $state_file ]]; then
  63                                 rm $state_file 2> /dev/null
  64                         fi
  65                         echo -n "1000" > /proc/lttng-test-filter-event
  66                 fi
  67         done
  68 }
  69
  70 function ust_event_generator_toggle_state
  71 {
  72         ust_event_generator_suspended=$((ust_event_generator_suspended==0))
  73
  74 }
  75 function ust_event_generator
  76 {
  77         state_file=$1
  78         ust_event_generator_suspended=0
  79         trap ust_event_generator_toggle_state SIGUSR1
  80         trap "exit" SIGTERM SIGINT
  81         while (true); do
  82                 if [[ $ust_event_generator_suspended -eq "1" ]]; then
  83                         touch $state_file
  84                         sleep 0.5
  85                 else
  86                         if [[ -f $state_file ]]; then
  87                                 rm $state_file 2> /dev/null
  88                         fi
  89                         taskset  -c 0 $TESTAPP_BIN -i $NR_ITER -w $NR_USEC_WAIT > /dev/null 2>&1
  90                 fi
  91         done
  92 }
  93
  94 function start_client {
  95         local pid=-1
  96         local output_file=$1
  97         local session_name=$2
  98         local channel_name=$3
  99         local domain_type=$4
 100         local buffer_usage_type=$5
 101         local buffer_usage_threshold_type=$6
 102         local buffer_usage_threshold_value=$7
 103         local nr_expected_notification=$8
 104
 105         ${CURDIR}/base_client ${session_name} ${channel_name} ${domain_type} ${buffer_usage_type} ${buffer_usage_threshold_type} ${buffer_usage_threshold_value} ${nr_expected_notification} > ${output_file} &
 106         pid=$!
 107
 108         app_pids+=("$pid")
 109 }
 110
 111 function wait_for_message ()
 112 {
 113         local directory=$1
 114         local file_pattern=$2
 115         local message=$3
 116
 117         for file in $directory/${file_pattern}*; do
 118                 while(true); do
 119                         # Check for "error" message
 120                         grep -q "error:" ${file}
 121                         app_error=$?
 122                         if [ $app_error -eq "0" ] ; then
 123                                 # An error occurred
 124                                 fail "Waiting message: error logged see file content: ${message}, ${file}"
 125                                 return 1
 126                         fi
 127
 128                         grep -q "${message}" ${file}
 129                         if [[ "$?" -ne "0" ]]; then
 130                                 # Lookup failed restart loop
 131                                 diag "Lookup failed sleep and retry grep for: ${message}, ${file}"
 132                                 sleep 0.25
 133                                 continue
 134                         fi
 135                         break
 136                 done
 137         done
 138         pass "Message received: ${message}"
 139         return 0
 140 }
 141
 142 function print_errors ()
 143 {
 144         local directory=$1
 145         local file_pattern=$2
 146
 147         for file in $directory/${file_pattern}*; do
 148                 # Check for "error" message
 149                 error_message=$(grep "error:" ${file})
 150                 if [[ "${error_message}x" != "x" ]]; then
 151                         diag "Errors for application ${file}:"
 152                         diag "${error_message}"
 153                 fi
 154         done
 155 }
 156
 157 function comm_consumerd ()
 158 {
 159         local message=$1
 160         local pipe=$2
 161         echo -ne "${message}" > "${pipe}"
 162 }
 163
 164 function stop_consumerd ()
 165 {
 166         local pipe=$1
 167         comm_consumerd "1" "$pipe"
 168 }
 169
 170 function resume_consumerd ()
 171 {
 172         local pipe=$1
 173         comm_consumerd "\0" "$pipe"
 174 }
 175
 176 function test_multi_app ()
 177 {
 178         local domain_type=$1
 179         local event_generator_pid=$2
 180
 181         local app_pids=()
 182         local low_output_file_pattern="low_app_output_file_"
 183         local high_output_file_pattern="high_app_output_file_"
 184         local output_dir=$(mktemp -d)
 185
 186         local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_multi_app")
 187         local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
 188
 189         local nr_notification_expected=5
 190         local nr_client_app=50
 191         local domain_string=""
 192         local event_name=""
 193
 194         case $domain_type in
 195                 ust)
 196                         domain_string=LTTNG_DOMAIN_UST
 197                         event_name="tp:tptest"
 198                         ;;
 199                 kernel)
 200                         domain_string=LTTNG_DOMAIN_KERNEL
 201                         event_name="lttng_test_filter_event"
 202                         ;;
 203                 *)
 204                         fail "Invalid domain type"
 205                         exit 1
 206                         ;;
 207         esac
 208
 209         # Setup
 210         LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
 211         start_lttng_sessiond
 212
 213         create_lttng_session_ok $SESSION_NAME $TRACE_PATH
 214         enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
 215         enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME
 216
 217         # Fetch consumerd testpoint pipe information
 218         # This is needed since the testpoint create a pipe with the consumer type suffixed
 219         consumerd_pipe=()
 220         for f in "$testpoint_pipe_path"*; do
 221                 consumerd_pipe+=("$f")
 222         done
 223
 224         for (( i = 0; i < $nr_client_app; i++ )); do
 225                 low_app_output_file=$output_dir/${low_output_file_pattern}${i}
 226                 high_app_output_file=$output_dir/${high_output_file_pattern}${i}
 227                 start_client $low_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string LOW RATIO 0.0 $nr_notification_expected
 228                 start_client $high_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 $nr_notification_expected
 229         done
 230
 231         wait_for_message $output_dir "${low_output_file_pattern}" "sync: ready"
 232         wait_for_message $output_dir "${high_output_file_pattern}" "sync: ready"
 233
 234         # Test notification reception
 235         for (( i = 0; i < $nr_notification_expected; i++ )); do
 236
 237                 # Stop consumerd consumption to force high notification
 238                 start_lttng_tracing_ok $SESSION_NAME
 239                 for pipe in "${consumerd_pipe[@]}"; do
 240                         stop_consumerd "${pipe}"
 241                 done
 242
 243                 wait_for_message $output_dir "${high_output_file_pattern}" "notification: high $i"
 244
 245                 # Put application in suspend mode to prevent double low
 246                 # notification and synchronize on state file.
 247                 kill -s SIGUSR1 $event_generator_pid
 248                 while [ ! -f "${TESTAPP_STATE_FILE}" ]; do
 249                         sleep 0.5
 250                 done
 251
 252                 # Resume consumerd
 253                 for pipe in "${consumerd_pipe[@]}"; do
 254                         resume_consumerd "${pipe}"
 255                 done
 256                 # Stop tracing forcing full buffer consumption
 257                 stop_lttng_tracing_ok $SESSION_NAME
 258
 259                 # Check for notifications reception
 260                 wait_for_message $output_dir "${low_output_file_pattern}" "notification: low $i"
 261                 ret=$?
 262                 ok $ret "Notifications $i received"
 263                 if [[ $ret -ne "0" ]]; then
 264                         # Error occurred bail out
 265                         break;
 266                 fi
 267
 268                 # Put application in active mode and synchronize on state file.
 269                 kill -s SIGUSR1 $event_generator_pid
 270                 while [ -f "${TESTAPP_STATE_FILE}" ]; do
 271                         sleep 0.5
 272                 done
 273         done
 274
 275         wait_for_message $output_dir "${low_output_file_pattern}" "exit: 0"
 276         ret=$?
 277         ok $ret "Application for low notification terminated normally"
 278         if [[ $ret -ne "0" ]]; then
 279                 print_errors $output_dir "${low_output_file_pattern}"
 280         fi
 281
 282         wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
 283         ret=$?
 284         ok $ret "Application for high notification terminated normally"
 285         if [[ $ret -ne "0" ]]; then
 286                 print_errors $output_dir "${high_output_file_pattern}"
 287         fi
 288
 289
 290         destroy_lttng_session_ok $SESSION_NAME
 291         stop_lttng_sessiond
 292
 293         for pipe in "${consumerd_pipe[@]}"; do
 294                 rm -rf "${pipe}"
 295         done
 296
 297         rm -rf $output_dir
 298 }
 299
 300 function test_multi_app_ust ()
 301 {
 302         diag "Multi client app UST notification"
 303         ust_event_generator $TESTAPP_STATE_FILE &
 304         local generator_pid=$!
 305
 306         test_multi_app ust $generator_pid
 307
 308         kill -s SIGTERM $generator_pid 2> /dev/null
 309         wait $generator_pid 2> /dev/null
 310         rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
 311 }
 312
 313 function test_multi_app_kernel ()
 314 {
 315         diag "Multi client app kernel notification"
 316         modprobe lttng-test
 317
 318         kernel_event_generator $TESTAPP_STATE_FILE &
 319         local generator_pid=$!
 320
 321         test_multi_app kernel $generator_pid
 322
 323
 324         kill -s SIGTERM $generator_pid 2> /dev/null
 325         wait $generator_pid 2> /dev/null
 326         rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
 327
 328         rmmod lttng-test
 329 }
 330
 331 function test_on_register_evaluation_ust ()
 332 {
 333         diag "On register notification UST"
 334
 335         # Start app in infinite loop
 336         ust_event_generator $TESTAPP_STATE_FILE &
 337         local generator_pid=$!
 338
 339         test_on_register_evaluation ust $generator_pid
 340
 341         kill -s SIGTERM $generator_pid 2> /dev/null
 342         wait $generator_pid 2> /dev/null
 343         rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
 344
 345 }
 346
 347 function test_on_register_evaluation_kernel()
 348 {
 349         diag "On register notification kernel"
 350
 351         modprobe lttng-test
 352
 353         kernel_event_generator $TESTAPP_STATE_FILE &
 354         local generator_pid=$!
 355
 356         test_on_register_evaluation kernel $generator_pid
 357
 358
 359         kill -s SIGTERM $generator_pid 2> /dev/null
 360         wait $generator_pid 2> /dev/null
 361         rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
 362
 363         rmmod lttng-test
 364 }
 365
 366 function test_on_register_evaluation ()
 367 {
 368         local domain_type=$1
 369         local event_generator_pid=$2
 370
 371         local app_pids=()
 372         local high_output_file_pattern="high_app_output_file_on_register_evaluation"
 373
 374         local output_dir=$(mktemp -d)
 375         local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_register_evaluation")
 376         local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
 377         local domain_string=""
 378         local event_name=""
 379
 380         case $domain_type in
 381                 ust)
 382                         domain_string=LTTNG_DOMAIN_UST
 383                         event_name="tp:tptest"
 384                         ;;
 385                 kernel)
 386                         domain_string=LTTNG_DOMAIN_KERNEL
 387                         event_name="lttng_test_filter_event"
 388                         ;;
 389                 *)
 390                         fail "Invalid domain type"
 391                         exit 1
 392                         ;;
 393         esac
 394
 395         # Setup
 396         LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
 397         start_lttng_sessiond
 398
 399         create_lttng_session_ok $SESSION_NAME $TRACE_PATH
 400         enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
 401         enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME
 402
 403         # Fetch consumerd testpoint pipe information
 404         # This is needed since the testpoint create a pipe with the consumer type suffixed
 405         consumerd_pipe=()
 406         for f in "$testpoint_pipe_path"*; do
 407                 consumerd_pipe+=("$f")
 408         done
 409
 410         high_app_output_file=${high_output_file_pattern}.first_receiver
 411         high_app_output_path=$output_dir/${high_app_output_file}
 412         start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1
 413
 414         wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
 415
 416         # Stop consumerd consumption to force high notification
 417         start_lttng_tracing_ok $SESSION_NAME
 418
 419         for pipe in "${consumerd_pipe[@]}"; do
 420                 stop_consumerd "${pipe}"
 421         done
 422
 423         wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"
 424
 425         # Start a second receiver, the receiver should receive a high
 426         # notification on subscription
 427         high_app_output_file=${high_output_file_pattern}.second_receiver
 428         high_app_output_path=$output_dir/${high_app_output_file}
 429         start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1
 430         wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
 431         wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"
 432
 433         # Resume consumerd
 434         for pipe in "${consumerd_pipe[@]}"; do
 435                 resume_consumerd "${pipe}"
 436         done
 437
 438         wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
 439         ret=$?
 440         ok $ret "Application for high notification terminated normally"
 441         if [[ $ret -ne "0" ]]; then
 442                 # Keep the file
 443                 print_errors "${high_output_file_pattern}"
 444         fi
 445
 446
 447         destroy_lttng_session_ok $SESSION_NAME
 448         stop_lttng_sessiond
 449
 450         kill -s SIGTERM $generator_pid 2> /dev/null
 451         wait $generator_pid 2> /dev/null
 452
 453         for pipe in "${consumerd_pipe[@]}"; do
 454                 rm -rf "${pipe}"
 455         done
 456
 457         rm -rf "$output_dir"
 458 }
 459
 460
 461 TESTS=(
 462         test_multi_app_ust
 463         test_on_register_evaluation_ust
 464 )
 465
 466 if [ "$(id -u)" == "0" ]; then
 467         validate_lttng_modules_present
 468         TESTS+=(
 469         test_multi_app_kernel
 470         test_on_register_evaluation_kernel
 471 )
 472 else
 473         skip 0 "Root access is needed. Skipping all kernel multi-app notification tests." $NUM_TEST_KERNEL
 474 fi
 475
 476
 477 for fct_test in ${TESTS[@]};
 478 do
 479         TRACE_PATH=$(mktemp -d)
 480
 481         ${fct_test}
 482         if [ $? -ne 0 ]; then
 483                 break;
 484         fi
 485
 486         # Only delete if successful
 487         rm -rf $TRACE_PATH
 488 done