tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18
  19 # To match 20201127-175802
  20 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  21 # The size of a long on this system
  22 system_long_bit_size=$(getconf LONG_BIT)
  23
  24 # Minimal kernel version supported for session daemon tests
  25 KERNEL_MAJOR_VERSION=2
  26 KERNEL_MINOR_VERSION=6
  27 KERNEL_PATCHLEVEL_VERSION=27
  28
  29 # We set the default UST register timeout and network and app socket timeout to
  30 # "wait forever", so that basic tests don't have to worry about hitting
  31 # timeouts on busy systems. Specialized tests should test those corner-cases.
  32 export LTTNG_UST_REGISTER_TIMEOUT=-1
  33 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  34 export LTTNG_APP_SOCKET_TIMEOUT=-1
  35
  36 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  37 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  38 # its own sessiond if none is running. It also ensures that 'lttng create'
  39 # fails when no sessiond is running.
  40 export LTTNG_SESSIOND_PATH="/bin/true"
  41
  42 source $TESTDIR/utils/tap/tap.sh
  43
  44 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  45         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  46 fi
  47
  48 # Enable job monitor mode.
  49 # Here we are mostly interested in the following from the monitor mode:
  50 #    All processes run in a separate process group.
  51 # This allows us to ensure that all subprocesses from all background tasks are
  52 # cleaned up correctly using signal to process group id.
  53 set -m
  54
  55 kill_background_jobs ()
  56 {
  57         local pids
  58         pids=$(jobs -p)
  59
  60         if [ -z "$pids" ]; then
  61                 # Empty
  62                 return 0
  63         fi
  64
  65         while read -r pid;
  66         do
  67                 # Use negative number to send the signal to the process group.
  68                 # This ensure that any subprocesses receive the signal.
  69                 # /dev/null is used since there is an acceptable race between
  70                 # the moments the pids are listed and the moment we send a
  71                 # signal.
  72                 kill -SIGTERM -- "-$pid" 2>/dev/null
  73         done <<< "$pids"
  74 }
  75
  76 function cleanup ()
  77 {
  78         # Try to kill daemons gracefully
  79         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  80         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  81
  82         # If daemons are still present, forcibly kill them
  83         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  84         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  85         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  86
  87         kill_background_jobs
  88 }
  89
  90 function full_cleanup ()
  91 {
  92         cleanup
  93         exit 1
  94 }
  95
  96 function LTTNG_BAIL_OUT ()
  97 {
  98         cleanup
  99         BAIL_OUT "$@"
 100 }
 101
 102 function null_pipes ()
 103 {
 104         exec 0>/dev/null
 105         exec 1>/dev/null
 106         exec 2>/dev/null
 107 }
 108
 109 trap full_cleanup SIGINT SIGTERM
 110
 111 # perl prove closes its child pipes before giving it a chance to run its
 112 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 113 # to allow those trap handlers to proceed.
 114
 115 trap null_pipes SIGPIPE
 116
 117 # Check pgrep from env, default to pgrep if none
 118 if [ -z "$PGREP" ]; then
 119         PGREP=pgrep
 120 fi
 121
 122 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 123 # identify the pids for multiple lttng related processes. The problem with "pgrep
 124 # -f" is that it ends up also looking at the arguments. We use a two stage
 125 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 126 # The second on perform grep on the basename of the first field of the
 127 # /proc/pid/cmdline of the previously identified pids. The first field
 128 # correspond to the actual command.
 129 function lttng_pgrep ()
 130 {
 131         local pattern=$1
 132         local possible_pids
 133         local full_command_no_argument
 134         local command_basename
 135
 136         possible_pids=$($PGREP -f "$pattern")
 137         if [ -z "$possible_pids" ]; then
 138                 return 0
 139         fi
 140
 141         while IFS= read -r pid ; do
 142                 # /proc/pid/cmdline is null separated.
 143                 if full_command_no_argument=$(cut -d '' -f 1 2>/dev/null < /proc/"$pid"/cmdline); then
 144                         command_basename=$(basename "$full_command_no_argument")
 145                         if grep -q "$pattern" <<< "$command_basename"; then
 146                                 echo "$pid"
 147                         fi
 148                 fi
 149         done <<< "$possible_pids"
 150         return 0
 151 }
 152
 153 function print_ok ()
 154 {
 155         # Check if we are a terminal
 156         if [ -t 1 ]; then
 157                 echo -e "\e[1;32mOK\e[0m"
 158         else
 159                 echo -e "OK"
 160         fi
 161 }
 162
 163 function print_fail ()
 164 {
 165         # Check if we are a terminal
 166         if [ -t 1 ]; then
 167                 echo -e "\e[1;31mFAIL\e[0m"
 168         else
 169                 echo -e "FAIL"
 170         fi
 171 }
 172
 173 function print_test_banner ()
 174 {
 175         local desc="$1"
 176         diag "$desc"
 177 }
 178
 179 function validate_kernel_version ()
 180 {
 181         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 182         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 183                 return 0
 184         fi
 185         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 186                 return 0
 187         fi
 188         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 189                 return 0
 190         fi
 191         return 1
 192 }
 193
 194 # Generate a random string
 195 #  $1 = number of characters; defaults to 16
 196 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 197 function randstring()
 198 {
 199         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 200         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 201         echo
 202 }
 203
 204 # Return the number of _configured_ CPUs.
 205 function conf_proc_count()
 206 {
 207         getconf _NPROCESSORS_CONF
 208         if [ $? -ne 0 ]; then
 209                 diag "Failed to get the number of configured CPUs"
 210         fi
 211         echo
 212 }
 213
 214 # Check if base lttng-modules are present.
 215 # Bail out on failure
 216 function validate_lttng_modules_present ()
 217 {
 218         # Check for loadable modules.
 219         modprobe -n lttng-tracer 2>/dev/null
 220         if [ $? -eq 0 ]; then
 221                 return 0
 222         fi
 223
 224         # Check for builtin modules.
 225         ls /proc/lttng > /dev/null 2>&1
 226         if [ $? -eq 0 ]; then
 227                 return 0
 228         fi
 229
 230         LTTNG_BAIL_OUT "LTTng modules not detected."
 231 }
 232
 233 function enable_kernel_lttng_event
 234 {
 235         local withtap="$1"
 236         local expected_to_fail="$2"
 237         local sess_name="$3"
 238         local event_name="$4"
 239         local channel_name="$5"
 240
 241         if [ -z "$event_name" ]; then
 242                 # Enable all event if no event name specified
 243                 event_name="-a"
 244         fi
 245
 246         if [ -z "$channel_name" ]; then
 247                 # default channel if none specified
 248                 chan=""
 249         else
 250                 chan="-c $channel_name"
 251         fi
 252
 253         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 254         ret=$?
 255         if [[ $expected_to_fail -eq "1" ]]; then
 256                 test $ret -ne "0"
 257                 ret=$?
 258                 if [ $withtap -eq "1" ]; then
 259                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 260                 fi
 261         else
 262                 if [ $withtap -eq "1" ]; then
 263                         ok $ret "Enable kernel event $event_name for session $sess_name"
 264                 fi
 265         fi
 266 }
 267
 268 function enable_kernel_lttng_event_ok ()
 269 {
 270         enable_kernel_lttng_event 1 0 "$@"
 271 }
 272
 273 function enable_kernel_lttng_event_fail ()
 274 {
 275         enable_kernel_lttng_event 1 1 "$@"
 276 }
 277
 278 function enable_kernel_lttng_event_notap ()
 279 {
 280         enable_kernel_lttng_event 0 0 "$@"
 281 }
 282
 283 # Old interface
 284 function lttng_enable_kernel_event
 285 {
 286         enable_kernel_lttng_event_ok "$@"
 287 }
 288
 289 function lttng_enable_kernel_syscall()
 290 {
 291         local expected_to_fail=$1
 292         local sess_name=$2
 293         local syscall_name=$3
 294         local channel_name=$4
 295
 296         if [ -z $syscall_name ]; then
 297                 # Enable all event if no syscall name specified
 298                 syscall_name="-a"
 299         fi
 300
 301         if [ -z $channel_name ]; then
 302                 # default channel if none specified
 303                 chan=""
 304         else
 305                 chan="-c $channel_name"
 306         fi
 307
 308         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 309         ret=$?
 310         if [[ $expected_to_fail -eq "1" ]]; then
 311                 test $ret -ne "0"
 312                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 313         else
 314                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 315         fi
 316 }
 317
 318 function lttng_enable_kernel_syscall_ok()
 319 {
 320         lttng_enable_kernel_syscall 0 "$@"
 321 }
 322
 323 function lttng_enable_kernel_syscall_fail()
 324 {
 325         lttng_enable_kernel_syscall 1 "$@"
 326 }
 327
 328 function lttng_disable_kernel_syscall()
 329 {
 330         local expected_to_fail=$1
 331         local sess_name=$2
 332         local syscall_name=$3
 333         local channel_name=$4
 334
 335         if [ -z $syscall_name ]; then
 336                 # Enable all event if no syscall name specified
 337                 syscall_name="-a"
 338         fi
 339
 340         if [ -z $channel_name ]; then
 341                 # default channel if none specified
 342                 chan=""
 343         else
 344                 chan="-c $channel_name"
 345         fi
 346
 347         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 348
 349         ret=$?
 350         if [[ $expected_to_fail -eq "1" ]]; then
 351                 test $ret -ne "0"
 352                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 353         else
 354                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 355         fi
 356 }
 357
 358 function lttng_disable_kernel_syscall_ok()
 359 {
 360         lttng_disable_kernel_syscall 0 "$@"
 361 }
 362
 363 function lttng_disable_kernel_syscall_fail()
 364 {
 365         lttng_disable_kernel_syscall 1 "$@"
 366 }
 367
 368 function lttng_enable_kernel_function_event ()
 369 {
 370         local expected_to_fail="$1"
 371         local sess_name="$2"
 372         local target="$3"
 373         local event_name="$4"
 374
 375         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 376         ret=$?
 377         if [[ $expected_to_fail -eq "1" ]]; then
 378                 test $ret -ne "0"
 379                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 380         else
 381                 ok $ret "Enable kernel function event for session $sess_name"
 382         fi
 383 }
 384
 385 function lttng_enable_kernel_function_event_ok ()
 386 {
 387         lttng_enable_kernel_function_event 0 "$@"
 388 }
 389
 390 function lttng_enable_kernel_userspace_probe_event ()
 391 {
 392         local expected_to_fail="$1"
 393         local sess_name="$2"
 394         local target="$3"
 395         local event_name="$4"
 396
 397         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 398         ret=$?
 399         if [[ $expected_to_fail -eq "1" ]]; then
 400                 test $ret -ne "0"
 401                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 402         else
 403                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 404         fi
 405 }
 406
 407 function lttng_enable_kernel_userspace_probe_event_fail ()
 408 {
 409         lttng_enable_kernel_userspace_probe_event 1 "$@"
 410 }
 411
 412 function lttng_enable_kernel_userspace_probe_event_ok ()
 413 {
 414         lttng_enable_kernel_userspace_probe_event 0 "$@"
 415 }
 416
 417 function disable_kernel_lttng_userspace_probe_event_ok ()
 418 {
 419         local sess_name="$1"
 420         local event_name="$2"
 421
 422         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 423         ok $? "Disable kernel event $target for session $sess_name"
 424 }
 425 function lttng_enable_kernel_channel()
 426 {
 427         local withtap=$1
 428         local expected_to_fail=$2
 429         local sess_name=$3
 430         local channel_name=$4
 431         local opts="${@:5}"
 432
 433         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 434         ret=$?
 435         if [[ $expected_to_fail -eq "1" ]]; then
 436                 test "$ret" -ne "0"
 437                 ret=$?
 438                 if [ $withtap -eq "1" ]; then
 439                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 440                 fi
 441         else
 442                 if [ $withtap -eq "1" ]; then
 443                         ok $ret "Enable channel $channel_name for session $sess_name"
 444                 fi
 445         fi
 446 }
 447
 448 function lttng_enable_kernel_channel_ok()
 449 {
 450         lttng_enable_kernel_channel 1 0 "$@"
 451 }
 452
 453 function lttng_enable_kernel_channel_fail()
 454 {
 455         lttng_enable_kernel_channel 1 1 "$@"
 456 }
 457
 458 function lttng_enable_kernel_channel_notap()
 459 {
 460         lttng_enable_kernel_channel 0 0 "$@"
 461 }
 462
 463 function enable_kernel_lttng_channel_ok()
 464 {
 465         lttng_enable_kernel_channel 1 0 "$@"
 466 }
 467
 468 function lttng_disable_kernel_channel()
 469 {
 470         local expected_to_fail=$1
 471         local sess_name=$2
 472         local channel_name=$3
 473
 474         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 475         ret=$?
 476         if [[ $expected_to_fail -eq "1" ]]; then
 477                 test "$ret" -ne "0"
 478                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 479         else
 480                 ok $ret "Disable channel $channel_name for session $sess_name"
 481         fi
 482 }
 483
 484 function lttng_disable_kernel_channel_ok()
 485 {
 486         lttng_disable_kernel_channel 0 "$@"
 487 }
 488
 489 function lttng_disable_kernel_channel_fail()
 490 {
 491         lttng_disable_kernel_channel 1 "$@"
 492 }
 493
 494 function start_lttng_relayd_opt()
 495 {
 496         local withtap=$1
 497         local process_mode=$2
 498         local opt=$3
 499
 500         DIR=$(readlink -f "$TESTDIR")
 501
 502         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 503                 # shellcheck disable=SC2086
 504                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 505                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 506                 if [ $? -eq 1 ]; then
 507                         if [ $withtap -eq "1" ]; then
 508                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 509                         fi
 510                         return 1
 511                 else
 512                         if [ $withtap -eq "1" ]; then
 513                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 514                         fi
 515                 fi
 516         else
 517                 pass "Start lttng-relayd (opt: $opt)"
 518         fi
 519 }
 520
 521 function start_lttng_relayd()
 522 {
 523         start_lttng_relayd_opt 1 "-b" "$@"
 524 }
 525
 526 function start_lttng_relayd_notap()
 527 {
 528         start_lttng_relayd_opt 0 "-b" "$@"
 529 }
 530
 531 function stop_lttng_relayd_opt()
 532 {
 533         local withtap=$1
 534         local is_cleanup=$2
 535         local signal=$3
 536         local timeout_s=$4
 537         local dtimeleft_s=
 538         local retval=0
 539         local pids
 540
 541         if [ -z "$signal" ]; then
 542                 signal="SIGTERM"
 543         fi
 544
 545
 546         # Multiply time by 2 to simplify integer arithmetic
 547         if [ -n "$timeout_s" ]; then
 548                 dtimeleft_s=$((timeout_s * 2))
 549         fi
 550
 551
 552         pids=$(lttng_pgrep "$RELAYD_MATCH")
 553         if [ -z "$pids" ]; then
 554                 if [ "$is_cleanup" -eq 1 ]; then
 555                         :
 556                 elif [ "$withtap" -eq "1" ]; then
 557                         fail "No relay daemon to kill"
 558                 else
 559                         LTTNG_BAIL_OUT "No relay daemon to kill"
 560                 fi
 561                 return 0
 562         fi
 563
 564         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 565
 566         # shellcheck disable=SC2086
 567         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 568                 retval=1
 569                 if [ "$withtap" -eq "1" ]; then
 570                         fail "Kill relay daemon"
 571                 fi
 572         else
 573                 out=1
 574                 while [ -n "$out" ]; do
 575                         out=$(lttng_pgrep "$RELAYD_MATCH")
 576                         if [ -n "$dtimeleft_s" ]; then
 577                                 if [ $dtimeleft_s -lt 0 ]; then
 578                                         out=
 579                                         retval=1
 580                                 fi
 581                                 dtimeleft_s=$((dtimeleft_s - 1))
 582                         fi
 583                         sleep 0.5
 584                 done
 585                 if [ "$withtap" -eq "1" ]; then
 586                         if [ "$retval" -eq "0" ]; then
 587                                 pass "Wait after kill relay daemon"
 588                         else
 589                                 fail "Wait after kill relay daemon"
 590                         fi
 591                 fi
 592         fi
 593         return $retval
 594 }
 595
 596 function stop_lttng_relayd()
 597 {
 598         stop_lttng_relayd_opt 1 0 "$@"
 599 }
 600
 601 function stop_lttng_relayd_notap()
 602 {
 603         stop_lttng_relayd_opt 0 0 "$@"
 604 }
 605
 606 function stop_lttng_relayd_cleanup()
 607 {
 608         stop_lttng_relayd_opt 0 1 "$@"
 609 }
 610
 611 #First arg: show tap output
 612 #Second argument: load path for automatic loading
 613 function start_lttng_sessiond_opt()
 614 {
 615         local withtap=$1
 616         local load_path=$2
 617
 618         # The rest of the arguments will be passed directly to lttng-sessiond.
 619         shift 2
 620
 621         local env_vars=""
 622         local consumerd=""
 623
 624         local long_bit_value=
 625         long_bit_value=$(getconf LONG_BIT)
 626
 627         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 628                 # Env variable requested no session daemon
 629                 return
 630         fi
 631
 632         DIR=$(readlink -f "$TESTDIR")
 633
 634         # Get long_bit value for 32/64 consumerd
 635         case "$long_bit_value" in
 636                 32)
 637                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 638                         ;;
 639                 64)
 640                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 641                         ;;
 642                 *)
 643                         return
 644                         ;;
 645         esac
 646
 647         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 648         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 649                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 650         fi
 651         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 652
 653         if ! validate_kernel_version; then
 654             fail "Start session daemon"
 655             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 656         fi
 657
 658         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 659         export LTTNG_SESSION_CONFIG_XSD_PATH
 660
 661         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 662                 # Have a load path ?
 663                 if [ -n "$load_path" ]; then
 664                         # shellcheck disable=SC2086
 665                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 666                 else
 667                         # shellcheck disable=SC2086
 668                         env $env_vars --background "$consumerd" "$@"
 669                 fi
 670                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 671                 status=$?
 672                 if [ "$withtap" -eq "1" ]; then
 673                         ok $status "Start session daemon"
 674                 fi
 675         fi
 676 }
 677
 678 function start_lttng_sessiond()
 679 {
 680         start_lttng_sessiond_opt 1 "$@"
 681 }
 682
 683 function start_lttng_sessiond_notap()
 684 {
 685         start_lttng_sessiond_opt 0 "$@"
 686 }
 687
 688 function stop_lttng_sessiond_opt()
 689 {
 690         local withtap=$1
 691         local is_cleanup=$2
 692         local signal=$3
 693         local timeout_s=$4
 694         local dtimeleft_s=
 695         local retval=0
 696         local runas_pids
 697         local pids
 698
 699         if [ -z "$signal" ]; then
 700                 signal=SIGTERM
 701         fi
 702
 703         # Multiply time by 2 to simplify integer arithmetic
 704         if [ -n "$timeout_s" ]; then
 705                 dtimeleft_s=$((timeout_s * 2))
 706         fi
 707
 708         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 709                 # Env variable requested no session daemon
 710                 return 0
 711         fi
 712
 713         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 714         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 715
 716         if [ -n "$runas_pids" ]; then
 717                 pids="$pids $runas_pids"
 718         fi
 719
 720         if [ -z "$pids" ]; then
 721                 if [ "$is_cleanup" -eq 1 ]; then
 722                         :
 723                 elif [ "$withtap" -eq "1" ]; then
 724                         fail "No session daemon to kill"
 725                 else
 726                         LTTNG_BAIL_OUT "No session daemon to kill"
 727                 fi
 728                 return 0
 729         fi
 730
 731         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 732
 733         # shellcheck disable=SC2086
 734         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 735                 retval=1
 736                 if [ "$withtap" -eq "1" ]; then
 737                         fail "Kill sessions daemon"
 738                 fi
 739         else
 740                 out=1
 741                 while [ -n "$out" ]; do
 742                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 743                         if [ -n "$dtimeleft_s" ]; then
 744                                 if [ $dtimeleft_s -lt 0 ]; then
 745                                         out=
 746                                         retval=1
 747                                 fi
 748                                 dtimeleft_s=$((dtimeleft_s - 1))
 749                         fi
 750                         sleep 0.5
 751                 done
 752                 out=1
 753                 while [ -n "$out" ]; do
 754                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 755                         if [ -n "$dtimeleft_s" ]; then
 756                                 if [ $dtimeleft_s -lt 0 ]; then
 757                                         out=
 758                                         retval=1
 759                                 fi
 760                                 dtimeleft_s=$((dtimeleft_s - 1))
 761                         fi
 762                         sleep 0.5
 763                 done
 764
 765                 if [ "$withtap" -eq "1" ]; then
 766                         if [ "$retval" -eq "0" ]; then
 767                                 pass "Wait after kill session daemon"
 768                         else
 769                                 fail "Wait after kill session daemon"
 770                         fi
 771                 fi
 772         fi
 773         if [ "$signal" = "SIGKILL" ]; then
 774                 if [ "$(id -u)" -eq "0" ]; then
 775                         local modules=
 776                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 777
 778                         if [ -n "$modules" ]; then
 779                                 diag "Unloading all LTTng modules"
 780                                 modprobe --remove "$modules"
 781                         fi
 782                 fi
 783         fi
 784
 785         return $retval
 786 }
 787
 788 function stop_lttng_sessiond()
 789 {
 790         stop_lttng_sessiond_opt 1 0 "$@"
 791 }
 792
 793 function stop_lttng_sessiond_notap()
 794 {
 795         stop_lttng_sessiond_opt 0 0 "$@"
 796 }
 797
 798 function stop_lttng_sessiond_cleanup()
 799 {
 800         stop_lttng_sessiond_opt 0 1 "$@"
 801 }
 802
 803 function sigstop_lttng_sessiond_opt()
 804 {
 805         local withtap=$1
 806         local signal=SIGSTOP
 807         local pids
 808
 809         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 810                 # Env variable requested no session daemon
 811                 return
 812         fi
 813
 814         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 815
 816         if [ "$withtap" -eq "1" ]; then
 817                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 818         fi
 819
 820         # shellcheck disable=SC2086
 821         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 822                 if [ "$withtap" -eq "1" ]; then
 823                         fail "Sending SIGSTOP to session daemon"
 824                 fi
 825         else
 826                 out=1
 827                 while [ $out -ne 0 ]; do
 828                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 829
 830                         # Wait until state becomes stopped for session
 831                         # daemon(s).
 832                         out=0
 833                         for sessiond_pid in $pids; do
 834                                 state="$(ps -p "$sessiond_pid" -o state= )"
 835                                 if [[ -n "$state" && "$state" != "T" ]]; then
 836                                         out=1
 837                                 fi
 838                         done
 839                         sleep 0.5
 840                 done
 841                 if [ "$withtap" -eq "1" ]; then
 842                         pass "Sending SIGSTOP to session daemon"
 843                 fi
 844         fi
 845 }
 846
 847 function sigstop_lttng_sessiond()
 848 {
 849         sigstop_lttng_sessiond_opt 1 "$@"
 850 }
 851
 852 function sigstop_lttng_sessiond_notap()
 853 {
 854         sigstop_lttng_sessiond_opt 0 "$@"
 855 }
 856
 857 function stop_lttng_consumerd_opt()
 858 {
 859         local withtap=$1
 860         local is_cleanup=$2
 861         local signal=$3
 862         local timeout_s=$4
 863         local dtimeleft_s=
 864         local retval=0
 865         local pids
 866
 867         if [ -z "$signal" ]; then
 868                 signal=SIGTERM
 869         fi
 870
 871         # Multiply time by 2 to simplify integer arithmetic
 872         if [ -n "$timeout_s" ]; then
 873                 dtimeleft_s=$((timeout_s * 2))
 874         fi
 875
 876         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 877
 878         if [ -z "$pids" ]; then
 879                 if [ "$is_cleanup" -eq 1 ]; then
 880                         :
 881                 elif [ "$withtap" -eq "1" ]; then
 882                         fail "No consumerd daemon to kill"
 883                 else
 884                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
 885                 fi
 886                 return 0
 887         fi
 888
 889         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 890
 891         # shellcheck disable=SC2086
 892         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 893                 retval=1
 894                 if [ "$withtap" -eq "1" ]; then
 895                         fail "Kill consumer daemon"
 896                 fi
 897         else
 898                 out=1
 899                 while [ $out -ne 0 ]; do
 900                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 901
 902                         # If consumerds are still present check their status.
 903                         # A zombie status qualifies the consumerd as *killed*
 904                         out=0
 905                         for consumer_pid in $pids; do
 906                                 state="$(ps -p "$consumer_pid" -o state= )"
 907                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 908                                         out=1
 909                                 fi
 910                         done
 911                         if [ -n "$dtimeleft_s" ]; then
 912                                 if [ $dtimeleft_s -lt 0 ]; then
 913                                         out=0
 914                                         retval=1
 915                                 fi
 916                                 dtimeleft_s=$((dtimeleft_s - 1))
 917                         fi
 918                         sleep 0.5
 919                 done
 920                 if [ "$withtap" -eq "1" ]; then
 921                         if [ "$retval" -eq "0" ]; then
 922                                 pass "Wait after kill consumer daemon"
 923                         else
 924                                 fail "Wait after kill consumer daemon"
 925                         fi
 926                 fi
 927         fi
 928
 929         return $retval
 930 }
 931
 932 function stop_lttng_consumerd()
 933 {
 934         stop_lttng_consumerd_opt 1 0 "$@"
 935 }
 936
 937 function stop_lttng_consumerd_notap()
 938 {
 939         stop_lttng_consumerd_opt 0 0 "$@"
 940 }
 941
 942 function stop_lttng_consumerd_cleanup()
 943 {
 944         stop_lttng_consumerd_opt 0 1 "$@"
 945 }
 946
 947 function sigstop_lttng_consumerd_opt()
 948 {
 949         local withtap=$1
 950         local signal=SIGSTOP
 951         local pids
 952
 953         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 954
 955         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 956
 957         # shellcheck disable=SC2086
 958         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 959         retval=$?
 960
 961         if [ $retval -eq 1 ]; then
 962                 if [ "$withtap" -eq "1" ]; then
 963                         fail "Sending SIGSTOP to consumer daemon"
 964                 fi
 965                 return 1
 966         else
 967                 out=1
 968                 while [ $out -ne 0 ]; do
 969                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 970
 971                         # Wait until state becomes stopped for all
 972                         # consumers.
 973                         out=0
 974                         for consumer_pid in $pids; do
 975                                 state="$(ps -p "$consumer_pid" -o state= )"
 976                                 if [[ -n "$state" && "$state" != "T" ]]; then
 977                                         out=1
 978                                 fi
 979                         done
 980                         sleep 0.5
 981                 done
 982                 if [ "$withtap" -eq "1" ]; then
 983                         pass "Sending SIGSTOP to consumer daemon"
 984                 fi
 985         fi
 986         return $retval
 987 }
 988
 989 function sigstop_lttng_consumerd()
 990 {
 991         sigstop_lttng_consumerd_opt 1 "$@"
 992 }
 993
 994 function sigstop_lttng_consumerd_notap()
 995 {
 996         sigstop_lttng_consumerd_opt 0 "$@"
 997 }
 998
 999 function list_lttng_with_opts ()
1000 {
1001         local opts=$1
1002         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1003         ok $? "Lttng-tool list command with option $opts"
1004 }
1005
1006 function create_lttng_session_no_output ()
1007 {
1008         local sess_name=$1
1009         local opts="${@:2}"
1010
1011         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1012         ok $? "Create session $sess_name in no-output mode"
1013 }
1014
1015 function create_lttng_session_uri () {
1016         local sess_name=$1
1017         local uri=$2
1018         local opts="${@:3}"
1019
1020         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1021         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1022 }
1023
1024 function create_lttng_session ()
1025 {
1026         local withtap=$1
1027         local expected_to_fail=$2
1028         local sess_name=$3
1029         local trace_path=$4
1030         local opt=$5
1031
1032         if [ -z "$trace_path" ]; then
1033                 # Use lttng-sessiond default output.
1034                 trace_path=""
1035         else
1036                 trace_path="-o $trace_path"
1037         fi
1038
1039         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create "$sess_name" $trace_path $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1040         ret=$?
1041         if [ $expected_to_fail -eq "1" ]; then
1042                 test "$ret" -ne "0"
1043                 ret=$?
1044                 if [ $withtap -eq "1" ]; then
1045                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1046                 fi
1047         else
1048                 if [ $withtap -eq "1" ]; then
1049                         ok $ret "Create session $sess_name in $trace_path"
1050                 fi
1051         fi
1052         return $ret
1053 }
1054
1055 function create_lttng_session_ok ()
1056 {
1057         create_lttng_session 1 0 "$@"
1058 }
1059
1060 function create_lttng_session_fail ()
1061 {
1062         create_lttng_session 1 1 "$@"
1063 }
1064
1065 function create_lttng_session_notap ()
1066 {
1067         create_lttng_session 0 0 "$@"
1068 }
1069
1070
1071 function enable_ust_lttng_channel ()
1072 {
1073         local withtap=$1
1074         local expected_to_fail=$2
1075         local sess_name=$3
1076         local channel_name=$4
1077         local opts="${@:5}"
1078
1079         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1080         ret=$?
1081         if [[ $expected_to_fail -eq "1" ]]; then
1082                 test "$ret" -ne "0"
1083                 ret=$?
1084                 if [ $withtap -eq "1" ]; then
1085                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1086                 fi
1087         else
1088                 if [ $withtap -eq "1" ]; then
1089                         ok $ret "Enable channel $channel_name for session $sess_name"
1090                 fi
1091         fi
1092         return $ret
1093 }
1094
1095 function enable_ust_lttng_channel_ok ()
1096 {
1097         enable_ust_lttng_channel 1 0 "$@"
1098 }
1099
1100 function enable_ust_lttng_channel_fail ()
1101 {
1102         enable_ust_lttng_channel 1 1 "$@"
1103 }
1104
1105 function enable_ust_lttng_channel_notap ()
1106 {
1107         enable_ust_lttng_channel 0 0 "$@"
1108 }
1109
1110 function disable_ust_lttng_channel()
1111 {
1112         local sess_name=$1
1113         local channel_name=$2
1114
1115         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1116         ok $? "Disable channel $channel_name for session $sess_name"
1117 }
1118
1119 function enable_lttng_mmap_overwrite_kernel_channel()
1120 {
1121         local sess_name=$1
1122         local channel_name=$2
1123
1124         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1125         ok $? "Enable channel $channel_name for session $sess_name"
1126 }
1127
1128 function enable_lttng_mmap_discard_small_kernel_channel()
1129 {
1130         local sess_name=$1
1131         local channel_name=$2
1132
1133         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1134         ok $? "Enable small discard channel $channel_name for session $sess_name"
1135 }
1136
1137 function enable_lttng_mmap_overwrite_small_kernel_channel()
1138 {
1139         local sess_name=$1
1140         local channel_name=$2
1141
1142         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1143         ok $? "Enable small discard channel $channel_name for session $sess_name"
1144 }
1145
1146 function enable_lttng_mmap_overwrite_ust_channel()
1147 {
1148         local sess_name=$1
1149         local channel_name=$2
1150
1151         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1152         ok $? "Enable channel $channel_name for session $sess_name"
1153 }
1154
1155 function enable_ust_lttng_event ()
1156 {
1157         local withtap=$1
1158         local expected_to_fail=$2
1159         local sess_name=$3
1160         local event_name="$4"
1161         local channel_name=$5
1162
1163         if [ -z $channel_name ]; then
1164                 # default channel if none specified
1165                 chan=""
1166         else
1167                 chan="-c $channel_name"
1168         fi
1169
1170         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1171         ret=$?
1172         if [[ $expected_to_fail -eq "1" ]]; then
1173                 test $ret -ne "0"
1174                 ret=$?
1175                 if [[ $withtap -eq "1" ]]; then
1176                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1177                 fi
1178         else
1179                 if [[ $withtap -eq "1" ]]; then
1180                         ok $ret "Enable ust event $event_name for session $sess_name"
1181                 fi
1182         fi
1183         return $ret
1184 }
1185
1186 function enable_ust_lttng_event_ok ()
1187 {
1188         enable_ust_lttng_event 1 0 "$@"
1189 }
1190
1191 function enable_ust_lttng_event_fail ()
1192 {
1193         enable_ust_lttng_event 1 1 "$@"
1194 }
1195
1196 function enable_ust_lttng_event_notap ()
1197 {
1198         enable_ust_lttng_event 0 0 "$@"
1199 }
1200
1201 function enable_jul_lttng_event()
1202 {
1203         sess_name=$1
1204         event_name="$2"
1205         channel_name=$3
1206
1207         if [ -z $channel_name ]; then
1208                 # default channel if none specified
1209                 chan=""
1210         else
1211                 chan="-c $channel_name"
1212         fi
1213
1214         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1215         ok $? "Enable JUL event $event_name for session $sess_name"
1216 }
1217
1218 function enable_jul_lttng_event_loglevel()
1219 {
1220         local sess_name=$1
1221         local event_name="$2"
1222         local loglevel=$3
1223         local channel_name=$4
1224
1225         if [ -z $channel_name ]; then
1226                 # default channel if none specified
1227                 chan=""
1228         else
1229                 chan="-c $channel_name"
1230         fi
1231
1232         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1233         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1234 }
1235
1236 function enable_log4j_lttng_event()
1237 {
1238         sess_name=$1
1239         event_name="$2"
1240         channel_name=$3
1241
1242         if [ -z $channel_name ]; then
1243                 # default channel if none specified
1244                 chan=""
1245         else
1246                 chan="-c $channel_name"
1247         fi
1248
1249         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1250         ok $? "Enable LOG4J event $event_name for session $sess_name"
1251 }
1252
1253 function enable_log4j_lttng_event_loglevel()
1254 {
1255         local sess_name=$1
1256         local event_name="$2"
1257         local loglevel=$3
1258         local channel_name=$4
1259
1260         if [ -z $channel_name ]; then
1261                 # default channel if none specified
1262                 chan=""
1263         else
1264                 chan="-c $channel_name"
1265         fi
1266
1267         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1268         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
1269 }
1270
1271 function enable_python_lttng_event()
1272 {
1273         sess_name=$1
1274         event_name="$2"
1275         channel_name=$3
1276
1277         if [ -z $channel_name ]; then
1278                 # default channel if none specified
1279                 chan=""
1280         else
1281                 chan="-c $channel_name"
1282         fi
1283
1284         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1285         ok $? "Enable Python event $event_name for session $sess_name"
1286 }
1287
1288 function enable_python_lttng_event_loglevel()
1289 {
1290         local sess_name=$1
1291         local event_name="$2"
1292         local loglevel=$3
1293         local channel_name=$4
1294
1295         if [ -z $channel_name ]; then
1296                 # default channel if none specified
1297                 chan=""
1298         else
1299                 chan="-c $channel_name"
1300         fi
1301
1302         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1303         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1304 }
1305
1306 function enable_ust_lttng_event_filter()
1307 {
1308         local sess_name="$1"
1309         local event_name="$2"
1310         local filter="$3"
1311         local channel_name=$4
1312
1313         if [ -z $channel_name ]; then
1314                 # default channel if none specified
1315                 chan=""
1316         else
1317                 chan="-c $channel_name"
1318         fi
1319
1320         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event $chan "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1321         ok $? "Enable event $event_name with filtering for session $sess_name"
1322 }
1323
1324 function enable_ust_lttng_event_loglevel()
1325 {
1326         local sess_name="$1"
1327         local event_name="$2"
1328         local loglevel="$3"
1329
1330         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1331         ok $? "Enable event $event_name with loglevel $loglevel"
1332 }
1333
1334 function enable_ust_lttng_event_loglevel_only()
1335 {
1336         local sess_name="$1"
1337         local event_name="$2"
1338         local loglevel="$3"
1339
1340         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1341         ok $? "Enable event $event_name with loglevel-only $loglevel"
1342 }
1343
1344 function disable_ust_lttng_event ()
1345 {
1346         local sess_name="$1"
1347         local event_name="$2"
1348         local channel_name="$3"
1349
1350         if [ -z $channel_name ]; then
1351                 # default channel if none specified
1352                 chan=""
1353         else
1354                 chan="-c $channel_name"
1355         fi
1356
1357         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1358         ok $? "Disable event $event_name for session $sess_name"
1359 }
1360
1361 function disable_jul_lttng_event ()
1362 {
1363         local sess_name="$1"
1364         local event_name="$2"
1365
1366         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1367         ok $? "Disable JUL event $event_name for session $sess_name"
1368 }
1369
1370 function disable_log4j_lttng_event ()
1371 {
1372         local sess_name="$1"
1373         local event_name="$2"
1374
1375         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1376         ok $? "Disable LOG4J event $event_name for session $sess_name"
1377 }
1378
1379 function disable_python_lttng_event ()
1380 {
1381         local sess_name="$1"
1382         local event_name="$2"
1383
1384         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1385         ok $? "Disable Python event $event_name for session $sess_name"
1386 }
1387
1388 function start_lttng_tracing_opt ()
1389 {
1390         local withtap=$1
1391         local expected_to_fail=$2
1392         local sess_name=$3
1393
1394         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1395         ret=$?
1396         if [[ $expected_to_fail -eq "1" ]]; then
1397                 test "$ret" -ne "0"
1398                 ret=$?
1399                 if [ $withtap -eq "1" ]; then
1400                         ok $? "Start tracing for session $sess_name failed as expected"
1401                 fi
1402         else
1403                 if [ $withtap -eq "1" ]; then
1404                         ok $ret "Start tracing for session $sess_name"
1405                 fi
1406         fi
1407 }
1408
1409 function start_lttng_tracing_ok ()
1410 {
1411         start_lttng_tracing_opt 1 0 "$@"
1412 }
1413
1414 function start_lttng_tracing_fail ()
1415 {
1416         start_lttng_tracing_opt 1 1 "$@"
1417 }
1418
1419 function start_lttng_tracing_notap ()
1420 {
1421         start_lttng_tracing_opt 0 1 "$@"
1422 }
1423
1424 function stop_lttng_tracing_opt ()
1425 {
1426         local withtap=$1
1427         local expected_to_fail=$2
1428         local sess_name=$3
1429
1430         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1431         ret=$?
1432         if [[ $expected_to_fail -eq "1" ]]; then
1433                 test "$ret" -ne "0"
1434                 ret=$?
1435                 if [ $withtap -eq "1" ]; then
1436                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1437                 fi
1438         else
1439                 if [ $withtap -eq "1" ]; then
1440                         ok $ret "Stop lttng tracing for session $sess_name"
1441                 fi
1442         fi
1443 }
1444
1445 function stop_lttng_tracing_ok ()
1446 {
1447         stop_lttng_tracing_opt 1 0 "$@"
1448 }
1449
1450 function stop_lttng_tracing_fail ()
1451 {
1452         stop_lttng_tracing_opt 1 1 "$@"
1453 }
1454
1455 function stop_lttng_tracing_notap ()
1456 {
1457         stop_lttng_tracing_opt 0 0 "$@"
1458 }
1459
1460 function destroy_lttng_session ()
1461 {
1462         local withtap=$1
1463         local expected_to_fail=$2
1464         local sess_name=$3
1465
1466         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1467         ret=$?
1468         if [[ $expected_to_fail -eq "1" ]]; then
1469                 test "$ret" -ne "0"
1470                 ret=$?
1471                 if [ $withtap -eq "1" ]; then
1472                         ok $ret "Destroy session $sess_name failed as expected"
1473                 fi
1474         else
1475                 if [ $withtap -eq "1" ]; then
1476                         ok $ret "Destroy session $sess_name"
1477                 fi
1478         fi
1479 }
1480
1481 function destroy_lttng_session_ok ()
1482 {
1483         destroy_lttng_session 1 0 "$@"
1484
1485 }
1486
1487 function destroy_lttng_session_fail ()
1488 {
1489         destroy_lttng_session 1 1 "$@"
1490 }
1491
1492 function destroy_lttng_session_notap ()
1493 {
1494         destroy_lttng_session 0 0 "$@"
1495 }
1496
1497 function destroy_lttng_sessions ()
1498 {
1499         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1500         ok $? "Destroy all lttng sessions"
1501 }
1502
1503 function lttng_snapshot_add_output ()
1504 {
1505         local expected_to_fail=$1
1506         local sess_name=$2
1507         local trace_path=$3
1508         local opts=$4
1509
1510         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1511         ret=$?
1512         if [[ $expected_to_fail -eq 1 ]]; then
1513                 test "$ret" -ne "0"
1514                 ok $? "Added snapshot output $trace_path failed as expected"
1515         else
1516                 ok $ret "Added snapshot output $trace_path"
1517         fi
1518 }
1519
1520 function lttng_snapshot_add_output_ok ()
1521 {
1522         lttng_snapshot_add_output 0 "$@"
1523 }
1524
1525 function lttng_snapshot_add_output_fail ()
1526 {
1527         lttng_snapshot_add_output 1 "$@"
1528 }
1529
1530 function lttng_snapshot_del_output ()
1531 {
1532         local expected_to_fail=$1
1533         local sess_name=$2
1534         local id=$3
1535
1536         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1537         ret=$?
1538         if [[ $expected_to_fail -eq "1" ]]; then
1539                 test "$ret" -ne "0"
1540                 ok $? "Deleted snapshot output id $id failed as expected"
1541         else
1542                 ok $ret "Deleted snapshot output id $id"
1543         fi
1544 }
1545
1546 function lttng_snapshot_del_output_ok ()
1547 {
1548         lttng_snapshot_del_output 0 "$@"
1549 }
1550
1551 function lttng_snapshot_del_output_fail ()
1552 {
1553         lttng_snapshot_del_output 1 "$@"
1554 }
1555
1556 function lttng_snapshot_record ()
1557 {
1558         local sess_name=$1
1559         local trace_path=$2
1560
1561         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s "$sess_name" "$trace_path" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1562         ok $? "Snapshot recorded"
1563 }
1564
1565 function lttng_snapshot_list ()
1566 {
1567         local sess_name=$1
1568         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1569         ok $? "Snapshot list"
1570 }
1571
1572 function lttng_save()
1573 {
1574         local sess_name=$1
1575         local opts=$2
1576
1577         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1578         ok $? "Session saved"
1579 }
1580
1581 function lttng_load()
1582 {
1583         local expected_to_fail=$1
1584         local opts=$2
1585
1586         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1587         ret=$?
1588         if [[ $expected_to_fail -eq "1" ]]; then
1589                 test $ret -ne "0"
1590                 ok $? "Load command failed as expected with opts: $opts"
1591         else
1592                 ok $ret "Load command with opts: $opts"
1593         fi
1594 }
1595
1596 function lttng_load_ok()
1597 {
1598         lttng_load 0 "$@"
1599 }
1600
1601 function lttng_load_fail()
1602 {
1603         lttng_load 1 "$@"
1604 }
1605
1606 function lttng_track()
1607 {
1608         local expected_to_fail="$1"
1609         shift 1
1610         local opts="$@"
1611         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1612         ret=$?
1613         if [[ $expected_to_fail -eq "1" ]]; then
1614                 test $ret -ne "0"
1615                 ok $? "Track command failed as expected with opts: $opts"
1616         else
1617                 ok $ret "Track command with opts: $opts"
1618         fi
1619 }
1620
1621 function lttng_track_ok()
1622 {
1623         lttng_track 0 "$@"
1624 }
1625
1626 function lttng_track_fail()
1627 {
1628         lttng_track 1 "$@"
1629 }
1630
1631 function lttng_untrack()
1632 {
1633         local expected_to_fail="$1"
1634         shift 1
1635         local opts="$@"
1636         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1637         ret=$?
1638         if [[ $expected_to_fail -eq "1" ]]; then
1639                 test $ret -ne "0"
1640                 ok $? "Untrack command failed as expected with opts: $opts"
1641         else
1642                 ok $ret "Untrack command with opts: $opts"
1643         fi
1644 }
1645
1646 function lttng_untrack_ok()
1647 {
1648         lttng_untrack 0 "$@"
1649 }
1650
1651 function lttng_untrack_fail()
1652 {
1653         lttng_untrack 1 "$@"
1654 }
1655
1656 function lttng_track_pid_ok()
1657 {
1658         PID=$1
1659         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1660         ok $? "Lttng track pid on the kernel domain"
1661 }
1662
1663 function lttng_untrack_kernel_all_ok()
1664 {
1665         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1666         ok $? "Lttng untrack all pid on the kernel domain"
1667 }
1668
1669 function lttng_track_ust_ok()
1670 {
1671         lttng_track_ok -u "$@"
1672 }
1673
1674 function lttng_track_ust_fail()
1675 {
1676         lttng_track_fail -u "$@"
1677 }
1678
1679 function lttng_track_kernel_ok()
1680 {
1681         lttng_track_ok -k "$@"
1682 }
1683
1684 function lttng_track_kernel_fail()
1685 {
1686         lttng_track_fail -k "$@"
1687 }
1688
1689 function lttng_untrack_ust_ok()
1690 {
1691         lttng_untrack_ok -u "$@"
1692 }
1693
1694 function lttng_untrack_ust_fail()
1695 {
1696         lttng_untrack_fail -u "$@"
1697 }
1698
1699 function lttng_untrack_kernel_ok()
1700 {
1701         lttng_untrack_ok -k "$@"
1702 }
1703
1704 function lttng_untrack_kernel_fail()
1705 {
1706         lttng_untrack_fail -k "$@"
1707 }
1708
1709 function lttng_add_context_list()
1710 {
1711         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1712         ret=$?
1713         ok $ret "Context listing"
1714 }
1715
1716 function add_context_lttng()
1717 {
1718         local expected_to_fail="$1"
1719         local domain="$2"
1720         local session_name="$3"
1721         local channel_name="$4"
1722         local type="$5"
1723
1724         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1725         ret=$?
1726         if [[ $expected_to_fail -eq "1" ]]; then
1727                 test $ret -ne "0"
1728                 ok $? "Add context command failed as expected for type: $type"
1729         else
1730                 ok $ret "Add context command for type: $type"
1731         fi
1732 }
1733
1734 function add_context_ust_ok()
1735 {
1736         add_context_lttng 0 -u "$@"
1737 }
1738
1739 function add_context_ust_fail()
1740 {
1741         add_context_lttng 1 -u "$@"
1742 }
1743
1744 function add_context_kernel_ok()
1745 {
1746         add_context_lttng 0 -k "$@"
1747 }
1748
1749 function add_context_kernel_fail()
1750 {
1751         add_context_lttng 1 -k "$@"
1752 }
1753
1754 function wait_live_trace_ready ()
1755 {
1756         local url=$1
1757         local zero_client_match=0
1758
1759         diag "Waiting for live trace at url: $url"
1760         while [ $zero_client_match -eq 0 ]; do
1761                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1762                 sleep 0.5
1763         done
1764         pass "Waiting for live trace at url: $url"
1765 }
1766
1767 function wait_live_viewer_connect ()
1768 {
1769         local url=$1
1770         local one_client_match=0
1771
1772         diag "Waiting for live viewers on url: $url"
1773         while [ $one_client_match -eq 0 ]; do
1774                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1775                 sleep 0.5
1776         done
1777         pass "Waiting for live viewers on url: $url"
1778 }
1779
1780 function validate_metadata_event ()
1781 {
1782         local event_name=$1
1783         local nr_event_id=$2
1784         local trace_path=$3
1785
1786         local metadata_file=$(find $trace_path -name "metadata")
1787         local metadata_path=$(dirname $metadata_file)
1788
1789         which $BABELTRACE_BIN >/dev/null
1790         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1791
1792         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1793
1794         if [ "$count" -ne "$nr_event_id" ]; then
1795                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1796                 diag "$count matching event id found in metadata"
1797         else
1798                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1799         fi
1800
1801 }
1802
1803 function trace_matches ()
1804 {
1805         local event_name=$1
1806         local nr_iter=$2
1807         local trace_path=$3
1808
1809         which $BABELTRACE_BIN >/dev/null
1810         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1811
1812         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1813
1814         if [ "$count" -ne "$nr_iter" ]; then
1815                 fail "Trace match"
1816                 diag "$count matching events found in trace"
1817         else
1818                 pass "Trace match"
1819         fi
1820 }
1821
1822 function trace_match_only()
1823 {
1824         local event_name=$1
1825         local nr_iter=$2
1826         local trace_path=$3
1827
1828         which $BABELTRACE_BIN >/dev/null
1829         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1830
1831         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1832         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1833
1834         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1835                 pass "Trace match with $total event $event_name"
1836         else
1837                 fail "Trace match"
1838                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1839         fi
1840 }
1841
1842 function validate_trace
1843 {
1844         local event_name=$1
1845         local trace_path=$2
1846
1847         which $BABELTRACE_BIN >/dev/null
1848         if [ $? -ne 0 ]; then
1849             skip 0 "Babeltrace binary not found. Skipping trace validation"
1850         fi
1851
1852         OLDIFS=$IFS
1853         IFS=","
1854         for i in $event_name; do
1855                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1856                 if [ "$traced" -ne 0 ]; then
1857                         pass "Validate trace for event $i, $traced events"
1858                 else
1859                         fail "Validate trace for event $i"
1860                         diag "Found $traced occurences of $i"
1861                 fi
1862         done
1863         ret=$?
1864         IFS=$OLDIFS
1865         return $ret
1866 }
1867
1868 function validate_trace_count
1869 {
1870         local event_name=$1
1871         local trace_path=$2
1872         local expected_count=$3
1873
1874         which $BABELTRACE_BIN >/dev/null
1875         if [ $? -ne 0 ]; then
1876             skip 0 "Babeltrace binary not found. Skipping trace validation"
1877         fi
1878
1879         cnt=0
1880         OLDIFS=$IFS
1881         IFS=","
1882         for i in $event_name; do
1883                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1884                 if [ "$traced" -ne 0 ]; then
1885                         pass "Validate trace for event $i, $traced events"
1886                 else
1887                         fail "Validate trace for event $i"
1888                         diag "Found $traced occurences of $i"
1889                 fi
1890                 cnt=$(($cnt + $traced))
1891         done
1892         IFS=$OLDIFS
1893         test $cnt -eq $expected_count
1894         ok $? "Read a total of $cnt events, expected $expected_count"
1895 }
1896
1897 function validate_trace_count_range_incl_min_excl_max
1898 {
1899         local event_name=$1
1900         local trace_path=$2
1901         local expected_min=$3
1902         local expected_max=$4
1903
1904         which $BABELTRACE_BIN >/dev/null
1905         if [ $? -ne 0 ]; then
1906             skip 0 "Babeltrace binary not found. Skipping trace validation"
1907         fi
1908
1909         cnt=0
1910         OLDIFS=$IFS
1911         IFS=","
1912         for i in $event_name; do
1913                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1914                 if [ "$traced" -ge $expected_min ]; then
1915                         pass "Validate trace for event $i, $traced events"
1916                 else
1917                         fail "Validate trace for event $i"
1918                         diag "Found $traced occurences of $i"
1919                 fi
1920                 cnt=$(($cnt + $traced))
1921         done
1922         IFS=$OLDIFS
1923         test $cnt -lt $expected_max
1924         ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
1925 }
1926
1927 function trace_first_line
1928 {
1929         local trace_path=$1
1930
1931         which $BABELTRACE_BIN >/dev/null
1932         if [ $? -ne 0 ]; then
1933             skip 0 "Babeltrace binary not found. Skipping trace validation"
1934         fi
1935
1936         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1937 }
1938
1939 function validate_trace_exp()
1940 {
1941         local event_exp=$1
1942         local trace_path=$2
1943
1944         which $BABELTRACE_BIN >/dev/null
1945         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1946
1947         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
1948         if [ "$traced" -ne 0 ]; then
1949                 pass "Validate trace for expression '${event_exp}', $traced events"
1950         else
1951                 fail "Validate trace for expression '${event_exp}'"
1952                 diag "Found $traced occurences of '${event_exp}'"
1953         fi
1954         ret=$?
1955         return $ret
1956 }
1957
1958 function validate_trace_only_exp()
1959 {
1960         local event_exp=$1
1961         local trace_path=$2
1962
1963         which $BABELTRACE_BIN >/dev/null
1964         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1965
1966         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
1967         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1968
1969         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
1970                 pass "Trace match with $total for expression '${event_exp}'"
1971         else
1972                 fail "Trace match"
1973                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
1974         fi
1975         ret=$?
1976         return $ret
1977 }
1978
1979 function validate_trace_empty()
1980 {
1981         local trace_path=$1
1982
1983         which $BABELTRACE_BIN >/dev/null
1984         if [ $? -ne 0 ]; then
1985             skip 0 "Babeltrace binary not found. Skipping trace validation"
1986         fi
1987
1988         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
1989         ret=$?
1990         if [ $ret -ne 0 ]; then
1991                 fail "Failed to parse trace"
1992                 return $ret
1993         fi
1994
1995         traced=$(echo -n "$events" | wc -l)
1996         if [ "$traced" -eq 0 ]; then
1997                 pass "Validate empty trace"
1998         else
1999                 fail "Validate empty trace"
2000                 diag "Found $traced events in trace"
2001         fi
2002         ret=$?
2003         return $ret
2004 }
2005
2006 function validate_directory_empty ()
2007 {
2008         local trace_path="$1"
2009
2010         # Do not double quote `$trace_path` below as we want wildcards to be
2011         # expanded.
2012         files="$(ls -A $trace_path)"
2013         ret=$?
2014         if [ $ret -ne 0 ]; then
2015                 fail "Failed to list content of directory \"$trace_path\""
2016                 return $ret
2017         fi
2018
2019         nb_files="$(echo -n "$files" | wc -l)"
2020         ok $nb_files "Directory \"$trace_path\" is empty"
2021 }
2022
2023 function validate_trace_session_ust_empty()
2024 {
2025         validate_directory_empty "$1"/ust
2026 }
2027
2028 function validate_trace_session_kernel_empty()
2029 {
2030         validate_trace_empty "$1"/kernel
2031 }
2032
2033 function regenerate_metadata ()
2034 {
2035         local expected_to_fail=$1
2036         local sess_name=$2
2037
2038         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2039         ret=$?
2040         if [[ $expected_to_fail -eq "1" ]]; then
2041                 test "$ret" -ne "0"
2042                 ok $? "Expected fail on regenerate metadata $sess_name"
2043         else
2044                 ok $ret "Metadata regenerate $sess_name"
2045         fi
2046 }
2047
2048 function regenerate_metadata_ok ()
2049 {
2050         regenerate_metadata 0 "$@"
2051 }
2052
2053 function regenerate_metadata_fail ()
2054 {
2055         regenerate_metadata 1 "$@"
2056 }
2057
2058 function regenerate_statedump ()
2059 {
2060         local expected_to_fail=$1
2061         local sess_name=$2
2062
2063         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2064         ret=$?
2065         if [[ $expected_to_fail -eq "1" ]]; then
2066                 test "$ret" -ne "0"
2067                 ok $? "Expected fail on regenerate statedump $sess_name"
2068         else
2069                 ok $ret "Statedump regenerate $sess_name"
2070         fi
2071 }
2072
2073 function regenerate_statedump_ok ()
2074 {
2075         regenerate_statedump 0 "$@"
2076 }
2077
2078 function regenerate_statedump_fail ()
2079 {
2080         regenerate_statedump 1 "$@"
2081 }
2082
2083 function rotate_session ()
2084 {
2085         local expected_to_fail=$1
2086         local sess_name=$2
2087
2088         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2089         ret=$?
2090         if [[ $expected_to_fail -eq "1" ]]; then
2091                 test "$ret" -ne "0"
2092                 ok $? "Expected fail on rotate session $sess_name"
2093         else
2094                 ok $ret "Rotate session $sess_name"
2095         fi
2096 }
2097
2098 function rotate_session_ok ()
2099 {
2100         rotate_session 0 "$@"
2101 }
2102
2103 function rotate_session_fail ()
2104 {
2105         rotate_session 1 "$@"
2106 }
2107
2108 function destructive_tests_enabled ()
2109 {
2110         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2111                 return 0
2112         else
2113                 return 1
2114         fi
2115 }
2116
2117 function lttng_enable_rotation_timer ()
2118 {
2119         local expected_to_fail=$1
2120         local sess_name=$2
2121         local period=$3
2122
2123         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2124         ret=$?
2125         if [[ $expected_to_fail -eq "1" ]]; then
2126                 test "$ret" -ne "0"
2127                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2128         else
2129                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2130         fi
2131 }
2132
2133 function lttng_enable_rotation_timer_ok ()
2134 {
2135         lttng_enable_rotation_timer 0 $@
2136 }
2137
2138 function lttng_enable_rotation_timer_fail ()
2139 {
2140         lttng_enable_rotation_timer 1 $@
2141 }
2142
2143 function lttng_enable_rotation_size ()
2144 {
2145         local expected_to_fail=$1
2146         local sess_name=$2
2147         local size=$3
2148
2149         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2150         ret=$?
2151         if [[ $expected_to_fail -eq "1" ]]; then
2152                 test "$ret" -ne "0"
2153                 ok $? "Expected fail on rotate session $sess_name"
2154         else
2155                 ok $ret "Rotate session $sess_name"
2156         fi
2157 }
2158
2159 function lttng_enable_rotation_size_ok ()
2160 {
2161         lttng_enable_rotation_size 0 $@
2162 }
2163
2164 function lttng_enable_rotation_size_fail ()
2165 {
2166         lttng_enable_rotation_size 1 $@
2167 }
2168
2169 function lttng_clear_session ()
2170 {
2171         local expected_to_fail=$1
2172         local sess_name=$2
2173
2174         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2175         ret=$?
2176         if [[ $expected_to_fail -eq "1" ]]; then
2177                 test "$ret" -ne "0"
2178                 ok $? "Expected fail on clear session $sess_name"
2179         else
2180                 ok $ret "Clear session $sess_name"
2181         fi
2182 }
2183
2184 function lttng_clear_session_ok ()
2185 {
2186         lttng_clear_session 0 $@
2187 }
2188
2189 function lttng_clear_session_fail ()
2190 {
2191         lttng_clear_session 1 $@
2192 }
2193
2194 function lttng_clear_all ()
2195 {
2196         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2197         ok $? "Clear all lttng sessions"
2198 }
2199
2200 function lttng_add_trigger()
2201 {
2202         local expected_to_fail="$1"
2203         local trigger_name="$2"
2204         shift 2
2205         local args=("$@")
2206
2207         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2208         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2209         ret=$?
2210         if [[ $expected_to_fail -eq "1" ]]; then
2211                 test "$ret" -ne "0"
2212                 ok $? "Add trigger $trigger_name failed as expected"
2213         else
2214                 ok $ret "Add trigger $trigger_name"
2215         fi
2216 }
2217
2218 function lttng_remove_trigger()
2219 {
2220         local expected_to_fail="$1"
2221         local trigger_name="$2"
2222         shift 2
2223
2224         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2225         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2226         ret=$?
2227         if [[ $expected_to_fail -eq "1" ]]; then
2228                 test "$ret" -ne "0"
2229                 ok $? "Remove trigger $trigger_name failed as expected"
2230         else
2231                 ok $ret "Remove trigger $trigger_name"
2232         fi
2233 }
2234
2235 function lttng_add_trigger_ok()
2236 {
2237         lttng_add_trigger 0 "$@"
2238 }
2239
2240 function lttng_add_trigger_fail()
2241 {
2242         lttng_add_trigger 1 "$@"
2243 }
2244
2245 function lttng_remove_trigger_ok()
2246 {
2247         lttng_remove_trigger 0 "$@"
2248 }
2249
2250 function list_triggers_matches_ok ()
2251 {
2252         local tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2253         local tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2254
2255         local test_name="$1"
2256         local expected_stdout_file="$2"
2257
2258         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2259
2260         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2261         ok $? "${test_name}: exit code is 0"
2262
2263         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2264         ok $? "${test_name}: expected stdout"
2265
2266         diff -u /dev/null "${tmp_stderr}"
2267         ok $? "${test_name}: expected stderr"
2268
2269         rm -f "${tmp_stdout}"
2270         rm -f "${tmp_stderr}"
2271 }
2272
2273 function validate_path_pattern ()
2274 {
2275         local message=$1
2276         local pattern=$2
2277         # Base path is only used in error case and is used to list the content
2278         # of the base path.
2279         local base_path=$3
2280
2281
2282         [ -f $pattern ]
2283         ret=$?
2284         ok $ret "$message"
2285
2286         if [ "$ret" -ne "0" ]; then
2287                 diag "Path pattern expected: $pattern"
2288                 # List the tracepath for more info. We use find as a recursive
2289                 # directory lister.
2290                 diag "The base path content:"
2291                 find "$base_path" -print
2292         fi
2293 }
2294
2295 function validate_trace_path_ust_uid ()
2296 {
2297         local trace_path=$1
2298         local session_name=$2
2299         local uid=$UID
2300         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2301
2302         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2303 }
2304
2305 function validate_trace_path_ust_uid_network ()
2306 {
2307         local trace_path=$1
2308         local session_name=$2
2309         local base_path=$3
2310         local uid=$UID
2311         local hostname=$HOSTNAME
2312         local pattern
2313         local ret
2314
2315         # If the session was given a network base path (e.g
2316         # 127.0.0.1/my/custom/path on creation, there is no session name
2317         # component to the path on the relayd side. Caller can simply not pass a
2318         # session name for this scenario.
2319         if [ -n "$session_name" ]; then
2320                 session_name="$session_name-$date_time_pattern"
2321                 if [ -n "$base_path" ]; then
2322                         fail "Session name and base path are mutually exclusive"
2323                         return
2324                 fi
2325         fi
2326
2327         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2328
2329         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2330 }
2331
2332 function validate_trace_path_ust_uid_snapshot_network ()
2333 {
2334         local trace_path=$1
2335         local session_name=$2
2336         local snapshot_name=$3
2337         local snapshot_number=$4
2338         local base_path=$5
2339         local hostname=$HOSTNAME
2340         local uid=$UID
2341         local pattern
2342         local ret
2343
2344         # If the session/output was given a network base path (e.g
2345         # 127.0.0.1/my/custom/path on creation, there is no session name
2346         # component to the path on the relayd side. Caller can simply not pass a
2347         # session name for this scenario.
2348         if [ -n "$session_name" ]; then
2349                 session_name="$session_name-$date_time_pattern"
2350                 if [ -n "$base_path" ]; then
2351                         fail "Session name and base path are mutually exclusive"
2352                         return
2353                 fi
2354         fi
2355
2356         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2357
2358         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2359 }
2360
2361 function validate_trace_path_ust_uid_snapshot ()
2362 {
2363         local trace_path=$1
2364         local session_name=$2
2365         local snapshot_name=$3
2366         local snapshot_number=$4
2367         local base_path=$5
2368         local uid=$UID
2369         local pattern
2370         local ret
2371
2372         # If the session/output was given a network base path (e.g
2373         # 127.0.0.1/my/custom/path) on creation, there is no session name
2374         # component to the path on the relayd side. Caller can simply not pass a
2375         # session name for this scenario.
2376         if [ -n "$session_name" ]; then
2377                 session_name="$session_name-$date_time_pattern"
2378                 if [ -n "$base_path" ]; then
2379                         fail "Session name and base path are mutually exclusive"
2380                         return
2381                 fi
2382         fi
2383
2384         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2385
2386         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2387 }
2388
2389 function validate_trace_path_ust_pid ()
2390 {
2391         local trace_path=$1
2392         local session_name=$2
2393         local app_string=$3
2394         local pid=$4
2395         local pattern
2396         local ret
2397
2398         # If the session was given a trace path on creation, there is no session
2399         # name component to the path. Caller can simply not pass a session name
2400         # for this scenario.
2401         if [ -n "$session_name" ]; then
2402                 session_name="$session_name-$date_time_pattern"
2403         fi
2404
2405         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2406
2407         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2408 }
2409
2410 function validate_trace_path_kernel ()
2411 {
2412         local trace_path=$1
2413         local session_name=$2
2414         local pattern
2415
2416         # If the session was given a trace path on creation, there is no session
2417         # name component to the path. Caller can simply not pass a session name
2418         # for this scenario.
2419         if [ -n "$session_name" ]; then
2420                 session_name="$session_name-$date_time_pattern"
2421         fi
2422
2423         pattern="$trace_path/$session_name/kernel/metadata"
2424
2425         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2426 }
2427
2428 function validate_trace_path_kernel_network ()
2429 {
2430         local trace_path=$1
2431         local session_name=$2
2432         local hostname=$HOSTNAME
2433         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2434
2435         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2436 }
2437
2438 function validate_trace_path_kernel_snapshot ()
2439 {
2440         local trace_path=$1
2441         local session_name=$2
2442         local snapshot_name=$3
2443         local snapshot_number=$4
2444         local base_path=$5
2445         local pattern
2446         local ret
2447
2448         # If the session/output was given a network base path (e.g
2449         # 127.0.0.1/my/custom/path on creation, there is no session name
2450         # component to the path on the relayd side. Caller can simply not pass a
2451         # session name for this scenario.
2452         if [ -n "$session_name" ]; then
2453                 session_name="$session_name-$date_time_pattern"
2454                 if [ -n "$base_path" ]; then
2455                         fail "Session name and base path are mutually exclusive"
2456                         return
2457                 fi
2458         fi
2459
2460         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2461
2462         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2463 }
2464
2465 function validate_trace_path_kernel_snapshot_network ()
2466 {
2467         local trace_path=$1
2468         local session_name=$2
2469         local snapshot_name=$3
2470         local snapshot_number=$4
2471         local base_path=$5
2472         local hostname=$HOSTNAME
2473         local pattern
2474         local ret
2475
2476         # If the session/output was given a network base path (e.g
2477         # 127.0.0.1/my/custom/path on creation, there is no session name
2478         # component to the path on the relayd side. Caller can simply not pass a
2479         # session name for this scenario.
2480         if [ -n "$session_name" ]; then
2481                 session_name="$session_name-$date_time_pattern"
2482                 if [ -n "$base_path" ]; then
2483                         fail "Session name and base path are mutually exclusive"
2484                         return
2485                 fi
2486         fi
2487
2488         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2489
2490         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2491 }