tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18
  19 # To match 20201127-175802
  20 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  21 # The size of a long on this system
  22 system_long_bit_size=$(getconf LONG_BIT)
  23
  24 # Minimal kernel version supported for session daemon tests
  25 KERNEL_MAJOR_VERSION=2
  26 KERNEL_MINOR_VERSION=6
  27 KERNEL_PATCHLEVEL_VERSION=27
  28
  29 # We set the default UST register timeout and network and app socket timeout to
  30 # "wait forever", so that basic tests don't have to worry about hitting
  31 # timeouts on busy systems. Specialized tests should test those corner-cases.
  32 export LTTNG_UST_REGISTER_TIMEOUT=-1
  33 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  34 export LTTNG_APP_SOCKET_TIMEOUT=-1
  35
  36 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  37 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  38 # its own sessiond if none is running. It also ensures that 'lttng create'
  39 # fails when no sessiond is running.
  40 export LTTNG_SESSIOND_PATH="/bin/true"
  41
  42 source $TESTDIR/utils/tap/tap.sh
  43
  44 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  45         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  46 fi
  47
  48 function full_cleanup ()
  49 {
  50         # Try to kill daemons gracefully
  51         stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  52         stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  53
  54         # If daemons are still present, forcibly kill them
  55         stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  56         stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  57         stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  58
  59         # Disable trap for SIGTERM since the following kill to the
  60         # pidgroup will be SIGTERM. Otherwise it loops.
  61         # The '-' before the pid number ($$) indicates 'kill' to signal the
  62         # whole process group.
  63         trap - SIGTERM && kill -- -$$
  64         exit 1
  65 }
  66
  67 function null_pipes ()
  68 {
  69         exec 0>/dev/null
  70         exec 1>/dev/null
  71         exec 2>/dev/null
  72 }
  73
  74 trap full_cleanup SIGINT SIGTERM
  75
  76 # perl prove closes its child pipes before giving it a chance to run its
  77 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
  78 # to allow those trap handlers to proceed.
  79
  80 trap null_pipes SIGPIPE
  81
  82 function print_ok ()
  83 {
  84         # Check if we are a terminal
  85         if [ -t 1 ]; then
  86                 echo -e "\e[1;32mOK\e[0m"
  87         else
  88                 echo -e "OK"
  89         fi
  90 }
  91
  92 function print_fail ()
  93 {
  94         # Check if we are a terminal
  95         if [ -t 1 ]; then
  96                 echo -e "\e[1;31mFAIL\e[0m"
  97         else
  98                 echo -e "FAIL"
  99         fi
 100 }
 101
 102 function print_test_banner ()
 103 {
 104         local desc="$1"
 105         diag "$desc"
 106 }
 107
 108 function validate_kernel_version ()
 109 {
 110         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 111         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 112                 return 0
 113         fi
 114         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 115                 return 0
 116         fi
 117         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 118                 return 0
 119         fi
 120         return 1
 121 }
 122
 123 # Generate a random string
 124 #  $1 = number of characters; defaults to 16
 125 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 126 function randstring()
 127 {
 128         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 129         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 130         echo
 131 }
 132
 133 # Return a space-separated string of online CPU IDs, based on
 134 # /sys/devices/system/cpu/online, or from 0 to nproc - 1 otherwise.
 135 function get_online_cpus()
 136 {
 137         local cpus=()
 138         local range_re
 139         if [ -f /sys/devices/system/cpu/online ]; then
 140                 range_re='([0-9]+)-([0-9]+)'
 141                 while read -r range ; do
 142                         if [[ "${range}" =~ ${range_re} ]] ; then
 143                                 mapfile -t -O "${#cpus[*]}" cpus <<< $(seq "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}")
 144                         else
 145                                 cpus+=("${range}")
 146                         fi
 147                 done < <(tr ',' $'\n' < /sys/devices/system/cpu/online)
 148         else
 149                 read -r -a cpus <<< $(seq 0 $(( $(conf_proc_count) - 1 )) )
 150         fi
 151         echo "${cpus[*]}"
 152 }
 153
 154 # Helpers for get_possible_cpus.
 155 function get_possible_cpus_count_from_sysfs_possible_mask()
 156 {
 157         local max_possible_cpu_id
 158
 159         # The Awk script extracts the highest CPU id from the possible CPU
 160         # mask. Assuming a numerical order, a field separator '-' and a record
 161         # separator ','. The last value parsed is the highest id.
 162         if [ -f /sys/devices/system/cpu/possible ]; then
 163                 max_possible_cpu_id=$(awk -F '-' 'BEGIN { RS = ","} { last = $NF } END { printf("%d\n", last) }' \
 164                                       /sys/devices/system/cpu/possible)
 165                 echo "$((max_possible_cpu_id+1))"
 166         else
 167                 echo "0"
 168         fi
 169 }
 170
 171 # This is a fallback if the possible CPU mask is not available. This will not
 172 # take into account unplugged CPUs.
 173 function get_max_cpus_count_from_sysfs_cpu_directories()
 174 {
 175         local max_possible_cpu_id=0
 176         local current_cpu_id
 177
 178         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 179                 current_cpu_id="${i#/sys/devices/system/cpu/cpu}"
 180                 if [ "$current_cpu_id" -gt "$max_possible_cpu_id" ]; then
 181                         max_possible_cpu_id="$current_cpu_id"
 182                 fi
 183         done
 184
 185         echo "$((max_possible_cpu_id+1))"
 186 }
 187
 188 # Return the number of possible CPUs.
 189 function get_possible_cpus_count()
 190 {
 191         local possible_cpus_count
 192         possible_cpus_count=$(get_possible_cpus_count_from_sysfs_possible_mask)
 193
 194         if [ "$possible_cpus_count" -eq "0" ]; then
 195                 local configured_cpus_count
 196                 configured_cpus_count=$(getconf _NPROCESSORS_CONF)
 197                 possible_cpus_count=$(get_max_cpus_count_from_sysfs_cpu_directories)
 198                 possible_cpus_count=$((configured_cpus_count > possible_cpus_count \
 199                                                              ? configured_cpus_count \
 200                                                              : possible_cpus_count))
 201         fi
 202
 203         echo "$possible_cpus_count"
 204 }
 205
 206 # Return the list of exposed CPU.
 207 #
 208 # NOTE! Use it like so:
 209 #
 210 # IFS=" " read -r -a VARIABLE <<< "$(get_exposed_cpus_list)"
 211 function get_exposed_cpus_list()
 212 {
 213         local list=()
 214
 215         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 216                 list+=("${i#/sys/devices/system/cpu/cpu}")
 217         done
 218
 219         echo "${list[@]}"
 220 }
 221
 222 # Return any available CPU found. Do not make assumption about the returned
 223 # value, e.g. that it could be 0.
 224 function get_any_available_cpu()
 225 {
 226         for cpu in $(get_online_cpus); do
 227                 echo "${cpu}"
 228                 break;
 229         done
 230 }
 231
 232 # Return the number of _configured_ CPUs.
 233 function conf_proc_count()
 234 {
 235         getconf _NPROCESSORS_CONF
 236         if [ $? -ne 0 ]; then
 237                 diag "Failed to get the number of configured CPUs"
 238         fi
 239         echo
 240 }
 241
 242 # Check if base lttng-modules are present.
 243 # Bail out on failure
 244 function validate_lttng_modules_present ()
 245 {
 246         # Check for loadable modules.
 247         modprobe -n lttng-tracer 2>/dev/null
 248         if [ $? -eq 0 ]; then
 249                 return 0
 250         fi
 251
 252         # Check for builtin modules.
 253         ls /proc/lttng > /dev/null 2>&1
 254         if [ $? -eq 0 ]; then
 255                 return 0
 256         fi
 257
 258         BAIL_OUT "LTTng modules not detected."
 259 }
 260
 261 # Run the lttng binary.
 262 #
 263 # The first two arguments are stdout and stderr redirect paths, respectively.
 264 # The rest of the arguments are forwarded to the lttng binary
 265 function _run_lttng_cmd
 266 {
 267         local stdout_dest="$1"
 268         local stderr_dest="$2"
 269         shift 2
 270
 271         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 272         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 273 }
 274
 275 function enable_kernel_lttng_event
 276 {
 277         local withtap="$1"
 278         local expected_to_fail="$2"
 279         local sess_name="$3"
 280         local event_name="$4"
 281         local channel_name="$5"
 282
 283         if [ -z "$event_name" ]; then
 284                 # Enable all event if no event name specified
 285                 event_name="-a"
 286         fi
 287
 288         if [ -z "$channel_name" ]; then
 289                 # default channel if none specified
 290                 chan=""
 291         else
 292                 chan="-c $channel_name"
 293         fi
 294
 295         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 296         ret=$?
 297         if [[ $expected_to_fail -eq "1" ]]; then
 298                 test $ret -ne "0"
 299                 ret=$?
 300                 if [ $withtap -eq "1" ]; then
 301                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 302                 fi
 303         else
 304                 if [ $withtap -eq "1" ]; then
 305                         ok $ret "Enable kernel event $event_name for session $sess_name"
 306                 fi
 307         fi
 308 }
 309
 310 function enable_kernel_lttng_event_ok ()
 311 {
 312         enable_kernel_lttng_event 1 0 "$@"
 313 }
 314
 315 function enable_kernel_lttng_event_fail ()
 316 {
 317         enable_kernel_lttng_event 1 1 "$@"
 318 }
 319
 320 function enable_kernel_lttng_event_notap ()
 321 {
 322         enable_kernel_lttng_event 0 0 "$@"
 323 }
 324
 325 # Old interface
 326 function lttng_enable_kernel_event
 327 {
 328         enable_kernel_lttng_event_ok "$@"
 329 }
 330
 331 function lttng_enable_kernel_syscall()
 332 {
 333         local expected_to_fail=$1
 334         local sess_name=$2
 335         local syscall_name=$3
 336         local channel_name=$4
 337
 338         if [ -z $syscall_name ]; then
 339                 # Enable all event if no syscall name specified
 340                 syscall_name="-a"
 341         fi
 342
 343         if [ -z $channel_name ]; then
 344                 # default channel if none specified
 345                 chan=""
 346         else
 347                 chan="-c $channel_name"
 348         fi
 349
 350         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 351         ret=$?
 352         if [[ $expected_to_fail -eq "1" ]]; then
 353                 test $ret -ne "0"
 354                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 355         else
 356                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 357         fi
 358 }
 359
 360 function lttng_enable_kernel_syscall_ok()
 361 {
 362         lttng_enable_kernel_syscall 0 "$@"
 363 }
 364
 365 function lttng_enable_kernel_syscall_fail()
 366 {
 367         lttng_enable_kernel_syscall 1 "$@"
 368 }
 369
 370 function lttng_disable_kernel_syscall()
 371 {
 372         local expected_to_fail=$1
 373         local sess_name=$2
 374         local syscall_name=$3
 375         local channel_name=$4
 376
 377         if [ -z $syscall_name ]; then
 378                 # Enable all event if no syscall name specified
 379                 syscall_name="-a"
 380         fi
 381
 382         if [ -z $channel_name ]; then
 383                 # default channel if none specified
 384                 chan=""
 385         else
 386                 chan="-c $channel_name"
 387         fi
 388
 389         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 390
 391         ret=$?
 392         if [[ $expected_to_fail -eq "1" ]]; then
 393                 test $ret -ne "0"
 394                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 395         else
 396                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 397         fi
 398 }
 399
 400 function lttng_disable_kernel_syscall_ok()
 401 {
 402         lttng_disable_kernel_syscall 0 "$@"
 403 }
 404
 405 function lttng_disable_kernel_syscall_fail()
 406 {
 407         lttng_disable_kernel_syscall 1 "$@"
 408 }
 409
 410 function lttng_enable_kernel_userspace_probe_event ()
 411 {
 412         local expected_to_fail="$1"
 413         local sess_name="$2"
 414         local target="$3"
 415         local event_name="$4"
 416
 417         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 418         ret=$?
 419         if [[ $expected_to_fail -eq "1" ]]; then
 420                 test $ret -ne "0"
 421                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 422         else
 423                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 424         fi
 425 }
 426
 427 function lttng_enable_kernel_userspace_probe_event_fail ()
 428 {
 429         lttng_enable_kernel_userspace_probe_event 1 "$@"
 430 }
 431
 432 function lttng_enable_kernel_userspace_probe_event_ok ()
 433 {
 434         lttng_enable_kernel_userspace_probe_event 0 "$@"
 435 }
 436
 437 function disable_kernel_lttng_userspace_probe_event_ok ()
 438 {
 439         local sess_name="$1"
 440         local event_name="$2"
 441
 442         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 443         ok $? "Disable kernel event $target for session $sess_name"
 444 }
 445 function lttng_enable_kernel_channel()
 446 {
 447         local withtap=$1
 448         local expected_to_fail=$2
 449         local sess_name=$3
 450         local channel_name=$4
 451         local opts="${@:5}"
 452
 453         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 454         ret=$?
 455         if [[ $expected_to_fail -eq "1" ]]; then
 456                 test "$ret" -ne "0"
 457                 ret=$?
 458                 if [ $withtap -eq "1" ]; then
 459                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 460                 fi
 461         else
 462                 if [ $withtap -eq "1" ]; then
 463                         ok $ret "Enable channel $channel_name for session $sess_name"
 464                 fi
 465         fi
 466 }
 467
 468 function lttng_enable_kernel_channel_ok()
 469 {
 470         lttng_enable_kernel_channel 1 0 "$@"
 471 }
 472
 473 function lttng_enable_kernel_channel_fail()
 474 {
 475         lttng_enable_kernel_channel 1 1 "$@"
 476 }
 477
 478 function lttng_enable_kernel_channel_notap()
 479 {
 480         lttng_enable_kernel_channel 0 0 "$@"
 481 }
 482
 483 function enable_kernel_lttng_channel_ok()
 484 {
 485         lttng_enable_kernel_channel 1 0 "$@"
 486 }
 487
 488 function lttng_disable_kernel_channel()
 489 {
 490         local expected_to_fail=$1
 491         local sess_name=$2
 492         local channel_name=$3
 493
 494         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 495         ret=$?
 496         if [[ $expected_to_fail -eq "1" ]]; then
 497                 test "$ret" -ne "0"
 498                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 499         else
 500                 ok $ret "Disable channel $channel_name for session $sess_name"
 501         fi
 502 }
 503
 504 function lttng_disable_kernel_channel_ok()
 505 {
 506         lttng_disable_kernel_channel 0 "$@"
 507 }
 508
 509 function lttng_disable_kernel_channel_fail()
 510 {
 511         lttng_disable_kernel_channel 1 "$@"
 512 }
 513
 514 function start_lttng_relayd_opt()
 515 {
 516         local withtap=$1
 517         local process_mode=$2
 518         local opt=$3
 519
 520         DIR=$(readlink -f "$TESTDIR")
 521
 522         if [ -z $(pgrep $RELAYD_MATCH) ]; then
 523                 # shellcheck disable=SC2086
 524                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 525                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 526                 if [ $? -eq 1 ]; then
 527                         if [ $withtap -eq "1" ]; then
 528                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 529                         fi
 530                         return 1
 531                 else
 532                         if [ $withtap -eq "1" ]; then
 533                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 534                         fi
 535                 fi
 536         else
 537                 pass "Start lttng-relayd (opt: $opt)"
 538         fi
 539 }
 540
 541 function start_lttng_relayd()
 542 {
 543         start_lttng_relayd_opt 1 "-b" "$@"
 544 }
 545
 546 function start_lttng_relayd_notap()
 547 {
 548         start_lttng_relayd_opt 0 "-b" "$@"
 549 }
 550
 551 function stop_lttng_relayd_opt()
 552 {
 553         local withtap=$1
 554         local signal=$2
 555
 556         if [ -z "$signal" ]; then
 557                 signal="SIGTERM"
 558         fi
 559
 560         local timeout_s=$3
 561         local dtimeleft_s=
 562
 563         # Multiply time by 2 to simplify integer arithmetic
 564         if [ -n "$timeout_s" ]; then
 565                 dtimeleft_s=$((timeout_s * 2))
 566         fi
 567
 568         local retval=0
 569         local pids=
 570
 571         pids=$(pgrep "$RELAYD_MATCH")
 572         if [ -z "$pids" ]; then
 573                 if [ "$withtap" -eq "1" ]; then
 574                         pass "No relay daemon to kill"
 575                 fi
 576                 return 0
 577         fi
 578
 579         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 580
 581         # shellcheck disable=SC2086
 582         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 583                 retval=1
 584                 if [ "$withtap" -eq "1" ]; then
 585                         fail "Kill relay daemon"
 586                 fi
 587         else
 588                 out=1
 589                 while [ -n "$out" ]; do
 590                         out=$(pgrep "$RELAYD_MATCH")
 591                         if [ -n "$dtimeleft_s" ]; then
 592                                 if [ $dtimeleft_s -lt 0 ]; then
 593                                         out=
 594                                         retval=1
 595                                 fi
 596                                 dtimeleft_s=$((dtimeleft_s - 1))
 597                         fi
 598                         sleep 0.5
 599                 done
 600                 if [ "$withtap" -eq "1" ]; then
 601                         if [ "$retval" -eq "0" ]; then
 602                                 pass "Wait after kill relay daemon"
 603                         else
 604                                 fail "Wait after kill relay daemon"
 605                         fi
 606                 fi
 607         fi
 608         return $retval
 609 }
 610
 611 function stop_lttng_relayd()
 612 {
 613         stop_lttng_relayd_opt 1 "$@"
 614 }
 615
 616 function stop_lttng_relayd_notap()
 617 {
 618         stop_lttng_relayd_opt 0 "$@"
 619 }
 620
 621 #First arg: show tap output
 622 #Second argument: load path for automatic loading
 623 function start_lttng_sessiond_opt()
 624 {
 625         local withtap=$1
 626         local load_path=$2
 627
 628         local env_vars=""
 629         local consumerd=""
 630
 631         local long_bit_value=
 632         long_bit_value=$(getconf LONG_BIT)
 633
 634         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 635                 # Env variable requested no session daemon
 636                 return
 637         fi
 638
 639         DIR=$(readlink -f "$TESTDIR")
 640
 641         # Get long_bit value for 32/64 consumerd
 642         case "$long_bit_value" in
 643                 32)
 644                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 645                         ;;
 646                 64)
 647                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 648                         ;;
 649                 *)
 650                         return
 651                         ;;
 652         esac
 653
 654         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 655         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 656                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 657         fi
 658         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 659
 660         if ! validate_kernel_version; then
 661             fail "Start session daemon"
 662             BAIL_OUT "*** Kernel too old for session daemon tests ***"
 663         fi
 664
 665         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 666         export LTTNG_SESSION_CONFIG_XSD_PATH
 667
 668         if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
 669                 # Have a load path ?
 670                 if [ -n "$load_path" ]; then
 671                         # shellcheck disable=SC2086
 672                         env $env_vars --load "$load_path" --background "$consumerd"
 673                 else
 674                         # shellcheck disable=SC2086
 675                         env $env_vars --background "$consumerd"
 676                 fi
 677                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 678                 status=$?
 679                 if [ "$withtap" -eq "1" ]; then
 680                         ok $status "Start session daemon"
 681                 fi
 682         fi
 683 }
 684
 685 function start_lttng_sessiond()
 686 {
 687         start_lttng_sessiond_opt 1 "$@"
 688 }
 689
 690 function start_lttng_sessiond_notap()
 691 {
 692         start_lttng_sessiond_opt 0 "$@"
 693 }
 694
 695 function stop_lttng_sessiond_opt()
 696 {
 697         local withtap=$1
 698         local signal=$2
 699
 700         if [ -z "$signal" ]; then
 701                 signal=SIGTERM
 702         fi
 703
 704         local timeout_s=$3
 705         local dtimeleft_s=
 706
 707         # Multiply time by 2 to simplify integer arithmetic
 708         if [ -n "$timeout_s" ]; then
 709                 dtimeleft_s=$((timeout_s * 2))
 710         fi
 711
 712         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 713                 # Env variable requested no session daemon
 714                 return 0
 715         fi
 716
 717         local retval=0
 718
 719         local runas_pids=
 720         runas_pids=$(pgrep "$RUNAS_MATCH")
 721
 722         local pids=
 723         pids=$(pgrep "$SESSIOND_MATCH")
 724
 725         if [ -n "$runas_pids" ]; then
 726                 pids="$pids $runas_pids"
 727         fi
 728
 729         if [ -z "$pids" ]; then
 730                 if [ "$withtap" -eq "1" ]; then
 731                         pass "No session daemon to kill"
 732                 fi
 733                 return 0
 734         fi
 735
 736         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 737
 738         # shellcheck disable=SC2086
 739         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 740                 retval=1
 741                 if [ "$withtap" -eq "1" ]; then
 742                         fail "Kill sessions daemon"
 743                 fi
 744         else
 745                 out=1
 746                 while [ -n "$out" ]; do
 747                         out=$(pgrep "${SESSIOND_MATCH}")
 748                         if [ -n "$dtimeleft_s" ]; then
 749                                 if [ $dtimeleft_s -lt 0 ]; then
 750                                         out=
 751                                         retval=1
 752                                 fi
 753                                 dtimeleft_s=$((dtimeleft_s - 1))
 754                         fi
 755                         sleep 0.5
 756                 done
 757                 out=1
 758                 while [ -n "$out" ]; do
 759                         out=$(pgrep "$CONSUMERD_MATCH")
 760                         if [ -n "$dtimeleft_s" ]; then
 761                                 if [ $dtimeleft_s -lt 0 ]; then
 762                                         out=
 763                                         retval=1
 764                                 fi
 765                                 dtimeleft_s=$((dtimeleft_s - 1))
 766                         fi
 767                         sleep 0.5
 768                 done
 769
 770                 if [ "$withtap" -eq "1" ]; then
 771                         if [ "$retval" -eq "0" ]; then
 772                                 pass "Wait after kill session daemon"
 773                         else
 774                                 fail "Wait after kill session daemon"
 775                         fi
 776                 fi
 777         fi
 778         if [ "$signal" = "SIGKILL" ]; then
 779                 if [ "$(id -u)" -eq "0" ]; then
 780                         local modules=
 781                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 782
 783                         if [ -n "$modules" ]; then
 784                                 diag "Unloading all LTTng modules"
 785                                 modprobe --remove "$modules"
 786                         fi
 787                 fi
 788         fi
 789
 790         return $retval
 791 }
 792
 793 function stop_lttng_sessiond()
 794 {
 795         stop_lttng_sessiond_opt 1 "$@"
 796 }
 797
 798 function stop_lttng_sessiond_notap()
 799 {
 800         stop_lttng_sessiond_opt 0 "$@"
 801 }
 802
 803 function sigstop_lttng_sessiond_opt()
 804 {
 805         local withtap=$1
 806         local signal=SIGSTOP
 807
 808         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 809                 # Env variable requested no session daemon
 810                 return
 811         fi
 812
 813         PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
 814
 815         if [ "$withtap" -eq "1" ]; then
 816                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
 817         fi
 818
 819         # shellcheck disable=SC2086
 820         if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 821                 if [ "$withtap" -eq "1" ]; then
 822                         fail "Sending SIGSTOP to session daemon"
 823                 fi
 824         else
 825                 out=1
 826                 while [ $out -ne 0 ]; do
 827                         pid="$(pgrep "$SESSIOND_MATCH")"
 828
 829                         # Wait until state becomes stopped for session
 830                         # daemon(s).
 831                         out=0
 832                         for sessiond_pid in $pid; do
 833                                 state="$(ps -p "$sessiond_pid" -o state= )"
 834                                 if [[ -n "$state" && "$state" != "T" ]]; then
 835                                         out=1
 836                                 fi
 837                         done
 838                         sleep 0.5
 839                 done
 840                 if [ "$withtap" -eq "1" ]; then
 841                         pass "Sending SIGSTOP to session daemon"
 842                 fi
 843         fi
 844 }
 845
 846 function sigstop_lttng_sessiond()
 847 {
 848         sigstop_lttng_sessiond_opt 1 "$@"
 849 }
 850
 851 function sigstop_lttng_sessiond_notap()
 852 {
 853         sigstop_lttng_sessiond_opt 0 "$@"
 854 }
 855
 856 function stop_lttng_consumerd_opt()
 857 {
 858         local withtap=$1
 859         local signal=$2
 860
 861         if [ -z "$signal" ]; then
 862                 signal=SIGTERM
 863         fi
 864
 865         local timeout_s=$3
 866         local dtimeleft_s=
 867
 868         # Multiply time by 2 to simplify integer arithmetic
 869         if [ -n "$timeout_s" ]; then
 870                 dtimeleft_s=$((timeout_s * 2))
 871         fi
 872
 873         local retval=0
 874
 875         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 876
 877         if [ -z "$PID_CONSUMERD" ]; then
 878                 if [ "$withtap" -eq "1" ]; then
 879                         pass "No consumer daemon to kill"
 880                 fi
 881                 return 0
 882         fi
 883
 884         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 885
 886         # shellcheck disable=SC2086
 887         if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 888                 retval=1
 889                 if [ "$withtap" -eq "1" ]; then
 890                         fail "Kill consumer daemon"
 891                 fi
 892         else
 893                 out=1
 894                 while [ $out -ne 0 ]; do
 895                         pid="$(pgrep "$CONSUMERD_MATCH")"
 896
 897                         # If consumerds are still present check their status.
 898                         # A zombie status qualifies the consumerd as *killed*
 899                         out=0
 900                         for consumer_pid in $pid; do
 901                                 state="$(ps -p "$consumer_pid" -o state= )"
 902                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 903                                         out=1
 904                                 fi
 905                         done
 906                         if [ -n "$dtimeleft_s" ]; then
 907                                 if [ $dtimeleft_s -lt 0 ]; then
 908                                         out=0
 909                                         retval=1
 910                                 fi
 911                                 dtimeleft_s=$((dtimeleft_s - 1))
 912                         fi
 913                         sleep 0.5
 914                 done
 915                 if [ "$withtap" -eq "1" ]; then
 916                         if [ "$retval" -eq "0" ]; then
 917                                 pass "Wait after kill consumer daemon"
 918                         else
 919                                 fail "Wait after kill consumer daemon"
 920                         fi
 921                 fi
 922         fi
 923
 924         return $retval
 925 }
 926
 927 function stop_lttng_consumerd()
 928 {
 929         stop_lttng_consumerd_opt 1 "$@"
 930 }
 931
 932 function stop_lttng_consumerd_notap()
 933 {
 934         stop_lttng_consumerd_opt 0 "$@"
 935 }
 936
 937 function sigstop_lttng_consumerd_opt()
 938 {
 939         local withtap=$1
 940         local signal=SIGSTOP
 941
 942         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 943
 944         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 945
 946         # shellcheck disable=SC2086
 947         kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 948         retval=$?
 949
 950         if [ $retval -eq 1 ]; then
 951                 if [ "$withtap" -eq "1" ]; then
 952                         fail "Sending SIGSTOP to consumer daemon"
 953                 fi
 954                 return 1
 955         else
 956                 out=1
 957                 while [ $out -ne 0 ]; do
 958                         pid="$(pgrep "$CONSUMERD_MATCH")"
 959
 960                         # Wait until state becomes stopped for all
 961                         # consumers.
 962                         out=0
 963                         for consumer_pid in $pid; do
 964                                 state="$(ps -p "$consumer_pid" -o state= )"
 965                                 if [[ -n "$state" && "$state" != "T" ]]; then
 966                                         out=1
 967                                 fi
 968                         done
 969                         sleep 0.5
 970                 done
 971                 if [ "$withtap" -eq "1" ]; then
 972                         pass "Sending SIGSTOP to consumer daemon"
 973                 fi
 974         fi
 975         return $retval
 976 }
 977
 978 function sigstop_lttng_consumerd()
 979 {
 980         sigstop_lttng_consumerd_opt 1 "$@"
 981 }
 982
 983 function sigstop_lttng_consumerd_notap()
 984 {
 985         sigstop_lttng_consumerd_opt 0 "$@"
 986 }
 987
 988 function list_lttng_with_opts ()
 989 {
 990         local opts=$1
 991         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 992         ok $? "Lttng-tool list command with option $opts"
 993 }
 994
 995 function create_lttng_session_no_output ()
 996 {
 997         local sess_name=$1
 998         local opts="${@:2}"
 999
1000         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1001         ok $? "Create session $sess_name in no-output mode"
1002 }
1003
1004 function create_lttng_session_uri () {
1005         local sess_name=$1
1006         local uri=$2
1007         local opts="${@:3}"
1008
1009         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1010         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1011 }
1012
1013 function create_lttng_session ()
1014 {
1015         local withtap=$1
1016         local expected_to_fail=$2
1017         local sess_name=$3
1018         local trace_path=$4
1019         local opt=$5
1020
1021         if [ -z "$trace_path" ]; then
1022                 # Use lttng-sessiond default output.
1023                 trace_path=""
1024         else
1025                 trace_path="-o $trace_path"
1026         fi
1027
1028         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create "$sess_name" $trace_path $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1029         ret=$?
1030         if [ $expected_to_fail -eq "1" ]; then
1031                 test "$ret" -ne "0"
1032                 ret=$?
1033                 if [ $withtap -eq "1" ]; then
1034                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1035                 fi
1036         else
1037                 if [ $withtap -eq "1" ]; then
1038                         ok $ret "Create session $sess_name in $trace_path"
1039                 fi
1040         fi
1041         return $ret
1042 }
1043
1044 function create_lttng_session_ok ()
1045 {
1046         create_lttng_session 1 0 "$@"
1047 }
1048
1049 function create_lttng_session_fail ()
1050 {
1051         create_lttng_session 1 1 "$@"
1052 }
1053
1054 function create_lttng_session_notap ()
1055 {
1056         create_lttng_session 0 0 "$@"
1057 }
1058
1059
1060 function enable_ust_lttng_channel ()
1061 {
1062         local withtap=$1
1063         local expected_to_fail=$2
1064         local sess_name=$3
1065         local channel_name=$4
1066         local opts="${@:5}"
1067
1068         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1069         ret=$?
1070         if [[ $expected_to_fail -eq "1" ]]; then
1071                 test "$ret" -ne "0"
1072                 ret=$?
1073                 if [ $withtap -eq "1" ]; then
1074                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1075                 fi
1076         else
1077                 if [ $withtap -eq "1" ]; then
1078                         ok $ret "Enable channel $channel_name for session $sess_name"
1079                 fi
1080         fi
1081         return $ret
1082 }
1083
1084 function enable_ust_lttng_channel_ok ()
1085 {
1086         enable_ust_lttng_channel 1 0 "$@"
1087 }
1088
1089 function enable_ust_lttng_channel_fail ()
1090 {
1091         enable_ust_lttng_channel 1 1 "$@"
1092 }
1093
1094 function enable_ust_lttng_channel_notap ()
1095 {
1096         enable_ust_lttng_channel 0 0 "$@"
1097 }
1098
1099 function disable_ust_lttng_channel()
1100 {
1101         local sess_name=$1
1102         local channel_name=$2
1103
1104         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1105         ok $? "Disable channel $channel_name for session $sess_name"
1106 }
1107
1108 function enable_lttng_mmap_overwrite_kernel_channel()
1109 {
1110         local sess_name=$1
1111         local channel_name=$2
1112
1113         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1114         ok $? "Enable channel $channel_name for session $sess_name"
1115 }
1116
1117 function enable_lttng_mmap_discard_small_kernel_channel()
1118 {
1119         local sess_name=$1
1120         local channel_name=$2
1121
1122         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1123         ok $? "Enable small discard channel $channel_name for session $sess_name"
1124 }
1125
1126 function enable_lttng_mmap_overwrite_small_kernel_channel()
1127 {
1128         local sess_name=$1
1129         local channel_name=$2
1130
1131         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1132         ok $? "Enable small discard channel $channel_name for session $sess_name"
1133 }
1134
1135 function enable_lttng_mmap_overwrite_ust_channel()
1136 {
1137         local sess_name=$1
1138         local channel_name=$2
1139
1140         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1141         ok $? "Enable channel $channel_name for session $sess_name"
1142 }
1143
1144 function enable_ust_lttng_event ()
1145 {
1146         local withtap=$1
1147         local expected_to_fail=$2
1148         local sess_name=$3
1149         local event_name="$4"
1150         local channel_name=$5
1151
1152         if [ -z $channel_name ]; then
1153                 # default channel if none specified
1154                 chan=""
1155         else
1156                 chan="-c $channel_name"
1157         fi
1158
1159         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1160         ret=$?
1161         if [[ $expected_to_fail -eq "1" ]]; then
1162                 test $ret -ne "0"
1163                 ret=$?
1164                 if [[ $withtap -eq "1" ]]; then
1165                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1166                 fi
1167         else
1168                 if [[ $withtap -eq "1" ]]; then
1169                         ok $ret "Enable ust event $event_name for session $sess_name"
1170                 fi
1171         fi
1172         return $ret
1173 }
1174
1175 function enable_ust_lttng_event_ok ()
1176 {
1177         enable_ust_lttng_event 1 0 "$@"
1178 }
1179
1180 function enable_ust_lttng_event_fail ()
1181 {
1182         enable_ust_lttng_event 1 1 "$@"
1183 }
1184
1185 function enable_ust_lttng_event_notap ()
1186 {
1187         enable_ust_lttng_event 0 0 "$@"
1188 }
1189
1190 function enable_jul_lttng_event()
1191 {
1192         sess_name=$1
1193         event_name="$2"
1194         channel_name=$3
1195
1196         if [ -z $channel_name ]; then
1197                 # default channel if none specified
1198                 chan=""
1199         else
1200                 chan="-c $channel_name"
1201         fi
1202
1203         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1204         ok $? "Enable JUL event $event_name for session $sess_name"
1205 }
1206
1207 function enable_jul_lttng_event_loglevel()
1208 {
1209         local sess_name=$1
1210         local event_name="$2"
1211         local loglevel=$3
1212         local channel_name=$4
1213
1214         if [ -z $channel_name ]; then
1215                 # default channel if none specified
1216                 chan=""
1217         else
1218                 chan="-c $channel_name"
1219         fi
1220
1221         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1222         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1223 }
1224
1225 function enable_log4j_lttng_event()
1226 {
1227         local sess_name=$1
1228         local event_name=$2
1229         local channel_name=$3
1230
1231         local chan_opt=()
1232
1233         # default channel if none specified
1234         if [ -n "$channel_name" ]; then
1235                 chan_opt=("-c" "$channel_name")
1236         fi
1237
1238         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1239                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1240         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1241 }
1242
1243 function enable_log4j_lttng_event_filter()
1244 {
1245         local sess_name=$1
1246         local event_name=$2
1247         local filter=$3
1248
1249         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1250                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1251         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1252 }
1253
1254 function enable_log4j_lttng_event_filter_loglevel_only()
1255 {
1256         local sess_name=$1
1257         local event_name=$2
1258         local filter=$3
1259         local loglevel=$4
1260
1261         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1262                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1263         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1264 }
1265
1266 function enable_log4j_lttng_event_loglevel()
1267 {
1268         local sess_name=$1
1269         local event_name=$2
1270         local loglevel=$3
1271         local channel_name=$4
1272
1273
1274         # default channel if none specified
1275         if [ -n "$channel_name" ]; then
1276                 chan_opt=("-c" "$channel_name")
1277         fi
1278
1279         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1280                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1281         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1282 }
1283
1284 function enable_log4j_lttng_event_loglevel_only()
1285 {
1286         local sess_name=$1
1287         local event_name=$2
1288         local loglevel=$3
1289         local channel_name=$4
1290
1291         local chan_opt=()
1292
1293         # default channel if none specified
1294         if [ -n "$channel_name" ]; then
1295                 chan_opt=("-c" "$channel_name")
1296         fi
1297
1298         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1299                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1300         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1301 }
1302
1303 function enable_python_lttng_event()
1304 {
1305         sess_name=$1
1306         event_name="$2"
1307         channel_name=$3
1308
1309         if [ -z $channel_name ]; then
1310                 # default channel if none specified
1311                 chan=""
1312         else
1313                 chan="-c $channel_name"
1314         fi
1315
1316         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1317         ok $? "Enable Python event $event_name for session $sess_name"
1318 }
1319
1320 function enable_python_lttng_event_loglevel()
1321 {
1322         local sess_name=$1
1323         local event_name="$2"
1324         local loglevel=$3
1325         local channel_name=$4
1326
1327         if [ -z $channel_name ]; then
1328                 # default channel if none specified
1329                 chan=""
1330         else
1331                 chan="-c $channel_name"
1332         fi
1333
1334         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1335         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1336 }
1337
1338 function enable_ust_lttng_event_filter()
1339 {
1340         local sess_name="$1"
1341         local event_name="$2"
1342         local filter="$3"
1343         local channel_name=$4
1344
1345         if [ -z $channel_name ]; then
1346                 # default channel if none specified
1347                 chan=""
1348         else
1349                 chan="-c $channel_name"
1350         fi
1351
1352         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event $chan "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1353         ok $? "Enable event $event_name with filtering for session $sess_name"
1354 }
1355
1356 function enable_ust_lttng_event_loglevel()
1357 {
1358         local sess_name="$1"
1359         local event_name="$2"
1360         local loglevel="$3"
1361
1362         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1363         ok $? "Enable event $event_name with loglevel $loglevel"
1364 }
1365
1366 function enable_ust_lttng_event_loglevel_only()
1367 {
1368         local sess_name="$1"
1369         local event_name="$2"
1370         local loglevel="$3"
1371
1372         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1373         ok $? "Enable event $event_name with loglevel-only $loglevel"
1374 }
1375
1376 function disable_ust_lttng_event ()
1377 {
1378         local sess_name="$1"
1379         local event_name="$2"
1380         local channel_name="$3"
1381
1382         if [ -z $channel_name ]; then
1383                 # default channel if none specified
1384                 chan=""
1385         else
1386                 chan="-c $channel_name"
1387         fi
1388
1389         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1390         ok $? "Disable event $event_name for session $sess_name"
1391 }
1392
1393 function disable_jul_lttng_event ()
1394 {
1395         local sess_name="$1"
1396         local event_name="$2"
1397
1398         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1399         ok $? "Disable JUL event $event_name for session $sess_name"
1400 }
1401
1402 function disable_log4j_lttng_event ()
1403 {
1404         local sess_name="$1"
1405         local event_name="$2"
1406
1407         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1408                 disable-event "$event_name" -s "$sess_name" --log4j
1409         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1410 }
1411
1412 function disable_python_lttng_event ()
1413 {
1414         local sess_name="$1"
1415         local event_name="$2"
1416
1417         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1418         ok $? "Disable Python event $event_name for session $sess_name"
1419 }
1420
1421 function start_lttng_tracing_opt ()
1422 {
1423         local withtap=$1
1424         local expected_to_fail=$2
1425         local sess_name=$3
1426
1427         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1428         ret=$?
1429         if [[ $expected_to_fail -eq "1" ]]; then
1430                 test "$ret" -ne "0"
1431                 ret=$?
1432                 if [ $withtap -eq "1" ]; then
1433                         ok $? "Start tracing for session $sess_name failed as expected"
1434                 fi
1435         else
1436                 if [ $withtap -eq "1" ]; then
1437                         ok $ret "Start tracing for session $sess_name"
1438                 fi
1439         fi
1440 }
1441
1442 function start_lttng_tracing_ok ()
1443 {
1444         start_lttng_tracing_opt 1 0 "$@"
1445 }
1446
1447 function start_lttng_tracing_fail ()
1448 {
1449         start_lttng_tracing_opt 1 1 "$@"
1450 }
1451
1452 function start_lttng_tracing_notap ()
1453 {
1454         start_lttng_tracing_opt 0 1 "$@"
1455 }
1456
1457 function stop_lttng_tracing_opt ()
1458 {
1459         local withtap=$1
1460         local expected_to_fail=$2
1461         local sess_name=$3
1462
1463         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1464         ret=$?
1465         if [[ $expected_to_fail -eq "1" ]]; then
1466                 test "$ret" -ne "0"
1467                 ret=$?
1468                 if [ $withtap -eq "1" ]; then
1469                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1470                 fi
1471         else
1472                 if [ $withtap -eq "1" ]; then
1473                         ok $ret "Stop lttng tracing for session $sess_name"
1474                 fi
1475         fi
1476 }
1477
1478 function stop_lttng_tracing_ok ()
1479 {
1480         stop_lttng_tracing_opt 1 0 "$@"
1481 }
1482
1483 function stop_lttng_tracing_fail ()
1484 {
1485         stop_lttng_tracing_opt 1 1 "$@"
1486 }
1487
1488 function stop_lttng_tracing_notap ()
1489 {
1490         stop_lttng_tracing_opt 0 0 "$@"
1491 }
1492
1493 function destroy_lttng_session ()
1494 {
1495         local withtap=$1
1496         local expected_to_fail=$2
1497         local sess_name=$3
1498
1499         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1500         ret=$?
1501         if [[ $expected_to_fail -eq "1" ]]; then
1502                 test "$ret" -ne "0"
1503                 ret=$?
1504                 if [ $withtap -eq "1" ]; then
1505                         ok $ret "Destroy session $sess_name failed as expected"
1506                 fi
1507         else
1508                 if [ $withtap -eq "1" ]; then
1509                         ok $ret "Destroy session $sess_name"
1510                 fi
1511         fi
1512 }
1513
1514 function destroy_lttng_session_ok ()
1515 {
1516         destroy_lttng_session 1 0 "$@"
1517
1518 }
1519
1520 function destroy_lttng_session_fail ()
1521 {
1522         destroy_lttng_session 1 1 "$@"
1523 }
1524
1525 function destroy_lttng_session_notap ()
1526 {
1527         destroy_lttng_session 0 0 "$@"
1528 }
1529
1530 function destroy_lttng_sessions ()
1531 {
1532         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1533         ok $? "Destroy all lttng sessions"
1534 }
1535
1536 function lttng_snapshot_add_output ()
1537 {
1538         local expected_to_fail=$1
1539         local sess_name=$2
1540         local trace_path=$3
1541         local opts=$4
1542
1543         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1544         ret=$?
1545         if [[ $expected_to_fail -eq 1 ]]; then
1546                 test "$ret" -ne "0"
1547                 ok $? "Added snapshot output $trace_path failed as expected"
1548         else
1549                 ok $ret "Added snapshot output $trace_path"
1550         fi
1551 }
1552
1553 function lttng_snapshot_add_output_ok ()
1554 {
1555         lttng_snapshot_add_output 0 "$@"
1556 }
1557
1558 function lttng_snapshot_add_output_fail ()
1559 {
1560         lttng_snapshot_add_output 1 "$@"
1561 }
1562
1563 function lttng_snapshot_del_output ()
1564 {
1565         local expected_to_fail=$1
1566         local sess_name=$2
1567         local id=$3
1568
1569         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1570         ret=$?
1571         if [[ $expected_to_fail -eq "1" ]]; then
1572                 test "$ret" -ne "0"
1573                 ok $? "Deleted snapshot output id $id failed as expected"
1574         else
1575                 ok $ret "Deleted snapshot output id $id"
1576         fi
1577 }
1578
1579 function lttng_snapshot_del_output_ok ()
1580 {
1581         lttng_snapshot_del_output 0 "$@"
1582 }
1583
1584 function lttng_snapshot_del_output_fail ()
1585 {
1586         lttng_snapshot_del_output 1 "$@"
1587 }
1588
1589 function lttng_snapshot_record ()
1590 {
1591         local sess_name=$1
1592         local trace_path=$2
1593
1594         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s "$sess_name" "$trace_path" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1595         ok $? "Snapshot recorded"
1596 }
1597
1598 function lttng_snapshot_list ()
1599 {
1600         local sess_name=$1
1601         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1602         ok $? "Snapshot list"
1603 }
1604
1605 function lttng_save()
1606 {
1607         local sess_name=$1
1608         local opts=$2
1609
1610         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1611         ok $? "Session saved"
1612 }
1613
1614 function lttng_load()
1615 {
1616         local expected_to_fail=$1
1617         local opts=$2
1618
1619         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1620         ret=$?
1621         if [[ $expected_to_fail -eq "1" ]]; then
1622                 test $ret -ne "0"
1623                 ok $? "Load command failed as expected with opts: $opts"
1624         else
1625                 ok $ret "Load command with opts: $opts"
1626         fi
1627 }
1628
1629 function lttng_load_ok()
1630 {
1631         lttng_load 0 "$@"
1632 }
1633
1634 function lttng_load_fail()
1635 {
1636         lttng_load 1 "$@"
1637 }
1638
1639 function lttng_track()
1640 {
1641         local expected_to_fail="$1"
1642         shift 1
1643         local opts="$@"
1644         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1645         ret=$?
1646         if [[ $expected_to_fail -eq "1" ]]; then
1647                 test $ret -ne "0"
1648                 ok $? "Track command failed as expected with opts: $opts"
1649         else
1650                 ok $ret "Track command with opts: $opts"
1651         fi
1652 }
1653
1654 function lttng_track_ok()
1655 {
1656         lttng_track 0 "$@"
1657 }
1658
1659 function lttng_track_fail()
1660 {
1661         lttng_track 1 "$@"
1662 }
1663
1664 function lttng_untrack()
1665 {
1666         local expected_to_fail="$1"
1667         shift 1
1668         local opts="$@"
1669         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1670         ret=$?
1671         if [[ $expected_to_fail -eq "1" ]]; then
1672                 test $ret -ne "0"
1673                 ok $? "Untrack command failed as expected with opts: $opts"
1674         else
1675                 ok $ret "Untrack command with opts: $opts"
1676         fi
1677 }
1678
1679 function lttng_untrack_ok()
1680 {
1681         lttng_untrack 0 "$@"
1682 }
1683
1684 function lttng_untrack_fail()
1685 {
1686         lttng_untrack 1 "$@"
1687 }
1688
1689 function lttng_track_pid_ok()
1690 {
1691         PID=$1
1692         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1693         ok $? "Lttng track pid on the kernel domain"
1694 }
1695
1696 function lttng_untrack_kernel_all_ok()
1697 {
1698         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1699         ok $? "Lttng untrack all pid on the kernel domain"
1700 }
1701
1702 function lttng_track_ust_ok()
1703 {
1704         lttng_track_ok -u "$@"
1705 }
1706
1707 function lttng_track_ust_fail()
1708 {
1709         lttng_track_fail -u "$@"
1710 }
1711
1712 function lttng_track_kernel_ok()
1713 {
1714         lttng_track_ok -k "$@"
1715 }
1716
1717 function lttng_track_kernel_fail()
1718 {
1719         lttng_track_fail -k "$@"
1720 }
1721
1722 function lttng_untrack_ust_ok()
1723 {
1724         lttng_untrack_ok -u "$@"
1725 }
1726
1727 function lttng_untrack_ust_fail()
1728 {
1729         lttng_untrack_fail -u "$@"
1730 }
1731
1732 function lttng_untrack_kernel_ok()
1733 {
1734         lttng_untrack_ok -k "$@"
1735 }
1736
1737 function lttng_untrack_kernel_fail()
1738 {
1739         lttng_untrack_fail -k "$@"
1740 }
1741
1742 function lttng_add_context_list()
1743 {
1744         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1745         ret=$?
1746         ok $ret "Context listing"
1747 }
1748
1749 function add_context_lttng()
1750 {
1751         local expected_to_fail="$1"
1752         local domain="$2"
1753         local session_name="$3"
1754         local channel_name="$4"
1755         local type="$5"
1756
1757         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1758         ret=$?
1759         if [[ $expected_to_fail -eq "1" ]]; then
1760                 test $ret -ne "0"
1761                 ok $? "Add context command failed as expected for type: $type"
1762         else
1763                 ok $ret "Add context command for type: $type"
1764         fi
1765 }
1766
1767 function add_context_ust_ok()
1768 {
1769         add_context_lttng 0 -u "$@"
1770 }
1771
1772 function add_context_ust_fail()
1773 {
1774         add_context_lttng 1 -u "$@"
1775 }
1776
1777 function add_context_kernel_ok()
1778 {
1779         add_context_lttng 0 -k "$@"
1780 }
1781
1782 function add_context_kernel_fail()
1783 {
1784         add_context_lttng 1 -k "$@"
1785 }
1786
1787 function wait_live_trace_ready ()
1788 {
1789         local url=$1
1790         local zero_client_match=0
1791
1792         diag "Waiting for live trace at url: $url"
1793         while [ $zero_client_match -eq 0 ]; do
1794                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1795                 sleep 0.5
1796         done
1797         pass "Waiting for live trace at url: $url"
1798 }
1799
1800 function wait_live_viewer_connect ()
1801 {
1802         local url=$1
1803         local one_client_match=0
1804
1805         diag "Waiting for live viewers on url: $url"
1806         while [ $one_client_match -eq 0 ]; do
1807                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1808                 sleep 0.5
1809         done
1810         pass "Waiting for live viewers on url: $url"
1811 }
1812
1813 function bail_out_if_no_babeltrace()
1814 {
1815         which "$BABELTRACE_BIN" >/dev/null
1816         if [ $? -ne 0 ]; then
1817                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
1818         fi
1819 }
1820
1821 function validate_metadata_event ()
1822 {
1823         local event_name=$1
1824         local nr_event_id=$2
1825         local trace_path=$3
1826
1827         local metadata_file=$(find $trace_path | grep metadata)
1828         local metadata_path=$(dirname $metadata_file)
1829
1830         which $BABELTRACE_BIN >/dev/null
1831         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1832
1833         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1834
1835         if [ "$count" -ne "$nr_event_id" ]; then
1836                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1837                 diag "$count matching event id found in metadata"
1838         else
1839                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1840         fi
1841
1842 }
1843
1844 function trace_matches ()
1845 {
1846         local event_name=$1
1847         local nr_iter=$2
1848         local trace_path=$3
1849
1850         which $BABELTRACE_BIN >/dev/null
1851         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1852
1853         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1854
1855         if [ "$count" -ne "$nr_iter" ]; then
1856                 fail "Trace match"
1857                 diag "$count matching events found in trace"
1858         else
1859                 pass "Trace match"
1860         fi
1861 }
1862
1863 function trace_match_only()
1864 {
1865         local event_name=$1
1866         local nr_iter=$2
1867         local trace_path=$3
1868
1869         which $BABELTRACE_BIN >/dev/null
1870         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1871
1872         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1873         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1874
1875         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1876                 pass "Trace match with $total event $event_name"
1877         else
1878                 fail "Trace match"
1879                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1880         fi
1881 }
1882
1883 function validate_trace
1884 {
1885         local event_name=$1
1886         local trace_path=$2
1887
1888         which $BABELTRACE_BIN >/dev/null
1889         if [ $? -ne 0 ]; then
1890             skip 0 "Babeltrace binary not found. Skipping trace validation"
1891         fi
1892
1893         OLDIFS=$IFS
1894         IFS=","
1895         for i in $event_name; do
1896                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1897                 if [ "$traced" -ne 0 ]; then
1898                         pass "Validate trace for event $i, $traced events"
1899                 else
1900                         fail "Validate trace for event $i"
1901                         diag "Found $traced occurences of $i"
1902                 fi
1903         done
1904         ret=$?
1905         IFS=$OLDIFS
1906         return $ret
1907 }
1908
1909 function validate_trace_count
1910 {
1911         local event_name=$1
1912         local trace_path=$2
1913         local expected_count=$3
1914
1915         which $BABELTRACE_BIN >/dev/null
1916         if [ $? -ne 0 ]; then
1917             skip 0 "Babeltrace binary not found. Skipping trace validation"
1918         fi
1919
1920         cnt=0
1921         OLDIFS=$IFS
1922         IFS=","
1923         for i in $event_name; do
1924                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1925                 if [ "$traced" -ne 0 ]; then
1926                         pass "Validate trace for event $i, $traced events"
1927                 else
1928                         fail "Validate trace for event $i"
1929                         diag "Found $traced occurences of $i"
1930                 fi
1931                 cnt=$(($cnt + $traced))
1932         done
1933         IFS=$OLDIFS
1934         test $cnt -eq $expected_count
1935         ok $? "Read a total of $cnt events, expected $expected_count"
1936 }
1937
1938 function validate_trace_count_range_incl_min_excl_max
1939 {
1940         local event_name=$1
1941         local trace_path=$2
1942         local expected_min=$3
1943         local expected_max=$4
1944
1945         which $BABELTRACE_BIN >/dev/null
1946         if [ $? -ne 0 ]; then
1947             skip 0 "Babeltrace binary not found. Skipping trace validation"
1948         fi
1949
1950         cnt=0
1951         OLDIFS=$IFS
1952         IFS=","
1953         for i in $event_name; do
1954                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1955                 if [ "$traced" -ge $expected_min ]; then
1956                         pass "Validate trace for event $i, $traced events"
1957                 else
1958                         fail "Validate trace for event $i"
1959                         diag "Found $traced occurences of $i"
1960                 fi
1961                 cnt=$(($cnt + $traced))
1962         done
1963         IFS=$OLDIFS
1964         test $cnt -lt $expected_max
1965         ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
1966 }
1967
1968 function trace_first_line
1969 {
1970         local trace_path=$1
1971
1972         which $BABELTRACE_BIN >/dev/null
1973         if [ $? -ne 0 ]; then
1974             skip 0 "Babeltrace binary not found. Skipping trace validation"
1975         fi
1976
1977         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1978 }
1979
1980 function validate_trace_exp()
1981 {
1982         local event_exp=$1
1983         local trace_path=$2
1984
1985         which $BABELTRACE_BIN >/dev/null
1986         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1987
1988         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
1989         if [ "$traced" -ne 0 ]; then
1990                 pass "Validate trace for expression '${event_exp}', $traced events"
1991         else
1992                 fail "Validate trace for expression '${event_exp}'"
1993                 diag "Found $traced occurences of '${event_exp}'"
1994         fi
1995         ret=$?
1996         return $ret
1997 }
1998
1999 function validate_trace_only_exp()
2000 {
2001         local event_exp=$1
2002         local trace_path=$2
2003
2004         which $BABELTRACE_BIN >/dev/null
2005         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
2006
2007         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
2008         local total=$($BABELTRACE_BIN $trace_path | wc -l)
2009
2010         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
2011                 pass "Trace match with $total for expression '${event_exp}'"
2012         else
2013                 fail "Trace match"
2014                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
2015         fi
2016         ret=$?
2017         return $ret
2018 }
2019
2020 function validate_trace_empty()
2021 {
2022         local trace_path=$1
2023
2024         which $BABELTRACE_BIN >/dev/null
2025         if [ $? -ne 0 ]; then
2026             skip 0 "Babeltrace binary not found. Skipping trace validation"
2027         fi
2028
2029         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
2030         ret=$?
2031         if [ $ret -ne 0 ]; then
2032                 fail "Failed to parse trace"
2033                 return $ret
2034         fi
2035
2036         traced=$(echo -n "$events" | wc -l)
2037         if [ "$traced" -eq 0 ]; then
2038                 pass "Validate empty trace"
2039         else
2040                 fail "Validate empty trace"
2041                 diag "Found $traced events in trace"
2042         fi
2043         ret=$?
2044         return $ret
2045 }
2046
2047 function validate_directory_empty ()
2048 {
2049         local trace_path="$1"
2050
2051         # Do not double quote `$trace_path` below as we want wildcards to be
2052         # expanded.
2053         files="$(ls -A $trace_path)"
2054         ret=$?
2055         if [ $ret -ne 0 ]; then
2056                 fail "Failed to list content of directory \"$trace_path\""
2057                 return $ret
2058         fi
2059
2060         nb_files="$(echo -n "$files" | wc -l)"
2061         ok $nb_files "Directory \"$trace_path\" is empty"
2062 }
2063
2064 function validate_trace_session_ust_empty()
2065 {
2066         validate_directory_empty "$1"/ust
2067 }
2068
2069 function validate_trace_session_kernel_empty()
2070 {
2071         validate_trace_empty "$1"/kernel
2072 }
2073
2074 function regenerate_metadata ()
2075 {
2076         local expected_to_fail=$1
2077         local sess_name=$2
2078
2079         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2080         ret=$?
2081         if [[ $expected_to_fail -eq "1" ]]; then
2082                 test "$ret" -ne "0"
2083                 ok $? "Expected fail on regenerate metadata $sess_name"
2084         else
2085                 ok $ret "Metadata regenerate $sess_name"
2086         fi
2087 }
2088
2089 function regenerate_metadata_ok ()
2090 {
2091         regenerate_metadata 0 "$@"
2092 }
2093
2094 function regenerate_metadata_fail ()
2095 {
2096         regenerate_metadata 1 "$@"
2097 }
2098
2099 function regenerate_statedump ()
2100 {
2101         local expected_to_fail=$1
2102         local sess_name=$2
2103
2104         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2105         ret=$?
2106         if [[ $expected_to_fail -eq "1" ]]; then
2107                 test "$ret" -ne "0"
2108                 ok $? "Expected fail on regenerate statedump $sess_name"
2109         else
2110                 ok $ret "Statedump regenerate $sess_name"
2111         fi
2112 }
2113
2114 function regenerate_statedump_ok ()
2115 {
2116         regenerate_statedump 0 "$@"
2117 }
2118
2119 function regenerate_statedump_fail ()
2120 {
2121         regenerate_statedump 1 "$@"
2122 }
2123
2124 function rotate_session ()
2125 {
2126         local expected_to_fail=$1
2127         local sess_name=$2
2128
2129         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2130         ret=$?
2131         if [[ $expected_to_fail -eq "1" ]]; then
2132                 test "$ret" -ne "0"
2133                 ok $? "Expected fail on rotate session $sess_name"
2134         else
2135                 ok $ret "Rotate session $sess_name"
2136         fi
2137 }
2138
2139 function rotate_session_ok ()
2140 {
2141         rotate_session 0 "$@"
2142 }
2143
2144 function rotate_session_fail ()
2145 {
2146         rotate_session 1 "$@"
2147 }
2148
2149 function destructive_tests_enabled ()
2150 {
2151         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2152                 return 0
2153         else
2154                 return 1
2155         fi
2156 }
2157
2158 function lttng_enable_rotation_timer ()
2159 {
2160         local expected_to_fail=$1
2161         local sess_name=$2
2162         local period=$3
2163
2164         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2165         ret=$?
2166         if [[ $expected_to_fail -eq "1" ]]; then
2167                 test "$ret" -ne "0"
2168                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2169         else
2170                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2171         fi
2172 }
2173
2174 function lttng_enable_rotation_timer_ok ()
2175 {
2176         lttng_enable_rotation_timer 0 $@
2177 }
2178
2179 function lttng_enable_rotation_timer_fail ()
2180 {
2181         lttng_enable_rotation_timer 1 $@
2182 }
2183
2184 function lttng_enable_rotation_size ()
2185 {
2186         local expected_to_fail=$1
2187         local sess_name=$2
2188         local size=$3
2189
2190         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2191         ret=$?
2192         if [[ $expected_to_fail -eq "1" ]]; then
2193                 test "$ret" -ne "0"
2194                 ok $? "Expected fail on rotate session $sess_name"
2195         else
2196                 ok $ret "Rotate session $sess_name"
2197         fi
2198 }
2199
2200 function lttng_enable_rotation_size_ok ()
2201 {
2202         lttng_enable_rotation_size 0 $@
2203 }
2204
2205 function lttng_enable_rotation_size_fail ()
2206 {
2207         lttng_enable_rotation_size 1 $@
2208 }
2209
2210 function lttng_clear_session ()
2211 {
2212         local expected_to_fail=$1
2213         local sess_name=$2
2214
2215         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2216         ret=$?
2217         if [[ $expected_to_fail -eq "1" ]]; then
2218                 test "$ret" -ne "0"
2219                 ok $? "Expected fail on clear session $sess_name"
2220         else
2221                 ok $ret "Clear session $sess_name"
2222         fi
2223 }
2224
2225 function lttng_clear_session_ok ()
2226 {
2227         lttng_clear_session 0 $@
2228 }
2229
2230 function lttng_clear_session_fail ()
2231 {
2232         lttng_clear_session 1 $@
2233 }
2234
2235 function lttng_clear_all ()
2236 {
2237         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2238         ok $? "Clear all lttng sessions"
2239 }
2240
2241 function validate_path_pattern ()
2242 {
2243         local message=$1
2244         local pattern=$2
2245         # Base path is only used in error case and is used to list the content
2246         # of the base path.
2247         local base_path=$3
2248
2249
2250         [ -f $pattern ]
2251         ret=$?
2252         ok $ret "$message"
2253
2254         if [ "$ret" -ne "0" ]; then
2255                 diag "Path pattern expected: $pattern"
2256                 # List the tracepath for more info. We use find as a recursive
2257                 # directory lister.
2258                 diag "The base path content:"
2259                 find "$base_path" -print
2260         fi
2261 }
2262
2263 function validate_trace_path_ust_uid ()
2264 {
2265         local trace_path=$1
2266         local session_name=$2
2267         local uid=$UID
2268         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2269
2270         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2271 }
2272
2273 function validate_trace_path_ust_uid_network ()
2274 {
2275         local trace_path=$1
2276         local session_name=$2
2277         local base_path=$3
2278         local uid=$UID
2279         local hostname=$HOSTNAME
2280         local pattern
2281         local ret
2282
2283         # If the session was given a network base path (e.g
2284         # 127.0.0.1/my/custom/path on creation, there is no session name
2285         # component to the path on the relayd side. Caller can simply not pass a
2286         # session name for this scenario.
2287         if [ -n "$session_name" ]; then
2288                 session_name="$session_name-$date_time_pattern"
2289                 if [ -n "$base_path" ]; then
2290                         fail "Session name and base path are mutually exclusive"
2291                         return
2292                 fi
2293         fi
2294
2295         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2296
2297         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2298 }
2299
2300 function validate_trace_path_ust_uid_snapshot_network ()
2301 {
2302         local trace_path=$1
2303         local session_name=$2
2304         local snapshot_name=$3
2305         local snapshot_number=$4
2306         local base_path=$5
2307         local hostname=$HOSTNAME
2308         local uid=$UID
2309         local pattern
2310         local ret
2311
2312         # If the session/output was given a network base path (e.g
2313         # 127.0.0.1/my/custom/path on creation, there is no session name
2314         # component to the path on the relayd side. Caller can simply not pass a
2315         # session name for this scenario.
2316         if [ -n "$session_name" ]; then
2317                 session_name="$session_name-$date_time_pattern"
2318                 if [ -n "$base_path" ]; then
2319                         fail "Session name and base path are mutually exclusive"
2320                         return
2321                 fi
2322         fi
2323
2324         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2325
2326         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2327 }
2328
2329 function validate_trace_path_ust_uid_snapshot ()
2330 {
2331         local trace_path=$1
2332         local session_name=$2
2333         local snapshot_name=$3
2334         local snapshot_number=$4
2335         local base_path=$5
2336         local uid=$UID
2337         local pattern
2338         local ret
2339
2340         # If the session/output was given a network base path (e.g
2341         # 127.0.0.1/my/custom/path) on creation, there is no session name
2342         # component to the path on the relayd side. Caller can simply not pass a
2343         # session name for this scenario.
2344         if [ -n "$session_name" ]; then
2345                 session_name="$session_name-$date_time_pattern"
2346                 if [ -n "$base_path" ]; then
2347                         fail "Session name and base path are mutually exclusive"
2348                         return
2349                 fi
2350         fi
2351
2352         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2353
2354         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2355 }
2356
2357 function validate_trace_path_ust_pid ()
2358 {
2359         local trace_path=$1
2360         local session_name=$2
2361         local app_string=$3
2362         local pid=$4
2363         local pattern
2364         local ret
2365
2366         # If the session was given a trace path on creation, there is no session
2367         # name component to the path. Caller can simply not pass a session name
2368         # for this scenario.
2369         if [ -n "$session_name" ]; then
2370                 session_name="$session_name-$date_time_pattern"
2371         fi
2372
2373         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2374
2375         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2376 }
2377
2378 function validate_trace_path_kernel ()
2379 {
2380         local trace_path=$1
2381         local session_name=$2
2382         local pattern
2383
2384         # If the session was given a trace path on creation, there is no session
2385         # name component to the path. Caller can simply not pass a session name
2386         # for this scenario.
2387         if [ -n "$session_name" ]; then
2388                 session_name="$session_name-$date_time_pattern"
2389         fi
2390
2391         pattern="$trace_path/$session_name/kernel/metadata"
2392
2393         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2394 }
2395
2396 function validate_trace_path_kernel_network ()
2397 {
2398         local trace_path=$1
2399         local session_name=$2
2400         local hostname=$HOSTNAME
2401         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2402
2403         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2404 }
2405
2406 function validate_trace_path_kernel_snapshot ()
2407 {
2408         local trace_path=$1
2409         local session_name=$2
2410         local snapshot_name=$3
2411         local snapshot_number=$4
2412         local base_path=$5
2413         local pattern
2414         local ret
2415
2416         # If the session/output was given a network base path (e.g
2417         # 127.0.0.1/my/custom/path on creation, there is no session name
2418         # component to the path on the relayd side. Caller can simply not pass a
2419         # session name for this scenario.
2420         if [ -n "$session_name" ]; then
2421                 session_name="$session_name-$date_time_pattern"
2422                 if [ -n "$base_path" ]; then
2423                         fail "Session name and base path are mutually exclusive"
2424                         return
2425                 fi
2426         fi
2427
2428         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2429
2430         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2431 }
2432
2433 function validate_trace_path_kernel_snapshot_network ()
2434 {
2435         local trace_path=$1
2436         local session_name=$2
2437         local snapshot_name=$3
2438         local snapshot_number=$4
2439         local base_path=$5
2440         local hostname=$HOSTNAME
2441         local pattern
2442         local ret
2443
2444         # If the session/output was given a network base path (e.g
2445         # 127.0.0.1/my/custom/path on creation, there is no session name
2446         # component to the path on the relayd side. Caller can simply not pass a
2447         # session name for this scenario.
2448         if [ -n "$session_name" ]; then
2449                 session_name="$session_name-$date_time_pattern"
2450                 if [ -n "$base_path" ]; then
2451                         fail "Session name and base path are mutually exclusive"
2452                         return
2453                 fi
2454         fi
2455
2456         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2457
2458         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2459 }