tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$(cut -d '' -f 1 2>/dev/null < /proc/"$pid"/cmdline); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 208         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 209         echo
 210 }
 211
 212 # Helpers for get_possible_cpus.
 213 function get_possible_cpus_count_from_sysfs_possible_mask()
 214 {
 215         local max_possible_cpu_id
 216
 217         # The Awk script extracts the highest CPU id from the possible CPU
 218         # mask. Assuming a numerical order, a field separator '-' and a record
 219         # separator ','. The last value parsed is the highest id.
 220         if [ -f /sys/devices/system/cpu/possible ]; then
 221                 max_possible_cpu_id=$(awk -F '-' 'BEGIN { RS = ","} { last = $NF } END { printf("%d\n", last) }' \
 222                                       /sys/devices/system/cpu/possible)
 223                 echo "$((max_possible_cpu_id+1))"
 224         else
 225                 echo "0"
 226         fi
 227 }
 228
 229 # This is a fallback if the possible CPU mask is not available. This will not
 230 # take into account unplugged CPUs.
 231 function get_max_cpus_count_from_sysfs_cpu_directories()
 232 {
 233         local max_possible_cpu_id=0
 234         local current_cpu_id
 235
 236         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 237                 current_cpu_id="${i#/sys/devices/system/cpu/cpu}"
 238                 if [ "$current_cpu_id" -gt "$max_possible_cpu_id" ]; then
 239                         max_possible_cpu_id="$current_cpu_id"
 240                 fi
 241         done
 242
 243         echo "$((max_possible_cpu_id+1))"
 244 }
 245
 246 # Return the number of possible CPUs.
 247 function get_possible_cpus_count()
 248 {
 249         local possible_cpus_count
 250         possible_cpus_count=$(get_possible_cpus_count_from_sysfs_possible_mask)
 251
 252         if [ "$possible_cpus_count" -eq "0" ]; then
 253                 local configured_cpus_count
 254                 configured_cpus_count=$(getconf _NPROCESSORS_CONF)
 255                 possible_cpus_count=$(get_max_cpus_count_from_sysfs_cpu_directories)
 256                 possible_cpus_count=$((configured_cpus_count > possible_cpus_count \
 257                                                              ? configured_cpus_count \
 258                                                              : possible_cpus_count))
 259         fi
 260
 261         echo "$possible_cpus_count"
 262 }
 263
 264 # Return the list of exposed CPU.
 265 #
 266 # NOTE! Use it like so:
 267 #
 268 # IFS=" " read -r -a VARIABLE <<< "$(get_exposed_cpus_list)"
 269 function get_exposed_cpus_list()
 270 {
 271         local list=()
 272
 273         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 274                 list+=("${i#/sys/devices/system/cpu/cpu}")
 275         done
 276
 277         echo "${list[@]}"
 278 }
 279
 280 # Return any available CPU found. Do not make assumption about the returned
 281 # value, e.g. that it could be 0.
 282 function get_any_available_cpu()
 283 {
 284         for cpu in /sys/devices/system/cpu/cpu[0-9]*; do
 285                 echo "${cpu#/sys/devices/system/cpu/cpu}"
 286                 break;
 287         done
 288 }
 289
 290 # Return the number of _configured_ CPUs.
 291 function conf_proc_count()
 292 {
 293         getconf _NPROCESSORS_CONF
 294         if [ $? -ne 0 ]; then
 295                 diag "Failed to get the number of configured CPUs"
 296         fi
 297         echo
 298 }
 299
 300 # Check if base lttng-modules are present.
 301 # Bail out on failure
 302 function validate_lttng_modules_present ()
 303 {
 304         # Check for loadable modules.
 305         modprobe -n lttng-tracer 2>/dev/null
 306         if [ $? -eq 0 ]; then
 307                 return 0
 308         fi
 309
 310         # Check for builtin modules.
 311         ls /proc/lttng > /dev/null 2>&1
 312         if [ $? -eq 0 ]; then
 313                 return 0
 314         fi
 315
 316         LTTNG_BAIL_OUT "LTTng modules not detected."
 317 }
 318
 319 # Run the lttng binary.
 320 #
 321 # The first two arguments are stdout and stderr redirect paths, respectively.
 322 # The rest of the arguments are forwarded to the lttng binary
 323 function _run_lttng_cmd
 324 {
 325         local stdout_dest="$1"
 326         local stderr_dest="$2"
 327         shift 2
 328
 329         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 330         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 331 }
 332
 333 function enable_kernel_lttng_event
 334 {
 335         local withtap="$1"
 336         local expected_to_fail="$2"
 337         local sess_name="$3"
 338         local event_name="$4"
 339         local channel_name="$5"
 340
 341         if [ -z "$event_name" ]; then
 342                 # Enable all event if no event name specified
 343                 event_name="-a"
 344         fi
 345
 346         if [ -z "$channel_name" ]; then
 347                 # default channel if none specified
 348                 chan=""
 349         else
 350                 chan="-c $channel_name"
 351         fi
 352
 353         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 354         ret=$?
 355         if [[ $expected_to_fail -eq "1" ]]; then
 356                 test $ret -ne "0"
 357                 ret=$?
 358                 if [ $withtap -eq "1" ]; then
 359                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 360                 fi
 361         else
 362                 if [ $withtap -eq "1" ]; then
 363                         ok $ret "Enable kernel event $event_name for session $sess_name"
 364                 fi
 365         fi
 366 }
 367
 368 function enable_kernel_lttng_event_ok ()
 369 {
 370         enable_kernel_lttng_event 1 0 "$@"
 371 }
 372
 373 function enable_kernel_lttng_event_fail ()
 374 {
 375         enable_kernel_lttng_event 1 1 "$@"
 376 }
 377
 378 function enable_kernel_lttng_event_notap ()
 379 {
 380         enable_kernel_lttng_event 0 0 "$@"
 381 }
 382
 383 # Old interface
 384 function lttng_enable_kernel_event
 385 {
 386         enable_kernel_lttng_event_ok "$@"
 387 }
 388
 389 function lttng_enable_kernel_syscall()
 390 {
 391         local expected_to_fail=$1
 392         local sess_name=$2
 393         local syscall_name=$3
 394         local channel_name=$4
 395
 396         if [ -z $syscall_name ]; then
 397                 # Enable all event if no syscall name specified
 398                 syscall_name="-a"
 399         fi
 400
 401         if [ -z $channel_name ]; then
 402                 # default channel if none specified
 403                 chan=""
 404         else
 405                 chan="-c $channel_name"
 406         fi
 407
 408         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 409         ret=$?
 410         if [[ $expected_to_fail -eq "1" ]]; then
 411                 test $ret -ne "0"
 412                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 413         else
 414                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 415         fi
 416 }
 417
 418 function lttng_enable_kernel_syscall_ok()
 419 {
 420         lttng_enable_kernel_syscall 0 "$@"
 421 }
 422
 423 function lttng_enable_kernel_syscall_fail()
 424 {
 425         lttng_enable_kernel_syscall 1 "$@"
 426 }
 427
 428 function lttng_disable_kernel_syscall()
 429 {
 430         local expected_to_fail=$1
 431         local sess_name=$2
 432         local syscall_name=$3
 433         local channel_name=$4
 434
 435         if [ -z $syscall_name ]; then
 436                 # Enable all event if no syscall name specified
 437                 syscall_name="-a"
 438         fi
 439
 440         if [ -z $channel_name ]; then
 441                 # default channel if none specified
 442                 chan=""
 443         else
 444                 chan="-c $channel_name"
 445         fi
 446
 447         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 448
 449         ret=$?
 450         if [[ $expected_to_fail -eq "1" ]]; then
 451                 test $ret -ne "0"
 452                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 453         else
 454                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 455         fi
 456 }
 457
 458 function lttng_disable_kernel_syscall_ok()
 459 {
 460         lttng_disable_kernel_syscall 0 "$@"
 461 }
 462
 463 function lttng_disable_kernel_syscall_fail()
 464 {
 465         lttng_disable_kernel_syscall 1 "$@"
 466 }
 467
 468 function lttng_enable_kernel_function_event ()
 469 {
 470         local expected_to_fail="$1"
 471         local sess_name="$2"
 472         local target="$3"
 473         local event_name="$4"
 474
 475         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 476         ret=$?
 477         if [[ $expected_to_fail -eq "1" ]]; then
 478                 test $ret -ne "0"
 479                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 480         else
 481                 ok $ret "Enable kernel function event for session $sess_name"
 482         fi
 483 }
 484
 485 function lttng_enable_kernel_function_event_ok ()
 486 {
 487         lttng_enable_kernel_function_event 0 "$@"
 488 }
 489
 490 function lttng_enable_kernel_userspace_probe_event ()
 491 {
 492         local expected_to_fail="$1"
 493         local sess_name="$2"
 494         local target="$3"
 495         local event_name="$4"
 496
 497         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 498         ret=$?
 499         if [[ $expected_to_fail -eq "1" ]]; then
 500                 test $ret -ne "0"
 501                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 502         else
 503                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 504         fi
 505 }
 506
 507 function lttng_enable_kernel_userspace_probe_event_fail ()
 508 {
 509         lttng_enable_kernel_userspace_probe_event 1 "$@"
 510 }
 511
 512 function lttng_enable_kernel_userspace_probe_event_ok ()
 513 {
 514         lttng_enable_kernel_userspace_probe_event 0 "$@"
 515 }
 516
 517 function disable_kernel_lttng_userspace_probe_event_ok ()
 518 {
 519         local sess_name="$1"
 520         local event_name="$2"
 521
 522         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 523         ok $? "Disable kernel event $target for session $sess_name"
 524 }
 525 function lttng_enable_kernel_channel()
 526 {
 527         local withtap=$1
 528         local expected_to_fail=$2
 529         local sess_name=$3
 530         local channel_name=$4
 531         local opts="${@:5}"
 532
 533         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 534         ret=$?
 535         if [[ $expected_to_fail -eq "1" ]]; then
 536                 test "$ret" -ne "0"
 537                 ret=$?
 538                 if [ $withtap -eq "1" ]; then
 539                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 540                 fi
 541         else
 542                 if [ $withtap -eq "1" ]; then
 543                         ok $ret "Enable channel $channel_name for session $sess_name"
 544                 fi
 545         fi
 546 }
 547
 548 function lttng_enable_kernel_channel_ok()
 549 {
 550         lttng_enable_kernel_channel 1 0 "$@"
 551 }
 552
 553 function lttng_enable_kernel_channel_fail()
 554 {
 555         lttng_enable_kernel_channel 1 1 "$@"
 556 }
 557
 558 function lttng_enable_kernel_channel_notap()
 559 {
 560         lttng_enable_kernel_channel 0 0 "$@"
 561 }
 562
 563 function enable_kernel_lttng_channel_ok()
 564 {
 565         lttng_enable_kernel_channel 1 0 "$@"
 566 }
 567
 568 function lttng_disable_kernel_channel()
 569 {
 570         local expected_to_fail=$1
 571         local sess_name=$2
 572         local channel_name=$3
 573
 574         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 575         ret=$?
 576         if [[ $expected_to_fail -eq "1" ]]; then
 577                 test "$ret" -ne "0"
 578                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 579         else
 580                 ok $ret "Disable channel $channel_name for session $sess_name"
 581         fi
 582 }
 583
 584 function lttng_disable_kernel_channel_ok()
 585 {
 586         lttng_disable_kernel_channel 0 "$@"
 587 }
 588
 589 function lttng_disable_kernel_channel_fail()
 590 {
 591         lttng_disable_kernel_channel 1 "$@"
 592 }
 593
 594 function start_lttng_relayd_opt()
 595 {
 596         local withtap=$1
 597         local process_mode=$2
 598         local opt=$3
 599
 600         DIR=$(readlink -f "$TESTDIR")
 601
 602         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 603                 # shellcheck disable=SC2086
 604                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 605                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 606                 if [ $? -eq 1 ]; then
 607                         if [ $withtap -eq "1" ]; then
 608                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 609                         fi
 610                         return 1
 611                 else
 612                         if [ $withtap -eq "1" ]; then
 613                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 614                         fi
 615                 fi
 616         else
 617                 pass "Start lttng-relayd (opt: $opt)"
 618         fi
 619 }
 620
 621 function start_lttng_relayd()
 622 {
 623         start_lttng_relayd_opt 1 "-b" "$@"
 624 }
 625
 626 function start_lttng_relayd_notap()
 627 {
 628         start_lttng_relayd_opt 0 "-b" "$@"
 629 }
 630
 631 function stop_lttng_relayd_opt()
 632 {
 633         local withtap=$1
 634         local is_cleanup=$2
 635         local signal=$3
 636         local timeout_s=$4
 637         local dtimeleft_s=
 638         local retval=0
 639         local pids
 640
 641         if [ -z "$signal" ]; then
 642                 signal="SIGTERM"
 643         fi
 644
 645
 646         # Multiply time by 2 to simplify integer arithmetic
 647         if [ -n "$timeout_s" ]; then
 648                 dtimeleft_s=$((timeout_s * 2))
 649         fi
 650
 651
 652         pids=$(lttng_pgrep "$RELAYD_MATCH")
 653         if [ -z "$pids" ]; then
 654                 if [ "$is_cleanup" -eq 1 ]; then
 655                         :
 656                 elif [ "$withtap" -eq "1" ]; then
 657                         fail "No relay daemon to kill"
 658                 else
 659                         LTTNG_BAIL_OUT "No relay daemon to kill"
 660                 fi
 661                 return 0
 662         fi
 663
 664         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 665
 666         # shellcheck disable=SC2086
 667         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 668                 retval=1
 669                 if [ "$withtap" -eq "1" ]; then
 670                         fail "Kill relay daemon"
 671                 fi
 672         else
 673                 out=1
 674                 while [ -n "$out" ]; do
 675                         out=$(lttng_pgrep "$RELAYD_MATCH")
 676                         if [ -n "$dtimeleft_s" ]; then
 677                                 if [ $dtimeleft_s -lt 0 ]; then
 678                                         out=
 679                                         retval=1
 680                                 fi
 681                                 dtimeleft_s=$((dtimeleft_s - 1))
 682                         fi
 683                         sleep 0.5
 684                 done
 685                 if [ "$withtap" -eq "1" ]; then
 686                         if [ "$retval" -eq "0" ]; then
 687                                 pass "Wait after kill relay daemon"
 688                         else
 689                                 fail "Wait after kill relay daemon"
 690                         fi
 691                 fi
 692         fi
 693         return $retval
 694 }
 695
 696 function stop_lttng_relayd()
 697 {
 698         stop_lttng_relayd_opt 1 0 "$@"
 699 }
 700
 701 function stop_lttng_relayd_notap()
 702 {
 703         stop_lttng_relayd_opt 0 0 "$@"
 704 }
 705
 706 function stop_lttng_relayd_cleanup()
 707 {
 708         stop_lttng_relayd_opt 0 1 "$@"
 709 }
 710
 711 #First arg: show tap output
 712 #Second argument: load path for automatic loading
 713 function start_lttng_sessiond_opt()
 714 {
 715         local withtap=$1
 716         local load_path=$2
 717
 718         # The rest of the arguments will be passed directly to lttng-sessiond.
 719         shift 2
 720
 721         local env_vars=""
 722         local consumerd=""
 723
 724         local long_bit_value=
 725         long_bit_value=$(getconf LONG_BIT)
 726
 727         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 728                 # Env variable requested no session daemon
 729                 return
 730         fi
 731
 732         DIR=$(readlink -f "$TESTDIR")
 733
 734         # Get long_bit value for 32/64 consumerd
 735         case "$long_bit_value" in
 736                 32)
 737                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 738                         ;;
 739                 64)
 740                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 741                         ;;
 742                 *)
 743                         return
 744                         ;;
 745         esac
 746
 747         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 748         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 749                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 750         fi
 751         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 752
 753         if ! validate_kernel_version; then
 754             fail "Start session daemon"
 755             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 756         fi
 757
 758         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 759         export LTTNG_SESSION_CONFIG_XSD_PATH
 760
 761         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 762                 # Have a load path ?
 763                 if [ -n "$load_path" ]; then
 764                         # shellcheck disable=SC2086
 765                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 766                 else
 767                         # shellcheck disable=SC2086
 768                         env $env_vars --background "$consumerd" "$@"
 769                 fi
 770                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 771                 status=$?
 772                 if [ "$withtap" -eq "1" ]; then
 773                         ok $status "Start session daemon"
 774                 fi
 775         fi
 776 }
 777
 778 function start_lttng_sessiond()
 779 {
 780         start_lttng_sessiond_opt 1 "$@"
 781 }
 782
 783 function start_lttng_sessiond_notap()
 784 {
 785         start_lttng_sessiond_opt 0 "$@"
 786 }
 787
 788 function stop_lttng_sessiond_opt()
 789 {
 790         local withtap=$1
 791         local is_cleanup=$2
 792         local signal=$3
 793         local timeout_s=$4
 794         local dtimeleft_s=
 795         local retval=0
 796         local runas_pids
 797         local pids
 798
 799         if [ -z "$signal" ]; then
 800                 signal=SIGTERM
 801         fi
 802
 803         # Multiply time by 2 to simplify integer arithmetic
 804         if [ -n "$timeout_s" ]; then
 805                 dtimeleft_s=$((timeout_s * 2))
 806         fi
 807
 808         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 809                 # Env variable requested no session daemon
 810                 return 0
 811         fi
 812
 813         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 814         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 815
 816         if [ -n "$runas_pids" ]; then
 817                 pids="$pids $runas_pids"
 818         fi
 819
 820         if [ -z "$pids" ]; then
 821                 if [ "$is_cleanup" -eq 1 ]; then
 822                         :
 823                 elif [ "$withtap" -eq "1" ]; then
 824                         fail "No session daemon to kill"
 825                 else
 826                         LTTNG_BAIL_OUT "No session daemon to kill"
 827                 fi
 828                 return 0
 829         fi
 830
 831         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 832
 833         # shellcheck disable=SC2086
 834         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 835                 retval=1
 836                 if [ "$withtap" -eq "1" ]; then
 837                         fail "Kill sessions daemon"
 838                 fi
 839         else
 840                 out=1
 841                 while [ -n "$out" ]; do
 842                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 843                         if [ -n "$dtimeleft_s" ]; then
 844                                 if [ $dtimeleft_s -lt 0 ]; then
 845                                         out=
 846                                         retval=1
 847                                 fi
 848                                 dtimeleft_s=$((dtimeleft_s - 1))
 849                         fi
 850                         sleep 0.5
 851                 done
 852                 out=1
 853                 while [ -n "$out" ]; do
 854                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 855                         if [ -n "$dtimeleft_s" ]; then
 856                                 if [ $dtimeleft_s -lt 0 ]; then
 857                                         out=
 858                                         retval=1
 859                                 fi
 860                                 dtimeleft_s=$((dtimeleft_s - 1))
 861                         fi
 862                         sleep 0.5
 863                 done
 864
 865                 if [ "$withtap" -eq "1" ]; then
 866                         if [ "$retval" -eq "0" ]; then
 867                                 pass "Wait after kill session daemon"
 868                         else
 869                                 fail "Wait after kill session daemon"
 870                         fi
 871                 fi
 872         fi
 873         if [ "$signal" = "SIGKILL" ]; then
 874                 if [ "$(id -u)" -eq "0" ]; then
 875                         local modules=
 876                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 877
 878                         if [ -n "$modules" ]; then
 879                                 diag "Unloading all LTTng modules"
 880                                 modprobe --remove "$modules"
 881                         fi
 882                 fi
 883         fi
 884
 885         return $retval
 886 }
 887
 888 function stop_lttng_sessiond()
 889 {
 890         stop_lttng_sessiond_opt 1 0 "$@"
 891 }
 892
 893 function stop_lttng_sessiond_notap()
 894 {
 895         stop_lttng_sessiond_opt 0 0 "$@"
 896 }
 897
 898 function stop_lttng_sessiond_cleanup()
 899 {
 900         stop_lttng_sessiond_opt 0 1 "$@"
 901 }
 902
 903 function sigstop_lttng_sessiond_opt()
 904 {
 905         local withtap=$1
 906         local signal=SIGSTOP
 907         local pids
 908
 909         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 910                 # Env variable requested no session daemon
 911                 return
 912         fi
 913
 914         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 915
 916         if [ "$withtap" -eq "1" ]; then
 917                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 918         fi
 919
 920         # shellcheck disable=SC2086
 921         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 922                 if [ "$withtap" -eq "1" ]; then
 923                         fail "Sending SIGSTOP to session daemon"
 924                 fi
 925         else
 926                 out=1
 927                 while [ $out -ne 0 ]; do
 928                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 929
 930                         # Wait until state becomes stopped for session
 931                         # daemon(s).
 932                         out=0
 933                         for sessiond_pid in $pids; do
 934                                 state="$(ps -p "$sessiond_pid" -o state= )"
 935                                 if [[ -n "$state" && "$state" != "T" ]]; then
 936                                         out=1
 937                                 fi
 938                         done
 939                         sleep 0.5
 940                 done
 941                 if [ "$withtap" -eq "1" ]; then
 942                         pass "Sending SIGSTOP to session daemon"
 943                 fi
 944         fi
 945 }
 946
 947 function sigstop_lttng_sessiond()
 948 {
 949         sigstop_lttng_sessiond_opt 1 "$@"
 950 }
 951
 952 function sigstop_lttng_sessiond_notap()
 953 {
 954         sigstop_lttng_sessiond_opt 0 "$@"
 955 }
 956
 957 function stop_lttng_consumerd_opt()
 958 {
 959         local withtap=$1
 960         local is_cleanup=$2
 961         local signal=$3
 962         local timeout_s=$4
 963         local dtimeleft_s=
 964         local retval=0
 965         local pids
 966
 967         if [ -z "$signal" ]; then
 968                 signal=SIGTERM
 969         fi
 970
 971         # Multiply time by 2 to simplify integer arithmetic
 972         if [ -n "$timeout_s" ]; then
 973                 dtimeleft_s=$((timeout_s * 2))
 974         fi
 975
 976         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 977
 978         if [ -z "$pids" ]; then
 979                 if [ "$is_cleanup" -eq 1 ]; then
 980                         :
 981                 elif [ "$withtap" -eq "1" ]; then
 982                         fail "No consumerd daemon to kill"
 983                 else
 984                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
 985                 fi
 986                 return 0
 987         fi
 988
 989         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 990
 991         # shellcheck disable=SC2086
 992         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 993                 retval=1
 994                 if [ "$withtap" -eq "1" ]; then
 995                         fail "Kill consumer daemon"
 996                 fi
 997         else
 998                 out=1
 999                 while [ $out -ne 0 ]; do
1000                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1001
1002                         # If consumerds are still present check their status.
1003                         # A zombie status qualifies the consumerd as *killed*
1004                         out=0
1005                         for consumer_pid in $pids; do
1006                                 state="$(ps -p "$consumer_pid" -o state= )"
1007                                 if [[ -n "$state" && "$state" != "Z" ]]; then
1008                                         out=1
1009                                 fi
1010                         done
1011                         if [ -n "$dtimeleft_s" ]; then
1012                                 if [ $dtimeleft_s -lt 0 ]; then
1013                                         out=0
1014                                         retval=1
1015                                 fi
1016                                 dtimeleft_s=$((dtimeleft_s - 1))
1017                         fi
1018                         sleep 0.5
1019                 done
1020                 if [ "$withtap" -eq "1" ]; then
1021                         if [ "$retval" -eq "0" ]; then
1022                                 pass "Wait after kill consumer daemon"
1023                         else
1024                                 fail "Wait after kill consumer daemon"
1025                         fi
1026                 fi
1027         fi
1028
1029         return $retval
1030 }
1031
1032 function stop_lttng_consumerd()
1033 {
1034         stop_lttng_consumerd_opt 1 0 "$@"
1035 }
1036
1037 function stop_lttng_consumerd_notap()
1038 {
1039         stop_lttng_consumerd_opt 0 0 "$@"
1040 }
1041
1042 function stop_lttng_consumerd_cleanup()
1043 {
1044         stop_lttng_consumerd_opt 0 1 "$@"
1045 }
1046
1047 function sigstop_lttng_consumerd_opt()
1048 {
1049         local withtap=$1
1050         local signal=SIGSTOP
1051         local pids
1052
1053         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1054
1055         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1056
1057         # shellcheck disable=SC2086
1058         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1059         retval=$?
1060
1061         if [ $retval -eq 1 ]; then
1062                 if [ "$withtap" -eq "1" ]; then
1063                         fail "Sending SIGSTOP to consumer daemon"
1064                 fi
1065                 return 1
1066         else
1067                 out=1
1068                 while [ $out -ne 0 ]; do
1069                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1070
1071                         # Wait until state becomes stopped for all
1072                         # consumers.
1073                         out=0
1074                         for consumer_pid in $pids; do
1075                                 state="$(ps -p "$consumer_pid" -o state= )"
1076                                 if [[ -n "$state" && "$state" != "T" ]]; then
1077                                         out=1
1078                                 fi
1079                         done
1080                         sleep 0.5
1081                 done
1082                 if [ "$withtap" -eq "1" ]; then
1083                         pass "Sending SIGSTOP to consumer daemon"
1084                 fi
1085         fi
1086         return $retval
1087 }
1088
1089 function sigstop_lttng_consumerd()
1090 {
1091         sigstop_lttng_consumerd_opt 1 "$@"
1092 }
1093
1094 function sigstop_lttng_consumerd_notap()
1095 {
1096         sigstop_lttng_consumerd_opt 0 "$@"
1097 }
1098
1099 function list_lttng_with_opts ()
1100 {
1101         local ret
1102         local withtap=$1
1103         shift
1104         local opts=$1
1105         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1106         ret=$?
1107         if [ $withtap -eq "1" ]; then
1108                 ok $ret "Lttng-tool list command with option $opts"
1109         fi
1110 }
1111
1112 function list_lttng_ok ()
1113 {
1114         list_lttng_with_opts 1 "$@"
1115 }
1116
1117 function list_lttng_notap ()
1118 {
1119         list_lttng_with_opts 0 "$@"
1120 }
1121
1122 function create_lttng_session_no_output ()
1123 {
1124         local sess_name=$1
1125         local opts="${@:2}"
1126
1127         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1128         ok $? "Create session $sess_name in no-output mode"
1129 }
1130
1131 function create_lttng_session_uri () {
1132         local sess_name=$1
1133         local uri=$2
1134         local opts="${@:3}"
1135
1136         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1137         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1138 }
1139
1140 function create_lttng_session ()
1141 {
1142         local withtap=$1
1143         local expected_to_fail=$2
1144         local sess_name=$3
1145         local trace_path=$4
1146         local opt=$5
1147
1148         if [ -z "$trace_path" ]; then
1149                 # Use lttng-sessiond default output.
1150                 trace_path=""
1151         else
1152                 trace_path="-o $trace_path"
1153         fi
1154
1155         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create "$sess_name" $trace_path $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1156         ret=$?
1157         if [ $expected_to_fail -eq "1" ]; then
1158                 test "$ret" -ne "0"
1159                 ret=$?
1160                 if [ $withtap -eq "1" ]; then
1161                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1162                 fi
1163         else
1164                 if [ $withtap -eq "1" ]; then
1165                         ok $ret "Create session $sess_name in $trace_path"
1166                 fi
1167         fi
1168         return $ret
1169 }
1170
1171 function create_lttng_session_ok ()
1172 {
1173         create_lttng_session 1 0 "$@"
1174 }
1175
1176 function create_lttng_session_fail ()
1177 {
1178         create_lttng_session 1 1 "$@"
1179 }
1180
1181 function create_lttng_session_notap ()
1182 {
1183         create_lttng_session 0 0 "$@"
1184 }
1185
1186
1187 function enable_ust_lttng_channel ()
1188 {
1189         local withtap=$1
1190         local expected_to_fail=$2
1191         local sess_name=$3
1192         local channel_name=$4
1193         local opts="${@:5}"
1194
1195         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1196         ret=$?
1197         if [[ $expected_to_fail -eq "1" ]]; then
1198                 test "$ret" -ne "0"
1199                 ret=$?
1200                 if [ $withtap -eq "1" ]; then
1201                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1202                 fi
1203         else
1204                 if [ $withtap -eq "1" ]; then
1205                         ok $ret "Enable channel $channel_name for session $sess_name"
1206                 fi
1207         fi
1208         return $ret
1209 }
1210
1211 function enable_ust_lttng_channel_ok ()
1212 {
1213         enable_ust_lttng_channel 1 0 "$@"
1214 }
1215
1216 function enable_ust_lttng_channel_fail ()
1217 {
1218         enable_ust_lttng_channel 1 1 "$@"
1219 }
1220
1221 function enable_ust_lttng_channel_notap ()
1222 {
1223         enable_ust_lttng_channel 0 0 "$@"
1224 }
1225
1226 function disable_ust_lttng_channel()
1227 {
1228         local sess_name=$1
1229         local channel_name=$2
1230
1231         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1232         ok $? "Disable channel $channel_name for session $sess_name"
1233 }
1234
1235 function enable_lttng_mmap_overwrite_kernel_channel()
1236 {
1237         local sess_name=$1
1238         local channel_name=$2
1239
1240         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1241         ok $? "Enable channel $channel_name for session $sess_name"
1242 }
1243
1244 function enable_lttng_mmap_discard_small_kernel_channel()
1245 {
1246         local sess_name=$1
1247         local channel_name=$2
1248
1249         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1250         ok $? "Enable small discard channel $channel_name for session $sess_name"
1251 }
1252
1253 function enable_lttng_mmap_overwrite_small_kernel_channel()
1254 {
1255         local sess_name=$1
1256         local channel_name=$2
1257
1258         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1259         ok $? "Enable small discard channel $channel_name for session $sess_name"
1260 }
1261
1262 function enable_lttng_mmap_overwrite_ust_channel()
1263 {
1264         local sess_name=$1
1265         local channel_name=$2
1266
1267         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1268         ok $? "Enable channel $channel_name for session $sess_name"
1269 }
1270
1271 function enable_ust_lttng_event ()
1272 {
1273         local withtap=$1
1274         local expected_to_fail=$2
1275         local sess_name=$3
1276         local event_name="$4"
1277         local channel_name=$5
1278
1279         if [ -z $channel_name ]; then
1280                 # default channel if none specified
1281                 chan=""
1282         else
1283                 chan="-c $channel_name"
1284         fi
1285
1286         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1287         ret=$?
1288         if [[ $expected_to_fail -eq "1" ]]; then
1289                 test $ret -ne "0"
1290                 ret=$?
1291                 if [[ $withtap -eq "1" ]]; then
1292                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1293                 fi
1294         else
1295                 if [[ $withtap -eq "1" ]]; then
1296                         ok $ret "Enable ust event $event_name for session $sess_name"
1297                 fi
1298         fi
1299         return $ret
1300 }
1301
1302 function enable_ust_lttng_event_ok ()
1303 {
1304         enable_ust_lttng_event 1 0 "$@"
1305 }
1306
1307 function enable_ust_lttng_event_fail ()
1308 {
1309         enable_ust_lttng_event 1 1 "$@"
1310 }
1311
1312 function enable_ust_lttng_event_notap ()
1313 {
1314         enable_ust_lttng_event 0 0 "$@"
1315 }
1316
1317 function enable_jul_lttng_event()
1318 {
1319         sess_name=$1
1320         event_name="$2"
1321         channel_name=$3
1322
1323         if [ -z $channel_name ]; then
1324                 # default channel if none specified
1325                 chan=""
1326         else
1327                 chan="-c $channel_name"
1328         fi
1329
1330         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1331         ok $? "Enable JUL event $event_name for session $sess_name"
1332 }
1333
1334 function enable_jul_lttng_event_loglevel()
1335 {
1336         local sess_name=$1
1337         local event_name="$2"
1338         local loglevel=$3
1339         local channel_name=$4
1340
1341         if [ -z $channel_name ]; then
1342                 # default channel if none specified
1343                 chan=""
1344         else
1345                 chan="-c $channel_name"
1346         fi
1347
1348         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1349         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1350 }
1351
1352 function enable_log4j_lttng_event()
1353 {
1354         local sess_name=$1
1355         local event_name=$2
1356         local channel_name=$3
1357
1358         local chan_opt=()
1359
1360         # default channel if none specified
1361         if [ -n "$channel_name" ]; then
1362                 chan_opt=("-c" "$channel_name")
1363         fi
1364
1365         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1366                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1367         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1368 }
1369
1370 function enable_log4j_lttng_event_filter()
1371 {
1372         local sess_name=$1
1373         local event_name=$2
1374         local filter=$3
1375
1376         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1377                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1378         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1379 }
1380
1381 function enable_log4j_lttng_event_filter_loglevel_only()
1382 {
1383         local sess_name=$1
1384         local event_name=$2
1385         local filter=$3
1386         local loglevel=$4
1387
1388         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1389                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1390         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1391 }
1392
1393 function enable_log4j_lttng_event_loglevel()
1394 {
1395         local sess_name=$1
1396         local event_name=$2
1397         local loglevel=$3
1398         local channel_name=$4
1399
1400
1401         # default channel if none specified
1402         if [ -n "$channel_name" ]; then
1403                 chan_opt=("-c" "$channel_name")
1404         fi
1405
1406         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1407                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1408         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1409 }
1410
1411 function enable_log4j_lttng_event_loglevel_only()
1412 {
1413         local sess_name=$1
1414         local event_name=$2
1415         local loglevel=$3
1416         local channel_name=$4
1417
1418         local chan_opt=()
1419
1420         # default channel if none specified
1421         if [ -n "$channel_name" ]; then
1422                 chan_opt=("-c" "$channel_name")
1423         fi
1424
1425         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1426                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1427         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1428 }
1429
1430 function enable_python_lttng_event()
1431 {
1432         sess_name=$1
1433         event_name="$2"
1434         channel_name=$3
1435
1436         if [ -z $channel_name ]; then
1437                 # default channel if none specified
1438                 chan=""
1439         else
1440                 chan="-c $channel_name"
1441         fi
1442
1443         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1444         ok $? "Enable Python event $event_name for session $sess_name"
1445 }
1446
1447 function enable_python_lttng_event_loglevel()
1448 {
1449         local sess_name=$1
1450         local event_name="$2"
1451         local loglevel=$3
1452         local channel_name=$4
1453
1454         if [ -z $channel_name ]; then
1455                 # default channel if none specified
1456                 chan=""
1457         else
1458                 chan="-c $channel_name"
1459         fi
1460
1461         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1462         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1463 }
1464
1465 function enable_ust_lttng_event_filter()
1466 {
1467         local sess_name="$1"
1468         local event_name="$2"
1469         local filter="$3"
1470         local channel_name=$4
1471
1472         if [ -z $channel_name ]; then
1473                 # default channel if none specified
1474                 chan=""
1475         else
1476                 chan="-c $channel_name"
1477         fi
1478
1479         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event $chan "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1480         ok $? "Enable event $event_name with filtering for session $sess_name"
1481 }
1482
1483 function enable_ust_lttng_event_loglevel()
1484 {
1485         local sess_name="$1"
1486         local event_name="$2"
1487         local loglevel="$3"
1488
1489         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1490         ok $? "Enable event $event_name with loglevel $loglevel"
1491 }
1492
1493 function enable_ust_lttng_event_loglevel_only()
1494 {
1495         local sess_name="$1"
1496         local event_name="$2"
1497         local loglevel="$3"
1498
1499         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1500         ok $? "Enable event $event_name with loglevel-only $loglevel"
1501 }
1502
1503 function disable_ust_lttng_event ()
1504 {
1505         local sess_name="$1"
1506         local event_name="$2"
1507         local channel_name="$3"
1508
1509         if [ -z $channel_name ]; then
1510                 # default channel if none specified
1511                 chan=""
1512         else
1513                 chan="-c $channel_name"
1514         fi
1515
1516         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1517         ok $? "Disable event $event_name for session $sess_name"
1518 }
1519
1520 function disable_jul_lttng_event ()
1521 {
1522         local sess_name="$1"
1523         local event_name="$2"
1524
1525         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1526         ok $? "Disable JUL event $event_name for session $sess_name"
1527 }
1528
1529 function disable_log4j_lttng_event ()
1530 {
1531         local sess_name="$1"
1532         local event_name="$2"
1533
1534         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1535                 disable-event "$event_name" -s "$sess_name" --log4j
1536         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1537 }
1538
1539 function disable_python_lttng_event ()
1540 {
1541         local sess_name="$1"
1542         local event_name="$2"
1543
1544         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1545         ok $? "Disable Python event $event_name for session $sess_name"
1546 }
1547
1548 function start_lttng_tracing_opt ()
1549 {
1550         local withtap=$1
1551         local expected_to_fail=$2
1552         local sess_name=$3
1553
1554         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1555         ret=$?
1556         if [[ $expected_to_fail -eq "1" ]]; then
1557                 test "$ret" -ne "0"
1558                 ret=$?
1559                 if [ $withtap -eq "1" ]; then
1560                         ok $? "Start tracing for session $sess_name failed as expected"
1561                 fi
1562         else
1563                 if [ $withtap -eq "1" ]; then
1564                         ok $ret "Start tracing for session $sess_name"
1565                 fi
1566         fi
1567 }
1568
1569 function start_lttng_tracing_ok ()
1570 {
1571         start_lttng_tracing_opt 1 0 "$@"
1572 }
1573
1574 function start_lttng_tracing_fail ()
1575 {
1576         start_lttng_tracing_opt 1 1 "$@"
1577 }
1578
1579 function start_lttng_tracing_notap ()
1580 {
1581         start_lttng_tracing_opt 0 1 "$@"
1582 }
1583
1584 function stop_lttng_tracing_opt ()
1585 {
1586         local withtap=$1
1587         local expected_to_fail=$2
1588         local sess_name=$3
1589
1590         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1591         ret=$?
1592         if [[ $expected_to_fail -eq "1" ]]; then
1593                 test "$ret" -ne "0"
1594                 ret=$?
1595                 if [ $withtap -eq "1" ]; then
1596                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1597                 fi
1598         else
1599                 if [ $withtap -eq "1" ]; then
1600                         ok $ret "Stop lttng tracing for session $sess_name"
1601                 fi
1602         fi
1603 }
1604
1605 function stop_lttng_tracing_ok ()
1606 {
1607         stop_lttng_tracing_opt 1 0 "$@"
1608 }
1609
1610 function stop_lttng_tracing_fail ()
1611 {
1612         stop_lttng_tracing_opt 1 1 "$@"
1613 }
1614
1615 function stop_lttng_tracing_notap ()
1616 {
1617         stop_lttng_tracing_opt 0 0 "$@"
1618 }
1619
1620 function destroy_lttng_session ()
1621 {
1622         local withtap=$1
1623         local expected_to_fail=$2
1624         local sess_name=$3
1625
1626         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1627         ret=$?
1628         if [[ $expected_to_fail -eq "1" ]]; then
1629                 test "$ret" -ne "0"
1630                 ret=$?
1631                 if [ $withtap -eq "1" ]; then
1632                         ok $ret "Destroy session $sess_name failed as expected"
1633                 fi
1634         else
1635                 if [ $withtap -eq "1" ]; then
1636                         ok $ret "Destroy session $sess_name"
1637                 fi
1638         fi
1639 }
1640
1641 function destroy_lttng_session_ok ()
1642 {
1643         destroy_lttng_session 1 0 "$@"
1644
1645 }
1646
1647 function destroy_lttng_session_fail ()
1648 {
1649         destroy_lttng_session 1 1 "$@"
1650 }
1651
1652 function destroy_lttng_session_notap ()
1653 {
1654         destroy_lttng_session 0 0 "$@"
1655 }
1656
1657 function destroy_lttng_sessions ()
1658 {
1659         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1660         ok $? "Destroy all lttng sessions"
1661 }
1662
1663 function lttng_snapshot_add_output ()
1664 {
1665         local expected_to_fail=$1
1666         local sess_name=$2
1667         local trace_path=$3
1668         local opts=$4
1669
1670         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1671         ret=$?
1672         if [[ $expected_to_fail -eq 1 ]]; then
1673                 test "$ret" -ne "0"
1674                 ok $? "Added snapshot output $trace_path failed as expected"
1675         else
1676                 ok $ret "Added snapshot output $trace_path"
1677         fi
1678 }
1679
1680 function lttng_snapshot_add_output_ok ()
1681 {
1682         lttng_snapshot_add_output 0 "$@"
1683 }
1684
1685 function lttng_snapshot_add_output_fail ()
1686 {
1687         lttng_snapshot_add_output 1 "$@"
1688 }
1689
1690 function lttng_snapshot_del_output ()
1691 {
1692         local expected_to_fail=$1
1693         local sess_name=$2
1694         local id=$3
1695
1696         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1697         ret=$?
1698         if [[ $expected_to_fail -eq "1" ]]; then
1699                 test "$ret" -ne "0"
1700                 ok $? "Deleted snapshot output id $id failed as expected"
1701         else
1702                 ok $ret "Deleted snapshot output id $id"
1703         fi
1704 }
1705
1706 function lttng_snapshot_del_output_ok ()
1707 {
1708         lttng_snapshot_del_output 0 "$@"
1709 }
1710
1711 function lttng_snapshot_del_output_fail ()
1712 {
1713         lttng_snapshot_del_output 1 "$@"
1714 }
1715
1716 function lttng_snapshot_record ()
1717 {
1718         local sess_name=$1
1719         local trace_path=$2
1720
1721         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s "$sess_name" "$trace_path" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1722         ok $? "Snapshot recorded"
1723 }
1724
1725 function lttng_snapshot_list ()
1726 {
1727         local sess_name=$1
1728         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1729         ok $? "Snapshot list"
1730 }
1731
1732 function lttng_save()
1733 {
1734         local sess_name=$1
1735         local opts=$2
1736
1737         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1738         ok $? "Session saved"
1739 }
1740
1741 function lttng_load()
1742 {
1743         local expected_to_fail=$1
1744         local opts=$2
1745
1746         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1747         ret=$?
1748         if [[ $expected_to_fail -eq "1" ]]; then
1749                 test $ret -ne "0"
1750                 ok $? "Load command failed as expected with opts: $opts"
1751         else
1752                 ok $ret "Load command with opts: $opts"
1753         fi
1754 }
1755
1756 function lttng_load_ok()
1757 {
1758         lttng_load 0 "$@"
1759 }
1760
1761 function lttng_load_fail()
1762 {
1763         lttng_load 1 "$@"
1764 }
1765
1766 function lttng_track()
1767 {
1768         local expected_to_fail="$1"
1769         shift 1
1770         local opts="$@"
1771         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1772         ret=$?
1773         if [[ $expected_to_fail -eq "1" ]]; then
1774                 test $ret -ne "0"
1775                 ok $? "Track command failed as expected with opts: $opts"
1776         else
1777                 ok $ret "Track command with opts: $opts"
1778         fi
1779 }
1780
1781 function lttng_track_ok()
1782 {
1783         lttng_track 0 "$@"
1784 }
1785
1786 function lttng_track_fail()
1787 {
1788         lttng_track 1 "$@"
1789 }
1790
1791 function lttng_untrack()
1792 {
1793         local expected_to_fail="$1"
1794         shift 1
1795         local opts="$@"
1796         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1797         ret=$?
1798         if [[ $expected_to_fail -eq "1" ]]; then
1799                 test $ret -ne "0"
1800                 ok $? "Untrack command failed as expected with opts: $opts"
1801         else
1802                 ok $ret "Untrack command with opts: $opts"
1803         fi
1804 }
1805
1806 function lttng_untrack_ok()
1807 {
1808         lttng_untrack 0 "$@"
1809 }
1810
1811 function lttng_untrack_fail()
1812 {
1813         lttng_untrack 1 "$@"
1814 }
1815
1816 function lttng_track_pid_ok()
1817 {
1818         PID=$1
1819         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1820         ok $? "Lttng track pid on the kernel domain"
1821 }
1822
1823 function lttng_untrack_kernel_all_ok()
1824 {
1825         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1826         ok $? "Lttng untrack all pid on the kernel domain"
1827 }
1828
1829 function lttng_track_ust_ok()
1830 {
1831         lttng_track_ok -u "$@"
1832 }
1833
1834 function lttng_track_ust_fail()
1835 {
1836         lttng_track_fail -u "$@"
1837 }
1838
1839 function lttng_track_kernel_ok()
1840 {
1841         lttng_track_ok -k "$@"
1842 }
1843
1844 function lttng_track_kernel_fail()
1845 {
1846         lttng_track_fail -k "$@"
1847 }
1848
1849 function lttng_untrack_ust_ok()
1850 {
1851         lttng_untrack_ok -u "$@"
1852 }
1853
1854 function lttng_untrack_ust_fail()
1855 {
1856         lttng_untrack_fail -u "$@"
1857 }
1858
1859 function lttng_untrack_kernel_ok()
1860 {
1861         lttng_untrack_ok -k "$@"
1862 }
1863
1864 function lttng_untrack_kernel_fail()
1865 {
1866         lttng_untrack_fail -k "$@"
1867 }
1868
1869 function lttng_add_context_list()
1870 {
1871         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1872         ret=$?
1873         ok $ret "Context listing"
1874 }
1875
1876 function add_context_lttng()
1877 {
1878         local expected_to_fail="$1"
1879         local domain="$2"
1880         local session_name="$3"
1881         local channel_name="$4"
1882         local type="$5"
1883
1884         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1885         ret=$?
1886         if [[ $expected_to_fail -eq "1" ]]; then
1887                 test $ret -ne "0"
1888                 ok $? "Add context command failed as expected for type: $type"
1889         else
1890                 ok $ret "Add context command for type: $type"
1891         fi
1892 }
1893
1894 function add_context_ust_ok()
1895 {
1896         add_context_lttng 0 -u "$@"
1897 }
1898
1899 function add_context_ust_fail()
1900 {
1901         add_context_lttng 1 -u "$@"
1902 }
1903
1904 function add_context_kernel_ok()
1905 {
1906         add_context_lttng 0 -k "$@"
1907 }
1908
1909 function add_context_kernel_fail()
1910 {
1911         add_context_lttng 1 -k "$@"
1912 }
1913
1914 function wait_live_trace_ready ()
1915 {
1916         local url=$1
1917         local zero_client_match=0
1918
1919         diag "Waiting for live trace at url: $url"
1920         while [ $zero_client_match -eq 0 ]; do
1921                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1922                 sleep 0.5
1923         done
1924         pass "Waiting for live trace at url: $url"
1925 }
1926
1927 function wait_live_viewer_connect ()
1928 {
1929         local url=$1
1930         local one_client_match=0
1931
1932         diag "Waiting for live viewers on url: $url"
1933         while [ $one_client_match -eq 0 ]; do
1934                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1935                 sleep 0.5
1936         done
1937         pass "Waiting for live viewers on url: $url"
1938 }
1939
1940 function bail_out_if_no_babeltrace()
1941 {
1942         which "$BABELTRACE_BIN" >/dev/null
1943         if [ $? -ne 0 ]; then
1944                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
1945         fi
1946 }
1947
1948 function validate_metadata_event ()
1949 {
1950         local event_name=$1
1951         local nr_event_id=$2
1952         local trace_path=$3
1953
1954         local metadata_file=$(find $trace_path -name "metadata")
1955         local metadata_path=$(dirname $metadata_file)
1956
1957         which $BABELTRACE_BIN >/dev/null
1958         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1959
1960         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1961
1962         if [ "$count" -ne "$nr_event_id" ]; then
1963                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1964                 diag "$count matching event id found in metadata"
1965         else
1966                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1967         fi
1968
1969 }
1970
1971 function trace_matches ()
1972 {
1973         local event_name=$1
1974         local nr_iter=$2
1975         local trace_path=$3
1976
1977         which $BABELTRACE_BIN >/dev/null
1978         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1979
1980         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1981
1982         if [ "$count" -ne "$nr_iter" ]; then
1983                 fail "Trace match"
1984                 diag "$count matching events found in trace"
1985         else
1986                 pass "Trace match"
1987         fi
1988 }
1989
1990 function trace_match_only()
1991 {
1992         local event_name=$1
1993         local nr_iter=$2
1994         local trace_path=$3
1995
1996         which $BABELTRACE_BIN >/dev/null
1997         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1998
1999         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
2000         local total=$($BABELTRACE_BIN $trace_path | wc -l)
2001
2002         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
2003                 pass "Trace match with $total event $event_name"
2004         else
2005                 fail "Trace match"
2006                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
2007         fi
2008 }
2009
2010 function validate_trace
2011 {
2012         local event_name=$1
2013         local trace_path=$2
2014
2015         which $BABELTRACE_BIN >/dev/null
2016         if [ $? -ne 0 ]; then
2017             skip 0 "Babeltrace binary not found. Skipping trace validation"
2018         fi
2019
2020         OLDIFS=$IFS
2021         IFS=","
2022         for i in $event_name; do
2023                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2024                 if [ "$traced" -ne 0 ]; then
2025                         pass "Validate trace for event $i, $traced events"
2026                 else
2027                         fail "Validate trace for event $i"
2028                         diag "Found $traced occurrences of $i"
2029                 fi
2030         done
2031         ret=$?
2032         IFS=$OLDIFS
2033         return $ret
2034 }
2035
2036 function validate_trace_count
2037 {
2038         local event_name=$1
2039         local trace_path=$2
2040         local expected_count=$3
2041
2042         which $BABELTRACE_BIN >/dev/null
2043         if [ $? -ne 0 ]; then
2044             skip 0 "Babeltrace binary not found. Skipping trace validation"
2045         fi
2046
2047         cnt=0
2048         OLDIFS=$IFS
2049         IFS=","
2050         for i in $event_name; do
2051                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2052                 if [ "$traced" -ne 0 ]; then
2053                         pass "Validate trace for event $i, $traced events"
2054                 else
2055                         fail "Validate trace for event $i"
2056                         diag "Found $traced occurrences of $i"
2057                 fi
2058                 cnt=$(($cnt + $traced))
2059         done
2060         IFS=$OLDIFS
2061         test $cnt -eq $expected_count
2062         ok $? "Read a total of $cnt events, expected $expected_count"
2063 }
2064
2065 function validate_trace_count_range_incl_min_excl_max
2066 {
2067         local event_name=$1
2068         local trace_path=$2
2069         local expected_min=$3
2070         local expected_max=$4
2071
2072         which $BABELTRACE_BIN >/dev/null
2073         if [ $? -ne 0 ]; then
2074             skip 0 "Babeltrace binary not found. Skipping trace validation"
2075         fi
2076
2077         cnt=0
2078         OLDIFS=$IFS
2079         IFS=","
2080         for i in $event_name; do
2081                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2082                 if [ "$traced" -ge $expected_min ]; then
2083                         pass "Validate trace for event $i, $traced events"
2084                 else
2085                         fail "Validate trace for event $i"
2086                         diag "Found $traced occurrences of $i"
2087                 fi
2088                 cnt=$(($cnt + $traced))
2089         done
2090         IFS=$OLDIFS
2091         test $cnt -lt $expected_max
2092         ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
2093 }
2094
2095 function trace_first_line
2096 {
2097         local trace_path=$1
2098
2099         which $BABELTRACE_BIN >/dev/null
2100         if [ $? -ne 0 ]; then
2101             skip 0 "Babeltrace binary not found. Skipping trace validation"
2102         fi
2103
2104         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
2105 }
2106
2107 function validate_trace_exp()
2108 {
2109         local event_exp=$1
2110         local trace_path=$2
2111
2112         which $BABELTRACE_BIN >/dev/null
2113         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
2114
2115         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
2116         if [ "$traced" -ne 0 ]; then
2117                 pass "Validate trace for expression '${event_exp}', $traced events"
2118         else
2119                 fail "Validate trace for expression '${event_exp}'"
2120                 diag "Found $traced occurrences of '${event_exp}'"
2121         fi
2122         ret=$?
2123         return $ret
2124 }
2125
2126 function validate_trace_only_exp()
2127 {
2128         local event_exp=$1
2129         local trace_path=$2
2130
2131         which $BABELTRACE_BIN >/dev/null
2132         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
2133
2134         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
2135         local total=$($BABELTRACE_BIN $trace_path | wc -l)
2136
2137         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
2138                 pass "Trace match with $total for expression '${event_exp}'"
2139         else
2140                 fail "Trace match"
2141                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
2142         fi
2143         ret=$?
2144         return $ret
2145 }
2146
2147 function validate_trace_empty()
2148 {
2149         local trace_path=$1
2150
2151         which $BABELTRACE_BIN >/dev/null
2152         if [ $? -ne 0 ]; then
2153             skip 0 "Babeltrace binary not found. Skipping trace validation"
2154         fi
2155
2156         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
2157         ret=$?
2158         if [ $ret -ne 0 ]; then
2159                 fail "Failed to parse trace"
2160                 return $ret
2161         fi
2162
2163         traced=$(echo -n "$events" | wc -l)
2164         if [ "$traced" -eq 0 ]; then
2165                 pass "Validate empty trace"
2166         else
2167                 fail "Validate empty trace"
2168                 diag "Found $traced events in trace"
2169         fi
2170         ret=$?
2171         return $ret
2172 }
2173
2174 function validate_directory_empty ()
2175 {
2176         local trace_path="$1"
2177
2178         # Do not double quote `$trace_path` below as we want wildcards to be
2179         # expanded.
2180         files="$(ls -A $trace_path)"
2181         ret=$?
2182         if [ $ret -ne 0 ]; then
2183                 fail "Failed to list content of directory \"$trace_path\""
2184                 return $ret
2185         fi
2186
2187         nb_files="$(echo -n "$files" | wc -l)"
2188         ok $nb_files "Directory \"$trace_path\" is empty"
2189 }
2190
2191 function validate_trace_session_ust_empty()
2192 {
2193         validate_directory_empty "$1"/ust
2194 }
2195
2196 function validate_trace_session_kernel_empty()
2197 {
2198         validate_trace_empty "$1"/kernel
2199 }
2200
2201 function regenerate_metadata ()
2202 {
2203         local expected_to_fail=$1
2204         local sess_name=$2
2205
2206         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2207         ret=$?
2208         if [[ $expected_to_fail -eq "1" ]]; then
2209                 test "$ret" -ne "0"
2210                 ok $? "Expected fail on regenerate metadata $sess_name"
2211         else
2212                 ok $ret "Metadata regenerate $sess_name"
2213         fi
2214 }
2215
2216 function regenerate_metadata_ok ()
2217 {
2218         regenerate_metadata 0 "$@"
2219 }
2220
2221 function regenerate_metadata_fail ()
2222 {
2223         regenerate_metadata 1 "$@"
2224 }
2225
2226 function regenerate_statedump ()
2227 {
2228         local expected_to_fail=$1
2229         local sess_name=$2
2230
2231         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2232         ret=$?
2233         if [[ $expected_to_fail -eq "1" ]]; then
2234                 test "$ret" -ne "0"
2235                 ok $? "Expected fail on regenerate statedump $sess_name"
2236         else
2237                 ok $ret "Statedump regenerate $sess_name"
2238         fi
2239 }
2240
2241 function regenerate_statedump_ok ()
2242 {
2243         regenerate_statedump 0 "$@"
2244 }
2245
2246 function regenerate_statedump_fail ()
2247 {
2248         regenerate_statedump 1 "$@"
2249 }
2250
2251 function rotate_session ()
2252 {
2253         local expected_to_fail=$1
2254         local sess_name=$2
2255
2256         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2257         ret=$?
2258         if [[ $expected_to_fail -eq "1" ]]; then
2259                 test "$ret" -ne "0"
2260                 ok $? "Expected fail on rotate session $sess_name"
2261         else
2262                 ok $ret "Rotate session $sess_name"
2263         fi
2264 }
2265
2266 function rotate_session_ok ()
2267 {
2268         rotate_session 0 "$@"
2269 }
2270
2271 function rotate_session_fail ()
2272 {
2273         rotate_session 1 "$@"
2274 }
2275
2276 function destructive_tests_enabled ()
2277 {
2278         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2279                 return 0
2280         else
2281                 return 1
2282         fi
2283 }
2284
2285 function lttng_enable_rotation_timer ()
2286 {
2287         local expected_to_fail=$1
2288         local sess_name=$2
2289         local period=$3
2290
2291         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2292         ret=$?
2293         if [[ $expected_to_fail -eq "1" ]]; then
2294                 test "$ret" -ne "0"
2295                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2296         else
2297                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2298         fi
2299 }
2300
2301 function lttng_enable_rotation_timer_ok ()
2302 {
2303         lttng_enable_rotation_timer 0 $@
2304 }
2305
2306 function lttng_enable_rotation_timer_fail ()
2307 {
2308         lttng_enable_rotation_timer 1 $@
2309 }
2310
2311 function lttng_enable_rotation_size ()
2312 {
2313         local expected_to_fail=$1
2314         local sess_name=$2
2315         local size=$3
2316
2317         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2318         ret=$?
2319         if [[ $expected_to_fail -eq "1" ]]; then
2320                 test "$ret" -ne "0"
2321                 ok $? "Expected fail on rotate session $sess_name"
2322         else
2323                 ok $ret "Rotate session $sess_name"
2324         fi
2325 }
2326
2327 function lttng_enable_rotation_size_ok ()
2328 {
2329         lttng_enable_rotation_size 0 $@
2330 }
2331
2332 function lttng_enable_rotation_size_fail ()
2333 {
2334         lttng_enable_rotation_size 1 $@
2335 }
2336
2337 function lttng_clear_session ()
2338 {
2339         local expected_to_fail=$1
2340         local sess_name=$2
2341
2342         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2343         ret=$?
2344         if [[ $expected_to_fail -eq "1" ]]; then
2345                 test "$ret" -ne "0"
2346                 ok $? "Expected fail on clear session $sess_name"
2347         else
2348                 ok $ret "Clear session $sess_name"
2349         fi
2350 }
2351
2352 function lttng_clear_session_ok ()
2353 {
2354         lttng_clear_session 0 $@
2355 }
2356
2357 function lttng_clear_session_fail ()
2358 {
2359         lttng_clear_session 1 $@
2360 }
2361
2362 function lttng_clear_all ()
2363 {
2364         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2365         ok $? "Clear all lttng sessions"
2366 }
2367
2368 function lttng_add_trigger()
2369 {
2370         local expected_to_fail="$1"
2371         local trigger_name="$2"
2372         shift 2
2373         local args=("$@")
2374
2375         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2376         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2377         ret=$?
2378         if [[ $expected_to_fail -eq "1" ]]; then
2379                 test "$ret" -ne "0"
2380                 ok $? "Add trigger $trigger_name failed as expected"
2381         else
2382                 ok $ret "Add trigger $trigger_name"
2383         fi
2384 }
2385
2386 function lttng_remove_trigger()
2387 {
2388         local expected_to_fail="$1"
2389         local trigger_name="$2"
2390         shift 2
2391
2392         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2393         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2394         ret=$?
2395         if [[ $expected_to_fail -eq "1" ]]; then
2396                 test "$ret" -ne "0"
2397                 ok $? "Remove trigger $trigger_name failed as expected"
2398         else
2399                 ok $ret "Remove trigger $trigger_name"
2400         fi
2401 }
2402
2403 function lttng_add_trigger_ok()
2404 {
2405         lttng_add_trigger 0 "$@"
2406 }
2407
2408 function lttng_add_trigger_fail()
2409 {
2410         lttng_add_trigger 1 "$@"
2411 }
2412
2413 function lttng_remove_trigger_ok()
2414 {
2415         lttng_remove_trigger 0 "$@"
2416 }
2417
2418 function list_triggers_matches_ok ()
2419 {
2420         local tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2421         local tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2422
2423         local test_name="$1"
2424         local expected_stdout_file="$2"
2425
2426         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2427
2428         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2429         ok $? "${test_name}: exit code is 0"
2430
2431         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2432         ok $? "${test_name}: expected stdout"
2433
2434         diff -u /dev/null "${tmp_stderr}"
2435         ok $? "${test_name}: expected stderr"
2436
2437         rm -f "${tmp_stdout}"
2438         rm -f "${tmp_stderr}"
2439 }
2440
2441 function list_triggers_matches_mi_ok ()
2442 {
2443         local tmp_stdout
2444         local tmp_stdout_raw
2445         local tmp_stderr
2446
2447         local test_name="$1"
2448         local expected_stdout_file="$2"
2449
2450         tmp_stdout_raw=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2451         tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2452         tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2453
2454         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2455
2456         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2457         ok $? "${test_name}: exit code is 0"
2458
2459         # Pretty-fy xml before further test.
2460         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2461
2462         $MI_VALIDATE "${tmp_stdout}"
2463         ok $? "list-trigger mi is valid"
2464
2465         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2466         ok $? "${test_name}: expected stdout"
2467
2468         diff -u /dev/null "${tmp_stderr}"
2469         ok $? "${test_name}: expected stderr"
2470
2471         rm -f "${tmp_stdout}"
2472         rm -f "${tmp_stdout_raw}"
2473         rm -f "${tmp_stderr}"
2474 }
2475
2476 function validate_path_pattern ()
2477 {
2478         local message=$1
2479         local pattern=$2
2480         # Base path is only used in error case and is used to list the content
2481         # of the base path.
2482         local base_path=$3
2483
2484
2485         [ -f $pattern ]
2486         ret=$?
2487         ok $ret "$message"
2488
2489         if [ "$ret" -ne "0" ]; then
2490                 diag "Path pattern expected: $pattern"
2491                 # List the tracepath for more info. We use find as a recursive
2492                 # directory lister.
2493                 diag "The base path content:"
2494                 find "$base_path" -print
2495         fi
2496 }
2497
2498 function validate_trace_path_ust_uid ()
2499 {
2500         local trace_path=$1
2501         local session_name=$2
2502         local uid=$UID
2503         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2504
2505         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2506 }
2507
2508 function validate_trace_path_ust_uid_network ()
2509 {
2510         local trace_path=$1
2511         local session_name=$2
2512         local base_path=$3
2513         local uid=$UID
2514         local hostname=$HOSTNAME
2515         local pattern
2516         local ret
2517
2518         # If the session was given a network base path (e.g
2519         # 127.0.0.1/my/custom/path on creation, there is no session name
2520         # component to the path on the relayd side. Caller can simply not pass a
2521         # session name for this scenario.
2522         if [ -n "$session_name" ]; then
2523                 session_name="$session_name-$date_time_pattern"
2524                 if [ -n "$base_path" ]; then
2525                         fail "Session name and base path are mutually exclusive"
2526                         return
2527                 fi
2528         fi
2529
2530         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2531
2532         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2533 }
2534
2535 function validate_trace_path_ust_uid_snapshot_network ()
2536 {
2537         local trace_path=$1
2538         local session_name=$2
2539         local snapshot_name=$3
2540         local snapshot_number=$4
2541         local base_path=$5
2542         local hostname=$HOSTNAME
2543         local uid=$UID
2544         local pattern
2545         local ret
2546
2547         # If the session/output was given a network base path (e.g
2548         # 127.0.0.1/my/custom/path on creation, there is no session name
2549         # component to the path on the relayd side. Caller can simply not pass a
2550         # session name for this scenario.
2551         if [ -n "$session_name" ]; then
2552                 session_name="$session_name-$date_time_pattern"
2553                 if [ -n "$base_path" ]; then
2554                         fail "Session name and base path are mutually exclusive"
2555                         return
2556                 fi
2557         fi
2558
2559         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2560
2561         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2562 }
2563
2564 function validate_trace_path_ust_uid_snapshot ()
2565 {
2566         local trace_path=$1
2567         local session_name=$2
2568         local snapshot_name=$3
2569         local snapshot_number=$4
2570         local base_path=$5
2571         local uid=$UID
2572         local pattern
2573         local ret
2574
2575         # If the session/output was given a network base path (e.g
2576         # 127.0.0.1/my/custom/path) on creation, there is no session name
2577         # component to the path on the relayd side. Caller can simply not pass a
2578         # session name for this scenario.
2579         if [ -n "$session_name" ]; then
2580                 session_name="$session_name-$date_time_pattern"
2581                 if [ -n "$base_path" ]; then
2582                         fail "Session name and base path are mutually exclusive"
2583                         return
2584                 fi
2585         fi
2586
2587         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2588
2589         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2590 }
2591
2592 function validate_trace_path_ust_pid ()
2593 {
2594         local trace_path=$1
2595         local session_name=$2
2596         local app_string=$3
2597         local pid=$4
2598         local pattern
2599         local ret
2600
2601         # If the session was given a trace path on creation, there is no session
2602         # name component to the path. Caller can simply not pass a session name
2603         # for this scenario.
2604         if [ -n "$session_name" ]; then
2605                 session_name="$session_name-$date_time_pattern"
2606         fi
2607
2608         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2609
2610         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2611 }
2612
2613 function validate_trace_path_kernel ()
2614 {
2615         local trace_path=$1
2616         local session_name=$2
2617         local pattern
2618
2619         # If the session was given a trace path on creation, there is no session
2620         # name component to the path. Caller can simply not pass a session name
2621         # for this scenario.
2622         if [ -n "$session_name" ]; then
2623                 session_name="$session_name-$date_time_pattern"
2624         fi
2625
2626         pattern="$trace_path/$session_name/kernel/metadata"
2627
2628         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2629 }
2630
2631 function validate_trace_path_kernel_network ()
2632 {
2633         local trace_path=$1
2634         local session_name=$2
2635         local hostname=$HOSTNAME
2636         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2637
2638         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2639 }
2640
2641 function validate_trace_path_kernel_snapshot ()
2642 {
2643         local trace_path=$1
2644         local session_name=$2
2645         local snapshot_name=$3
2646         local snapshot_number=$4
2647         local base_path=$5
2648         local pattern
2649         local ret
2650
2651         # If the session/output was given a network base path (e.g
2652         # 127.0.0.1/my/custom/path on creation, there is no session name
2653         # component to the path on the relayd side. Caller can simply not pass a
2654         # session name for this scenario.
2655         if [ -n "$session_name" ]; then
2656                 session_name="$session_name-$date_time_pattern"
2657                 if [ -n "$base_path" ]; then
2658                         fail "Session name and base path are mutually exclusive"
2659                         return
2660                 fi
2661         fi
2662
2663         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2664
2665         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2666 }
2667
2668 function validate_trace_path_kernel_snapshot_network ()
2669 {
2670         local trace_path=$1
2671         local session_name=$2
2672         local snapshot_name=$3
2673         local snapshot_number=$4
2674         local base_path=$5
2675         local hostname=$HOSTNAME
2676         local pattern
2677         local ret
2678
2679         # If the session/output was given a network base path (e.g
2680         # 127.0.0.1/my/custom/path on creation, there is no session name
2681         # component to the path on the relayd side. Caller can simply not pass a
2682         # session name for this scenario.
2683         if [ -n "$session_name" ]; then
2684                 session_name="$session_name-$date_time_pattern"
2685                 if [ -n "$base_path" ]; then
2686                         fail "Session name and base path are mutually exclusive"
2687                         return
2688                 fi
2689         fi
2690
2691         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2692
2693         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2694 }