tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$(cut -d '' -f 1 2>/dev/null < /proc/"$pid"/cmdline); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 208         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 209         echo
 210 }
 211
 212 # Return a space-separated string of online CPU IDs, based on
 213 # /sys/devices/system/cpu/online, or from 0 to nproc - 1 otherwise.
 214 function get_online_cpus()
 215 {
 216         local cpus=()
 217         local range_re
 218         if [ -f /sys/devices/system/cpu/online ]; then
 219                 range_re='([0-9]+)-([0-9]+)'
 220                 while read -r range ; do
 221                         if [[ "${range}" =~ ${range_re} ]] ; then
 222                                 mapfile -t -O "${#cpus[*]}" cpus <<< $(seq "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}")
 223                         else
 224                                 cpus+=("${range}")
 225                         fi
 226                 done < <(tr ',' $'\n' < /sys/devices/system/cpu/online)
 227         else
 228                 read -r -a cpus <<< $(seq 0 $(( $(conf_proc_count) - 1 )) )
 229         fi
 230         echo "${cpus[*]}"
 231 }
 232
 233 # Helpers for get_possible_cpus.
 234 function get_possible_cpus_count_from_sysfs_possible_mask()
 235 {
 236         local max_possible_cpu_id
 237
 238         # The Awk script extracts the highest CPU id from the possible CPU
 239         # mask. Assuming a numerical order, a field separator '-' and a record
 240         # separator ','. The last value parsed is the highest id.
 241         if [ -f /sys/devices/system/cpu/possible ]; then
 242                 max_possible_cpu_id=$(awk -F '-' 'BEGIN { RS = ","} { last = $NF } END { printf("%d\n", last) }' \
 243                                       /sys/devices/system/cpu/possible)
 244                 echo "$((max_possible_cpu_id+1))"
 245         else
 246                 echo "0"
 247         fi
 248 }
 249
 250 # This is a fallback if the possible CPU mask is not available. This will not
 251 # take into account unplugged CPUs.
 252 function get_max_cpus_count_from_sysfs_cpu_directories()
 253 {
 254         local max_possible_cpu_id=0
 255         local current_cpu_id
 256
 257         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 258                 current_cpu_id="${i#/sys/devices/system/cpu/cpu}"
 259                 if [ "$current_cpu_id" -gt "$max_possible_cpu_id" ]; then
 260                         max_possible_cpu_id="$current_cpu_id"
 261                 fi
 262         done
 263
 264         echo "$((max_possible_cpu_id+1))"
 265 }
 266
 267 # Return the number of possible CPUs.
 268 function get_possible_cpus_count()
 269 {
 270         local possible_cpus_count
 271         possible_cpus_count=$(get_possible_cpus_count_from_sysfs_possible_mask)
 272
 273         if [ "$possible_cpus_count" -eq "0" ]; then
 274                 local configured_cpus_count
 275                 configured_cpus_count=$(getconf _NPROCESSORS_CONF)
 276                 possible_cpus_count=$(get_max_cpus_count_from_sysfs_cpu_directories)
 277                 possible_cpus_count=$((configured_cpus_count > possible_cpus_count \
 278                                                              ? configured_cpus_count \
 279                                                              : possible_cpus_count))
 280         fi
 281
 282         echo "$possible_cpus_count"
 283 }
 284
 285 # Return the list of exposed CPU.
 286 #
 287 # NOTE! Use it like so:
 288 #
 289 # IFS=" " read -r -a VARIABLE <<< "$(get_exposed_cpus_list)"
 290 function get_exposed_cpus_list()
 291 {
 292         local list=()
 293
 294         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 295                 list+=("${i#/sys/devices/system/cpu/cpu}")
 296         done
 297
 298         echo "${list[@]}"
 299 }
 300
 301 # Return any available CPU found. Do not make assumption about the returned
 302 # value, e.g. that it could be 0.
 303 function get_any_available_cpu()
 304 {
 305         for cpu in $(get_online_cpus); do
 306                 echo "${cpu}"
 307                 break;
 308         done
 309 }
 310
 311 # Return the number of _configured_ CPUs.
 312 function conf_proc_count()
 313 {
 314         getconf _NPROCESSORS_CONF
 315         if [ $? -ne 0 ]; then
 316                 diag "Failed to get the number of configured CPUs"
 317         fi
 318         echo
 319 }
 320
 321 # Check if base lttng-modules are present.
 322 # Bail out on failure
 323 function validate_lttng_modules_present ()
 324 {
 325         # Check for loadable modules.
 326         modprobe -n lttng-tracer 2>/dev/null
 327         if [ $? -eq 0 ]; then
 328                 return 0
 329         fi
 330
 331         # Check for builtin modules.
 332         ls /proc/lttng > /dev/null 2>&1
 333         if [ $? -eq 0 ]; then
 334                 return 0
 335         fi
 336
 337         LTTNG_BAIL_OUT "LTTng modules not detected."
 338 }
 339
 340 # Run the lttng binary.
 341 #
 342 # The first two arguments are stdout and stderr redirect paths, respectively.
 343 # The rest of the arguments are forwarded to the lttng binary
 344 function _run_lttng_cmd
 345 {
 346         local stdout_dest="$1"
 347         local stderr_dest="$2"
 348         shift 2
 349
 350         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 351         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 352 }
 353
 354 function enable_kernel_lttng_event
 355 {
 356         local withtap="$1"
 357         local expected_to_fail="$2"
 358         local sess_name="$3"
 359         local event_name="$4"
 360         local channel_name="$5"
 361
 362         if [ -z "$event_name" ]; then
 363                 # Enable all event if no event name specified
 364                 event_name="-a"
 365         fi
 366
 367         if [ -z "$channel_name" ]; then
 368                 # default channel if none specified
 369                 chan=""
 370         else
 371                 chan="-c $channel_name"
 372         fi
 373
 374         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 375         ret=$?
 376         if [[ $expected_to_fail -eq "1" ]]; then
 377                 test $ret -ne "0"
 378                 ret=$?
 379                 if [ $withtap -eq "1" ]; then
 380                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 381                 fi
 382         else
 383                 if [ $withtap -eq "1" ]; then
 384                         ok $ret "Enable kernel event $event_name for session $sess_name"
 385                 fi
 386         fi
 387 }
 388
 389 function enable_kernel_lttng_event_ok ()
 390 {
 391         enable_kernel_lttng_event 1 0 "$@"
 392 }
 393
 394 function enable_kernel_lttng_event_fail ()
 395 {
 396         enable_kernel_lttng_event 1 1 "$@"
 397 }
 398
 399 function enable_kernel_lttng_event_notap ()
 400 {
 401         enable_kernel_lttng_event 0 0 "$@"
 402 }
 403
 404 # Old interface
 405 function lttng_enable_kernel_event
 406 {
 407         enable_kernel_lttng_event_ok "$@"
 408 }
 409
 410 function lttng_enable_kernel_syscall()
 411 {
 412         local expected_to_fail=$1
 413         local sess_name=$2
 414         local syscall_name=$3
 415         local channel_name=$4
 416
 417         if [ -z $syscall_name ]; then
 418                 # Enable all event if no syscall name specified
 419                 syscall_name="-a"
 420         fi
 421
 422         if [ -z $channel_name ]; then
 423                 # default channel if none specified
 424                 chan=""
 425         else
 426                 chan="-c $channel_name"
 427         fi
 428
 429         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 430         ret=$?
 431         if [[ $expected_to_fail -eq "1" ]]; then
 432                 test $ret -ne "0"
 433                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 434         else
 435                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 436         fi
 437 }
 438
 439 function lttng_enable_kernel_syscall_ok()
 440 {
 441         lttng_enable_kernel_syscall 0 "$@"
 442 }
 443
 444 function lttng_enable_kernel_syscall_fail()
 445 {
 446         lttng_enable_kernel_syscall 1 "$@"
 447 }
 448
 449 function lttng_disable_kernel_syscall()
 450 {
 451         local expected_to_fail=$1
 452         local sess_name=$2
 453         local syscall_name=$3
 454         local channel_name=$4
 455
 456         if [ -z $syscall_name ]; then
 457                 # Enable all event if no syscall name specified
 458                 syscall_name="-a"
 459         fi
 460
 461         if [ -z $channel_name ]; then
 462                 # default channel if none specified
 463                 chan=""
 464         else
 465                 chan="-c $channel_name"
 466         fi
 467
 468         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 469
 470         ret=$?
 471         if [[ $expected_to_fail -eq "1" ]]; then
 472                 test $ret -ne "0"
 473                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 474         else
 475                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 476         fi
 477 }
 478
 479 function lttng_disable_kernel_syscall_ok()
 480 {
 481         lttng_disable_kernel_syscall 0 "$@"
 482 }
 483
 484 function lttng_disable_kernel_syscall_fail()
 485 {
 486         lttng_disable_kernel_syscall 1 "$@"
 487 }
 488
 489 function lttng_enable_kernel_function_event ()
 490 {
 491         local expected_to_fail="$1"
 492         local sess_name="$2"
 493         local target="$3"
 494         local event_name="$4"
 495
 496         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 497         ret=$?
 498         if [[ $expected_to_fail -eq "1" ]]; then
 499                 test $ret -ne "0"
 500                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 501         else
 502                 ok $ret "Enable kernel function event for session $sess_name"
 503         fi
 504 }
 505
 506 function lttng_enable_kernel_function_event_ok ()
 507 {
 508         lttng_enable_kernel_function_event 0 "$@"
 509 }
 510
 511 function lttng_enable_kernel_userspace_probe_event ()
 512 {
 513         local expected_to_fail="$1"
 514         local sess_name="$2"
 515         local target="$3"
 516         local event_name="$4"
 517
 518         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 519         ret=$?
 520         if [[ $expected_to_fail -eq "1" ]]; then
 521                 test $ret -ne "0"
 522                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 523         else
 524                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 525         fi
 526 }
 527
 528 function lttng_enable_kernel_userspace_probe_event_fail ()
 529 {
 530         lttng_enable_kernel_userspace_probe_event 1 "$@"
 531 }
 532
 533 function lttng_enable_kernel_userspace_probe_event_ok ()
 534 {
 535         lttng_enable_kernel_userspace_probe_event 0 "$@"
 536 }
 537
 538 function disable_kernel_lttng_userspace_probe_event_ok ()
 539 {
 540         local sess_name="$1"
 541         local event_name="$2"
 542
 543         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 544         ok $? "Disable kernel event $target for session $sess_name"
 545 }
 546 function lttng_enable_kernel_channel()
 547 {
 548         local withtap=$1
 549         local expected_to_fail=$2
 550         local sess_name=$3
 551         local channel_name=$4
 552         local opts="${@:5}"
 553
 554         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 555         ret=$?
 556         if [[ $expected_to_fail -eq "1" ]]; then
 557                 test "$ret" -ne "0"
 558                 ret=$?
 559                 if [ $withtap -eq "1" ]; then
 560                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 561                 fi
 562         else
 563                 if [ $withtap -eq "1" ]; then
 564                         ok $ret "Enable channel $channel_name for session $sess_name"
 565                 fi
 566         fi
 567 }
 568
 569 function lttng_enable_kernel_channel_ok()
 570 {
 571         lttng_enable_kernel_channel 1 0 "$@"
 572 }
 573
 574 function lttng_enable_kernel_channel_fail()
 575 {
 576         lttng_enable_kernel_channel 1 1 "$@"
 577 }
 578
 579 function lttng_enable_kernel_channel_notap()
 580 {
 581         lttng_enable_kernel_channel 0 0 "$@"
 582 }
 583
 584 function enable_kernel_lttng_channel_ok()
 585 {
 586         lttng_enable_kernel_channel 1 0 "$@"
 587 }
 588
 589 function lttng_disable_kernel_channel()
 590 {
 591         local expected_to_fail=$1
 592         local sess_name=$2
 593         local channel_name=$3
 594
 595         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 596         ret=$?
 597         if [[ $expected_to_fail -eq "1" ]]; then
 598                 test "$ret" -ne "0"
 599                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 600         else
 601                 ok $ret "Disable channel $channel_name for session $sess_name"
 602         fi
 603 }
 604
 605 function lttng_disable_kernel_channel_ok()
 606 {
 607         lttng_disable_kernel_channel 0 "$@"
 608 }
 609
 610 function lttng_disable_kernel_channel_fail()
 611 {
 612         lttng_disable_kernel_channel 1 "$@"
 613 }
 614
 615 function start_lttng_relayd_opt()
 616 {
 617         local withtap=$1
 618         local process_mode=$2
 619         local opt=$3
 620
 621         DIR=$(readlink -f "$TESTDIR")
 622
 623         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 624                 # shellcheck disable=SC2086
 625                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 626                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 627                 if [ $? -eq 1 ]; then
 628                         if [ $withtap -eq "1" ]; then
 629                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 630                         fi
 631                         return 1
 632                 else
 633                         if [ $withtap -eq "1" ]; then
 634                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 635                         fi
 636                 fi
 637         else
 638                 pass "Start lttng-relayd (opt: $opt)"
 639         fi
 640 }
 641
 642 function start_lttng_relayd()
 643 {
 644         start_lttng_relayd_opt 1 "-b" "$@"
 645 }
 646
 647 function start_lttng_relayd_notap()
 648 {
 649         start_lttng_relayd_opt 0 "-b" "$@"
 650 }
 651
 652 function stop_lttng_relayd_opt()
 653 {
 654         local withtap=$1
 655         local is_cleanup=$2
 656         local signal=$3
 657         local timeout_s=$4
 658         local dtimeleft_s=
 659         local retval=0
 660         local pids
 661
 662         if [ -z "$signal" ]; then
 663                 signal="SIGTERM"
 664         fi
 665
 666
 667         # Multiply time by 2 to simplify integer arithmetic
 668         if [ -n "$timeout_s" ]; then
 669                 dtimeleft_s=$((timeout_s * 2))
 670         fi
 671
 672
 673         pids=$(lttng_pgrep "$RELAYD_MATCH")
 674         if [ -z "$pids" ]; then
 675                 if [ "$is_cleanup" -eq 1 ]; then
 676                         :
 677                 elif [ "$withtap" -eq "1" ]; then
 678                         fail "No relay daemon to kill"
 679                 else
 680                         LTTNG_BAIL_OUT "No relay daemon to kill"
 681                 fi
 682                 return 0
 683         fi
 684
 685         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 686
 687         # shellcheck disable=SC2086
 688         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 689                 retval=1
 690                 if [ "$withtap" -eq "1" ]; then
 691                         fail "Kill relay daemon"
 692                 fi
 693         else
 694                 out=1
 695                 while [ -n "$out" ]; do
 696                         out=$(lttng_pgrep "$RELAYD_MATCH")
 697                         if [ -n "$dtimeleft_s" ]; then
 698                                 if [ $dtimeleft_s -lt 0 ]; then
 699                                         out=
 700                                         retval=1
 701                                 fi
 702                                 dtimeleft_s=$((dtimeleft_s - 1))
 703                         fi
 704                         sleep 0.5
 705                 done
 706                 if [ "$withtap" -eq "1" ]; then
 707                         if [ "$retval" -eq "0" ]; then
 708                                 pass "Wait after kill relay daemon"
 709                         else
 710                                 fail "Wait after kill relay daemon"
 711                         fi
 712                 fi
 713         fi
 714         return $retval
 715 }
 716
 717 function stop_lttng_relayd()
 718 {
 719         stop_lttng_relayd_opt 1 0 "$@"
 720 }
 721
 722 function stop_lttng_relayd_notap()
 723 {
 724         stop_lttng_relayd_opt 0 0 "$@"
 725 }
 726
 727 function stop_lttng_relayd_cleanup()
 728 {
 729         stop_lttng_relayd_opt 0 1 "$@"
 730 }
 731
 732 #First arg: show tap output
 733 #Second argument: load path for automatic loading
 734 function start_lttng_sessiond_opt()
 735 {
 736         local withtap=$1
 737         local load_path=$2
 738
 739         # The rest of the arguments will be passed directly to lttng-sessiond.
 740         shift 2
 741
 742         local env_vars=""
 743         local consumerd=""
 744
 745         local long_bit_value=
 746         long_bit_value=$(getconf LONG_BIT)
 747
 748         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 749                 # Env variable requested no session daemon
 750                 return
 751         fi
 752
 753         DIR=$(readlink -f "$TESTDIR")
 754
 755         # Get long_bit value for 32/64 consumerd
 756         case "$long_bit_value" in
 757                 32)
 758                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 759                         ;;
 760                 64)
 761                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 762                         ;;
 763                 *)
 764                         return
 765                         ;;
 766         esac
 767
 768         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 769         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 770                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 771         fi
 772         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 773
 774         if ! validate_kernel_version; then
 775             fail "Start session daemon"
 776             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 777         fi
 778
 779         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 780         export LTTNG_SESSION_CONFIG_XSD_PATH
 781
 782         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 783                 # Have a load path ?
 784                 if [ -n "$load_path" ]; then
 785                         # shellcheck disable=SC2086
 786                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 787                 else
 788                         # shellcheck disable=SC2086
 789                         env $env_vars --background "$consumerd" "$@"
 790                 fi
 791                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 792                 status=$?
 793                 if [ "$withtap" -eq "1" ]; then
 794                         ok $status "Start session daemon"
 795                 fi
 796         fi
 797 }
 798
 799 function start_lttng_sessiond()
 800 {
 801         start_lttng_sessiond_opt 1 "$@"
 802 }
 803
 804 function start_lttng_sessiond_notap()
 805 {
 806         start_lttng_sessiond_opt 0 "$@"
 807 }
 808
 809 function stop_lttng_sessiond_opt()
 810 {
 811         local withtap=$1
 812         local is_cleanup=$2
 813         local signal=$3
 814         local timeout_s=$4
 815         local dtimeleft_s=
 816         local retval=0
 817         local runas_pids
 818         local pids
 819
 820         if [ -z "$signal" ]; then
 821                 signal=SIGTERM
 822         fi
 823
 824         # Multiply time by 2 to simplify integer arithmetic
 825         if [ -n "$timeout_s" ]; then
 826                 dtimeleft_s=$((timeout_s * 2))
 827         fi
 828
 829         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 830                 # Env variable requested no session daemon
 831                 return 0
 832         fi
 833
 834         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 835         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 836
 837         if [ -n "$runas_pids" ]; then
 838                 pids="$pids $runas_pids"
 839         fi
 840
 841         if [ -z "$pids" ]; then
 842                 if [ "$is_cleanup" -eq 1 ]; then
 843                         :
 844                 elif [ "$withtap" -eq "1" ]; then
 845                         fail "No session daemon to kill"
 846                 else
 847                         LTTNG_BAIL_OUT "No session daemon to kill"
 848                 fi
 849                 return 0
 850         fi
 851
 852         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 853
 854         # shellcheck disable=SC2086
 855         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 856                 retval=1
 857                 if [ "$withtap" -eq "1" ]; then
 858                         fail "Kill sessions daemon"
 859                 fi
 860         else
 861                 out=1
 862                 while [ -n "$out" ]; do
 863                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 864                         if [ -n "$dtimeleft_s" ]; then
 865                                 if [ $dtimeleft_s -lt 0 ]; then
 866                                         out=
 867                                         retval=1
 868                                 fi
 869                                 dtimeleft_s=$((dtimeleft_s - 1))
 870                         fi
 871                         sleep 0.5
 872                 done
 873                 out=1
 874                 while [ -n "$out" ]; do
 875                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 876                         if [ -n "$dtimeleft_s" ]; then
 877                                 if [ $dtimeleft_s -lt 0 ]; then
 878                                         out=
 879                                         retval=1
 880                                 fi
 881                                 dtimeleft_s=$((dtimeleft_s - 1))
 882                         fi
 883                         sleep 0.5
 884                 done
 885
 886                 if [ "$withtap" -eq "1" ]; then
 887                         if [ "$retval" -eq "0" ]; then
 888                                 pass "Wait after kill session daemon"
 889                         else
 890                                 fail "Wait after kill session daemon"
 891                         fi
 892                 fi
 893         fi
 894         if [ "$signal" = "SIGKILL" ]; then
 895                 if [ "$(id -u)" -eq "0" ]; then
 896                         local modules=
 897                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 898
 899                         if [ -n "$modules" ]; then
 900                                 diag "Unloading all LTTng modules"
 901                                 modprobe --remove "$modules"
 902                         fi
 903                 fi
 904         fi
 905
 906         return $retval
 907 }
 908
 909 function stop_lttng_sessiond()
 910 {
 911         stop_lttng_sessiond_opt 1 0 "$@"
 912 }
 913
 914 function stop_lttng_sessiond_notap()
 915 {
 916         stop_lttng_sessiond_opt 0 0 "$@"
 917 }
 918
 919 function stop_lttng_sessiond_cleanup()
 920 {
 921         stop_lttng_sessiond_opt 0 1 "$@"
 922 }
 923
 924 function sigstop_lttng_sessiond_opt()
 925 {
 926         local withtap=$1
 927         local signal=SIGSTOP
 928         local pids
 929
 930         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 931                 # Env variable requested no session daemon
 932                 return
 933         fi
 934
 935         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 936
 937         if [ "$withtap" -eq "1" ]; then
 938                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 939         fi
 940
 941         # shellcheck disable=SC2086
 942         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 943                 if [ "$withtap" -eq "1" ]; then
 944                         fail "Sending SIGSTOP to session daemon"
 945                 fi
 946         else
 947                 out=1
 948                 while [ $out -ne 0 ]; do
 949                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 950
 951                         # Wait until state becomes stopped for session
 952                         # daemon(s).
 953                         out=0
 954                         for sessiond_pid in $pids; do
 955                                 state="$(ps -p "$sessiond_pid" -o state= )"
 956                                 if [[ -n "$state" && "$state" != "T" ]]; then
 957                                         out=1
 958                                 fi
 959                         done
 960                         sleep 0.5
 961                 done
 962                 if [ "$withtap" -eq "1" ]; then
 963                         pass "Sending SIGSTOP to session daemon"
 964                 fi
 965         fi
 966 }
 967
 968 function sigstop_lttng_sessiond()
 969 {
 970         sigstop_lttng_sessiond_opt 1 "$@"
 971 }
 972
 973 function sigstop_lttng_sessiond_notap()
 974 {
 975         sigstop_lttng_sessiond_opt 0 "$@"
 976 }
 977
 978 function stop_lttng_consumerd_opt()
 979 {
 980         local withtap=$1
 981         local is_cleanup=$2
 982         local signal=$3
 983         local timeout_s=$4
 984         local dtimeleft_s=
 985         local retval=0
 986         local pids
 987
 988         if [ -z "$signal" ]; then
 989                 signal=SIGTERM
 990         fi
 991
 992         # Multiply time by 2 to simplify integer arithmetic
 993         if [ -n "$timeout_s" ]; then
 994                 dtimeleft_s=$((timeout_s * 2))
 995         fi
 996
 997         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 998
 999         if [ -z "$pids" ]; then
1000                 if [ "$is_cleanup" -eq 1 ]; then
1001                         :
1002                 elif [ "$withtap" -eq "1" ]; then
1003                         fail "No consumerd daemon to kill"
1004                 else
1005                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
1006                 fi
1007                 return 0
1008         fi
1009
1010         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1011
1012         # shellcheck disable=SC2086
1013         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
1014                 retval=1
1015                 if [ "$withtap" -eq "1" ]; then
1016                         fail "Kill consumer daemon"
1017                 fi
1018         else
1019                 out=1
1020                 while [ $out -ne 0 ]; do
1021                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1022
1023                         # If consumerds are still present check their status.
1024                         # A zombie status qualifies the consumerd as *killed*
1025                         out=0
1026                         for consumer_pid in $pids; do
1027                                 state="$(ps -p "$consumer_pid" -o state= )"
1028                                 if [[ -n "$state" && "$state" != "Z" ]]; then
1029                                         out=1
1030                                 fi
1031                         done
1032                         if [ -n "$dtimeleft_s" ]; then
1033                                 if [ $dtimeleft_s -lt 0 ]; then
1034                                         out=0
1035                                         retval=1
1036                                 fi
1037                                 dtimeleft_s=$((dtimeleft_s - 1))
1038                         fi
1039                         sleep 0.5
1040                 done
1041                 if [ "$withtap" -eq "1" ]; then
1042                         if [ "$retval" -eq "0" ]; then
1043                                 pass "Wait after kill consumer daemon"
1044                         else
1045                                 fail "Wait after kill consumer daemon"
1046                         fi
1047                 fi
1048         fi
1049
1050         return $retval
1051 }
1052
1053 function stop_lttng_consumerd()
1054 {
1055         stop_lttng_consumerd_opt 1 0 "$@"
1056 }
1057
1058 function stop_lttng_consumerd_notap()
1059 {
1060         stop_lttng_consumerd_opt 0 0 "$@"
1061 }
1062
1063 function stop_lttng_consumerd_cleanup()
1064 {
1065         stop_lttng_consumerd_opt 0 1 "$@"
1066 }
1067
1068 function sigstop_lttng_consumerd_opt()
1069 {
1070         local withtap=$1
1071         local signal=SIGSTOP
1072         local pids
1073
1074         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1075
1076         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1077
1078         # shellcheck disable=SC2086
1079         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1080         retval=$?
1081
1082         if [ $retval -eq 1 ]; then
1083                 if [ "$withtap" -eq "1" ]; then
1084                         fail "Sending SIGSTOP to consumer daemon"
1085                 fi
1086                 return 1
1087         else
1088                 out=1
1089                 while [ $out -ne 0 ]; do
1090                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1091
1092                         # Wait until state becomes stopped for all
1093                         # consumers.
1094                         out=0
1095                         for consumer_pid in $pids; do
1096                                 state="$(ps -p "$consumer_pid" -o state= )"
1097                                 if [[ -n "$state" && "$state" != "T" ]]; then
1098                                         out=1
1099                                 fi
1100                         done
1101                         sleep 0.5
1102                 done
1103                 if [ "$withtap" -eq "1" ]; then
1104                         pass "Sending SIGSTOP to consumer daemon"
1105                 fi
1106         fi
1107         return $retval
1108 }
1109
1110 function sigstop_lttng_consumerd()
1111 {
1112         sigstop_lttng_consumerd_opt 1 "$@"
1113 }
1114
1115 function sigstop_lttng_consumerd_notap()
1116 {
1117         sigstop_lttng_consumerd_opt 0 "$@"
1118 }
1119
1120 function list_lttng_with_opts ()
1121 {
1122         local ret
1123         local withtap=$1
1124         shift
1125         local opts=$1
1126         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1127         ret=$?
1128         if [ $withtap -eq "1" ]; then
1129                 ok $ret "Lttng-tool list command with option $opts"
1130         fi
1131 }
1132
1133 function list_lttng_ok ()
1134 {
1135         list_lttng_with_opts 1 "$@"
1136 }
1137
1138 function list_lttng_notap ()
1139 {
1140         list_lttng_with_opts 0 "$@"
1141 }
1142
1143 function create_lttng_session_no_output ()
1144 {
1145         local sess_name=$1
1146         local opts="${@:2}"
1147
1148         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1149         ok $? "Create session $sess_name in no-output mode"
1150 }
1151
1152 function create_lttng_session_uri () {
1153         local sess_name=$1
1154         local uri=$2
1155         local opts="${@:3}"
1156
1157         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1158         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1159 }
1160
1161 function create_lttng_session ()
1162 {
1163         local withtap=$1
1164         local expected_to_fail=$2
1165         local sess_name=$3
1166         local trace_path=$4
1167         local opt=$5
1168
1169         if [ -z "$trace_path" ]; then
1170                 # Use lttng-sessiond default output.
1171                 trace_path=""
1172         else
1173                 trace_path="-o $trace_path"
1174         fi
1175
1176         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create "$sess_name" $trace_path $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1177         ret=$?
1178         if [ $expected_to_fail -eq "1" ]; then
1179                 test "$ret" -ne "0"
1180                 ret=$?
1181                 if [ $withtap -eq "1" ]; then
1182                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1183                 fi
1184         else
1185                 if [ $withtap -eq "1" ]; then
1186                         ok $ret "Create session $sess_name in $trace_path"
1187                 fi
1188         fi
1189         return $ret
1190 }
1191
1192 function create_lttng_session_ok ()
1193 {
1194         create_lttng_session 1 0 "$@"
1195 }
1196
1197 function create_lttng_session_fail ()
1198 {
1199         create_lttng_session 1 1 "$@"
1200 }
1201
1202 function create_lttng_session_notap ()
1203 {
1204         create_lttng_session 0 0 "$@"
1205 }
1206
1207
1208 function enable_ust_lttng_channel ()
1209 {
1210         local withtap=$1
1211         local expected_to_fail=$2
1212         local sess_name=$3
1213         local channel_name=$4
1214         local opts="${@:5}"
1215
1216         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1217         ret=$?
1218         if [[ $expected_to_fail -eq "1" ]]; then
1219                 test "$ret" -ne "0"
1220                 ret=$?
1221                 if [ $withtap -eq "1" ]; then
1222                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1223                 fi
1224         else
1225                 if [ $withtap -eq "1" ]; then
1226                         ok $ret "Enable channel $channel_name for session $sess_name"
1227                 fi
1228         fi
1229         return $ret
1230 }
1231
1232 function enable_ust_lttng_channel_ok ()
1233 {
1234         enable_ust_lttng_channel 1 0 "$@"
1235 }
1236
1237 function enable_ust_lttng_channel_fail ()
1238 {
1239         enable_ust_lttng_channel 1 1 "$@"
1240 }
1241
1242 function enable_ust_lttng_channel_notap ()
1243 {
1244         enable_ust_lttng_channel 0 0 "$@"
1245 }
1246
1247 function disable_ust_lttng_channel()
1248 {
1249         local sess_name=$1
1250         local channel_name=$2
1251
1252         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1253         ok $? "Disable channel $channel_name for session $sess_name"
1254 }
1255
1256 function enable_lttng_mmap_overwrite_kernel_channel()
1257 {
1258         local sess_name=$1
1259         local channel_name=$2
1260
1261         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1262         ok $? "Enable channel $channel_name for session $sess_name"
1263 }
1264
1265 function enable_lttng_mmap_discard_small_kernel_channel()
1266 {
1267         local sess_name=$1
1268         local channel_name=$2
1269
1270         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1271         ok $? "Enable small discard channel $channel_name for session $sess_name"
1272 }
1273
1274 function enable_lttng_mmap_overwrite_small_kernel_channel()
1275 {
1276         local sess_name=$1
1277         local channel_name=$2
1278
1279         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1280         ok $? "Enable small discard channel $channel_name for session $sess_name"
1281 }
1282
1283 function enable_lttng_mmap_overwrite_ust_channel()
1284 {
1285         local sess_name=$1
1286         local channel_name=$2
1287
1288         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1289         ok $? "Enable channel $channel_name for session $sess_name"
1290 }
1291
1292 function enable_ust_lttng_event ()
1293 {
1294         local withtap=$1
1295         local expected_to_fail=$2
1296         local sess_name=$3
1297         local event_name="$4"
1298         local channel_name=$5
1299
1300         if [ -z $channel_name ]; then
1301                 # default channel if none specified
1302                 chan=""
1303         else
1304                 chan="-c $channel_name"
1305         fi
1306
1307         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1308         ret=$?
1309         if [[ $expected_to_fail -eq "1" ]]; then
1310                 test $ret -ne "0"
1311                 ret=$?
1312                 if [[ $withtap -eq "1" ]]; then
1313                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1314                 fi
1315         else
1316                 if [[ $withtap -eq "1" ]]; then
1317                         ok $ret "Enable ust event $event_name for session $sess_name"
1318                 fi
1319         fi
1320         return $ret
1321 }
1322
1323 function enable_ust_lttng_event_ok ()
1324 {
1325         enable_ust_lttng_event 1 0 "$@"
1326 }
1327
1328 function enable_ust_lttng_event_fail ()
1329 {
1330         enable_ust_lttng_event 1 1 "$@"
1331 }
1332
1333 function enable_ust_lttng_event_notap ()
1334 {
1335         enable_ust_lttng_event 0 0 "$@"
1336 }
1337
1338 function enable_jul_lttng_event()
1339 {
1340         sess_name=$1
1341         event_name="$2"
1342         channel_name=$3
1343
1344         if [ -z $channel_name ]; then
1345                 # default channel if none specified
1346                 chan=""
1347         else
1348                 chan="-c $channel_name"
1349         fi
1350
1351         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1352         ok $? "Enable JUL event $event_name for session $sess_name"
1353 }
1354
1355 function enable_jul_lttng_event_loglevel()
1356 {
1357         local sess_name=$1
1358         local event_name="$2"
1359         local loglevel=$3
1360         local channel_name=$4
1361
1362         if [ -z $channel_name ]; then
1363                 # default channel if none specified
1364                 chan=""
1365         else
1366                 chan="-c $channel_name"
1367         fi
1368
1369         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1370         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1371 }
1372
1373 function enable_log4j_lttng_event()
1374 {
1375         local sess_name=$1
1376         local event_name=$2
1377         local channel_name=$3
1378
1379         local chan_opt=()
1380
1381         # default channel if none specified
1382         if [ -n "$channel_name" ]; then
1383                 chan_opt=("-c" "$channel_name")
1384         fi
1385
1386         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1387                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1388         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1389 }
1390
1391 function enable_log4j_lttng_event_filter()
1392 {
1393         local sess_name=$1
1394         local event_name=$2
1395         local filter=$3
1396
1397         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1398                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1399         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1400 }
1401
1402 function enable_log4j_lttng_event_filter_loglevel_only()
1403 {
1404         local sess_name=$1
1405         local event_name=$2
1406         local filter=$3
1407         local loglevel=$4
1408
1409         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1410                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1411         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1412 }
1413
1414 function enable_log4j_lttng_event_loglevel()
1415 {
1416         local sess_name=$1
1417         local event_name=$2
1418         local loglevel=$3
1419         local channel_name=$4
1420
1421
1422         # default channel if none specified
1423         if [ -n "$channel_name" ]; then
1424                 chan_opt=("-c" "$channel_name")
1425         fi
1426
1427         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1428                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1429         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1430 }
1431
1432 function enable_log4j_lttng_event_loglevel_only()
1433 {
1434         local sess_name=$1
1435         local event_name=$2
1436         local loglevel=$3
1437         local channel_name=$4
1438
1439         local chan_opt=()
1440
1441         # default channel if none specified
1442         if [ -n "$channel_name" ]; then
1443                 chan_opt=("-c" "$channel_name")
1444         fi
1445
1446         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1447                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1448         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1449 }
1450
1451 function enable_python_lttng_event()
1452 {
1453         sess_name=$1
1454         event_name="$2"
1455         channel_name=$3
1456
1457         if [ -z $channel_name ]; then
1458                 # default channel if none specified
1459                 chan=""
1460         else
1461                 chan="-c $channel_name"
1462         fi
1463
1464         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1465         ok $? "Enable Python event $event_name for session $sess_name"
1466 }
1467
1468 function enable_python_lttng_event_loglevel()
1469 {
1470         local sess_name=$1
1471         local event_name="$2"
1472         local loglevel=$3
1473         local channel_name=$4
1474
1475         if [ -z $channel_name ]; then
1476                 # default channel if none specified
1477                 chan=""
1478         else
1479                 chan="-c $channel_name"
1480         fi
1481
1482         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1483         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1484 }
1485
1486 function enable_ust_lttng_event_filter()
1487 {
1488         local sess_name="$1"
1489         local event_name="$2"
1490         local filter="$3"
1491         local channel_name=$4
1492
1493         if [ -z $channel_name ]; then
1494                 # default channel if none specified
1495                 chan=""
1496         else
1497                 chan="-c $channel_name"
1498         fi
1499
1500         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event $chan "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1501         ok $? "Enable event $event_name with filtering for session $sess_name"
1502 }
1503
1504 function enable_ust_lttng_event_loglevel()
1505 {
1506         local sess_name="$1"
1507         local event_name="$2"
1508         local loglevel="$3"
1509         local channel_name="$4"
1510         local chan=()
1511         if [ -n "${channel_name}" ] ; then
1512                 chan=('-c' "${channel_name}")
1513         fi
1514
1515         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "${chan[@]}" "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1516         ok $? "Enable event $event_name with loglevel $loglevel"
1517 }
1518
1519 function enable_ust_lttng_event_loglevel_only()
1520 {
1521         local sess_name="$1"
1522         local event_name="$2"
1523         local loglevel="$3"
1524         local channel_name="$4"
1525         local chan=()
1526         if [ -n "${channel_name}" ] ; then
1527                 chan=('-c' "${channel_name}")
1528         fi
1529
1530         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "${chan[@]}" "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1531         ok $? "Enable event $event_name with loglevel-only $loglevel"
1532 }
1533
1534 function disable_ust_lttng_event ()
1535 {
1536         local sess_name="$1"
1537         local event_name="$2"
1538         local channel_name="$3"
1539
1540         if [ -z $channel_name ]; then
1541                 # default channel if none specified
1542                 chan=""
1543         else
1544                 chan="-c $channel_name"
1545         fi
1546
1547         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1548         ok $? "Disable event $event_name for session $sess_name"
1549 }
1550
1551 function disable_jul_lttng_event ()
1552 {
1553         local sess_name="$1"
1554         local event_name="$2"
1555
1556         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1557         ok $? "Disable JUL event $event_name for session $sess_name"
1558 }
1559
1560 function disable_log4j_lttng_event ()
1561 {
1562         local sess_name="$1"
1563         local event_name="$2"
1564
1565         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1566                 disable-event "$event_name" -s "$sess_name" --log4j
1567         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1568 }
1569
1570 function disable_python_lttng_event ()
1571 {
1572         local sess_name="$1"
1573         local event_name="$2"
1574
1575         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1576         ok $? "Disable Python event $event_name for session $sess_name"
1577 }
1578
1579 function start_lttng_tracing_opt ()
1580 {
1581         local withtap=$1
1582         local expected_to_fail=$2
1583         local sess_name=$3
1584
1585         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1586         ret=$?
1587         if [[ $expected_to_fail -eq "1" ]]; then
1588                 test "$ret" -ne "0"
1589                 ret=$?
1590                 if [ $withtap -eq "1" ]; then
1591                         ok $? "Start tracing for session $sess_name failed as expected"
1592                 fi
1593         else
1594                 if [ $withtap -eq "1" ]; then
1595                         ok $ret "Start tracing for session $sess_name"
1596                 fi
1597         fi
1598 }
1599
1600 function start_lttng_tracing_ok ()
1601 {
1602         start_lttng_tracing_opt 1 0 "$@"
1603 }
1604
1605 function start_lttng_tracing_fail ()
1606 {
1607         start_lttng_tracing_opt 1 1 "$@"
1608 }
1609
1610 function start_lttng_tracing_notap ()
1611 {
1612         start_lttng_tracing_opt 0 1 "$@"
1613 }
1614
1615 function stop_lttng_tracing_opt ()
1616 {
1617         local withtap=$1
1618         local expected_to_fail=$2
1619         local sess_name=$3
1620
1621         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1622         ret=$?
1623         if [[ $expected_to_fail -eq "1" ]]; then
1624                 test "$ret" -ne "0"
1625                 ret=$?
1626                 if [ $withtap -eq "1" ]; then
1627                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1628                 fi
1629         else
1630                 if [ $withtap -eq "1" ]; then
1631                         ok $ret "Stop lttng tracing for session $sess_name"
1632                 fi
1633         fi
1634 }
1635
1636 function stop_lttng_tracing_ok ()
1637 {
1638         stop_lttng_tracing_opt 1 0 "$@"
1639 }
1640
1641 function stop_lttng_tracing_fail ()
1642 {
1643         stop_lttng_tracing_opt 1 1 "$@"
1644 }
1645
1646 function stop_lttng_tracing_notap ()
1647 {
1648         stop_lttng_tracing_opt 0 0 "$@"
1649 }
1650
1651 function destroy_lttng_session ()
1652 {
1653         local withtap=$1
1654         local expected_to_fail=$2
1655         local sess_name=$3
1656
1657         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1658         ret=$?
1659         if [[ $expected_to_fail -eq "1" ]]; then
1660                 test "$ret" -ne "0"
1661                 ret=$?
1662                 if [ $withtap -eq "1" ]; then
1663                         ok $ret "Destroy session $sess_name failed as expected"
1664                 fi
1665         else
1666                 if [ $withtap -eq "1" ]; then
1667                         ok $ret "Destroy session $sess_name"
1668                 fi
1669         fi
1670 }
1671
1672 function destroy_lttng_session_ok ()
1673 {
1674         destroy_lttng_session 1 0 "$@"
1675
1676 }
1677
1678 function destroy_lttng_session_fail ()
1679 {
1680         destroy_lttng_session 1 1 "$@"
1681 }
1682
1683 function destroy_lttng_session_notap ()
1684 {
1685         destroy_lttng_session 0 0 "$@"
1686 }
1687
1688 function destroy_lttng_sessions ()
1689 {
1690         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1691         ok $? "Destroy all lttng sessions"
1692 }
1693
1694 function lttng_snapshot_add_output ()
1695 {
1696         local expected_to_fail=$1
1697         local sess_name=$2
1698         local trace_path=$3
1699         local opts=$4
1700
1701         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1702         ret=$?
1703         if [[ $expected_to_fail -eq 1 ]]; then
1704                 test "$ret" -ne "0"
1705                 ok $? "Added snapshot output $trace_path failed as expected"
1706         else
1707                 ok $ret "Added snapshot output $trace_path"
1708         fi
1709 }
1710
1711 function lttng_snapshot_add_output_ok ()
1712 {
1713         lttng_snapshot_add_output 0 "$@"
1714 }
1715
1716 function lttng_snapshot_add_output_fail ()
1717 {
1718         lttng_snapshot_add_output 1 "$@"
1719 }
1720
1721 function lttng_snapshot_del_output ()
1722 {
1723         local expected_to_fail=$1
1724         local sess_name=$2
1725         local id=$3
1726
1727         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1728         ret=$?
1729         if [[ $expected_to_fail -eq "1" ]]; then
1730                 test "$ret" -ne "0"
1731                 ok $? "Deleted snapshot output id $id failed as expected"
1732         else
1733                 ok $ret "Deleted snapshot output id $id"
1734         fi
1735 }
1736
1737 function lttng_snapshot_del_output_ok ()
1738 {
1739         lttng_snapshot_del_output 0 "$@"
1740 }
1741
1742 function lttng_snapshot_del_output_fail ()
1743 {
1744         lttng_snapshot_del_output 1 "$@"
1745 }
1746
1747 function lttng_snapshot_record ()
1748 {
1749         local sess_name=$1
1750         local trace_path=$2
1751
1752         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s "$sess_name" "$trace_path" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1753         ok $? "Snapshot recorded"
1754 }
1755
1756 function lttng_snapshot_list ()
1757 {
1758         local sess_name=$1
1759         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1760         ok $? "Snapshot list"
1761 }
1762
1763 function lttng_save()
1764 {
1765         local sess_name=$1
1766         local opts=$2
1767
1768         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1769         ok $? "Session saved"
1770 }
1771
1772 function lttng_load()
1773 {
1774         local expected_to_fail=$1
1775         local opts=$2
1776
1777         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1778         ret=$?
1779         if [[ $expected_to_fail -eq "1" ]]; then
1780                 test $ret -ne "0"
1781                 ok $? "Load command failed as expected with opts: $opts"
1782         else
1783                 ok $ret "Load command with opts: $opts"
1784         fi
1785 }
1786
1787 function lttng_load_ok()
1788 {
1789         lttng_load 0 "$@"
1790 }
1791
1792 function lttng_load_fail()
1793 {
1794         lttng_load 1 "$@"
1795 }
1796
1797 function lttng_track()
1798 {
1799         local expected_to_fail="$1"
1800         shift 1
1801         local opts="$@"
1802         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1803         ret=$?
1804         if [[ $expected_to_fail -eq "1" ]]; then
1805                 test $ret -ne "0"
1806                 ok $? "Track command failed as expected with opts: $opts"
1807         else
1808                 ok $ret "Track command with opts: $opts"
1809         fi
1810 }
1811
1812 function lttng_track_ok()
1813 {
1814         lttng_track 0 "$@"
1815 }
1816
1817 function lttng_track_fail()
1818 {
1819         lttng_track 1 "$@"
1820 }
1821
1822 function lttng_untrack()
1823 {
1824         local expected_to_fail="$1"
1825         shift 1
1826         local opts="$@"
1827         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1828         ret=$?
1829         if [[ $expected_to_fail -eq "1" ]]; then
1830                 test $ret -ne "0"
1831                 ok $? "Untrack command failed as expected with opts: $opts"
1832         else
1833                 ok $ret "Untrack command with opts: $opts"
1834         fi
1835 }
1836
1837 function lttng_untrack_ok()
1838 {
1839         lttng_untrack 0 "$@"
1840 }
1841
1842 function lttng_untrack_fail()
1843 {
1844         lttng_untrack 1 "$@"
1845 }
1846
1847 function lttng_track_pid_ok()
1848 {
1849         PID=$1
1850         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1851         ok $? "Lttng track pid on the kernel domain"
1852 }
1853
1854 function lttng_untrack_kernel_all_ok()
1855 {
1856         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1857         ok $? "Lttng untrack all pid on the kernel domain"
1858 }
1859
1860 function lttng_track_ust_ok()
1861 {
1862         lttng_track_ok -u "$@"
1863 }
1864
1865 function lttng_track_ust_fail()
1866 {
1867         lttng_track_fail -u "$@"
1868 }
1869
1870 function lttng_track_kernel_ok()
1871 {
1872         lttng_track_ok -k "$@"
1873 }
1874
1875 function lttng_track_kernel_fail()
1876 {
1877         lttng_track_fail -k "$@"
1878 }
1879
1880 function lttng_untrack_ust_ok()
1881 {
1882         lttng_untrack_ok -u "$@"
1883 }
1884
1885 function lttng_untrack_ust_fail()
1886 {
1887         lttng_untrack_fail -u "$@"
1888 }
1889
1890 function lttng_untrack_kernel_ok()
1891 {
1892         lttng_untrack_ok -k "$@"
1893 }
1894
1895 function lttng_untrack_kernel_fail()
1896 {
1897         lttng_untrack_fail -k "$@"
1898 }
1899
1900 function lttng_add_context_list()
1901 {
1902         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1903         ret=$?
1904         ok $ret "Context listing"
1905 }
1906
1907 function add_context_lttng()
1908 {
1909         local expected_to_fail="$1"
1910         local domain="$2"
1911         local session_name="$3"
1912         local channel_name="$4"
1913         local type="$5"
1914
1915         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1916         ret=$?
1917         if [[ $expected_to_fail -eq "1" ]]; then
1918                 test $ret -ne "0"
1919                 ok $? "Add context command failed as expected for type: $type"
1920         else
1921                 ok $ret "Add context command for type: $type"
1922         fi
1923 }
1924
1925 function add_context_ust_ok()
1926 {
1927         add_context_lttng 0 -u "$@"
1928 }
1929
1930 function add_context_ust_fail()
1931 {
1932         add_context_lttng 1 -u "$@"
1933 }
1934
1935 function add_context_kernel_ok()
1936 {
1937         add_context_lttng 0 -k "$@"
1938 }
1939
1940 function add_context_kernel_fail()
1941 {
1942         add_context_lttng 1 -k "$@"
1943 }
1944
1945 function wait_live_trace_ready ()
1946 {
1947         local url=$1
1948         local zero_client_match=0
1949
1950         diag "Waiting for live trace at url: $url"
1951         while [ $zero_client_match -eq 0 ]; do
1952                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1953                 sleep 0.5
1954         done
1955         pass "Waiting for live trace at url: $url"
1956 }
1957
1958 function wait_live_viewer_connect ()
1959 {
1960         local url=$1
1961         local one_client_match=0
1962
1963         diag "Waiting for live viewers on url: $url"
1964         while [ $one_client_match -eq 0 ]; do
1965                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1966                 sleep 0.5
1967         done
1968         pass "Waiting for live viewers on url: $url"
1969 }
1970
1971 function bail_out_if_no_babeltrace()
1972 {
1973         which "$BABELTRACE_BIN" >/dev/null
1974         if [ $? -ne 0 ]; then
1975                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
1976         fi
1977 }
1978
1979 function validate_metadata_event ()
1980 {
1981         local event_name=$1
1982         local nr_event_id=$2
1983         local trace_path=$3
1984
1985         local metadata_file=$(find $trace_path -name "metadata")
1986         local metadata_path=$(dirname $metadata_file)
1987
1988         which $BABELTRACE_BIN >/dev/null
1989         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1990
1991         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1992
1993         if [ "$count" -ne "$nr_event_id" ]; then
1994                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1995                 diag "$count matching event id found in metadata"
1996         else
1997                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1998         fi
1999
2000 }
2001
2002 function trace_matches ()
2003 {
2004         local event_name=$1
2005         local nr_iter=$2
2006         local trace_path=$3
2007
2008         which $BABELTRACE_BIN >/dev/null
2009         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
2010
2011         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
2012
2013         if [ "$count" -ne "$nr_iter" ]; then
2014                 fail "Trace match"
2015                 diag "$count matching events found in trace"
2016         else
2017                 pass "Trace match"
2018         fi
2019 }
2020
2021 function trace_match_only()
2022 {
2023         local event_name=$1
2024         local nr_iter=$2
2025         local trace_path=$3
2026
2027         which $BABELTRACE_BIN >/dev/null
2028         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
2029
2030         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
2031         local total=$($BABELTRACE_BIN $trace_path | wc -l)
2032
2033         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
2034                 pass "Trace match with $total event $event_name"
2035         else
2036                 fail "Trace match"
2037                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
2038         fi
2039 }
2040
2041 function validate_trace
2042 {
2043         local event_name=$1
2044         local trace_path=$2
2045
2046         which $BABELTRACE_BIN >/dev/null
2047         if [ $? -ne 0 ]; then
2048             skip 0 "Babeltrace binary not found. Skipping trace validation"
2049         fi
2050
2051         OLDIFS=$IFS
2052         IFS=","
2053         for i in $event_name; do
2054                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2055                 if [ "$traced" -ne 0 ]; then
2056                         pass "Validate trace for event $i, $traced events"
2057                 else
2058                         fail "Validate trace for event $i"
2059                         diag "Found $traced occurrences of $i"
2060                 fi
2061         done
2062         ret=$?
2063         IFS=$OLDIFS
2064         return $ret
2065 }
2066
2067 function validate_trace_count
2068 {
2069         local event_name=$1
2070         local trace_path=$2
2071         local expected_count=$3
2072
2073         which $BABELTRACE_BIN >/dev/null
2074         if [ $? -ne 0 ]; then
2075             skip 0 "Babeltrace binary not found. Skipping trace validation"
2076         fi
2077
2078         cnt=0
2079         OLDIFS=$IFS
2080         IFS=","
2081         for i in $event_name; do
2082                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2083                 if [ "$traced" -ne 0 ]; then
2084                         pass "Validate trace for event $i, $traced events"
2085                 else
2086                         fail "Validate trace for event $i"
2087                         diag "Found $traced occurrences of $i"
2088                 fi
2089                 cnt=$(($cnt + $traced))
2090         done
2091         IFS=$OLDIFS
2092         test $cnt -eq $expected_count
2093         ok $? "Read a total of $cnt events, expected $expected_count"
2094 }
2095
2096 function validate_trace_count_range_incl_min_excl_max
2097 {
2098         local event_name=$1
2099         local trace_path=$2
2100         local expected_min=$3
2101         local expected_max=$4
2102
2103         which $BABELTRACE_BIN >/dev/null
2104         if [ $? -ne 0 ]; then
2105             skip 0 "Babeltrace binary not found. Skipping trace validation"
2106         fi
2107
2108         cnt=0
2109         OLDIFS=$IFS
2110         IFS=","
2111         for i in $event_name; do
2112                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2113                 if [ "$traced" -ge $expected_min ]; then
2114                         pass "Validate trace for event $i, $traced events"
2115                 else
2116                         fail "Validate trace for event $i"
2117                         diag "Found $traced occurrences of $i"
2118                 fi
2119                 cnt=$(($cnt + $traced))
2120         done
2121         IFS=$OLDIFS
2122         test $cnt -lt $expected_max
2123         ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
2124 }
2125
2126 function trace_first_line
2127 {
2128         local trace_path=$1
2129
2130         which $BABELTRACE_BIN >/dev/null
2131         if [ $? -ne 0 ]; then
2132             skip 0 "Babeltrace binary not found. Skipping trace validation"
2133         fi
2134
2135         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
2136 }
2137
2138 function validate_trace_exp()
2139 {
2140         local event_exp=$1
2141         local trace_path=$2
2142
2143         which $BABELTRACE_BIN >/dev/null
2144         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
2145
2146         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
2147         if [ "$traced" -ne 0 ]; then
2148                 pass "Validate trace for expression '${event_exp}', $traced events"
2149         else
2150                 fail "Validate trace for expression '${event_exp}'"
2151                 diag "Found $traced occurrences of '${event_exp}'"
2152         fi
2153         ret=$?
2154         return $ret
2155 }
2156
2157 function validate_trace_only_exp()
2158 {
2159         local event_exp=$1
2160         local trace_path=$2
2161
2162         which $BABELTRACE_BIN >/dev/null
2163         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
2164
2165         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
2166         local total=$($BABELTRACE_BIN $trace_path | wc -l)
2167
2168         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
2169                 pass "Trace match with $total for expression '${event_exp}'"
2170         else
2171                 fail "Trace match"
2172                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
2173         fi
2174         ret=$?
2175         return $ret
2176 }
2177
2178 function validate_trace_empty()
2179 {
2180         local trace_path=$1
2181
2182         which $BABELTRACE_BIN >/dev/null
2183         if [ $? -ne 0 ]; then
2184             skip 0 "Babeltrace binary not found. Skipping trace validation"
2185         fi
2186
2187         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
2188         ret=$?
2189         if [ $ret -ne 0 ]; then
2190                 fail "Failed to parse trace"
2191                 return $ret
2192         fi
2193
2194         traced=$(echo -n "$events" | wc -l)
2195         if [ "$traced" -eq 0 ]; then
2196                 pass "Validate empty trace"
2197         else
2198                 fail "Validate empty trace"
2199                 diag "Found $traced events in trace"
2200         fi
2201         ret=$?
2202         return $ret
2203 }
2204
2205 function validate_directory_empty ()
2206 {
2207         local trace_path="$1"
2208
2209         # Do not double quote `$trace_path` below as we want wildcards to be
2210         # expanded.
2211         files="$(ls -A $trace_path)"
2212         ret=$?
2213         if [ $ret -ne 0 ]; then
2214                 fail "Failed to list content of directory \"$trace_path\""
2215                 return $ret
2216         fi
2217
2218         nb_files="$(echo -n "$files" | wc -l)"
2219         ok $nb_files "Directory \"$trace_path\" is empty"
2220 }
2221
2222 function validate_trace_session_ust_empty()
2223 {
2224         validate_directory_empty "$1"/ust
2225 }
2226
2227 function validate_trace_session_kernel_empty()
2228 {
2229         validate_trace_empty "$1"/kernel
2230 }
2231
2232 function regenerate_metadata ()
2233 {
2234         local expected_to_fail=$1
2235         local sess_name=$2
2236
2237         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2238         ret=$?
2239         if [[ $expected_to_fail -eq "1" ]]; then
2240                 test "$ret" -ne "0"
2241                 ok $? "Expected fail on regenerate metadata $sess_name"
2242         else
2243                 ok $ret "Metadata regenerate $sess_name"
2244         fi
2245 }
2246
2247 function regenerate_metadata_ok ()
2248 {
2249         regenerate_metadata 0 "$@"
2250 }
2251
2252 function regenerate_metadata_fail ()
2253 {
2254         regenerate_metadata 1 "$@"
2255 }
2256
2257 function regenerate_statedump ()
2258 {
2259         local expected_to_fail=$1
2260         local sess_name=$2
2261
2262         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2263         ret=$?
2264         if [[ $expected_to_fail -eq "1" ]]; then
2265                 test "$ret" -ne "0"
2266                 ok $? "Expected fail on regenerate statedump $sess_name"
2267         else
2268                 ok $ret "Statedump regenerate $sess_name"
2269         fi
2270 }
2271
2272 function regenerate_statedump_ok ()
2273 {
2274         regenerate_statedump 0 "$@"
2275 }
2276
2277 function regenerate_statedump_fail ()
2278 {
2279         regenerate_statedump 1 "$@"
2280 }
2281
2282 function rotate_session ()
2283 {
2284         local expected_to_fail=$1
2285         local sess_name=$2
2286
2287         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2288         ret=$?
2289         if [[ $expected_to_fail -eq "1" ]]; then
2290                 test "$ret" -ne "0"
2291                 ok $? "Expected fail on rotate session $sess_name"
2292         else
2293                 ok $ret "Rotate session $sess_name"
2294         fi
2295 }
2296
2297 function rotate_session_ok ()
2298 {
2299         rotate_session 0 "$@"
2300 }
2301
2302 function rotate_session_fail ()
2303 {
2304         rotate_session 1 "$@"
2305 }
2306
2307 function destructive_tests_enabled ()
2308 {
2309         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2310                 return 0
2311         else
2312                 return 1
2313         fi
2314 }
2315
2316 function lttng_enable_rotation_timer ()
2317 {
2318         local expected_to_fail=$1
2319         local sess_name=$2
2320         local period=$3
2321
2322         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2323         ret=$?
2324         if [[ $expected_to_fail -eq "1" ]]; then
2325                 test "$ret" -ne "0"
2326                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2327         else
2328                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2329         fi
2330 }
2331
2332 function lttng_enable_rotation_timer_ok ()
2333 {
2334         lttng_enable_rotation_timer 0 $@
2335 }
2336
2337 function lttng_enable_rotation_timer_fail ()
2338 {
2339         lttng_enable_rotation_timer 1 $@
2340 }
2341
2342 function lttng_enable_rotation_size ()
2343 {
2344         local expected_to_fail=$1
2345         local sess_name=$2
2346         local size=$3
2347
2348         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2349         ret=$?
2350         if [[ $expected_to_fail -eq "1" ]]; then
2351                 test "$ret" -ne "0"
2352                 ok $? "Expected fail on rotate session $sess_name"
2353         else
2354                 ok $ret "Rotate session $sess_name"
2355         fi
2356 }
2357
2358 function lttng_enable_rotation_size_ok ()
2359 {
2360         lttng_enable_rotation_size 0 $@
2361 }
2362
2363 function lttng_enable_rotation_size_fail ()
2364 {
2365         lttng_enable_rotation_size 1 $@
2366 }
2367
2368 function lttng_clear_session ()
2369 {
2370         local expected_to_fail=$1
2371         local sess_name=$2
2372
2373         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2374         ret=$?
2375         if [[ $expected_to_fail -eq "1" ]]; then
2376                 test "$ret" -ne "0"
2377                 ok $? "Expected fail on clear session $sess_name"
2378         else
2379                 ok $ret "Clear session $sess_name"
2380         fi
2381 }
2382
2383 function lttng_clear_session_ok ()
2384 {
2385         lttng_clear_session 0 $@
2386 }
2387
2388 function lttng_clear_session_fail ()
2389 {
2390         lttng_clear_session 1 $@
2391 }
2392
2393 function lttng_clear_all ()
2394 {
2395         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2396         ok $? "Clear all lttng sessions"
2397 }
2398
2399 function lttng_add_trigger()
2400 {
2401         local expected_to_fail="$1"
2402         local trigger_name="$2"
2403         shift 2
2404         local args=("$@")
2405
2406         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2407         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2408         ret=$?
2409         if [[ $expected_to_fail -eq "1" ]]; then
2410                 test "$ret" -ne "0"
2411                 ok $? "Add trigger $trigger_name failed as expected"
2412         else
2413                 ok $ret "Add trigger $trigger_name"
2414         fi
2415 }
2416
2417 function lttng_remove_trigger()
2418 {
2419         local expected_to_fail="$1"
2420         local trigger_name="$2"
2421         shift 2
2422
2423         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2424         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2425         ret=$?
2426         if [[ $expected_to_fail -eq "1" ]]; then
2427                 test "$ret" -ne "0"
2428                 ok $? "Remove trigger $trigger_name failed as expected"
2429         else
2430                 ok $ret "Remove trigger $trigger_name"
2431         fi
2432 }
2433
2434 function lttng_add_trigger_ok()
2435 {
2436         lttng_add_trigger 0 "$@"
2437 }
2438
2439 function lttng_add_trigger_fail()
2440 {
2441         lttng_add_trigger 1 "$@"
2442 }
2443
2444 function lttng_remove_trigger_ok()
2445 {
2446         lttng_remove_trigger 0 "$@"
2447 }
2448
2449 function list_triggers_matches_ok ()
2450 {
2451         local tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2452         local tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2453
2454         local test_name="$1"
2455         local expected_stdout_file="$2"
2456
2457         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2458
2459         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2460         ok $? "${test_name}: exit code is 0"
2461
2462         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2463         ok $? "${test_name}: expected stdout"
2464
2465         diff -u /dev/null "${tmp_stderr}"
2466         ok $? "${test_name}: expected stderr"
2467
2468         rm -f "${tmp_stdout}"
2469         rm -f "${tmp_stderr}"
2470 }
2471
2472 function list_triggers_matches_mi_ok ()
2473 {
2474         local tmp_stdout
2475         local tmp_stdout_raw
2476         local tmp_stderr
2477
2478         local test_name="$1"
2479         local expected_stdout_file="$2"
2480
2481         tmp_stdout_raw=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2482         tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2483         tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2484
2485         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2486
2487         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2488         ok $? "${test_name}: exit code is 0"
2489
2490         # Pretty-fy xml before further test.
2491         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2492
2493         $MI_VALIDATE "${tmp_stdout}"
2494         ok $? "list-trigger mi is valid"
2495
2496         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2497         ok $? "${test_name}: expected stdout"
2498
2499         diff -u /dev/null "${tmp_stderr}"
2500         ok $? "${test_name}: expected stderr"
2501
2502         rm -f "${tmp_stdout}"
2503         rm -f "${tmp_stdout_raw}"
2504         rm -f "${tmp_stderr}"
2505 }
2506
2507 function validate_path_pattern ()
2508 {
2509         local message=$1
2510         local pattern=$2
2511         # Base path is only used in error case and is used to list the content
2512         # of the base path.
2513         local base_path=$3
2514
2515
2516         [ -f $pattern ]
2517         ret=$?
2518         ok $ret "$message"
2519
2520         if [ "$ret" -ne "0" ]; then
2521                 diag "Path pattern expected: $pattern"
2522                 # List the tracepath for more info. We use find as a recursive
2523                 # directory lister.
2524                 diag "The base path content:"
2525                 find "$base_path" -print
2526         fi
2527 }
2528
2529 function validate_trace_path_ust_uid ()
2530 {
2531         local trace_path=$1
2532         local session_name=$2
2533         local uid=$UID
2534         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2535
2536         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2537 }
2538
2539 function validate_trace_path_ust_uid_network ()
2540 {
2541         local trace_path=$1
2542         local session_name=$2
2543         local base_path=$3
2544         local uid=$UID
2545         local hostname=$HOSTNAME
2546         local pattern
2547         local ret
2548
2549         # If the session was given a network base path (e.g
2550         # 127.0.0.1/my/custom/path on creation, there is no session name
2551         # component to the path on the relayd side. Caller can simply not pass a
2552         # session name for this scenario.
2553         if [ -n "$session_name" ]; then
2554                 session_name="$session_name-$date_time_pattern"
2555                 if [ -n "$base_path" ]; then
2556                         fail "Session name and base path are mutually exclusive"
2557                         return
2558                 fi
2559         fi
2560
2561         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2562
2563         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2564 }
2565
2566 function validate_trace_path_ust_uid_snapshot_network ()
2567 {
2568         local trace_path=$1
2569         local session_name=$2
2570         local snapshot_name=$3
2571         local snapshot_number=$4
2572         local base_path=$5
2573         local hostname=$HOSTNAME
2574         local uid=$UID
2575         local pattern
2576         local ret
2577
2578         # If the session/output was given a network base path (e.g
2579         # 127.0.0.1/my/custom/path on creation, there is no session name
2580         # component to the path on the relayd side. Caller can simply not pass a
2581         # session name for this scenario.
2582         if [ -n "$session_name" ]; then
2583                 session_name="$session_name-$date_time_pattern"
2584                 if [ -n "$base_path" ]; then
2585                         fail "Session name and base path are mutually exclusive"
2586                         return
2587                 fi
2588         fi
2589
2590         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2591
2592         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2593 }
2594
2595 function validate_trace_path_ust_uid_snapshot ()
2596 {
2597         local trace_path=$1
2598         local session_name=$2
2599         local snapshot_name=$3
2600         local snapshot_number=$4
2601         local base_path=$5
2602         local uid=$UID
2603         local pattern
2604         local ret
2605
2606         # If the session/output was given a network base path (e.g
2607         # 127.0.0.1/my/custom/path) on creation, there is no session name
2608         # component to the path on the relayd side. Caller can simply not pass a
2609         # session name for this scenario.
2610         if [ -n "$session_name" ]; then
2611                 session_name="$session_name-$date_time_pattern"
2612                 if [ -n "$base_path" ]; then
2613                         fail "Session name and base path are mutually exclusive"
2614                         return
2615                 fi
2616         fi
2617
2618         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2619
2620         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2621 }
2622
2623 function validate_trace_path_ust_pid ()
2624 {
2625         local trace_path=$1
2626         local session_name=$2
2627         local app_string=$3
2628         local pid=$4
2629         local pattern
2630         local ret
2631
2632         # If the session was given a trace path on creation, there is no session
2633         # name component to the path. Caller can simply not pass a session name
2634         # for this scenario.
2635         if [ -n "$session_name" ]; then
2636                 session_name="$session_name-$date_time_pattern"
2637         fi
2638
2639         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2640
2641         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2642 }
2643
2644 function validate_trace_path_kernel ()
2645 {
2646         local trace_path=$1
2647         local session_name=$2
2648         local pattern
2649
2650         # If the session was given a trace path on creation, there is no session
2651         # name component to the path. Caller can simply not pass a session name
2652         # for this scenario.
2653         if [ -n "$session_name" ]; then
2654                 session_name="$session_name-$date_time_pattern"
2655         fi
2656
2657         pattern="$trace_path/$session_name/kernel/metadata"
2658
2659         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2660 }
2661
2662 function validate_trace_path_kernel_network ()
2663 {
2664         local trace_path=$1
2665         local session_name=$2
2666         local hostname=$HOSTNAME
2667         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2668
2669         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2670 }
2671
2672 function validate_trace_path_kernel_snapshot ()
2673 {
2674         local trace_path=$1
2675         local session_name=$2
2676         local snapshot_name=$3
2677         local snapshot_number=$4
2678         local base_path=$5
2679         local pattern
2680         local ret
2681
2682         # If the session/output was given a network base path (e.g
2683         # 127.0.0.1/my/custom/path on creation, there is no session name
2684         # component to the path on the relayd side. Caller can simply not pass a
2685         # session name for this scenario.
2686         if [ -n "$session_name" ]; then
2687                 session_name="$session_name-$date_time_pattern"
2688                 if [ -n "$base_path" ]; then
2689                         fail "Session name and base path are mutually exclusive"
2690                         return
2691                 fi
2692         fi
2693
2694         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2695
2696         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2697 }
2698
2699 function validate_trace_path_kernel_snapshot_network ()
2700 {
2701         local trace_path=$1
2702         local session_name=$2
2703         local snapshot_name=$3
2704         local snapshot_number=$4
2705         local base_path=$5
2706         local hostname=$HOSTNAME
2707         local pattern
2708         local ret
2709
2710         # If the session/output was given a network base path (e.g
2711         # 127.0.0.1/my/custom/path on creation, there is no session name
2712         # component to the path on the relayd side. Caller can simply not pass a
2713         # session name for this scenario.
2714         if [ -n "$session_name" ]; then
2715                 session_name="$session_name-$date_time_pattern"
2716                 if [ -n "$base_path" ]; then
2717                         fail "Session name and base path are mutually exclusive"
2718                         return
2719                 fi
2720         fi
2721
2722         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2723
2724         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2725 }