tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace2"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$( (tr '\0' '\n' < /proc/"$pid"/cmdline) 2>/dev/null | head -n1); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         local len="${1:-16}"
 208
 209         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 210         # /dev/urandom isn't guaranteed to generate valid multi-byte characters.
 211         # Specifying the C locale eliminates the "Illegal byte sequence" error
 212         # that 'tr' outputs in such cases.
 213         LC_CTYPE=C tr -cd "$CHAR" < /dev/urandom 2>/dev/null | head -c "$len" 2>/dev/null
 214         echo
 215 }
 216
 217 # Helpers for get_possible_cpus.
 218 function get_possible_cpus_count_from_sysfs_possible_mask()
 219 {
 220         local max_possible_cpu_id
 221
 222         # The Awk script extracts the highest CPU id from the possible CPU
 223         # mask. Assuming a numerical order, a field separator '-' and a record
 224         # separator ','. The last value parsed is the highest id.
 225         if [ -f /sys/devices/system/cpu/possible ]; then
 226                 max_possible_cpu_id=$(awk -F '-' 'BEGIN { RS = ","} { last = $NF } END { printf("%d\n", last) }' \
 227                                       /sys/devices/system/cpu/possible)
 228                 echo "$((max_possible_cpu_id+1))"
 229         else
 230                 echo "0"
 231         fi
 232 }
 233
 234 # This is a fallback if the possible CPU mask is not available. This will not
 235 # take into account unplugged CPUs.
 236 function get_max_cpus_count_from_sysfs_cpu_directories()
 237 {
 238         local max_possible_cpu_id=0
 239         local current_cpu_id
 240
 241         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 242                 current_cpu_id="${i#/sys/devices/system/cpu/cpu}"
 243                 if [ "$current_cpu_id" -gt "$max_possible_cpu_id" ]; then
 244                         max_possible_cpu_id="$current_cpu_id"
 245                 fi
 246         done
 247
 248         echo "$((max_possible_cpu_id+1))"
 249 }
 250
 251 # Return the number of possible CPUs.
 252 function get_possible_cpus_count()
 253 {
 254         local possible_cpus_count
 255         possible_cpus_count=$(get_possible_cpus_count_from_sysfs_possible_mask)
 256
 257         if [ "$possible_cpus_count" -eq "0" ]; then
 258                 local configured_cpus_count
 259                 configured_cpus_count=$(getconf _NPROCESSORS_CONF)
 260                 possible_cpus_count=$(get_max_cpus_count_from_sysfs_cpu_directories)
 261                 possible_cpus_count=$((configured_cpus_count > possible_cpus_count \
 262                                                              ? configured_cpus_count \
 263                                                              : possible_cpus_count))
 264         fi
 265
 266         echo "$possible_cpus_count"
 267 }
 268
 269 # Return the list of exposed CPU.
 270 #
 271 # NOTE! Use it like so:
 272 #
 273 # IFS=" " read -r -a VARIABLE <<< "$(get_exposed_cpus_list)"
 274 function get_exposed_cpus_list()
 275 {
 276         local list=()
 277
 278         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 279                 list+=("${i#/sys/devices/system/cpu/cpu}")
 280         done
 281
 282         echo "${list[@]}"
 283 }
 284
 285 # Return any available CPU found. Do not make assumption about the returned
 286 # value, e.g. that it could be 0.
 287 function get_any_available_cpu()
 288 {
 289         for cpu in /sys/devices/system/cpu/cpu[0-9]*; do
 290                 echo "${cpu#/sys/devices/system/cpu/cpu}"
 291                 break;
 292         done
 293 }
 294
 295 # Return the number of _configured_ CPUs.
 296 function conf_proc_count()
 297 {
 298         getconf _NPROCESSORS_CONF
 299         if [ $? -ne 0 ]; then
 300                 diag "Failed to get the number of configured CPUs"
 301         fi
 302         echo
 303 }
 304
 305 # Check if base lttng-modules are present.
 306 # Bail out on failure
 307 function validate_lttng_modules_present ()
 308 {
 309         # Check for loadable modules.
 310         modprobe -n lttng-tracer 2>/dev/null
 311         if [ $? -eq 0 ]; then
 312                 return 0
 313         fi
 314
 315         # Check for builtin modules.
 316         ls /proc/lttng > /dev/null 2>&1
 317         if [ $? -eq 0 ]; then
 318                 return 0
 319         fi
 320
 321         LTTNG_BAIL_OUT "LTTng modules not detected."
 322 }
 323
 324 # Run the lttng binary.
 325 #
 326 # The first two arguments are stdout and stderr redirect paths, respectively.
 327 # The rest of the arguments are forwarded to the lttng binary
 328 function _run_lttng_cmd
 329 {
 330         local stdout_dest="$1"
 331         local stderr_dest="$2"
 332         shift 2
 333
 334         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 335         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 336 }
 337
 338 function enable_kernel_lttng_event
 339 {
 340         local withtap="$1"
 341         local expected_to_fail="$2"
 342         local sess_name="$3"
 343         local event_name="$4"
 344         local channel_name="$5"
 345
 346         if [ -z "$event_name" ]; then
 347                 # Enable all event if no event name specified
 348                 event_name="-a"
 349         fi
 350
 351         if [ -z "$channel_name" ]; then
 352                 # default channel if none specified
 353                 chan=""
 354         else
 355                 chan="-c $channel_name"
 356         fi
 357
 358         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 359                 enable-event "$event_name" $chan -s $sess_name -k
 360         ret=$?
 361         if [[ $expected_to_fail -eq "1" ]]; then
 362                 test $ret -ne "0"
 363                 ret=$?
 364                 if [ $withtap -eq "1" ]; then
 365                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 366                 fi
 367         else
 368                 if [ $withtap -eq "1" ]; then
 369                         ok $ret "Enable kernel event $event_name for session $sess_name"
 370                 fi
 371         fi
 372 }
 373
 374 function enable_kernel_lttng_event_ok ()
 375 {
 376         enable_kernel_lttng_event 1 0 "$@"
 377 }
 378
 379 function enable_kernel_lttng_event_fail ()
 380 {
 381         enable_kernel_lttng_event 1 1 "$@"
 382 }
 383
 384 function enable_kernel_lttng_event_notap ()
 385 {
 386         enable_kernel_lttng_event 0 0 "$@"
 387 }
 388
 389 # Old interface
 390 function lttng_enable_kernel_event
 391 {
 392         enable_kernel_lttng_event_ok "$@"
 393 }
 394
 395 function lttng_enable_kernel_syscall()
 396 {
 397         local expected_to_fail=$1
 398         local sess_name=$2
 399         local syscall_name=$3
 400         local channel_name=$4
 401
 402         if [ -z $syscall_name ]; then
 403                 # Enable all event if no syscall name specified
 404                 syscall_name="-a"
 405         fi
 406
 407         if [ -z $channel_name ]; then
 408                 # default channel if none specified
 409                 chan=""
 410         else
 411                 chan="-c $channel_name"
 412         fi
 413
 414         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 415                 enable-event --syscall "$syscall_name" $chan -s $sess_name -k
 416         ret=$?
 417         if [[ $expected_to_fail -eq "1" ]]; then
 418                 test $ret -ne "0"
 419                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 420         else
 421                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 422         fi
 423 }
 424
 425 function lttng_enable_kernel_syscall_ok()
 426 {
 427         lttng_enable_kernel_syscall 0 "$@"
 428 }
 429
 430 function lttng_enable_kernel_syscall_fail()
 431 {
 432         lttng_enable_kernel_syscall 1 "$@"
 433 }
 434
 435 function lttng_disable_kernel_syscall()
 436 {
 437         local expected_to_fail=$1
 438         local sess_name=$2
 439         local syscall_name=$3
 440         local channel_name=$4
 441
 442         if [ -z $syscall_name ]; then
 443                 # Enable all event if no syscall name specified
 444                 syscall_name="-a"
 445         fi
 446
 447         if [ -z $channel_name ]; then
 448                 # default channel if none specified
 449                 chan=""
 450         else
 451                 chan="-c $channel_name"
 452         fi
 453
 454         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 455                 disable-event --syscall "$syscall_name" $chan -s $sess_name -k
 456
 457         ret=$?
 458         if [[ $expected_to_fail -eq "1" ]]; then
 459                 test $ret -ne "0"
 460                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 461         else
 462                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 463         fi
 464 }
 465
 466 function lttng_disable_kernel_syscall_ok()
 467 {
 468         lttng_disable_kernel_syscall 0 "$@"
 469 }
 470
 471 function lttng_disable_kernel_syscall_fail()
 472 {
 473         lttng_disable_kernel_syscall 1 "$@"
 474 }
 475
 476 function lttng_enable_kernel_function_event ()
 477 {
 478         local expected_to_fail="$1"
 479         local sess_name="$2"
 480         local target="$3"
 481         local event_name="$4"
 482
 483         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 484         ret=$?
 485         if [[ $expected_to_fail -eq "1" ]]; then
 486                 test $ret -ne "0"
 487                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 488         else
 489                 ok $ret "Enable kernel function event for session $sess_name"
 490         fi
 491 }
 492
 493 function lttng_enable_kernel_function_event_ok ()
 494 {
 495         lttng_enable_kernel_function_event 0 "$@"
 496 }
 497
 498 function lttng_enable_kernel_userspace_probe_event ()
 499 {
 500         local expected_to_fail="$1"
 501         local sess_name="$2"
 502         local target="$3"
 503         local event_name="$4"
 504
 505         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 506         ret=$?
 507         if [[ $expected_to_fail -eq "1" ]]; then
 508                 test $ret -ne "0"
 509                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 510         else
 511                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 512         fi
 513 }
 514
 515 function lttng_enable_kernel_userspace_probe_event_fail ()
 516 {
 517         lttng_enable_kernel_userspace_probe_event 1 "$@"
 518 }
 519
 520 function lttng_enable_kernel_userspace_probe_event_ok ()
 521 {
 522         lttng_enable_kernel_userspace_probe_event 0 "$@"
 523 }
 524
 525 function disable_kernel_lttng_userspace_probe_event_ok ()
 526 {
 527         local sess_name="$1"
 528         local event_name="$2"
 529
 530         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 531         ok $? "Disable kernel event $target for session $sess_name"
 532 }
 533 function lttng_enable_kernel_channel()
 534 {
 535         local withtap=$1
 536         local expected_to_fail=$2
 537         local sess_name=$3
 538         local channel_name=$4
 539         local opts="${@:5}"
 540
 541         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 542                 enable-channel -k $channel_name -s $sess_name $opts
 543         ret=$?
 544         if [[ $expected_to_fail -eq "1" ]]; then
 545                 test "$ret" -ne "0"
 546                 ret=$?
 547                 if [ $withtap -eq "1" ]; then
 548                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 549                 fi
 550         else
 551                 if [ $withtap -eq "1" ]; then
 552                         ok $ret "Enable channel $channel_name for session $sess_name"
 553                 fi
 554         fi
 555 }
 556
 557 function lttng_enable_kernel_channel_ok()
 558 {
 559         lttng_enable_kernel_channel 1 0 "$@"
 560 }
 561
 562 function lttng_enable_kernel_channel_fail()
 563 {
 564         lttng_enable_kernel_channel 1 1 "$@"
 565 }
 566
 567 function lttng_enable_kernel_channel_notap()
 568 {
 569         lttng_enable_kernel_channel 0 0 "$@"
 570 }
 571
 572 function enable_kernel_lttng_channel_ok()
 573 {
 574         lttng_enable_kernel_channel 1 0 "$@"
 575 }
 576
 577 function lttng_disable_kernel_channel()
 578 {
 579         local expected_to_fail=$1
 580         local sess_name=$2
 581         local channel_name=$3
 582
 583         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 584                 disable-channel -k $channel_name -s $sess_name
 585         ret=$?
 586         if [[ $expected_to_fail -eq "1" ]]; then
 587                 test "$ret" -ne "0"
 588                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 589         else
 590                 ok $ret "Disable channel $channel_name for session $sess_name"
 591         fi
 592 }
 593
 594 function lttng_disable_kernel_channel_ok()
 595 {
 596         lttng_disable_kernel_channel 0 "$@"
 597 }
 598
 599 function lttng_disable_kernel_channel_fail()
 600 {
 601         lttng_disable_kernel_channel 1 "$@"
 602 }
 603
 604 function start_lttng_relayd_opt()
 605 {
 606         local withtap=$1
 607         local process_mode=$2
 608         local opt=$3
 609
 610         DIR=$(readlink -f "$TESTDIR")
 611
 612         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 613                 # shellcheck disable=SC2086
 614                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 615                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 616                 if [ $? -eq 1 ]; then
 617                         if [ $withtap -eq "1" ]; then
 618                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 619                         fi
 620                         return 1
 621                 else
 622                         if [ $withtap -eq "1" ]; then
 623                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 624                         fi
 625                 fi
 626         else
 627                 pass "Start lttng-relayd (opt: $opt)"
 628         fi
 629 }
 630
 631 function start_lttng_relayd()
 632 {
 633         start_lttng_relayd_opt 1 "-b" "$@"
 634 }
 635
 636 function start_lttng_relayd_notap()
 637 {
 638         start_lttng_relayd_opt 0 "-b" "$@"
 639 }
 640
 641 function stop_lttng_relayd_opt()
 642 {
 643         local withtap=$1
 644         local is_cleanup=$2
 645         local signal=$3
 646         local timeout_s=$4
 647         local dtimeleft_s=
 648         local retval=0
 649         local pids
 650
 651         if [ -z "$signal" ]; then
 652                 signal="SIGTERM"
 653         fi
 654
 655
 656         # Multiply time by 2 to simplify integer arithmetic
 657         if [ -n "$timeout_s" ]; then
 658                 dtimeleft_s=$((timeout_s * 2))
 659         fi
 660
 661
 662         pids=$(lttng_pgrep "$RELAYD_MATCH")
 663         if [ -z "$pids" ]; then
 664                 if [ "$is_cleanup" -eq 1 ]; then
 665                         :
 666                 elif [ "$withtap" -eq "1" ]; then
 667                         fail "No relay daemon to kill"
 668                 else
 669                         LTTNG_BAIL_OUT "No relay daemon to kill"
 670                 fi
 671                 return 0
 672         fi
 673
 674         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 675
 676         # shellcheck disable=SC2086
 677         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 678                 retval=1
 679                 if [ "$withtap" -eq "1" ]; then
 680                         fail "Kill relay daemon"
 681                 fi
 682         else
 683                 out=1
 684                 while [ -n "$out" ]; do
 685                         out=$(lttng_pgrep "$RELAYD_MATCH")
 686                         if [ -n "$dtimeleft_s" ]; then
 687                                 if [ $dtimeleft_s -lt 0 ]; then
 688                                         out=
 689                                         retval=1
 690                                 fi
 691                                 dtimeleft_s=$((dtimeleft_s - 1))
 692                         fi
 693                         sleep 0.5
 694                 done
 695                 if [ "$withtap" -eq "1" ]; then
 696                         if [ "$retval" -eq "0" ]; then
 697                                 pass "Wait after kill relay daemon"
 698                         else
 699                                 fail "Wait after kill relay daemon"
 700                         fi
 701                 fi
 702         fi
 703         return $retval
 704 }
 705
 706 function stop_lttng_relayd()
 707 {
 708         stop_lttng_relayd_opt 1 0 "$@"
 709 }
 710
 711 function stop_lttng_relayd_notap()
 712 {
 713         stop_lttng_relayd_opt 0 0 "$@"
 714 }
 715
 716 function stop_lttng_relayd_cleanup()
 717 {
 718         stop_lttng_relayd_opt 0 1 "$@"
 719 }
 720
 721 #First arg: show tap output
 722 #Second argument: load path for automatic loading
 723 function start_lttng_sessiond_opt()
 724 {
 725         local withtap=$1
 726         local load_path=$2
 727
 728         # The rest of the arguments will be passed directly to lttng-sessiond.
 729         shift 2
 730
 731         local env_vars=""
 732         local consumerd=""
 733
 734         local long_bit_value=
 735         long_bit_value=$(getconf LONG_BIT)
 736
 737         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 738                 # Env variable requested no session daemon
 739                 return
 740         fi
 741
 742         DIR=$(readlink -f "$TESTDIR")
 743
 744         # Get long_bit value for 32/64 consumerd
 745         case "$long_bit_value" in
 746                 32)
 747                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 748                         ;;
 749                 64)
 750                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 751                         ;;
 752                 *)
 753                         return
 754                         ;;
 755         esac
 756
 757         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 758         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 759                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 760         fi
 761         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 762
 763         if ! validate_kernel_version; then
 764             fail "Start session daemon"
 765             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 766         fi
 767
 768         diag "export LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/"
 769         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/"}"
 770         export LTTNG_SESSION_CONFIG_XSD_PATH
 771
 772         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 773                 # Have a load path ?
 774                 if [ -n "$load_path" ]; then
 775                         diag "env $env_vars --load $load_path --background $consumerd $@"
 776                         # shellcheck disable=SC2086
 777                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 778                 else
 779                         diag "env $env_vars --background $consumerd $@"
 780                         # shellcheck disable=SC2086
 781                         env $env_vars --background "$consumerd" "$@"
 782                 fi
 783                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 784                 status=$?
 785                 if [ "$withtap" -eq "1" ]; then
 786                         ok $status "Start session daemon"
 787                 fi
 788         fi
 789 }
 790
 791 function start_lttng_sessiond()
 792 {
 793         start_lttng_sessiond_opt 1 "$@"
 794 }
 795
 796 function start_lttng_sessiond_notap()
 797 {
 798         start_lttng_sessiond_opt 0 "$@"
 799 }
 800
 801 function stop_lttng_sessiond_opt()
 802 {
 803         local withtap=$1
 804         local is_cleanup=$2
 805         local signal=$3
 806         local timeout_s=$4
 807         local dtimeleft_s=
 808         local retval=0
 809         local runas_pids
 810         local pids
 811
 812         if [ -z "$signal" ]; then
 813                 signal=SIGTERM
 814         fi
 815
 816         # Multiply time by 2 to simplify integer arithmetic
 817         if [ -n "$timeout_s" ]; then
 818                 dtimeleft_s=$((timeout_s * 2))
 819         fi
 820
 821         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 822                 # Env variable requested no session daemon
 823                 return 0
 824         fi
 825
 826         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 827         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 828
 829         if [ -n "$runas_pids" ]; then
 830                 pids="$pids $runas_pids"
 831         fi
 832
 833         if [ -z "$pids" ]; then
 834                 if [ "$is_cleanup" -eq 1 ]; then
 835                         :
 836                 elif [ "$withtap" -eq "1" ]; then
 837                         fail "No session daemon to kill"
 838                 else
 839                         LTTNG_BAIL_OUT "No session daemon to kill"
 840                 fi
 841                 return 0
 842         fi
 843
 844         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 845
 846         # shellcheck disable=SC2086
 847         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 848                 retval=1
 849                 if [ "$withtap" -eq "1" ]; then
 850                         fail "Kill sessions daemon"
 851                 fi
 852         else
 853                 out=1
 854                 while [ -n "$out" ]; do
 855                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 856                         if [ -n "$dtimeleft_s" ]; then
 857                                 if [ $dtimeleft_s -lt 0 ]; then
 858                                         out=
 859                                         retval=1
 860                                 fi
 861                                 dtimeleft_s=$((dtimeleft_s - 1))
 862                         fi
 863                         sleep 0.5
 864                 done
 865                 out=1
 866                 while [ -n "$out" ]; do
 867                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 868                         if [ -n "$dtimeleft_s" ]; then
 869                                 if [ $dtimeleft_s -lt 0 ]; then
 870                                         out=
 871                                         retval=1
 872                                 fi
 873                                 dtimeleft_s=$((dtimeleft_s - 1))
 874                         fi
 875                         sleep 0.5
 876                 done
 877
 878                 if [ "$withtap" -eq "1" ]; then
 879                         if [ "$retval" -eq "0" ]; then
 880                                 pass "Wait after kill session daemon"
 881                         else
 882                                 fail "Wait after kill session daemon"
 883                         fi
 884                 fi
 885         fi
 886         if [ "$signal" = "SIGKILL" ]; then
 887                 if [ "$(id -u)" -eq "0" ]; then
 888                         local modules=
 889                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 890
 891                         if [ -n "$modules" ]; then
 892                                 diag "Unloading all LTTng modules"
 893                                 modprobe --remove "$modules"
 894                         fi
 895                 fi
 896         fi
 897
 898         return $retval
 899 }
 900
 901 function stop_lttng_sessiond()
 902 {
 903         stop_lttng_sessiond_opt 1 0 "$@"
 904 }
 905
 906 function stop_lttng_sessiond_notap()
 907 {
 908         stop_lttng_sessiond_opt 0 0 "$@"
 909 }
 910
 911 function stop_lttng_sessiond_cleanup()
 912 {
 913         stop_lttng_sessiond_opt 0 1 "$@"
 914 }
 915
 916 function sigstop_lttng_sessiond_opt()
 917 {
 918         local withtap=$1
 919         local signal=SIGSTOP
 920         local pids
 921
 922         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 923                 # Env variable requested no session daemon
 924                 return
 925         fi
 926
 927         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 928
 929         if [ "$withtap" -eq "1" ]; then
 930                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 931         fi
 932
 933         # shellcheck disable=SC2086
 934         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 935                 if [ "$withtap" -eq "1" ]; then
 936                         fail "Sending SIGSTOP to session daemon"
 937                 fi
 938         else
 939                 out=1
 940                 while [ $out -ne 0 ]; do
 941                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 942
 943                         # Wait until state becomes stopped for session
 944                         # daemon(s).
 945                         out=0
 946                         for sessiond_pid in $pids; do
 947                                 state="$(ps -p "$sessiond_pid" -o state= )"
 948                                 if [[ -n "$state" && "$state" != "T" ]]; then
 949                                         out=1
 950                                 fi
 951                         done
 952                         sleep 0.5
 953                 done
 954                 if [ "$withtap" -eq "1" ]; then
 955                         pass "Sending SIGSTOP to session daemon"
 956                 fi
 957         fi
 958 }
 959
 960 function sigstop_lttng_sessiond()
 961 {
 962         sigstop_lttng_sessiond_opt 1 "$@"
 963 }
 964
 965 function sigstop_lttng_sessiond_notap()
 966 {
 967         sigstop_lttng_sessiond_opt 0 "$@"
 968 }
 969
 970 function stop_lttng_consumerd_opt()
 971 {
 972         local withtap=$1
 973         local is_cleanup=$2
 974         local signal=$3
 975         local timeout_s=$4
 976         local dtimeleft_s=
 977         local retval=0
 978         local pids
 979
 980         if [ -z "$signal" ]; then
 981                 signal=SIGTERM
 982         fi
 983
 984         # Multiply time by 2 to simplify integer arithmetic
 985         if [ -n "$timeout_s" ]; then
 986                 dtimeleft_s=$((timeout_s * 2))
 987         fi
 988
 989         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 990
 991         if [ -z "$pids" ]; then
 992                 if [ "$is_cleanup" -eq 1 ]; then
 993                         :
 994                 elif [ "$withtap" -eq "1" ]; then
 995                         fail "No consumerd daemon to kill"
 996                 else
 997                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
 998                 fi
 999                 return 0
1000         fi
1001
1002         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1003
1004         # shellcheck disable=SC2086
1005         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
1006                 retval=1
1007                 if [ "$withtap" -eq "1" ]; then
1008                         fail "Kill consumer daemon"
1009                 fi
1010         else
1011                 out=1
1012                 while [ $out -ne 0 ]; do
1013                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1014
1015                         # If consumerds are still present check their status.
1016                         # A zombie status qualifies the consumerd as *killed*
1017                         out=0
1018                         for consumer_pid in $pids; do
1019                                 state="$(ps -p "$consumer_pid" -o state= )"
1020                                 if [[ -n "$state" && "$state" != "Z" ]]; then
1021                                         out=1
1022                                 fi
1023                         done
1024                         if [ -n "$dtimeleft_s" ]; then
1025                                 if [ $dtimeleft_s -lt 0 ]; then
1026                                         out=0
1027                                         retval=1
1028                                 fi
1029                                 dtimeleft_s=$((dtimeleft_s - 1))
1030                         fi
1031                         sleep 0.5
1032                 done
1033                 if [ "$withtap" -eq "1" ]; then
1034                         if [ "$retval" -eq "0" ]; then
1035                                 pass "Wait after kill consumer daemon"
1036                         else
1037                                 fail "Wait after kill consumer daemon"
1038                         fi
1039                 fi
1040         fi
1041
1042         return $retval
1043 }
1044
1045 function stop_lttng_consumerd()
1046 {
1047         stop_lttng_consumerd_opt 1 0 "$@"
1048 }
1049
1050 function stop_lttng_consumerd_notap()
1051 {
1052         stop_lttng_consumerd_opt 0 0 "$@"
1053 }
1054
1055 function stop_lttng_consumerd_cleanup()
1056 {
1057         stop_lttng_consumerd_opt 0 1 "$@"
1058 }
1059
1060 function sigstop_lttng_consumerd_opt()
1061 {
1062         local withtap=$1
1063         local signal=SIGSTOP
1064         local pids
1065
1066         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1067
1068         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1069
1070         # shellcheck disable=SC2086
1071         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1072         retval=$?
1073
1074         if [ $retval -eq 1 ]; then
1075                 if [ "$withtap" -eq "1" ]; then
1076                         fail "Sending SIGSTOP to consumer daemon"
1077                 fi
1078                 return 1
1079         else
1080                 out=1
1081                 while [ $out -ne 0 ]; do
1082                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1083
1084                         # Wait until state becomes stopped for all
1085                         # consumers.
1086                         out=0
1087                         for consumer_pid in $pids; do
1088                                 state="$(ps -p "$consumer_pid" -o state= )"
1089                                 if [[ -n "$state" && "$state" != "T" ]]; then
1090                                         out=1
1091                                 fi
1092                         done
1093                         sleep 0.5
1094                 done
1095                 if [ "$withtap" -eq "1" ]; then
1096                         pass "Sending SIGSTOP to consumer daemon"
1097                 fi
1098         fi
1099         return $retval
1100 }
1101
1102 function sigstop_lttng_consumerd()
1103 {
1104         sigstop_lttng_consumerd_opt 1 "$@"
1105 }
1106
1107 function sigstop_lttng_consumerd_notap()
1108 {
1109         sigstop_lttng_consumerd_opt 0 "$@"
1110 }
1111
1112 function list_lttng_with_opts ()
1113 {
1114         local ret
1115         local withtap=$1
1116         shift
1117         local opts=$1
1118         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1119                 list $opts
1120         ret=$?
1121         if [ $withtap -eq "1" ]; then
1122                 ok $ret "Lttng-tool list command with option $opts"
1123         fi
1124 }
1125
1126 function list_lttng_ok ()
1127 {
1128         list_lttng_with_opts 1 "$@"
1129 }
1130
1131 function list_lttng_notap ()
1132 {
1133         list_lttng_with_opts 0 "$@"
1134 }
1135
1136 function create_lttng_session_no_output ()
1137 {
1138         local sess_name=$1
1139         local opts="${@:2}"
1140
1141         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1142                 create $sess_name --no-output $opts
1143         ok $? "Create session $sess_name in no-output mode"
1144 }
1145
1146 function create_lttng_session_uri () {
1147         local sess_name=$1
1148         local uri=$2
1149         local opts="${@:3}"
1150
1151         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1152                 create $sess_name -U $uri $opts
1153         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1154 }
1155
1156 function create_lttng_session ()
1157 {
1158         local withtap=$1
1159         local expected_to_fail=$2
1160         local sess_name=$3
1161         local trace_path=$4
1162         local opt=$5
1163
1164         if [ -z "$trace_path" ]; then
1165                 # Use lttng-sessiond default output.
1166                 trace_path=""
1167         else
1168                 trace_path="-o $trace_path"
1169         fi
1170
1171         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1172                 create "$sess_name" $trace_path $opt
1173         ret=$?
1174         if [ $expected_to_fail -eq "1" ]; then
1175                 test "$ret" -ne "0"
1176                 ret=$?
1177                 if [ $withtap -eq "1" ]; then
1178                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1179                 fi
1180         else
1181                 if [ $withtap -eq "1" ]; then
1182                         ok $ret "Create session $sess_name in $trace_path"
1183                 fi
1184         fi
1185         return $ret
1186 }
1187
1188 function create_lttng_session_ok ()
1189 {
1190         create_lttng_session 1 0 "$@"
1191 }
1192
1193 function create_lttng_session_fail ()
1194 {
1195         create_lttng_session 1 1 "$@"
1196 }
1197
1198 function create_lttng_session_notap ()
1199 {
1200         create_lttng_session 0 0 "$@"
1201 }
1202
1203
1204 function enable_ust_lttng_channel ()
1205 {
1206         local withtap=$1
1207         local expected_to_fail=$2
1208         local sess_name=$3
1209         local channel_name=$4
1210         local opts="${@:5}"
1211
1212         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1213                 enable-channel -u $channel_name -s $sess_name $opts
1214         ret=$?
1215         if [[ $expected_to_fail -eq "1" ]]; then
1216                 test "$ret" -ne "0"
1217                 ret=$?
1218                 if [ $withtap -eq "1" ]; then
1219                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1220                 fi
1221         else
1222                 if [ $withtap -eq "1" ]; then
1223                         ok $ret "Enable channel $channel_name for session $sess_name"
1224                 fi
1225         fi
1226         return $ret
1227 }
1228
1229 function enable_ust_lttng_channel_ok ()
1230 {
1231         enable_ust_lttng_channel 1 0 "$@"
1232 }
1233
1234 function enable_ust_lttng_channel_fail ()
1235 {
1236         enable_ust_lttng_channel 1 1 "$@"
1237 }
1238
1239 function enable_ust_lttng_channel_notap ()
1240 {
1241         enable_ust_lttng_channel 0 0 "$@"
1242 }
1243
1244 function disable_ust_lttng_channel()
1245 {
1246         local sess_name=$1
1247         local channel_name=$2
1248
1249         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1250                 disable-channel -u $channel_name -s $sess_name
1251         ok $? "Disable channel $channel_name for session $sess_name"
1252 }
1253
1254 function enable_lttng_mmap_overwrite_kernel_channel()
1255 {
1256         local sess_name=$1
1257         local channel_name=$2
1258
1259         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1260                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite
1261         ok $? "Enable channel $channel_name for session $sess_name"
1262 }
1263
1264 function enable_lttng_mmap_discard_small_kernel_channel()
1265 {
1266         local sess_name=$1
1267         local channel_name=$2
1268
1269         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1270                 enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1271         ok $? "Enable small discard channel $channel_name for session $sess_name"
1272 }
1273
1274 function enable_lttng_mmap_overwrite_small_kernel_channel()
1275 {
1276         local sess_name=$1
1277         local channel_name=$2
1278
1279         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1280                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1281         ok $? "Enable small discard channel $channel_name for session $sess_name"
1282 }
1283
1284 function enable_lttng_mmap_overwrite_ust_channel()
1285 {
1286         local sess_name=$1
1287         local channel_name=$2
1288
1289         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1290                 enable-channel -s $sess_name $channel_name -u --output mmap --overwrite
1291         ok $? "Enable channel $channel_name for session $sess_name"
1292 }
1293
1294 function enable_ust_lttng_event ()
1295 {
1296         local withtap=$1
1297         local expected_to_fail=$2
1298         local sess_name=$3
1299         local event_name="$4"
1300         local channel_name=$5
1301
1302         if [ -z $channel_name ]; then
1303                 # default channel if none specified
1304                 chan=""
1305         else
1306                 chan="-c $channel_name"
1307         fi
1308
1309         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1310                 enable-event "$event_name" $chan -s "$sess_name" -u
1311         ret=$?
1312         if [[ $expected_to_fail -eq "1" ]]; then
1313                 test $ret -ne "0"
1314                 ret=$?
1315                 if [[ $withtap -eq "1" ]]; then
1316                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1317                 fi
1318         else
1319                 if [[ $withtap -eq "1" ]]; then
1320                         ok $ret "Enable ust event $event_name for session $sess_name"
1321                 fi
1322         fi
1323         return $ret
1324 }
1325
1326 function enable_ust_lttng_event_ok ()
1327 {
1328         enable_ust_lttng_event 1 0 "$@"
1329 }
1330
1331 function enable_ust_lttng_event_fail ()
1332 {
1333         enable_ust_lttng_event 1 1 "$@"
1334 }
1335
1336 function enable_ust_lttng_event_notap ()
1337 {
1338         enable_ust_lttng_event 0 0 "$@"
1339 }
1340
1341 function enable_jul_lttng_event()
1342 {
1343         sess_name=$1
1344         event_name="$2"
1345         channel_name=$3
1346
1347         if [ -z $channel_name ]; then
1348                 # default channel if none specified
1349                 chan=""
1350         else
1351                 chan="-c $channel_name"
1352         fi
1353
1354         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1355                 enable-event "$event_name" $chan -s $sess_name -j
1356         ok $? "Enable JUL event $event_name for session $sess_name"
1357 }
1358
1359 function enable_jul_lttng_event_loglevel()
1360 {
1361         local sess_name=$1
1362         local event_name="$2"
1363         local loglevel=$3
1364         local channel_name=$4
1365
1366         if [ -z $channel_name ]; then
1367                 # default channel if none specified
1368                 chan=""
1369         else
1370                 chan="-c $channel_name"
1371         fi
1372
1373         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1374                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j
1375         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1376 }
1377
1378 function enable_log4j_lttng_event()
1379 {
1380         local sess_name=$1
1381         local event_name=$2
1382         local channel_name=$3
1383
1384         local chan_opt=()
1385
1386         # default channel if none specified
1387         if [ -n "$channel_name" ]; then
1388                 chan_opt=("-c" "$channel_name")
1389         fi
1390
1391         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1392                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1393         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1394 }
1395
1396 function enable_log4j_lttng_event_filter()
1397 {
1398         local sess_name=$1
1399         local event_name=$2
1400         local filter=$3
1401
1402         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1403                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1404         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1405 }
1406
1407 function enable_log4j_lttng_event_filter_loglevel_only()
1408 {
1409         local sess_name=$1
1410         local event_name=$2
1411         local filter=$3
1412         local loglevel=$4
1413
1414         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1415                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1416         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1417 }
1418
1419 function enable_log4j_lttng_event_loglevel()
1420 {
1421         local sess_name=$1
1422         local event_name=$2
1423         local loglevel=$3
1424         local channel_name=$4
1425
1426
1427         # default channel if none specified
1428         if [ -n "$channel_name" ]; then
1429                 chan_opt=("-c" "$channel_name")
1430         fi
1431
1432         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1433                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1434         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1435 }
1436
1437 function enable_log4j_lttng_event_loglevel_only()
1438 {
1439         local sess_name=$1
1440         local event_name=$2
1441         local loglevel=$3
1442         local channel_name=$4
1443
1444         local chan_opt=()
1445
1446         # default channel if none specified
1447         if [ -n "$channel_name" ]; then
1448                 chan_opt=("-c" "$channel_name")
1449         fi
1450
1451         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1452                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1453         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1454 }
1455
1456 function enable_python_lttng_event()
1457 {
1458         sess_name=$1
1459         event_name="$2"
1460         channel_name=$3
1461
1462         if [ -z $channel_name ]; then
1463                 # default channel if none specified
1464                 chan=""
1465         else
1466                 chan="-c $channel_name"
1467         fi
1468
1469         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1470                 enable-event "$event_name" $chan -s $sess_name -p
1471         ok $? "Enable Python event $event_name for session $sess_name"
1472 }
1473
1474 function enable_python_lttng_event_loglevel()
1475 {
1476         local sess_name=$1
1477         local event_name="$2"
1478         local loglevel=$3
1479         local channel_name=$4
1480
1481         if [ -z $channel_name ]; then
1482                 # default channel if none specified
1483                 chan=""
1484         else
1485                 chan="-c $channel_name"
1486         fi
1487
1488         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1489                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p
1490         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1491 }
1492
1493 function enable_ust_lttng_event_filter()
1494 {
1495         local sess_name="$1"
1496         local event_name="$2"
1497         local filter="$3"
1498         local channel_name=$4
1499
1500         if [ -z $channel_name ]; then
1501                 # default channel if none specified
1502                 chan=""
1503         else
1504                 chan="-c $channel_name"
1505         fi
1506
1507         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1508                 enable-event $chan "$event_name" -s $sess_name -u --filter "$filter"
1509         ok $? "Enable event $event_name with filtering for session $sess_name"
1510 }
1511
1512 function enable_ust_lttng_event_loglevel()
1513 {
1514         local sess_name="$1"
1515         local event_name="$2"
1516         local loglevel="$3"
1517         local channel_name="$4"
1518         local chan=()
1519         if [ -n "${channel_name}" ] ; then
1520                 chan=('-c' "${channel_name}")
1521         fi
1522
1523         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1524                 enable-event "${chan[@]}" "$event_name" -s "${sess_name}" -u --loglevel="${loglevel}"
1525         ok $? "Enable event $event_name with loglevel $loglevel"
1526 }
1527
1528 function enable_ust_lttng_event_loglevel_only()
1529 {
1530         local sess_name="$1"
1531         local event_name="$2"
1532         local loglevel="$3"
1533         local channel_name="$4"
1534         local chan=()
1535         if [ -n "${channel_name}" ] ; then
1536                 chan=('-c' "${channel_name}")
1537         fi
1538
1539         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1540                 enable-event "${chan[@]}" "$event_name" -s "${sess_name}" -u --loglevel-only "${loglevel}"
1541         ok $? "Enable event $event_name with loglevel-only $loglevel"
1542 }
1543
1544 function disable_ust_lttng_event ()
1545 {
1546         local sess_name="$1"
1547         local event_name="$2"
1548         local channel_name="$3"
1549
1550         if [ -z $channel_name ]; then
1551                 # default channel if none specified
1552                 chan=""
1553         else
1554                 chan="-c $channel_name"
1555         fi
1556
1557         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1558                 disable-event "$event_name" -s $sess_name $chan -u
1559         ok $? "Disable event $event_name for session $sess_name"
1560 }
1561
1562 function disable_jul_lttng_event ()
1563 {
1564         local sess_name="$1"
1565         local event_name="$2"
1566
1567         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1568         ok $? "Disable JUL event $event_name for session $sess_name"
1569 }
1570
1571 function disable_log4j_lttng_event ()
1572 {
1573         local sess_name="$1"
1574         local event_name="$2"
1575
1576         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1577                 disable-event "$event_name" -s "$sess_name" --log4j
1578         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1579 }
1580
1581 function disable_python_lttng_event ()
1582 {
1583         local sess_name="$1"
1584         local event_name="$2"
1585
1586         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1587                 disable-event "$event_name" -s $sess_name -p
1588         ok $? "Disable Python event $event_name for session $sess_name"
1589 }
1590
1591 function start_lttng_tracing_opt ()
1592 {
1593         local withtap=$1
1594         local expected_to_fail=$2
1595         local sess_name=$3
1596
1597         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1598                 start $sess_name
1599         ret=$?
1600         if [[ $expected_to_fail -eq "1" ]]; then
1601                 test "$ret" -ne "0"
1602                 ret=$?
1603                 if [ $withtap -eq "1" ]; then
1604                         ok $? "Start tracing for session $sess_name failed as expected"
1605                 fi
1606         else
1607                 if [ $withtap -eq "1" ]; then
1608                         ok $ret "Start tracing for session $sess_name"
1609                 fi
1610         fi
1611 }
1612
1613 function start_lttng_tracing_ok ()
1614 {
1615         start_lttng_tracing_opt 1 0 "$@"
1616 }
1617
1618 function start_lttng_tracing_fail ()
1619 {
1620         start_lttng_tracing_opt 1 1 "$@"
1621 }
1622
1623 function start_lttng_tracing_notap ()
1624 {
1625         start_lttng_tracing_opt 0 1 "$@"
1626 }
1627
1628 function stop_lttng_tracing_opt ()
1629 {
1630         local withtap=$1
1631         local expected_to_fail=$2
1632         local sess_name=$3
1633
1634         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1635                 stop $sess_name
1636         ret=$?
1637         if [[ $expected_to_fail -eq "1" ]]; then
1638                 test "$ret" -ne "0"
1639                 ret=$?
1640                 if [ $withtap -eq "1" ]; then
1641                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1642                 fi
1643         else
1644                 if [ $withtap -eq "1" ]; then
1645                         ok $ret "Stop lttng tracing for session $sess_name"
1646                 fi
1647         fi
1648 }
1649
1650 function stop_lttng_tracing_ok ()
1651 {
1652         stop_lttng_tracing_opt 1 0 "$@"
1653 }
1654
1655 function stop_lttng_tracing_fail ()
1656 {
1657         stop_lttng_tracing_opt 1 1 "$@"
1658 }
1659
1660 function stop_lttng_tracing_notap ()
1661 {
1662         stop_lttng_tracing_opt 0 0 "$@"
1663 }
1664
1665 function destroy_lttng_session ()
1666 {
1667         local withtap=$1
1668         local expected_to_fail=$2
1669         local sess_name=$3
1670
1671         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1672                 destroy $sess_name
1673         ret=$?
1674         if [[ $expected_to_fail -eq "1" ]]; then
1675                 test "$ret" -ne "0"
1676                 ret=$?
1677                 if [ $withtap -eq "1" ]; then
1678                         ok $ret "Destroy session $sess_name failed as expected"
1679                 fi
1680         else
1681                 if [ $withtap -eq "1" ]; then
1682                         ok $ret "Destroy session $sess_name"
1683                 fi
1684         fi
1685 }
1686
1687 function destroy_lttng_session_ok ()
1688 {
1689         destroy_lttng_session 1 0 "$@"
1690
1691 }
1692
1693 function destroy_lttng_session_fail ()
1694 {
1695         destroy_lttng_session 1 1 "$@"
1696 }
1697
1698 function destroy_lttng_session_notap ()
1699 {
1700         destroy_lttng_session 0 0 "$@"
1701 }
1702
1703 function destroy_lttng_sessions ()
1704 {
1705         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1706                 destroy --all
1707         ok $? "Destroy all lttng sessions"
1708 }
1709
1710 function lttng_snapshot_add_output ()
1711 {
1712         local expected_to_fail=$1
1713         local sess_name=$2
1714         local trace_path=$3
1715         local opts=$4
1716
1717         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1718                 snapshot add-output -s $sess_name $trace_path $opts
1719         ret=$?
1720         if [[ $expected_to_fail -eq 1 ]]; then
1721                 test "$ret" -ne "0"
1722                 ok $? "Added snapshot output $trace_path failed as expected"
1723         else
1724                 ok $ret "Added snapshot output $trace_path"
1725         fi
1726 }
1727
1728 function lttng_snapshot_add_output_ok ()
1729 {
1730         lttng_snapshot_add_output 0 "$@"
1731 }
1732
1733 function lttng_snapshot_add_output_fail ()
1734 {
1735         lttng_snapshot_add_output 1 "$@"
1736 }
1737
1738 function lttng_snapshot_del_output ()
1739 {
1740         local expected_to_fail=$1
1741         local sess_name=$2
1742         local id=$3
1743
1744         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1745                 snapshot del-output -s $sess_name $id
1746         ret=$?
1747         if [[ $expected_to_fail -eq "1" ]]; then
1748                 test "$ret" -ne "0"
1749                 ok $? "Deleted snapshot output id $id failed as expected"
1750         else
1751                 ok $ret "Deleted snapshot output id $id"
1752         fi
1753 }
1754
1755 function lttng_snapshot_del_output_ok ()
1756 {
1757         lttng_snapshot_del_output 0 "$@"
1758 }
1759
1760 function lttng_snapshot_del_output_fail ()
1761 {
1762         lttng_snapshot_del_output 1 "$@"
1763 }
1764
1765 function lttng_snapshot_record ()
1766 {
1767         local sess_name=$1
1768         local trace_path=$2
1769
1770         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1771                 snapshot record -s "$sess_name" "$trace_path"
1772         ok $? "Snapshot recorded"
1773 }
1774
1775 function lttng_snapshot_list ()
1776 {
1777         local sess_name=$1
1778         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1779                 snapshot list-output -s $sess_name
1780         ok $? "Snapshot list"
1781 }
1782
1783 function lttng_save()
1784 {
1785         local sess_name=$1
1786         local opts=$2
1787
1788         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1789                 save $sess_name $opts
1790         ok $? "Session saved"
1791 }
1792
1793 function lttng_load()
1794 {
1795         local expected_to_fail=$1
1796         local opts=$2
1797
1798         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1799                 load $opts
1800         ret=$?
1801         if [[ $expected_to_fail -eq "1" ]]; then
1802                 test $ret -ne "0"
1803                 ok $? "Load command failed as expected with opts: $opts"
1804         else
1805                 ok $ret "Load command with opts: $opts"
1806         fi
1807 }
1808
1809 function lttng_load_ok()
1810 {
1811         lttng_load 0 "$@"
1812 }
1813
1814 function lttng_load_fail()
1815 {
1816         lttng_load 1 "$@"
1817 }
1818
1819 function lttng_track()
1820 {
1821         local expected_to_fail="$1"
1822         shift 1
1823         local opts="$@"
1824         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1825         ret=$?
1826         if [[ $expected_to_fail -eq "1" ]]; then
1827                 test $ret -ne "0"
1828                 ok $? "Track command failed as expected with opts: $opts"
1829         else
1830                 ok $ret "Track command with opts: $opts"
1831         fi
1832 }
1833
1834 function lttng_track_ok()
1835 {
1836         lttng_track 0 "$@"
1837 }
1838
1839 function lttng_track_fail()
1840 {
1841         lttng_track 1 "$@"
1842 }
1843
1844 function lttng_untrack()
1845 {
1846         local expected_to_fail="$1"
1847         shift 1
1848         local opts="$@"
1849         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1850         ret=$?
1851         if [[ $expected_to_fail -eq "1" ]]; then
1852                 test $ret -ne "0"
1853                 ok $? "Untrack command failed as expected with opts: $opts"
1854         else
1855                 ok $ret "Untrack command with opts: $opts"
1856         fi
1857 }
1858
1859 function lttng_untrack_ok()
1860 {
1861         lttng_untrack 0 "$@"
1862 }
1863
1864 function lttng_untrack_fail()
1865 {
1866         lttng_untrack 1 "$@"
1867 }
1868
1869 function lttng_track_pid_ok()
1870 {
1871         PID=$1
1872         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1873         ok $? "Lttng track pid on the kernel domain"
1874 }
1875
1876 function lttng_untrack_kernel_all_ok()
1877 {
1878         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1879         ok $? "Lttng untrack all pid on the kernel domain"
1880 }
1881
1882 function lttng_track_ust_ok()
1883 {
1884         lttng_track_ok -u "$@"
1885 }
1886
1887 function lttng_track_ust_fail()
1888 {
1889         lttng_track_fail -u "$@"
1890 }
1891
1892 function lttng_track_kernel_ok()
1893 {
1894         lttng_track_ok -k "$@"
1895 }
1896
1897 function lttng_track_kernel_fail()
1898 {
1899         lttng_track_fail -k "$@"
1900 }
1901
1902 function lttng_untrack_ust_ok()
1903 {
1904         lttng_untrack_ok -u "$@"
1905 }
1906
1907 function lttng_untrack_ust_fail()
1908 {
1909         lttng_untrack_fail -u "$@"
1910 }
1911
1912 function lttng_untrack_kernel_ok()
1913 {
1914         lttng_untrack_ok -k "$@"
1915 }
1916
1917 function lttng_untrack_kernel_fail()
1918 {
1919         lttng_untrack_fail -k "$@"
1920 }
1921
1922 function lttng_add_context_list()
1923 {
1924         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1925                 add-context --list
1926         ret=$?
1927         ok $ret "Context listing"
1928 }
1929
1930 function add_context_lttng()
1931 {
1932         local expected_to_fail="$1"
1933         local domain="$2"
1934         local session_name="$3"
1935         local channel_name="$4"
1936         local type="$5"
1937
1938         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1939                 add-context -s $session_name -c $channel_name -t $type $domain
1940         ret=$?
1941         if [[ $expected_to_fail -eq "1" ]]; then
1942                 test $ret -ne "0"
1943                 ok $? "Add context command failed as expected for type: $type"
1944         else
1945                 ok $ret "Add context command for type: $type"
1946         fi
1947 }
1948
1949 function add_context_ust_ok()
1950 {
1951         add_context_lttng 0 -u "$@"
1952 }
1953
1954 function add_context_ust_fail()
1955 {
1956         add_context_lttng 1 -u "$@"
1957 }
1958
1959 function add_context_kernel_ok()
1960 {
1961         add_context_lttng 0 -k "$@"
1962 }
1963
1964 function add_context_kernel_fail()
1965 {
1966         add_context_lttng 1 -k "$@"
1967 }
1968
1969 function wait_live_trace_ready ()
1970 {
1971         local url=$1
1972         local zero_client_match=0
1973
1974         diag "Waiting for live trace at url: $url"
1975         while [ $zero_client_match -eq 0 ]; do
1976                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1977                 sleep 0.5
1978         done
1979         pass "Waiting for live trace at url: $url"
1980 }
1981
1982 function wait_live_viewer_connect ()
1983 {
1984         local url=$1
1985         local one_client_match=0
1986
1987         diag "Waiting for live viewers on url: $url"
1988         while [ $one_client_match -eq 0 ]; do
1989                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1990                 sleep 0.5
1991         done
1992         pass "Waiting for live viewers on url: $url"
1993 }
1994
1995 function bail_out_if_no_babeltrace()
1996 {
1997         which "$BABELTRACE_BIN" >/dev/null
1998         if [ $? -ne 0 ]; then
1999                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
2000         fi
2001 }
2002
2003 # Check that the trace metadata contains '$expected' event ids matching '$event_name'.
2004 function validate_metadata_event()
2005 {
2006         local event_name=$1
2007         local expected=$2
2008         local trace_path=$3
2009
2010         local metadata_file
2011         local metadata_path
2012         local count
2013
2014         metadata_file=$(find "$trace_path" -name "metadata")
2015         metadata_path=$(dirname "$metadata_file")
2016
2017         bail_out_if_no_babeltrace
2018
2019         count=$($BABELTRACE_BIN --output-format=ctf-metadata "$metadata_path" | grep -c "$event_name")
2020
2021         test "$count" -eq "$expected"
2022         ok $? "Found $count / $expected metadata event id matching '$event_name'"
2023 }
2024
2025 # Check that the trace contains '$expected' events matching '$event_name', other
2026 # events not matching '$event_name' can be present.
2027 function trace_matches()
2028 {
2029         local event_name=$1
2030         local expected=$2
2031         local trace_path=$3
2032
2033         local count
2034         local total
2035
2036         bail_out_if_no_babeltrace
2037
2038         count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2039         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2040
2041         test "$count" -eq "$expected"
2042
2043         ok $? "Found $count / $expected events matching '$event_name' out of $total events"
2044 }
2045
2046 # Check that the trace contains '$expected' events matching '$event_name' and no
2047 # other events.
2048 function trace_match_only()
2049 {
2050         local event_name=$1
2051         local expected=$2
2052         local trace_path=$3
2053
2054         local count
2055         local total
2056
2057         bail_out_if_no_babeltrace
2058
2059         count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2060         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2061
2062         test "$expected" -eq "$count" && test "$total" -eq "$expected"
2063
2064         ok $? "Found $count / $expected events matching '$event_name' amongst $total events"
2065 }
2066
2067 # Check that the trace contains at least 1 event matching each name in the
2068 # comma separated list '$event_names'.
2069 function validate_trace()
2070 {
2071         local event_names=$1
2072         local trace_path=$2
2073
2074         local count
2075
2076         bail_out_if_no_babeltrace
2077
2078         OLDIFS=$IFS
2079         IFS=","
2080         for event_name in $event_names; do
2081                 # trace_path is unquoted since callers make use of globbing
2082                 count=$($BABELTRACE_BIN $trace_path | grep -c "$event_name")
2083                 test "$count" -gt 0
2084                 ok $? "Found $count events matching '$event_name'"
2085         done
2086         IFS=$OLDIFS
2087 }
2088
2089 # Check that the trace contains at least 1 event matching each name in the
2090 # comma separated list '$event_names' and a total of '$expected' events.
2091 function validate_trace_count()
2092 {
2093         local event_names=$1
2094         local trace_path=$2
2095         local expected=$3
2096
2097         local count
2098         local total=0
2099
2100         bail_out_if_no_babeltrace
2101
2102         OLDIFS=$IFS
2103         IFS=","
2104         for event_name in $event_names; do
2105                 count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2106                 test "$count" -gt 0
2107                 ok $? "Found '$count' events matching '$event_name'"
2108                 total=$(( total + count ))
2109         done
2110         IFS=$OLDIFS
2111         test $total -eq "$expected"
2112         ok $? "Found $total events, expected $expected events"
2113 }
2114
2115 # Check that the trace contains at least '$expected_min' event matching each
2116 # name in the comma separated list '$event_names' and a total at least
2117 # '$expected_min' and less than '$expected_max' events.
2118 function validate_trace_count_range_incl_min_excl_max()
2119 {
2120         local event_names=$1
2121         local trace_path=$2
2122         local expected_min=$3
2123         local expected_max=$4
2124
2125         local count
2126         local total=0
2127
2128         bail_out_if_no_babeltrace
2129
2130         OLDIFS=$IFS
2131         IFS=","
2132         for event_name in $event_names; do
2133                 count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2134                 test "$count" -ge "$expected_min"
2135                 ok $? "Found $count events matching '$event_name', expected at least $expected_min"
2136                 total=$(( total + count ))
2137         done
2138         IFS=$OLDIFS
2139         test $total -ge "$expected_min" && test $total -lt "$expected_max"
2140         ok $? "Found a total of $total events, expected at least $expected_min and less than $expected_max"
2141 }
2142
2143 function trace_first_line()
2144 {
2145         local trace_path=$1
2146
2147         $BABELTRACE_BIN "$trace_path" | head -n 1
2148 }
2149
2150 # Check that the trace contains at least 1 event matching the grep extended
2151 # regexp '$event_exp'.
2152 function validate_trace_exp()
2153 {
2154         local event_exp=$1
2155         local trace_path=$2
2156
2157         local count
2158
2159         bail_out_if_no_babeltrace
2160
2161         # event_exp is unquoted since it contains multiple grep arguments
2162         count=$($BABELTRACE_BIN "$trace_path" | grep -c --extended-regexp $event_exp)
2163         test "$count" -gt 0
2164         ok $? "Found $count events matching expression '$event_exp'"
2165 }
2166
2167 # Check that the trace contains at least 1 event matching the grep extended
2168 # regexp '$event_exp' and zero event not matching it.
2169 function validate_trace_only_exp()
2170 {
2171         local event_exp=$1
2172         local trace_path=$2
2173
2174         local count
2175         local total
2176
2177         bail_out_if_no_babeltrace
2178
2179         # event_exp is unquoted since it contains multiple grep arguments
2180         count=$($BABELTRACE_BIN "$trace_path" | grep -c --extended-regexp $event_exp)
2181         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2182
2183         test  "$count" -gt 0 && test "$total" -eq "$count"
2184         ok $? "Found $count events matching expression '$event_exp' amongst $total events"
2185 }
2186
2187 # Check that the trace is valid and contains 0 event.
2188 function validate_trace_empty()
2189 {
2190         local trace_path=$1
2191
2192         local ret
2193         local count
2194
2195         bail_out_if_no_babeltrace
2196
2197         events=$($BABELTRACE_BIN "$trace_path")
2198         ret=$?
2199         if [ $ret -ne 0 ]; then
2200                 fail "Failed to parse trace"
2201                 return $ret
2202         fi
2203
2204         count=$(echo -n "$events" | wc -l)
2205         test "$count" -eq 0
2206         ok $? "Validate trace is empty, found $count events"
2207 }
2208
2209 function validate_directory_empty ()
2210 {
2211         local trace_path="$1"
2212
2213         local files
2214         local ret
2215         local nb_files
2216
2217         # Do not double quote `$trace_path` below as we want wildcards to be
2218         # expanded.
2219         files="$(ls -A $trace_path)"
2220         ret=$?
2221         if [ $ret -ne 0 ]; then
2222                 fail "Failed to list content of directory \"$trace_path\""
2223                 return $ret
2224         fi
2225
2226         nb_files="$(echo -n "$files" | wc -l)"
2227         test "$nb_files" -eq 0
2228         ok $? "Directory \"$trace_path\" is empty"
2229 }
2230
2231 function validate_trace_session_ust_empty()
2232 {
2233         validate_directory_empty "$1"/ust
2234 }
2235
2236 function validate_trace_session_kernel_empty()
2237 {
2238         validate_trace_empty "$1"/kernel
2239 }
2240
2241 function regenerate_metadata ()
2242 {
2243         local expected_to_fail=$1
2244         local sess_name=$2
2245
2246         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2247                 regenerate metadata -s $sess_name
2248         ret=$?
2249         if [[ $expected_to_fail -eq "1" ]]; then
2250                 test "$ret" -ne "0"
2251                 ok $? "Expected fail on regenerate metadata $sess_name"
2252         else
2253                 ok $ret "Metadata regenerate $sess_name"
2254         fi
2255 }
2256
2257 function regenerate_metadata_ok ()
2258 {
2259         regenerate_metadata 0 "$@"
2260 }
2261
2262 function regenerate_metadata_fail ()
2263 {
2264         regenerate_metadata 1 "$@"
2265 }
2266
2267 function regenerate_statedump ()
2268 {
2269         local expected_to_fail=$1
2270         local sess_name=$2
2271
2272         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2273                 regenerate statedump -s $sess_name
2274         ret=$?
2275         if [[ $expected_to_fail -eq "1" ]]; then
2276                 test "$ret" -ne "0"
2277                 ok $? "Expected fail on regenerate statedump $sess_name"
2278         else
2279                 ok $ret "Statedump regenerate $sess_name"
2280         fi
2281 }
2282
2283 function regenerate_statedump_ok ()
2284 {
2285         regenerate_statedump 0 "$@"
2286 }
2287
2288 function regenerate_statedump_fail ()
2289 {
2290         regenerate_statedump 1 "$@"
2291 }
2292
2293 function rotate_session ()
2294 {
2295         local expected_to_fail=$1
2296         local sess_name=$2
2297
2298         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2299                 rotate $sess_name
2300         ret=$?
2301         if [[ $expected_to_fail -eq "1" ]]; then
2302                 test "$ret" -ne "0"
2303                 ok $? "Expected fail on rotate session $sess_name"
2304         else
2305                 ok $ret "Rotate session $sess_name"
2306         fi
2307 }
2308
2309 function rotate_session_ok ()
2310 {
2311         rotate_session 0 "$@"
2312 }
2313
2314 function rotate_session_fail ()
2315 {
2316         rotate_session 1 "$@"
2317 }
2318
2319 function destructive_tests_enabled ()
2320 {
2321         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2322                 return 0
2323         else
2324                 return 1
2325         fi
2326 }
2327
2328 function lttng_enable_rotation_timer ()
2329 {
2330         local expected_to_fail=$1
2331         local sess_name=$2
2332         local period=$3
2333
2334         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2335                 enable-rotation -s $sess_name --timer $period
2336         ret=$?
2337         if [[ $expected_to_fail -eq "1" ]]; then
2338                 test "$ret" -ne "0"
2339                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2340         else
2341                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2342         fi
2343 }
2344
2345 function lttng_enable_rotation_timer_ok ()
2346 {
2347         lttng_enable_rotation_timer 0 $@
2348 }
2349
2350 function lttng_enable_rotation_timer_fail ()
2351 {
2352         lttng_enable_rotation_timer 1 $@
2353 }
2354
2355 function lttng_enable_rotation_size ()
2356 {
2357         local expected_to_fail=$1
2358         local sess_name=$2
2359         local size=$3
2360
2361         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2362                 enable-rotation -s $sess_name --size $size
2363         ret=$?
2364         if [[ $expected_to_fail -eq "1" ]]; then
2365                 test "$ret" -ne "0"
2366                 ok $? "Expected to fail to set a periodic rotation of session $sess_name" "every " $size " bytes"
2367         else
2368                 ok $ret "Set a scheduled rotation of session $sess_name" "every " $size " bytes"
2369         fi
2370 }
2371
2372 function lttng_enable_rotation_size_ok ()
2373 {
2374         lttng_enable_rotation_size 0 $@
2375 }
2376
2377 function lttng_enable_rotation_size_fail ()
2378 {
2379         lttng_enable_rotation_size 1 $@
2380 }
2381
2382 function lttng_clear_session ()
2383 {
2384         local expected_to_fail=$1
2385         local sess_name=$2
2386
2387         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2388                 clear $sess_name
2389         ret=$?
2390         if [[ $expected_to_fail -eq "1" ]]; then
2391                 test "$ret" -ne "0"
2392                 ok $? "Expected fail on clear session $sess_name"
2393         else
2394                 ok $ret "Clear session $sess_name"
2395         fi
2396 }
2397
2398 function lttng_clear_session_ok ()
2399 {
2400         lttng_clear_session 0 $@
2401 }
2402
2403 function lttng_clear_session_fail ()
2404 {
2405         lttng_clear_session 1 $@
2406 }
2407
2408 function lttng_clear_all ()
2409 {
2410         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2411                 clear --all
2412         ok $? "Clear all lttng sessions"
2413 }
2414
2415 function lttng_add_trigger()
2416 {
2417         local expected_to_fail="$1"
2418         local trigger_name="$2"
2419         shift 2
2420         local args=("$@")
2421
2422         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2423         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2424         ret=$?
2425         if [[ $expected_to_fail -eq "1" ]]; then
2426                 test "$ret" -ne "0"
2427                 ok $? "Add trigger $trigger_name failed as expected"
2428         else
2429                 ok $ret "Add trigger $trigger_name"
2430         fi
2431 }
2432
2433 function lttng_remove_trigger()
2434 {
2435         local expected_to_fail="$1"
2436         local trigger_name="$2"
2437         shift 2
2438
2439         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2440         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2441         ret=$?
2442         if [[ $expected_to_fail -eq "1" ]]; then
2443                 test "$ret" -ne "0"
2444                 ok $? "Remove trigger $trigger_name failed as expected"
2445         else
2446                 ok $ret "Remove trigger $trigger_name"
2447         fi
2448 }
2449
2450 function lttng_add_trigger_ok()
2451 {
2452         lttng_add_trigger 0 "$@"
2453 }
2454
2455 function lttng_add_trigger_fail()
2456 {
2457         lttng_add_trigger 1 "$@"
2458 }
2459
2460 function lttng_remove_trigger_ok()
2461 {
2462         lttng_remove_trigger 0 "$@"
2463 }
2464
2465 function list_triggers_matches_ok ()
2466 {
2467         local tmp_stdout=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2468         local tmp_stderr=$(mktemp -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2469
2470         local test_name="$1"
2471         local expected_stdout_file="$2"
2472
2473         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2474
2475         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2476         ok $? "${test_name}: exit code is 0"
2477
2478         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2479         ok $? "${test_name}: expected stdout"
2480
2481         diff -u /dev/null "${tmp_stderr}"
2482         ok $? "${test_name}: expected stderr"
2483
2484         rm -f "${tmp_stdout}"
2485         rm -f "${tmp_stderr}"
2486 }
2487
2488 function list_triggers_matches_mi_ok ()
2489 {
2490         local tmp_stdout
2491         local tmp_stdout_raw
2492         local tmp_stderr
2493
2494         local test_name="$1"
2495         local expected_stdout_file="$2"
2496
2497         tmp_stdout_raw=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2498         tmp_stdout=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2499         tmp_stderr=$(mktemp -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2500
2501         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2502
2503         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2504         ok $? "${test_name}: exit code is 0"
2505
2506         # Pretty-fy xml before further test.
2507         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2508
2509         $MI_VALIDATE "${tmp_stdout}"
2510         ok $? "list-trigger mi is valid"
2511
2512         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2513         ok $? "${test_name}: expected stdout"
2514
2515         diff -u /dev/null "${tmp_stderr}"
2516         ok $? "${test_name}: expected stderr"
2517
2518         rm -f "${tmp_stdout}"
2519         rm -f "${tmp_stdout_raw}"
2520         rm -f "${tmp_stderr}"
2521 }
2522
2523 function validate_path_pattern ()
2524 {
2525         local message=$1
2526         local pattern=$2
2527         # Base path is only used in error case and is used to list the content
2528         # of the base path.
2529         local base_path=$3
2530
2531
2532         [ -f $pattern ]
2533         ret=$?
2534         ok $ret "$message"
2535
2536         if [ "$ret" -ne "0" ]; then
2537                 diag "Path pattern expected: $pattern"
2538                 # List the tracepath for more info. We use find as a recursive
2539                 # directory lister.
2540                 diag "The base path content:"
2541                 find "$base_path" -print
2542         fi
2543 }
2544
2545 function validate_trace_path_ust_uid ()
2546 {
2547         local trace_path=$1
2548         local session_name=$2
2549         local uid=$UID
2550         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2551
2552         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2553 }
2554
2555 function validate_trace_path_ust_uid_network ()
2556 {
2557         local trace_path=$1
2558         local session_name=$2
2559         local base_path=$3
2560         local uid=$UID
2561         local hostname=$HOSTNAME
2562         local pattern
2563         local ret
2564
2565         # If the session was given a network base path (e.g
2566         # 127.0.0.1/my/custom/path on creation, there is no session name
2567         # component to the path on the relayd side. Caller can simply not pass a
2568         # session name for this scenario.
2569         if [ -n "$session_name" ]; then
2570                 session_name="$session_name-$date_time_pattern"
2571                 if [ -n "$base_path" ]; then
2572                         fail "Session name and base path are mutually exclusive"
2573                         return
2574                 fi
2575         fi
2576
2577         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2578
2579         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2580 }
2581
2582 function validate_trace_path_ust_uid_snapshot_network ()
2583 {
2584         local trace_path=$1
2585         local session_name=$2
2586         local snapshot_name=$3
2587         local snapshot_number=$4
2588         local base_path=$5
2589         local hostname=$HOSTNAME
2590         local uid=$UID
2591         local pattern
2592         local ret
2593
2594         # If the session/output was given a network base path (e.g
2595         # 127.0.0.1/my/custom/path on creation, there is no session name
2596         # component to the path on the relayd side. Caller can simply not pass a
2597         # session name for this scenario.
2598         if [ -n "$session_name" ]; then
2599                 session_name="$session_name-$date_time_pattern"
2600                 if [ -n "$base_path" ]; then
2601                         fail "Session name and base path are mutually exclusive"
2602                         return
2603                 fi
2604         fi
2605
2606         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2607
2608         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2609 }
2610
2611 function validate_trace_path_ust_uid_snapshot ()
2612 {
2613         local trace_path=$1
2614         local session_name=$2
2615         local snapshot_name=$3
2616         local snapshot_number=$4
2617         local base_path=$5
2618         local uid=$UID
2619         local pattern
2620         local ret
2621
2622         # If the session/output was given a network base path (e.g
2623         # 127.0.0.1/my/custom/path) on creation, there is no session name
2624         # component to the path on the relayd side. Caller can simply not pass a
2625         # session name for this scenario.
2626         if [ -n "$session_name" ]; then
2627                 session_name="$session_name-$date_time_pattern"
2628                 if [ -n "$base_path" ]; then
2629                         fail "Session name and base path are mutually exclusive"
2630                         return
2631                 fi
2632         fi
2633
2634         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2635
2636         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2637 }
2638
2639 function validate_trace_path_ust_pid ()
2640 {
2641         local trace_path=$1
2642         local session_name=$2
2643         local app_string=$3
2644         local pid=$4
2645         local pattern
2646         local ret
2647
2648         # If the session was given a trace path on creation, there is no session
2649         # name component to the path. Caller can simply not pass a session name
2650         # for this scenario.
2651         if [ -n "$session_name" ]; then
2652                 session_name="$session_name-$date_time_pattern"
2653         fi
2654
2655         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2656
2657         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2658 }
2659
2660 function validate_trace_path_kernel ()
2661 {
2662         local trace_path=$1
2663         local session_name=$2
2664         local pattern
2665
2666         # If the session was given a trace path on creation, there is no session
2667         # name component to the path. Caller can simply not pass a session name
2668         # for this scenario.
2669         if [ -n "$session_name" ]; then
2670                 session_name="$session_name-$date_time_pattern"
2671         fi
2672
2673         pattern="$trace_path/$session_name/kernel/metadata"
2674
2675         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2676 }
2677
2678 function validate_trace_path_kernel_network ()
2679 {
2680         local trace_path=$1
2681         local session_name=$2
2682         local hostname=$HOSTNAME
2683         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2684
2685         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2686 }
2687
2688 function validate_trace_path_kernel_snapshot ()
2689 {
2690         local trace_path=$1
2691         local session_name=$2
2692         local snapshot_name=$3
2693         local snapshot_number=$4
2694         local base_path=$5
2695         local pattern
2696         local ret
2697
2698         # If the session/output was given a network base path (e.g
2699         # 127.0.0.1/my/custom/path on creation, there is no session name
2700         # component to the path on the relayd side. Caller can simply not pass a
2701         # session name for this scenario.
2702         if [ -n "$session_name" ]; then
2703                 session_name="$session_name-$date_time_pattern"
2704                 if [ -n "$base_path" ]; then
2705                         fail "Session name and base path are mutually exclusive"
2706                         return
2707                 fi
2708         fi
2709
2710         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2711
2712         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2713 }
2714
2715 function validate_trace_path_kernel_snapshot_network ()
2716 {
2717         local trace_path=$1
2718         local session_name=$2
2719         local snapshot_name=$3
2720         local snapshot_number=$4
2721         local base_path=$5
2722         local hostname=$HOSTNAME
2723         local pattern
2724         local ret
2725
2726         # If the session/output was given a network base path (e.g
2727         # 127.0.0.1/my/custom/path on creation, there is no session name
2728         # component to the path on the relayd side. Caller can simply not pass a
2729         # session name for this scenario.
2730         if [ -n "$session_name" ]; then
2731                 session_name="$session_name-$date_time_pattern"
2732                 if [ -n "$base_path" ]; then
2733                         fail "Session name and base path are mutually exclusive"
2734                         return
2735                 fi
2736         fi
2737
2738         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2739
2740         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2741 }