tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace2"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$( (tr '\0' '\n' < /proc/"$pid"/cmdline) 2>/dev/null | head -n1); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         local len="${1:-16}"
 208
 209         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 210         # /dev/urandom isn't guaranteed to generate valid multi-byte characters.
 211         # Specifying the C locale eliminates the "Illegal byte sequence" error
 212         # that 'tr' outputs in such cases.
 213         LC_CTYPE=C tr -cd "$CHAR" < /dev/urandom 2>/dev/null | head -c "$len" 2>/dev/null
 214         echo
 215 }
 216
 217 # Return a space-separated string of online CPU IDs, based on
 218 # /sys/devices/system/cpu/online, or from 0 to nproc - 1 otherwise.
 219 function get_online_cpus()
 220 {
 221         local cpus=()
 222         local range_re
 223         if [ -f /sys/devices/system/cpu/online ]; then
 224                 range_re='([0-9]+)-([0-9]+)'
 225                 while read -r range ; do
 226                         if [[ "${range}" =~ ${range_re} ]] ; then
 227                                 mapfile -t -O "${#cpus[*]}" cpus <<< $(seq "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}")
 228                         else
 229                                 cpus+=("${range}")
 230                         fi
 231                 done < <(tr ',' $'\n' < /sys/devices/system/cpu/online)
 232         else
 233                 read -r -a cpus <<< $(seq 0 $(( $(conf_proc_count) - 1 )) )
 234         fi
 235         echo "${cpus[*]}"
 236 }
 237
 238 # Helpers for get_possible_cpus.
 239 function get_possible_cpus_count_from_sysfs_possible_mask()
 240 {
 241         local max_possible_cpu_id
 242
 243         # The Awk script extracts the highest CPU id from the possible CPU
 244         # mask. Assuming a numerical order, a field separator '-' and a record
 245         # separator ','. The last value parsed is the highest id.
 246         if [ -f /sys/devices/system/cpu/possible ]; then
 247                 max_possible_cpu_id=$(awk -F '-' 'BEGIN { RS = ","} { last = $NF } END { printf("%d\n", last) }' \
 248                                       /sys/devices/system/cpu/possible)
 249                 echo "$((max_possible_cpu_id+1))"
 250         else
 251                 echo "0"
 252         fi
 253 }
 254
 255 # This is a fallback if the possible CPU mask is not available. This will not
 256 # take into account unplugged CPUs.
 257 function get_max_cpus_count_from_sysfs_cpu_directories()
 258 {
 259         local max_possible_cpu_id=0
 260         local current_cpu_id
 261
 262         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 263                 current_cpu_id="${i#/sys/devices/system/cpu/cpu}"
 264                 if [ "$current_cpu_id" -gt "$max_possible_cpu_id" ]; then
 265                         max_possible_cpu_id="$current_cpu_id"
 266                 fi
 267         done
 268
 269         echo "$((max_possible_cpu_id+1))"
 270 }
 271
 272 # Return the number of possible CPUs.
 273 function get_possible_cpus_count()
 274 {
 275         local possible_cpus_count
 276         possible_cpus_count=$(get_possible_cpus_count_from_sysfs_possible_mask)
 277
 278         if [ "$possible_cpus_count" -eq "0" ]; then
 279                 local configured_cpus_count
 280                 configured_cpus_count=$(getconf _NPROCESSORS_CONF)
 281                 possible_cpus_count=$(get_max_cpus_count_from_sysfs_cpu_directories)
 282                 possible_cpus_count=$((configured_cpus_count > possible_cpus_count \
 283                                                              ? configured_cpus_count \
 284                                                              : possible_cpus_count))
 285         fi
 286
 287         echo "$possible_cpus_count"
 288 }
 289
 290 # Return the list of exposed CPU.
 291 #
 292 # NOTE! Use it like so:
 293 #
 294 # IFS=" " read -r -a VARIABLE <<< "$(get_exposed_cpus_list)"
 295 function get_exposed_cpus_list()
 296 {
 297         local list=()
 298
 299         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 300                 list+=("${i#/sys/devices/system/cpu/cpu}")
 301         done
 302
 303         echo "${list[@]}"
 304 }
 305
 306 # Return any available CPU found. Do not make assumption about the returned
 307 # value, e.g. that it could be 0.
 308 function get_any_available_cpu()
 309 {
 310         for cpu in $(get_online_cpus); do
 311                 echo "${cpu}"
 312                 break;
 313         done
 314 }
 315
 316 # Return the number of _configured_ CPUs.
 317 function conf_proc_count()
 318 {
 319         getconf _NPROCESSORS_CONF
 320         if [ $? -ne 0 ]; then
 321                 diag "Failed to get the number of configured CPUs"
 322         fi
 323         echo
 324 }
 325
 326 # Check if base lttng-modules are present.
 327 # Bail out on failure
 328 function validate_lttng_modules_present ()
 329 {
 330         # Check for loadable modules.
 331         modprobe -n lttng-tracer 2>/dev/null
 332         if [ $? -eq 0 ]; then
 333                 return 0
 334         fi
 335
 336         # Check for builtin modules.
 337         ls /proc/lttng > /dev/null 2>&1
 338         if [ $? -eq 0 ]; then
 339                 return 0
 340         fi
 341
 342         LTTNG_BAIL_OUT "LTTng modules not detected."
 343 }
 344
 345 # Run the lttng binary.
 346 #
 347 # The first two arguments are stdout and stderr redirect paths, respectively.
 348 # The rest of the arguments are forwarded to the lttng binary
 349 function _run_lttng_cmd
 350 {
 351         local stdout_dest="$1"
 352         local stderr_dest="$2"
 353         shift 2
 354
 355         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 356         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 357 }
 358
 359 function enable_kernel_lttng_event
 360 {
 361         local withtap="$1"
 362         local expected_to_fail="$2"
 363         local sess_name="$3"
 364         local event_name="$4"
 365         local channel_name="$5"
 366
 367         if [ -z "$event_name" ]; then
 368                 # Enable all event if no event name specified
 369                 event_name="-a"
 370         fi
 371
 372         if [ -z "$channel_name" ]; then
 373                 # default channel if none specified
 374                 chan=""
 375         else
 376                 chan="-c $channel_name"
 377         fi
 378
 379         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 380                 enable-event "$event_name" $chan -s $sess_name -k
 381         ret=$?
 382         if [[ $expected_to_fail -eq "1" ]]; then
 383                 test $ret -ne "0"
 384                 ret=$?
 385                 if [ $withtap -eq "1" ]; then
 386                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 387                 fi
 388         else
 389                 if [ $withtap -eq "1" ]; then
 390                         ok $ret "Enable kernel event $event_name for session $sess_name"
 391                 fi
 392         fi
 393 }
 394
 395 function enable_kernel_lttng_event_ok ()
 396 {
 397         enable_kernel_lttng_event 1 0 "$@"
 398 }
 399
 400 function enable_kernel_lttng_event_fail ()
 401 {
 402         enable_kernel_lttng_event 1 1 "$@"
 403 }
 404
 405 function enable_kernel_lttng_event_notap ()
 406 {
 407         enable_kernel_lttng_event 0 0 "$@"
 408 }
 409
 410 # Old interface
 411 function lttng_enable_kernel_event
 412 {
 413         enable_kernel_lttng_event_ok "$@"
 414 }
 415
 416 function lttng_enable_kernel_syscall()
 417 {
 418         local expected_to_fail=$1
 419         local sess_name=$2
 420         local syscall_name=$3
 421         local channel_name=$4
 422
 423         if [ -z $syscall_name ]; then
 424                 # Enable all event if no syscall name specified
 425                 syscall_name="-a"
 426         fi
 427
 428         if [ -z $channel_name ]; then
 429                 # default channel if none specified
 430                 chan=""
 431         else
 432                 chan="-c $channel_name"
 433         fi
 434
 435         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 436                 enable-event --syscall "$syscall_name" $chan -s $sess_name -k
 437         ret=$?
 438         if [[ $expected_to_fail -eq "1" ]]; then
 439                 test $ret -ne "0"
 440                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 441         else
 442                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 443         fi
 444 }
 445
 446 function lttng_enable_kernel_syscall_ok()
 447 {
 448         lttng_enable_kernel_syscall 0 "$@"
 449 }
 450
 451 function lttng_enable_kernel_syscall_fail()
 452 {
 453         lttng_enable_kernel_syscall 1 "$@"
 454 }
 455
 456 function lttng_disable_kernel_syscall()
 457 {
 458         local expected_to_fail=$1
 459         local sess_name=$2
 460         local syscall_name=$3
 461         local channel_name=$4
 462
 463         if [ -z $syscall_name ]; then
 464                 # Enable all event if no syscall name specified
 465                 syscall_name="-a"
 466         fi
 467
 468         if [ -z $channel_name ]; then
 469                 # default channel if none specified
 470                 chan=""
 471         else
 472                 chan="-c $channel_name"
 473         fi
 474
 475         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 476                 disable-event --syscall "$syscall_name" $chan -s $sess_name -k
 477
 478         ret=$?
 479         if [[ $expected_to_fail -eq "1" ]]; then
 480                 test $ret -ne "0"
 481                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 482         else
 483                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 484         fi
 485 }
 486
 487 function lttng_disable_kernel_syscall_ok()
 488 {
 489         lttng_disable_kernel_syscall 0 "$@"
 490 }
 491
 492 function lttng_disable_kernel_syscall_fail()
 493 {
 494         lttng_disable_kernel_syscall 1 "$@"
 495 }
 496
 497 function lttng_enable_kernel_function_event ()
 498 {
 499         local expected_to_fail="$1"
 500         local sess_name="$2"
 501         local target="$3"
 502         local event_name="$4"
 503
 504         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 505         ret=$?
 506         if [[ $expected_to_fail -eq "1" ]]; then
 507                 test $ret -ne "0"
 508                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 509         else
 510                 ok $ret "Enable kernel function event for session $sess_name"
 511         fi
 512 }
 513
 514 function lttng_enable_kernel_function_event_ok ()
 515 {
 516         lttng_enable_kernel_function_event 0 "$@"
 517 }
 518
 519 function lttng_enable_kernel_userspace_probe_event ()
 520 {
 521         local expected_to_fail="$1"
 522         local sess_name="$2"
 523         local target="$3"
 524         local event_name="$4"
 525
 526         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 527         ret=$?
 528         if [[ $expected_to_fail -eq "1" ]]; then
 529                 test $ret -ne "0"
 530                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 531         else
 532                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 533         fi
 534 }
 535
 536 function lttng_enable_kernel_userspace_probe_event_fail ()
 537 {
 538         lttng_enable_kernel_userspace_probe_event 1 "$@"
 539 }
 540
 541 function lttng_enable_kernel_userspace_probe_event_ok ()
 542 {
 543         lttng_enable_kernel_userspace_probe_event 0 "$@"
 544 }
 545
 546 function disable_kernel_lttng_userspace_probe_event_ok ()
 547 {
 548         local sess_name="$1"
 549         local event_name="$2"
 550
 551         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 552         ok $? "Disable kernel event $target for session $sess_name"
 553 }
 554 function lttng_enable_kernel_channel()
 555 {
 556         local withtap=$1
 557         local expected_to_fail=$2
 558         local sess_name=$3
 559         local channel_name=$4
 560         local opts="${@:5}"
 561
 562         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 563                 enable-channel -k $channel_name -s $sess_name $opts
 564         ret=$?
 565         if [[ $expected_to_fail -eq "1" ]]; then
 566                 test "$ret" -ne "0"
 567                 ret=$?
 568                 if [ $withtap -eq "1" ]; then
 569                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 570                 fi
 571         else
 572                 if [ $withtap -eq "1" ]; then
 573                         ok $ret "Enable channel $channel_name for session $sess_name"
 574                 fi
 575         fi
 576 }
 577
 578 function lttng_enable_kernel_channel_ok()
 579 {
 580         lttng_enable_kernel_channel 1 0 "$@"
 581 }
 582
 583 function lttng_enable_kernel_channel_fail()
 584 {
 585         lttng_enable_kernel_channel 1 1 "$@"
 586 }
 587
 588 function lttng_enable_kernel_channel_notap()
 589 {
 590         lttng_enable_kernel_channel 0 0 "$@"
 591 }
 592
 593 function enable_kernel_lttng_channel_ok()
 594 {
 595         lttng_enable_kernel_channel 1 0 "$@"
 596 }
 597
 598 function lttng_disable_kernel_channel()
 599 {
 600         local expected_to_fail=$1
 601         local sess_name=$2
 602         local channel_name=$3
 603
 604         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 605                 disable-channel -k $channel_name -s $sess_name
 606         ret=$?
 607         if [[ $expected_to_fail -eq "1" ]]; then
 608                 test "$ret" -ne "0"
 609                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 610         else
 611                 ok $ret "Disable channel $channel_name for session $sess_name"
 612         fi
 613 }
 614
 615 function lttng_disable_kernel_channel_ok()
 616 {
 617         lttng_disable_kernel_channel 0 "$@"
 618 }
 619
 620 function lttng_disable_kernel_channel_fail()
 621 {
 622         lttng_disable_kernel_channel 1 "$@"
 623 }
 624
 625 function start_lttng_relayd_opt()
 626 {
 627         local withtap=$1
 628         local process_mode=$2
 629         local opt=$3
 630
 631         DIR=$(readlink -f "$TESTDIR")
 632
 633         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 634                 # shellcheck disable=SC2086
 635                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 636                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt -vvv >>/tmp/relayd.log 2>&1 &
 637                 if [ $? -eq 1 ]; then
 638                         if [ $withtap -eq "1" ]; then
 639                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 640                         fi
 641                         return 1
 642                 else
 643                         if [ $withtap -eq "1" ]; then
 644                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 645                         fi
 646                 fi
 647         else
 648                 pass "Start lttng-relayd (opt: $opt)"
 649         fi
 650 }
 651
 652 function start_lttng_relayd()
 653 {
 654         start_lttng_relayd_opt 1 "-b" "$@"
 655 }
 656
 657 function start_lttng_relayd_notap()
 658 {
 659         start_lttng_relayd_opt 0 "-b" "$@"
 660 }
 661
 662 function stop_lttng_relayd_opt()
 663 {
 664         local withtap=$1
 665         local is_cleanup=$2
 666         local signal=$3
 667         local timeout_s=$4
 668         local dtimeleft_s=
 669         local retval=0
 670         local pids
 671
 672         if [ -z "$signal" ]; then
 673                 signal="SIGTERM"
 674         fi
 675
 676
 677         # Multiply time by 2 to simplify integer arithmetic
 678         # Multiply time by 5 to adjust for sleeping every 0.1s
 679         if [ -n "$timeout_s" ]; then
 680                 dtimeleft_s=$((timeout_s * 2 * 5))
 681         fi
 682
 683
 684         pids=$(lttng_pgrep "$RELAYD_MATCH")
 685         if [ -z "$pids" ]; then
 686                 if [ "$is_cleanup" -eq 1 ]; then
 687                         :
 688                 elif [ "$withtap" -eq "1" ]; then
 689                         fail "No relay daemon to kill"
 690                 else
 691                         LTTNG_BAIL_OUT "No relay daemon to kill"
 692                 fi
 693                 return 0
 694         fi
 695
 696         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 697
 698         # shellcheck disable=SC2086
 699         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 700                 retval=1
 701                 if [ "$withtap" -eq "1" ]; then
 702                         fail "Kill relay daemon"
 703                 fi
 704         else
 705                 out=1
 706                 while [ -n "$out" ]; do
 707                         out=$(lttng_pgrep "$RELAYD_MATCH")
 708                         if [ -n "$dtimeleft_s" ]; then
 709                                 if [ $dtimeleft_s -lt 0 ]; then
 710                                         out=
 711                                         retval=1
 712                                 fi
 713                                 dtimeleft_s=$((dtimeleft_s - 1))
 714                         fi
 715                         sleep 0.1
 716                 done
 717                 if [ "$withtap" -eq "1" ]; then
 718                         if [ "$retval" -eq "0" ]; then
 719                                 pass "Wait after kill relay daemon"
 720                         else
 721                                 fail "Wait after kill relay daemon"
 722                         fi
 723                 fi
 724         fi
 725         return $retval
 726 }
 727
 728 function stop_lttng_relayd()
 729 {
 730         stop_lttng_relayd_opt 1 0 "$@"
 731 }
 732
 733 function stop_lttng_relayd_notap()
 734 {
 735         stop_lttng_relayd_opt 0 0 "$@"
 736 }
 737
 738 function stop_lttng_relayd_cleanup()
 739 {
 740         stop_lttng_relayd_opt 0 1 "$@"
 741 }
 742
 743 #First arg: show tap output
 744 #Second argument: load path for automatic loading
 745 function start_lttng_sessiond_opt()
 746 {
 747         local withtap=$1
 748         local load_path=$2
 749
 750         # The rest of the arguments will be passed directly to lttng-sessiond.
 751         shift 2
 752
 753         local env_vars=""
 754         local consumerd=""
 755
 756         local long_bit_value=
 757         long_bit_value=$(getconf LONG_BIT)
 758
 759         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 760                 # Env variable requested no session daemon
 761                 return
 762         fi
 763
 764         DIR=$(readlink -f "$TESTDIR")
 765
 766         # Get long_bit value for 32/64 consumerd
 767         case "$long_bit_value" in
 768                 32)
 769                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 770                         ;;
 771                 64)
 772                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 773                         ;;
 774                 *)
 775                         return
 776                         ;;
 777         esac
 778
 779         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 780         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 781                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 782         fi
 783         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 784
 785         if ! validate_kernel_version; then
 786             fail "Start session daemon"
 787             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 788         fi
 789
 790         diag "export LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/"
 791         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/"}"
 792         export LTTNG_SESSION_CONFIG_XSD_PATH
 793
 794         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 795                 # Have a load path ?
 796                 if [ -n "$load_path" ]; then
 797                         diag "env $env_vars --load $load_path --background $consumerd $@"
 798                         # shellcheck disable=SC2086
 799                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 800                 else
 801                         diag "env $env_vars --background $consumerd $@"
 802                         # shellcheck disable=SC2086
 803                         env $env_vars --background "$consumerd" "$@"
 804                 fi
 805                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 806                 status=$?
 807                 if [ "$withtap" -eq "1" ]; then
 808                         ok $status "Start session daemon"
 809                 fi
 810         fi
 811 }
 812
 813 function start_lttng_sessiond()
 814 {
 815         start_lttng_sessiond_opt 1 "$@"
 816 }
 817
 818 function start_lttng_sessiond_notap()
 819 {
 820         start_lttng_sessiond_opt 0 "$@"
 821 }
 822
 823 function stop_lttng_sessiond_opt()
 824 {
 825         local withtap=$1
 826         local is_cleanup=$2
 827         local signal=$3
 828         local timeout_s=$4
 829         local dtimeleft_s=
 830         local retval=0
 831         local runas_pids
 832         local pids
 833
 834         if [ -z "$signal" ]; then
 835                 signal=SIGTERM
 836         fi
 837
 838         # Multiply time by 2 to simplify integer arithmetic
 839         # Multiply time by 5 to adjust for sleeping every 0.1s
 840         if [ -n "$timeout_s" ]; then
 841                 dtimeleft_s=$((timeout_s * 2 * 5))
 842         fi
 843
 844         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 845                 # Env variable requested no session daemon
 846                 return 0
 847         fi
 848
 849         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 850         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 851
 852         if [ -n "$runas_pids" ]; then
 853                 pids="$pids $runas_pids"
 854         fi
 855
 856         if [ -z "$pids" ]; then
 857                 if [ "$is_cleanup" -eq 1 ]; then
 858                         :
 859                 elif [ "$withtap" -eq "1" ]; then
 860                         fail "No session daemon to kill"
 861                 else
 862                         LTTNG_BAIL_OUT "No session daemon to kill"
 863                 fi
 864                 return 0
 865         fi
 866
 867         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 868
 869         # shellcheck disable=SC2086
 870         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 871                 retval=1
 872                 if [ "$withtap" -eq "1" ]; then
 873                         fail "Kill sessions daemon"
 874                 fi
 875         else
 876                 out=1
 877                 while [ -n "$out" ]; do
 878                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 879                         if [ -n "$dtimeleft_s" ]; then
 880                                 if [ $dtimeleft_s -lt 0 ]; then
 881                                         out=
 882                                         retval=1
 883                                 fi
 884                                 dtimeleft_s=$((dtimeleft_s - 1))
 885                         fi
 886                         sleep 0.1
 887                 done
 888                 out=1
 889                 while [ -n "$out" ]; do
 890                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 891                         if [ -n "$dtimeleft_s" ]; then
 892                                 if [ $dtimeleft_s -lt 0 ]; then
 893                                         out=
 894                                         retval=1
 895                                 fi
 896                                 dtimeleft_s=$((dtimeleft_s - 1))
 897                         fi
 898                         sleep 0.1
 899                 done
 900
 901                 if [ "$withtap" -eq "1" ]; then
 902                         if [ "$retval" -eq "0" ]; then
 903                                 pass "Wait after kill session daemon"
 904                         else
 905                                 fail "Wait after kill session daemon"
 906                         fi
 907                 fi
 908         fi
 909         if [ "$signal" = "SIGKILL" ]; then
 910                 if [ "$(id -u)" -eq "0" ]; then
 911                         local modules=
 912                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 913
 914                         if [ -n "$modules" ]; then
 915                                 diag "Unloading all LTTng modules"
 916                                 modprobe --remove "$modules"
 917                         fi
 918                 fi
 919         fi
 920
 921         return $retval
 922 }
 923
 924 function stop_lttng_sessiond()
 925 {
 926         stop_lttng_sessiond_opt 1 0 "$@"
 927 }
 928
 929 function stop_lttng_sessiond_notap()
 930 {
 931         stop_lttng_sessiond_opt 0 0 "$@"
 932 }
 933
 934 function stop_lttng_sessiond_cleanup()
 935 {
 936         stop_lttng_sessiond_opt 0 1 "$@"
 937 }
 938
 939 function sigstop_lttng_sessiond_opt()
 940 {
 941         local withtap=$1
 942         local signal=SIGSTOP
 943         local pids
 944
 945         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 946                 # Env variable requested no session daemon
 947                 return
 948         fi
 949
 950         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 951
 952         if [ "$withtap" -eq "1" ]; then
 953                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 954         fi
 955
 956         # shellcheck disable=SC2086
 957         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 958                 if [ "$withtap" -eq "1" ]; then
 959                         fail "Sending SIGSTOP to session daemon"
 960                 fi
 961         else
 962                 out=1
 963                 while [ $out -ne 0 ]; do
 964                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 965
 966                         # Wait until state becomes stopped for session
 967                         # daemon(s).
 968                         out=0
 969                         for sessiond_pid in $pids; do
 970                                 state="$(ps -p "$sessiond_pid" -o state= )"
 971                                 if [[ -n "$state" && "$state" != "T" ]]; then
 972                                         out=1
 973                                 fi
 974                         done
 975                         sleep 0.5
 976                 done
 977                 if [ "$withtap" -eq "1" ]; then
 978                         pass "Sending SIGSTOP to session daemon"
 979                 fi
 980         fi
 981 }
 982
 983 function sigstop_lttng_sessiond()
 984 {
 985         sigstop_lttng_sessiond_opt 1 "$@"
 986 }
 987
 988 function sigstop_lttng_sessiond_notap()
 989 {
 990         sigstop_lttng_sessiond_opt 0 "$@"
 991 }
 992
 993 function stop_lttng_consumerd_opt()
 994 {
 995         local withtap=$1
 996         local is_cleanup=$2
 997         local signal=$3
 998         local timeout_s=$4
 999         local dtimeleft_s=
1000         local retval=0
1001         local pids
1002
1003         if [ -z "$signal" ]; then
1004                 signal=SIGTERM
1005         fi
1006
1007         # Multiply time by 2 to simplify integer arithmetic
1008         # Multiply time by 5 to adjust for sleeping every 0.1s
1009         if [ -n "$timeout_s" ]; then
1010                 dtimeleft_s=$((timeout_s * 2 * 5))
1011         fi
1012
1013         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1014
1015         if [ -z "$pids" ]; then
1016                 if [ "$is_cleanup" -eq 1 ]; then
1017                         :
1018                 elif [ "$withtap" -eq "1" ]; then
1019                         fail "No consumerd daemon to kill"
1020                 else
1021                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
1022                 fi
1023                 return 0
1024         fi
1025
1026         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1027
1028         # shellcheck disable=SC2086
1029         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
1030                 retval=1
1031                 if [ "$withtap" -eq "1" ]; then
1032                         fail "Kill consumer daemon"
1033                 fi
1034         else
1035                 out=1
1036                 while [ $out -ne 0 ]; do
1037                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1038
1039                         # If consumerds are still present check their status.
1040                         # A zombie status qualifies the consumerd as *killed*
1041                         out=0
1042                         for consumer_pid in $pids; do
1043                                 state="$(ps -p "$consumer_pid" -o state= )"
1044                                 if [[ -n "$state" && "$state" != "Z" ]]; then
1045                                         out=1
1046                                 fi
1047                         done
1048                         if [ -n "$dtimeleft_s" ]; then
1049                                 if [ $dtimeleft_s -lt 0 ]; then
1050                                         out=0
1051                                         retval=1
1052                                 fi
1053                                 dtimeleft_s=$((dtimeleft_s - 1))
1054                         fi
1055                         sleep 0.1
1056                 done
1057                 if [ "$withtap" -eq "1" ]; then
1058                         if [ "$retval" -eq "0" ]; then
1059                                 pass "Wait after kill consumer daemon"
1060                         else
1061                                 fail "Wait after kill consumer daemon"
1062                         fi
1063                 fi
1064         fi
1065
1066         return $retval
1067 }
1068
1069 function stop_lttng_consumerd()
1070 {
1071         stop_lttng_consumerd_opt 1 0 "$@"
1072 }
1073
1074 function stop_lttng_consumerd_notap()
1075 {
1076         stop_lttng_consumerd_opt 0 0 "$@"
1077 }
1078
1079 function stop_lttng_consumerd_cleanup()
1080 {
1081         stop_lttng_consumerd_opt 0 1 "$@"
1082 }
1083
1084 function sigstop_lttng_consumerd_opt()
1085 {
1086         local withtap=$1
1087         local signal=SIGSTOP
1088         local pids
1089
1090         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1091
1092         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1093
1094         # shellcheck disable=SC2086
1095         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1096         retval=$?
1097
1098         if [ $retval -eq 1 ]; then
1099                 if [ "$withtap" -eq "1" ]; then
1100                         fail "Sending SIGSTOP to consumer daemon"
1101                 fi
1102                 return 1
1103         else
1104                 out=1
1105                 while [ $out -ne 0 ]; do
1106                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1107
1108                         # Wait until state becomes stopped for all
1109                         # consumers.
1110                         out=0
1111                         for consumer_pid in $pids; do
1112                                 state="$(ps -p "$consumer_pid" -o state= )"
1113                                 if [[ -n "$state" && "$state" != "T" ]]; then
1114                                         out=1
1115                                 fi
1116                         done
1117                         sleep 0.5
1118                 done
1119                 if [ "$withtap" -eq "1" ]; then
1120                         pass "Sending SIGSTOP to consumer daemon"
1121                 fi
1122         fi
1123         return $retval
1124 }
1125
1126 function sigstop_lttng_consumerd()
1127 {
1128         sigstop_lttng_consumerd_opt 1 "$@"
1129 }
1130
1131 function sigstop_lttng_consumerd_notap()
1132 {
1133         sigstop_lttng_consumerd_opt 0 "$@"
1134 }
1135
1136 function list_lttng_with_opts ()
1137 {
1138         local ret
1139         local withtap=$1
1140         shift
1141         local opts=$1
1142         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1143                 list $opts
1144         ret=$?
1145         if [ $withtap -eq "1" ]; then
1146                 ok $ret "Lttng-tool list command with option $opts"
1147         fi
1148 }
1149
1150 function list_lttng_ok ()
1151 {
1152         list_lttng_with_opts 1 "$@"
1153 }
1154
1155 function list_lttng_notap ()
1156 {
1157         list_lttng_with_opts 0 "$@"
1158 }
1159
1160 function create_lttng_session_no_output ()
1161 {
1162         local sess_name=$1
1163         local opts="${@:2}"
1164
1165         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1166                 create $sess_name --no-output $opts
1167         ok $? "Create session $sess_name in no-output mode"
1168 }
1169
1170 function create_lttng_session_uri () {
1171         local sess_name=$1
1172         local uri=$2
1173         local opts="${@:3}"
1174
1175         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1176                 create $sess_name -U $uri $opts
1177         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1178 }
1179
1180 function create_lttng_session ()
1181 {
1182         local withtap=$1
1183         local expected_to_fail=$2
1184         local sess_name=$3
1185         local trace_path=$4
1186         local opt=$5
1187
1188         if [ -z "$trace_path" ]; then
1189                 # Use lttng-sessiond default output.
1190                 trace_path=""
1191         else
1192                 trace_path="-o $trace_path"
1193         fi
1194
1195         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1196                 create "$sess_name" $trace_path $opt
1197         ret=$?
1198         if [ $expected_to_fail -eq "1" ]; then
1199                 test "$ret" -ne "0"
1200                 ret=$?
1201                 if [ $withtap -eq "1" ]; then
1202                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1203                 fi
1204         else
1205                 if [ $withtap -eq "1" ]; then
1206                         ok $ret "Create session $sess_name in $trace_path"
1207                 fi
1208         fi
1209         return $ret
1210 }
1211
1212 function create_lttng_session_ok ()
1213 {
1214         create_lttng_session 1 0 "$@"
1215 }
1216
1217 function create_lttng_session_fail ()
1218 {
1219         create_lttng_session 1 1 "$@"
1220 }
1221
1222 function create_lttng_session_notap ()
1223 {
1224         create_lttng_session 0 0 "$@"
1225 }
1226
1227
1228 function enable_ust_lttng_channel ()
1229 {
1230         local withtap=$1
1231         local expected_to_fail=$2
1232         local sess_name=$3
1233         local channel_name=$4
1234         local opts="${@:5}"
1235
1236         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1237                 enable-channel -u $channel_name -s $sess_name $opts
1238         ret=$?
1239         if [[ $expected_to_fail -eq "1" ]]; then
1240                 test "$ret" -ne "0"
1241                 ret=$?
1242                 if [ $withtap -eq "1" ]; then
1243                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1244                 fi
1245         else
1246                 if [ $withtap -eq "1" ]; then
1247                         ok $ret "Enable channel $channel_name for session $sess_name"
1248                 fi
1249         fi
1250         return $ret
1251 }
1252
1253 function enable_ust_lttng_channel_ok ()
1254 {
1255         enable_ust_lttng_channel 1 0 "$@"
1256 }
1257
1258 function enable_ust_lttng_channel_fail ()
1259 {
1260         enable_ust_lttng_channel 1 1 "$@"
1261 }
1262
1263 function enable_ust_lttng_channel_notap ()
1264 {
1265         enable_ust_lttng_channel 0 0 "$@"
1266 }
1267
1268 function disable_ust_lttng_channel()
1269 {
1270         local sess_name=$1
1271         local channel_name=$2
1272
1273         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1274                 disable-channel -u $channel_name -s $sess_name
1275         ok $? "Disable channel $channel_name for session $sess_name"
1276 }
1277
1278 function enable_lttng_mmap_overwrite_kernel_channel()
1279 {
1280         local sess_name=$1
1281         local channel_name=$2
1282
1283         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1284                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite
1285         ok $? "Enable channel $channel_name for session $sess_name"
1286 }
1287
1288 function enable_lttng_mmap_discard_small_kernel_channel()
1289 {
1290         local sess_name=$1
1291         local channel_name=$2
1292
1293         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1294                 enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1295         ok $? "Enable small discard channel $channel_name for session $sess_name"
1296 }
1297
1298 function enable_lttng_mmap_overwrite_small_kernel_channel()
1299 {
1300         local sess_name=$1
1301         local channel_name=$2
1302
1303         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1304                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1305         ok $? "Enable small discard channel $channel_name for session $sess_name"
1306 }
1307
1308 function enable_lttng_mmap_overwrite_ust_channel()
1309 {
1310         local sess_name=$1
1311         local channel_name=$2
1312
1313         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1314                 enable-channel -s $sess_name $channel_name -u --output mmap --overwrite
1315         ok $? "Enable channel $channel_name for session $sess_name"
1316 }
1317
1318 function enable_ust_lttng_event ()
1319 {
1320         local withtap=$1
1321         local expected_to_fail=$2
1322         local sess_name=$3
1323         local event_name="$4"
1324         local channel_name=$5
1325
1326         if [ -z $channel_name ]; then
1327                 # default channel if none specified
1328                 chan=""
1329         else
1330                 chan="-c $channel_name"
1331         fi
1332
1333         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1334                 enable-event "$event_name" $chan -s "$sess_name" -u
1335         ret=$?
1336         if [[ $expected_to_fail -eq "1" ]]; then
1337                 test $ret -ne "0"
1338                 ret=$?
1339                 if [[ $withtap -eq "1" ]]; then
1340                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1341                 fi
1342         else
1343                 if [[ $withtap -eq "1" ]]; then
1344                         ok $ret "Enable ust event $event_name for session $sess_name"
1345                 fi
1346         fi
1347         return $ret
1348 }
1349
1350 function enable_ust_lttng_event_ok ()
1351 {
1352         enable_ust_lttng_event 1 0 "$@"
1353 }
1354
1355 function enable_ust_lttng_event_fail ()
1356 {
1357         enable_ust_lttng_event 1 1 "$@"
1358 }
1359
1360 function enable_ust_lttng_event_notap ()
1361 {
1362         enable_ust_lttng_event 0 0 "$@"
1363 }
1364
1365 function enable_jul_lttng_event()
1366 {
1367         sess_name=$1
1368         event_name="$2"
1369         channel_name=$3
1370
1371         if [ -z $channel_name ]; then
1372                 # default channel if none specified
1373                 chan=""
1374         else
1375                 chan="-c $channel_name"
1376         fi
1377
1378         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1379                 enable-event "$event_name" $chan -s $sess_name -j
1380         ok $? "Enable JUL event $event_name for session $sess_name"
1381 }
1382
1383 function enable_jul_lttng_event_loglevel()
1384 {
1385         local sess_name=$1
1386         local event_name="$2"
1387         local loglevel=$3
1388         local channel_name=$4
1389
1390         if [ -z $channel_name ]; then
1391                 # default channel if none specified
1392                 chan=""
1393         else
1394                 chan="-c $channel_name"
1395         fi
1396
1397         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1398                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j
1399         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1400 }
1401
1402 function enable_log4j_lttng_event()
1403 {
1404         local sess_name=$1
1405         local event_name=$2
1406         local channel_name=$3
1407
1408         local chan_opt=()
1409
1410         # default channel if none specified
1411         if [ -n "$channel_name" ]; then
1412                 chan_opt=("-c" "$channel_name")
1413         fi
1414
1415         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1416                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1417         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1418 }
1419
1420 function enable_log4j_lttng_event_filter()
1421 {
1422         local sess_name=$1
1423         local event_name=$2
1424         local filter=$3
1425
1426         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1427                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1428         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1429 }
1430
1431 function enable_log4j_lttng_event_filter_loglevel_only()
1432 {
1433         local sess_name=$1
1434         local event_name=$2
1435         local filter=$3
1436         local loglevel=$4
1437
1438         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1439                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1440         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1441 }
1442
1443 function enable_log4j_lttng_event_loglevel()
1444 {
1445         local sess_name=$1
1446         local event_name=$2
1447         local loglevel=$3
1448         local channel_name=$4
1449
1450
1451         # default channel if none specified
1452         if [ -n "$channel_name" ]; then
1453                 chan_opt=("-c" "$channel_name")
1454         fi
1455
1456         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1457                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1458         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1459 }
1460
1461 function enable_log4j_lttng_event_loglevel_only()
1462 {
1463         local sess_name=$1
1464         local event_name=$2
1465         local loglevel=$3
1466         local channel_name=$4
1467
1468         local chan_opt=()
1469
1470         # default channel if none specified
1471         if [ -n "$channel_name" ]; then
1472                 chan_opt=("-c" "$channel_name")
1473         fi
1474
1475         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1476                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1477         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1478 }
1479
1480 function enable_python_lttng_event()
1481 {
1482         sess_name=$1
1483         event_name="$2"
1484         channel_name=$3
1485
1486         if [ -z $channel_name ]; then
1487                 # default channel if none specified
1488                 chan=""
1489         else
1490                 chan="-c $channel_name"
1491         fi
1492
1493         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1494                 enable-event "$event_name" $chan -s $sess_name -p
1495         ok $? "Enable Python event $event_name for session $sess_name"
1496 }
1497
1498 function enable_python_lttng_event_loglevel()
1499 {
1500         local sess_name=$1
1501         local event_name="$2"
1502         local loglevel=$3
1503         local channel_name=$4
1504
1505         if [ -z $channel_name ]; then
1506                 # default channel if none specified
1507                 chan=""
1508         else
1509                 chan="-c $channel_name"
1510         fi
1511
1512         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1513                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p
1514         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1515 }
1516
1517 function enable_ust_lttng_event_filter()
1518 {
1519         local sess_name="$1"
1520         local event_name="$2"
1521         local filter="$3"
1522         local channel_name=$4
1523
1524         if [ -z $channel_name ]; then
1525                 # default channel if none specified
1526                 chan=""
1527         else
1528                 chan="-c $channel_name"
1529         fi
1530
1531         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1532                 enable-event $chan "$event_name" -s $sess_name -u --filter "$filter"
1533         ok $? "Enable event $event_name with filtering for session $sess_name"
1534 }
1535
1536 function enable_ust_lttng_event_loglevel()
1537 {
1538         local sess_name="$1"
1539         local event_name="$2"
1540         local loglevel="$3"
1541         local channel_name="$4"
1542         local chan=()
1543         if [ -n "${channel_name}" ] ; then
1544                 chan=('-c' "${channel_name}")
1545         fi
1546
1547         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1548                 enable-event "${chan[@]}" "$event_name" -s "${sess_name}" -u --loglevel="${loglevel}"
1549         ok $? "Enable event $event_name with loglevel $loglevel"
1550 }
1551
1552 function enable_ust_lttng_event_loglevel_only()
1553 {
1554         local sess_name="$1"
1555         local event_name="$2"
1556         local loglevel="$3"
1557         local channel_name="$4"
1558         local chan=()
1559         if [ -n "${channel_name}" ] ; then
1560                 chan=('-c' "${channel_name}")
1561         fi
1562
1563         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1564                 enable-event "${chan[@]}" "$event_name" -s "${sess_name}" -u --loglevel-only "${loglevel}"
1565         ok $? "Enable event $event_name with loglevel-only $loglevel"
1566 }
1567
1568 function disable_ust_lttng_event ()
1569 {
1570         local sess_name="$1"
1571         local event_name="$2"
1572         local channel_name="$3"
1573
1574         if [ -z $channel_name ]; then
1575                 # default channel if none specified
1576                 chan=""
1577         else
1578                 chan="-c $channel_name"
1579         fi
1580
1581         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1582                 disable-event "$event_name" -s $sess_name $chan -u
1583         ok $? "Disable event $event_name for session $sess_name"
1584 }
1585
1586 function disable_jul_lttng_event ()
1587 {
1588         local sess_name="$1"
1589         local event_name="$2"
1590
1591         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1592         ok $? "Disable JUL event $event_name for session $sess_name"
1593 }
1594
1595 function disable_log4j_lttng_event ()
1596 {
1597         local sess_name="$1"
1598         local event_name="$2"
1599
1600         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1601                 disable-event "$event_name" -s "$sess_name" --log4j
1602         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1603 }
1604
1605 function disable_python_lttng_event ()
1606 {
1607         local sess_name="$1"
1608         local event_name="$2"
1609
1610         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1611                 disable-event "$event_name" -s $sess_name -p
1612         ok $? "Disable Python event $event_name for session $sess_name"
1613 }
1614
1615 function start_lttng_tracing_opt ()
1616 {
1617         local withtap=$1
1618         local expected_to_fail=$2
1619         local sess_name=$3
1620
1621         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1622                 start $sess_name
1623         ret=$?
1624         if [[ $expected_to_fail -eq "1" ]]; then
1625                 test "$ret" -ne "0"
1626                 ret=$?
1627                 if [ $withtap -eq "1" ]; then
1628                         ok $? "Start tracing for session $sess_name failed as expected"
1629                 fi
1630         else
1631                 if [ $withtap -eq "1" ]; then
1632                         ok $ret "Start tracing for session $sess_name"
1633                 fi
1634         fi
1635 }
1636
1637 function start_lttng_tracing_ok ()
1638 {
1639         start_lttng_tracing_opt 1 0 "$@"
1640 }
1641
1642 function start_lttng_tracing_fail ()
1643 {
1644         start_lttng_tracing_opt 1 1 "$@"
1645 }
1646
1647 function start_lttng_tracing_notap ()
1648 {
1649         start_lttng_tracing_opt 0 1 "$@"
1650 }
1651
1652 function stop_lttng_tracing_opt ()
1653 {
1654         local withtap=$1
1655         local expected_to_fail=$2
1656         local sess_name=$3
1657
1658         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1659                 stop $sess_name
1660         ret=$?
1661         if [[ $expected_to_fail -eq "1" ]]; then
1662                 test "$ret" -ne "0"
1663                 ret=$?
1664                 if [ $withtap -eq "1" ]; then
1665                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1666                 fi
1667         else
1668                 if [ $withtap -eq "1" ]; then
1669                         ok $ret "Stop lttng tracing for session $sess_name"
1670                 fi
1671         fi
1672 }
1673
1674 function stop_lttng_tracing_ok ()
1675 {
1676         stop_lttng_tracing_opt 1 0 "$@"
1677 }
1678
1679 function stop_lttng_tracing_fail ()
1680 {
1681         stop_lttng_tracing_opt 1 1 "$@"
1682 }
1683
1684 function stop_lttng_tracing_notap ()
1685 {
1686         stop_lttng_tracing_opt 0 0 "$@"
1687 }
1688
1689 function destroy_lttng_session ()
1690 {
1691         local withtap=$1
1692         local expected_to_fail=$2
1693         local sess_name=$3
1694         shift 3
1695
1696         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1697                 destroy $sess_name $@
1698         ret=$?
1699         if [[ $expected_to_fail -eq "1" ]]; then
1700                 test "$ret" -ne "0"
1701                 ret=$?
1702                 if [ $withtap -eq "1" ]; then
1703                         ok $ret "Destroy session $sess_name failed as expected"
1704                 fi
1705         else
1706                 if [ $withtap -eq "1" ]; then
1707                         ok $ret "Destroy session $sess_name"
1708                 fi
1709         fi
1710 }
1711
1712 function destroy_lttng_session_ok ()
1713 {
1714         destroy_lttng_session 1 0 "$@"
1715
1716 }
1717
1718 function destroy_lttng_session_fail ()
1719 {
1720         destroy_lttng_session 1 1 "$@"
1721 }
1722
1723 function destroy_lttng_session_notap ()
1724 {
1725         destroy_lttng_session 0 0 "$@"
1726 }
1727
1728 function destroy_lttng_sessions ()
1729 {
1730         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1731                 destroy --all
1732         ok $? "Destroy all lttng sessions"
1733 }
1734
1735 function lttng_snapshot_add_output ()
1736 {
1737         local expected_to_fail=$1
1738         local sess_name=$2
1739         local trace_path=$3
1740         local opts=$4
1741
1742         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1743                 snapshot add-output -s $sess_name $trace_path $opts
1744         ret=$?
1745         if [[ $expected_to_fail -eq 1 ]]; then
1746                 test "$ret" -ne "0"
1747                 ok $? "Added snapshot output $trace_path failed as expected"
1748         else
1749                 ok $ret "Added snapshot output $trace_path"
1750         fi
1751 }
1752
1753 function lttng_snapshot_add_output_ok ()
1754 {
1755         lttng_snapshot_add_output 0 "$@"
1756 }
1757
1758 function lttng_snapshot_add_output_fail ()
1759 {
1760         lttng_snapshot_add_output 1 "$@"
1761 }
1762
1763 function lttng_snapshot_del_output ()
1764 {
1765         local expected_to_fail=$1
1766         local sess_name=$2
1767         local id=$3
1768
1769         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1770                 snapshot del-output -s $sess_name $id
1771         ret=$?
1772         if [[ $expected_to_fail -eq "1" ]]; then
1773                 test "$ret" -ne "0"
1774                 ok $? "Deleted snapshot output id $id failed as expected"
1775         else
1776                 ok $ret "Deleted snapshot output id $id"
1777         fi
1778 }
1779
1780 function lttng_snapshot_del_output_ok ()
1781 {
1782         lttng_snapshot_del_output 0 "$@"
1783 }
1784
1785 function lttng_snapshot_del_output_fail ()
1786 {
1787         lttng_snapshot_del_output 1 "$@"
1788 }
1789
1790 function lttng_snapshot_record ()
1791 {
1792         local sess_name=$1
1793         local trace_path=$2
1794
1795         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1796                 snapshot record -s "$sess_name" "$trace_path"
1797         ok $? "Snapshot recorded"
1798 }
1799
1800 function lttng_snapshot_list ()
1801 {
1802         local sess_name=$1
1803         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1804                 snapshot list-output -s $sess_name
1805         ok $? "Snapshot list"
1806 }
1807
1808 function lttng_save()
1809 {
1810         local sess_name=$1
1811         local opts=$2
1812
1813         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1814                 save $sess_name $opts
1815         ok $? "Session saved"
1816 }
1817
1818 function lttng_load()
1819 {
1820         local expected_to_fail=$1
1821         local opts=$2
1822
1823         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1824                 load $opts
1825         ret=$?
1826         if [[ $expected_to_fail -eq "1" ]]; then
1827                 test $ret -ne "0"
1828                 ok $? "Load command failed as expected with opts: $opts"
1829         else
1830                 ok $ret "Load command with opts: $opts"
1831         fi
1832 }
1833
1834 function lttng_load_ok()
1835 {
1836         lttng_load 0 "$@"
1837 }
1838
1839 function lttng_load_fail()
1840 {
1841         lttng_load 1 "$@"
1842 }
1843
1844 function lttng_track()
1845 {
1846         local expected_to_fail="$1"
1847         shift 1
1848         local opts="$@"
1849         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1850         ret=$?
1851         if [[ $expected_to_fail -eq "1" ]]; then
1852                 test $ret -ne "0"
1853                 ok $? "Track command failed as expected with opts: $opts"
1854         else
1855                 ok $ret "Track command with opts: $opts"
1856         fi
1857 }
1858
1859 function lttng_track_ok()
1860 {
1861         lttng_track 0 "$@"
1862 }
1863
1864 function lttng_track_fail()
1865 {
1866         lttng_track 1 "$@"
1867 }
1868
1869 function lttng_untrack()
1870 {
1871         local expected_to_fail="$1"
1872         shift 1
1873         local opts="$@"
1874         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1875         ret=$?
1876         if [[ $expected_to_fail -eq "1" ]]; then
1877                 test $ret -ne "0"
1878                 ok $? "Untrack command failed as expected with opts: $opts"
1879         else
1880                 ok $ret "Untrack command with opts: $opts"
1881         fi
1882 }
1883
1884 function lttng_untrack_ok()
1885 {
1886         lttng_untrack 0 "$@"
1887 }
1888
1889 function lttng_untrack_fail()
1890 {
1891         lttng_untrack 1 "$@"
1892 }
1893
1894 function lttng_track_pid_ok()
1895 {
1896         PID=$1
1897         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1898         ok $? "Lttng track pid on the kernel domain"
1899 }
1900
1901 function lttng_untrack_kernel_all_ok()
1902 {
1903         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1904         ok $? "Lttng untrack all pid on the kernel domain"
1905 }
1906
1907 function lttng_track_ust_ok()
1908 {
1909         lttng_track_ok -u "$@"
1910 }
1911
1912 function lttng_track_ust_fail()
1913 {
1914         lttng_track_fail -u "$@"
1915 }
1916
1917 function lttng_track_kernel_ok()
1918 {
1919         lttng_track_ok -k "$@"
1920 }
1921
1922 function lttng_track_kernel_fail()
1923 {
1924         lttng_track_fail -k "$@"
1925 }
1926
1927 function lttng_untrack_ust_ok()
1928 {
1929         lttng_untrack_ok -u "$@"
1930 }
1931
1932 function lttng_untrack_ust_fail()
1933 {
1934         lttng_untrack_fail -u "$@"
1935 }
1936
1937 function lttng_untrack_kernel_ok()
1938 {
1939         lttng_untrack_ok -k "$@"
1940 }
1941
1942 function lttng_untrack_kernel_fail()
1943 {
1944         lttng_untrack_fail -k "$@"
1945 }
1946
1947 function lttng_add_context_list()
1948 {
1949         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1950                 add-context --list
1951         ret=$?
1952         ok $ret "Context listing"
1953 }
1954
1955 function add_context_lttng()
1956 {
1957         local expected_to_fail="$1"
1958         local domain="$2"
1959         local session_name="$3"
1960         local channel_name="$4"
1961         local type="$5"
1962
1963         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1964                 add-context -s $session_name -c $channel_name -t $type $domain
1965         ret=$?
1966         if [[ $expected_to_fail -eq "1" ]]; then
1967                 test $ret -ne "0"
1968                 ok $? "Add context command failed as expected for type: $type"
1969         else
1970                 ok $ret "Add context command for type: $type"
1971         fi
1972 }
1973
1974 function add_context_ust_ok()
1975 {
1976         add_context_lttng 0 -u "$@"
1977 }
1978
1979 function add_context_ust_fail()
1980 {
1981         add_context_lttng 1 -u "$@"
1982 }
1983
1984 function add_context_kernel_ok()
1985 {
1986         add_context_lttng 0 -k "$@"
1987 }
1988
1989 function add_context_kernel_fail()
1990 {
1991         add_context_lttng 1 -k "$@"
1992 }
1993
1994 function wait_live_trace_ready ()
1995 {
1996         local url=$1
1997         local zero_client_match=0
1998
1999         diag "Waiting for live trace at url: $url"
2000         while [ $zero_client_match -eq 0 ]; do
2001                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
2002                 sleep 0.1
2003         done
2004         pass "Waiting for live trace at url: $url"
2005 }
2006
2007 function wait_live_viewer_connect ()
2008 {
2009         local url=$1
2010         local one_client_match=0
2011
2012         diag "Waiting for live viewers on url: $url"
2013         while [ $one_client_match -eq 0 ]; do
2014                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
2015                 sleep 0.1
2016         done
2017         pass "Waiting for live viewers on url: $url"
2018 }
2019
2020 function bail_out_if_no_babeltrace()
2021 {
2022         which "$BABELTRACE_BIN" >/dev/null
2023         if [ $? -ne 0 ]; then
2024                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
2025         fi
2026 }
2027
2028 # Check that the trace metadata contains '$expected' event ids matching '$event_name'.
2029 function validate_metadata_event()
2030 {
2031         local event_name=$1
2032         local expected=$2
2033         local trace_path=$3
2034
2035         local metadata_file
2036         local metadata_path
2037         local count
2038
2039         metadata_file=$(find "$trace_path" -name "metadata")
2040         metadata_path=$(dirname "$metadata_file")
2041
2042         bail_out_if_no_babeltrace
2043
2044         count=$($BABELTRACE_BIN --output-format=ctf-metadata "$metadata_path" | grep -c "$event_name")
2045
2046         test "$count" -eq "$expected"
2047         ok $? "Found $count / $expected metadata event id matching '$event_name'"
2048 }
2049
2050 # Check that the trace contains '$expected' events matching '$event_name', other
2051 # events not matching '$event_name' can be present.
2052 function trace_matches()
2053 {
2054         local event_name=$1
2055         local expected=$2
2056         local trace_path=$3
2057
2058         local count
2059         local total
2060
2061         bail_out_if_no_babeltrace
2062
2063         count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2064         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2065
2066         test "$count" -eq "$expected"
2067
2068         ok $? "Found $count / $expected events matching '$event_name' out of $total events"
2069 }
2070
2071 # Check that the trace contains '$expected' events matching '$event_name' and no
2072 # other events.
2073 function trace_match_only()
2074 {
2075         local event_name=$1
2076         local expected=$2
2077         local trace_path=$3
2078
2079         local count
2080         local total
2081
2082         bail_out_if_no_babeltrace
2083
2084         count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2085         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2086
2087         test "$expected" -eq "$count" && test "$total" -eq "$expected"
2088
2089         ok $? "Found $count / $expected events matching '$event_name' amongst $total events"
2090 }
2091
2092 # Check that the trace contains at least 1 event matching each name in the
2093 # comma separated list '$event_names'.
2094 function validate_trace()
2095 {
2096         local event_names=$1
2097         local trace_path=$2
2098
2099         local count
2100
2101         bail_out_if_no_babeltrace
2102
2103         OLDIFS=$IFS
2104         IFS=","
2105         for event_name in $event_names; do
2106                 # trace_path is unquoted since callers make use of globbing
2107                 count=$($BABELTRACE_BIN $trace_path | grep -c "$event_name")
2108                 test "$count" -gt 0
2109                 ok $? "Found $count events matching '$event_name'"
2110         done
2111         IFS=$OLDIFS
2112 }
2113
2114 # Check that the trace contains at least 1 event matching each name in the
2115 # comma separated list '$event_names' and a total of '$expected' events.
2116 function validate_trace_count()
2117 {
2118         local event_names=$1
2119         local trace_path=$2
2120         local expected=$3
2121
2122         local count
2123         local total=0
2124
2125         bail_out_if_no_babeltrace
2126
2127         OLDIFS=$IFS
2128         IFS=","
2129         for event_name in $event_names; do
2130                 count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2131                 test "$count" -gt 0
2132                 ok $? "Found '$count' events matching '$event_name'"
2133                 total=$(( total + count ))
2134         done
2135         IFS=$OLDIFS
2136         test $total -eq "$expected"
2137         ok $? "Found $total events, expected $expected events"
2138 }
2139
2140 # Check that the trace contains at least '$expected_min' event matching each
2141 # name in the comma separated list '$event_names' and a total at least
2142 # '$expected_min' and less than '$expected_max' events.
2143 function validate_trace_count_range_incl_min_excl_max()
2144 {
2145         local event_names=$1
2146         local trace_path=$2
2147         local expected_min=$3
2148         local expected_max=$4
2149
2150         local count
2151         local total=0
2152
2153         bail_out_if_no_babeltrace
2154
2155         OLDIFS=$IFS
2156         IFS=","
2157         for event_name in $event_names; do
2158                 count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2159                 test "$count" -ge "$expected_min"
2160                 ok $? "Found $count events matching '$event_name', expected at least $expected_min"
2161                 total=$(( total + count ))
2162         done
2163         IFS=$OLDIFS
2164         test $total -ge "$expected_min" && test $total -lt "$expected_max"
2165         ok $? "Found a total of $total events, expected at least $expected_min and less than $expected_max"
2166 }
2167
2168 function trace_first_line()
2169 {
2170         local trace_path=$1
2171
2172         $BABELTRACE_BIN "$trace_path" | head -n 1
2173 }
2174
2175 # Check that the trace contains at least 1 event matching the grep extended
2176 # regexp '$event_exp'.
2177 function validate_trace_exp()
2178 {
2179         local event_exp=$1
2180         local trace_path=$2
2181
2182         local count
2183
2184         bail_out_if_no_babeltrace
2185
2186         # event_exp is unquoted since it contains multiple grep arguments
2187         count=$($BABELTRACE_BIN "$trace_path" | grep -c --extended-regexp $event_exp)
2188         test "$count" -gt 0
2189         ok $? "Found $count events matching expression '$event_exp'"
2190 }
2191
2192 # Check that the trace contains at least 1 event matching the grep extended
2193 # regexp '$event_exp' and zero event not matching it.
2194 function validate_trace_only_exp()
2195 {
2196         local event_exp=$1
2197         local trace_path=$2
2198
2199         local count
2200         local total
2201
2202         bail_out_if_no_babeltrace
2203
2204         # event_exp is unquoted since it contains multiple grep arguments
2205         count=$($BABELTRACE_BIN "$trace_path" | grep -c --extended-regexp $event_exp)
2206         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2207
2208         test  "$count" -gt 0 && test "$total" -eq "$count"
2209         ok $? "Found $count events matching expression '$event_exp' amongst $total events"
2210 }
2211
2212 # Check that the trace is valid and contains 0 event.
2213 function validate_trace_empty()
2214 {
2215         local trace_path=$1
2216
2217         local ret
2218         local count
2219
2220         bail_out_if_no_babeltrace
2221
2222         events=$($BABELTRACE_BIN "$trace_path")
2223         ret=$?
2224         if [ $ret -ne 0 ]; then
2225                 fail "Failed to parse trace"
2226                 return $ret
2227         fi
2228
2229         count=$(echo -n "$events" | wc -l)
2230         test "$count" -eq 0
2231         ok $? "Validate trace is empty, found $count events"
2232 }
2233
2234 function validate_directory_empty ()
2235 {
2236         local trace_path="$1"
2237
2238         local files
2239         local ret
2240         local nb_files
2241
2242         # Do not double quote `$trace_path` below as we want wildcards to be
2243         # expanded.
2244         files="$(ls -A $trace_path)"
2245         ret=$?
2246         if [ $ret -ne 0 ]; then
2247                 fail "Failed to list content of directory \"$trace_path\""
2248                 return $ret
2249         fi
2250
2251         nb_files="$(echo -n "$files" | wc -l)"
2252         test "$nb_files" -eq 0
2253         ok $? "Directory \"$trace_path\" is empty"
2254 }
2255
2256 function validate_trace_session_ust_empty()
2257 {
2258         validate_directory_empty "$1"/ust
2259 }
2260
2261 function validate_trace_session_kernel_empty()
2262 {
2263         validate_trace_empty "$1"/kernel
2264 }
2265
2266 function regenerate_metadata ()
2267 {
2268         local expected_to_fail=$1
2269         local sess_name=$2
2270
2271         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2272                 regenerate metadata -s $sess_name
2273         ret=$?
2274         if [[ $expected_to_fail -eq "1" ]]; then
2275                 test "$ret" -ne "0"
2276                 ok $? "Expected fail on regenerate metadata $sess_name"
2277         else
2278                 ok $ret "Metadata regenerate $sess_name"
2279         fi
2280 }
2281
2282 function regenerate_metadata_ok ()
2283 {
2284         regenerate_metadata 0 "$@"
2285 }
2286
2287 function regenerate_metadata_fail ()
2288 {
2289         regenerate_metadata 1 "$@"
2290 }
2291
2292 function regenerate_statedump ()
2293 {
2294         local expected_to_fail=$1
2295         local sess_name=$2
2296
2297         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2298                 regenerate statedump -s $sess_name
2299         ret=$?
2300         if [[ $expected_to_fail -eq "1" ]]; then
2301                 test "$ret" -ne "0"
2302                 ok $? "Expected fail on regenerate statedump $sess_name"
2303         else
2304                 ok $ret "Statedump regenerate $sess_name"
2305         fi
2306 }
2307
2308 function regenerate_statedump_ok ()
2309 {
2310         regenerate_statedump 0 "$@"
2311 }
2312
2313 function regenerate_statedump_fail ()
2314 {
2315         regenerate_statedump 1 "$@"
2316 }
2317
2318 function rotate_session ()
2319 {
2320         local expected_to_fail=$1
2321         local sess_name=$2
2322
2323         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2324                 rotate $sess_name
2325         ret=$?
2326         if [[ $expected_to_fail -eq "1" ]]; then
2327                 test "$ret" -ne "0"
2328                 ok $? "Expected fail on rotate session $sess_name"
2329         else
2330                 ok $ret "Rotate session $sess_name"
2331         fi
2332 }
2333
2334 function rotate_session_ok ()
2335 {
2336         rotate_session 0 "$@"
2337 }
2338
2339 function rotate_session_fail ()
2340 {
2341         rotate_session 1 "$@"
2342 }
2343
2344 function destructive_tests_enabled ()
2345 {
2346         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2347                 return 0
2348         else
2349                 return 1
2350         fi
2351 }
2352
2353 function lttng_enable_rotation_timer ()
2354 {
2355         local expected_to_fail=$1
2356         local sess_name=$2
2357         local period=$3
2358
2359         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2360                 enable-rotation -s $sess_name --timer $period
2361         ret=$?
2362         if [[ $expected_to_fail -eq "1" ]]; then
2363                 test "$ret" -ne "0"
2364                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2365         else
2366                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2367         fi
2368 }
2369
2370 function lttng_enable_rotation_timer_ok ()
2371 {
2372         lttng_enable_rotation_timer 0 $@
2373 }
2374
2375 function lttng_enable_rotation_timer_fail ()
2376 {
2377         lttng_enable_rotation_timer 1 $@
2378 }
2379
2380 function lttng_enable_rotation_size ()
2381 {
2382         local expected_to_fail=$1
2383         local sess_name=$2
2384         local size=$3
2385
2386         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2387                 enable-rotation -s $sess_name --size $size
2388         ret=$?
2389         if [[ $expected_to_fail -eq "1" ]]; then
2390                 test "$ret" -ne "0"
2391                 ok $? "Expected to fail to set a periodic rotation of session $sess_name" "every " $size " bytes"
2392         else
2393                 ok $ret "Set a scheduled rotation of session $sess_name" "every " $size " bytes"
2394         fi
2395 }
2396
2397 function lttng_enable_rotation_size_ok ()
2398 {
2399         lttng_enable_rotation_size 0 $@
2400 }
2401
2402 function lttng_enable_rotation_size_fail ()
2403 {
2404         lttng_enable_rotation_size 1 $@
2405 }
2406
2407 function lttng_clear_session ()
2408 {
2409         local expected_to_fail=$1
2410         local sess_name=$2
2411
2412         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2413                 clear $sess_name
2414         ret=$?
2415         if [[ $expected_to_fail -eq "1" ]]; then
2416                 test "$ret" -ne "0"
2417                 ok $? "Expected fail on clear session $sess_name"
2418         else
2419                 ok $ret "Clear session $sess_name"
2420         fi
2421 }
2422
2423 function lttng_clear_session_ok ()
2424 {
2425         lttng_clear_session 0 $@
2426 }
2427
2428 function lttng_clear_session_fail ()
2429 {
2430         lttng_clear_session 1 $@
2431 }
2432
2433 function lttng_clear_all ()
2434 {
2435         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2436                 clear --all
2437         ok $? "Clear all lttng sessions"
2438 }
2439
2440 function lttng_add_trigger()
2441 {
2442         local expected_to_fail="$1"
2443         local trigger_name="$2"
2444         shift 2
2445         local args=("$@")
2446
2447         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2448         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2449         ret=$?
2450         if [[ $expected_to_fail -eq "1" ]]; then
2451                 test "$ret" -ne "0"
2452                 ok $? "Add trigger $trigger_name failed as expected"
2453         else
2454                 ok $ret "Add trigger $trigger_name"
2455         fi
2456 }
2457
2458 function lttng_remove_trigger()
2459 {
2460         local expected_to_fail="$1"
2461         local trigger_name="$2"
2462         shift 2
2463
2464         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2465         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2466         ret=$?
2467         if [[ $expected_to_fail -eq "1" ]]; then
2468                 test "$ret" -ne "0"
2469                 ok $? "Remove trigger $trigger_name failed as expected"
2470         else
2471                 ok $ret "Remove trigger $trigger_name"
2472         fi
2473 }
2474
2475 function lttng_add_trigger_ok()
2476 {
2477         lttng_add_trigger 0 "$@"
2478 }
2479
2480 function lttng_add_trigger_fail()
2481 {
2482         lttng_add_trigger 1 "$@"
2483 }
2484
2485 function lttng_remove_trigger_ok()
2486 {
2487         lttng_remove_trigger 0 "$@"
2488 }
2489
2490 function list_triggers_matches_ok ()
2491 {
2492         local tmp_stdout=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2493         local tmp_stderr=$(mktemp -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2494
2495         local test_name="$1"
2496         local expected_stdout_file="$2"
2497
2498         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2499
2500         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2501         ok $? "${test_name}: exit code is 0"
2502
2503         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2504         ok $? "${test_name}: expected stdout"
2505
2506         diff -u /dev/null "${tmp_stderr}"
2507         ok $? "${test_name}: expected stderr"
2508
2509         rm -f "${tmp_stdout}"
2510         rm -f "${tmp_stderr}"
2511 }
2512
2513 function list_triggers_matches_mi_ok ()
2514 {
2515         local tmp_stdout
2516         local tmp_stdout_raw
2517         local tmp_stderr
2518
2519         local test_name="$1"
2520         local expected_stdout_file="$2"
2521
2522         tmp_stdout_raw=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2523         tmp_stdout=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2524         tmp_stderr=$(mktemp -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2525
2526         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2527
2528         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2529         ok $? "${test_name}: exit code is 0"
2530
2531         # Pretty-fy xml before further test.
2532         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2533
2534         $MI_VALIDATE "${tmp_stdout}"
2535         ok $? "list-trigger mi is valid"
2536
2537         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2538         ok $? "${test_name}: expected stdout"
2539
2540         diff -u /dev/null "${tmp_stderr}"
2541         ok $? "${test_name}: expected stderr"
2542
2543         rm -f "${tmp_stdout}"
2544         rm -f "${tmp_stdout_raw}"
2545         rm -f "${tmp_stderr}"
2546 }
2547
2548 function validate_path_pattern ()
2549 {
2550         local message=$1
2551         local pattern=$2
2552         # Base path is only used in error case and is used to list the content
2553         # of the base path.
2554         local base_path=$3
2555
2556
2557         [ -f $pattern ]
2558         ret=$?
2559         ok $ret "$message"
2560
2561         if [ "$ret" -ne "0" ]; then
2562                 diag "Path pattern expected: $pattern"
2563                 # List the tracepath for more info. We use find as a recursive
2564                 # directory lister.
2565                 diag "The base path content:"
2566                 find "$base_path" -print
2567         fi
2568 }
2569
2570 function validate_trace_path_ust_uid ()
2571 {
2572         local trace_path=$1
2573         local session_name=$2
2574         local uid=$UID
2575         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2576
2577         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2578 }
2579
2580 function validate_trace_path_ust_uid_network ()
2581 {
2582         local trace_path=$1
2583         local session_name=$2
2584         local base_path=$3
2585         local uid=$UID
2586         local hostname=$HOSTNAME
2587         local pattern
2588         local ret
2589
2590         # If the session was given a network base path (e.g
2591         # 127.0.0.1/my/custom/path on creation, there is no session name
2592         # component to the path on the relayd side. Caller can simply not pass a
2593         # session name for this scenario.
2594         if [ -n "$session_name" ]; then
2595                 session_name="$session_name-$date_time_pattern"
2596                 if [ -n "$base_path" ]; then
2597                         fail "Session name and base path are mutually exclusive"
2598                         return
2599                 fi
2600         fi
2601
2602         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2603
2604         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2605 }
2606
2607 function validate_trace_path_ust_uid_snapshot_network ()
2608 {
2609         local trace_path=$1
2610         local session_name=$2
2611         local snapshot_name=$3
2612         local snapshot_number=$4
2613         local base_path=$5
2614         local hostname=$HOSTNAME
2615         local uid=$UID
2616         local pattern
2617         local ret
2618
2619         # If the session/output was given a network base path (e.g
2620         # 127.0.0.1/my/custom/path on creation, there is no session name
2621         # component to the path on the relayd side. Caller can simply not pass a
2622         # session name for this scenario.
2623         if [ -n "$session_name" ]; then
2624                 session_name="$session_name-$date_time_pattern"
2625                 if [ -n "$base_path" ]; then
2626                         fail "Session name and base path are mutually exclusive"
2627                         return
2628                 fi
2629         fi
2630
2631         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2632
2633         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2634 }
2635
2636 function validate_trace_path_ust_uid_snapshot ()
2637 {
2638         local trace_path=$1
2639         local session_name=$2
2640         local snapshot_name=$3
2641         local snapshot_number=$4
2642         local base_path=$5
2643         local uid=$UID
2644         local pattern
2645         local ret
2646
2647         # If the session/output was given a network base path (e.g
2648         # 127.0.0.1/my/custom/path) on creation, there is no session name
2649         # component to the path on the relayd side. Caller can simply not pass a
2650         # session name for this scenario.
2651         if [ -n "$session_name" ]; then
2652                 session_name="$session_name-$date_time_pattern"
2653                 if [ -n "$base_path" ]; then
2654                         fail "Session name and base path are mutually exclusive"
2655                         return
2656                 fi
2657         fi
2658
2659         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2660
2661         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2662 }
2663
2664 function validate_trace_path_ust_pid ()
2665 {
2666         local trace_path=$1
2667         local session_name=$2
2668         local app_string=$3
2669         local pid=$4
2670         local pattern
2671         local ret
2672
2673         # If the session was given a trace path on creation, there is no session
2674         # name component to the path. Caller can simply not pass a session name
2675         # for this scenario.
2676         if [ -n "$session_name" ]; then
2677                 session_name="$session_name-$date_time_pattern"
2678         fi
2679
2680         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2681
2682         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2683 }
2684
2685 function validate_trace_path_kernel ()
2686 {
2687         local trace_path=$1
2688         local session_name=$2
2689         local pattern
2690
2691         # If the session was given a trace path on creation, there is no session
2692         # name component to the path. Caller can simply not pass a session name
2693         # for this scenario.
2694         if [ -n "$session_name" ]; then
2695                 session_name="$session_name-$date_time_pattern"
2696         fi
2697
2698         pattern="$trace_path/$session_name/kernel/metadata"
2699
2700         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2701 }
2702
2703 function validate_trace_path_kernel_network ()
2704 {
2705         local trace_path=$1
2706         local session_name=$2
2707         local hostname=$HOSTNAME
2708         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2709
2710         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2711 }
2712
2713 function validate_trace_path_kernel_snapshot ()
2714 {
2715         local trace_path=$1
2716         local session_name=$2
2717         local snapshot_name=$3
2718         local snapshot_number=$4
2719         local base_path=$5
2720         local pattern
2721         local ret
2722
2723         # If the session/output was given a network base path (e.g
2724         # 127.0.0.1/my/custom/path on creation, there is no session name
2725         # component to the path on the relayd side. Caller can simply not pass a
2726         # session name for this scenario.
2727         if [ -n "$session_name" ]; then
2728                 session_name="$session_name-$date_time_pattern"
2729                 if [ -n "$base_path" ]; then
2730                         fail "Session name and base path are mutually exclusive"
2731                         return
2732                 fi
2733         fi
2734
2735         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2736
2737         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2738 }
2739
2740 function validate_trace_path_kernel_snapshot_network ()
2741 {
2742         local trace_path=$1
2743         local session_name=$2
2744         local snapshot_name=$3
2745         local snapshot_number=$4
2746         local base_path=$5
2747         local hostname=$HOSTNAME
2748         local pattern
2749         local ret
2750
2751         # If the session/output was given a network base path (e.g
2752         # 127.0.0.1/my/custom/path on creation, there is no session name
2753         # component to the path on the relayd side. Caller can simply not pass a
2754         # session name for this scenario.
2755         if [ -n "$session_name" ]; then
2756                 session_name="$session_name-$date_time_pattern"
2757                 if [ -n "$base_path" ]; then
2758                         fail "Session name and base path are mutually exclusive"
2759                         return
2760                 fi
2761         fi
2762
2763         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2764
2765         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2766 }