tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace2"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$( (tr '\0' '\n' < /proc/"$pid"/cmdline) 2>/dev/null | head -n1); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         local len="${1:-16}"
 208
 209         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 210         # /dev/urandom isn't guaranteed to generate valid multi-byte characters.
 211         # Specifying the C locale eliminates the "Illegal byte sequence" error
 212         # that 'tr' outputs in such cases.
 213         LC_CTYPE=C tr -cd "$CHAR" < /dev/urandom 2>/dev/null | head -c "$len" 2>/dev/null
 214         echo
 215 }
 216
 217 # Return a space-separated string of online CPU IDs, based on
 218 # /sys/devices/system/cpu/online, or from 0 to nproc - 1 otherwise.
 219 function get_online_cpus()
 220 {
 221         local cpus=()
 222         local range_re
 223         if [ -f /sys/devices/system/cpu/online ]; then
 224                 range_re='([0-9]+)-([0-9]+)'
 225                 while read -r range ; do
 226                         if [[ "${range}" =~ ${range_re} ]] ; then
 227                                 mapfile -t -O "${#cpus[*]}" cpus <<< $(seq "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}")
 228                         else
 229                                 cpus+=("${range}")
 230                         fi
 231                 done < <(tr ',' $'\n' < /sys/devices/system/cpu/online)
 232         else
 233                 read -r -a cpus <<< $(seq 0 $(( $(conf_proc_count) - 1 )) )
 234         fi
 235         echo "${cpus[*]}"
 236 }
 237
 238 # Helpers for get_possible_cpus.
 239 function get_possible_cpus_count_from_sysfs_possible_mask()
 240 {
 241         local max_possible_cpu_id
 242
 243         # The Awk script extracts the highest CPU id from the possible CPU
 244         # mask. Assuming a numerical order, a field separator '-' and a record
 245         # separator ','. The last value parsed is the highest id.
 246         if [ -f /sys/devices/system/cpu/possible ]; then
 247                 max_possible_cpu_id=$(awk -F '-' 'BEGIN { RS = ","} { last = $NF } END { printf("%d\n", last) }' \
 248                                       /sys/devices/system/cpu/possible)
 249                 echo "$((max_possible_cpu_id+1))"
 250         else
 251                 echo "0"
 252         fi
 253 }
 254
 255 # This is a fallback if the possible CPU mask is not available. This will not
 256 # take into account unplugged CPUs.
 257 function get_max_cpus_count_from_sysfs_cpu_directories()
 258 {
 259         local max_possible_cpu_id=0
 260         local current_cpu_id
 261
 262         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 263                 current_cpu_id="${i#/sys/devices/system/cpu/cpu}"
 264                 if [ "$current_cpu_id" -gt "$max_possible_cpu_id" ]; then
 265                         max_possible_cpu_id="$current_cpu_id"
 266                 fi
 267         done
 268
 269         echo "$((max_possible_cpu_id+1))"
 270 }
 271
 272 # Return the number of possible CPUs.
 273 function get_possible_cpus_count()
 274 {
 275         local possible_cpus_count
 276         possible_cpus_count=$(get_possible_cpus_count_from_sysfs_possible_mask)
 277
 278         if [ "$possible_cpus_count" -eq "0" ]; then
 279                 local configured_cpus_count
 280                 configured_cpus_count=$(getconf _NPROCESSORS_CONF)
 281                 possible_cpus_count=$(get_max_cpus_count_from_sysfs_cpu_directories)
 282                 possible_cpus_count=$((configured_cpus_count > possible_cpus_count \
 283                                                              ? configured_cpus_count \
 284                                                              : possible_cpus_count))
 285         fi
 286
 287         echo "$possible_cpus_count"
 288 }
 289
 290 # Return the list of exposed CPU.
 291 #
 292 # NOTE! Use it like so:
 293 #
 294 # IFS=" " read -r -a VARIABLE <<< "$(get_exposed_cpus_list)"
 295 function get_exposed_cpus_list()
 296 {
 297         local list=()
 298
 299         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 300                 list+=("${i#/sys/devices/system/cpu/cpu}")
 301         done
 302
 303         echo "${list[@]}"
 304 }
 305
 306 # Return any available CPU found. Do not make assumption about the returned
 307 # value, e.g. that it could be 0.
 308 function get_any_available_cpu()
 309 {
 310         for cpu in $(get_online_cpus); do
 311                 echo "${cpu}"
 312                 break;
 313         done
 314 }
 315
 316 # Return the number of _configured_ CPUs.
 317 function conf_proc_count()
 318 {
 319         getconf _NPROCESSORS_CONF
 320         if [ $? -ne 0 ]; then
 321                 diag "Failed to get the number of configured CPUs"
 322         fi
 323         echo
 324 }
 325
 326 # Check if base lttng-modules are present.
 327 # Bail out on failure
 328 function validate_lttng_modules_present ()
 329 {
 330         # Check for loadable modules.
 331         modprobe -n lttng-tracer 2>/dev/null
 332         if [ $? -eq 0 ]; then
 333                 return 0
 334         fi
 335
 336         # Check for builtin modules.
 337         ls /proc/lttng > /dev/null 2>&1
 338         if [ $? -eq 0 ]; then
 339                 return 0
 340         fi
 341
 342         LTTNG_BAIL_OUT "LTTng modules not detected."
 343 }
 344
 345 # Run the lttng binary.
 346 #
 347 # The first two arguments are stdout and stderr redirect paths, respectively.
 348 # The rest of the arguments are forwarded to the lttng binary
 349 function _run_lttng_cmd
 350 {
 351         local stdout_dest="$1"
 352         local stderr_dest="$2"
 353         shift 2
 354
 355         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 356         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 357 }
 358
 359 function enable_kernel_lttng_event
 360 {
 361         local withtap="$1"
 362         local expected_to_fail="$2"
 363         local sess_name="$3"
 364         local event_name="$4"
 365         local channel_name="$5"
 366
 367         if [ -z "$event_name" ]; then
 368                 # Enable all event if no event name specified
 369                 event_name="-a"
 370         fi
 371
 372         if [ -z "$channel_name" ]; then
 373                 # default channel if none specified
 374                 chan=""
 375         else
 376                 chan="-c $channel_name"
 377         fi
 378
 379         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 380                 enable-event "$event_name" $chan -s $sess_name -k
 381         ret=$?
 382         if [[ $expected_to_fail -eq "1" ]]; then
 383                 test $ret -ne "0"
 384                 ret=$?
 385                 if [ $withtap -eq "1" ]; then
 386                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 387                 fi
 388         else
 389                 if [ $withtap -eq "1" ]; then
 390                         ok $ret "Enable kernel event $event_name for session $sess_name"
 391                 fi
 392         fi
 393 }
 394
 395 function enable_kernel_lttng_event_ok ()
 396 {
 397         enable_kernel_lttng_event 1 0 "$@"
 398 }
 399
 400 function enable_kernel_lttng_event_fail ()
 401 {
 402         enable_kernel_lttng_event 1 1 "$@"
 403 }
 404
 405 function enable_kernel_lttng_event_notap ()
 406 {
 407         enable_kernel_lttng_event 0 0 "$@"
 408 }
 409
 410 # Old interface
 411 function lttng_enable_kernel_event
 412 {
 413         enable_kernel_lttng_event_ok "$@"
 414 }
 415
 416 function lttng_enable_kernel_syscall()
 417 {
 418         local expected_to_fail=$1
 419         local sess_name=$2
 420         local syscall_name=$3
 421         local channel_name=$4
 422
 423         if [ -z $syscall_name ]; then
 424                 # Enable all event if no syscall name specified
 425                 syscall_name="-a"
 426         fi
 427
 428         if [ -z $channel_name ]; then
 429                 # default channel if none specified
 430                 chan=""
 431         else
 432                 chan="-c $channel_name"
 433         fi
 434
 435         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 436                 enable-event --syscall "$syscall_name" $chan -s $sess_name -k
 437         ret=$?
 438         if [[ $expected_to_fail -eq "1" ]]; then
 439                 test $ret -ne "0"
 440                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 441         else
 442                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 443         fi
 444 }
 445
 446 function lttng_enable_kernel_syscall_ok()
 447 {
 448         lttng_enable_kernel_syscall 0 "$@"
 449 }
 450
 451 function lttng_enable_kernel_syscall_fail()
 452 {
 453         lttng_enable_kernel_syscall 1 "$@"
 454 }
 455
 456 function lttng_disable_kernel_syscall()
 457 {
 458         local expected_to_fail=$1
 459         local sess_name=$2
 460         local syscall_name=$3
 461         local channel_name=$4
 462
 463         if [ -z $syscall_name ]; then
 464                 # Enable all event if no syscall name specified
 465                 syscall_name="-a"
 466         fi
 467
 468         if [ -z $channel_name ]; then
 469                 # default channel if none specified
 470                 chan=""
 471         else
 472                 chan="-c $channel_name"
 473         fi
 474
 475         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 476                 disable-event --syscall "$syscall_name" $chan -s $sess_name -k
 477
 478         ret=$?
 479         if [[ $expected_to_fail -eq "1" ]]; then
 480                 test $ret -ne "0"
 481                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 482         else
 483                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 484         fi
 485 }
 486
 487 function lttng_disable_kernel_syscall_ok()
 488 {
 489         lttng_disable_kernel_syscall 0 "$@"
 490 }
 491
 492 function lttng_disable_kernel_syscall_fail()
 493 {
 494         lttng_disable_kernel_syscall 1 "$@"
 495 }
 496
 497 function lttng_enable_kernel_function_event ()
 498 {
 499         local expected_to_fail="$1"
 500         local sess_name="$2"
 501         local target="$3"
 502         local event_name="$4"
 503
 504         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 505         ret=$?
 506         if [[ $expected_to_fail -eq "1" ]]; then
 507                 test $ret -ne "0"
 508                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 509         else
 510                 ok $ret "Enable kernel function event for session $sess_name"
 511         fi
 512 }
 513
 514 function lttng_enable_kernel_function_event_ok ()
 515 {
 516         lttng_enable_kernel_function_event 0 "$@"
 517 }
 518
 519 function lttng_enable_kernel_userspace_probe_event ()
 520 {
 521         local expected_to_fail="$1"
 522         local sess_name="$2"
 523         local target="$3"
 524         local event_name="$4"
 525
 526         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 527         ret=$?
 528         if [[ $expected_to_fail -eq "1" ]]; then
 529                 test $ret -ne "0"
 530                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 531         else
 532                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 533         fi
 534 }
 535
 536 function lttng_enable_kernel_userspace_probe_event_fail ()
 537 {
 538         lttng_enable_kernel_userspace_probe_event 1 "$@"
 539 }
 540
 541 function lttng_enable_kernel_userspace_probe_event_ok ()
 542 {
 543         lttng_enable_kernel_userspace_probe_event 0 "$@"
 544 }
 545
 546 function disable_kernel_lttng_userspace_probe_event_ok ()
 547 {
 548         local sess_name="$1"
 549         local event_name="$2"
 550
 551         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 552         ok $? "Disable kernel event $target for session $sess_name"
 553 }
 554 function lttng_enable_kernel_channel()
 555 {
 556         local withtap=$1
 557         local expected_to_fail=$2
 558         local sess_name=$3
 559         local channel_name=$4
 560         local opts="${@:5}"
 561
 562         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 563                 enable-channel -k $channel_name -s $sess_name $opts
 564         ret=$?
 565         if [[ $expected_to_fail -eq "1" ]]; then
 566                 test "$ret" -ne "0"
 567                 ret=$?
 568                 if [ $withtap -eq "1" ]; then
 569                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 570                 fi
 571         else
 572                 if [ $withtap -eq "1" ]; then
 573                         ok $ret "Enable channel $channel_name for session $sess_name"
 574                 fi
 575         fi
 576 }
 577
 578 function lttng_enable_kernel_channel_ok()
 579 {
 580         lttng_enable_kernel_channel 1 0 "$@"
 581 }
 582
 583 function lttng_enable_kernel_channel_fail()
 584 {
 585         lttng_enable_kernel_channel 1 1 "$@"
 586 }
 587
 588 function lttng_enable_kernel_channel_notap()
 589 {
 590         lttng_enable_kernel_channel 0 0 "$@"
 591 }
 592
 593 function enable_kernel_lttng_channel_ok()
 594 {
 595         lttng_enable_kernel_channel 1 0 "$@"
 596 }
 597
 598 function lttng_disable_kernel_channel()
 599 {
 600         local expected_to_fail=$1
 601         local sess_name=$2
 602         local channel_name=$3
 603
 604         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 605                 disable-channel -k $channel_name -s $sess_name
 606         ret=$?
 607         if [[ $expected_to_fail -eq "1" ]]; then
 608                 test "$ret" -ne "0"
 609                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 610         else
 611                 ok $ret "Disable channel $channel_name for session $sess_name"
 612         fi
 613 }
 614
 615 function lttng_disable_kernel_channel_ok()
 616 {
 617         lttng_disable_kernel_channel 0 "$@"
 618 }
 619
 620 function lttng_disable_kernel_channel_fail()
 621 {
 622         lttng_disable_kernel_channel 1 "$@"
 623 }
 624
 625 function start_lttng_relayd_opt()
 626 {
 627         local withtap=$1
 628         local process_mode=$2
 629         local opt=$3
 630
 631         DIR=$(readlink -f "$TESTDIR")
 632
 633         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 634                 # shellcheck disable=SC2086
 635                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 636                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt -vvv >>/tmp/relayd.log 2>&1 &
 637                 if [ $? -eq 1 ]; then
 638                         if [ $withtap -eq "1" ]; then
 639                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 640                         fi
 641                         return 1
 642                 else
 643                         if [ $withtap -eq "1" ]; then
 644                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 645                         fi
 646                 fi
 647         else
 648                 pass "Start lttng-relayd (opt: $opt)"
 649         fi
 650 }
 651
 652 function start_lttng_relayd()
 653 {
 654         start_lttng_relayd_opt 1 "-b" "$@"
 655 }
 656
 657 function start_lttng_relayd_notap()
 658 {
 659         start_lttng_relayd_opt 0 "-b" "$@"
 660 }
 661
 662 function stop_lttng_relayd_opt()
 663 {
 664         local withtap=$1
 665         local is_cleanup=$2
 666         local signal=$3
 667         local timeout_s=$4
 668         local dtimeleft_s=
 669         local retval=0
 670         local pids
 671
 672         if [ -z "$signal" ]; then
 673                 signal="SIGTERM"
 674         fi
 675
 676
 677         # Multiply time by 2 to simplify integer arithmetic
 678         if [ -n "$timeout_s" ]; then
 679                 dtimeleft_s=$((timeout_s * 2))
 680         fi
 681
 682
 683         pids=$(lttng_pgrep "$RELAYD_MATCH")
 684         if [ -z "$pids" ]; then
 685                 if [ "$is_cleanup" -eq 1 ]; then
 686                         :
 687                 elif [ "$withtap" -eq "1" ]; then
 688                         fail "No relay daemon to kill"
 689                 else
 690                         LTTNG_BAIL_OUT "No relay daemon to kill"
 691                 fi
 692                 return 0
 693         fi
 694
 695         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 696
 697         # shellcheck disable=SC2086
 698         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 699                 retval=1
 700                 if [ "$withtap" -eq "1" ]; then
 701                         fail "Kill relay daemon"
 702                 fi
 703         else
 704                 out=1
 705                 while [ -n "$out" ]; do
 706                         out=$(lttng_pgrep "$RELAYD_MATCH")
 707                         if [ -n "$dtimeleft_s" ]; then
 708                                 if [ $dtimeleft_s -lt 0 ]; then
 709                                         out=
 710                                         retval=1
 711                                 fi
 712                                 dtimeleft_s=$((dtimeleft_s - 1))
 713                         fi
 714                         sleep 0.5
 715                 done
 716                 if [ "$withtap" -eq "1" ]; then
 717                         if [ "$retval" -eq "0" ]; then
 718                                 pass "Wait after kill relay daemon"
 719                         else
 720                                 fail "Wait after kill relay daemon"
 721                         fi
 722                 fi
 723         fi
 724         return $retval
 725 }
 726
 727 function stop_lttng_relayd()
 728 {
 729         stop_lttng_relayd_opt 1 0 "$@"
 730 }
 731
 732 function stop_lttng_relayd_notap()
 733 {
 734         stop_lttng_relayd_opt 0 0 "$@"
 735 }
 736
 737 function stop_lttng_relayd_cleanup()
 738 {
 739         stop_lttng_relayd_opt 0 1 "$@"
 740 }
 741
 742 #First arg: show tap output
 743 #Second argument: load path for automatic loading
 744 function start_lttng_sessiond_opt()
 745 {
 746         local withtap=$1
 747         local load_path=$2
 748
 749         # The rest of the arguments will be passed directly to lttng-sessiond.
 750         shift 2
 751
 752         local env_vars=""
 753         local consumerd=""
 754
 755         local long_bit_value=
 756         long_bit_value=$(getconf LONG_BIT)
 757
 758         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 759                 # Env variable requested no session daemon
 760                 return
 761         fi
 762
 763         DIR=$(readlink -f "$TESTDIR")
 764
 765         # Get long_bit value for 32/64 consumerd
 766         case "$long_bit_value" in
 767                 32)
 768                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 769                         ;;
 770                 64)
 771                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 772                         ;;
 773                 *)
 774                         return
 775                         ;;
 776         esac
 777
 778         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 779         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 780                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 781         fi
 782         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 783
 784         if ! validate_kernel_version; then
 785             fail "Start session daemon"
 786             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 787         fi
 788
 789         diag "export LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/"
 790         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/"}"
 791         export LTTNG_SESSION_CONFIG_XSD_PATH
 792
 793         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 794                 # Have a load path ?
 795                 if [ -n "$load_path" ]; then
 796                         diag "env $env_vars --load $load_path --background $consumerd $@"
 797                         # shellcheck disable=SC2086
 798                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 799                 else
 800                         diag "env $env_vars --background $consumerd $@"
 801                         # shellcheck disable=SC2086
 802                         env $env_vars --background "$consumerd" "$@"
 803                 fi
 804                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 805                 status=$?
 806                 if [ "$withtap" -eq "1" ]; then
 807                         ok $status "Start session daemon"
 808                 fi
 809         fi
 810 }
 811
 812 function start_lttng_sessiond()
 813 {
 814         start_lttng_sessiond_opt 1 "$@"
 815 }
 816
 817 function start_lttng_sessiond_notap()
 818 {
 819         start_lttng_sessiond_opt 0 "$@"
 820 }
 821
 822 function stop_lttng_sessiond_opt()
 823 {
 824         local withtap=$1
 825         local is_cleanup=$2
 826         local signal=$3
 827         local timeout_s=$4
 828         local dtimeleft_s=
 829         local retval=0
 830         local runas_pids
 831         local pids
 832
 833         if [ -z "$signal" ]; then
 834                 signal=SIGTERM
 835         fi
 836
 837         # Multiply time by 2 to simplify integer arithmetic
 838         if [ -n "$timeout_s" ]; then
 839                 dtimeleft_s=$((timeout_s * 2))
 840         fi
 841
 842         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 843                 # Env variable requested no session daemon
 844                 return 0
 845         fi
 846
 847         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 848         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 849
 850         if [ -n "$runas_pids" ]; then
 851                 pids="$pids $runas_pids"
 852         fi
 853
 854         if [ -z "$pids" ]; then
 855                 if [ "$is_cleanup" -eq 1 ]; then
 856                         :
 857                 elif [ "$withtap" -eq "1" ]; then
 858                         fail "No session daemon to kill"
 859                 else
 860                         LTTNG_BAIL_OUT "No session daemon to kill"
 861                 fi
 862                 return 0
 863         fi
 864
 865         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 866
 867         # shellcheck disable=SC2086
 868         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 869                 retval=1
 870                 if [ "$withtap" -eq "1" ]; then
 871                         fail "Kill sessions daemon"
 872                 fi
 873         else
 874                 out=1
 875                 while [ -n "$out" ]; do
 876                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 877                         if [ -n "$dtimeleft_s" ]; then
 878                                 if [ $dtimeleft_s -lt 0 ]; then
 879                                         out=
 880                                         retval=1
 881                                 fi
 882                                 dtimeleft_s=$((dtimeleft_s - 1))
 883                         fi
 884                         sleep 0.5
 885                 done
 886                 out=1
 887                 while [ -n "$out" ]; do
 888                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 889                         if [ -n "$dtimeleft_s" ]; then
 890                                 if [ $dtimeleft_s -lt 0 ]; then
 891                                         out=
 892                                         retval=1
 893                                 fi
 894                                 dtimeleft_s=$((dtimeleft_s - 1))
 895                         fi
 896                         sleep 0.5
 897                 done
 898
 899                 if [ "$withtap" -eq "1" ]; then
 900                         if [ "$retval" -eq "0" ]; then
 901                                 pass "Wait after kill session daemon"
 902                         else
 903                                 fail "Wait after kill session daemon"
 904                         fi
 905                 fi
 906         fi
 907         if [ "$signal" = "SIGKILL" ]; then
 908                 if [ "$(id -u)" -eq "0" ]; then
 909                         local modules=
 910                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 911
 912                         if [ -n "$modules" ]; then
 913                                 diag "Unloading all LTTng modules"
 914                                 modprobe --remove "$modules"
 915                         fi
 916                 fi
 917         fi
 918
 919         return $retval
 920 }
 921
 922 function stop_lttng_sessiond()
 923 {
 924         stop_lttng_sessiond_opt 1 0 "$@"
 925 }
 926
 927 function stop_lttng_sessiond_notap()
 928 {
 929         stop_lttng_sessiond_opt 0 0 "$@"
 930 }
 931
 932 function stop_lttng_sessiond_cleanup()
 933 {
 934         stop_lttng_sessiond_opt 0 1 "$@"
 935 }
 936
 937 function sigstop_lttng_sessiond_opt()
 938 {
 939         local withtap=$1
 940         local signal=SIGSTOP
 941         local pids
 942
 943         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 944                 # Env variable requested no session daemon
 945                 return
 946         fi
 947
 948         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 949
 950         if [ "$withtap" -eq "1" ]; then
 951                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 952         fi
 953
 954         # shellcheck disable=SC2086
 955         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 956                 if [ "$withtap" -eq "1" ]; then
 957                         fail "Sending SIGSTOP to session daemon"
 958                 fi
 959         else
 960                 out=1
 961                 while [ $out -ne 0 ]; do
 962                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 963
 964                         # Wait until state becomes stopped for session
 965                         # daemon(s).
 966                         out=0
 967                         for sessiond_pid in $pids; do
 968                                 state="$(ps -p "$sessiond_pid" -o state= )"
 969                                 if [[ -n "$state" && "$state" != "T" ]]; then
 970                                         out=1
 971                                 fi
 972                         done
 973                         sleep 0.5
 974                 done
 975                 if [ "$withtap" -eq "1" ]; then
 976                         pass "Sending SIGSTOP to session daemon"
 977                 fi
 978         fi
 979 }
 980
 981 function sigstop_lttng_sessiond()
 982 {
 983         sigstop_lttng_sessiond_opt 1 "$@"
 984 }
 985
 986 function sigstop_lttng_sessiond_notap()
 987 {
 988         sigstop_lttng_sessiond_opt 0 "$@"
 989 }
 990
 991 function stop_lttng_consumerd_opt()
 992 {
 993         local withtap=$1
 994         local is_cleanup=$2
 995         local signal=$3
 996         local timeout_s=$4
 997         local dtimeleft_s=
 998         local retval=0
 999         local pids
1000
1001         if [ -z "$signal" ]; then
1002                 signal=SIGTERM
1003         fi
1004
1005         # Multiply time by 2 to simplify integer arithmetic
1006         if [ -n "$timeout_s" ]; then
1007                 dtimeleft_s=$((timeout_s * 2))
1008         fi
1009
1010         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1011
1012         if [ -z "$pids" ]; then
1013                 if [ "$is_cleanup" -eq 1 ]; then
1014                         :
1015                 elif [ "$withtap" -eq "1" ]; then
1016                         fail "No consumerd daemon to kill"
1017                 else
1018                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
1019                 fi
1020                 return 0
1021         fi
1022
1023         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1024
1025         # shellcheck disable=SC2086
1026         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
1027                 retval=1
1028                 if [ "$withtap" -eq "1" ]; then
1029                         fail "Kill consumer daemon"
1030                 fi
1031         else
1032                 out=1
1033                 while [ $out -ne 0 ]; do
1034                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1035
1036                         # If consumerds are still present check their status.
1037                         # A zombie status qualifies the consumerd as *killed*
1038                         out=0
1039                         for consumer_pid in $pids; do
1040                                 state="$(ps -p "$consumer_pid" -o state= )"
1041                                 if [[ -n "$state" && "$state" != "Z" ]]; then
1042                                         out=1
1043                                 fi
1044                         done
1045                         if [ -n "$dtimeleft_s" ]; then
1046                                 if [ $dtimeleft_s -lt 0 ]; then
1047                                         out=0
1048                                         retval=1
1049                                 fi
1050                                 dtimeleft_s=$((dtimeleft_s - 1))
1051                         fi
1052                         sleep 0.5
1053                 done
1054                 if [ "$withtap" -eq "1" ]; then
1055                         if [ "$retval" -eq "0" ]; then
1056                                 pass "Wait after kill consumer daemon"
1057                         else
1058                                 fail "Wait after kill consumer daemon"
1059                         fi
1060                 fi
1061         fi
1062
1063         return $retval
1064 }
1065
1066 function stop_lttng_consumerd()
1067 {
1068         stop_lttng_consumerd_opt 1 0 "$@"
1069 }
1070
1071 function stop_lttng_consumerd_notap()
1072 {
1073         stop_lttng_consumerd_opt 0 0 "$@"
1074 }
1075
1076 function stop_lttng_consumerd_cleanup()
1077 {
1078         stop_lttng_consumerd_opt 0 1 "$@"
1079 }
1080
1081 function sigstop_lttng_consumerd_opt()
1082 {
1083         local withtap=$1
1084         local signal=SIGSTOP
1085         local pids
1086
1087         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1088
1089         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1090
1091         # shellcheck disable=SC2086
1092         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1093         retval=$?
1094
1095         if [ $retval -eq 1 ]; then
1096                 if [ "$withtap" -eq "1" ]; then
1097                         fail "Sending SIGSTOP to consumer daemon"
1098                 fi
1099                 return 1
1100         else
1101                 out=1
1102                 while [ $out -ne 0 ]; do
1103                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1104
1105                         # Wait until state becomes stopped for all
1106                         # consumers.
1107                         out=0
1108                         for consumer_pid in $pids; do
1109                                 state="$(ps -p "$consumer_pid" -o state= )"
1110                                 if [[ -n "$state" && "$state" != "T" ]]; then
1111                                         out=1
1112                                 fi
1113                         done
1114                         sleep 0.5
1115                 done
1116                 if [ "$withtap" -eq "1" ]; then
1117                         pass "Sending SIGSTOP to consumer daemon"
1118                 fi
1119         fi
1120         return $retval
1121 }
1122
1123 function sigstop_lttng_consumerd()
1124 {
1125         sigstop_lttng_consumerd_opt 1 "$@"
1126 }
1127
1128 function sigstop_lttng_consumerd_notap()
1129 {
1130         sigstop_lttng_consumerd_opt 0 "$@"
1131 }
1132
1133 function list_lttng_with_opts ()
1134 {
1135         local ret
1136         local withtap=$1
1137         shift
1138         local opts=$1
1139         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1140                 list $opts
1141         ret=$?
1142         if [ $withtap -eq "1" ]; then
1143                 ok $ret "Lttng-tool list command with option $opts"
1144         fi
1145 }
1146
1147 function list_lttng_ok ()
1148 {
1149         list_lttng_with_opts 1 "$@"
1150 }
1151
1152 function list_lttng_notap ()
1153 {
1154         list_lttng_with_opts 0 "$@"
1155 }
1156
1157 function create_lttng_session_no_output ()
1158 {
1159         local sess_name=$1
1160         local opts="${@:2}"
1161
1162         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1163                 create $sess_name --no-output $opts
1164         ok $? "Create session $sess_name in no-output mode"
1165 }
1166
1167 function create_lttng_session_uri () {
1168         local sess_name=$1
1169         local uri=$2
1170         local opts="${@:3}"
1171
1172         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1173                 create $sess_name -U $uri $opts
1174         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1175 }
1176
1177 function create_lttng_session ()
1178 {
1179         local withtap=$1
1180         local expected_to_fail=$2
1181         local sess_name=$3
1182         local trace_path=$4
1183         local opt=$5
1184
1185         if [ -z "$trace_path" ]; then
1186                 # Use lttng-sessiond default output.
1187                 trace_path=""
1188         else
1189                 trace_path="-o $trace_path"
1190         fi
1191
1192         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1193                 create "$sess_name" $trace_path $opt
1194         ret=$?
1195         if [ $expected_to_fail -eq "1" ]; then
1196                 test "$ret" -ne "0"
1197                 ret=$?
1198                 if [ $withtap -eq "1" ]; then
1199                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1200                 fi
1201         else
1202                 if [ $withtap -eq "1" ]; then
1203                         ok $ret "Create session $sess_name in $trace_path"
1204                 fi
1205         fi
1206         return $ret
1207 }
1208
1209 function create_lttng_session_ok ()
1210 {
1211         create_lttng_session 1 0 "$@"
1212 }
1213
1214 function create_lttng_session_fail ()
1215 {
1216         create_lttng_session 1 1 "$@"
1217 }
1218
1219 function create_lttng_session_notap ()
1220 {
1221         create_lttng_session 0 0 "$@"
1222 }
1223
1224
1225 function enable_ust_lttng_channel ()
1226 {
1227         local withtap=$1
1228         local expected_to_fail=$2
1229         local sess_name=$3
1230         local channel_name=$4
1231         local opts="${@:5}"
1232
1233         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1234                 enable-channel -u $channel_name -s $sess_name $opts
1235         ret=$?
1236         if [[ $expected_to_fail -eq "1" ]]; then
1237                 test "$ret" -ne "0"
1238                 ret=$?
1239                 if [ $withtap -eq "1" ]; then
1240                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1241                 fi
1242         else
1243                 if [ $withtap -eq "1" ]; then
1244                         ok $ret "Enable channel $channel_name for session $sess_name"
1245                 fi
1246         fi
1247         return $ret
1248 }
1249
1250 function enable_ust_lttng_channel_ok ()
1251 {
1252         enable_ust_lttng_channel 1 0 "$@"
1253 }
1254
1255 function enable_ust_lttng_channel_fail ()
1256 {
1257         enable_ust_lttng_channel 1 1 "$@"
1258 }
1259
1260 function enable_ust_lttng_channel_notap ()
1261 {
1262         enable_ust_lttng_channel 0 0 "$@"
1263 }
1264
1265 function disable_ust_lttng_channel()
1266 {
1267         local sess_name=$1
1268         local channel_name=$2
1269
1270         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1271                 disable-channel -u $channel_name -s $sess_name
1272         ok $? "Disable channel $channel_name for session $sess_name"
1273 }
1274
1275 function enable_lttng_mmap_overwrite_kernel_channel()
1276 {
1277         local sess_name=$1
1278         local channel_name=$2
1279
1280         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1281                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite
1282         ok $? "Enable channel $channel_name for session $sess_name"
1283 }
1284
1285 function enable_lttng_mmap_discard_small_kernel_channel()
1286 {
1287         local sess_name=$1
1288         local channel_name=$2
1289
1290         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1291                 enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1292         ok $? "Enable small discard channel $channel_name for session $sess_name"
1293 }
1294
1295 function enable_lttng_mmap_overwrite_small_kernel_channel()
1296 {
1297         local sess_name=$1
1298         local channel_name=$2
1299
1300         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1301                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1302         ok $? "Enable small discard channel $channel_name for session $sess_name"
1303 }
1304
1305 function enable_lttng_mmap_overwrite_ust_channel()
1306 {
1307         local sess_name=$1
1308         local channel_name=$2
1309
1310         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1311                 enable-channel -s $sess_name $channel_name -u --output mmap --overwrite
1312         ok $? "Enable channel $channel_name for session $sess_name"
1313 }
1314
1315 function enable_ust_lttng_event ()
1316 {
1317         local withtap=$1
1318         local expected_to_fail=$2
1319         local sess_name=$3
1320         local event_name="$4"
1321         local channel_name=$5
1322
1323         if [ -z $channel_name ]; then
1324                 # default channel if none specified
1325                 chan=""
1326         else
1327                 chan="-c $channel_name"
1328         fi
1329
1330         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1331                 enable-event "$event_name" $chan -s "$sess_name" -u
1332         ret=$?
1333         if [[ $expected_to_fail -eq "1" ]]; then
1334                 test $ret -ne "0"
1335                 ret=$?
1336                 if [[ $withtap -eq "1" ]]; then
1337                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1338                 fi
1339         else
1340                 if [[ $withtap -eq "1" ]]; then
1341                         ok $ret "Enable ust event $event_name for session $sess_name"
1342                 fi
1343         fi
1344         return $ret
1345 }
1346
1347 function enable_ust_lttng_event_ok ()
1348 {
1349         enable_ust_lttng_event 1 0 "$@"
1350 }
1351
1352 function enable_ust_lttng_event_fail ()
1353 {
1354         enable_ust_lttng_event 1 1 "$@"
1355 }
1356
1357 function enable_ust_lttng_event_notap ()
1358 {
1359         enable_ust_lttng_event 0 0 "$@"
1360 }
1361
1362 function enable_jul_lttng_event()
1363 {
1364         sess_name=$1
1365         event_name="$2"
1366         channel_name=$3
1367
1368         if [ -z $channel_name ]; then
1369                 # default channel if none specified
1370                 chan=""
1371         else
1372                 chan="-c $channel_name"
1373         fi
1374
1375         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1376                 enable-event "$event_name" $chan -s $sess_name -j
1377         ok $? "Enable JUL event $event_name for session $sess_name"
1378 }
1379
1380 function enable_jul_lttng_event_loglevel()
1381 {
1382         local sess_name=$1
1383         local event_name="$2"
1384         local loglevel=$3
1385         local channel_name=$4
1386
1387         if [ -z $channel_name ]; then
1388                 # default channel if none specified
1389                 chan=""
1390         else
1391                 chan="-c $channel_name"
1392         fi
1393
1394         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1395                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j
1396         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1397 }
1398
1399 function enable_log4j_lttng_event()
1400 {
1401         local sess_name=$1
1402         local event_name=$2
1403         local channel_name=$3
1404
1405         local chan_opt=()
1406
1407         # default channel if none specified
1408         if [ -n "$channel_name" ]; then
1409                 chan_opt=("-c" "$channel_name")
1410         fi
1411
1412         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1413                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1414         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1415 }
1416
1417 function enable_log4j_lttng_event_filter()
1418 {
1419         local sess_name=$1
1420         local event_name=$2
1421         local filter=$3
1422
1423         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1424                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1425         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1426 }
1427
1428 function enable_log4j_lttng_event_filter_loglevel_only()
1429 {
1430         local sess_name=$1
1431         local event_name=$2
1432         local filter=$3
1433         local loglevel=$4
1434
1435         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1436                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1437         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1438 }
1439
1440 function enable_log4j_lttng_event_loglevel()
1441 {
1442         local sess_name=$1
1443         local event_name=$2
1444         local loglevel=$3
1445         local channel_name=$4
1446
1447
1448         # default channel if none specified
1449         if [ -n "$channel_name" ]; then
1450                 chan_opt=("-c" "$channel_name")
1451         fi
1452
1453         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1454                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1455         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1456 }
1457
1458 function enable_log4j_lttng_event_loglevel_only()
1459 {
1460         local sess_name=$1
1461         local event_name=$2
1462         local loglevel=$3
1463         local channel_name=$4
1464
1465         local chan_opt=()
1466
1467         # default channel if none specified
1468         if [ -n "$channel_name" ]; then
1469                 chan_opt=("-c" "$channel_name")
1470         fi
1471
1472         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1473                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1474         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1475 }
1476
1477 function enable_python_lttng_event()
1478 {
1479         sess_name=$1
1480         event_name="$2"
1481         channel_name=$3
1482
1483         if [ -z $channel_name ]; then
1484                 # default channel if none specified
1485                 chan=""
1486         else
1487                 chan="-c $channel_name"
1488         fi
1489
1490         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1491                 enable-event "$event_name" $chan -s $sess_name -p
1492         ok $? "Enable Python event $event_name for session $sess_name"
1493 }
1494
1495 function enable_python_lttng_event_loglevel()
1496 {
1497         local sess_name=$1
1498         local event_name="$2"
1499         local loglevel=$3
1500         local channel_name=$4
1501
1502         if [ -z $channel_name ]; then
1503                 # default channel if none specified
1504                 chan=""
1505         else
1506                 chan="-c $channel_name"
1507         fi
1508
1509         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1510                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p
1511         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1512 }
1513
1514 function enable_ust_lttng_event_filter()
1515 {
1516         local sess_name="$1"
1517         local event_name="$2"
1518         local filter="$3"
1519         local channel_name=$4
1520
1521         if [ -z $channel_name ]; then
1522                 # default channel if none specified
1523                 chan=""
1524         else
1525                 chan="-c $channel_name"
1526         fi
1527
1528         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1529                 enable-event $chan "$event_name" -s $sess_name -u --filter "$filter"
1530         ok $? "Enable event $event_name with filtering for session $sess_name"
1531 }
1532
1533 function enable_ust_lttng_event_loglevel()
1534 {
1535         local sess_name="$1"
1536         local event_name="$2"
1537         local loglevel="$3"
1538         local channel_name="$4"
1539         local chan=()
1540         if [ -n "${channel_name}" ] ; then
1541                 chan=('-c' "${channel_name}")
1542         fi
1543
1544         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1545                 enable-event "${chan[@]}" "$event_name" -s "${sess_name}" -u --loglevel="${loglevel}"
1546         ok $? "Enable event $event_name with loglevel $loglevel"
1547 }
1548
1549 function enable_ust_lttng_event_loglevel_only()
1550 {
1551         local sess_name="$1"
1552         local event_name="$2"
1553         local loglevel="$3"
1554         local channel_name="$4"
1555         local chan=()
1556         if [ -n "${channel_name}" ] ; then
1557                 chan=('-c' "${channel_name}")
1558         fi
1559
1560         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1561                 enable-event "${chan[@]}" "$event_name" -s "${sess_name}" -u --loglevel-only "${loglevel}"
1562         ok $? "Enable event $event_name with loglevel-only $loglevel"
1563 }
1564
1565 function disable_ust_lttng_event ()
1566 {
1567         local sess_name="$1"
1568         local event_name="$2"
1569         local channel_name="$3"
1570
1571         if [ -z $channel_name ]; then
1572                 # default channel if none specified
1573                 chan=""
1574         else
1575                 chan="-c $channel_name"
1576         fi
1577
1578         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1579                 disable-event "$event_name" -s $sess_name $chan -u
1580         ok $? "Disable event $event_name for session $sess_name"
1581 }
1582
1583 function disable_jul_lttng_event ()
1584 {
1585         local sess_name="$1"
1586         local event_name="$2"
1587
1588         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1589         ok $? "Disable JUL event $event_name for session $sess_name"
1590 }
1591
1592 function disable_log4j_lttng_event ()
1593 {
1594         local sess_name="$1"
1595         local event_name="$2"
1596
1597         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1598                 disable-event "$event_name" -s "$sess_name" --log4j
1599         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1600 }
1601
1602 function disable_python_lttng_event ()
1603 {
1604         local sess_name="$1"
1605         local event_name="$2"
1606
1607         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1608                 disable-event "$event_name" -s $sess_name -p
1609         ok $? "Disable Python event $event_name for session $sess_name"
1610 }
1611
1612 function start_lttng_tracing_opt ()
1613 {
1614         local withtap=$1
1615         local expected_to_fail=$2
1616         local sess_name=$3
1617
1618         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1619                 start $sess_name
1620         ret=$?
1621         if [[ $expected_to_fail -eq "1" ]]; then
1622                 test "$ret" -ne "0"
1623                 ret=$?
1624                 if [ $withtap -eq "1" ]; then
1625                         ok $? "Start tracing for session $sess_name failed as expected"
1626                 fi
1627         else
1628                 if [ $withtap -eq "1" ]; then
1629                         ok $ret "Start tracing for session $sess_name"
1630                 fi
1631         fi
1632 }
1633
1634 function start_lttng_tracing_ok ()
1635 {
1636         start_lttng_tracing_opt 1 0 "$@"
1637 }
1638
1639 function start_lttng_tracing_fail ()
1640 {
1641         start_lttng_tracing_opt 1 1 "$@"
1642 }
1643
1644 function start_lttng_tracing_notap ()
1645 {
1646         start_lttng_tracing_opt 0 1 "$@"
1647 }
1648
1649 function stop_lttng_tracing_opt ()
1650 {
1651         local withtap=$1
1652         local expected_to_fail=$2
1653         local sess_name=$3
1654
1655         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1656                 stop $sess_name
1657         ret=$?
1658         if [[ $expected_to_fail -eq "1" ]]; then
1659                 test "$ret" -ne "0"
1660                 ret=$?
1661                 if [ $withtap -eq "1" ]; then
1662                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1663                 fi
1664         else
1665                 if [ $withtap -eq "1" ]; then
1666                         ok $ret "Stop lttng tracing for session $sess_name"
1667                 fi
1668         fi
1669 }
1670
1671 function stop_lttng_tracing_ok ()
1672 {
1673         stop_lttng_tracing_opt 1 0 "$@"
1674 }
1675
1676 function stop_lttng_tracing_fail ()
1677 {
1678         stop_lttng_tracing_opt 1 1 "$@"
1679 }
1680
1681 function stop_lttng_tracing_notap ()
1682 {
1683         stop_lttng_tracing_opt 0 0 "$@"
1684 }
1685
1686 function destroy_lttng_session ()
1687 {
1688         local withtap=$1
1689         local expected_to_fail=$2
1690         local sess_name=$3
1691
1692         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1693                 destroy $sess_name
1694         ret=$?
1695         if [[ $expected_to_fail -eq "1" ]]; then
1696                 test "$ret" -ne "0"
1697                 ret=$?
1698                 if [ $withtap -eq "1" ]; then
1699                         ok $ret "Destroy session $sess_name failed as expected"
1700                 fi
1701         else
1702                 if [ $withtap -eq "1" ]; then
1703                         ok $ret "Destroy session $sess_name"
1704                 fi
1705         fi
1706 }
1707
1708 function destroy_lttng_session_ok ()
1709 {
1710         destroy_lttng_session 1 0 "$@"
1711
1712 }
1713
1714 function destroy_lttng_session_fail ()
1715 {
1716         destroy_lttng_session 1 1 "$@"
1717 }
1718
1719 function destroy_lttng_session_notap ()
1720 {
1721         destroy_lttng_session 0 0 "$@"
1722 }
1723
1724 function destroy_lttng_sessions ()
1725 {
1726         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1727                 destroy --all
1728         ok $? "Destroy all lttng sessions"
1729 }
1730
1731 function lttng_snapshot_add_output ()
1732 {
1733         local expected_to_fail=$1
1734         local sess_name=$2
1735         local trace_path=$3
1736         local opts=$4
1737
1738         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1739                 snapshot add-output -s $sess_name $trace_path $opts
1740         ret=$?
1741         if [[ $expected_to_fail -eq 1 ]]; then
1742                 test "$ret" -ne "0"
1743                 ok $? "Added snapshot output $trace_path failed as expected"
1744         else
1745                 ok $ret "Added snapshot output $trace_path"
1746         fi
1747 }
1748
1749 function lttng_snapshot_add_output_ok ()
1750 {
1751         lttng_snapshot_add_output 0 "$@"
1752 }
1753
1754 function lttng_snapshot_add_output_fail ()
1755 {
1756         lttng_snapshot_add_output 1 "$@"
1757 }
1758
1759 function lttng_snapshot_del_output ()
1760 {
1761         local expected_to_fail=$1
1762         local sess_name=$2
1763         local id=$3
1764
1765         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1766                 snapshot del-output -s $sess_name $id
1767         ret=$?
1768         if [[ $expected_to_fail -eq "1" ]]; then
1769                 test "$ret" -ne "0"
1770                 ok $? "Deleted snapshot output id $id failed as expected"
1771         else
1772                 ok $ret "Deleted snapshot output id $id"
1773         fi
1774 }
1775
1776 function lttng_snapshot_del_output_ok ()
1777 {
1778         lttng_snapshot_del_output 0 "$@"
1779 }
1780
1781 function lttng_snapshot_del_output_fail ()
1782 {
1783         lttng_snapshot_del_output 1 "$@"
1784 }
1785
1786 function lttng_snapshot_record ()
1787 {
1788         local sess_name=$1
1789         local trace_path=$2
1790
1791         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1792                 snapshot record -s "$sess_name" "$trace_path"
1793         ok $? "Snapshot recorded"
1794 }
1795
1796 function lttng_snapshot_list ()
1797 {
1798         local sess_name=$1
1799         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1800                 snapshot list-output -s $sess_name
1801         ok $? "Snapshot list"
1802 }
1803
1804 function lttng_save()
1805 {
1806         local sess_name=$1
1807         local opts=$2
1808
1809         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1810                 save $sess_name $opts
1811         ok $? "Session saved"
1812 }
1813
1814 function lttng_load()
1815 {
1816         local expected_to_fail=$1
1817         local opts=$2
1818
1819         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1820                 load $opts
1821         ret=$?
1822         if [[ $expected_to_fail -eq "1" ]]; then
1823                 test $ret -ne "0"
1824                 ok $? "Load command failed as expected with opts: $opts"
1825         else
1826                 ok $ret "Load command with opts: $opts"
1827         fi
1828 }
1829
1830 function lttng_load_ok()
1831 {
1832         lttng_load 0 "$@"
1833 }
1834
1835 function lttng_load_fail()
1836 {
1837         lttng_load 1 "$@"
1838 }
1839
1840 function lttng_track()
1841 {
1842         local expected_to_fail="$1"
1843         shift 1
1844         local opts="$@"
1845         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1846         ret=$?
1847         if [[ $expected_to_fail -eq "1" ]]; then
1848                 test $ret -ne "0"
1849                 ok $? "Track command failed as expected with opts: $opts"
1850         else
1851                 ok $ret "Track command with opts: $opts"
1852         fi
1853 }
1854
1855 function lttng_track_ok()
1856 {
1857         lttng_track 0 "$@"
1858 }
1859
1860 function lttng_track_fail()
1861 {
1862         lttng_track 1 "$@"
1863 }
1864
1865 function lttng_untrack()
1866 {
1867         local expected_to_fail="$1"
1868         shift 1
1869         local opts="$@"
1870         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1871         ret=$?
1872         if [[ $expected_to_fail -eq "1" ]]; then
1873                 test $ret -ne "0"
1874                 ok $? "Untrack command failed as expected with opts: $opts"
1875         else
1876                 ok $ret "Untrack command with opts: $opts"
1877         fi
1878 }
1879
1880 function lttng_untrack_ok()
1881 {
1882         lttng_untrack 0 "$@"
1883 }
1884
1885 function lttng_untrack_fail()
1886 {
1887         lttng_untrack 1 "$@"
1888 }
1889
1890 function lttng_track_pid_ok()
1891 {
1892         PID=$1
1893         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1894         ok $? "Lttng track pid on the kernel domain"
1895 }
1896
1897 function lttng_untrack_kernel_all_ok()
1898 {
1899         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1900         ok $? "Lttng untrack all pid on the kernel domain"
1901 }
1902
1903 function lttng_track_ust_ok()
1904 {
1905         lttng_track_ok -u "$@"
1906 }
1907
1908 function lttng_track_ust_fail()
1909 {
1910         lttng_track_fail -u "$@"
1911 }
1912
1913 function lttng_track_kernel_ok()
1914 {
1915         lttng_track_ok -k "$@"
1916 }
1917
1918 function lttng_track_kernel_fail()
1919 {
1920         lttng_track_fail -k "$@"
1921 }
1922
1923 function lttng_untrack_ust_ok()
1924 {
1925         lttng_untrack_ok -u "$@"
1926 }
1927
1928 function lttng_untrack_ust_fail()
1929 {
1930         lttng_untrack_fail -u "$@"
1931 }
1932
1933 function lttng_untrack_kernel_ok()
1934 {
1935         lttng_untrack_ok -k "$@"
1936 }
1937
1938 function lttng_untrack_kernel_fail()
1939 {
1940         lttng_untrack_fail -k "$@"
1941 }
1942
1943 function lttng_add_context_list()
1944 {
1945         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1946                 add-context --list
1947         ret=$?
1948         ok $ret "Context listing"
1949 }
1950
1951 function add_context_lttng()
1952 {
1953         local expected_to_fail="$1"
1954         local domain="$2"
1955         local session_name="$3"
1956         local channel_name="$4"
1957         local type="$5"
1958
1959         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1960                 add-context -s $session_name -c $channel_name -t $type $domain
1961         ret=$?
1962         if [[ $expected_to_fail -eq "1" ]]; then
1963                 test $ret -ne "0"
1964                 ok $? "Add context command failed as expected for type: $type"
1965         else
1966                 ok $ret "Add context command for type: $type"
1967         fi
1968 }
1969
1970 function add_context_ust_ok()
1971 {
1972         add_context_lttng 0 -u "$@"
1973 }
1974
1975 function add_context_ust_fail()
1976 {
1977         add_context_lttng 1 -u "$@"
1978 }
1979
1980 function add_context_kernel_ok()
1981 {
1982         add_context_lttng 0 -k "$@"
1983 }
1984
1985 function add_context_kernel_fail()
1986 {
1987         add_context_lttng 1 -k "$@"
1988 }
1989
1990 function wait_live_trace_ready ()
1991 {
1992         local url=$1
1993         local zero_client_match=0
1994
1995         diag "Waiting for live trace at url: $url"
1996         while [ $zero_client_match -eq 0 ]; do
1997                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1998                 sleep 0.5
1999         done
2000         pass "Waiting for live trace at url: $url"
2001 }
2002
2003 function wait_live_viewer_connect ()
2004 {
2005         local url=$1
2006         local one_client_match=0
2007
2008         diag "Waiting for live viewers on url: $url"
2009         while [ $one_client_match -eq 0 ]; do
2010                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
2011                 sleep 0.5
2012         done
2013         pass "Waiting for live viewers on url: $url"
2014 }
2015
2016 function bail_out_if_no_babeltrace()
2017 {
2018         which "$BABELTRACE_BIN" >/dev/null
2019         if [ $? -ne 0 ]; then
2020                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
2021         fi
2022 }
2023
2024 # Check that the trace metadata contains '$expected' event ids matching '$event_name'.
2025 function validate_metadata_event()
2026 {
2027         local event_name=$1
2028         local expected=$2
2029         local trace_path=$3
2030
2031         local metadata_file
2032         local metadata_path
2033         local count
2034
2035         metadata_file=$(find "$trace_path" -name "metadata")
2036         metadata_path=$(dirname "$metadata_file")
2037
2038         bail_out_if_no_babeltrace
2039
2040         count=$($BABELTRACE_BIN --output-format=ctf-metadata "$metadata_path" | grep -c "$event_name")
2041
2042         test "$count" -eq "$expected"
2043         ok $? "Found $count / $expected metadata event id matching '$event_name'"
2044 }
2045
2046 # Check that the trace contains '$expected' events matching '$event_name', other
2047 # events not matching '$event_name' can be present.
2048 function trace_matches()
2049 {
2050         local event_name=$1
2051         local expected=$2
2052         local trace_path=$3
2053
2054         local count
2055         local total
2056
2057         bail_out_if_no_babeltrace
2058
2059         count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2060         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2061
2062         test "$count" -eq "$expected"
2063
2064         ok $? "Found $count / $expected events matching '$event_name' out of $total events"
2065 }
2066
2067 # Check that the trace contains '$expected' events matching '$event_name' and no
2068 # other events.
2069 function trace_match_only()
2070 {
2071         local event_name=$1
2072         local expected=$2
2073         local trace_path=$3
2074
2075         local count
2076         local total
2077
2078         bail_out_if_no_babeltrace
2079
2080         count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2081         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2082
2083         test "$expected" -eq "$count" && test "$total" -eq "$expected"
2084
2085         ok $? "Found $count / $expected events matching '$event_name' amongst $total events"
2086 }
2087
2088 # Check that the trace contains at least 1 event matching each name in the
2089 # comma separated list '$event_names'.
2090 function validate_trace()
2091 {
2092         local event_names=$1
2093         local trace_path=$2
2094
2095         local count
2096
2097         bail_out_if_no_babeltrace
2098
2099         OLDIFS=$IFS
2100         IFS=","
2101         for event_name in $event_names; do
2102                 # trace_path is unquoted since callers make use of globbing
2103                 count=$($BABELTRACE_BIN $trace_path | grep -c "$event_name")
2104                 test "$count" -gt 0
2105                 ok $? "Found $count events matching '$event_name'"
2106         done
2107         IFS=$OLDIFS
2108 }
2109
2110 # Check that the trace contains at least 1 event matching each name in the
2111 # comma separated list '$event_names' and a total of '$expected' events.
2112 function validate_trace_count()
2113 {
2114         local event_names=$1
2115         local trace_path=$2
2116         local expected=$3
2117
2118         local count
2119         local total=0
2120
2121         bail_out_if_no_babeltrace
2122
2123         OLDIFS=$IFS
2124         IFS=","
2125         for event_name in $event_names; do
2126                 count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2127                 test "$count" -gt 0
2128                 ok $? "Found '$count' events matching '$event_name'"
2129                 total=$(( total + count ))
2130         done
2131         IFS=$OLDIFS
2132         test $total -eq "$expected"
2133         ok $? "Found $total events, expected $expected events"
2134 }
2135
2136 # Check that the trace contains at least '$expected_min' event matching each
2137 # name in the comma separated list '$event_names' and a total at least
2138 # '$expected_min' and less than '$expected_max' events.
2139 function validate_trace_count_range_incl_min_excl_max()
2140 {
2141         local event_names=$1
2142         local trace_path=$2
2143         local expected_min=$3
2144         local expected_max=$4
2145
2146         local count
2147         local total=0
2148
2149         bail_out_if_no_babeltrace
2150
2151         OLDIFS=$IFS
2152         IFS=","
2153         for event_name in $event_names; do
2154                 count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2155                 test "$count" -ge "$expected_min"
2156                 ok $? "Found $count events matching '$event_name', expected at least $expected_min"
2157                 total=$(( total + count ))
2158         done
2159         IFS=$OLDIFS
2160         test $total -ge "$expected_min" && test $total -lt "$expected_max"
2161         ok $? "Found a total of $total events, expected at least $expected_min and less than $expected_max"
2162 }
2163
2164 function trace_first_line()
2165 {
2166         local trace_path=$1
2167
2168         $BABELTRACE_BIN "$trace_path" | head -n 1
2169 }
2170
2171 # Check that the trace contains at least 1 event matching the grep extended
2172 # regexp '$event_exp'.
2173 function validate_trace_exp()
2174 {
2175         local event_exp=$1
2176         local trace_path=$2
2177
2178         local count
2179
2180         bail_out_if_no_babeltrace
2181
2182         # event_exp is unquoted since it contains multiple grep arguments
2183         count=$($BABELTRACE_BIN "$trace_path" | grep -c --extended-regexp $event_exp)
2184         test "$count" -gt 0
2185         ok $? "Found $count events matching expression '$event_exp'"
2186 }
2187
2188 # Check that the trace contains at least 1 event matching the grep extended
2189 # regexp '$event_exp' and zero event not matching it.
2190 function validate_trace_only_exp()
2191 {
2192         local event_exp=$1
2193         local trace_path=$2
2194
2195         local count
2196         local total
2197
2198         bail_out_if_no_babeltrace
2199
2200         # event_exp is unquoted since it contains multiple grep arguments
2201         count=$($BABELTRACE_BIN "$trace_path" | grep -c --extended-regexp $event_exp)
2202         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2203
2204         test  "$count" -gt 0 && test "$total" -eq "$count"
2205         ok $? "Found $count events matching expression '$event_exp' amongst $total events"
2206 }
2207
2208 # Check that the trace is valid and contains 0 event.
2209 function validate_trace_empty()
2210 {
2211         local trace_path=$1
2212
2213         local ret
2214         local count
2215
2216         bail_out_if_no_babeltrace
2217
2218         events=$($BABELTRACE_BIN "$trace_path")
2219         ret=$?
2220         if [ $ret -ne 0 ]; then
2221                 fail "Failed to parse trace"
2222                 return $ret
2223         fi
2224
2225         count=$(echo -n "$events" | wc -l)
2226         test "$count" -eq 0
2227         ok $? "Validate trace is empty, found $count events"
2228 }
2229
2230 function validate_directory_empty ()
2231 {
2232         local trace_path="$1"
2233
2234         local files
2235         local ret
2236         local nb_files
2237
2238         # Do not double quote `$trace_path` below as we want wildcards to be
2239         # expanded.
2240         files="$(ls -A $trace_path)"
2241         ret=$?
2242         if [ $ret -ne 0 ]; then
2243                 fail "Failed to list content of directory \"$trace_path\""
2244                 return $ret
2245         fi
2246
2247         nb_files="$(echo -n "$files" | wc -l)"
2248         test "$nb_files" -eq 0
2249         ok $? "Directory \"$trace_path\" is empty"
2250 }
2251
2252 function validate_trace_session_ust_empty()
2253 {
2254         validate_directory_empty "$1"/ust
2255 }
2256
2257 function validate_trace_session_kernel_empty()
2258 {
2259         validate_trace_empty "$1"/kernel
2260 }
2261
2262 function regenerate_metadata ()
2263 {
2264         local expected_to_fail=$1
2265         local sess_name=$2
2266
2267         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2268                 regenerate metadata -s $sess_name
2269         ret=$?
2270         if [[ $expected_to_fail -eq "1" ]]; then
2271                 test "$ret" -ne "0"
2272                 ok $? "Expected fail on regenerate metadata $sess_name"
2273         else
2274                 ok $ret "Metadata regenerate $sess_name"
2275         fi
2276 }
2277
2278 function regenerate_metadata_ok ()
2279 {
2280         regenerate_metadata 0 "$@"
2281 }
2282
2283 function regenerate_metadata_fail ()
2284 {
2285         regenerate_metadata 1 "$@"
2286 }
2287
2288 function regenerate_statedump ()
2289 {
2290         local expected_to_fail=$1
2291         local sess_name=$2
2292
2293         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2294                 regenerate statedump -s $sess_name
2295         ret=$?
2296         if [[ $expected_to_fail -eq "1" ]]; then
2297                 test "$ret" -ne "0"
2298                 ok $? "Expected fail on regenerate statedump $sess_name"
2299         else
2300                 ok $ret "Statedump regenerate $sess_name"
2301         fi
2302 }
2303
2304 function regenerate_statedump_ok ()
2305 {
2306         regenerate_statedump 0 "$@"
2307 }
2308
2309 function regenerate_statedump_fail ()
2310 {
2311         regenerate_statedump 1 "$@"
2312 }
2313
2314 function rotate_session ()
2315 {
2316         local expected_to_fail=$1
2317         local sess_name=$2
2318
2319         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2320                 rotate $sess_name
2321         ret=$?
2322         if [[ $expected_to_fail -eq "1" ]]; then
2323                 test "$ret" -ne "0"
2324                 ok $? "Expected fail on rotate session $sess_name"
2325         else
2326                 ok $ret "Rotate session $sess_name"
2327         fi
2328 }
2329
2330 function rotate_session_ok ()
2331 {
2332         rotate_session 0 "$@"
2333 }
2334
2335 function rotate_session_fail ()
2336 {
2337         rotate_session 1 "$@"
2338 }
2339
2340 function destructive_tests_enabled ()
2341 {
2342         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2343                 return 0
2344         else
2345                 return 1
2346         fi
2347 }
2348
2349 function lttng_enable_rotation_timer ()
2350 {
2351         local expected_to_fail=$1
2352         local sess_name=$2
2353         local period=$3
2354
2355         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2356                 enable-rotation -s $sess_name --timer $period
2357         ret=$?
2358         if [[ $expected_to_fail -eq "1" ]]; then
2359                 test "$ret" -ne "0"
2360                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2361         else
2362                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2363         fi
2364 }
2365
2366 function lttng_enable_rotation_timer_ok ()
2367 {
2368         lttng_enable_rotation_timer 0 $@
2369 }
2370
2371 function lttng_enable_rotation_timer_fail ()
2372 {
2373         lttng_enable_rotation_timer 1 $@
2374 }
2375
2376 function lttng_enable_rotation_size ()
2377 {
2378         local expected_to_fail=$1
2379         local sess_name=$2
2380         local size=$3
2381
2382         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2383                 enable-rotation -s $sess_name --size $size
2384         ret=$?
2385         if [[ $expected_to_fail -eq "1" ]]; then
2386                 test "$ret" -ne "0"
2387                 ok $? "Expected to fail to set a periodic rotation of session $sess_name" "every " $size " bytes"
2388         else
2389                 ok $ret "Set a scheduled rotation of session $sess_name" "every " $size " bytes"
2390         fi
2391 }
2392
2393 function lttng_enable_rotation_size_ok ()
2394 {
2395         lttng_enable_rotation_size 0 $@
2396 }
2397
2398 function lttng_enable_rotation_size_fail ()
2399 {
2400         lttng_enable_rotation_size 1 $@
2401 }
2402
2403 function lttng_clear_session ()
2404 {
2405         local expected_to_fail=$1
2406         local sess_name=$2
2407
2408         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2409                 clear $sess_name
2410         ret=$?
2411         if [[ $expected_to_fail -eq "1" ]]; then
2412                 test "$ret" -ne "0"
2413                 ok $? "Expected fail on clear session $sess_name"
2414         else
2415                 ok $ret "Clear session $sess_name"
2416         fi
2417 }
2418
2419 function lttng_clear_session_ok ()
2420 {
2421         lttng_clear_session 0 $@
2422 }
2423
2424 function lttng_clear_session_fail ()
2425 {
2426         lttng_clear_session 1 $@
2427 }
2428
2429 function lttng_clear_all ()
2430 {
2431         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2432                 clear --all
2433         ok $? "Clear all lttng sessions"
2434 }
2435
2436 function lttng_add_trigger()
2437 {
2438         local expected_to_fail="$1"
2439         local trigger_name="$2"
2440         shift 2
2441         local args=("$@")
2442
2443         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2444         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2445         ret=$?
2446         if [[ $expected_to_fail -eq "1" ]]; then
2447                 test "$ret" -ne "0"
2448                 ok $? "Add trigger $trigger_name failed as expected"
2449         else
2450                 ok $ret "Add trigger $trigger_name"
2451         fi
2452 }
2453
2454 function lttng_remove_trigger()
2455 {
2456         local expected_to_fail="$1"
2457         local trigger_name="$2"
2458         shift 2
2459
2460         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2461         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2462         ret=$?
2463         if [[ $expected_to_fail -eq "1" ]]; then
2464                 test "$ret" -ne "0"
2465                 ok $? "Remove trigger $trigger_name failed as expected"
2466         else
2467                 ok $ret "Remove trigger $trigger_name"
2468         fi
2469 }
2470
2471 function lttng_add_trigger_ok()
2472 {
2473         lttng_add_trigger 0 "$@"
2474 }
2475
2476 function lttng_add_trigger_fail()
2477 {
2478         lttng_add_trigger 1 "$@"
2479 }
2480
2481 function lttng_remove_trigger_ok()
2482 {
2483         lttng_remove_trigger 0 "$@"
2484 }
2485
2486 function list_triggers_matches_ok ()
2487 {
2488         local tmp_stdout=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2489         local tmp_stderr=$(mktemp -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2490
2491         local test_name="$1"
2492         local expected_stdout_file="$2"
2493
2494         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2495
2496         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2497         ok $? "${test_name}: exit code is 0"
2498
2499         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2500         ok $? "${test_name}: expected stdout"
2501
2502         diff -u /dev/null "${tmp_stderr}"
2503         ok $? "${test_name}: expected stderr"
2504
2505         rm -f "${tmp_stdout}"
2506         rm -f "${tmp_stderr}"
2507 }
2508
2509 function list_triggers_matches_mi_ok ()
2510 {
2511         local tmp_stdout
2512         local tmp_stdout_raw
2513         local tmp_stderr
2514
2515         local test_name="$1"
2516         local expected_stdout_file="$2"
2517
2518         tmp_stdout_raw=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2519         tmp_stdout=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2520         tmp_stderr=$(mktemp -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2521
2522         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2523
2524         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2525         ok $? "${test_name}: exit code is 0"
2526
2527         # Pretty-fy xml before further test.
2528         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2529
2530         $MI_VALIDATE "${tmp_stdout}"
2531         ok $? "list-trigger mi is valid"
2532
2533         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2534         ok $? "${test_name}: expected stdout"
2535
2536         diff -u /dev/null "${tmp_stderr}"
2537         ok $? "${test_name}: expected stderr"
2538
2539         rm -f "${tmp_stdout}"
2540         rm -f "${tmp_stdout_raw}"
2541         rm -f "${tmp_stderr}"
2542 }
2543
2544 function validate_path_pattern ()
2545 {
2546         local message=$1
2547         local pattern=$2
2548         # Base path is only used in error case and is used to list the content
2549         # of the base path.
2550         local base_path=$3
2551
2552
2553         [ -f $pattern ]
2554         ret=$?
2555         ok $ret "$message"
2556
2557         if [ "$ret" -ne "0" ]; then
2558                 diag "Path pattern expected: $pattern"
2559                 # List the tracepath for more info. We use find as a recursive
2560                 # directory lister.
2561                 diag "The base path content:"
2562                 find "$base_path" -print
2563         fi
2564 }
2565
2566 function validate_trace_path_ust_uid ()
2567 {
2568         local trace_path=$1
2569         local session_name=$2
2570         local uid=$UID
2571         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2572
2573         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2574 }
2575
2576 function validate_trace_path_ust_uid_network ()
2577 {
2578         local trace_path=$1
2579         local session_name=$2
2580         local base_path=$3
2581         local uid=$UID
2582         local hostname=$HOSTNAME
2583         local pattern
2584         local ret
2585
2586         # If the session was given a network base path (e.g
2587         # 127.0.0.1/my/custom/path on creation, there is no session name
2588         # component to the path on the relayd side. Caller can simply not pass a
2589         # session name for this scenario.
2590         if [ -n "$session_name" ]; then
2591                 session_name="$session_name-$date_time_pattern"
2592                 if [ -n "$base_path" ]; then
2593                         fail "Session name and base path are mutually exclusive"
2594                         return
2595                 fi
2596         fi
2597
2598         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2599
2600         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2601 }
2602
2603 function validate_trace_path_ust_uid_snapshot_network ()
2604 {
2605         local trace_path=$1
2606         local session_name=$2
2607         local snapshot_name=$3
2608         local snapshot_number=$4
2609         local base_path=$5
2610         local hostname=$HOSTNAME
2611         local uid=$UID
2612         local pattern
2613         local ret
2614
2615         # If the session/output was given a network base path (e.g
2616         # 127.0.0.1/my/custom/path on creation, there is no session name
2617         # component to the path on the relayd side. Caller can simply not pass a
2618         # session name for this scenario.
2619         if [ -n "$session_name" ]; then
2620                 session_name="$session_name-$date_time_pattern"
2621                 if [ -n "$base_path" ]; then
2622                         fail "Session name and base path are mutually exclusive"
2623                         return
2624                 fi
2625         fi
2626
2627         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2628
2629         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2630 }
2631
2632 function validate_trace_path_ust_uid_snapshot ()
2633 {
2634         local trace_path=$1
2635         local session_name=$2
2636         local snapshot_name=$3
2637         local snapshot_number=$4
2638         local base_path=$5
2639         local uid=$UID
2640         local pattern
2641         local ret
2642
2643         # If the session/output was given a network base path (e.g
2644         # 127.0.0.1/my/custom/path) on creation, there is no session name
2645         # component to the path on the relayd side. Caller can simply not pass a
2646         # session name for this scenario.
2647         if [ -n "$session_name" ]; then
2648                 session_name="$session_name-$date_time_pattern"
2649                 if [ -n "$base_path" ]; then
2650                         fail "Session name and base path are mutually exclusive"
2651                         return
2652                 fi
2653         fi
2654
2655         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2656
2657         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2658 }
2659
2660 function validate_trace_path_ust_pid ()
2661 {
2662         local trace_path=$1
2663         local session_name=$2
2664         local app_string=$3
2665         local pid=$4
2666         local pattern
2667         local ret
2668
2669         # If the session was given a trace path on creation, there is no session
2670         # name component to the path. Caller can simply not pass a session name
2671         # for this scenario.
2672         if [ -n "$session_name" ]; then
2673                 session_name="$session_name-$date_time_pattern"
2674         fi
2675
2676         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2677
2678         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2679 }
2680
2681 function validate_trace_path_kernel ()
2682 {
2683         local trace_path=$1
2684         local session_name=$2
2685         local pattern
2686
2687         # If the session was given a trace path on creation, there is no session
2688         # name component to the path. Caller can simply not pass a session name
2689         # for this scenario.
2690         if [ -n "$session_name" ]; then
2691                 session_name="$session_name-$date_time_pattern"
2692         fi
2693
2694         pattern="$trace_path/$session_name/kernel/metadata"
2695
2696         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2697 }
2698
2699 function validate_trace_path_kernel_network ()
2700 {
2701         local trace_path=$1
2702         local session_name=$2
2703         local hostname=$HOSTNAME
2704         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2705
2706         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2707 }
2708
2709 function validate_trace_path_kernel_snapshot ()
2710 {
2711         local trace_path=$1
2712         local session_name=$2
2713         local snapshot_name=$3
2714         local snapshot_number=$4
2715         local base_path=$5
2716         local pattern
2717         local ret
2718
2719         # If the session/output was given a network base path (e.g
2720         # 127.0.0.1/my/custom/path on creation, there is no session name
2721         # component to the path on the relayd side. Caller can simply not pass a
2722         # session name for this scenario.
2723         if [ -n "$session_name" ]; then
2724                 session_name="$session_name-$date_time_pattern"
2725                 if [ -n "$base_path" ]; then
2726                         fail "Session name and base path are mutually exclusive"
2727                         return
2728                 fi
2729         fi
2730
2731         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2732
2733         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2734 }
2735
2736 function validate_trace_path_kernel_snapshot_network ()
2737 {
2738         local trace_path=$1
2739         local session_name=$2
2740         local snapshot_name=$3
2741         local snapshot_number=$4
2742         local base_path=$5
2743         local hostname=$HOSTNAME
2744         local pattern
2745         local ret
2746
2747         # If the session/output was given a network base path (e.g
2748         # 127.0.0.1/my/custom/path on creation, there is no session name
2749         # component to the path on the relayd side. Caller can simply not pass a
2750         # session name for this scenario.
2751         if [ -n "$session_name" ]; then
2752                 session_name="$session_name-$date_time_pattern"
2753                 if [ -n "$base_path" ]; then
2754                         fail "Session name and base path are mutually exclusive"
2755                         return
2756                 fi
2757         fi
2758
2759         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2760
2761         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2762 }