tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$(cut -d '' -f 1 2>/dev/null < /proc/"$pid"/cmdline); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 208         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 209         echo
 210 }
 211
 212 # Helpers for get_possible_cpus.
 213 function get_possible_cpus_count_from_sysfs_possible_mask()
 214 {
 215         local max_possible_cpu_id
 216
 217         # The Awk script extracts the highest CPU id from the possible CPU
 218         # mask. Assuming a numerical order, a field separator '-' and a record
 219         # separator ','. The last value parsed is the highest id.
 220         if [ -f /sys/devices/system/cpu/possible ]; then
 221                 max_possible_cpu_id=$(awk -F '-' 'BEGIN { RS = ","} { last = $NF } END { printf("%d\n", last) }' \
 222                                       /sys/devices/system/cpu/possible)
 223                 echo "$((max_possible_cpu_id+1))"
 224         else
 225                 echo "0"
 226         fi
 227 }
 228
 229 # This is a fallback if the possible CPU mask is not available. This will not
 230 # take into account unplugged CPUs.
 231 function get_max_cpus_count_from_sysfs_cpu_directories()
 232 {
 233         local max_possible_cpu_id=0
 234         local current_cpu_id
 235
 236         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 237                 current_cpu_id="${i#/sys/devices/system/cpu/cpu}"
 238                 if [ "$current_cpu_id" -gt "$max_possible_cpu_id" ]; then
 239                         max_possible_cpu_id="$current_cpu_id"
 240                 fi
 241         done
 242
 243         echo "$((max_possible_cpu_id+1))"
 244 }
 245
 246 # Return the number of possible CPUs.
 247 function get_possible_cpus_count()
 248 {
 249         local possible_cpus_count
 250         possible_cpus_count=$(get_possible_cpus_count_from_sysfs_possible_mask)
 251
 252         if [ "$possible_cpus_count" -eq "0" ]; then
 253                 local configured_cpus_count
 254                 configured_cpus_count=$(getconf _NPROCESSORS_CONF)
 255                 possible_cpus_count=$(get_max_cpus_count_from_sysfs_cpu_directories)
 256                 possible_cpus_count=$((configured_cpus_count > possible_cpus_count \
 257                                                              ? configured_cpus_count \
 258                                                              : possible_cpus_count))
 259         fi
 260
 261         echo "$possible_cpus_count"
 262 }
 263
 264 # Return the list of exposed CPU.
 265 #
 266 # NOTE! Use it like so:
 267 #
 268 # IFS=" " read -r -a VARIABLE <<< "$(get_exposed_cpus_list)"
 269 function get_exposed_cpus_list()
 270 {
 271         local list=()
 272
 273         for i in /sys/devices/system/cpu/cpu[0-9]*; do
 274                 list+=("${i#/sys/devices/system/cpu/cpu}")
 275         done
 276
 277         echo "${list[@]}"
 278 }
 279
 280 # Return the number of _configured_ CPUs.
 281 function conf_proc_count()
 282 {
 283         getconf _NPROCESSORS_CONF
 284         if [ $? -ne 0 ]; then
 285                 diag "Failed to get the number of configured CPUs"
 286         fi
 287         echo
 288 }
 289
 290 # Check if base lttng-modules are present.
 291 # Bail out on failure
 292 function validate_lttng_modules_present ()
 293 {
 294         # Check for loadable modules.
 295         modprobe -n lttng-tracer 2>/dev/null
 296         if [ $? -eq 0 ]; then
 297                 return 0
 298         fi
 299
 300         # Check for builtin modules.
 301         ls /proc/lttng > /dev/null 2>&1
 302         if [ $? -eq 0 ]; then
 303                 return 0
 304         fi
 305
 306         LTTNG_BAIL_OUT "LTTng modules not detected."
 307 }
 308
 309 # Run the lttng binary.
 310 #
 311 # The first two arguments are stdout and stderr redirect paths, respectively.
 312 # The rest of the arguments are forwarded to the lttng binary
 313 function _run_lttng_cmd
 314 {
 315         local stdout_dest="$1"
 316         local stderr_dest="$2"
 317         shift 2
 318
 319         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 320         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 321 }
 322
 323 function enable_kernel_lttng_event
 324 {
 325         local withtap="$1"
 326         local expected_to_fail="$2"
 327         local sess_name="$3"
 328         local event_name="$4"
 329         local channel_name="$5"
 330
 331         if [ -z "$event_name" ]; then
 332                 # Enable all event if no event name specified
 333                 event_name="-a"
 334         fi
 335
 336         if [ -z "$channel_name" ]; then
 337                 # default channel if none specified
 338                 chan=""
 339         else
 340                 chan="-c $channel_name"
 341         fi
 342
 343         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 344         ret=$?
 345         if [[ $expected_to_fail -eq "1" ]]; then
 346                 test $ret -ne "0"
 347                 ret=$?
 348                 if [ $withtap -eq "1" ]; then
 349                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 350                 fi
 351         else
 352                 if [ $withtap -eq "1" ]; then
 353                         ok $ret "Enable kernel event $event_name for session $sess_name"
 354                 fi
 355         fi
 356 }
 357
 358 function enable_kernel_lttng_event_ok ()
 359 {
 360         enable_kernel_lttng_event 1 0 "$@"
 361 }
 362
 363 function enable_kernel_lttng_event_fail ()
 364 {
 365         enable_kernel_lttng_event 1 1 "$@"
 366 }
 367
 368 function enable_kernel_lttng_event_notap ()
 369 {
 370         enable_kernel_lttng_event 0 0 "$@"
 371 }
 372
 373 # Old interface
 374 function lttng_enable_kernel_event
 375 {
 376         enable_kernel_lttng_event_ok "$@"
 377 }
 378
 379 function lttng_enable_kernel_syscall()
 380 {
 381         local expected_to_fail=$1
 382         local sess_name=$2
 383         local syscall_name=$3
 384         local channel_name=$4
 385
 386         if [ -z $syscall_name ]; then
 387                 # Enable all event if no syscall name specified
 388                 syscall_name="-a"
 389         fi
 390
 391         if [ -z $channel_name ]; then
 392                 # default channel if none specified
 393                 chan=""
 394         else
 395                 chan="-c $channel_name"
 396         fi
 397
 398         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 399         ret=$?
 400         if [[ $expected_to_fail -eq "1" ]]; then
 401                 test $ret -ne "0"
 402                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 403         else
 404                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 405         fi
 406 }
 407
 408 function lttng_enable_kernel_syscall_ok()
 409 {
 410         lttng_enable_kernel_syscall 0 "$@"
 411 }
 412
 413 function lttng_enable_kernel_syscall_fail()
 414 {
 415         lttng_enable_kernel_syscall 1 "$@"
 416 }
 417
 418 function lttng_disable_kernel_syscall()
 419 {
 420         local expected_to_fail=$1
 421         local sess_name=$2
 422         local syscall_name=$3
 423         local channel_name=$4
 424
 425         if [ -z $syscall_name ]; then
 426                 # Enable all event if no syscall name specified
 427                 syscall_name="-a"
 428         fi
 429
 430         if [ -z $channel_name ]; then
 431                 # default channel if none specified
 432                 chan=""
 433         else
 434                 chan="-c $channel_name"
 435         fi
 436
 437         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 438
 439         ret=$?
 440         if [[ $expected_to_fail -eq "1" ]]; then
 441                 test $ret -ne "0"
 442                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 443         else
 444                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 445         fi
 446 }
 447
 448 function lttng_disable_kernel_syscall_ok()
 449 {
 450         lttng_disable_kernel_syscall 0 "$@"
 451 }
 452
 453 function lttng_disable_kernel_syscall_fail()
 454 {
 455         lttng_disable_kernel_syscall 1 "$@"
 456 }
 457
 458 function lttng_enable_kernel_function_event ()
 459 {
 460         local expected_to_fail="$1"
 461         local sess_name="$2"
 462         local target="$3"
 463         local event_name="$4"
 464
 465         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 466         ret=$?
 467         if [[ $expected_to_fail -eq "1" ]]; then
 468                 test $ret -ne "0"
 469                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 470         else
 471                 ok $ret "Enable kernel function event for session $sess_name"
 472         fi
 473 }
 474
 475 function lttng_enable_kernel_function_event_ok ()
 476 {
 477         lttng_enable_kernel_function_event 0 "$@"
 478 }
 479
 480 function lttng_enable_kernel_userspace_probe_event ()
 481 {
 482         local expected_to_fail="$1"
 483         local sess_name="$2"
 484         local target="$3"
 485         local event_name="$4"
 486
 487         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 488         ret=$?
 489         if [[ $expected_to_fail -eq "1" ]]; then
 490                 test $ret -ne "0"
 491                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 492         else
 493                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 494         fi
 495 }
 496
 497 function lttng_enable_kernel_userspace_probe_event_fail ()
 498 {
 499         lttng_enable_kernel_userspace_probe_event 1 "$@"
 500 }
 501
 502 function lttng_enable_kernel_userspace_probe_event_ok ()
 503 {
 504         lttng_enable_kernel_userspace_probe_event 0 "$@"
 505 }
 506
 507 function disable_kernel_lttng_userspace_probe_event_ok ()
 508 {
 509         local sess_name="$1"
 510         local event_name="$2"
 511
 512         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 513         ok $? "Disable kernel event $target for session $sess_name"
 514 }
 515 function lttng_enable_kernel_channel()
 516 {
 517         local withtap=$1
 518         local expected_to_fail=$2
 519         local sess_name=$3
 520         local channel_name=$4
 521         local opts="${@:5}"
 522
 523         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 524         ret=$?
 525         if [[ $expected_to_fail -eq "1" ]]; then
 526                 test "$ret" -ne "0"
 527                 ret=$?
 528                 if [ $withtap -eq "1" ]; then
 529                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 530                 fi
 531         else
 532                 if [ $withtap -eq "1" ]; then
 533                         ok $ret "Enable channel $channel_name for session $sess_name"
 534                 fi
 535         fi
 536 }
 537
 538 function lttng_enable_kernel_channel_ok()
 539 {
 540         lttng_enable_kernel_channel 1 0 "$@"
 541 }
 542
 543 function lttng_enable_kernel_channel_fail()
 544 {
 545         lttng_enable_kernel_channel 1 1 "$@"
 546 }
 547
 548 function lttng_enable_kernel_channel_notap()
 549 {
 550         lttng_enable_kernel_channel 0 0 "$@"
 551 }
 552
 553 function enable_kernel_lttng_channel_ok()
 554 {
 555         lttng_enable_kernel_channel 1 0 "$@"
 556 }
 557
 558 function lttng_disable_kernel_channel()
 559 {
 560         local expected_to_fail=$1
 561         local sess_name=$2
 562         local channel_name=$3
 563
 564         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 565         ret=$?
 566         if [[ $expected_to_fail -eq "1" ]]; then
 567                 test "$ret" -ne "0"
 568                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 569         else
 570                 ok $ret "Disable channel $channel_name for session $sess_name"
 571         fi
 572 }
 573
 574 function lttng_disable_kernel_channel_ok()
 575 {
 576         lttng_disable_kernel_channel 0 "$@"
 577 }
 578
 579 function lttng_disable_kernel_channel_fail()
 580 {
 581         lttng_disable_kernel_channel 1 "$@"
 582 }
 583
 584 function start_lttng_relayd_opt()
 585 {
 586         local withtap=$1
 587         local process_mode=$2
 588         local opt=$3
 589
 590         DIR=$(readlink -f "$TESTDIR")
 591
 592         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 593                 # shellcheck disable=SC2086
 594                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 595                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 596                 if [ $? -eq 1 ]; then
 597                         if [ $withtap -eq "1" ]; then
 598                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 599                         fi
 600                         return 1
 601                 else
 602                         if [ $withtap -eq "1" ]; then
 603                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 604                         fi
 605                 fi
 606         else
 607                 pass "Start lttng-relayd (opt: $opt)"
 608         fi
 609 }
 610
 611 function start_lttng_relayd()
 612 {
 613         start_lttng_relayd_opt 1 "-b" "$@"
 614 }
 615
 616 function start_lttng_relayd_notap()
 617 {
 618         start_lttng_relayd_opt 0 "-b" "$@"
 619 }
 620
 621 function stop_lttng_relayd_opt()
 622 {
 623         local withtap=$1
 624         local is_cleanup=$2
 625         local signal=$3
 626         local timeout_s=$4
 627         local dtimeleft_s=
 628         local retval=0
 629         local pids
 630
 631         if [ -z "$signal" ]; then
 632                 signal="SIGTERM"
 633         fi
 634
 635
 636         # Multiply time by 2 to simplify integer arithmetic
 637         if [ -n "$timeout_s" ]; then
 638                 dtimeleft_s=$((timeout_s * 2))
 639         fi
 640
 641
 642         pids=$(lttng_pgrep "$RELAYD_MATCH")
 643         if [ -z "$pids" ]; then
 644                 if [ "$is_cleanup" -eq 1 ]; then
 645                         :
 646                 elif [ "$withtap" -eq "1" ]; then
 647                         fail "No relay daemon to kill"
 648                 else
 649                         LTTNG_BAIL_OUT "No relay daemon to kill"
 650                 fi
 651                 return 0
 652         fi
 653
 654         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 655
 656         # shellcheck disable=SC2086
 657         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 658                 retval=1
 659                 if [ "$withtap" -eq "1" ]; then
 660                         fail "Kill relay daemon"
 661                 fi
 662         else
 663                 out=1
 664                 while [ -n "$out" ]; do
 665                         out=$(lttng_pgrep "$RELAYD_MATCH")
 666                         if [ -n "$dtimeleft_s" ]; then
 667                                 if [ $dtimeleft_s -lt 0 ]; then
 668                                         out=
 669                                         retval=1
 670                                 fi
 671                                 dtimeleft_s=$((dtimeleft_s - 1))
 672                         fi
 673                         sleep 0.5
 674                 done
 675                 if [ "$withtap" -eq "1" ]; then
 676                         if [ "$retval" -eq "0" ]; then
 677                                 pass "Wait after kill relay daemon"
 678                         else
 679                                 fail "Wait after kill relay daemon"
 680                         fi
 681                 fi
 682         fi
 683         return $retval
 684 }
 685
 686 function stop_lttng_relayd()
 687 {
 688         stop_lttng_relayd_opt 1 0 "$@"
 689 }
 690
 691 function stop_lttng_relayd_notap()
 692 {
 693         stop_lttng_relayd_opt 0 0 "$@"
 694 }
 695
 696 function stop_lttng_relayd_cleanup()
 697 {
 698         stop_lttng_relayd_opt 0 1 "$@"
 699 }
 700
 701 #First arg: show tap output
 702 #Second argument: load path for automatic loading
 703 function start_lttng_sessiond_opt()
 704 {
 705         local withtap=$1
 706         local load_path=$2
 707
 708         # The rest of the arguments will be passed directly to lttng-sessiond.
 709         shift 2
 710
 711         local env_vars=""
 712         local consumerd=""
 713
 714         local long_bit_value=
 715         long_bit_value=$(getconf LONG_BIT)
 716
 717         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 718                 # Env variable requested no session daemon
 719                 return
 720         fi
 721
 722         DIR=$(readlink -f "$TESTDIR")
 723
 724         # Get long_bit value for 32/64 consumerd
 725         case "$long_bit_value" in
 726                 32)
 727                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 728                         ;;
 729                 64)
 730                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 731                         ;;
 732                 *)
 733                         return
 734                         ;;
 735         esac
 736
 737         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 738         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 739                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 740         fi
 741         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 742
 743         if ! validate_kernel_version; then
 744             fail "Start session daemon"
 745             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 746         fi
 747
 748         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 749         export LTTNG_SESSION_CONFIG_XSD_PATH
 750
 751         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 752                 # Have a load path ?
 753                 if [ -n "$load_path" ]; then
 754                         # shellcheck disable=SC2086
 755                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 756                 else
 757                         # shellcheck disable=SC2086
 758                         env $env_vars --background "$consumerd" "$@"
 759                 fi
 760                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 761                 status=$?
 762                 if [ "$withtap" -eq "1" ]; then
 763                         ok $status "Start session daemon"
 764                 fi
 765         fi
 766 }
 767
 768 function start_lttng_sessiond()
 769 {
 770         start_lttng_sessiond_opt 1 "$@"
 771 }
 772
 773 function start_lttng_sessiond_notap()
 774 {
 775         start_lttng_sessiond_opt 0 "$@"
 776 }
 777
 778 function stop_lttng_sessiond_opt()
 779 {
 780         local withtap=$1
 781         local is_cleanup=$2
 782         local signal=$3
 783         local timeout_s=$4
 784         local dtimeleft_s=
 785         local retval=0
 786         local runas_pids
 787         local pids
 788
 789         if [ -z "$signal" ]; then
 790                 signal=SIGTERM
 791         fi
 792
 793         # Multiply time by 2 to simplify integer arithmetic
 794         if [ -n "$timeout_s" ]; then
 795                 dtimeleft_s=$((timeout_s * 2))
 796         fi
 797
 798         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 799                 # Env variable requested no session daemon
 800                 return 0
 801         fi
 802
 803         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 804         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 805
 806         if [ -n "$runas_pids" ]; then
 807                 pids="$pids $runas_pids"
 808         fi
 809
 810         if [ -z "$pids" ]; then
 811                 if [ "$is_cleanup" -eq 1 ]; then
 812                         :
 813                 elif [ "$withtap" -eq "1" ]; then
 814                         fail "No session daemon to kill"
 815                 else
 816                         LTTNG_BAIL_OUT "No session daemon to kill"
 817                 fi
 818                 return 0
 819         fi
 820
 821         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 822
 823         # shellcheck disable=SC2086
 824         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 825                 retval=1
 826                 if [ "$withtap" -eq "1" ]; then
 827                         fail "Kill sessions daemon"
 828                 fi
 829         else
 830                 out=1
 831                 while [ -n "$out" ]; do
 832                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 833                         if [ -n "$dtimeleft_s" ]; then
 834                                 if [ $dtimeleft_s -lt 0 ]; then
 835                                         out=
 836                                         retval=1
 837                                 fi
 838                                 dtimeleft_s=$((dtimeleft_s - 1))
 839                         fi
 840                         sleep 0.5
 841                 done
 842                 out=1
 843                 while [ -n "$out" ]; do
 844                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 845                         if [ -n "$dtimeleft_s" ]; then
 846                                 if [ $dtimeleft_s -lt 0 ]; then
 847                                         out=
 848                                         retval=1
 849                                 fi
 850                                 dtimeleft_s=$((dtimeleft_s - 1))
 851                         fi
 852                         sleep 0.5
 853                 done
 854
 855                 if [ "$withtap" -eq "1" ]; then
 856                         if [ "$retval" -eq "0" ]; then
 857                                 pass "Wait after kill session daemon"
 858                         else
 859                                 fail "Wait after kill session daemon"
 860                         fi
 861                 fi
 862         fi
 863         if [ "$signal" = "SIGKILL" ]; then
 864                 if [ "$(id -u)" -eq "0" ]; then
 865                         local modules=
 866                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 867
 868                         if [ -n "$modules" ]; then
 869                                 diag "Unloading all LTTng modules"
 870                                 modprobe --remove "$modules"
 871                         fi
 872                 fi
 873         fi
 874
 875         return $retval
 876 }
 877
 878 function stop_lttng_sessiond()
 879 {
 880         stop_lttng_sessiond_opt 1 0 "$@"
 881 }
 882
 883 function stop_lttng_sessiond_notap()
 884 {
 885         stop_lttng_sessiond_opt 0 0 "$@"
 886 }
 887
 888 function stop_lttng_sessiond_cleanup()
 889 {
 890         stop_lttng_sessiond_opt 0 1 "$@"
 891 }
 892
 893 function sigstop_lttng_sessiond_opt()
 894 {
 895         local withtap=$1
 896         local signal=SIGSTOP
 897         local pids
 898
 899         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 900                 # Env variable requested no session daemon
 901                 return
 902         fi
 903
 904         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 905
 906         if [ "$withtap" -eq "1" ]; then
 907                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 908         fi
 909
 910         # shellcheck disable=SC2086
 911         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 912                 if [ "$withtap" -eq "1" ]; then
 913                         fail "Sending SIGSTOP to session daemon"
 914                 fi
 915         else
 916                 out=1
 917                 while [ $out -ne 0 ]; do
 918                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 919
 920                         # Wait until state becomes stopped for session
 921                         # daemon(s).
 922                         out=0
 923                         for sessiond_pid in $pids; do
 924                                 state="$(ps -p "$sessiond_pid" -o state= )"
 925                                 if [[ -n "$state" && "$state" != "T" ]]; then
 926                                         out=1
 927                                 fi
 928                         done
 929                         sleep 0.5
 930                 done
 931                 if [ "$withtap" -eq "1" ]; then
 932                         pass "Sending SIGSTOP to session daemon"
 933                 fi
 934         fi
 935 }
 936
 937 function sigstop_lttng_sessiond()
 938 {
 939         sigstop_lttng_sessiond_opt 1 "$@"
 940 }
 941
 942 function sigstop_lttng_sessiond_notap()
 943 {
 944         sigstop_lttng_sessiond_opt 0 "$@"
 945 }
 946
 947 function stop_lttng_consumerd_opt()
 948 {
 949         local withtap=$1
 950         local is_cleanup=$2
 951         local signal=$3
 952         local timeout_s=$4
 953         local dtimeleft_s=
 954         local retval=0
 955         local pids
 956
 957         if [ -z "$signal" ]; then
 958                 signal=SIGTERM
 959         fi
 960
 961         # Multiply time by 2 to simplify integer arithmetic
 962         if [ -n "$timeout_s" ]; then
 963                 dtimeleft_s=$((timeout_s * 2))
 964         fi
 965
 966         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 967
 968         if [ -z "$pids" ]; then
 969                 if [ "$is_cleanup" -eq 1 ]; then
 970                         :
 971                 elif [ "$withtap" -eq "1" ]; then
 972                         fail "No consumerd daemon to kill"
 973                 else
 974                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
 975                 fi
 976                 return 0
 977         fi
 978
 979         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 980
 981         # shellcheck disable=SC2086
 982         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 983                 retval=1
 984                 if [ "$withtap" -eq "1" ]; then
 985                         fail "Kill consumer daemon"
 986                 fi
 987         else
 988                 out=1
 989                 while [ $out -ne 0 ]; do
 990                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 991
 992                         # If consumerds are still present check their status.
 993                         # A zombie status qualifies the consumerd as *killed*
 994                         out=0
 995                         for consumer_pid in $pids; do
 996                                 state="$(ps -p "$consumer_pid" -o state= )"
 997                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 998                                         out=1
 999                                 fi
1000                         done
1001                         if [ -n "$dtimeleft_s" ]; then
1002                                 if [ $dtimeleft_s -lt 0 ]; then
1003                                         out=0
1004                                         retval=1
1005                                 fi
1006                                 dtimeleft_s=$((dtimeleft_s - 1))
1007                         fi
1008                         sleep 0.5
1009                 done
1010                 if [ "$withtap" -eq "1" ]; then
1011                         if [ "$retval" -eq "0" ]; then
1012                                 pass "Wait after kill consumer daemon"
1013                         else
1014                                 fail "Wait after kill consumer daemon"
1015                         fi
1016                 fi
1017         fi
1018
1019         return $retval
1020 }
1021
1022 function stop_lttng_consumerd()
1023 {
1024         stop_lttng_consumerd_opt 1 0 "$@"
1025 }
1026
1027 function stop_lttng_consumerd_notap()
1028 {
1029         stop_lttng_consumerd_opt 0 0 "$@"
1030 }
1031
1032 function stop_lttng_consumerd_cleanup()
1033 {
1034         stop_lttng_consumerd_opt 0 1 "$@"
1035 }
1036
1037 function sigstop_lttng_consumerd_opt()
1038 {
1039         local withtap=$1
1040         local signal=SIGSTOP
1041         local pids
1042
1043         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1044
1045         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1046
1047         # shellcheck disable=SC2086
1048         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1049         retval=$?
1050
1051         if [ $retval -eq 1 ]; then
1052                 if [ "$withtap" -eq "1" ]; then
1053                         fail "Sending SIGSTOP to consumer daemon"
1054                 fi
1055                 return 1
1056         else
1057                 out=1
1058                 while [ $out -ne 0 ]; do
1059                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1060
1061                         # Wait until state becomes stopped for all
1062                         # consumers.
1063                         out=0
1064                         for consumer_pid in $pids; do
1065                                 state="$(ps -p "$consumer_pid" -o state= )"
1066                                 if [[ -n "$state" && "$state" != "T" ]]; then
1067                                         out=1
1068                                 fi
1069                         done
1070                         sleep 0.5
1071                 done
1072                 if [ "$withtap" -eq "1" ]; then
1073                         pass "Sending SIGSTOP to consumer daemon"
1074                 fi
1075         fi
1076         return $retval
1077 }
1078
1079 function sigstop_lttng_consumerd()
1080 {
1081         sigstop_lttng_consumerd_opt 1 "$@"
1082 }
1083
1084 function sigstop_lttng_consumerd_notap()
1085 {
1086         sigstop_lttng_consumerd_opt 0 "$@"
1087 }
1088
1089 function list_lttng_with_opts ()
1090 {
1091         local ret
1092         local withtap=$1
1093         shift
1094         local opts=$1
1095         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1096         ret=$?
1097         if [ $withtap -eq "1" ]; then
1098                 ok $ret "Lttng-tool list command with option $opts"
1099         fi
1100 }
1101
1102 function list_lttng_ok ()
1103 {
1104         list_lttng_with_opts 1 "$@"
1105 }
1106
1107 function list_lttng_notap ()
1108 {
1109         list_lttng_with_opts 0 "$@"
1110 }
1111
1112 function create_lttng_session_no_output ()
1113 {
1114         local sess_name=$1
1115         local opts="${@:2}"
1116
1117         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1118         ok $? "Create session $sess_name in no-output mode"
1119 }
1120
1121 function create_lttng_session_uri () {
1122         local sess_name=$1
1123         local uri=$2
1124         local opts="${@:3}"
1125
1126         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1127         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1128 }
1129
1130 function create_lttng_session ()
1131 {
1132         local withtap=$1
1133         local expected_to_fail=$2
1134         local sess_name=$3
1135         local trace_path=$4
1136         local opt=$5
1137
1138         if [ -z "$trace_path" ]; then
1139                 # Use lttng-sessiond default output.
1140                 trace_path=""
1141         else
1142                 trace_path="-o $trace_path"
1143         fi
1144
1145         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create "$sess_name" $trace_path $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1146         ret=$?
1147         if [ $expected_to_fail -eq "1" ]; then
1148                 test "$ret" -ne "0"
1149                 ret=$?
1150                 if [ $withtap -eq "1" ]; then
1151                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1152                 fi
1153         else
1154                 if [ $withtap -eq "1" ]; then
1155                         ok $ret "Create session $sess_name in $trace_path"
1156                 fi
1157         fi
1158         return $ret
1159 }
1160
1161 function create_lttng_session_ok ()
1162 {
1163         create_lttng_session 1 0 "$@"
1164 }
1165
1166 function create_lttng_session_fail ()
1167 {
1168         create_lttng_session 1 1 "$@"
1169 }
1170
1171 function create_lttng_session_notap ()
1172 {
1173         create_lttng_session 0 0 "$@"
1174 }
1175
1176
1177 function enable_ust_lttng_channel ()
1178 {
1179         local withtap=$1
1180         local expected_to_fail=$2
1181         local sess_name=$3
1182         local channel_name=$4
1183         local opts="${@:5}"
1184
1185         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1186         ret=$?
1187         if [[ $expected_to_fail -eq "1" ]]; then
1188                 test "$ret" -ne "0"
1189                 ret=$?
1190                 if [ $withtap -eq "1" ]; then
1191                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1192                 fi
1193         else
1194                 if [ $withtap -eq "1" ]; then
1195                         ok $ret "Enable channel $channel_name for session $sess_name"
1196                 fi
1197         fi
1198         return $ret
1199 }
1200
1201 function enable_ust_lttng_channel_ok ()
1202 {
1203         enable_ust_lttng_channel 1 0 "$@"
1204 }
1205
1206 function enable_ust_lttng_channel_fail ()
1207 {
1208         enable_ust_lttng_channel 1 1 "$@"
1209 }
1210
1211 function enable_ust_lttng_channel_notap ()
1212 {
1213         enable_ust_lttng_channel 0 0 "$@"
1214 }
1215
1216 function disable_ust_lttng_channel()
1217 {
1218         local sess_name=$1
1219         local channel_name=$2
1220
1221         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1222         ok $? "Disable channel $channel_name for session $sess_name"
1223 }
1224
1225 function enable_lttng_mmap_overwrite_kernel_channel()
1226 {
1227         local sess_name=$1
1228         local channel_name=$2
1229
1230         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1231         ok $? "Enable channel $channel_name for session $sess_name"
1232 }
1233
1234 function enable_lttng_mmap_discard_small_kernel_channel()
1235 {
1236         local sess_name=$1
1237         local channel_name=$2
1238
1239         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1240         ok $? "Enable small discard channel $channel_name for session $sess_name"
1241 }
1242
1243 function enable_lttng_mmap_overwrite_small_kernel_channel()
1244 {
1245         local sess_name=$1
1246         local channel_name=$2
1247
1248         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1249         ok $? "Enable small discard channel $channel_name for session $sess_name"
1250 }
1251
1252 function enable_lttng_mmap_overwrite_ust_channel()
1253 {
1254         local sess_name=$1
1255         local channel_name=$2
1256
1257         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1258         ok $? "Enable channel $channel_name for session $sess_name"
1259 }
1260
1261 function enable_ust_lttng_event ()
1262 {
1263         local withtap=$1
1264         local expected_to_fail=$2
1265         local sess_name=$3
1266         local event_name="$4"
1267         local channel_name=$5
1268
1269         if [ -z $channel_name ]; then
1270                 # default channel if none specified
1271                 chan=""
1272         else
1273                 chan="-c $channel_name"
1274         fi
1275
1276         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1277         ret=$?
1278         if [[ $expected_to_fail -eq "1" ]]; then
1279                 test $ret -ne "0"
1280                 ret=$?
1281                 if [[ $withtap -eq "1" ]]; then
1282                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1283                 fi
1284         else
1285                 if [[ $withtap -eq "1" ]]; then
1286                         ok $ret "Enable ust event $event_name for session $sess_name"
1287                 fi
1288         fi
1289         return $ret
1290 }
1291
1292 function enable_ust_lttng_event_ok ()
1293 {
1294         enable_ust_lttng_event 1 0 "$@"
1295 }
1296
1297 function enable_ust_lttng_event_fail ()
1298 {
1299         enable_ust_lttng_event 1 1 "$@"
1300 }
1301
1302 function enable_ust_lttng_event_notap ()
1303 {
1304         enable_ust_lttng_event 0 0 "$@"
1305 }
1306
1307 function enable_jul_lttng_event()
1308 {
1309         sess_name=$1
1310         event_name="$2"
1311         channel_name=$3
1312
1313         if [ -z $channel_name ]; then
1314                 # default channel if none specified
1315                 chan=""
1316         else
1317                 chan="-c $channel_name"
1318         fi
1319
1320         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1321         ok $? "Enable JUL event $event_name for session $sess_name"
1322 }
1323
1324 function enable_jul_lttng_event_loglevel()
1325 {
1326         local sess_name=$1
1327         local event_name="$2"
1328         local loglevel=$3
1329         local channel_name=$4
1330
1331         if [ -z $channel_name ]; then
1332                 # default channel if none specified
1333                 chan=""
1334         else
1335                 chan="-c $channel_name"
1336         fi
1337
1338         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1339         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1340 }
1341
1342 function enable_log4j_lttng_event()
1343 {
1344         local sess_name=$1
1345         local event_name=$2
1346         local channel_name=$3
1347
1348         local chan_opt=()
1349
1350         # default channel if none specified
1351         if [ -n "$channel_name" ]; then
1352                 chan_opt=("-c" "$channel_name")
1353         fi
1354
1355         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1356                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1357         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1358 }
1359
1360 function enable_log4j_lttng_event_filter()
1361 {
1362         local sess_name=$1
1363         local event_name=$2
1364         local filter=$3
1365
1366         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1367                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1368         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1369 }
1370
1371 function enable_log4j_lttng_event_filter_loglevel_only()
1372 {
1373         local sess_name=$1
1374         local event_name=$2
1375         local filter=$3
1376         local loglevel=$4
1377
1378         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1379                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1380         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1381 }
1382
1383 function enable_log4j_lttng_event_loglevel()
1384 {
1385         local sess_name=$1
1386         local event_name=$2
1387         local loglevel=$3
1388         local channel_name=$4
1389
1390
1391         # default channel if none specified
1392         if [ -n "$channel_name" ]; then
1393                 chan_opt=("-c" "$channel_name")
1394         fi
1395
1396         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1397                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1398         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1399 }
1400
1401 function enable_log4j_lttng_event_loglevel_only()
1402 {
1403         local sess_name=$1
1404         local event_name=$2
1405         local loglevel=$3
1406         local channel_name=$4
1407
1408         local chan_opt=()
1409
1410         # default channel if none specified
1411         if [ -n "$channel_name" ]; then
1412                 chan_opt=("-c" "$channel_name")
1413         fi
1414
1415         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1416                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1417         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1418 }
1419
1420 function enable_python_lttng_event()
1421 {
1422         sess_name=$1
1423         event_name="$2"
1424         channel_name=$3
1425
1426         if [ -z $channel_name ]; then
1427                 # default channel if none specified
1428                 chan=""
1429         else
1430                 chan="-c $channel_name"
1431         fi
1432
1433         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1434         ok $? "Enable Python event $event_name for session $sess_name"
1435 }
1436
1437 function enable_python_lttng_event_loglevel()
1438 {
1439         local sess_name=$1
1440         local event_name="$2"
1441         local loglevel=$3
1442         local channel_name=$4
1443
1444         if [ -z $channel_name ]; then
1445                 # default channel if none specified
1446                 chan=""
1447         else
1448                 chan="-c $channel_name"
1449         fi
1450
1451         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1452         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1453 }
1454
1455 function enable_ust_lttng_event_filter()
1456 {
1457         local sess_name="$1"
1458         local event_name="$2"
1459         local filter="$3"
1460         local channel_name=$4
1461
1462         if [ -z $channel_name ]; then
1463                 # default channel if none specified
1464                 chan=""
1465         else
1466                 chan="-c $channel_name"
1467         fi
1468
1469         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event $chan "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1470         ok $? "Enable event $event_name with filtering for session $sess_name"
1471 }
1472
1473 function enable_ust_lttng_event_loglevel()
1474 {
1475         local sess_name="$1"
1476         local event_name="$2"
1477         local loglevel="$3"
1478
1479         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1480         ok $? "Enable event $event_name with loglevel $loglevel"
1481 }
1482
1483 function enable_ust_lttng_event_loglevel_only()
1484 {
1485         local sess_name="$1"
1486         local event_name="$2"
1487         local loglevel="$3"
1488
1489         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1490         ok $? "Enable event $event_name with loglevel-only $loglevel"
1491 }
1492
1493 function disable_ust_lttng_event ()
1494 {
1495         local sess_name="$1"
1496         local event_name="$2"
1497         local channel_name="$3"
1498
1499         if [ -z $channel_name ]; then
1500                 # default channel if none specified
1501                 chan=""
1502         else
1503                 chan="-c $channel_name"
1504         fi
1505
1506         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1507         ok $? "Disable event $event_name for session $sess_name"
1508 }
1509
1510 function disable_jul_lttng_event ()
1511 {
1512         local sess_name="$1"
1513         local event_name="$2"
1514
1515         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1516         ok $? "Disable JUL event $event_name for session $sess_name"
1517 }
1518
1519 function disable_log4j_lttng_event ()
1520 {
1521         local sess_name="$1"
1522         local event_name="$2"
1523
1524         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1525                 disable-event "$event_name" -s "$sess_name" --log4j
1526         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1527 }
1528
1529 function disable_python_lttng_event ()
1530 {
1531         local sess_name="$1"
1532         local event_name="$2"
1533
1534         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1535         ok $? "Disable Python event $event_name for session $sess_name"
1536 }
1537
1538 function start_lttng_tracing_opt ()
1539 {
1540         local withtap=$1
1541         local expected_to_fail=$2
1542         local sess_name=$3
1543
1544         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1545         ret=$?
1546         if [[ $expected_to_fail -eq "1" ]]; then
1547                 test "$ret" -ne "0"
1548                 ret=$?
1549                 if [ $withtap -eq "1" ]; then
1550                         ok $? "Start tracing for session $sess_name failed as expected"
1551                 fi
1552         else
1553                 if [ $withtap -eq "1" ]; then
1554                         ok $ret "Start tracing for session $sess_name"
1555                 fi
1556         fi
1557 }
1558
1559 function start_lttng_tracing_ok ()
1560 {
1561         start_lttng_tracing_opt 1 0 "$@"
1562 }
1563
1564 function start_lttng_tracing_fail ()
1565 {
1566         start_lttng_tracing_opt 1 1 "$@"
1567 }
1568
1569 function start_lttng_tracing_notap ()
1570 {
1571         start_lttng_tracing_opt 0 1 "$@"
1572 }
1573
1574 function stop_lttng_tracing_opt ()
1575 {
1576         local withtap=$1
1577         local expected_to_fail=$2
1578         local sess_name=$3
1579
1580         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1581         ret=$?
1582         if [[ $expected_to_fail -eq "1" ]]; then
1583                 test "$ret" -ne "0"
1584                 ret=$?
1585                 if [ $withtap -eq "1" ]; then
1586                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1587                 fi
1588         else
1589                 if [ $withtap -eq "1" ]; then
1590                         ok $ret "Stop lttng tracing for session $sess_name"
1591                 fi
1592         fi
1593 }
1594
1595 function stop_lttng_tracing_ok ()
1596 {
1597         stop_lttng_tracing_opt 1 0 "$@"
1598 }
1599
1600 function stop_lttng_tracing_fail ()
1601 {
1602         stop_lttng_tracing_opt 1 1 "$@"
1603 }
1604
1605 function stop_lttng_tracing_notap ()
1606 {
1607         stop_lttng_tracing_opt 0 0 "$@"
1608 }
1609
1610 function destroy_lttng_session ()
1611 {
1612         local withtap=$1
1613         local expected_to_fail=$2
1614         local sess_name=$3
1615
1616         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1617         ret=$?
1618         if [[ $expected_to_fail -eq "1" ]]; then
1619                 test "$ret" -ne "0"
1620                 ret=$?
1621                 if [ $withtap -eq "1" ]; then
1622                         ok $ret "Destroy session $sess_name failed as expected"
1623                 fi
1624         else
1625                 if [ $withtap -eq "1" ]; then
1626                         ok $ret "Destroy session $sess_name"
1627                 fi
1628         fi
1629 }
1630
1631 function destroy_lttng_session_ok ()
1632 {
1633         destroy_lttng_session 1 0 "$@"
1634
1635 }
1636
1637 function destroy_lttng_session_fail ()
1638 {
1639         destroy_lttng_session 1 1 "$@"
1640 }
1641
1642 function destroy_lttng_session_notap ()
1643 {
1644         destroy_lttng_session 0 0 "$@"
1645 }
1646
1647 function destroy_lttng_sessions ()
1648 {
1649         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1650         ok $? "Destroy all lttng sessions"
1651 }
1652
1653 function lttng_snapshot_add_output ()
1654 {
1655         local expected_to_fail=$1
1656         local sess_name=$2
1657         local trace_path=$3
1658         local opts=$4
1659
1660         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1661         ret=$?
1662         if [[ $expected_to_fail -eq 1 ]]; then
1663                 test "$ret" -ne "0"
1664                 ok $? "Added snapshot output $trace_path failed as expected"
1665         else
1666                 ok $ret "Added snapshot output $trace_path"
1667         fi
1668 }
1669
1670 function lttng_snapshot_add_output_ok ()
1671 {
1672         lttng_snapshot_add_output 0 "$@"
1673 }
1674
1675 function lttng_snapshot_add_output_fail ()
1676 {
1677         lttng_snapshot_add_output 1 "$@"
1678 }
1679
1680 function lttng_snapshot_del_output ()
1681 {
1682         local expected_to_fail=$1
1683         local sess_name=$2
1684         local id=$3
1685
1686         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1687         ret=$?
1688         if [[ $expected_to_fail -eq "1" ]]; then
1689                 test "$ret" -ne "0"
1690                 ok $? "Deleted snapshot output id $id failed as expected"
1691         else
1692                 ok $ret "Deleted snapshot output id $id"
1693         fi
1694 }
1695
1696 function lttng_snapshot_del_output_ok ()
1697 {
1698         lttng_snapshot_del_output 0 "$@"
1699 }
1700
1701 function lttng_snapshot_del_output_fail ()
1702 {
1703         lttng_snapshot_del_output 1 "$@"
1704 }
1705
1706 function lttng_snapshot_record ()
1707 {
1708         local sess_name=$1
1709         local trace_path=$2
1710
1711         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s "$sess_name" "$trace_path" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1712         ok $? "Snapshot recorded"
1713 }
1714
1715 function lttng_snapshot_list ()
1716 {
1717         local sess_name=$1
1718         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1719         ok $? "Snapshot list"
1720 }
1721
1722 function lttng_save()
1723 {
1724         local sess_name=$1
1725         local opts=$2
1726
1727         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1728         ok $? "Session saved"
1729 }
1730
1731 function lttng_load()
1732 {
1733         local expected_to_fail=$1
1734         local opts=$2
1735
1736         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1737         ret=$?
1738         if [[ $expected_to_fail -eq "1" ]]; then
1739                 test $ret -ne "0"
1740                 ok $? "Load command failed as expected with opts: $opts"
1741         else
1742                 ok $ret "Load command with opts: $opts"
1743         fi
1744 }
1745
1746 function lttng_load_ok()
1747 {
1748         lttng_load 0 "$@"
1749 }
1750
1751 function lttng_load_fail()
1752 {
1753         lttng_load 1 "$@"
1754 }
1755
1756 function lttng_track()
1757 {
1758         local expected_to_fail="$1"
1759         shift 1
1760         local opts="$@"
1761         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1762         ret=$?
1763         if [[ $expected_to_fail -eq "1" ]]; then
1764                 test $ret -ne "0"
1765                 ok $? "Track command failed as expected with opts: $opts"
1766         else
1767                 ok $ret "Track command with opts: $opts"
1768         fi
1769 }
1770
1771 function lttng_track_ok()
1772 {
1773         lttng_track 0 "$@"
1774 }
1775
1776 function lttng_track_fail()
1777 {
1778         lttng_track 1 "$@"
1779 }
1780
1781 function lttng_untrack()
1782 {
1783         local expected_to_fail="$1"
1784         shift 1
1785         local opts="$@"
1786         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1787         ret=$?
1788         if [[ $expected_to_fail -eq "1" ]]; then
1789                 test $ret -ne "0"
1790                 ok $? "Untrack command failed as expected with opts: $opts"
1791         else
1792                 ok $ret "Untrack command with opts: $opts"
1793         fi
1794 }
1795
1796 function lttng_untrack_ok()
1797 {
1798         lttng_untrack 0 "$@"
1799 }
1800
1801 function lttng_untrack_fail()
1802 {
1803         lttng_untrack 1 "$@"
1804 }
1805
1806 function lttng_track_pid_ok()
1807 {
1808         PID=$1
1809         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1810         ok $? "Lttng track pid on the kernel domain"
1811 }
1812
1813 function lttng_untrack_kernel_all_ok()
1814 {
1815         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1816         ok $? "Lttng untrack all pid on the kernel domain"
1817 }
1818
1819 function lttng_track_ust_ok()
1820 {
1821         lttng_track_ok -u "$@"
1822 }
1823
1824 function lttng_track_ust_fail()
1825 {
1826         lttng_track_fail -u "$@"
1827 }
1828
1829 function lttng_track_kernel_ok()
1830 {
1831         lttng_track_ok -k "$@"
1832 }
1833
1834 function lttng_track_kernel_fail()
1835 {
1836         lttng_track_fail -k "$@"
1837 }
1838
1839 function lttng_untrack_ust_ok()
1840 {
1841         lttng_untrack_ok -u "$@"
1842 }
1843
1844 function lttng_untrack_ust_fail()
1845 {
1846         lttng_untrack_fail -u "$@"
1847 }
1848
1849 function lttng_untrack_kernel_ok()
1850 {
1851         lttng_untrack_ok -k "$@"
1852 }
1853
1854 function lttng_untrack_kernel_fail()
1855 {
1856         lttng_untrack_fail -k "$@"
1857 }
1858
1859 function lttng_add_context_list()
1860 {
1861         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1862         ret=$?
1863         ok $ret "Context listing"
1864 }
1865
1866 function add_context_lttng()
1867 {
1868         local expected_to_fail="$1"
1869         local domain="$2"
1870         local session_name="$3"
1871         local channel_name="$4"
1872         local type="$5"
1873
1874         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1875         ret=$?
1876         if [[ $expected_to_fail -eq "1" ]]; then
1877                 test $ret -ne "0"
1878                 ok $? "Add context command failed as expected for type: $type"
1879         else
1880                 ok $ret "Add context command for type: $type"
1881         fi
1882 }
1883
1884 function add_context_ust_ok()
1885 {
1886         add_context_lttng 0 -u "$@"
1887 }
1888
1889 function add_context_ust_fail()
1890 {
1891         add_context_lttng 1 -u "$@"
1892 }
1893
1894 function add_context_kernel_ok()
1895 {
1896         add_context_lttng 0 -k "$@"
1897 }
1898
1899 function add_context_kernel_fail()
1900 {
1901         add_context_lttng 1 -k "$@"
1902 }
1903
1904 function wait_live_trace_ready ()
1905 {
1906         local url=$1
1907         local zero_client_match=0
1908
1909         diag "Waiting for live trace at url: $url"
1910         while [ $zero_client_match -eq 0 ]; do
1911                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1912                 sleep 0.5
1913         done
1914         pass "Waiting for live trace at url: $url"
1915 }
1916
1917 function wait_live_viewer_connect ()
1918 {
1919         local url=$1
1920         local one_client_match=0
1921
1922         diag "Waiting for live viewers on url: $url"
1923         while [ $one_client_match -eq 0 ]; do
1924                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1925                 sleep 0.5
1926         done
1927         pass "Waiting for live viewers on url: $url"
1928 }
1929
1930 function bail_out_if_no_babeltrace()
1931 {
1932         which "$BABELTRACE_BIN" >/dev/null
1933         if [ $? -ne 0 ]; then
1934                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
1935         fi
1936 }
1937
1938 function validate_metadata_event ()
1939 {
1940         local event_name=$1
1941         local nr_event_id=$2
1942         local trace_path=$3
1943
1944         local metadata_file=$(find $trace_path -name "metadata")
1945         local metadata_path=$(dirname $metadata_file)
1946
1947         which $BABELTRACE_BIN >/dev/null
1948         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1949
1950         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1951
1952         if [ "$count" -ne "$nr_event_id" ]; then
1953                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1954                 diag "$count matching event id found in metadata"
1955         else
1956                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1957         fi
1958
1959 }
1960
1961 function trace_matches ()
1962 {
1963         local event_name=$1
1964         local nr_iter=$2
1965         local trace_path=$3
1966
1967         which $BABELTRACE_BIN >/dev/null
1968         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1969
1970         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1971
1972         if [ "$count" -ne "$nr_iter" ]; then
1973                 fail "Trace match"
1974                 diag "$count matching events found in trace"
1975         else
1976                 pass "Trace match"
1977         fi
1978 }
1979
1980 function trace_match_only()
1981 {
1982         local event_name=$1
1983         local nr_iter=$2
1984         local trace_path=$3
1985
1986         which $BABELTRACE_BIN >/dev/null
1987         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1988
1989         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1990         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1991
1992         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1993                 pass "Trace match with $total event $event_name"
1994         else
1995                 fail "Trace match"
1996                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1997         fi
1998 }
1999
2000 function validate_trace
2001 {
2002         local event_name=$1
2003         local trace_path=$2
2004
2005         which $BABELTRACE_BIN >/dev/null
2006         if [ $? -ne 0 ]; then
2007             skip 0 "Babeltrace binary not found. Skipping trace validation"
2008         fi
2009
2010         OLDIFS=$IFS
2011         IFS=","
2012         for i in $event_name; do
2013                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2014                 if [ "$traced" -ne 0 ]; then
2015                         pass "Validate trace for event $i, $traced events"
2016                 else
2017                         fail "Validate trace for event $i"
2018                         diag "Found $traced occurrences of $i"
2019                 fi
2020         done
2021         ret=$?
2022         IFS=$OLDIFS
2023         return $ret
2024 }
2025
2026 function validate_trace_count
2027 {
2028         local event_name=$1
2029         local trace_path=$2
2030         local expected_count=$3
2031
2032         which $BABELTRACE_BIN >/dev/null
2033         if [ $? -ne 0 ]; then
2034             skip 0 "Babeltrace binary not found. Skipping trace validation"
2035         fi
2036
2037         cnt=0
2038         OLDIFS=$IFS
2039         IFS=","
2040         for i in $event_name; do
2041                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2042                 if [ "$traced" -ne 0 ]; then
2043                         pass "Validate trace for event $i, $traced events"
2044                 else
2045                         fail "Validate trace for event $i"
2046                         diag "Found $traced occurrences of $i"
2047                 fi
2048                 cnt=$(($cnt + $traced))
2049         done
2050         IFS=$OLDIFS
2051         test $cnt -eq $expected_count
2052         ok $? "Read a total of $cnt events, expected $expected_count"
2053 }
2054
2055 function validate_trace_count_range_incl_min_excl_max
2056 {
2057         local event_name=$1
2058         local trace_path=$2
2059         local expected_min=$3
2060         local expected_max=$4
2061
2062         which $BABELTRACE_BIN >/dev/null
2063         if [ $? -ne 0 ]; then
2064             skip 0 "Babeltrace binary not found. Skipping trace validation"
2065         fi
2066
2067         cnt=0
2068         OLDIFS=$IFS
2069         IFS=","
2070         for i in $event_name; do
2071                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2072                 if [ "$traced" -ge $expected_min ]; then
2073                         pass "Validate trace for event $i, $traced events"
2074                 else
2075                         fail "Validate trace for event $i"
2076                         diag "Found $traced occurrences of $i"
2077                 fi
2078                 cnt=$(($cnt + $traced))
2079         done
2080         IFS=$OLDIFS
2081         test $cnt -lt $expected_max
2082         ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
2083 }
2084
2085 function trace_first_line
2086 {
2087         local trace_path=$1
2088
2089         which $BABELTRACE_BIN >/dev/null
2090         if [ $? -ne 0 ]; then
2091             skip 0 "Babeltrace binary not found. Skipping trace validation"
2092         fi
2093
2094         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
2095 }
2096
2097 function validate_trace_exp()
2098 {
2099         local event_exp=$1
2100         local trace_path=$2
2101
2102         which $BABELTRACE_BIN >/dev/null
2103         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
2104
2105         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
2106         if [ "$traced" -ne 0 ]; then
2107                 pass "Validate trace for expression '${event_exp}', $traced events"
2108         else
2109                 fail "Validate trace for expression '${event_exp}'"
2110                 diag "Found $traced occurrences of '${event_exp}'"
2111         fi
2112         ret=$?
2113         return $ret
2114 }
2115
2116 function validate_trace_only_exp()
2117 {
2118         local event_exp=$1
2119         local trace_path=$2
2120
2121         which $BABELTRACE_BIN >/dev/null
2122         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
2123
2124         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
2125         local total=$($BABELTRACE_BIN $trace_path | wc -l)
2126
2127         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
2128                 pass "Trace match with $total for expression '${event_exp}'"
2129         else
2130                 fail "Trace match"
2131                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
2132         fi
2133         ret=$?
2134         return $ret
2135 }
2136
2137 function validate_trace_empty()
2138 {
2139         local trace_path=$1
2140
2141         which $BABELTRACE_BIN >/dev/null
2142         if [ $? -ne 0 ]; then
2143             skip 0 "Babeltrace binary not found. Skipping trace validation"
2144         fi
2145
2146         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
2147         ret=$?
2148         if [ $ret -ne 0 ]; then
2149                 fail "Failed to parse trace"
2150                 return $ret
2151         fi
2152
2153         traced=$(echo -n "$events" | wc -l)
2154         if [ "$traced" -eq 0 ]; then
2155                 pass "Validate empty trace"
2156         else
2157                 fail "Validate empty trace"
2158                 diag "Found $traced events in trace"
2159         fi
2160         ret=$?
2161         return $ret
2162 }
2163
2164 function validate_directory_empty ()
2165 {
2166         local trace_path="$1"
2167
2168         # Do not double quote `$trace_path` below as we want wildcards to be
2169         # expanded.
2170         files="$(ls -A $trace_path)"
2171         ret=$?
2172         if [ $ret -ne 0 ]; then
2173                 fail "Failed to list content of directory \"$trace_path\""
2174                 return $ret
2175         fi
2176
2177         nb_files="$(echo -n "$files" | wc -l)"
2178         ok $nb_files "Directory \"$trace_path\" is empty"
2179 }
2180
2181 function validate_trace_session_ust_empty()
2182 {
2183         validate_directory_empty "$1"/ust
2184 }
2185
2186 function validate_trace_session_kernel_empty()
2187 {
2188         validate_trace_empty "$1"/kernel
2189 }
2190
2191 function regenerate_metadata ()
2192 {
2193         local expected_to_fail=$1
2194         local sess_name=$2
2195
2196         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2197         ret=$?
2198         if [[ $expected_to_fail -eq "1" ]]; then
2199                 test "$ret" -ne "0"
2200                 ok $? "Expected fail on regenerate metadata $sess_name"
2201         else
2202                 ok $ret "Metadata regenerate $sess_name"
2203         fi
2204 }
2205
2206 function regenerate_metadata_ok ()
2207 {
2208         regenerate_metadata 0 "$@"
2209 }
2210
2211 function regenerate_metadata_fail ()
2212 {
2213         regenerate_metadata 1 "$@"
2214 }
2215
2216 function regenerate_statedump ()
2217 {
2218         local expected_to_fail=$1
2219         local sess_name=$2
2220
2221         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2222         ret=$?
2223         if [[ $expected_to_fail -eq "1" ]]; then
2224                 test "$ret" -ne "0"
2225                 ok $? "Expected fail on regenerate statedump $sess_name"
2226         else
2227                 ok $ret "Statedump regenerate $sess_name"
2228         fi
2229 }
2230
2231 function regenerate_statedump_ok ()
2232 {
2233         regenerate_statedump 0 "$@"
2234 }
2235
2236 function regenerate_statedump_fail ()
2237 {
2238         regenerate_statedump 1 "$@"
2239 }
2240
2241 function rotate_session ()
2242 {
2243         local expected_to_fail=$1
2244         local sess_name=$2
2245
2246         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2247         ret=$?
2248         if [[ $expected_to_fail -eq "1" ]]; then
2249                 test "$ret" -ne "0"
2250                 ok $? "Expected fail on rotate session $sess_name"
2251         else
2252                 ok $ret "Rotate session $sess_name"
2253         fi
2254 }
2255
2256 function rotate_session_ok ()
2257 {
2258         rotate_session 0 "$@"
2259 }
2260
2261 function rotate_session_fail ()
2262 {
2263         rotate_session 1 "$@"
2264 }
2265
2266 function destructive_tests_enabled ()
2267 {
2268         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2269                 return 0
2270         else
2271                 return 1
2272         fi
2273 }
2274
2275 function lttng_enable_rotation_timer ()
2276 {
2277         local expected_to_fail=$1
2278         local sess_name=$2
2279         local period=$3
2280
2281         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2282         ret=$?
2283         if [[ $expected_to_fail -eq "1" ]]; then
2284                 test "$ret" -ne "0"
2285                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2286         else
2287                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2288         fi
2289 }
2290
2291 function lttng_enable_rotation_timer_ok ()
2292 {
2293         lttng_enable_rotation_timer 0 $@
2294 }
2295
2296 function lttng_enable_rotation_timer_fail ()
2297 {
2298         lttng_enable_rotation_timer 1 $@
2299 }
2300
2301 function lttng_enable_rotation_size ()
2302 {
2303         local expected_to_fail=$1
2304         local sess_name=$2
2305         local size=$3
2306
2307         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2308         ret=$?
2309         if [[ $expected_to_fail -eq "1" ]]; then
2310                 test "$ret" -ne "0"
2311                 ok $? "Expected fail on rotate session $sess_name"
2312         else
2313                 ok $ret "Rotate session $sess_name"
2314         fi
2315 }
2316
2317 function lttng_enable_rotation_size_ok ()
2318 {
2319         lttng_enable_rotation_size 0 $@
2320 }
2321
2322 function lttng_enable_rotation_size_fail ()
2323 {
2324         lttng_enable_rotation_size 1 $@
2325 }
2326
2327 function lttng_clear_session ()
2328 {
2329         local expected_to_fail=$1
2330         local sess_name=$2
2331
2332         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2333         ret=$?
2334         if [[ $expected_to_fail -eq "1" ]]; then
2335                 test "$ret" -ne "0"
2336                 ok $? "Expected fail on clear session $sess_name"
2337         else
2338                 ok $ret "Clear session $sess_name"
2339         fi
2340 }
2341
2342 function lttng_clear_session_ok ()
2343 {
2344         lttng_clear_session 0 $@
2345 }
2346
2347 function lttng_clear_session_fail ()
2348 {
2349         lttng_clear_session 1 $@
2350 }
2351
2352 function lttng_clear_all ()
2353 {
2354         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2355         ok $? "Clear all lttng sessions"
2356 }
2357
2358 function lttng_add_trigger()
2359 {
2360         local expected_to_fail="$1"
2361         local trigger_name="$2"
2362         shift 2
2363         local args=("$@")
2364
2365         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2366         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2367         ret=$?
2368         if [[ $expected_to_fail -eq "1" ]]; then
2369                 test "$ret" -ne "0"
2370                 ok $? "Add trigger $trigger_name failed as expected"
2371         else
2372                 ok $ret "Add trigger $trigger_name"
2373         fi
2374 }
2375
2376 function lttng_remove_trigger()
2377 {
2378         local expected_to_fail="$1"
2379         local trigger_name="$2"
2380         shift 2
2381
2382         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2383         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2384         ret=$?
2385         if [[ $expected_to_fail -eq "1" ]]; then
2386                 test "$ret" -ne "0"
2387                 ok $? "Remove trigger $trigger_name failed as expected"
2388         else
2389                 ok $ret "Remove trigger $trigger_name"
2390         fi
2391 }
2392
2393 function lttng_add_trigger_ok()
2394 {
2395         lttng_add_trigger 0 "$@"
2396 }
2397
2398 function lttng_add_trigger_fail()
2399 {
2400         lttng_add_trigger 1 "$@"
2401 }
2402
2403 function lttng_remove_trigger_ok()
2404 {
2405         lttng_remove_trigger 0 "$@"
2406 }
2407
2408 function list_triggers_matches_ok ()
2409 {
2410         local tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2411         local tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2412
2413         local test_name="$1"
2414         local expected_stdout_file="$2"
2415
2416         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2417
2418         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2419         ok $? "${test_name}: exit code is 0"
2420
2421         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2422         ok $? "${test_name}: expected stdout"
2423
2424         diff -u /dev/null "${tmp_stderr}"
2425         ok $? "${test_name}: expected stderr"
2426
2427         rm -f "${tmp_stdout}"
2428         rm -f "${tmp_stderr}"
2429 }
2430
2431 function list_triggers_matches_mi_ok ()
2432 {
2433         local tmp_stdout
2434         local tmp_stdout_raw
2435         local tmp_stderr
2436
2437         local test_name="$1"
2438         local expected_stdout_file="$2"
2439
2440         tmp_stdout_raw=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2441         tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2442         tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2443
2444         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2445
2446         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2447         ok $? "${test_name}: exit code is 0"
2448
2449         # Pretty-fy xml before further test.
2450         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2451
2452         $MI_VALIDATE "${tmp_stdout}"
2453         ok $? "list-trigger mi is valid"
2454
2455         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2456         ok $? "${test_name}: expected stdout"
2457
2458         diff -u /dev/null "${tmp_stderr}"
2459         ok $? "${test_name}: expected stderr"
2460
2461         rm -f "${tmp_stdout}"
2462         rm -f "${tmp_stdout_raw}"
2463         rm -f "${tmp_stderr}"
2464 }
2465
2466 function validate_path_pattern ()
2467 {
2468         local message=$1
2469         local pattern=$2
2470         # Base path is only used in error case and is used to list the content
2471         # of the base path.
2472         local base_path=$3
2473
2474
2475         [ -f $pattern ]
2476         ret=$?
2477         ok $ret "$message"
2478
2479         if [ "$ret" -ne "0" ]; then
2480                 diag "Path pattern expected: $pattern"
2481                 # List the tracepath for more info. We use find as a recursive
2482                 # directory lister.
2483                 diag "The base path content:"
2484                 find "$base_path" -print
2485         fi
2486 }
2487
2488 function validate_trace_path_ust_uid ()
2489 {
2490         local trace_path=$1
2491         local session_name=$2
2492         local uid=$UID
2493         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2494
2495         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2496 }
2497
2498 function validate_trace_path_ust_uid_network ()
2499 {
2500         local trace_path=$1
2501         local session_name=$2
2502         local base_path=$3
2503         local uid=$UID
2504         local hostname=$HOSTNAME
2505         local pattern
2506         local ret
2507
2508         # If the session was given a network base path (e.g
2509         # 127.0.0.1/my/custom/path on creation, there is no session name
2510         # component to the path on the relayd side. Caller can simply not pass a
2511         # session name for this scenario.
2512         if [ -n "$session_name" ]; then
2513                 session_name="$session_name-$date_time_pattern"
2514                 if [ -n "$base_path" ]; then
2515                         fail "Session name and base path are mutually exclusive"
2516                         return
2517                 fi
2518         fi
2519
2520         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2521
2522         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2523 }
2524
2525 function validate_trace_path_ust_uid_snapshot_network ()
2526 {
2527         local trace_path=$1
2528         local session_name=$2
2529         local snapshot_name=$3
2530         local snapshot_number=$4
2531         local base_path=$5
2532         local hostname=$HOSTNAME
2533         local uid=$UID
2534         local pattern
2535         local ret
2536
2537         # If the session/output was given a network base path (e.g
2538         # 127.0.0.1/my/custom/path on creation, there is no session name
2539         # component to the path on the relayd side. Caller can simply not pass a
2540         # session name for this scenario.
2541         if [ -n "$session_name" ]; then
2542                 session_name="$session_name-$date_time_pattern"
2543                 if [ -n "$base_path" ]; then
2544                         fail "Session name and base path are mutually exclusive"
2545                         return
2546                 fi
2547         fi
2548
2549         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2550
2551         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2552 }
2553
2554 function validate_trace_path_ust_uid_snapshot ()
2555 {
2556         local trace_path=$1
2557         local session_name=$2
2558         local snapshot_name=$3
2559         local snapshot_number=$4
2560         local base_path=$5
2561         local uid=$UID
2562         local pattern
2563         local ret
2564
2565         # If the session/output was given a network base path (e.g
2566         # 127.0.0.1/my/custom/path) on creation, there is no session name
2567         # component to the path on the relayd side. Caller can simply not pass a
2568         # session name for this scenario.
2569         if [ -n "$session_name" ]; then
2570                 session_name="$session_name-$date_time_pattern"
2571                 if [ -n "$base_path" ]; then
2572                         fail "Session name and base path are mutually exclusive"
2573                         return
2574                 fi
2575         fi
2576
2577         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2578
2579         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2580 }
2581
2582 function validate_trace_path_ust_pid ()
2583 {
2584         local trace_path=$1
2585         local session_name=$2
2586         local app_string=$3
2587         local pid=$4
2588         local pattern
2589         local ret
2590
2591         # If the session was given a trace path on creation, there is no session
2592         # name component to the path. Caller can simply not pass a session name
2593         # for this scenario.
2594         if [ -n "$session_name" ]; then
2595                 session_name="$session_name-$date_time_pattern"
2596         fi
2597
2598         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2599
2600         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2601 }
2602
2603 function validate_trace_path_kernel ()
2604 {
2605         local trace_path=$1
2606         local session_name=$2
2607         local pattern
2608
2609         # If the session was given a trace path on creation, there is no session
2610         # name component to the path. Caller can simply not pass a session name
2611         # for this scenario.
2612         if [ -n "$session_name" ]; then
2613                 session_name="$session_name-$date_time_pattern"
2614         fi
2615
2616         pattern="$trace_path/$session_name/kernel/metadata"
2617
2618         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2619 }
2620
2621 function validate_trace_path_kernel_network ()
2622 {
2623         local trace_path=$1
2624         local session_name=$2
2625         local hostname=$HOSTNAME
2626         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2627
2628         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2629 }
2630
2631 function validate_trace_path_kernel_snapshot ()
2632 {
2633         local trace_path=$1
2634         local session_name=$2
2635         local snapshot_name=$3
2636         local snapshot_number=$4
2637         local base_path=$5
2638         local pattern
2639         local ret
2640
2641         # If the session/output was given a network base path (e.g
2642         # 127.0.0.1/my/custom/path on creation, there is no session name
2643         # component to the path on the relayd side. Caller can simply not pass a
2644         # session name for this scenario.
2645         if [ -n "$session_name" ]; then
2646                 session_name="$session_name-$date_time_pattern"
2647                 if [ -n "$base_path" ]; then
2648                         fail "Session name and base path are mutually exclusive"
2649                         return
2650                 fi
2651         fi
2652
2653         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2654
2655         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2656 }
2657
2658 function validate_trace_path_kernel_snapshot_network ()
2659 {
2660         local trace_path=$1
2661         local session_name=$2
2662         local snapshot_name=$3
2663         local snapshot_number=$4
2664         local base_path=$5
2665         local hostname=$HOSTNAME
2666         local pattern
2667         local ret
2668
2669         # If the session/output was given a network base path (e.g
2670         # 127.0.0.1/my/custom/path on creation, there is no session name
2671         # component to the path on the relayd side. Caller can simply not pass a
2672         # session name for this scenario.
2673         if [ -n "$session_name" ]; then
2674                 session_name="$session_name-$date_time_pattern"
2675                 if [ -n "$base_path" ]; then
2676                         fail "Session name and base path are mutually exclusive"
2677                         return
2678                 fi
2679         fi
2680
2681         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2682
2683         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2684 }