tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$(cut -d '' -f 1 2>/dev/null < /proc/"$pid"/cmdline); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 208         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 209         echo
 210 }
 211
 212 # Helpers for get_possible_cpus.
 213 function get_possible_cpus_count_from_sysfs_possible_mask()
 214 {
 215         local max_possible_cpu_id=$(cut -d '-' -f 2 < /sys/devices/system/cpu/possible)
 216         echo $((max_possible_cpu_id+1))
 217 }
 218
 219 function get_max_cpus_count_from_sysfs_cpu_directories()
 220 {
 221         local max_possible_cpu_id= \
 222                 $(find /sys/devices/system/cpu/ -mindepth 1 -maxdepth 1 -regex ".+cpu[0-9]+" | \
 223                   sed -e 's/cpu//g' | \
 224                   awk -F '/' '{ if ($NF > N) N = $NF } END { print N }')
 225         echo $((max_possible_cpu_id+1))
 226 }
 227
 228 # Return the number of possible CPUs.
 229 function get_possible_cpus_count()
 230 {
 231         local possible_cpus_count=$(get_possible_cpus_count_from_sysfs_possible_mask)
 232
 233         if [ $? -ne 0 ]; then
 234                 possible_cpus_count=$(get_max_cpus_count_from_sysfs_cpu_directories)
 235                 local configured_cpus_count=$(getconf _NPROCESSORS_CONF)
 236                 possible_cpus_count=$(($configured_cpus_count > $possible_cpus_count \
 237                                                               ? $configured_cpus_count \
 238                                                               : $possible_cpus_count))
 239         fi
 240
 241         echo $possible_cpus_count
 242 }
 243
 244 # Return the number of _configured_ CPUs.
 245 function conf_proc_count()
 246 {
 247         getconf _NPROCESSORS_CONF
 248         if [ $? -ne 0 ]; then
 249                 diag "Failed to get the number of configured CPUs"
 250         fi
 251         echo
 252 }
 253
 254 # Check if base lttng-modules are present.
 255 # Bail out on failure
 256 function validate_lttng_modules_present ()
 257 {
 258         # Check for loadable modules.
 259         modprobe -n lttng-tracer 2>/dev/null
 260         if [ $? -eq 0 ]; then
 261                 return 0
 262         fi
 263
 264         # Check for builtin modules.
 265         ls /proc/lttng > /dev/null 2>&1
 266         if [ $? -eq 0 ]; then
 267                 return 0
 268         fi
 269
 270         LTTNG_BAIL_OUT "LTTng modules not detected."
 271 }
 272
 273 # Run the lttng binary.
 274 #
 275 # The first two arguments are stdout and stderr redirect paths, respectively.
 276 # The rest of the arguments are forwarded to the lttng binary
 277 function _run_lttng_cmd
 278 {
 279         local stdout_dest="$1"
 280         local stderr_dest="$2"
 281         shift 2
 282
 283         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 284         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 285 }
 286
 287 function enable_kernel_lttng_event
 288 {
 289         local withtap="$1"
 290         local expected_to_fail="$2"
 291         local sess_name="$3"
 292         local event_name="$4"
 293         local channel_name="$5"
 294
 295         if [ -z "$event_name" ]; then
 296                 # Enable all event if no event name specified
 297                 event_name="-a"
 298         fi
 299
 300         if [ -z "$channel_name" ]; then
 301                 # default channel if none specified
 302                 chan=""
 303         else
 304                 chan="-c $channel_name"
 305         fi
 306
 307         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 308         ret=$?
 309         if [[ $expected_to_fail -eq "1" ]]; then
 310                 test $ret -ne "0"
 311                 ret=$?
 312                 if [ $withtap -eq "1" ]; then
 313                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 314                 fi
 315         else
 316                 if [ $withtap -eq "1" ]; then
 317                         ok $ret "Enable kernel event $event_name for session $sess_name"
 318                 fi
 319         fi
 320 }
 321
 322 function enable_kernel_lttng_event_ok ()
 323 {
 324         enable_kernel_lttng_event 1 0 "$@"
 325 }
 326
 327 function enable_kernel_lttng_event_fail ()
 328 {
 329         enable_kernel_lttng_event 1 1 "$@"
 330 }
 331
 332 function enable_kernel_lttng_event_notap ()
 333 {
 334         enable_kernel_lttng_event 0 0 "$@"
 335 }
 336
 337 # Old interface
 338 function lttng_enable_kernel_event
 339 {
 340         enable_kernel_lttng_event_ok "$@"
 341 }
 342
 343 function lttng_enable_kernel_syscall()
 344 {
 345         local expected_to_fail=$1
 346         local sess_name=$2
 347         local syscall_name=$3
 348         local channel_name=$4
 349
 350         if [ -z $syscall_name ]; then
 351                 # Enable all event if no syscall name specified
 352                 syscall_name="-a"
 353         fi
 354
 355         if [ -z $channel_name ]; then
 356                 # default channel if none specified
 357                 chan=""
 358         else
 359                 chan="-c $channel_name"
 360         fi
 361
 362         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 363         ret=$?
 364         if [[ $expected_to_fail -eq "1" ]]; then
 365                 test $ret -ne "0"
 366                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 367         else
 368                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 369         fi
 370 }
 371
 372 function lttng_enable_kernel_syscall_ok()
 373 {
 374         lttng_enable_kernel_syscall 0 "$@"
 375 }
 376
 377 function lttng_enable_kernel_syscall_fail()
 378 {
 379         lttng_enable_kernel_syscall 1 "$@"
 380 }
 381
 382 function lttng_disable_kernel_syscall()
 383 {
 384         local expected_to_fail=$1
 385         local sess_name=$2
 386         local syscall_name=$3
 387         local channel_name=$4
 388
 389         if [ -z $syscall_name ]; then
 390                 # Enable all event if no syscall name specified
 391                 syscall_name="-a"
 392         fi
 393
 394         if [ -z $channel_name ]; then
 395                 # default channel if none specified
 396                 chan=""
 397         else
 398                 chan="-c $channel_name"
 399         fi
 400
 401         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 402
 403         ret=$?
 404         if [[ $expected_to_fail -eq "1" ]]; then
 405                 test $ret -ne "0"
 406                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 407         else
 408                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 409         fi
 410 }
 411
 412 function lttng_disable_kernel_syscall_ok()
 413 {
 414         lttng_disable_kernel_syscall 0 "$@"
 415 }
 416
 417 function lttng_disable_kernel_syscall_fail()
 418 {
 419         lttng_disable_kernel_syscall 1 "$@"
 420 }
 421
 422 function lttng_enable_kernel_function_event ()
 423 {
 424         local expected_to_fail="$1"
 425         local sess_name="$2"
 426         local target="$3"
 427         local event_name="$4"
 428
 429         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 430         ret=$?
 431         if [[ $expected_to_fail -eq "1" ]]; then
 432                 test $ret -ne "0"
 433                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 434         else
 435                 ok $ret "Enable kernel function event for session $sess_name"
 436         fi
 437 }
 438
 439 function lttng_enable_kernel_function_event_ok ()
 440 {
 441         lttng_enable_kernel_function_event 0 "$@"
 442 }
 443
 444 function lttng_enable_kernel_userspace_probe_event ()
 445 {
 446         local expected_to_fail="$1"
 447         local sess_name="$2"
 448         local target="$3"
 449         local event_name="$4"
 450
 451         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 452         ret=$?
 453         if [[ $expected_to_fail -eq "1" ]]; then
 454                 test $ret -ne "0"
 455                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 456         else
 457                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 458         fi
 459 }
 460
 461 function lttng_enable_kernel_userspace_probe_event_fail ()
 462 {
 463         lttng_enable_kernel_userspace_probe_event 1 "$@"
 464 }
 465
 466 function lttng_enable_kernel_userspace_probe_event_ok ()
 467 {
 468         lttng_enable_kernel_userspace_probe_event 0 "$@"
 469 }
 470
 471 function disable_kernel_lttng_userspace_probe_event_ok ()
 472 {
 473         local sess_name="$1"
 474         local event_name="$2"
 475
 476         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 477         ok $? "Disable kernel event $target for session $sess_name"
 478 }
 479 function lttng_enable_kernel_channel()
 480 {
 481         local withtap=$1
 482         local expected_to_fail=$2
 483         local sess_name=$3
 484         local channel_name=$4
 485         local opts="${@:5}"
 486
 487         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 488         ret=$?
 489         if [[ $expected_to_fail -eq "1" ]]; then
 490                 test "$ret" -ne "0"
 491                 ret=$?
 492                 if [ $withtap -eq "1" ]; then
 493                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 494                 fi
 495         else
 496                 if [ $withtap -eq "1" ]; then
 497                         ok $ret "Enable channel $channel_name for session $sess_name"
 498                 fi
 499         fi
 500 }
 501
 502 function lttng_enable_kernel_channel_ok()
 503 {
 504         lttng_enable_kernel_channel 1 0 "$@"
 505 }
 506
 507 function lttng_enable_kernel_channel_fail()
 508 {
 509         lttng_enable_kernel_channel 1 1 "$@"
 510 }
 511
 512 function lttng_enable_kernel_channel_notap()
 513 {
 514         lttng_enable_kernel_channel 0 0 "$@"
 515 }
 516
 517 function enable_kernel_lttng_channel_ok()
 518 {
 519         lttng_enable_kernel_channel 1 0 "$@"
 520 }
 521
 522 function lttng_disable_kernel_channel()
 523 {
 524         local expected_to_fail=$1
 525         local sess_name=$2
 526         local channel_name=$3
 527
 528         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 529         ret=$?
 530         if [[ $expected_to_fail -eq "1" ]]; then
 531                 test "$ret" -ne "0"
 532                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 533         else
 534                 ok $ret "Disable channel $channel_name for session $sess_name"
 535         fi
 536 }
 537
 538 function lttng_disable_kernel_channel_ok()
 539 {
 540         lttng_disable_kernel_channel 0 "$@"
 541 }
 542
 543 function lttng_disable_kernel_channel_fail()
 544 {
 545         lttng_disable_kernel_channel 1 "$@"
 546 }
 547
 548 function start_lttng_relayd_opt()
 549 {
 550         local withtap=$1
 551         local process_mode=$2
 552         local opt=$3
 553
 554         DIR=$(readlink -f "$TESTDIR")
 555
 556         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 557                 # shellcheck disable=SC2086
 558                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 559                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 560                 if [ $? -eq 1 ]; then
 561                         if [ $withtap -eq "1" ]; then
 562                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 563                         fi
 564                         return 1
 565                 else
 566                         if [ $withtap -eq "1" ]; then
 567                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 568                         fi
 569                 fi
 570         else
 571                 pass "Start lttng-relayd (opt: $opt)"
 572         fi
 573 }
 574
 575 function start_lttng_relayd()
 576 {
 577         start_lttng_relayd_opt 1 "-b" "$@"
 578 }
 579
 580 function start_lttng_relayd_notap()
 581 {
 582         start_lttng_relayd_opt 0 "-b" "$@"
 583 }
 584
 585 function stop_lttng_relayd_opt()
 586 {
 587         local withtap=$1
 588         local is_cleanup=$2
 589         local signal=$3
 590         local timeout_s=$4
 591         local dtimeleft_s=
 592         local retval=0
 593         local pids
 594
 595         if [ -z "$signal" ]; then
 596                 signal="SIGTERM"
 597         fi
 598
 599
 600         # Multiply time by 2 to simplify integer arithmetic
 601         if [ -n "$timeout_s" ]; then
 602                 dtimeleft_s=$((timeout_s * 2))
 603         fi
 604
 605
 606         pids=$(lttng_pgrep "$RELAYD_MATCH")
 607         if [ -z "$pids" ]; then
 608                 if [ "$is_cleanup" -eq 1 ]; then
 609                         :
 610                 elif [ "$withtap" -eq "1" ]; then
 611                         fail "No relay daemon to kill"
 612                 else
 613                         LTTNG_BAIL_OUT "No relay daemon to kill"
 614                 fi
 615                 return 0
 616         fi
 617
 618         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 619
 620         # shellcheck disable=SC2086
 621         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 622                 retval=1
 623                 if [ "$withtap" -eq "1" ]; then
 624                         fail "Kill relay daemon"
 625                 fi
 626         else
 627                 out=1
 628                 while [ -n "$out" ]; do
 629                         out=$(lttng_pgrep "$RELAYD_MATCH")
 630                         if [ -n "$dtimeleft_s" ]; then
 631                                 if [ $dtimeleft_s -lt 0 ]; then
 632                                         out=
 633                                         retval=1
 634                                 fi
 635                                 dtimeleft_s=$((dtimeleft_s - 1))
 636                         fi
 637                         sleep 0.5
 638                 done
 639                 if [ "$withtap" -eq "1" ]; then
 640                         if [ "$retval" -eq "0" ]; then
 641                                 pass "Wait after kill relay daemon"
 642                         else
 643                                 fail "Wait after kill relay daemon"
 644                         fi
 645                 fi
 646         fi
 647         return $retval
 648 }
 649
 650 function stop_lttng_relayd()
 651 {
 652         stop_lttng_relayd_opt 1 0 "$@"
 653 }
 654
 655 function stop_lttng_relayd_notap()
 656 {
 657         stop_lttng_relayd_opt 0 0 "$@"
 658 }
 659
 660 function stop_lttng_relayd_cleanup()
 661 {
 662         stop_lttng_relayd_opt 0 1 "$@"
 663 }
 664
 665 #First arg: show tap output
 666 #Second argument: load path for automatic loading
 667 function start_lttng_sessiond_opt()
 668 {
 669         local withtap=$1
 670         local load_path=$2
 671
 672         # The rest of the arguments will be passed directly to lttng-sessiond.
 673         shift 2
 674
 675         local env_vars=""
 676         local consumerd=""
 677
 678         local long_bit_value=
 679         long_bit_value=$(getconf LONG_BIT)
 680
 681         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 682                 # Env variable requested no session daemon
 683                 return
 684         fi
 685
 686         DIR=$(readlink -f "$TESTDIR")
 687
 688         # Get long_bit value for 32/64 consumerd
 689         case "$long_bit_value" in
 690                 32)
 691                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 692                         ;;
 693                 64)
 694                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 695                         ;;
 696                 *)
 697                         return
 698                         ;;
 699         esac
 700
 701         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 702         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 703                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 704         fi
 705         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 706
 707         if ! validate_kernel_version; then
 708             fail "Start session daemon"
 709             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 710         fi
 711
 712         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 713         export LTTNG_SESSION_CONFIG_XSD_PATH
 714
 715         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 716                 # Have a load path ?
 717                 if [ -n "$load_path" ]; then
 718                         # shellcheck disable=SC2086
 719                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 720                 else
 721                         # shellcheck disable=SC2086
 722                         env $env_vars --background "$consumerd" "$@"
 723                 fi
 724                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 725                 status=$?
 726                 if [ "$withtap" -eq "1" ]; then
 727                         ok $status "Start session daemon"
 728                 fi
 729         fi
 730 }
 731
 732 function start_lttng_sessiond()
 733 {
 734         start_lttng_sessiond_opt 1 "$@"
 735 }
 736
 737 function start_lttng_sessiond_notap()
 738 {
 739         start_lttng_sessiond_opt 0 "$@"
 740 }
 741
 742 function stop_lttng_sessiond_opt()
 743 {
 744         local withtap=$1
 745         local is_cleanup=$2
 746         local signal=$3
 747         local timeout_s=$4
 748         local dtimeleft_s=
 749         local retval=0
 750         local runas_pids
 751         local pids
 752
 753         if [ -z "$signal" ]; then
 754                 signal=SIGTERM
 755         fi
 756
 757         # Multiply time by 2 to simplify integer arithmetic
 758         if [ -n "$timeout_s" ]; then
 759                 dtimeleft_s=$((timeout_s * 2))
 760         fi
 761
 762         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 763                 # Env variable requested no session daemon
 764                 return 0
 765         fi
 766
 767         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 768         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 769
 770         if [ -n "$runas_pids" ]; then
 771                 pids="$pids $runas_pids"
 772         fi
 773
 774         if [ -z "$pids" ]; then
 775                 if [ "$is_cleanup" -eq 1 ]; then
 776                         :
 777                 elif [ "$withtap" -eq "1" ]; then
 778                         fail "No session daemon to kill"
 779                 else
 780                         LTTNG_BAIL_OUT "No session daemon to kill"
 781                 fi
 782                 return 0
 783         fi
 784
 785         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 786
 787         # shellcheck disable=SC2086
 788         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 789                 retval=1
 790                 if [ "$withtap" -eq "1" ]; then
 791                         fail "Kill sessions daemon"
 792                 fi
 793         else
 794                 out=1
 795                 while [ -n "$out" ]; do
 796                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 797                         if [ -n "$dtimeleft_s" ]; then
 798                                 if [ $dtimeleft_s -lt 0 ]; then
 799                                         out=
 800                                         retval=1
 801                                 fi
 802                                 dtimeleft_s=$((dtimeleft_s - 1))
 803                         fi
 804                         sleep 0.5
 805                 done
 806                 out=1
 807                 while [ -n "$out" ]; do
 808                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 809                         if [ -n "$dtimeleft_s" ]; then
 810                                 if [ $dtimeleft_s -lt 0 ]; then
 811                                         out=
 812                                         retval=1
 813                                 fi
 814                                 dtimeleft_s=$((dtimeleft_s - 1))
 815                         fi
 816                         sleep 0.5
 817                 done
 818
 819                 if [ "$withtap" -eq "1" ]; then
 820                         if [ "$retval" -eq "0" ]; then
 821                                 pass "Wait after kill session daemon"
 822                         else
 823                                 fail "Wait after kill session daemon"
 824                         fi
 825                 fi
 826         fi
 827         if [ "$signal" = "SIGKILL" ]; then
 828                 if [ "$(id -u)" -eq "0" ]; then
 829                         local modules=
 830                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 831
 832                         if [ -n "$modules" ]; then
 833                                 diag "Unloading all LTTng modules"
 834                                 modprobe --remove "$modules"
 835                         fi
 836                 fi
 837         fi
 838
 839         return $retval
 840 }
 841
 842 function stop_lttng_sessiond()
 843 {
 844         stop_lttng_sessiond_opt 1 0 "$@"
 845 }
 846
 847 function stop_lttng_sessiond_notap()
 848 {
 849         stop_lttng_sessiond_opt 0 0 "$@"
 850 }
 851
 852 function stop_lttng_sessiond_cleanup()
 853 {
 854         stop_lttng_sessiond_opt 0 1 "$@"
 855 }
 856
 857 function sigstop_lttng_sessiond_opt()
 858 {
 859         local withtap=$1
 860         local signal=SIGSTOP
 861         local pids
 862
 863         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 864                 # Env variable requested no session daemon
 865                 return
 866         fi
 867
 868         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 869
 870         if [ "$withtap" -eq "1" ]; then
 871                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 872         fi
 873
 874         # shellcheck disable=SC2086
 875         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 876                 if [ "$withtap" -eq "1" ]; then
 877                         fail "Sending SIGSTOP to session daemon"
 878                 fi
 879         else
 880                 out=1
 881                 while [ $out -ne 0 ]; do
 882                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 883
 884                         # Wait until state becomes stopped for session
 885                         # daemon(s).
 886                         out=0
 887                         for sessiond_pid in $pids; do
 888                                 state="$(ps -p "$sessiond_pid" -o state= )"
 889                                 if [[ -n "$state" && "$state" != "T" ]]; then
 890                                         out=1
 891                                 fi
 892                         done
 893                         sleep 0.5
 894                 done
 895                 if [ "$withtap" -eq "1" ]; then
 896                         pass "Sending SIGSTOP to session daemon"
 897                 fi
 898         fi
 899 }
 900
 901 function sigstop_lttng_sessiond()
 902 {
 903         sigstop_lttng_sessiond_opt 1 "$@"
 904 }
 905
 906 function sigstop_lttng_sessiond_notap()
 907 {
 908         sigstop_lttng_sessiond_opt 0 "$@"
 909 }
 910
 911 function stop_lttng_consumerd_opt()
 912 {
 913         local withtap=$1
 914         local is_cleanup=$2
 915         local signal=$3
 916         local timeout_s=$4
 917         local dtimeleft_s=
 918         local retval=0
 919         local pids
 920
 921         if [ -z "$signal" ]; then
 922                 signal=SIGTERM
 923         fi
 924
 925         # Multiply time by 2 to simplify integer arithmetic
 926         if [ -n "$timeout_s" ]; then
 927                 dtimeleft_s=$((timeout_s * 2))
 928         fi
 929
 930         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 931
 932         if [ -z "$pids" ]; then
 933                 if [ "$is_cleanup" -eq 1 ]; then
 934                         :
 935                 elif [ "$withtap" -eq "1" ]; then
 936                         fail "No consumerd daemon to kill"
 937                 else
 938                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
 939                 fi
 940                 return 0
 941         fi
 942
 943         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 944
 945         # shellcheck disable=SC2086
 946         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 947                 retval=1
 948                 if [ "$withtap" -eq "1" ]; then
 949                         fail "Kill consumer daemon"
 950                 fi
 951         else
 952                 out=1
 953                 while [ $out -ne 0 ]; do
 954                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 955
 956                         # If consumerds are still present check their status.
 957                         # A zombie status qualifies the consumerd as *killed*
 958                         out=0
 959                         for consumer_pid in $pids; do
 960                                 state="$(ps -p "$consumer_pid" -o state= )"
 961                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 962                                         out=1
 963                                 fi
 964                         done
 965                         if [ -n "$dtimeleft_s" ]; then
 966                                 if [ $dtimeleft_s -lt 0 ]; then
 967                                         out=0
 968                                         retval=1
 969                                 fi
 970                                 dtimeleft_s=$((dtimeleft_s - 1))
 971                         fi
 972                         sleep 0.5
 973                 done
 974                 if [ "$withtap" -eq "1" ]; then
 975                         if [ "$retval" -eq "0" ]; then
 976                                 pass "Wait after kill consumer daemon"
 977                         else
 978                                 fail "Wait after kill consumer daemon"
 979                         fi
 980                 fi
 981         fi
 982
 983         return $retval
 984 }
 985
 986 function stop_lttng_consumerd()
 987 {
 988         stop_lttng_consumerd_opt 1 0 "$@"
 989 }
 990
 991 function stop_lttng_consumerd_notap()
 992 {
 993         stop_lttng_consumerd_opt 0 0 "$@"
 994 }
 995
 996 function stop_lttng_consumerd_cleanup()
 997 {
 998         stop_lttng_consumerd_opt 0 1 "$@"
 999 }
1000
1001 function sigstop_lttng_consumerd_opt()
1002 {
1003         local withtap=$1
1004         local signal=SIGSTOP
1005         local pids
1006
1007         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1008
1009         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1010
1011         # shellcheck disable=SC2086
1012         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1013         retval=$?
1014
1015         if [ $retval -eq 1 ]; then
1016                 if [ "$withtap" -eq "1" ]; then
1017                         fail "Sending SIGSTOP to consumer daemon"
1018                 fi
1019                 return 1
1020         else
1021                 out=1
1022                 while [ $out -ne 0 ]; do
1023                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1024
1025                         # Wait until state becomes stopped for all
1026                         # consumers.
1027                         out=0
1028                         for consumer_pid in $pids; do
1029                                 state="$(ps -p "$consumer_pid" -o state= )"
1030                                 if [[ -n "$state" && "$state" != "T" ]]; then
1031                                         out=1
1032                                 fi
1033                         done
1034                         sleep 0.5
1035                 done
1036                 if [ "$withtap" -eq "1" ]; then
1037                         pass "Sending SIGSTOP to consumer daemon"
1038                 fi
1039         fi
1040         return $retval
1041 }
1042
1043 function sigstop_lttng_consumerd()
1044 {
1045         sigstop_lttng_consumerd_opt 1 "$@"
1046 }
1047
1048 function sigstop_lttng_consumerd_notap()
1049 {
1050         sigstop_lttng_consumerd_opt 0 "$@"
1051 }
1052
1053 function list_lttng_with_opts ()
1054 {
1055         local ret
1056         local withtap=$1
1057         shift
1058         local opts=$1
1059         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1060         ret=$?
1061         if [ $withtap -eq "1" ]; then
1062                 ok $ret "Lttng-tool list command with option $opts"
1063         fi
1064 }
1065
1066 function list_lttng_ok ()
1067 {
1068         list_lttng_with_opts 1 "$@"
1069 }
1070
1071 function list_lttng_notap ()
1072 {
1073         list_lttng_with_opts 0 "$@"
1074 }
1075
1076 function create_lttng_session_no_output ()
1077 {
1078         local sess_name=$1
1079         local opts="${@:2}"
1080
1081         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1082         ok $? "Create session $sess_name in no-output mode"
1083 }
1084
1085 function create_lttng_session_uri () {
1086         local sess_name=$1
1087         local uri=$2
1088         local opts="${@:3}"
1089
1090         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1091         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1092 }
1093
1094 function create_lttng_session ()
1095 {
1096         local withtap=$1
1097         local expected_to_fail=$2
1098         local sess_name=$3
1099         local trace_path=$4
1100         local opt=$5
1101
1102         if [ -z "$trace_path" ]; then
1103                 # Use lttng-sessiond default output.
1104                 trace_path=""
1105         else
1106                 trace_path="-o $trace_path"
1107         fi
1108
1109         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create "$sess_name" $trace_path $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1110         ret=$?
1111         if [ $expected_to_fail -eq "1" ]; then
1112                 test "$ret" -ne "0"
1113                 ret=$?
1114                 if [ $withtap -eq "1" ]; then
1115                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1116                 fi
1117         else
1118                 if [ $withtap -eq "1" ]; then
1119                         ok $ret "Create session $sess_name in $trace_path"
1120                 fi
1121         fi
1122         return $ret
1123 }
1124
1125 function create_lttng_session_ok ()
1126 {
1127         create_lttng_session 1 0 "$@"
1128 }
1129
1130 function create_lttng_session_fail ()
1131 {
1132         create_lttng_session 1 1 "$@"
1133 }
1134
1135 function create_lttng_session_notap ()
1136 {
1137         create_lttng_session 0 0 "$@"
1138 }
1139
1140
1141 function enable_ust_lttng_channel ()
1142 {
1143         local withtap=$1
1144         local expected_to_fail=$2
1145         local sess_name=$3
1146         local channel_name=$4
1147         local opts="${@:5}"
1148
1149         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1150         ret=$?
1151         if [[ $expected_to_fail -eq "1" ]]; then
1152                 test "$ret" -ne "0"
1153                 ret=$?
1154                 if [ $withtap -eq "1" ]; then
1155                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1156                 fi
1157         else
1158                 if [ $withtap -eq "1" ]; then
1159                         ok $ret "Enable channel $channel_name for session $sess_name"
1160                 fi
1161         fi
1162         return $ret
1163 }
1164
1165 function enable_ust_lttng_channel_ok ()
1166 {
1167         enable_ust_lttng_channel 1 0 "$@"
1168 }
1169
1170 function enable_ust_lttng_channel_fail ()
1171 {
1172         enable_ust_lttng_channel 1 1 "$@"
1173 }
1174
1175 function enable_ust_lttng_channel_notap ()
1176 {
1177         enable_ust_lttng_channel 0 0 "$@"
1178 }
1179
1180 function disable_ust_lttng_channel()
1181 {
1182         local sess_name=$1
1183         local channel_name=$2
1184
1185         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1186         ok $? "Disable channel $channel_name for session $sess_name"
1187 }
1188
1189 function enable_lttng_mmap_overwrite_kernel_channel()
1190 {
1191         local sess_name=$1
1192         local channel_name=$2
1193
1194         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1195         ok $? "Enable channel $channel_name for session $sess_name"
1196 }
1197
1198 function enable_lttng_mmap_discard_small_kernel_channel()
1199 {
1200         local sess_name=$1
1201         local channel_name=$2
1202
1203         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1204         ok $? "Enable small discard channel $channel_name for session $sess_name"
1205 }
1206
1207 function enable_lttng_mmap_overwrite_small_kernel_channel()
1208 {
1209         local sess_name=$1
1210         local channel_name=$2
1211
1212         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1213         ok $? "Enable small discard channel $channel_name for session $sess_name"
1214 }
1215
1216 function enable_lttng_mmap_overwrite_ust_channel()
1217 {
1218         local sess_name=$1
1219         local channel_name=$2
1220
1221         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1222         ok $? "Enable channel $channel_name for session $sess_name"
1223 }
1224
1225 function enable_ust_lttng_event ()
1226 {
1227         local withtap=$1
1228         local expected_to_fail=$2
1229         local sess_name=$3
1230         local event_name="$4"
1231         local channel_name=$5
1232
1233         if [ -z $channel_name ]; then
1234                 # default channel if none specified
1235                 chan=""
1236         else
1237                 chan="-c $channel_name"
1238         fi
1239
1240         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1241         ret=$?
1242         if [[ $expected_to_fail -eq "1" ]]; then
1243                 test $ret -ne "0"
1244                 ret=$?
1245                 if [[ $withtap -eq "1" ]]; then
1246                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1247                 fi
1248         else
1249                 if [[ $withtap -eq "1" ]]; then
1250                         ok $ret "Enable ust event $event_name for session $sess_name"
1251                 fi
1252         fi
1253         return $ret
1254 }
1255
1256 function enable_ust_lttng_event_ok ()
1257 {
1258         enable_ust_lttng_event 1 0 "$@"
1259 }
1260
1261 function enable_ust_lttng_event_fail ()
1262 {
1263         enable_ust_lttng_event 1 1 "$@"
1264 }
1265
1266 function enable_ust_lttng_event_notap ()
1267 {
1268         enable_ust_lttng_event 0 0 "$@"
1269 }
1270
1271 function enable_jul_lttng_event()
1272 {
1273         sess_name=$1
1274         event_name="$2"
1275         channel_name=$3
1276
1277         if [ -z $channel_name ]; then
1278                 # default channel if none specified
1279                 chan=""
1280         else
1281                 chan="-c $channel_name"
1282         fi
1283
1284         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1285         ok $? "Enable JUL event $event_name for session $sess_name"
1286 }
1287
1288 function enable_jul_lttng_event_loglevel()
1289 {
1290         local sess_name=$1
1291         local event_name="$2"
1292         local loglevel=$3
1293         local channel_name=$4
1294
1295         if [ -z $channel_name ]; then
1296                 # default channel if none specified
1297                 chan=""
1298         else
1299                 chan="-c $channel_name"
1300         fi
1301
1302         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1303         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1304 }
1305
1306 function enable_log4j_lttng_event()
1307 {
1308         local sess_name=$1
1309         local event_name=$2
1310         local channel_name=$3
1311
1312         local chan_opt=()
1313
1314         # default channel if none specified
1315         if [ -n "$channel_name" ]; then
1316                 chan_opt=("-c" "$channel_name")
1317         fi
1318
1319         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1320                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1321         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1322 }
1323
1324 function enable_log4j_lttng_event_filter()
1325 {
1326         local sess_name=$1
1327         local event_name=$2
1328         local filter=$3
1329
1330         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1331                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1332         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1333 }
1334
1335 function enable_log4j_lttng_event_filter_loglevel_only()
1336 {
1337         local sess_name=$1
1338         local event_name=$2
1339         local filter=$3
1340         local loglevel=$4
1341
1342         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1343                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1344         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1345 }
1346
1347 function enable_log4j_lttng_event_loglevel()
1348 {
1349         local sess_name=$1
1350         local event_name=$2
1351         local loglevel=$3
1352         local channel_name=$4
1353
1354
1355         # default channel if none specified
1356         if [ -n "$channel_name" ]; then
1357                 chan_opt=("-c" "$channel_name")
1358         fi
1359
1360         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1361                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1362         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1363 }
1364
1365 function enable_log4j_lttng_event_loglevel_only()
1366 {
1367         local sess_name=$1
1368         local event_name=$2
1369         local loglevel=$3
1370         local channel_name=$4
1371
1372         local chan_opt=()
1373
1374         # default channel if none specified
1375         if [ -n "$channel_name" ]; then
1376                 chan_opt=("-c" "$channel_name")
1377         fi
1378
1379         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1380                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1381         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1382 }
1383
1384 function enable_python_lttng_event()
1385 {
1386         sess_name=$1
1387         event_name="$2"
1388         channel_name=$3
1389
1390         if [ -z $channel_name ]; then
1391                 # default channel if none specified
1392                 chan=""
1393         else
1394                 chan="-c $channel_name"
1395         fi
1396
1397         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1398         ok $? "Enable Python event $event_name for session $sess_name"
1399 }
1400
1401 function enable_python_lttng_event_loglevel()
1402 {
1403         local sess_name=$1
1404         local event_name="$2"
1405         local loglevel=$3
1406         local channel_name=$4
1407
1408         if [ -z $channel_name ]; then
1409                 # default channel if none specified
1410                 chan=""
1411         else
1412                 chan="-c $channel_name"
1413         fi
1414
1415         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1416         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1417 }
1418
1419 function enable_ust_lttng_event_filter()
1420 {
1421         local sess_name="$1"
1422         local event_name="$2"
1423         local filter="$3"
1424         local channel_name=$4
1425
1426         if [ -z $channel_name ]; then
1427                 # default channel if none specified
1428                 chan=""
1429         else
1430                 chan="-c $channel_name"
1431         fi
1432
1433         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event $chan "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1434         ok $? "Enable event $event_name with filtering for session $sess_name"
1435 }
1436
1437 function enable_ust_lttng_event_loglevel()
1438 {
1439         local sess_name="$1"
1440         local event_name="$2"
1441         local loglevel="$3"
1442
1443         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1444         ok $? "Enable event $event_name with loglevel $loglevel"
1445 }
1446
1447 function enable_ust_lttng_event_loglevel_only()
1448 {
1449         local sess_name="$1"
1450         local event_name="$2"
1451         local loglevel="$3"
1452
1453         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1454         ok $? "Enable event $event_name with loglevel-only $loglevel"
1455 }
1456
1457 function disable_ust_lttng_event ()
1458 {
1459         local sess_name="$1"
1460         local event_name="$2"
1461         local channel_name="$3"
1462
1463         if [ -z $channel_name ]; then
1464                 # default channel if none specified
1465                 chan=""
1466         else
1467                 chan="-c $channel_name"
1468         fi
1469
1470         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1471         ok $? "Disable event $event_name for session $sess_name"
1472 }
1473
1474 function disable_jul_lttng_event ()
1475 {
1476         local sess_name="$1"
1477         local event_name="$2"
1478
1479         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1480         ok $? "Disable JUL event $event_name for session $sess_name"
1481 }
1482
1483 function disable_log4j_lttng_event ()
1484 {
1485         local sess_name="$1"
1486         local event_name="$2"
1487
1488         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1489                 disable-event "$event_name" -s "$sess_name" --log4j
1490         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1491 }
1492
1493 function disable_python_lttng_event ()
1494 {
1495         local sess_name="$1"
1496         local event_name="$2"
1497
1498         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1499         ok $? "Disable Python event $event_name for session $sess_name"
1500 }
1501
1502 function start_lttng_tracing_opt ()
1503 {
1504         local withtap=$1
1505         local expected_to_fail=$2
1506         local sess_name=$3
1507
1508         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1509         ret=$?
1510         if [[ $expected_to_fail -eq "1" ]]; then
1511                 test "$ret" -ne "0"
1512                 ret=$?
1513                 if [ $withtap -eq "1" ]; then
1514                         ok $? "Start tracing for session $sess_name failed as expected"
1515                 fi
1516         else
1517                 if [ $withtap -eq "1" ]; then
1518                         ok $ret "Start tracing for session $sess_name"
1519                 fi
1520         fi
1521 }
1522
1523 function start_lttng_tracing_ok ()
1524 {
1525         start_lttng_tracing_opt 1 0 "$@"
1526 }
1527
1528 function start_lttng_tracing_fail ()
1529 {
1530         start_lttng_tracing_opt 1 1 "$@"
1531 }
1532
1533 function start_lttng_tracing_notap ()
1534 {
1535         start_lttng_tracing_opt 0 1 "$@"
1536 }
1537
1538 function stop_lttng_tracing_opt ()
1539 {
1540         local withtap=$1
1541         local expected_to_fail=$2
1542         local sess_name=$3
1543
1544         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1545         ret=$?
1546         if [[ $expected_to_fail -eq "1" ]]; then
1547                 test "$ret" -ne "0"
1548                 ret=$?
1549                 if [ $withtap -eq "1" ]; then
1550                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1551                 fi
1552         else
1553                 if [ $withtap -eq "1" ]; then
1554                         ok $ret "Stop lttng tracing for session $sess_name"
1555                 fi
1556         fi
1557 }
1558
1559 function stop_lttng_tracing_ok ()
1560 {
1561         stop_lttng_tracing_opt 1 0 "$@"
1562 }
1563
1564 function stop_lttng_tracing_fail ()
1565 {
1566         stop_lttng_tracing_opt 1 1 "$@"
1567 }
1568
1569 function stop_lttng_tracing_notap ()
1570 {
1571         stop_lttng_tracing_opt 0 0 "$@"
1572 }
1573
1574 function destroy_lttng_session ()
1575 {
1576         local withtap=$1
1577         local expected_to_fail=$2
1578         local sess_name=$3
1579
1580         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1581         ret=$?
1582         if [[ $expected_to_fail -eq "1" ]]; then
1583                 test "$ret" -ne "0"
1584                 ret=$?
1585                 if [ $withtap -eq "1" ]; then
1586                         ok $ret "Destroy session $sess_name failed as expected"
1587                 fi
1588         else
1589                 if [ $withtap -eq "1" ]; then
1590                         ok $ret "Destroy session $sess_name"
1591                 fi
1592         fi
1593 }
1594
1595 function destroy_lttng_session_ok ()
1596 {
1597         destroy_lttng_session 1 0 "$@"
1598
1599 }
1600
1601 function destroy_lttng_session_fail ()
1602 {
1603         destroy_lttng_session 1 1 "$@"
1604 }
1605
1606 function destroy_lttng_session_notap ()
1607 {
1608         destroy_lttng_session 0 0 "$@"
1609 }
1610
1611 function destroy_lttng_sessions ()
1612 {
1613         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1614         ok $? "Destroy all lttng sessions"
1615 }
1616
1617 function lttng_snapshot_add_output ()
1618 {
1619         local expected_to_fail=$1
1620         local sess_name=$2
1621         local trace_path=$3
1622         local opts=$4
1623
1624         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1625         ret=$?
1626         if [[ $expected_to_fail -eq 1 ]]; then
1627                 test "$ret" -ne "0"
1628                 ok $? "Added snapshot output $trace_path failed as expected"
1629         else
1630                 ok $ret "Added snapshot output $trace_path"
1631         fi
1632 }
1633
1634 function lttng_snapshot_add_output_ok ()
1635 {
1636         lttng_snapshot_add_output 0 "$@"
1637 }
1638
1639 function lttng_snapshot_add_output_fail ()
1640 {
1641         lttng_snapshot_add_output 1 "$@"
1642 }
1643
1644 function lttng_snapshot_del_output ()
1645 {
1646         local expected_to_fail=$1
1647         local sess_name=$2
1648         local id=$3
1649
1650         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1651         ret=$?
1652         if [[ $expected_to_fail -eq "1" ]]; then
1653                 test "$ret" -ne "0"
1654                 ok $? "Deleted snapshot output id $id failed as expected"
1655         else
1656                 ok $ret "Deleted snapshot output id $id"
1657         fi
1658 }
1659
1660 function lttng_snapshot_del_output_ok ()
1661 {
1662         lttng_snapshot_del_output 0 "$@"
1663 }
1664
1665 function lttng_snapshot_del_output_fail ()
1666 {
1667         lttng_snapshot_del_output 1 "$@"
1668 }
1669
1670 function lttng_snapshot_record ()
1671 {
1672         local sess_name=$1
1673         local trace_path=$2
1674
1675         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s "$sess_name" "$trace_path" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1676         ok $? "Snapshot recorded"
1677 }
1678
1679 function lttng_snapshot_list ()
1680 {
1681         local sess_name=$1
1682         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1683         ok $? "Snapshot list"
1684 }
1685
1686 function lttng_save()
1687 {
1688         local sess_name=$1
1689         local opts=$2
1690
1691         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1692         ok $? "Session saved"
1693 }
1694
1695 function lttng_load()
1696 {
1697         local expected_to_fail=$1
1698         local opts=$2
1699
1700         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1701         ret=$?
1702         if [[ $expected_to_fail -eq "1" ]]; then
1703                 test $ret -ne "0"
1704                 ok $? "Load command failed as expected with opts: $opts"
1705         else
1706                 ok $ret "Load command with opts: $opts"
1707         fi
1708 }
1709
1710 function lttng_load_ok()
1711 {
1712         lttng_load 0 "$@"
1713 }
1714
1715 function lttng_load_fail()
1716 {
1717         lttng_load 1 "$@"
1718 }
1719
1720 function lttng_track()
1721 {
1722         local expected_to_fail="$1"
1723         shift 1
1724         local opts="$@"
1725         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1726         ret=$?
1727         if [[ $expected_to_fail -eq "1" ]]; then
1728                 test $ret -ne "0"
1729                 ok $? "Track command failed as expected with opts: $opts"
1730         else
1731                 ok $ret "Track command with opts: $opts"
1732         fi
1733 }
1734
1735 function lttng_track_ok()
1736 {
1737         lttng_track 0 "$@"
1738 }
1739
1740 function lttng_track_fail()
1741 {
1742         lttng_track 1 "$@"
1743 }
1744
1745 function lttng_untrack()
1746 {
1747         local expected_to_fail="$1"
1748         shift 1
1749         local opts="$@"
1750         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1751         ret=$?
1752         if [[ $expected_to_fail -eq "1" ]]; then
1753                 test $ret -ne "0"
1754                 ok $? "Untrack command failed as expected with opts: $opts"
1755         else
1756                 ok $ret "Untrack command with opts: $opts"
1757         fi
1758 }
1759
1760 function lttng_untrack_ok()
1761 {
1762         lttng_untrack 0 "$@"
1763 }
1764
1765 function lttng_untrack_fail()
1766 {
1767         lttng_untrack 1 "$@"
1768 }
1769
1770 function lttng_track_pid_ok()
1771 {
1772         PID=$1
1773         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1774         ok $? "Lttng track pid on the kernel domain"
1775 }
1776
1777 function lttng_untrack_kernel_all_ok()
1778 {
1779         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1780         ok $? "Lttng untrack all pid on the kernel domain"
1781 }
1782
1783 function lttng_track_ust_ok()
1784 {
1785         lttng_track_ok -u "$@"
1786 }
1787
1788 function lttng_track_ust_fail()
1789 {
1790         lttng_track_fail -u "$@"
1791 }
1792
1793 function lttng_track_kernel_ok()
1794 {
1795         lttng_track_ok -k "$@"
1796 }
1797
1798 function lttng_track_kernel_fail()
1799 {
1800         lttng_track_fail -k "$@"
1801 }
1802
1803 function lttng_untrack_ust_ok()
1804 {
1805         lttng_untrack_ok -u "$@"
1806 }
1807
1808 function lttng_untrack_ust_fail()
1809 {
1810         lttng_untrack_fail -u "$@"
1811 }
1812
1813 function lttng_untrack_kernel_ok()
1814 {
1815         lttng_untrack_ok -k "$@"
1816 }
1817
1818 function lttng_untrack_kernel_fail()
1819 {
1820         lttng_untrack_fail -k "$@"
1821 }
1822
1823 function lttng_add_context_list()
1824 {
1825         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1826         ret=$?
1827         ok $ret "Context listing"
1828 }
1829
1830 function add_context_lttng()
1831 {
1832         local expected_to_fail="$1"
1833         local domain="$2"
1834         local session_name="$3"
1835         local channel_name="$4"
1836         local type="$5"
1837
1838         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1839         ret=$?
1840         if [[ $expected_to_fail -eq "1" ]]; then
1841                 test $ret -ne "0"
1842                 ok $? "Add context command failed as expected for type: $type"
1843         else
1844                 ok $ret "Add context command for type: $type"
1845         fi
1846 }
1847
1848 function add_context_ust_ok()
1849 {
1850         add_context_lttng 0 -u "$@"
1851 }
1852
1853 function add_context_ust_fail()
1854 {
1855         add_context_lttng 1 -u "$@"
1856 }
1857
1858 function add_context_kernel_ok()
1859 {
1860         add_context_lttng 0 -k "$@"
1861 }
1862
1863 function add_context_kernel_fail()
1864 {
1865         add_context_lttng 1 -k "$@"
1866 }
1867
1868 function wait_live_trace_ready ()
1869 {
1870         local url=$1
1871         local zero_client_match=0
1872
1873         diag "Waiting for live trace at url: $url"
1874         while [ $zero_client_match -eq 0 ]; do
1875                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1876                 sleep 0.5
1877         done
1878         pass "Waiting for live trace at url: $url"
1879 }
1880
1881 function wait_live_viewer_connect ()
1882 {
1883         local url=$1
1884         local one_client_match=0
1885
1886         diag "Waiting for live viewers on url: $url"
1887         while [ $one_client_match -eq 0 ]; do
1888                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1889                 sleep 0.5
1890         done
1891         pass "Waiting for live viewers on url: $url"
1892 }
1893
1894 function bail_out_if_no_babeltrace()
1895 {
1896         which "$BABELTRACE_BIN" >/dev/null
1897         if [ $? -ne 0 ]; then
1898                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
1899         fi
1900 }
1901
1902 function validate_metadata_event ()
1903 {
1904         local event_name=$1
1905         local nr_event_id=$2
1906         local trace_path=$3
1907
1908         local metadata_file=$(find $trace_path -name "metadata")
1909         local metadata_path=$(dirname $metadata_file)
1910
1911         which $BABELTRACE_BIN >/dev/null
1912         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1913
1914         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1915
1916         if [ "$count" -ne "$nr_event_id" ]; then
1917                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1918                 diag "$count matching event id found in metadata"
1919         else
1920                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1921         fi
1922
1923 }
1924
1925 function trace_matches ()
1926 {
1927         local event_name=$1
1928         local nr_iter=$2
1929         local trace_path=$3
1930
1931         which $BABELTRACE_BIN >/dev/null
1932         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1933
1934         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1935
1936         if [ "$count" -ne "$nr_iter" ]; then
1937                 fail "Trace match"
1938                 diag "$count matching events found in trace"
1939         else
1940                 pass "Trace match"
1941         fi
1942 }
1943
1944 function trace_match_only()
1945 {
1946         local event_name=$1
1947         local nr_iter=$2
1948         local trace_path=$3
1949
1950         which $BABELTRACE_BIN >/dev/null
1951         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1952
1953         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1954         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1955
1956         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1957                 pass "Trace match with $total event $event_name"
1958         else
1959                 fail "Trace match"
1960                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1961         fi
1962 }
1963
1964 function validate_trace
1965 {
1966         local event_name=$1
1967         local trace_path=$2
1968
1969         which $BABELTRACE_BIN >/dev/null
1970         if [ $? -ne 0 ]; then
1971             skip 0 "Babeltrace binary not found. Skipping trace validation"
1972         fi
1973
1974         OLDIFS=$IFS
1975         IFS=","
1976         for i in $event_name; do
1977                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1978                 if [ "$traced" -ne 0 ]; then
1979                         pass "Validate trace for event $i, $traced events"
1980                 else
1981                         fail "Validate trace for event $i"
1982                         diag "Found $traced occurrences of $i"
1983                 fi
1984         done
1985         ret=$?
1986         IFS=$OLDIFS
1987         return $ret
1988 }
1989
1990 function validate_trace_count
1991 {
1992         local event_name=$1
1993         local trace_path=$2
1994         local expected_count=$3
1995
1996         which $BABELTRACE_BIN >/dev/null
1997         if [ $? -ne 0 ]; then
1998             skip 0 "Babeltrace binary not found. Skipping trace validation"
1999         fi
2000
2001         cnt=0
2002         OLDIFS=$IFS
2003         IFS=","
2004         for i in $event_name; do
2005                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2006                 if [ "$traced" -ne 0 ]; then
2007                         pass "Validate trace for event $i, $traced events"
2008                 else
2009                         fail "Validate trace for event $i"
2010                         diag "Found $traced occurrences of $i"
2011                 fi
2012                 cnt=$(($cnt + $traced))
2013         done
2014         IFS=$OLDIFS
2015         test $cnt -eq $expected_count
2016         ok $? "Read a total of $cnt events, expected $expected_count"
2017 }
2018
2019 function validate_trace_count_range_incl_min_excl_max
2020 {
2021         local event_name=$1
2022         local trace_path=$2
2023         local expected_min=$3
2024         local expected_max=$4
2025
2026         which $BABELTRACE_BIN >/dev/null
2027         if [ $? -ne 0 ]; then
2028             skip 0 "Babeltrace binary not found. Skipping trace validation"
2029         fi
2030
2031         cnt=0
2032         OLDIFS=$IFS
2033         IFS=","
2034         for i in $event_name; do
2035                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2036                 if [ "$traced" -ge $expected_min ]; then
2037                         pass "Validate trace for event $i, $traced events"
2038                 else
2039                         fail "Validate trace for event $i"
2040                         diag "Found $traced occurrences of $i"
2041                 fi
2042                 cnt=$(($cnt + $traced))
2043         done
2044         IFS=$OLDIFS
2045         test $cnt -lt $expected_max
2046         ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
2047 }
2048
2049 function trace_first_line
2050 {
2051         local trace_path=$1
2052
2053         which $BABELTRACE_BIN >/dev/null
2054         if [ $? -ne 0 ]; then
2055             skip 0 "Babeltrace binary not found. Skipping trace validation"
2056         fi
2057
2058         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
2059 }
2060
2061 function validate_trace_exp()
2062 {
2063         local event_exp=$1
2064         local trace_path=$2
2065
2066         which $BABELTRACE_BIN >/dev/null
2067         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
2068
2069         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
2070         if [ "$traced" -ne 0 ]; then
2071                 pass "Validate trace for expression '${event_exp}', $traced events"
2072         else
2073                 fail "Validate trace for expression '${event_exp}'"
2074                 diag "Found $traced occurrences of '${event_exp}'"
2075         fi
2076         ret=$?
2077         return $ret
2078 }
2079
2080 function validate_trace_only_exp()
2081 {
2082         local event_exp=$1
2083         local trace_path=$2
2084
2085         which $BABELTRACE_BIN >/dev/null
2086         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
2087
2088         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
2089         local total=$($BABELTRACE_BIN $trace_path | wc -l)
2090
2091         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
2092                 pass "Trace match with $total for expression '${event_exp}'"
2093         else
2094                 fail "Trace match"
2095                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
2096         fi
2097         ret=$?
2098         return $ret
2099 }
2100
2101 function validate_trace_empty()
2102 {
2103         local trace_path=$1
2104
2105         which $BABELTRACE_BIN >/dev/null
2106         if [ $? -ne 0 ]; then
2107             skip 0 "Babeltrace binary not found. Skipping trace validation"
2108         fi
2109
2110         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
2111         ret=$?
2112         if [ $ret -ne 0 ]; then
2113                 fail "Failed to parse trace"
2114                 return $ret
2115         fi
2116
2117         traced=$(echo -n "$events" | wc -l)
2118         if [ "$traced" -eq 0 ]; then
2119                 pass "Validate empty trace"
2120         else
2121                 fail "Validate empty trace"
2122                 diag "Found $traced events in trace"
2123         fi
2124         ret=$?
2125         return $ret
2126 }
2127
2128 function validate_directory_empty ()
2129 {
2130         local trace_path="$1"
2131
2132         # Do not double quote `$trace_path` below as we want wildcards to be
2133         # expanded.
2134         files="$(ls -A $trace_path)"
2135         ret=$?
2136         if [ $ret -ne 0 ]; then
2137                 fail "Failed to list content of directory \"$trace_path\""
2138                 return $ret
2139         fi
2140
2141         nb_files="$(echo -n "$files" | wc -l)"
2142         ok $nb_files "Directory \"$trace_path\" is empty"
2143 }
2144
2145 function validate_trace_session_ust_empty()
2146 {
2147         validate_directory_empty "$1"/ust
2148 }
2149
2150 function validate_trace_session_kernel_empty()
2151 {
2152         validate_trace_empty "$1"/kernel
2153 }
2154
2155 function regenerate_metadata ()
2156 {
2157         local expected_to_fail=$1
2158         local sess_name=$2
2159
2160         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2161         ret=$?
2162         if [[ $expected_to_fail -eq "1" ]]; then
2163                 test "$ret" -ne "0"
2164                 ok $? "Expected fail on regenerate metadata $sess_name"
2165         else
2166                 ok $ret "Metadata regenerate $sess_name"
2167         fi
2168 }
2169
2170 function regenerate_metadata_ok ()
2171 {
2172         regenerate_metadata 0 "$@"
2173 }
2174
2175 function regenerate_metadata_fail ()
2176 {
2177         regenerate_metadata 1 "$@"
2178 }
2179
2180 function regenerate_statedump ()
2181 {
2182         local expected_to_fail=$1
2183         local sess_name=$2
2184
2185         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2186         ret=$?
2187         if [[ $expected_to_fail -eq "1" ]]; then
2188                 test "$ret" -ne "0"
2189                 ok $? "Expected fail on regenerate statedump $sess_name"
2190         else
2191                 ok $ret "Statedump regenerate $sess_name"
2192         fi
2193 }
2194
2195 function regenerate_statedump_ok ()
2196 {
2197         regenerate_statedump 0 "$@"
2198 }
2199
2200 function regenerate_statedump_fail ()
2201 {
2202         regenerate_statedump 1 "$@"
2203 }
2204
2205 function rotate_session ()
2206 {
2207         local expected_to_fail=$1
2208         local sess_name=$2
2209
2210         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2211         ret=$?
2212         if [[ $expected_to_fail -eq "1" ]]; then
2213                 test "$ret" -ne "0"
2214                 ok $? "Expected fail on rotate session $sess_name"
2215         else
2216                 ok $ret "Rotate session $sess_name"
2217         fi
2218 }
2219
2220 function rotate_session_ok ()
2221 {
2222         rotate_session 0 "$@"
2223 }
2224
2225 function rotate_session_fail ()
2226 {
2227         rotate_session 1 "$@"
2228 }
2229
2230 function destructive_tests_enabled ()
2231 {
2232         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2233                 return 0
2234         else
2235                 return 1
2236         fi
2237 }
2238
2239 function lttng_enable_rotation_timer ()
2240 {
2241         local expected_to_fail=$1
2242         local sess_name=$2
2243         local period=$3
2244
2245         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2246         ret=$?
2247         if [[ $expected_to_fail -eq "1" ]]; then
2248                 test "$ret" -ne "0"
2249                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2250         else
2251                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2252         fi
2253 }
2254
2255 function lttng_enable_rotation_timer_ok ()
2256 {
2257         lttng_enable_rotation_timer 0 $@
2258 }
2259
2260 function lttng_enable_rotation_timer_fail ()
2261 {
2262         lttng_enable_rotation_timer 1 $@
2263 }
2264
2265 function lttng_enable_rotation_size ()
2266 {
2267         local expected_to_fail=$1
2268         local sess_name=$2
2269         local size=$3
2270
2271         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2272         ret=$?
2273         if [[ $expected_to_fail -eq "1" ]]; then
2274                 test "$ret" -ne "0"
2275                 ok $? "Expected fail on rotate session $sess_name"
2276         else
2277                 ok $ret "Rotate session $sess_name"
2278         fi
2279 }
2280
2281 function lttng_enable_rotation_size_ok ()
2282 {
2283         lttng_enable_rotation_size 0 $@
2284 }
2285
2286 function lttng_enable_rotation_size_fail ()
2287 {
2288         lttng_enable_rotation_size 1 $@
2289 }
2290
2291 function lttng_clear_session ()
2292 {
2293         local expected_to_fail=$1
2294         local sess_name=$2
2295
2296         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2297         ret=$?
2298         if [[ $expected_to_fail -eq "1" ]]; then
2299                 test "$ret" -ne "0"
2300                 ok $? "Expected fail on clear session $sess_name"
2301         else
2302                 ok $ret "Clear session $sess_name"
2303         fi
2304 }
2305
2306 function lttng_clear_session_ok ()
2307 {
2308         lttng_clear_session 0 $@
2309 }
2310
2311 function lttng_clear_session_fail ()
2312 {
2313         lttng_clear_session 1 $@
2314 }
2315
2316 function lttng_clear_all ()
2317 {
2318         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2319         ok $? "Clear all lttng sessions"
2320 }
2321
2322 function lttng_add_trigger()
2323 {
2324         local expected_to_fail="$1"
2325         local trigger_name="$2"
2326         shift 2
2327         local args=("$@")
2328
2329         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2330         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2331         ret=$?
2332         if [[ $expected_to_fail -eq "1" ]]; then
2333                 test "$ret" -ne "0"
2334                 ok $? "Add trigger $trigger_name failed as expected"
2335         else
2336                 ok $ret "Add trigger $trigger_name"
2337         fi
2338 }
2339
2340 function lttng_remove_trigger()
2341 {
2342         local expected_to_fail="$1"
2343         local trigger_name="$2"
2344         shift 2
2345
2346         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2347         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2348         ret=$?
2349         if [[ $expected_to_fail -eq "1" ]]; then
2350                 test "$ret" -ne "0"
2351                 ok $? "Remove trigger $trigger_name failed as expected"
2352         else
2353                 ok $ret "Remove trigger $trigger_name"
2354         fi
2355 }
2356
2357 function lttng_add_trigger_ok()
2358 {
2359         lttng_add_trigger 0 "$@"
2360 }
2361
2362 function lttng_add_trigger_fail()
2363 {
2364         lttng_add_trigger 1 "$@"
2365 }
2366
2367 function lttng_remove_trigger_ok()
2368 {
2369         lttng_remove_trigger 0 "$@"
2370 }
2371
2372 function list_triggers_matches_ok ()
2373 {
2374         local tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2375         local tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2376
2377         local test_name="$1"
2378         local expected_stdout_file="$2"
2379
2380         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2381
2382         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2383         ok $? "${test_name}: exit code is 0"
2384
2385         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2386         ok $? "${test_name}: expected stdout"
2387
2388         diff -u /dev/null "${tmp_stderr}"
2389         ok $? "${test_name}: expected stderr"
2390
2391         rm -f "${tmp_stdout}"
2392         rm -f "${tmp_stderr}"
2393 }
2394
2395 function list_triggers_matches_mi_ok ()
2396 {
2397         local tmp_stdout
2398         local tmp_stdout_raw
2399         local tmp_stderr
2400
2401         local test_name="$1"
2402         local expected_stdout_file="$2"
2403
2404         tmp_stdout_raw=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2405         tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2406         tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2407
2408         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2409
2410         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2411         ok $? "${test_name}: exit code is 0"
2412
2413         # Pretty-fy xml before further test.
2414         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2415
2416         $MI_VALIDATE "${tmp_stdout}"
2417         ok $? "list-trigger mi is valid"
2418
2419         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2420         ok $? "${test_name}: expected stdout"
2421
2422         diff -u /dev/null "${tmp_stderr}"
2423         ok $? "${test_name}: expected stderr"
2424
2425         rm -f "${tmp_stdout}"
2426         rm -f "${tmp_stdout_raw}"
2427         rm -f "${tmp_stderr}"
2428 }
2429
2430 function validate_path_pattern ()
2431 {
2432         local message=$1
2433         local pattern=$2
2434         # Base path is only used in error case and is used to list the content
2435         # of the base path.
2436         local base_path=$3
2437
2438
2439         [ -f $pattern ]
2440         ret=$?
2441         ok $ret "$message"
2442
2443         if [ "$ret" -ne "0" ]; then
2444                 diag "Path pattern expected: $pattern"
2445                 # List the tracepath for more info. We use find as a recursive
2446                 # directory lister.
2447                 diag "The base path content:"
2448                 find "$base_path" -print
2449         fi
2450 }
2451
2452 function validate_trace_path_ust_uid ()
2453 {
2454         local trace_path=$1
2455         local session_name=$2
2456         local uid=$UID
2457         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2458
2459         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2460 }
2461
2462 function validate_trace_path_ust_uid_network ()
2463 {
2464         local trace_path=$1
2465         local session_name=$2
2466         local base_path=$3
2467         local uid=$UID
2468         local hostname=$HOSTNAME
2469         local pattern
2470         local ret
2471
2472         # If the session was given a network base path (e.g
2473         # 127.0.0.1/my/custom/path on creation, there is no session name
2474         # component to the path on the relayd side. Caller can simply not pass a
2475         # session name for this scenario.
2476         if [ -n "$session_name" ]; then
2477                 session_name="$session_name-$date_time_pattern"
2478                 if [ -n "$base_path" ]; then
2479                         fail "Session name and base path are mutually exclusive"
2480                         return
2481                 fi
2482         fi
2483
2484         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2485
2486         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2487 }
2488
2489 function validate_trace_path_ust_uid_snapshot_network ()
2490 {
2491         local trace_path=$1
2492         local session_name=$2
2493         local snapshot_name=$3
2494         local snapshot_number=$4
2495         local base_path=$5
2496         local hostname=$HOSTNAME
2497         local uid=$UID
2498         local pattern
2499         local ret
2500
2501         # If the session/output was given a network base path (e.g
2502         # 127.0.0.1/my/custom/path on creation, there is no session name
2503         # component to the path on the relayd side. Caller can simply not pass a
2504         # session name for this scenario.
2505         if [ -n "$session_name" ]; then
2506                 session_name="$session_name-$date_time_pattern"
2507                 if [ -n "$base_path" ]; then
2508                         fail "Session name and base path are mutually exclusive"
2509                         return
2510                 fi
2511         fi
2512
2513         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2514
2515         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2516 }
2517
2518 function validate_trace_path_ust_uid_snapshot ()
2519 {
2520         local trace_path=$1
2521         local session_name=$2
2522         local snapshot_name=$3
2523         local snapshot_number=$4
2524         local base_path=$5
2525         local uid=$UID
2526         local pattern
2527         local ret
2528
2529         # If the session/output was given a network base path (e.g
2530         # 127.0.0.1/my/custom/path) on creation, there is no session name
2531         # component to the path on the relayd side. Caller can simply not pass a
2532         # session name for this scenario.
2533         if [ -n "$session_name" ]; then
2534                 session_name="$session_name-$date_time_pattern"
2535                 if [ -n "$base_path" ]; then
2536                         fail "Session name and base path are mutually exclusive"
2537                         return
2538                 fi
2539         fi
2540
2541         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2542
2543         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2544 }
2545
2546 function validate_trace_path_ust_pid ()
2547 {
2548         local trace_path=$1
2549         local session_name=$2
2550         local app_string=$3
2551         local pid=$4
2552         local pattern
2553         local ret
2554
2555         # If the session was given a trace path on creation, there is no session
2556         # name component to the path. Caller can simply not pass a session name
2557         # for this scenario.
2558         if [ -n "$session_name" ]; then
2559                 session_name="$session_name-$date_time_pattern"
2560         fi
2561
2562         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2563
2564         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2565 }
2566
2567 function validate_trace_path_kernel ()
2568 {
2569         local trace_path=$1
2570         local session_name=$2
2571         local pattern
2572
2573         # If the session was given a trace path on creation, there is no session
2574         # name component to the path. Caller can simply not pass a session name
2575         # for this scenario.
2576         if [ -n "$session_name" ]; then
2577                 session_name="$session_name-$date_time_pattern"
2578         fi
2579
2580         pattern="$trace_path/$session_name/kernel/metadata"
2581
2582         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2583 }
2584
2585 function validate_trace_path_kernel_network ()
2586 {
2587         local trace_path=$1
2588         local session_name=$2
2589         local hostname=$HOSTNAME
2590         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2591
2592         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2593 }
2594
2595 function validate_trace_path_kernel_snapshot ()
2596 {
2597         local trace_path=$1
2598         local session_name=$2
2599         local snapshot_name=$3
2600         local snapshot_number=$4
2601         local base_path=$5
2602         local pattern
2603         local ret
2604
2605         # If the session/output was given a network base path (e.g
2606         # 127.0.0.1/my/custom/path on creation, there is no session name
2607         # component to the path on the relayd side. Caller can simply not pass a
2608         # session name for this scenario.
2609         if [ -n "$session_name" ]; then
2610                 session_name="$session_name-$date_time_pattern"
2611                 if [ -n "$base_path" ]; then
2612                         fail "Session name and base path are mutually exclusive"
2613                         return
2614                 fi
2615         fi
2616
2617         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2618
2619         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2620 }
2621
2622 function validate_trace_path_kernel_snapshot_network ()
2623 {
2624         local trace_path=$1
2625         local session_name=$2
2626         local snapshot_name=$3
2627         local snapshot_number=$4
2628         local base_path=$5
2629         local hostname=$HOSTNAME
2630         local pattern
2631         local ret
2632
2633         # If the session/output was given a network base path (e.g
2634         # 127.0.0.1/my/custom/path on creation, there is no session name
2635         # component to the path on the relayd side. Caller can simply not pass a
2636         # session name for this scenario.
2637         if [ -n "$session_name" ]; then
2638                 session_name="$session_name-$date_time_pattern"
2639                 if [ -n "$base_path" ]; then
2640                         fail "Session name and base path are mutually exclusive"
2641                         return
2642                 fi
2643         fi
2644
2645         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2646
2647         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2648 }