tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace2"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$(tr '\0' '\n' < /proc/"$pid"/cmdline 2>/dev/null | head -n1); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         local len="${1:-16}"
 208
 209         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 210         # /dev/urandom isn't guaranteed to generate valid multi-byte characters.
 211         # Specifying the C locale eliminates the "Illegal byte sequence" error
 212         # that 'tr' outputs in such cases.
 213         LC_CTYPE=C tr -cd "$CHAR" < /dev/urandom 2>/dev/null | head -c "$len" 2>/dev/null
 214         echo
 215 }
 216
 217 # Helpers for get_possible_cpus.
 218 function get_possible_cpus_count_from_sysfs_possible_mask()
 219 {
 220         local max_possible_cpu_id=$(cut -d '-' -f 2 < /sys/devices/system/cpu/possible)
 221         echo $((max_possible_cpu_id+1))
 222 }
 223
 224 function get_max_cpus_count_from_sysfs_cpu_directories()
 225 {
 226         local max_possible_cpu_id= \
 227                 $(find /sys/devices/system/cpu/ -mindepth 1 -maxdepth 1 -regex ".+cpu[0-9]+" | \
 228                   sed -e 's/cpu//g' | \
 229                   awk -F '/' '{ if ($NF > N) N = $NF } END { print N }')
 230         echo $((max_possible_cpu_id+1))
 231 }
 232
 233 # Return the number of possible CPUs.
 234 function get_possible_cpus_count()
 235 {
 236         local possible_cpus_count=$(get_possible_cpus_count_from_sysfs_possible_mask)
 237
 238         if [ $? -ne 0 ]; then
 239                 possible_cpus_count=$(get_max_cpus_count_from_sysfs_cpu_directories)
 240                 local configured_cpus_count=$(getconf _NPROCESSORS_CONF)
 241                 possible_cpus_count=$(($configured_cpus_count > $possible_cpus_count \
 242                                                               ? $configured_cpus_count \
 243                                                               : $possible_cpus_count))
 244         fi
 245
 246         echo $possible_cpus_count
 247 }
 248
 249 # Return the number of _configured_ CPUs.
 250 function conf_proc_count()
 251 {
 252         getconf _NPROCESSORS_CONF
 253         if [ $? -ne 0 ]; then
 254                 diag "Failed to get the number of configured CPUs"
 255         fi
 256         echo
 257 }
 258
 259 # Check if base lttng-modules are present.
 260 # Bail out on failure
 261 function validate_lttng_modules_present ()
 262 {
 263         # Check for loadable modules.
 264         modprobe -n lttng-tracer 2>/dev/null
 265         if [ $? -eq 0 ]; then
 266                 return 0
 267         fi
 268
 269         # Check for builtin modules.
 270         ls /proc/lttng > /dev/null 2>&1
 271         if [ $? -eq 0 ]; then
 272                 return 0
 273         fi
 274
 275         LTTNG_BAIL_OUT "LTTng modules not detected."
 276 }
 277
 278 # Run the lttng binary.
 279 #
 280 # The first two arguments are stdout and stderr redirect paths, respectively.
 281 # The rest of the arguments are forwarded to the lttng binary
 282 function _run_lttng_cmd
 283 {
 284         local stdout_dest="$1"
 285         local stderr_dest="$2"
 286         shift 2
 287
 288         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 289         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 290 }
 291
 292 function enable_kernel_lttng_event
 293 {
 294         local withtap="$1"
 295         local expected_to_fail="$2"
 296         local sess_name="$3"
 297         local event_name="$4"
 298         local channel_name="$5"
 299
 300         if [ -z "$event_name" ]; then
 301                 # Enable all event if no event name specified
 302                 event_name="-a"
 303         fi
 304
 305         if [ -z "$channel_name" ]; then
 306                 # default channel if none specified
 307                 chan=""
 308         else
 309                 chan="-c $channel_name"
 310         fi
 311
 312         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 313                 enable-event "$event_name" $chan -s $sess_name -k
 314         ret=$?
 315         if [[ $expected_to_fail -eq "1" ]]; then
 316                 test $ret -ne "0"
 317                 ret=$?
 318                 if [ $withtap -eq "1" ]; then
 319                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 320                 fi
 321         else
 322                 if [ $withtap -eq "1" ]; then
 323                         ok $ret "Enable kernel event $event_name for session $sess_name"
 324                 fi
 325         fi
 326 }
 327
 328 function enable_kernel_lttng_event_ok ()
 329 {
 330         enable_kernel_lttng_event 1 0 "$@"
 331 }
 332
 333 function enable_kernel_lttng_event_fail ()
 334 {
 335         enable_kernel_lttng_event 1 1 "$@"
 336 }
 337
 338 function enable_kernel_lttng_event_notap ()
 339 {
 340         enable_kernel_lttng_event 0 0 "$@"
 341 }
 342
 343 # Old interface
 344 function lttng_enable_kernel_event
 345 {
 346         enable_kernel_lttng_event_ok "$@"
 347 }
 348
 349 function lttng_enable_kernel_syscall()
 350 {
 351         local expected_to_fail=$1
 352         local sess_name=$2
 353         local syscall_name=$3
 354         local channel_name=$4
 355
 356         if [ -z $syscall_name ]; then
 357                 # Enable all event if no syscall name specified
 358                 syscall_name="-a"
 359         fi
 360
 361         if [ -z $channel_name ]; then
 362                 # default channel if none specified
 363                 chan=""
 364         else
 365                 chan="-c $channel_name"
 366         fi
 367
 368         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 369                 enable-event --syscall "$syscall_name" $chan -s $sess_name -k
 370         ret=$?
 371         if [[ $expected_to_fail -eq "1" ]]; then
 372                 test $ret -ne "0"
 373                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 374         else
 375                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 376         fi
 377 }
 378
 379 function lttng_enable_kernel_syscall_ok()
 380 {
 381         lttng_enable_kernel_syscall 0 "$@"
 382 }
 383
 384 function lttng_enable_kernel_syscall_fail()
 385 {
 386         lttng_enable_kernel_syscall 1 "$@"
 387 }
 388
 389 function lttng_disable_kernel_syscall()
 390 {
 391         local expected_to_fail=$1
 392         local sess_name=$2
 393         local syscall_name=$3
 394         local channel_name=$4
 395
 396         if [ -z $syscall_name ]; then
 397                 # Enable all event if no syscall name specified
 398                 syscall_name="-a"
 399         fi
 400
 401         if [ -z $channel_name ]; then
 402                 # default channel if none specified
 403                 chan=""
 404         else
 405                 chan="-c $channel_name"
 406         fi
 407
 408         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 409                 disable-event --syscall "$syscall_name" $chan -s $sess_name -k
 410
 411         ret=$?
 412         if [[ $expected_to_fail -eq "1" ]]; then
 413                 test $ret -ne "0"
 414                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 415         else
 416                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 417         fi
 418 }
 419
 420 function lttng_disable_kernel_syscall_ok()
 421 {
 422         lttng_disable_kernel_syscall 0 "$@"
 423 }
 424
 425 function lttng_disable_kernel_syscall_fail()
 426 {
 427         lttng_disable_kernel_syscall 1 "$@"
 428 }
 429
 430 function lttng_enable_kernel_function_event ()
 431 {
 432         local expected_to_fail="$1"
 433         local sess_name="$2"
 434         local target="$3"
 435         local event_name="$4"
 436
 437         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 438         ret=$?
 439         if [[ $expected_to_fail -eq "1" ]]; then
 440                 test $ret -ne "0"
 441                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 442         else
 443                 ok $ret "Enable kernel function event for session $sess_name"
 444         fi
 445 }
 446
 447 function lttng_enable_kernel_function_event_ok ()
 448 {
 449         lttng_enable_kernel_function_event 0 "$@"
 450 }
 451
 452 function lttng_enable_kernel_userspace_probe_event ()
 453 {
 454         local expected_to_fail="$1"
 455         local sess_name="$2"
 456         local target="$3"
 457         local event_name="$4"
 458
 459         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 460         ret=$?
 461         if [[ $expected_to_fail -eq "1" ]]; then
 462                 test $ret -ne "0"
 463                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 464         else
 465                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 466         fi
 467 }
 468
 469 function lttng_enable_kernel_userspace_probe_event_fail ()
 470 {
 471         lttng_enable_kernel_userspace_probe_event 1 "$@"
 472 }
 473
 474 function lttng_enable_kernel_userspace_probe_event_ok ()
 475 {
 476         lttng_enable_kernel_userspace_probe_event 0 "$@"
 477 }
 478
 479 function disable_kernel_lttng_userspace_probe_event_ok ()
 480 {
 481         local sess_name="$1"
 482         local event_name="$2"
 483
 484         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 485         ok $? "Disable kernel event $target for session $sess_name"
 486 }
 487 function lttng_enable_kernel_channel()
 488 {
 489         local withtap=$1
 490         local expected_to_fail=$2
 491         local sess_name=$3
 492         local channel_name=$4
 493         local opts="${@:5}"
 494
 495         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 496                 enable-channel -k $channel_name -s $sess_name $opts
 497         ret=$?
 498         if [[ $expected_to_fail -eq "1" ]]; then
 499                 test "$ret" -ne "0"
 500                 ret=$?
 501                 if [ $withtap -eq "1" ]; then
 502                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 503                 fi
 504         else
 505                 if [ $withtap -eq "1" ]; then
 506                         ok $ret "Enable channel $channel_name for session $sess_name"
 507                 fi
 508         fi
 509 }
 510
 511 function lttng_enable_kernel_channel_ok()
 512 {
 513         lttng_enable_kernel_channel 1 0 "$@"
 514 }
 515
 516 function lttng_enable_kernel_channel_fail()
 517 {
 518         lttng_enable_kernel_channel 1 1 "$@"
 519 }
 520
 521 function lttng_enable_kernel_channel_notap()
 522 {
 523         lttng_enable_kernel_channel 0 0 "$@"
 524 }
 525
 526 function enable_kernel_lttng_channel_ok()
 527 {
 528         lttng_enable_kernel_channel 1 0 "$@"
 529 }
 530
 531 function lttng_disable_kernel_channel()
 532 {
 533         local expected_to_fail=$1
 534         local sess_name=$2
 535         local channel_name=$3
 536
 537         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 538                 disable-channel -k $channel_name -s $sess_name
 539         ret=$?
 540         if [[ $expected_to_fail -eq "1" ]]; then
 541                 test "$ret" -ne "0"
 542                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 543         else
 544                 ok $ret "Disable channel $channel_name for session $sess_name"
 545         fi
 546 }
 547
 548 function lttng_disable_kernel_channel_ok()
 549 {
 550         lttng_disable_kernel_channel 0 "$@"
 551 }
 552
 553 function lttng_disable_kernel_channel_fail()
 554 {
 555         lttng_disable_kernel_channel 1 "$@"
 556 }
 557
 558 function start_lttng_relayd_opt()
 559 {
 560         local withtap=$1
 561         local process_mode=$2
 562         local opt=$3
 563
 564         DIR=$(readlink -f "$TESTDIR")
 565
 566         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 567                 # shellcheck disable=SC2086
 568                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 569                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 570                 if [ $? -eq 1 ]; then
 571                         if [ $withtap -eq "1" ]; then
 572                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 573                         fi
 574                         return 1
 575                 else
 576                         if [ $withtap -eq "1" ]; then
 577                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 578                         fi
 579                 fi
 580         else
 581                 pass "Start lttng-relayd (opt: $opt)"
 582         fi
 583 }
 584
 585 function start_lttng_relayd()
 586 {
 587         start_lttng_relayd_opt 1 "-b" "$@"
 588 }
 589
 590 function start_lttng_relayd_notap()
 591 {
 592         start_lttng_relayd_opt 0 "-b" "$@"
 593 }
 594
 595 function stop_lttng_relayd_opt()
 596 {
 597         local withtap=$1
 598         local is_cleanup=$2
 599         local signal=$3
 600         local timeout_s=$4
 601         local dtimeleft_s=
 602         local retval=0
 603         local pids
 604
 605         if [ -z "$signal" ]; then
 606                 signal="SIGTERM"
 607         fi
 608
 609
 610         # Multiply time by 2 to simplify integer arithmetic
 611         if [ -n "$timeout_s" ]; then
 612                 dtimeleft_s=$((timeout_s * 2))
 613         fi
 614
 615
 616         pids=$(lttng_pgrep "$RELAYD_MATCH")
 617         if [ -z "$pids" ]; then
 618                 if [ "$is_cleanup" -eq 1 ]; then
 619                         :
 620                 elif [ "$withtap" -eq "1" ]; then
 621                         fail "No relay daemon to kill"
 622                 else
 623                         LTTNG_BAIL_OUT "No relay daemon to kill"
 624                 fi
 625                 return 0
 626         fi
 627
 628         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 629
 630         # shellcheck disable=SC2086
 631         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 632                 retval=1
 633                 if [ "$withtap" -eq "1" ]; then
 634                         fail "Kill relay daemon"
 635                 fi
 636         else
 637                 out=1
 638                 while [ -n "$out" ]; do
 639                         out=$(lttng_pgrep "$RELAYD_MATCH")
 640                         if [ -n "$dtimeleft_s" ]; then
 641                                 if [ $dtimeleft_s -lt 0 ]; then
 642                                         out=
 643                                         retval=1
 644                                 fi
 645                                 dtimeleft_s=$((dtimeleft_s - 1))
 646                         fi
 647                         sleep 0.5
 648                 done
 649                 if [ "$withtap" -eq "1" ]; then
 650                         if [ "$retval" -eq "0" ]; then
 651                                 pass "Wait after kill relay daemon"
 652                         else
 653                                 fail "Wait after kill relay daemon"
 654                         fi
 655                 fi
 656         fi
 657         return $retval
 658 }
 659
 660 function stop_lttng_relayd()
 661 {
 662         stop_lttng_relayd_opt 1 0 "$@"
 663 }
 664
 665 function stop_lttng_relayd_notap()
 666 {
 667         stop_lttng_relayd_opt 0 0 "$@"
 668 }
 669
 670 function stop_lttng_relayd_cleanup()
 671 {
 672         stop_lttng_relayd_opt 0 1 "$@"
 673 }
 674
 675 #First arg: show tap output
 676 #Second argument: load path for automatic loading
 677 function start_lttng_sessiond_opt()
 678 {
 679         local withtap=$1
 680         local load_path=$2
 681
 682         # The rest of the arguments will be passed directly to lttng-sessiond.
 683         shift 2
 684
 685         local env_vars=""
 686         local consumerd=""
 687
 688         local long_bit_value=
 689         long_bit_value=$(getconf LONG_BIT)
 690
 691         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 692                 # Env variable requested no session daemon
 693                 return
 694         fi
 695
 696         DIR=$(readlink -f "$TESTDIR")
 697
 698         # Get long_bit value for 32/64 consumerd
 699         case "$long_bit_value" in
 700                 32)
 701                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 702                         ;;
 703                 64)
 704                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 705                         ;;
 706                 *)
 707                         return
 708                         ;;
 709         esac
 710
 711         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 712         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 713                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 714         fi
 715         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 716
 717         if ! validate_kernel_version; then
 718             fail "Start session daemon"
 719             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 720         fi
 721
 722         diag "export LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/"
 723         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/"}"
 724         export LTTNG_SESSION_CONFIG_XSD_PATH
 725
 726         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 727                 # Have a load path ?
 728                 if [ -n "$load_path" ]; then
 729                         diag "env $env_vars --load $load_path --background $consumerd $@"
 730                         # shellcheck disable=SC2086
 731                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 732                 else
 733                         diag "env $env_vars --background $consumerd $@"
 734                         # shellcheck disable=SC2086
 735                         env $env_vars --background "$consumerd" "$@"
 736                 fi
 737                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 738                 status=$?
 739                 if [ "$withtap" -eq "1" ]; then
 740                         ok $status "Start session daemon"
 741                 fi
 742         fi
 743 }
 744
 745 function start_lttng_sessiond()
 746 {
 747         start_lttng_sessiond_opt 1 "$@"
 748 }
 749
 750 function start_lttng_sessiond_notap()
 751 {
 752         start_lttng_sessiond_opt 0 "$@"
 753 }
 754
 755 function stop_lttng_sessiond_opt()
 756 {
 757         local withtap=$1
 758         local is_cleanup=$2
 759         local signal=$3
 760         local timeout_s=$4
 761         local dtimeleft_s=
 762         local retval=0
 763         local runas_pids
 764         local pids
 765
 766         if [ -z "$signal" ]; then
 767                 signal=SIGTERM
 768         fi
 769
 770         # Multiply time by 2 to simplify integer arithmetic
 771         if [ -n "$timeout_s" ]; then
 772                 dtimeleft_s=$((timeout_s * 2))
 773         fi
 774
 775         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 776                 # Env variable requested no session daemon
 777                 return 0
 778         fi
 779
 780         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 781         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 782
 783         if [ -n "$runas_pids" ]; then
 784                 pids="$pids $runas_pids"
 785         fi
 786
 787         if [ -z "$pids" ]; then
 788                 if [ "$is_cleanup" -eq 1 ]; then
 789                         :
 790                 elif [ "$withtap" -eq "1" ]; then
 791                         fail "No session daemon to kill"
 792                 else
 793                         LTTNG_BAIL_OUT "No session daemon to kill"
 794                 fi
 795                 return 0
 796         fi
 797
 798         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 799
 800         # shellcheck disable=SC2086
 801         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 802                 retval=1
 803                 if [ "$withtap" -eq "1" ]; then
 804                         fail "Kill sessions daemon"
 805                 fi
 806         else
 807                 out=1
 808                 while [ -n "$out" ]; do
 809                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 810                         if [ -n "$dtimeleft_s" ]; then
 811                                 if [ $dtimeleft_s -lt 0 ]; then
 812                                         out=
 813                                         retval=1
 814                                 fi
 815                                 dtimeleft_s=$((dtimeleft_s - 1))
 816                         fi
 817                         sleep 0.5
 818                 done
 819                 out=1
 820                 while [ -n "$out" ]; do
 821                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 822                         if [ -n "$dtimeleft_s" ]; then
 823                                 if [ $dtimeleft_s -lt 0 ]; then
 824                                         out=
 825                                         retval=1
 826                                 fi
 827                                 dtimeleft_s=$((dtimeleft_s - 1))
 828                         fi
 829                         sleep 0.5
 830                 done
 831
 832                 if [ "$withtap" -eq "1" ]; then
 833                         if [ "$retval" -eq "0" ]; then
 834                                 pass "Wait after kill session daemon"
 835                         else
 836                                 fail "Wait after kill session daemon"
 837                         fi
 838                 fi
 839         fi
 840         if [ "$signal" = "SIGKILL" ]; then
 841                 if [ "$(id -u)" -eq "0" ]; then
 842                         local modules=
 843                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 844
 845                         if [ -n "$modules" ]; then
 846                                 diag "Unloading all LTTng modules"
 847                                 modprobe --remove "$modules"
 848                         fi
 849                 fi
 850         fi
 851
 852         return $retval
 853 }
 854
 855 function stop_lttng_sessiond()
 856 {
 857         stop_lttng_sessiond_opt 1 0 "$@"
 858 }
 859
 860 function stop_lttng_sessiond_notap()
 861 {
 862         stop_lttng_sessiond_opt 0 0 "$@"
 863 }
 864
 865 function stop_lttng_sessiond_cleanup()
 866 {
 867         stop_lttng_sessiond_opt 0 1 "$@"
 868 }
 869
 870 function sigstop_lttng_sessiond_opt()
 871 {
 872         local withtap=$1
 873         local signal=SIGSTOP
 874         local pids
 875
 876         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 877                 # Env variable requested no session daemon
 878                 return
 879         fi
 880
 881         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 882
 883         if [ "$withtap" -eq "1" ]; then
 884                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 885         fi
 886
 887         # shellcheck disable=SC2086
 888         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 889                 if [ "$withtap" -eq "1" ]; then
 890                         fail "Sending SIGSTOP to session daemon"
 891                 fi
 892         else
 893                 out=1
 894                 while [ $out -ne 0 ]; do
 895                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 896
 897                         # Wait until state becomes stopped for session
 898                         # daemon(s).
 899                         out=0
 900                         for sessiond_pid in $pids; do
 901                                 state="$(ps -p "$sessiond_pid" -o state= )"
 902                                 if [[ -n "$state" && "$state" != "T" ]]; then
 903                                         out=1
 904                                 fi
 905                         done
 906                         sleep 0.5
 907                 done
 908                 if [ "$withtap" -eq "1" ]; then
 909                         pass "Sending SIGSTOP to session daemon"
 910                 fi
 911         fi
 912 }
 913
 914 function sigstop_lttng_sessiond()
 915 {
 916         sigstop_lttng_sessiond_opt 1 "$@"
 917 }
 918
 919 function sigstop_lttng_sessiond_notap()
 920 {
 921         sigstop_lttng_sessiond_opt 0 "$@"
 922 }
 923
 924 function stop_lttng_consumerd_opt()
 925 {
 926         local withtap=$1
 927         local is_cleanup=$2
 928         local signal=$3
 929         local timeout_s=$4
 930         local dtimeleft_s=
 931         local retval=0
 932         local pids
 933
 934         if [ -z "$signal" ]; then
 935                 signal=SIGTERM
 936         fi
 937
 938         # Multiply time by 2 to simplify integer arithmetic
 939         if [ -n "$timeout_s" ]; then
 940                 dtimeleft_s=$((timeout_s * 2))
 941         fi
 942
 943         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 944
 945         if [ -z "$pids" ]; then
 946                 if [ "$is_cleanup" -eq 1 ]; then
 947                         :
 948                 elif [ "$withtap" -eq "1" ]; then
 949                         fail "No consumerd daemon to kill"
 950                 else
 951                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
 952                 fi
 953                 return 0
 954         fi
 955
 956         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 957
 958         # shellcheck disable=SC2086
 959         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 960                 retval=1
 961                 if [ "$withtap" -eq "1" ]; then
 962                         fail "Kill consumer daemon"
 963                 fi
 964         else
 965                 out=1
 966                 while [ $out -ne 0 ]; do
 967                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 968
 969                         # If consumerds are still present check their status.
 970                         # A zombie status qualifies the consumerd as *killed*
 971                         out=0
 972                         for consumer_pid in $pids; do
 973                                 state="$(ps -p "$consumer_pid" -o state= )"
 974                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 975                                         out=1
 976                                 fi
 977                         done
 978                         if [ -n "$dtimeleft_s" ]; then
 979                                 if [ $dtimeleft_s -lt 0 ]; then
 980                                         out=0
 981                                         retval=1
 982                                 fi
 983                                 dtimeleft_s=$((dtimeleft_s - 1))
 984                         fi
 985                         sleep 0.5
 986                 done
 987                 if [ "$withtap" -eq "1" ]; then
 988                         if [ "$retval" -eq "0" ]; then
 989                                 pass "Wait after kill consumer daemon"
 990                         else
 991                                 fail "Wait after kill consumer daemon"
 992                         fi
 993                 fi
 994         fi
 995
 996         return $retval
 997 }
 998
 999 function stop_lttng_consumerd()
1000 {
1001         stop_lttng_consumerd_opt 1 0 "$@"
1002 }
1003
1004 function stop_lttng_consumerd_notap()
1005 {
1006         stop_lttng_consumerd_opt 0 0 "$@"
1007 }
1008
1009 function stop_lttng_consumerd_cleanup()
1010 {
1011         stop_lttng_consumerd_opt 0 1 "$@"
1012 }
1013
1014 function sigstop_lttng_consumerd_opt()
1015 {
1016         local withtap=$1
1017         local signal=SIGSTOP
1018         local pids
1019
1020         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1021
1022         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
1023
1024         # shellcheck disable=SC2086
1025         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1026         retval=$?
1027
1028         if [ $retval -eq 1 ]; then
1029                 if [ "$withtap" -eq "1" ]; then
1030                         fail "Sending SIGSTOP to consumer daemon"
1031                 fi
1032                 return 1
1033         else
1034                 out=1
1035                 while [ $out -ne 0 ]; do
1036                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1037
1038                         # Wait until state becomes stopped for all
1039                         # consumers.
1040                         out=0
1041                         for consumer_pid in $pids; do
1042                                 state="$(ps -p "$consumer_pid" -o state= )"
1043                                 if [[ -n "$state" && "$state" != "T" ]]; then
1044                                         out=1
1045                                 fi
1046                         done
1047                         sleep 0.5
1048                 done
1049                 if [ "$withtap" -eq "1" ]; then
1050                         pass "Sending SIGSTOP to consumer daemon"
1051                 fi
1052         fi
1053         return $retval
1054 }
1055
1056 function sigstop_lttng_consumerd()
1057 {
1058         sigstop_lttng_consumerd_opt 1 "$@"
1059 }
1060
1061 function sigstop_lttng_consumerd_notap()
1062 {
1063         sigstop_lttng_consumerd_opt 0 "$@"
1064 }
1065
1066 function list_lttng_with_opts ()
1067 {
1068         local ret
1069         local withtap=$1
1070         shift
1071         local opts=$1
1072         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1073                 list $opts
1074         ret=$?
1075         if [ $withtap -eq "1" ]; then
1076                 ok $ret "Lttng-tool list command with option $opts"
1077         fi
1078 }
1079
1080 function list_lttng_ok ()
1081 {
1082         list_lttng_with_opts 1 "$@"
1083 }
1084
1085 function list_lttng_notap ()
1086 {
1087         list_lttng_with_opts 0 "$@"
1088 }
1089
1090 function create_lttng_session_no_output ()
1091 {
1092         local sess_name=$1
1093         local opts="${@:2}"
1094
1095         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1096                 create $sess_name --no-output $opts
1097         ok $? "Create session $sess_name in no-output mode"
1098 }
1099
1100 function create_lttng_session_uri () {
1101         local sess_name=$1
1102         local uri=$2
1103         local opts="${@:3}"
1104
1105         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1106                 create $sess_name -U $uri $opts
1107         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1108 }
1109
1110 function create_lttng_session ()
1111 {
1112         local withtap=$1
1113         local expected_to_fail=$2
1114         local sess_name=$3
1115         local trace_path=$4
1116         local opt=$5
1117
1118         if [ -z "$trace_path" ]; then
1119                 # Use lttng-sessiond default output.
1120                 trace_path=""
1121         else
1122                 trace_path="-o $trace_path"
1123         fi
1124
1125         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1126                 create "$sess_name" $trace_path $opt
1127         ret=$?
1128         if [ $expected_to_fail -eq "1" ]; then
1129                 test "$ret" -ne "0"
1130                 ret=$?
1131                 if [ $withtap -eq "1" ]; then
1132                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1133                 fi
1134         else
1135                 if [ $withtap -eq "1" ]; then
1136                         ok $ret "Create session $sess_name in $trace_path"
1137                 fi
1138         fi
1139         return $ret
1140 }
1141
1142 function create_lttng_session_ok ()
1143 {
1144         create_lttng_session 1 0 "$@"
1145 }
1146
1147 function create_lttng_session_fail ()
1148 {
1149         create_lttng_session 1 1 "$@"
1150 }
1151
1152 function create_lttng_session_notap ()
1153 {
1154         create_lttng_session 0 0 "$@"
1155 }
1156
1157
1158 function enable_ust_lttng_channel ()
1159 {
1160         local withtap=$1
1161         local expected_to_fail=$2
1162         local sess_name=$3
1163         local channel_name=$4
1164         local opts="${@:5}"
1165
1166         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1167                 enable-channel -u $channel_name -s $sess_name $opts
1168         ret=$?
1169         if [[ $expected_to_fail -eq "1" ]]; then
1170                 test "$ret" -ne "0"
1171                 ret=$?
1172                 if [ $withtap -eq "1" ]; then
1173                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1174                 fi
1175         else
1176                 if [ $withtap -eq "1" ]; then
1177                         ok $ret "Enable channel $channel_name for session $sess_name"
1178                 fi
1179         fi
1180         return $ret
1181 }
1182
1183 function enable_ust_lttng_channel_ok ()
1184 {
1185         enable_ust_lttng_channel 1 0 "$@"
1186 }
1187
1188 function enable_ust_lttng_channel_fail ()
1189 {
1190         enable_ust_lttng_channel 1 1 "$@"
1191 }
1192
1193 function enable_ust_lttng_channel_notap ()
1194 {
1195         enable_ust_lttng_channel 0 0 "$@"
1196 }
1197
1198 function disable_ust_lttng_channel()
1199 {
1200         local sess_name=$1
1201         local channel_name=$2
1202
1203         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1204                 disable-channel -u $channel_name -s $sess_name
1205         ok $? "Disable channel $channel_name for session $sess_name"
1206 }
1207
1208 function enable_lttng_mmap_overwrite_kernel_channel()
1209 {
1210         local sess_name=$1
1211         local channel_name=$2
1212
1213         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1214                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite
1215         ok $? "Enable channel $channel_name for session $sess_name"
1216 }
1217
1218 function enable_lttng_mmap_discard_small_kernel_channel()
1219 {
1220         local sess_name=$1
1221         local channel_name=$2
1222
1223         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1224                 enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1225         ok $? "Enable small discard channel $channel_name for session $sess_name"
1226 }
1227
1228 function enable_lttng_mmap_overwrite_small_kernel_channel()
1229 {
1230         local sess_name=$1
1231         local channel_name=$2
1232
1233         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1234                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1235         ok $? "Enable small discard channel $channel_name for session $sess_name"
1236 }
1237
1238 function enable_lttng_mmap_overwrite_ust_channel()
1239 {
1240         local sess_name=$1
1241         local channel_name=$2
1242
1243         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1244                 enable-channel -s $sess_name $channel_name -u --output mmap --overwrite
1245         ok $? "Enable channel $channel_name for session $sess_name"
1246 }
1247
1248 function enable_ust_lttng_event ()
1249 {
1250         local withtap=$1
1251         local expected_to_fail=$2
1252         local sess_name=$3
1253         local event_name="$4"
1254         local channel_name=$5
1255
1256         if [ -z $channel_name ]; then
1257                 # default channel if none specified
1258                 chan=""
1259         else
1260                 chan="-c $channel_name"
1261         fi
1262
1263         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1264                 enable-event "$event_name" $chan -s "$sess_name" -u
1265         ret=$?
1266         if [[ $expected_to_fail -eq "1" ]]; then
1267                 test $ret -ne "0"
1268                 ret=$?
1269                 if [[ $withtap -eq "1" ]]; then
1270                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1271                 fi
1272         else
1273                 if [[ $withtap -eq "1" ]]; then
1274                         ok $ret "Enable ust event $event_name for session $sess_name"
1275                 fi
1276         fi
1277         return $ret
1278 }
1279
1280 function enable_ust_lttng_event_ok ()
1281 {
1282         enable_ust_lttng_event 1 0 "$@"
1283 }
1284
1285 function enable_ust_lttng_event_fail ()
1286 {
1287         enable_ust_lttng_event 1 1 "$@"
1288 }
1289
1290 function enable_ust_lttng_event_notap ()
1291 {
1292         enable_ust_lttng_event 0 0 "$@"
1293 }
1294
1295 function enable_jul_lttng_event()
1296 {
1297         sess_name=$1
1298         event_name="$2"
1299         channel_name=$3
1300
1301         if [ -z $channel_name ]; then
1302                 # default channel if none specified
1303                 chan=""
1304         else
1305                 chan="-c $channel_name"
1306         fi
1307
1308         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1309                 enable-event "$event_name" $chan -s $sess_name -j
1310         ok $? "Enable JUL event $event_name for session $sess_name"
1311 }
1312
1313 function enable_jul_lttng_event_loglevel()
1314 {
1315         local sess_name=$1
1316         local event_name="$2"
1317         local loglevel=$3
1318         local channel_name=$4
1319
1320         if [ -z $channel_name ]; then
1321                 # default channel if none specified
1322                 chan=""
1323         else
1324                 chan="-c $channel_name"
1325         fi
1326
1327         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1328                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j
1329         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1330 }
1331
1332 function enable_log4j_lttng_event()
1333 {
1334         local sess_name=$1
1335         local event_name=$2
1336         local channel_name=$3
1337
1338         local chan_opt=()
1339
1340         # default channel if none specified
1341         if [ -n "$channel_name" ]; then
1342                 chan_opt=("-c" "$channel_name")
1343         fi
1344
1345         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1346                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1347         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1348 }
1349
1350 function enable_log4j_lttng_event_filter()
1351 {
1352         local sess_name=$1
1353         local event_name=$2
1354         local filter=$3
1355
1356         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1357                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1358         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1359 }
1360
1361 function enable_log4j_lttng_event_filter_loglevel_only()
1362 {
1363         local sess_name=$1
1364         local event_name=$2
1365         local filter=$3
1366         local loglevel=$4
1367
1368         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1369                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1370         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1371 }
1372
1373 function enable_log4j_lttng_event_loglevel()
1374 {
1375         local sess_name=$1
1376         local event_name=$2
1377         local loglevel=$3
1378         local channel_name=$4
1379
1380
1381         # default channel if none specified
1382         if [ -n "$channel_name" ]; then
1383                 chan_opt=("-c" "$channel_name")
1384         fi
1385
1386         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1387                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1388         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1389 }
1390
1391 function enable_log4j_lttng_event_loglevel_only()
1392 {
1393         local sess_name=$1
1394         local event_name=$2
1395         local loglevel=$3
1396         local channel_name=$4
1397
1398         local chan_opt=()
1399
1400         # default channel if none specified
1401         if [ -n "$channel_name" ]; then
1402                 chan_opt=("-c" "$channel_name")
1403         fi
1404
1405         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1406                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1407         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1408 }
1409
1410 function enable_python_lttng_event()
1411 {
1412         sess_name=$1
1413         event_name="$2"
1414         channel_name=$3
1415
1416         if [ -z $channel_name ]; then
1417                 # default channel if none specified
1418                 chan=""
1419         else
1420                 chan="-c $channel_name"
1421         fi
1422
1423         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1424                 enable-event "$event_name" $chan -s $sess_name -p
1425         ok $? "Enable Python event $event_name for session $sess_name"
1426 }
1427
1428 function enable_python_lttng_event_loglevel()
1429 {
1430         local sess_name=$1
1431         local event_name="$2"
1432         local loglevel=$3
1433         local channel_name=$4
1434
1435         if [ -z $channel_name ]; then
1436                 # default channel if none specified
1437                 chan=""
1438         else
1439                 chan="-c $channel_name"
1440         fi
1441
1442         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1443                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p
1444         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1445 }
1446
1447 function enable_ust_lttng_event_filter()
1448 {
1449         local sess_name="$1"
1450         local event_name="$2"
1451         local filter="$3"
1452         local channel_name=$4
1453
1454         if [ -z $channel_name ]; then
1455                 # default channel if none specified
1456                 chan=""
1457         else
1458                 chan="-c $channel_name"
1459         fi
1460
1461         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1462                 enable-event $chan "$event_name" -s $sess_name -u --filter "$filter"
1463         ok $? "Enable event $event_name with filtering for session $sess_name"
1464 }
1465
1466 function enable_ust_lttng_event_loglevel()
1467 {
1468         local sess_name="$1"
1469         local event_name="$2"
1470         local loglevel="$3"
1471
1472         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1473                 enable-event "$event_name" -s $sess_name -u --loglevel $loglevel
1474         ok $? "Enable event $event_name with loglevel $loglevel"
1475 }
1476
1477 function enable_ust_lttng_event_loglevel_only()
1478 {
1479         local sess_name="$1"
1480         local event_name="$2"
1481         local loglevel="$3"
1482
1483         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1484                 enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel
1485         ok $? "Enable event $event_name with loglevel-only $loglevel"
1486 }
1487
1488 function disable_ust_lttng_event ()
1489 {
1490         local sess_name="$1"
1491         local event_name="$2"
1492         local channel_name="$3"
1493
1494         if [ -z $channel_name ]; then
1495                 # default channel if none specified
1496                 chan=""
1497         else
1498                 chan="-c $channel_name"
1499         fi
1500
1501         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1502                 disable-event "$event_name" -s $sess_name $chan -u
1503         ok $? "Disable event $event_name for session $sess_name"
1504 }
1505
1506 function disable_jul_lttng_event ()
1507 {
1508         local sess_name="$1"
1509         local event_name="$2"
1510
1511         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1512         ok $? "Disable JUL event $event_name for session $sess_name"
1513 }
1514
1515 function disable_log4j_lttng_event ()
1516 {
1517         local sess_name="$1"
1518         local event_name="$2"
1519
1520         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1521                 disable-event "$event_name" -s "$sess_name" --log4j
1522         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1523 }
1524
1525 function disable_python_lttng_event ()
1526 {
1527         local sess_name="$1"
1528         local event_name="$2"
1529
1530         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1531                 disable-event "$event_name" -s $sess_name -p
1532         ok $? "Disable Python event $event_name for session $sess_name"
1533 }
1534
1535 function start_lttng_tracing_opt ()
1536 {
1537         local withtap=$1
1538         local expected_to_fail=$2
1539         local sess_name=$3
1540
1541         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1542                 start $sess_name
1543         ret=$?
1544         if [[ $expected_to_fail -eq "1" ]]; then
1545                 test "$ret" -ne "0"
1546                 ret=$?
1547                 if [ $withtap -eq "1" ]; then
1548                         ok $? "Start tracing for session $sess_name failed as expected"
1549                 fi
1550         else
1551                 if [ $withtap -eq "1" ]; then
1552                         ok $ret "Start tracing for session $sess_name"
1553                 fi
1554         fi
1555 }
1556
1557 function start_lttng_tracing_ok ()
1558 {
1559         start_lttng_tracing_opt 1 0 "$@"
1560 }
1561
1562 function start_lttng_tracing_fail ()
1563 {
1564         start_lttng_tracing_opt 1 1 "$@"
1565 }
1566
1567 function start_lttng_tracing_notap ()
1568 {
1569         start_lttng_tracing_opt 0 1 "$@"
1570 }
1571
1572 function stop_lttng_tracing_opt ()
1573 {
1574         local withtap=$1
1575         local expected_to_fail=$2
1576         local sess_name=$3
1577
1578         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1579                 stop $sess_name
1580         ret=$?
1581         if [[ $expected_to_fail -eq "1" ]]; then
1582                 test "$ret" -ne "0"
1583                 ret=$?
1584                 if [ $withtap -eq "1" ]; then
1585                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1586                 fi
1587         else
1588                 if [ $withtap -eq "1" ]; then
1589                         ok $ret "Stop lttng tracing for session $sess_name"
1590                 fi
1591         fi
1592 }
1593
1594 function stop_lttng_tracing_ok ()
1595 {
1596         stop_lttng_tracing_opt 1 0 "$@"
1597 }
1598
1599 function stop_lttng_tracing_fail ()
1600 {
1601         stop_lttng_tracing_opt 1 1 "$@"
1602 }
1603
1604 function stop_lttng_tracing_notap ()
1605 {
1606         stop_lttng_tracing_opt 0 0 "$@"
1607 }
1608
1609 function destroy_lttng_session ()
1610 {
1611         local withtap=$1
1612         local expected_to_fail=$2
1613         local sess_name=$3
1614
1615         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1616                 destroy $sess_name
1617         ret=$?
1618         if [[ $expected_to_fail -eq "1" ]]; then
1619                 test "$ret" -ne "0"
1620                 ret=$?
1621                 if [ $withtap -eq "1" ]; then
1622                         ok $ret "Destroy session $sess_name failed as expected"
1623                 fi
1624         else
1625                 if [ $withtap -eq "1" ]; then
1626                         ok $ret "Destroy session $sess_name"
1627                 fi
1628         fi
1629 }
1630
1631 function destroy_lttng_session_ok ()
1632 {
1633         destroy_lttng_session 1 0 "$@"
1634
1635 }
1636
1637 function destroy_lttng_session_fail ()
1638 {
1639         destroy_lttng_session 1 1 "$@"
1640 }
1641
1642 function destroy_lttng_session_notap ()
1643 {
1644         destroy_lttng_session 0 0 "$@"
1645 }
1646
1647 function destroy_lttng_sessions ()
1648 {
1649         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1650                 destroy --all
1651         ok $? "Destroy all lttng sessions"
1652 }
1653
1654 function lttng_snapshot_add_output ()
1655 {
1656         local expected_to_fail=$1
1657         local sess_name=$2
1658         local trace_path=$3
1659         local opts=$4
1660
1661         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1662                 snapshot add-output -s $sess_name $trace_path $opts
1663         ret=$?
1664         if [[ $expected_to_fail -eq 1 ]]; then
1665                 test "$ret" -ne "0"
1666                 ok $? "Added snapshot output $trace_path failed as expected"
1667         else
1668                 ok $ret "Added snapshot output $trace_path"
1669         fi
1670 }
1671
1672 function lttng_snapshot_add_output_ok ()
1673 {
1674         lttng_snapshot_add_output 0 "$@"
1675 }
1676
1677 function lttng_snapshot_add_output_fail ()
1678 {
1679         lttng_snapshot_add_output 1 "$@"
1680 }
1681
1682 function lttng_snapshot_del_output ()
1683 {
1684         local expected_to_fail=$1
1685         local sess_name=$2
1686         local id=$3
1687
1688         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1689                 snapshot del-output -s $sess_name $id
1690         ret=$?
1691         if [[ $expected_to_fail -eq "1" ]]; then
1692                 test "$ret" -ne "0"
1693                 ok $? "Deleted snapshot output id $id failed as expected"
1694         else
1695                 ok $ret "Deleted snapshot output id $id"
1696         fi
1697 }
1698
1699 function lttng_snapshot_del_output_ok ()
1700 {
1701         lttng_snapshot_del_output 0 "$@"
1702 }
1703
1704 function lttng_snapshot_del_output_fail ()
1705 {
1706         lttng_snapshot_del_output 1 "$@"
1707 }
1708
1709 function lttng_snapshot_record ()
1710 {
1711         local sess_name=$1
1712         local trace_path=$2
1713
1714         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1715                 snapshot record -s "$sess_name" "$trace_path"
1716         ok $? "Snapshot recorded"
1717 }
1718
1719 function lttng_snapshot_list ()
1720 {
1721         local sess_name=$1
1722         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1723                 snapshot list-output -s $sess_name
1724         ok $? "Snapshot list"
1725 }
1726
1727 function lttng_save()
1728 {
1729         local sess_name=$1
1730         local opts=$2
1731
1732         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1733                 save $sess_name $opts
1734         ok $? "Session saved"
1735 }
1736
1737 function lttng_load()
1738 {
1739         local expected_to_fail=$1
1740         local opts=$2
1741
1742         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1743                 load $opts
1744         ret=$?
1745         if [[ $expected_to_fail -eq "1" ]]; then
1746                 test $ret -ne "0"
1747                 ok $? "Load command failed as expected with opts: $opts"
1748         else
1749                 ok $ret "Load command with opts: $opts"
1750         fi
1751 }
1752
1753 function lttng_load_ok()
1754 {
1755         lttng_load 0 "$@"
1756 }
1757
1758 function lttng_load_fail()
1759 {
1760         lttng_load 1 "$@"
1761 }
1762
1763 function lttng_track()
1764 {
1765         local expected_to_fail="$1"
1766         shift 1
1767         local opts="$@"
1768         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1769         ret=$?
1770         if [[ $expected_to_fail -eq "1" ]]; then
1771                 test $ret -ne "0"
1772                 ok $? "Track command failed as expected with opts: $opts"
1773         else
1774                 ok $ret "Track command with opts: $opts"
1775         fi
1776 }
1777
1778 function lttng_track_ok()
1779 {
1780         lttng_track 0 "$@"
1781 }
1782
1783 function lttng_track_fail()
1784 {
1785         lttng_track 1 "$@"
1786 }
1787
1788 function lttng_untrack()
1789 {
1790         local expected_to_fail="$1"
1791         shift 1
1792         local opts="$@"
1793         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1794         ret=$?
1795         if [[ $expected_to_fail -eq "1" ]]; then
1796                 test $ret -ne "0"
1797                 ok $? "Untrack command failed as expected with opts: $opts"
1798         else
1799                 ok $ret "Untrack command with opts: $opts"
1800         fi
1801 }
1802
1803 function lttng_untrack_ok()
1804 {
1805         lttng_untrack 0 "$@"
1806 }
1807
1808 function lttng_untrack_fail()
1809 {
1810         lttng_untrack 1 "$@"
1811 }
1812
1813 function lttng_track_pid_ok()
1814 {
1815         PID=$1
1816         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1817         ok $? "Lttng track pid on the kernel domain"
1818 }
1819
1820 function lttng_untrack_kernel_all_ok()
1821 {
1822         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1823         ok $? "Lttng untrack all pid on the kernel domain"
1824 }
1825
1826 function lttng_track_ust_ok()
1827 {
1828         lttng_track_ok -u "$@"
1829 }
1830
1831 function lttng_track_ust_fail()
1832 {
1833         lttng_track_fail -u "$@"
1834 }
1835
1836 function lttng_track_kernel_ok()
1837 {
1838         lttng_track_ok -k "$@"
1839 }
1840
1841 function lttng_track_kernel_fail()
1842 {
1843         lttng_track_fail -k "$@"
1844 }
1845
1846 function lttng_untrack_ust_ok()
1847 {
1848         lttng_untrack_ok -u "$@"
1849 }
1850
1851 function lttng_untrack_ust_fail()
1852 {
1853         lttng_untrack_fail -u "$@"
1854 }
1855
1856 function lttng_untrack_kernel_ok()
1857 {
1858         lttng_untrack_ok -k "$@"
1859 }
1860
1861 function lttng_untrack_kernel_fail()
1862 {
1863         lttng_untrack_fail -k "$@"
1864 }
1865
1866 function lttng_add_context_list()
1867 {
1868         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1869                 add-context --list
1870         ret=$?
1871         ok $ret "Context listing"
1872 }
1873
1874 function add_context_lttng()
1875 {
1876         local expected_to_fail="$1"
1877         local domain="$2"
1878         local session_name="$3"
1879         local channel_name="$4"
1880         local type="$5"
1881
1882         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1883                 add-context -s $session_name -c $channel_name -t $type $domain
1884         ret=$?
1885         if [[ $expected_to_fail -eq "1" ]]; then
1886                 test $ret -ne "0"
1887                 ok $? "Add context command failed as expected for type: $type"
1888         else
1889                 ok $ret "Add context command for type: $type"
1890         fi
1891 }
1892
1893 function add_context_ust_ok()
1894 {
1895         add_context_lttng 0 -u "$@"
1896 }
1897
1898 function add_context_ust_fail()
1899 {
1900         add_context_lttng 1 -u "$@"
1901 }
1902
1903 function add_context_kernel_ok()
1904 {
1905         add_context_lttng 0 -k "$@"
1906 }
1907
1908 function add_context_kernel_fail()
1909 {
1910         add_context_lttng 1 -k "$@"
1911 }
1912
1913 function wait_live_trace_ready ()
1914 {
1915         local url=$1
1916         local zero_client_match=0
1917
1918         diag "Waiting for live trace at url: $url"
1919         while [ $zero_client_match -eq 0 ]; do
1920                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1921                 sleep 0.5
1922         done
1923         pass "Waiting for live trace at url: $url"
1924 }
1925
1926 function wait_live_viewer_connect ()
1927 {
1928         local url=$1
1929         local one_client_match=0
1930
1931         diag "Waiting for live viewers on url: $url"
1932         while [ $one_client_match -eq 0 ]; do
1933                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1934                 sleep 0.5
1935         done
1936         pass "Waiting for live viewers on url: $url"
1937 }
1938
1939 function bail_out_if_no_babeltrace()
1940 {
1941         which "$BABELTRACE_BIN" >/dev/null
1942         if [ $? -ne 0 ]; then
1943                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
1944         fi
1945 }
1946
1947 # Check that the trace metadata contains '$expected' event ids matching '$event_name'.
1948 function validate_metadata_event()
1949 {
1950         local event_name=$1
1951         local expected=$2
1952         local trace_path=$3
1953
1954         local metadata_file
1955         local metadata_path
1956         local count
1957
1958         metadata_file=$(find "$trace_path" -name "metadata")
1959         metadata_path=$(dirname "$metadata_file")
1960
1961         bail_out_if_no_babeltrace
1962
1963         count=$($BABELTRACE_BIN --output-format=ctf-metadata "$metadata_path" | grep -c "$event_name")
1964
1965         test "$count" -eq "$expected"
1966         ok $? "Found $count / $expected metadata event id matching '$event_name'"
1967 }
1968
1969 # Check that the trace contains '$expected' events matching '$event_name', other
1970 # events not matching '$event_name' can be present.
1971 function trace_matches()
1972 {
1973         local event_name=$1
1974         local expected=$2
1975         local trace_path=$3
1976
1977         local count
1978         local total
1979
1980         bail_out_if_no_babeltrace
1981
1982         count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
1983         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
1984
1985         test "$count" -eq "$expected"
1986
1987         ok $? "Found $count / $expected events matching '$event_name' out of $total events"
1988 }
1989
1990 # Check that the trace contains '$expected' events matching '$event_name' and no
1991 # other events.
1992 function trace_match_only()
1993 {
1994         local event_name=$1
1995         local expected=$2
1996         local trace_path=$3
1997
1998         local count
1999         local total
2000
2001         bail_out_if_no_babeltrace
2002
2003         count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2004         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2005
2006         test "$expected" -eq "$count" && test "$total" -eq "$expected"
2007
2008         ok $? "Found $count / $expected events matching '$event_name' amongst $total events"
2009 }
2010
2011 # Check that the trace contains at least 1 event matching each name in the
2012 # comma separated list '$event_names'.
2013 function validate_trace()
2014 {
2015         local event_names=$1
2016         local trace_path=$2
2017
2018         local count
2019
2020         bail_out_if_no_babeltrace
2021
2022         OLDIFS=$IFS
2023         IFS=","
2024         for event_name in $event_names; do
2025                 # trace_path is unquoted since callers make use of globbing
2026                 count=$($BABELTRACE_BIN $trace_path | grep -c "$event_name")
2027                 test "$count" -gt 0
2028                 ok $? "Found $count events matching '$event_name'"
2029         done
2030         IFS=$OLDIFS
2031 }
2032
2033 # Check that the trace contains at least 1 event matching each name in the
2034 # comma separated list '$event_names' and a total of '$expected' events.
2035 function validate_trace_count()
2036 {
2037         local event_names=$1
2038         local trace_path=$2
2039         local expected=$3
2040
2041         local count
2042         local total=0
2043
2044         bail_out_if_no_babeltrace
2045
2046         OLDIFS=$IFS
2047         IFS=","
2048         for event_name in $event_names; do
2049                 count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2050                 test "$count" -gt 0
2051                 ok $? "Found '$count' events matching '$event_name'"
2052                 total=$(( total + count ))
2053         done
2054         IFS=$OLDIFS
2055         test $total -eq "$expected"
2056         ok $? "Found $total events, expected $expected events"
2057 }
2058
2059 # Check that the trace contains at least '$expected_min' event matching each
2060 # name in the comma separated list '$event_names' and a total at least
2061 # '$expected_min' and less than '$expected_max' events.
2062 function validate_trace_count_range_incl_min_excl_max()
2063 {
2064         local event_names=$1
2065         local trace_path=$2
2066         local expected_min=$3
2067         local expected_max=$4
2068
2069         local count
2070         local total=0
2071
2072         bail_out_if_no_babeltrace
2073
2074         OLDIFS=$IFS
2075         IFS=","
2076         for event_name in $event_names; do
2077                 count=$($BABELTRACE_BIN "$trace_path" | grep -c "$event_name")
2078                 test "$count" -ge "$expected_min"
2079                 ok $? "Found $count events matching '$event_name', expected at least $expected_min"
2080                 total=$(( total + count ))
2081         done
2082         IFS=$OLDIFS
2083         test $total -ge "$expected_min" && test $total -lt "$expected_max"
2084         ok $? "Found a total of $total events, expected at least $expected_min and less than $expected_max"
2085 }
2086
2087 function trace_first_line()
2088 {
2089         local trace_path=$1
2090
2091         $BABELTRACE_BIN "$trace_path" | head -n 1
2092 }
2093
2094 # Check that the trace contains at least 1 event matching the grep extended
2095 # regexp '$event_exp'.
2096 function validate_trace_exp()
2097 {
2098         local event_exp=$1
2099         local trace_path=$2
2100
2101         local count
2102
2103         bail_out_if_no_babeltrace
2104
2105         # event_exp is unquoted since it contains multiple grep arguments
2106         count=$($BABELTRACE_BIN "$trace_path" | grep -c --extended-regexp $event_exp)
2107         test "$count" -gt 0
2108         ok $? "Found $count events matching expression '$event_exp'"
2109 }
2110
2111 # Check that the trace contains at least 1 event matching the grep extended
2112 # regexp '$event_exp' and zero event not matching it.
2113 function validate_trace_only_exp()
2114 {
2115         local event_exp=$1
2116         local trace_path=$2
2117
2118         local count
2119         local total
2120
2121         bail_out_if_no_babeltrace
2122
2123         # event_exp is unquoted since it contains multiple grep arguments
2124         count=$($BABELTRACE_BIN "$trace_path" | grep -c --extended-regexp $event_exp)
2125         total=$($BABELTRACE_BIN "$trace_path" | wc -l)
2126
2127         test  "$count" -gt 0 && test "$total" -eq "$count"
2128         ok $? "Found $count events matching expression '$event_exp' amongst $total events"
2129 }
2130
2131 # Check that the trace is valid and contains 0 event.
2132 function validate_trace_empty()
2133 {
2134         local trace_path=$1
2135
2136         local ret
2137         local count
2138
2139         bail_out_if_no_babeltrace
2140
2141         events=$($BABELTRACE_BIN "$trace_path")
2142         ret=$?
2143         if [ $ret -ne 0 ]; then
2144                 fail "Failed to parse trace"
2145                 return $ret
2146         fi
2147
2148         count=$(echo -n "$events" | wc -l)
2149         test "$count" -eq 0
2150         ok $? "Validate trace is empty, found $count events"
2151 }
2152
2153 function validate_directory_empty ()
2154 {
2155         local trace_path="$1"
2156
2157         local files
2158         local ret
2159         local nb_files
2160
2161         # Do not double quote `$trace_path` below as we want wildcards to be
2162         # expanded.
2163         files="$(ls -A $trace_path)"
2164         ret=$?
2165         if [ $ret -ne 0 ]; then
2166                 fail "Failed to list content of directory \"$trace_path\""
2167                 return $ret
2168         fi
2169
2170         nb_files="$(echo -n "$files" | wc -l)"
2171         test "$nb_files" -eq 0
2172         ok $? "Directory \"$trace_path\" is empty"
2173 }
2174
2175 function validate_trace_session_ust_empty()
2176 {
2177         validate_directory_empty "$1"/ust
2178 }
2179
2180 function validate_trace_session_kernel_empty()
2181 {
2182         validate_trace_empty "$1"/kernel
2183 }
2184
2185 function regenerate_metadata ()
2186 {
2187         local expected_to_fail=$1
2188         local sess_name=$2
2189
2190         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2191                 regenerate metadata -s $sess_name
2192         ret=$?
2193         if [[ $expected_to_fail -eq "1" ]]; then
2194                 test "$ret" -ne "0"
2195                 ok $? "Expected fail on regenerate metadata $sess_name"
2196         else
2197                 ok $ret "Metadata regenerate $sess_name"
2198         fi
2199 }
2200
2201 function regenerate_metadata_ok ()
2202 {
2203         regenerate_metadata 0 "$@"
2204 }
2205
2206 function regenerate_metadata_fail ()
2207 {
2208         regenerate_metadata 1 "$@"
2209 }
2210
2211 function regenerate_statedump ()
2212 {
2213         local expected_to_fail=$1
2214         local sess_name=$2
2215
2216         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2217                 regenerate statedump -s $sess_name
2218         ret=$?
2219         if [[ $expected_to_fail -eq "1" ]]; then
2220                 test "$ret" -ne "0"
2221                 ok $? "Expected fail on regenerate statedump $sess_name"
2222         else
2223                 ok $ret "Statedump regenerate $sess_name"
2224         fi
2225 }
2226
2227 function regenerate_statedump_ok ()
2228 {
2229         regenerate_statedump 0 "$@"
2230 }
2231
2232 function regenerate_statedump_fail ()
2233 {
2234         regenerate_statedump 1 "$@"
2235 }
2236
2237 function rotate_session ()
2238 {
2239         local expected_to_fail=$1
2240         local sess_name=$2
2241
2242         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2243                 rotate $sess_name
2244         ret=$?
2245         if [[ $expected_to_fail -eq "1" ]]; then
2246                 test "$ret" -ne "0"
2247                 ok $? "Expected fail on rotate session $sess_name"
2248         else
2249                 ok $ret "Rotate session $sess_name"
2250         fi
2251 }
2252
2253 function rotate_session_ok ()
2254 {
2255         rotate_session 0 "$@"
2256 }
2257
2258 function rotate_session_fail ()
2259 {
2260         rotate_session 1 "$@"
2261 }
2262
2263 function destructive_tests_enabled ()
2264 {
2265         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2266                 return 0
2267         else
2268                 return 1
2269         fi
2270 }
2271
2272 function lttng_enable_rotation_timer ()
2273 {
2274         local expected_to_fail=$1
2275         local sess_name=$2
2276         local period=$3
2277
2278         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2279                 enable-rotation -s $sess_name --timer $period
2280         ret=$?
2281         if [[ $expected_to_fail -eq "1" ]]; then
2282                 test "$ret" -ne "0"
2283                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2284         else
2285                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2286         fi
2287 }
2288
2289 function lttng_enable_rotation_timer_ok ()
2290 {
2291         lttng_enable_rotation_timer 0 $@
2292 }
2293
2294 function lttng_enable_rotation_timer_fail ()
2295 {
2296         lttng_enable_rotation_timer 1 $@
2297 }
2298
2299 function lttng_enable_rotation_size ()
2300 {
2301         local expected_to_fail=$1
2302         local sess_name=$2
2303         local size=$3
2304
2305         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2306                 enable-rotation -s $sess_name --size $size
2307         ret=$?
2308         if [[ $expected_to_fail -eq "1" ]]; then
2309                 test "$ret" -ne "0"
2310                 ok $? "Expected to fail to set a periodic rotation of session $sess_name" "every " $size " bytes"
2311         else
2312                 ok $ret "Set a scheduled rotation of session $sess_name" "every " $size " bytes"
2313         fi
2314 }
2315
2316 function lttng_enable_rotation_size_ok ()
2317 {
2318         lttng_enable_rotation_size 0 $@
2319 }
2320
2321 function lttng_enable_rotation_size_fail ()
2322 {
2323         lttng_enable_rotation_size 1 $@
2324 }
2325
2326 function lttng_clear_session ()
2327 {
2328         local expected_to_fail=$1
2329         local sess_name=$2
2330
2331         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2332                 clear $sess_name
2333         ret=$?
2334         if [[ $expected_to_fail -eq "1" ]]; then
2335                 test "$ret" -ne "0"
2336                 ok $? "Expected fail on clear session $sess_name"
2337         else
2338                 ok $ret "Clear session $sess_name"
2339         fi
2340 }
2341
2342 function lttng_clear_session_ok ()
2343 {
2344         lttng_clear_session 0 $@
2345 }
2346
2347 function lttng_clear_session_fail ()
2348 {
2349         lttng_clear_session 1 $@
2350 }
2351
2352 function lttng_clear_all ()
2353 {
2354         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2355                 clear --all
2356         ok $? "Clear all lttng sessions"
2357 }
2358
2359 function lttng_add_trigger()
2360 {
2361         local expected_to_fail="$1"
2362         local trigger_name="$2"
2363         shift 2
2364         local args=("$@")
2365
2366         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2367         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2368         ret=$?
2369         if [[ $expected_to_fail -eq "1" ]]; then
2370                 test "$ret" -ne "0"
2371                 ok $? "Add trigger $trigger_name failed as expected"
2372         else
2373                 ok $ret "Add trigger $trigger_name"
2374         fi
2375 }
2376
2377 function lttng_remove_trigger()
2378 {
2379         local expected_to_fail="$1"
2380         local trigger_name="$2"
2381         shift 2
2382
2383         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2384         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2385         ret=$?
2386         if [[ $expected_to_fail -eq "1" ]]; then
2387                 test "$ret" -ne "0"
2388                 ok $? "Remove trigger $trigger_name failed as expected"
2389         else
2390                 ok $ret "Remove trigger $trigger_name"
2391         fi
2392 }
2393
2394 function lttng_add_trigger_ok()
2395 {
2396         lttng_add_trigger 0 "$@"
2397 }
2398
2399 function lttng_add_trigger_fail()
2400 {
2401         lttng_add_trigger 1 "$@"
2402 }
2403
2404 function lttng_remove_trigger_ok()
2405 {
2406         lttng_remove_trigger 0 "$@"
2407 }
2408
2409 function list_triggers_matches_ok ()
2410 {
2411         local tmp_stdout=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2412         local tmp_stderr=$(mktemp -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2413
2414         local test_name="$1"
2415         local expected_stdout_file="$2"
2416
2417         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2418
2419         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2420         ok $? "${test_name}: exit code is 0"
2421
2422         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2423         ok $? "${test_name}: expected stdout"
2424
2425         diff -u /dev/null "${tmp_stderr}"
2426         ok $? "${test_name}: expected stderr"
2427
2428         rm -f "${tmp_stdout}"
2429         rm -f "${tmp_stderr}"
2430 }
2431
2432 function list_triggers_matches_mi_ok ()
2433 {
2434         local tmp_stdout
2435         local tmp_stdout_raw
2436         local tmp_stderr
2437
2438         local test_name="$1"
2439         local expected_stdout_file="$2"
2440
2441         tmp_stdout_raw=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2442         tmp_stdout=$(mktemp -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2443         tmp_stderr=$(mktemp -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2444
2445         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2446
2447         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2448         ok $? "${test_name}: exit code is 0"
2449
2450         # Pretty-fy xml before further test.
2451         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2452
2453         $MI_VALIDATE "${tmp_stdout}"
2454         ok $? "list-trigger mi is valid"
2455
2456         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2457         ok $? "${test_name}: expected stdout"
2458
2459         diff -u /dev/null "${tmp_stderr}"
2460         ok $? "${test_name}: expected stderr"
2461
2462         rm -f "${tmp_stdout}"
2463         rm -f "${tmp_stdout_raw}"
2464         rm -f "${tmp_stderr}"
2465 }
2466
2467 function validate_path_pattern ()
2468 {
2469         local message=$1
2470         local pattern=$2
2471         # Base path is only used in error case and is used to list the content
2472         # of the base path.
2473         local base_path=$3
2474
2475
2476         [ -f $pattern ]
2477         ret=$?
2478         ok $ret "$message"
2479
2480         if [ "$ret" -ne "0" ]; then
2481                 diag "Path pattern expected: $pattern"
2482                 # List the tracepath for more info. We use find as a recursive
2483                 # directory lister.
2484                 diag "The base path content:"
2485                 find "$base_path" -print
2486         fi
2487 }
2488
2489 function validate_trace_path_ust_uid ()
2490 {
2491         local trace_path=$1
2492         local session_name=$2
2493         local uid=$UID
2494         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2495
2496         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2497 }
2498
2499 function validate_trace_path_ust_uid_network ()
2500 {
2501         local trace_path=$1
2502         local session_name=$2
2503         local base_path=$3
2504         local uid=$UID
2505         local hostname=$HOSTNAME
2506         local pattern
2507         local ret
2508
2509         # If the session was given a network base path (e.g
2510         # 127.0.0.1/my/custom/path on creation, there is no session name
2511         # component to the path on the relayd side. Caller can simply not pass a
2512         # session name for this scenario.
2513         if [ -n "$session_name" ]; then
2514                 session_name="$session_name-$date_time_pattern"
2515                 if [ -n "$base_path" ]; then
2516                         fail "Session name and base path are mutually exclusive"
2517                         return
2518                 fi
2519         fi
2520
2521         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2522
2523         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2524 }
2525
2526 function validate_trace_path_ust_uid_snapshot_network ()
2527 {
2528         local trace_path=$1
2529         local session_name=$2
2530         local snapshot_name=$3
2531         local snapshot_number=$4
2532         local base_path=$5
2533         local hostname=$HOSTNAME
2534         local uid=$UID
2535         local pattern
2536         local ret
2537
2538         # If the session/output was given a network base path (e.g
2539         # 127.0.0.1/my/custom/path on creation, there is no session name
2540         # component to the path on the relayd side. Caller can simply not pass a
2541         # session name for this scenario.
2542         if [ -n "$session_name" ]; then
2543                 session_name="$session_name-$date_time_pattern"
2544                 if [ -n "$base_path" ]; then
2545                         fail "Session name and base path are mutually exclusive"
2546                         return
2547                 fi
2548         fi
2549
2550         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2551
2552         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2553 }
2554
2555 function validate_trace_path_ust_uid_snapshot ()
2556 {
2557         local trace_path=$1
2558         local session_name=$2
2559         local snapshot_name=$3
2560         local snapshot_number=$4
2561         local base_path=$5
2562         local uid=$UID
2563         local pattern
2564         local ret
2565
2566         # If the session/output was given a network base path (e.g
2567         # 127.0.0.1/my/custom/path) on creation, there is no session name
2568         # component to the path on the relayd side. Caller can simply not pass a
2569         # session name for this scenario.
2570         if [ -n "$session_name" ]; then
2571                 session_name="$session_name-$date_time_pattern"
2572                 if [ -n "$base_path" ]; then
2573                         fail "Session name and base path are mutually exclusive"
2574                         return
2575                 fi
2576         fi
2577
2578         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2579
2580         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2581 }
2582
2583 function validate_trace_path_ust_pid ()
2584 {
2585         local trace_path=$1
2586         local session_name=$2
2587         local app_string=$3
2588         local pid=$4
2589         local pattern
2590         local ret
2591
2592         # If the session was given a trace path on creation, there is no session
2593         # name component to the path. Caller can simply not pass a session name
2594         # for this scenario.
2595         if [ -n "$session_name" ]; then
2596                 session_name="$session_name-$date_time_pattern"
2597         fi
2598
2599         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2600
2601         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2602 }
2603
2604 function validate_trace_path_kernel ()
2605 {
2606         local trace_path=$1
2607         local session_name=$2
2608         local pattern
2609
2610         # If the session was given a trace path on creation, there is no session
2611         # name component to the path. Caller can simply not pass a session name
2612         # for this scenario.
2613         if [ -n "$session_name" ]; then
2614                 session_name="$session_name-$date_time_pattern"
2615         fi
2616
2617         pattern="$trace_path/$session_name/kernel/metadata"
2618
2619         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2620 }
2621
2622 function validate_trace_path_kernel_network ()
2623 {
2624         local trace_path=$1
2625         local session_name=$2
2626         local hostname=$HOSTNAME
2627         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2628
2629         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2630 }
2631
2632 function validate_trace_path_kernel_snapshot ()
2633 {
2634         local trace_path=$1
2635         local session_name=$2
2636         local snapshot_name=$3
2637         local snapshot_number=$4
2638         local base_path=$5
2639         local pattern
2640         local ret
2641
2642         # If the session/output was given a network base path (e.g
2643         # 127.0.0.1/my/custom/path on creation, there is no session name
2644         # component to the path on the relayd side. Caller can simply not pass a
2645         # session name for this scenario.
2646         if [ -n "$session_name" ]; then
2647                 session_name="$session_name-$date_time_pattern"
2648                 if [ -n "$base_path" ]; then
2649                         fail "Session name and base path are mutually exclusive"
2650                         return
2651                 fi
2652         fi
2653
2654         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2655
2656         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2657 }
2658
2659 function validate_trace_path_kernel_snapshot_network ()
2660 {
2661         local trace_path=$1
2662         local session_name=$2
2663         local snapshot_name=$3
2664         local snapshot_number=$4
2665         local base_path=$5
2666         local hostname=$HOSTNAME
2667         local pattern
2668         local ret
2669
2670         # If the session/output was given a network base path (e.g
2671         # 127.0.0.1/my/custom/path on creation, there is no session name
2672         # component to the path on the relayd side. Caller can simply not pass a
2673         # session name for this scenario.
2674         if [ -n "$session_name" ]; then
2675                 session_name="$session_name-$date_time_pattern"
2676                 if [ -n "$base_path" ]; then
2677                         fail "Session name and base path are mutually exclusive"
2678                         return
2679                 fi
2680         fi
2681
2682         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2683
2684         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2685 }