tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace2"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$(cut -d '' -f 1 2>/dev/null < /proc/"$pid"/cmdline); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 208         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 209         echo
 210 }
 211
 212 # Return the number of _configured_ CPUs.
 213 function conf_proc_count()
 214 {
 215         getconf _NPROCESSORS_CONF
 216         if [ $? -ne 0 ]; then
 217                 diag "Failed to get the number of configured CPUs"
 218         fi
 219         echo
 220 }
 221
 222 # Check if base lttng-modules are present.
 223 # Bail out on failure
 224 function validate_lttng_modules_present ()
 225 {
 226         # Check for loadable modules.
 227         modprobe -n lttng-tracer 2>/dev/null
 228         if [ $? -eq 0 ]; then
 229                 return 0
 230         fi
 231
 232         # Check for builtin modules.
 233         ls /proc/lttng > /dev/null 2>&1
 234         if [ $? -eq 0 ]; then
 235                 return 0
 236         fi
 237
 238         LTTNG_BAIL_OUT "LTTng modules not detected."
 239 }
 240
 241 # Run the lttng binary.
 242 #
 243 # The first two arguments are stdout and stderr redirect paths, respectively.
 244 # The rest of the arguments are forwarded to the lttng binary
 245 function _run_lttng_cmd
 246 {
 247         local stdout_dest="$1"
 248         local stderr_dest="$2"
 249         shift 2
 250
 251         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 252         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 253 }
 254
 255 function enable_kernel_lttng_event
 256 {
 257         local withtap="$1"
 258         local expected_to_fail="$2"
 259         local sess_name="$3"
 260         local event_name="$4"
 261         local channel_name="$5"
 262
 263         if [ -z "$event_name" ]; then
 264                 # Enable all event if no event name specified
 265                 event_name="-a"
 266         fi
 267
 268         if [ -z "$channel_name" ]; then
 269                 # default channel if none specified
 270                 chan=""
 271         else
 272                 chan="-c $channel_name"
 273         fi
 274
 275         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 276                 enable-event "$event_name" $chan -s $sess_name -k
 277         ret=$?
 278         if [[ $expected_to_fail -eq "1" ]]; then
 279                 test $ret -ne "0"
 280                 ret=$?
 281                 if [ $withtap -eq "1" ]; then
 282                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 283                 fi
 284         else
 285                 if [ $withtap -eq "1" ]; then
 286                         ok $ret "Enable kernel event $event_name for session $sess_name"
 287                 fi
 288         fi
 289 }
 290
 291 function enable_kernel_lttng_event_ok ()
 292 {
 293         enable_kernel_lttng_event 1 0 "$@"
 294 }
 295
 296 function enable_kernel_lttng_event_fail ()
 297 {
 298         enable_kernel_lttng_event 1 1 "$@"
 299 }
 300
 301 function enable_kernel_lttng_event_notap ()
 302 {
 303         enable_kernel_lttng_event 0 0 "$@"
 304 }
 305
 306 # Old interface
 307 function lttng_enable_kernel_event
 308 {
 309         enable_kernel_lttng_event_ok "$@"
 310 }
 311
 312 function lttng_enable_kernel_syscall()
 313 {
 314         local expected_to_fail=$1
 315         local sess_name=$2
 316         local syscall_name=$3
 317         local channel_name=$4
 318
 319         if [ -z $syscall_name ]; then
 320                 # Enable all event if no syscall name specified
 321                 syscall_name="-a"
 322         fi
 323
 324         if [ -z $channel_name ]; then
 325                 # default channel if none specified
 326                 chan=""
 327         else
 328                 chan="-c $channel_name"
 329         fi
 330
 331         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 332                 enable-event --syscall "$syscall_name" $chan -s $sess_name -k
 333         ret=$?
 334         if [[ $expected_to_fail -eq "1" ]]; then
 335                 test $ret -ne "0"
 336                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 337         else
 338                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 339         fi
 340 }
 341
 342 function lttng_enable_kernel_syscall_ok()
 343 {
 344         lttng_enable_kernel_syscall 0 "$@"
 345 }
 346
 347 function lttng_enable_kernel_syscall_fail()
 348 {
 349         lttng_enable_kernel_syscall 1 "$@"
 350 }
 351
 352 function lttng_disable_kernel_syscall()
 353 {
 354         local expected_to_fail=$1
 355         local sess_name=$2
 356         local syscall_name=$3
 357         local channel_name=$4
 358
 359         if [ -z $syscall_name ]; then
 360                 # Enable all event if no syscall name specified
 361                 syscall_name="-a"
 362         fi
 363
 364         if [ -z $channel_name ]; then
 365                 # default channel if none specified
 366                 chan=""
 367         else
 368                 chan="-c $channel_name"
 369         fi
 370
 371         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 372                 disable-event --syscall "$syscall_name" $chan -s $sess_name -k
 373
 374         ret=$?
 375         if [[ $expected_to_fail -eq "1" ]]; then
 376                 test $ret -ne "0"
 377                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 378         else
 379                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 380         fi
 381 }
 382
 383 function lttng_disable_kernel_syscall_ok()
 384 {
 385         lttng_disable_kernel_syscall 0 "$@"
 386 }
 387
 388 function lttng_disable_kernel_syscall_fail()
 389 {
 390         lttng_disable_kernel_syscall 1 "$@"
 391 }
 392
 393 function lttng_enable_kernel_function_event ()
 394 {
 395         local expected_to_fail="$1"
 396         local sess_name="$2"
 397         local target="$3"
 398         local event_name="$4"
 399
 400         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 401         ret=$?
 402         if [[ $expected_to_fail -eq "1" ]]; then
 403                 test $ret -ne "0"
 404                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 405         else
 406                 ok $ret "Enable kernel function event for session $sess_name"
 407         fi
 408 }
 409
 410 function lttng_enable_kernel_function_event_ok ()
 411 {
 412         lttng_enable_kernel_function_event 0 "$@"
 413 }
 414
 415 function lttng_enable_kernel_userspace_probe_event ()
 416 {
 417         local expected_to_fail="$1"
 418         local sess_name="$2"
 419         local target="$3"
 420         local event_name="$4"
 421
 422         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 423         ret=$?
 424         if [[ $expected_to_fail -eq "1" ]]; then
 425                 test $ret -ne "0"
 426                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 427         else
 428                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 429         fi
 430 }
 431
 432 function lttng_enable_kernel_userspace_probe_event_fail ()
 433 {
 434         lttng_enable_kernel_userspace_probe_event 1 "$@"
 435 }
 436
 437 function lttng_enable_kernel_userspace_probe_event_ok ()
 438 {
 439         lttng_enable_kernel_userspace_probe_event 0 "$@"
 440 }
 441
 442 function disable_kernel_lttng_userspace_probe_event_ok ()
 443 {
 444         local sess_name="$1"
 445         local event_name="$2"
 446
 447         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 448         ok $? "Disable kernel event $target for session $sess_name"
 449 }
 450 function lttng_enable_kernel_channel()
 451 {
 452         local withtap=$1
 453         local expected_to_fail=$2
 454         local sess_name=$3
 455         local channel_name=$4
 456         local opts="${@:5}"
 457
 458         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 459                 enable-channel -k $channel_name -s $sess_name $opts
 460         ret=$?
 461         if [[ $expected_to_fail -eq "1" ]]; then
 462                 test "$ret" -ne "0"
 463                 ret=$?
 464                 if [ $withtap -eq "1" ]; then
 465                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 466                 fi
 467         else
 468                 if [ $withtap -eq "1" ]; then
 469                         ok $ret "Enable channel $channel_name for session $sess_name"
 470                 fi
 471         fi
 472 }
 473
 474 function lttng_enable_kernel_channel_ok()
 475 {
 476         lttng_enable_kernel_channel 1 0 "$@"
 477 }
 478
 479 function lttng_enable_kernel_channel_fail()
 480 {
 481         lttng_enable_kernel_channel 1 1 "$@"
 482 }
 483
 484 function lttng_enable_kernel_channel_notap()
 485 {
 486         lttng_enable_kernel_channel 0 0 "$@"
 487 }
 488
 489 function enable_kernel_lttng_channel_ok()
 490 {
 491         lttng_enable_kernel_channel 1 0 "$@"
 492 }
 493
 494 function lttng_disable_kernel_channel()
 495 {
 496         local expected_to_fail=$1
 497         local sess_name=$2
 498         local channel_name=$3
 499
 500         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 501                 disable-channel -k $channel_name -s $sess_name
 502         ret=$?
 503         if [[ $expected_to_fail -eq "1" ]]; then
 504                 test "$ret" -ne "0"
 505                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 506         else
 507                 ok $ret "Disable channel $channel_name for session $sess_name"
 508         fi
 509 }
 510
 511 function lttng_disable_kernel_channel_ok()
 512 {
 513         lttng_disable_kernel_channel 0 "$@"
 514 }
 515
 516 function lttng_disable_kernel_channel_fail()
 517 {
 518         lttng_disable_kernel_channel 1 "$@"
 519 }
 520
 521 function start_lttng_relayd_opt()
 522 {
 523         local withtap=$1
 524         local process_mode=$2
 525         local opt=$3
 526
 527         DIR=$(readlink -f "$TESTDIR")
 528
 529         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 530                 # shellcheck disable=SC2086
 531                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 532                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 533                 if [ $? -eq 1 ]; then
 534                         if [ $withtap -eq "1" ]; then
 535                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 536                         fi
 537                         return 1
 538                 else
 539                         if [ $withtap -eq "1" ]; then
 540                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 541                         fi
 542                 fi
 543         else
 544                 pass "Start lttng-relayd (opt: $opt)"
 545         fi
 546 }
 547
 548 function start_lttng_relayd()
 549 {
 550         start_lttng_relayd_opt 1 "-b" "$@"
 551 }
 552
 553 function start_lttng_relayd_notap()
 554 {
 555         start_lttng_relayd_opt 0 "-b" "$@"
 556 }
 557
 558 function stop_lttng_relayd_opt()
 559 {
 560         local withtap=$1
 561         local is_cleanup=$2
 562         local signal=$3
 563         local timeout_s=$4
 564         local dtimeleft_s=
 565         local retval=0
 566         local pids
 567
 568         if [ -z "$signal" ]; then
 569                 signal="SIGTERM"
 570         fi
 571
 572
 573         # Multiply time by 2 to simplify integer arithmetic
 574         if [ -n "$timeout_s" ]; then
 575                 dtimeleft_s=$((timeout_s * 2))
 576         fi
 577
 578
 579         pids=$(lttng_pgrep "$RELAYD_MATCH")
 580         if [ -z "$pids" ]; then
 581                 if [ "$is_cleanup" -eq 1 ]; then
 582                         :
 583                 elif [ "$withtap" -eq "1" ]; then
 584                         fail "No relay daemon to kill"
 585                 else
 586                         LTTNG_BAIL_OUT "No relay daemon to kill"
 587                 fi
 588                 return 0
 589         fi
 590
 591         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 592
 593         # shellcheck disable=SC2086
 594         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 595                 retval=1
 596                 if [ "$withtap" -eq "1" ]; then
 597                         fail "Kill relay daemon"
 598                 fi
 599         else
 600                 out=1
 601                 while [ -n "$out" ]; do
 602                         out=$(lttng_pgrep "$RELAYD_MATCH")
 603                         if [ -n "$dtimeleft_s" ]; then
 604                                 if [ $dtimeleft_s -lt 0 ]; then
 605                                         out=
 606                                         retval=1
 607                                 fi
 608                                 dtimeleft_s=$((dtimeleft_s - 1))
 609                         fi
 610                         sleep 0.5
 611                 done
 612                 if [ "$withtap" -eq "1" ]; then
 613                         if [ "$retval" -eq "0" ]; then
 614                                 pass "Wait after kill relay daemon"
 615                         else
 616                                 fail "Wait after kill relay daemon"
 617                         fi
 618                 fi
 619         fi
 620         return $retval
 621 }
 622
 623 function stop_lttng_relayd()
 624 {
 625         stop_lttng_relayd_opt 1 0 "$@"
 626 }
 627
 628 function stop_lttng_relayd_notap()
 629 {
 630         stop_lttng_relayd_opt 0 0 "$@"
 631 }
 632
 633 function stop_lttng_relayd_cleanup()
 634 {
 635         stop_lttng_relayd_opt 0 1 "$@"
 636 }
 637
 638 #First arg: show tap output
 639 #Second argument: load path for automatic loading
 640 function start_lttng_sessiond_opt()
 641 {
 642         local withtap=$1
 643         local load_path=$2
 644
 645         # The rest of the arguments will be passed directly to lttng-sessiond.
 646         shift 2
 647
 648         local env_vars=""
 649         local consumerd=""
 650
 651         local long_bit_value=
 652         long_bit_value=$(getconf LONG_BIT)
 653
 654         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 655                 # Env variable requested no session daemon
 656                 return
 657         fi
 658
 659         DIR=$(readlink -f "$TESTDIR")
 660
 661         # Get long_bit value for 32/64 consumerd
 662         case "$long_bit_value" in
 663                 32)
 664                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 665                         ;;
 666                 64)
 667                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 668                         ;;
 669                 *)
 670                         return
 671                         ;;
 672         esac
 673
 674         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 675         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 676                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 677         fi
 678         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 679
 680         if ! validate_kernel_version; then
 681             fail "Start session daemon"
 682             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 683         fi
 684
 685         diag "export LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/config/"
 686         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 687         export LTTNG_SESSION_CONFIG_XSD_PATH
 688
 689         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 690                 # Have a load path ?
 691                 if [ -n "$load_path" ]; then
 692                         diag "env $env_vars --load $load_path --background $consumerd $@"
 693                         # shellcheck disable=SC2086
 694                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 695                 else
 696                         diag "env $env_vars --background $consumerd $@"
 697                         # shellcheck disable=SC2086
 698                         env $env_vars --background "$consumerd" "$@"
 699                 fi
 700                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 701                 status=$?
 702                 if [ "$withtap" -eq "1" ]; then
 703                         ok $status "Start session daemon"
 704                 fi
 705         fi
 706 }
 707
 708 function start_lttng_sessiond()
 709 {
 710         start_lttng_sessiond_opt 1 "$@"
 711 }
 712
 713 function start_lttng_sessiond_notap()
 714 {
 715         start_lttng_sessiond_opt 0 "$@"
 716 }
 717
 718 function stop_lttng_sessiond_opt()
 719 {
 720         local withtap=$1
 721         local is_cleanup=$2
 722         local signal=$3
 723         local timeout_s=$4
 724         local dtimeleft_s=
 725         local retval=0
 726         local runas_pids
 727         local pids
 728
 729         if [ -z "$signal" ]; then
 730                 signal=SIGTERM
 731         fi
 732
 733         # Multiply time by 2 to simplify integer arithmetic
 734         if [ -n "$timeout_s" ]; then
 735                 dtimeleft_s=$((timeout_s * 2))
 736         fi
 737
 738         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 739                 # Env variable requested no session daemon
 740                 return 0
 741         fi
 742
 743         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 744         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 745
 746         if [ -n "$runas_pids" ]; then
 747                 pids="$pids $runas_pids"
 748         fi
 749
 750         if [ -z "$pids" ]; then
 751                 if [ "$is_cleanup" -eq 1 ]; then
 752                         :
 753                 elif [ "$withtap" -eq "1" ]; then
 754                         fail "No session daemon to kill"
 755                 else
 756                         LTTNG_BAIL_OUT "No session daemon to kill"
 757                 fi
 758                 return 0
 759         fi
 760
 761         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 762
 763         # shellcheck disable=SC2086
 764         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 765                 retval=1
 766                 if [ "$withtap" -eq "1" ]; then
 767                         fail "Kill sessions daemon"
 768                 fi
 769         else
 770                 out=1
 771                 while [ -n "$out" ]; do
 772                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 773                         if [ -n "$dtimeleft_s" ]; then
 774                                 if [ $dtimeleft_s -lt 0 ]; then
 775                                         out=
 776                                         retval=1
 777                                 fi
 778                                 dtimeleft_s=$((dtimeleft_s - 1))
 779                         fi
 780                         sleep 0.5
 781                 done
 782                 out=1
 783                 while [ -n "$out" ]; do
 784                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 785                         if [ -n "$dtimeleft_s" ]; then
 786                                 if [ $dtimeleft_s -lt 0 ]; then
 787                                         out=
 788                                         retval=1
 789                                 fi
 790                                 dtimeleft_s=$((dtimeleft_s - 1))
 791                         fi
 792                         sleep 0.5
 793                 done
 794
 795                 if [ "$withtap" -eq "1" ]; then
 796                         if [ "$retval" -eq "0" ]; then
 797                                 pass "Wait after kill session daemon"
 798                         else
 799                                 fail "Wait after kill session daemon"
 800                         fi
 801                 fi
 802         fi
 803         if [ "$signal" = "SIGKILL" ]; then
 804                 if [ "$(id -u)" -eq "0" ]; then
 805                         local modules=
 806                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 807
 808                         if [ -n "$modules" ]; then
 809                                 diag "Unloading all LTTng modules"
 810                                 modprobe --remove "$modules"
 811                         fi
 812                 fi
 813         fi
 814
 815         return $retval
 816 }
 817
 818 function stop_lttng_sessiond()
 819 {
 820         stop_lttng_sessiond_opt 1 0 "$@"
 821 }
 822
 823 function stop_lttng_sessiond_notap()
 824 {
 825         stop_lttng_sessiond_opt 0 0 "$@"
 826 }
 827
 828 function stop_lttng_sessiond_cleanup()
 829 {
 830         stop_lttng_sessiond_opt 0 1 "$@"
 831 }
 832
 833 function sigstop_lttng_sessiond_opt()
 834 {
 835         local withtap=$1
 836         local signal=SIGSTOP
 837         local pids
 838
 839         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 840                 # Env variable requested no session daemon
 841                 return
 842         fi
 843
 844         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 845
 846         if [ "$withtap" -eq "1" ]; then
 847                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 848         fi
 849
 850         # shellcheck disable=SC2086
 851         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 852                 if [ "$withtap" -eq "1" ]; then
 853                         fail "Sending SIGSTOP to session daemon"
 854                 fi
 855         else
 856                 out=1
 857                 while [ $out -ne 0 ]; do
 858                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 859
 860                         # Wait until state becomes stopped for session
 861                         # daemon(s).
 862                         out=0
 863                         for sessiond_pid in $pids; do
 864                                 state="$(ps -p "$sessiond_pid" -o state= )"
 865                                 if [[ -n "$state" && "$state" != "T" ]]; then
 866                                         out=1
 867                                 fi
 868                         done
 869                         sleep 0.5
 870                 done
 871                 if [ "$withtap" -eq "1" ]; then
 872                         pass "Sending SIGSTOP to session daemon"
 873                 fi
 874         fi
 875 }
 876
 877 function sigstop_lttng_sessiond()
 878 {
 879         sigstop_lttng_sessiond_opt 1 "$@"
 880 }
 881
 882 function sigstop_lttng_sessiond_notap()
 883 {
 884         sigstop_lttng_sessiond_opt 0 "$@"
 885 }
 886
 887 function stop_lttng_consumerd_opt()
 888 {
 889         local withtap=$1
 890         local is_cleanup=$2
 891         local signal=$3
 892         local timeout_s=$4
 893         local dtimeleft_s=
 894         local retval=0
 895         local pids
 896
 897         if [ -z "$signal" ]; then
 898                 signal=SIGTERM
 899         fi
 900
 901         # Multiply time by 2 to simplify integer arithmetic
 902         if [ -n "$timeout_s" ]; then
 903                 dtimeleft_s=$((timeout_s * 2))
 904         fi
 905
 906         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 907
 908         if [ -z "$pids" ]; then
 909                 if [ "$is_cleanup" -eq 1 ]; then
 910                         :
 911                 elif [ "$withtap" -eq "1" ]; then
 912                         fail "No consumerd daemon to kill"
 913                 else
 914                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
 915                 fi
 916                 return 0
 917         fi
 918
 919         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 920
 921         # shellcheck disable=SC2086
 922         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 923                 retval=1
 924                 if [ "$withtap" -eq "1" ]; then
 925                         fail "Kill consumer daemon"
 926                 fi
 927         else
 928                 out=1
 929                 while [ $out -ne 0 ]; do
 930                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 931
 932                         # If consumerds are still present check their status.
 933                         # A zombie status qualifies the consumerd as *killed*
 934                         out=0
 935                         for consumer_pid in $pids; do
 936                                 state="$(ps -p "$consumer_pid" -o state= )"
 937                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 938                                         out=1
 939                                 fi
 940                         done
 941                         if [ -n "$dtimeleft_s" ]; then
 942                                 if [ $dtimeleft_s -lt 0 ]; then
 943                                         out=0
 944                                         retval=1
 945                                 fi
 946                                 dtimeleft_s=$((dtimeleft_s - 1))
 947                         fi
 948                         sleep 0.5
 949                 done
 950                 if [ "$withtap" -eq "1" ]; then
 951                         if [ "$retval" -eq "0" ]; then
 952                                 pass "Wait after kill consumer daemon"
 953                         else
 954                                 fail "Wait after kill consumer daemon"
 955                         fi
 956                 fi
 957         fi
 958
 959         return $retval
 960 }
 961
 962 function stop_lttng_consumerd()
 963 {
 964         stop_lttng_consumerd_opt 1 0 "$@"
 965 }
 966
 967 function stop_lttng_consumerd_notap()
 968 {
 969         stop_lttng_consumerd_opt 0 0 "$@"
 970 }
 971
 972 function stop_lttng_consumerd_cleanup()
 973 {
 974         stop_lttng_consumerd_opt 0 1 "$@"
 975 }
 976
 977 function sigstop_lttng_consumerd_opt()
 978 {
 979         local withtap=$1
 980         local signal=SIGSTOP
 981         local pids
 982
 983         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 984
 985         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 986
 987         # shellcheck disable=SC2086
 988         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 989         retval=$?
 990
 991         if [ $retval -eq 1 ]; then
 992                 if [ "$withtap" -eq "1" ]; then
 993                         fail "Sending SIGSTOP to consumer daemon"
 994                 fi
 995                 return 1
 996         else
 997                 out=1
 998                 while [ $out -ne 0 ]; do
 999                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1000
1001                         # Wait until state becomes stopped for all
1002                         # consumers.
1003                         out=0
1004                         for consumer_pid in $pids; do
1005                                 state="$(ps -p "$consumer_pid" -o state= )"
1006                                 if [[ -n "$state" && "$state" != "T" ]]; then
1007                                         out=1
1008                                 fi
1009                         done
1010                         sleep 0.5
1011                 done
1012                 if [ "$withtap" -eq "1" ]; then
1013                         pass "Sending SIGSTOP to consumer daemon"
1014                 fi
1015         fi
1016         return $retval
1017 }
1018
1019 function sigstop_lttng_consumerd()
1020 {
1021         sigstop_lttng_consumerd_opt 1 "$@"
1022 }
1023
1024 function sigstop_lttng_consumerd_notap()
1025 {
1026         sigstop_lttng_consumerd_opt 0 "$@"
1027 }
1028
1029 function list_lttng_with_opts ()
1030 {
1031         local opts=$1
1032         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1033                 list $opts
1034         ok $? "Lttng-tool list command with option $opts"
1035 }
1036
1037 function create_lttng_session_no_output ()
1038 {
1039         local sess_name=$1
1040         local opts="${@:2}"
1041
1042         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1043                 create $sess_name --no-output $opts
1044         ok $? "Create session $sess_name in no-output mode"
1045 }
1046
1047 function create_lttng_session_uri () {
1048         local sess_name=$1
1049         local uri=$2
1050         local opts="${@:3}"
1051
1052         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1053                 create $sess_name -U $uri $opts
1054         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1055 }
1056
1057 function create_lttng_session ()
1058 {
1059         local withtap=$1
1060         local expected_to_fail=$2
1061         local sess_name=$3
1062         local trace_path=$4
1063         local opt=$5
1064
1065         if [ -z "$trace_path" ]; then
1066                 # Use lttng-sessiond default output.
1067                 trace_path=""
1068         else
1069                 trace_path="-o $trace_path"
1070         fi
1071
1072         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1073                 create "$sess_name" $trace_path $opt
1074         ret=$?
1075         if [ $expected_to_fail -eq "1" ]; then
1076                 test "$ret" -ne "0"
1077                 ret=$?
1078                 if [ $withtap -eq "1" ]; then
1079                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1080                 fi
1081         else
1082                 if [ $withtap -eq "1" ]; then
1083                         ok $ret "Create session $sess_name in $trace_path"
1084                 fi
1085         fi
1086         return $ret
1087 }
1088
1089 function create_lttng_session_ok ()
1090 {
1091         create_lttng_session 1 0 "$@"
1092 }
1093
1094 function create_lttng_session_fail ()
1095 {
1096         create_lttng_session 1 1 "$@"
1097 }
1098
1099 function create_lttng_session_notap ()
1100 {
1101         create_lttng_session 0 0 "$@"
1102 }
1103
1104
1105 function enable_ust_lttng_channel ()
1106 {
1107         local withtap=$1
1108         local expected_to_fail=$2
1109         local sess_name=$3
1110         local channel_name=$4
1111         local opts="${@:5}"
1112
1113         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1114                 enable-channel -u $channel_name -s $sess_name $opts
1115         ret=$?
1116         if [[ $expected_to_fail -eq "1" ]]; then
1117                 test "$ret" -ne "0"
1118                 ret=$?
1119                 if [ $withtap -eq "1" ]; then
1120                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1121                 fi
1122         else
1123                 if [ $withtap -eq "1" ]; then
1124                         ok $ret "Enable channel $channel_name for session $sess_name"
1125                 fi
1126         fi
1127         return $ret
1128 }
1129
1130 function enable_ust_lttng_channel_ok ()
1131 {
1132         enable_ust_lttng_channel 1 0 "$@"
1133 }
1134
1135 function enable_ust_lttng_channel_fail ()
1136 {
1137         enable_ust_lttng_channel 1 1 "$@"
1138 }
1139
1140 function enable_ust_lttng_channel_notap ()
1141 {
1142         enable_ust_lttng_channel 0 0 "$@"
1143 }
1144
1145 function disable_ust_lttng_channel()
1146 {
1147         local sess_name=$1
1148         local channel_name=$2
1149
1150         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1151                 disable-channel -u $channel_name -s $sess_name
1152         ok $? "Disable channel $channel_name for session $sess_name"
1153 }
1154
1155 function enable_lttng_mmap_overwrite_kernel_channel()
1156 {
1157         local sess_name=$1
1158         local channel_name=$2
1159
1160         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1161                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite
1162         ok $? "Enable channel $channel_name for session $sess_name"
1163 }
1164
1165 function enable_lttng_mmap_discard_small_kernel_channel()
1166 {
1167         local sess_name=$1
1168         local channel_name=$2
1169
1170         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1171                 enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1172         ok $? "Enable small discard channel $channel_name for session $sess_name"
1173 }
1174
1175 function enable_lttng_mmap_overwrite_small_kernel_channel()
1176 {
1177         local sess_name=$1
1178         local channel_name=$2
1179
1180         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1181                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1182         ok $? "Enable small discard channel $channel_name for session $sess_name"
1183 }
1184
1185 function enable_lttng_mmap_overwrite_ust_channel()
1186 {
1187         local sess_name=$1
1188         local channel_name=$2
1189
1190         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1191                 enable-channel -s $sess_name $channel_name -u --output mmap --overwrite
1192         ok $? "Enable channel $channel_name for session $sess_name"
1193 }
1194
1195 function enable_ust_lttng_event ()
1196 {
1197         local withtap=$1
1198         local expected_to_fail=$2
1199         local sess_name=$3
1200         local event_name="$4"
1201         local channel_name=$5
1202
1203         if [ -z $channel_name ]; then
1204                 # default channel if none specified
1205                 chan=""
1206         else
1207                 chan="-c $channel_name"
1208         fi
1209
1210         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1211                 enable-event "$event_name" $chan -s "$sess_name" -u
1212         ret=$?
1213         if [[ $expected_to_fail -eq "1" ]]; then
1214                 test $ret -ne "0"
1215                 ret=$?
1216                 if [[ $withtap -eq "1" ]]; then
1217                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1218                 fi
1219         else
1220                 if [[ $withtap -eq "1" ]]; then
1221                         ok $ret "Enable ust event $event_name for session $sess_name"
1222                 fi
1223         fi
1224         return $ret
1225 }
1226
1227 function enable_ust_lttng_event_ok ()
1228 {
1229         enable_ust_lttng_event 1 0 "$@"
1230 }
1231
1232 function enable_ust_lttng_event_fail ()
1233 {
1234         enable_ust_lttng_event 1 1 "$@"
1235 }
1236
1237 function enable_ust_lttng_event_notap ()
1238 {
1239         enable_ust_lttng_event 0 0 "$@"
1240 }
1241
1242 function enable_jul_lttng_event()
1243 {
1244         sess_name=$1
1245         event_name="$2"
1246         channel_name=$3
1247
1248         if [ -z $channel_name ]; then
1249                 # default channel if none specified
1250                 chan=""
1251         else
1252                 chan="-c $channel_name"
1253         fi
1254
1255         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1256                 enable-event "$event_name" $chan -s $sess_name -j
1257         ok $? "Enable JUL event $event_name for session $sess_name"
1258 }
1259
1260 function enable_jul_lttng_event_loglevel()
1261 {
1262         local sess_name=$1
1263         local event_name="$2"
1264         local loglevel=$3
1265         local channel_name=$4
1266
1267         if [ -z $channel_name ]; then
1268                 # default channel if none specified
1269                 chan=""
1270         else
1271                 chan="-c $channel_name"
1272         fi
1273
1274         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1275                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j
1276         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1277 }
1278
1279 function enable_log4j_lttng_event()
1280 {
1281         sess_name=$1
1282         event_name="$2"
1283         channel_name=$3
1284
1285         if [ -z $channel_name ]; then
1286                 # default channel if none specified
1287                 chan=""
1288         else
1289                 chan="-c $channel_name"
1290         fi
1291
1292         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1293                 enable-event "$event_name" $chan -s $sess_name -l
1294         ok $? "Enable LOG4J event $event_name for session $sess_name"
1295 }
1296
1297 function enable_log4j_lttng_event_loglevel()
1298 {
1299         local sess_name=$1
1300         local event_name="$2"
1301         local loglevel=$3
1302         local channel_name=$4
1303
1304         if [ -z $channel_name ]; then
1305                 # default channel if none specified
1306                 chan=""
1307         else
1308                 chan="-c $channel_name"
1309         fi
1310
1311         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1312                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l
1313         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
1314 }
1315
1316 function enable_python_lttng_event()
1317 {
1318         sess_name=$1
1319         event_name="$2"
1320         channel_name=$3
1321
1322         if [ -z $channel_name ]; then
1323                 # default channel if none specified
1324                 chan=""
1325         else
1326                 chan="-c $channel_name"
1327         fi
1328
1329         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1330                 enable-event "$event_name" $chan -s $sess_name -p
1331         ok $? "Enable Python event $event_name for session $sess_name"
1332 }
1333
1334 function enable_python_lttng_event_loglevel()
1335 {
1336         local sess_name=$1
1337         local event_name="$2"
1338         local loglevel=$3
1339         local channel_name=$4
1340
1341         if [ -z $channel_name ]; then
1342                 # default channel if none specified
1343                 chan=""
1344         else
1345                 chan="-c $channel_name"
1346         fi
1347
1348         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1349                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p
1350         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1351 }
1352
1353 function enable_ust_lttng_event_filter()
1354 {
1355         local sess_name="$1"
1356         local event_name="$2"
1357         local filter="$3"
1358         local channel_name=$4
1359
1360         if [ -z $channel_name ]; then
1361                 # default channel if none specified
1362                 chan=""
1363         else
1364                 chan="-c $channel_name"
1365         fi
1366
1367         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1368                 enable-event $chan "$event_name" -s $sess_name -u --filter "$filter"
1369         ok $? "Enable event $event_name with filtering for session $sess_name"
1370 }
1371
1372 function enable_ust_lttng_event_loglevel()
1373 {
1374         local sess_name="$1"
1375         local event_name="$2"
1376         local loglevel="$3"
1377
1378         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1379                 enable-event "$event_name" -s $sess_name -u --loglevel $loglevel
1380         ok $? "Enable event $event_name with loglevel $loglevel"
1381 }
1382
1383 function enable_ust_lttng_event_loglevel_only()
1384 {
1385         local sess_name="$1"
1386         local event_name="$2"
1387         local loglevel="$3"
1388
1389         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1390                 enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel
1391         ok $? "Enable event $event_name with loglevel-only $loglevel"
1392 }
1393
1394 function disable_ust_lttng_event ()
1395 {
1396         local sess_name="$1"
1397         local event_name="$2"
1398         local channel_name="$3"
1399
1400         if [ -z $channel_name ]; then
1401                 # default channel if none specified
1402                 chan=""
1403         else
1404                 chan="-c $channel_name"
1405         fi
1406
1407         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1408                 disable-event "$event_name" -s $sess_name $chan -u
1409         ok $? "Disable event $event_name for session $sess_name"
1410 }
1411
1412 function disable_jul_lttng_event ()
1413 {
1414         local sess_name="$1"
1415         local event_name="$2"
1416
1417         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1418         ok $? "Disable JUL event $event_name for session $sess_name"
1419 }
1420
1421 function disable_log4j_lttng_event ()
1422 {
1423         local sess_name="$1"
1424         local event_name="$2"
1425
1426         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1427         ok $? "Disable LOG4J event $event_name for session $sess_name"
1428 }
1429
1430 function disable_python_lttng_event ()
1431 {
1432         local sess_name="$1"
1433         local event_name="$2"
1434
1435         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1436                 disable-event "$event_name" -s $sess_name -p
1437         ok $? "Disable Python event $event_name for session $sess_name"
1438 }
1439
1440 function start_lttng_tracing_opt ()
1441 {
1442         local withtap=$1
1443         local expected_to_fail=$2
1444         local sess_name=$3
1445
1446         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1447                 start $sess_name
1448         ret=$?
1449         if [[ $expected_to_fail -eq "1" ]]; then
1450                 test "$ret" -ne "0"
1451                 ret=$?
1452                 if [ $withtap -eq "1" ]; then
1453                         ok $? "Start tracing for session $sess_name failed as expected"
1454                 fi
1455         else
1456                 if [ $withtap -eq "1" ]; then
1457                         ok $ret "Start tracing for session $sess_name"
1458                 fi
1459         fi
1460 }
1461
1462 function start_lttng_tracing_ok ()
1463 {
1464         start_lttng_tracing_opt 1 0 "$@"
1465 }
1466
1467 function start_lttng_tracing_fail ()
1468 {
1469         start_lttng_tracing_opt 1 1 "$@"
1470 }
1471
1472 function start_lttng_tracing_notap ()
1473 {
1474         start_lttng_tracing_opt 0 1 "$@"
1475 }
1476
1477 function stop_lttng_tracing_opt ()
1478 {
1479         local withtap=$1
1480         local expected_to_fail=$2
1481         local sess_name=$3
1482
1483         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1484                 stop $sess_name
1485         ret=$?
1486         if [[ $expected_to_fail -eq "1" ]]; then
1487                 test "$ret" -ne "0"
1488                 ret=$?
1489                 if [ $withtap -eq "1" ]; then
1490                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1491                 fi
1492         else
1493                 if [ $withtap -eq "1" ]; then
1494                         ok $ret "Stop lttng tracing for session $sess_name"
1495                 fi
1496         fi
1497 }
1498
1499 function stop_lttng_tracing_ok ()
1500 {
1501         stop_lttng_tracing_opt 1 0 "$@"
1502 }
1503
1504 function stop_lttng_tracing_fail ()
1505 {
1506         stop_lttng_tracing_opt 1 1 "$@"
1507 }
1508
1509 function stop_lttng_tracing_notap ()
1510 {
1511         stop_lttng_tracing_opt 0 0 "$@"
1512 }
1513
1514 function destroy_lttng_session ()
1515 {
1516         local withtap=$1
1517         local expected_to_fail=$2
1518         local sess_name=$3
1519
1520         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1521                 destroy $sess_name
1522         ret=$?
1523         if [[ $expected_to_fail -eq "1" ]]; then
1524                 test "$ret" -ne "0"
1525                 ret=$?
1526                 if [ $withtap -eq "1" ]; then
1527                         ok $ret "Destroy session $sess_name failed as expected"
1528                 fi
1529         else
1530                 if [ $withtap -eq "1" ]; then
1531                         ok $ret "Destroy session $sess_name"
1532                 fi
1533         fi
1534 }
1535
1536 function destroy_lttng_session_ok ()
1537 {
1538         destroy_lttng_session 1 0 "$@"
1539
1540 }
1541
1542 function destroy_lttng_session_fail ()
1543 {
1544         destroy_lttng_session 1 1 "$@"
1545 }
1546
1547 function destroy_lttng_session_notap ()
1548 {
1549         destroy_lttng_session 0 0 "$@"
1550 }
1551
1552 function destroy_lttng_sessions ()
1553 {
1554         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1555                 destroy --all
1556         ok $? "Destroy all lttng sessions"
1557 }
1558
1559 function lttng_snapshot_add_output ()
1560 {
1561         local expected_to_fail=$1
1562         local sess_name=$2
1563         local trace_path=$3
1564         local opts=$4
1565
1566         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1567                 snapshot add-output -s $sess_name $trace_path $opts
1568         ret=$?
1569         if [[ $expected_to_fail -eq 1 ]]; then
1570                 test "$ret" -ne "0"
1571                 ok $? "Added snapshot output $trace_path failed as expected"
1572         else
1573                 ok $ret "Added snapshot output $trace_path"
1574         fi
1575 }
1576
1577 function lttng_snapshot_add_output_ok ()
1578 {
1579         lttng_snapshot_add_output 0 "$@"
1580 }
1581
1582 function lttng_snapshot_add_output_fail ()
1583 {
1584         lttng_snapshot_add_output 1 "$@"
1585 }
1586
1587 function lttng_snapshot_del_output ()
1588 {
1589         local expected_to_fail=$1
1590         local sess_name=$2
1591         local id=$3
1592
1593         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1594                 snapshot del-output -s $sess_name $id
1595         ret=$?
1596         if [[ $expected_to_fail -eq "1" ]]; then
1597                 test "$ret" -ne "0"
1598                 ok $? "Deleted snapshot output id $id failed as expected"
1599         else
1600                 ok $ret "Deleted snapshot output id $id"
1601         fi
1602 }
1603
1604 function lttng_snapshot_del_output_ok ()
1605 {
1606         lttng_snapshot_del_output 0 "$@"
1607 }
1608
1609 function lttng_snapshot_del_output_fail ()
1610 {
1611         lttng_snapshot_del_output 1 "$@"
1612 }
1613
1614 function lttng_snapshot_record ()
1615 {
1616         local sess_name=$1
1617         local trace_path=$2
1618
1619         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1620                 snapshot record -s "$sess_name" "$trace_path"
1621         ok $? "Snapshot recorded"
1622 }
1623
1624 function lttng_snapshot_list ()
1625 {
1626         local sess_name=$1
1627         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1628                 snapshot list-output -s $sess_name
1629         ok $? "Snapshot list"
1630 }
1631
1632 function lttng_save()
1633 {
1634         local sess_name=$1
1635         local opts=$2
1636
1637         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1638                 save $sess_name $opts
1639         ok $? "Session saved"
1640 }
1641
1642 function lttng_load()
1643 {
1644         local expected_to_fail=$1
1645         local opts=$2
1646
1647         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1648                 load $opts
1649         ret=$?
1650         if [[ $expected_to_fail -eq "1" ]]; then
1651                 test $ret -ne "0"
1652                 ok $? "Load command failed as expected with opts: $opts"
1653         else
1654                 ok $ret "Load command with opts: $opts"
1655         fi
1656 }
1657
1658 function lttng_load_ok()
1659 {
1660         lttng_load 0 "$@"
1661 }
1662
1663 function lttng_load_fail()
1664 {
1665         lttng_load 1 "$@"
1666 }
1667
1668 function lttng_track()
1669 {
1670         local expected_to_fail="$1"
1671         shift 1
1672         local opts="$@"
1673         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1674         ret=$?
1675         if [[ $expected_to_fail -eq "1" ]]; then
1676                 test $ret -ne "0"
1677                 ok $? "Track command failed as expected with opts: $opts"
1678         else
1679                 ok $ret "Track command with opts: $opts"
1680         fi
1681 }
1682
1683 function lttng_track_ok()
1684 {
1685         lttng_track 0 "$@"
1686 }
1687
1688 function lttng_track_fail()
1689 {
1690         lttng_track 1 "$@"
1691 }
1692
1693 function lttng_untrack()
1694 {
1695         local expected_to_fail="$1"
1696         shift 1
1697         local opts="$@"
1698         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1699         ret=$?
1700         if [[ $expected_to_fail -eq "1" ]]; then
1701                 test $ret -ne "0"
1702                 ok $? "Untrack command failed as expected with opts: $opts"
1703         else
1704                 ok $ret "Untrack command with opts: $opts"
1705         fi
1706 }
1707
1708 function lttng_untrack_ok()
1709 {
1710         lttng_untrack 0 "$@"
1711 }
1712
1713 function lttng_untrack_fail()
1714 {
1715         lttng_untrack 1 "$@"
1716 }
1717
1718 function lttng_track_pid_ok()
1719 {
1720         PID=$1
1721         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1722         ok $? "Lttng track pid on the kernel domain"
1723 }
1724
1725 function lttng_untrack_kernel_all_ok()
1726 {
1727         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1728         ok $? "Lttng untrack all pid on the kernel domain"
1729 }
1730
1731 function lttng_track_ust_ok()
1732 {
1733         lttng_track_ok -u "$@"
1734 }
1735
1736 function lttng_track_ust_fail()
1737 {
1738         lttng_track_fail -u "$@"
1739 }
1740
1741 function lttng_track_kernel_ok()
1742 {
1743         lttng_track_ok -k "$@"
1744 }
1745
1746 function lttng_track_kernel_fail()
1747 {
1748         lttng_track_fail -k "$@"
1749 }
1750
1751 function lttng_untrack_ust_ok()
1752 {
1753         lttng_untrack_ok -u "$@"
1754 }
1755
1756 function lttng_untrack_ust_fail()
1757 {
1758         lttng_untrack_fail -u "$@"
1759 }
1760
1761 function lttng_untrack_kernel_ok()
1762 {
1763         lttng_untrack_ok -k "$@"
1764 }
1765
1766 function lttng_untrack_kernel_fail()
1767 {
1768         lttng_untrack_fail -k "$@"
1769 }
1770
1771 function lttng_add_context_list()
1772 {
1773         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1774                 add-context --list
1775         ret=$?
1776         ok $ret "Context listing"
1777 }
1778
1779 function add_context_lttng()
1780 {
1781         local expected_to_fail="$1"
1782         local domain="$2"
1783         local session_name="$3"
1784         local channel_name="$4"
1785         local type="$5"
1786
1787         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1788                 add-context -s $session_name -c $channel_name -t $type $domain
1789         ret=$?
1790         if [[ $expected_to_fail -eq "1" ]]; then
1791                 test $ret -ne "0"
1792                 ok $? "Add context command failed as expected for type: $type"
1793         else
1794                 ok $ret "Add context command for type: $type"
1795         fi
1796 }
1797
1798 function add_context_ust_ok()
1799 {
1800         add_context_lttng 0 -u "$@"
1801 }
1802
1803 function add_context_ust_fail()
1804 {
1805         add_context_lttng 1 -u "$@"
1806 }
1807
1808 function add_context_kernel_ok()
1809 {
1810         add_context_lttng 0 -k "$@"
1811 }
1812
1813 function add_context_kernel_fail()
1814 {
1815         add_context_lttng 1 -k "$@"
1816 }
1817
1818 function wait_live_trace_ready ()
1819 {
1820         local url=$1
1821         local zero_client_match=0
1822
1823         diag "Waiting for live trace at url: $url"
1824         while [ $zero_client_match -eq 0 ]; do
1825                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1826                 sleep 0.5
1827         done
1828         pass "Waiting for live trace at url: $url"
1829 }
1830
1831 function wait_live_viewer_connect ()
1832 {
1833         local url=$1
1834         local one_client_match=0
1835
1836         diag "Waiting for live viewers on url: $url"
1837         while [ $one_client_match -eq 0 ]; do
1838                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1839                 sleep 0.5
1840         done
1841         pass "Waiting for live viewers on url: $url"
1842 }
1843
1844 function bail_out_if_no_babeltrace()
1845 {
1846         which "$BABELTRACE_BIN" >/dev/null
1847         if [ $? -ne 0 ]; then
1848                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
1849         fi
1850 }
1851
1852 function validate_metadata_event ()
1853 {
1854         local event_name=$1
1855         local nr_event_id=$2
1856         local trace_path=$3
1857
1858         local metadata_file=$(find $trace_path -name "metadata")
1859         local metadata_path=$(dirname $metadata_file)
1860
1861         bail_out_if_no_babeltrace
1862
1863         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1864
1865         if [ "$count" -ne "$nr_event_id" ]; then
1866                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1867                 diag "$count matching event id found in metadata"
1868         else
1869                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1870         fi
1871
1872 }
1873
1874 function trace_matches ()
1875 {
1876         local event_name=$1
1877         local nr_iter=$2
1878         local trace_path=$3
1879
1880         bail_out_if_no_babeltrace
1881
1882         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1883
1884         if [ "$count" -ne "$nr_iter" ]; then
1885                 fail "Trace match"
1886                 diag "$count matching events found in trace"
1887         else
1888                 pass "Trace match"
1889         fi
1890 }
1891
1892 function trace_match_only()
1893 {
1894         local event_name=$1
1895         local nr_iter=$2
1896         local trace_path=$3
1897
1898         bail_out_if_no_babeltrace
1899         #which "$BABELTRACE_BIN" >/dev/null
1900         #skip $? -ne 0 "\"$BABELTRACE_BIN\" binary not found. Skipping trace comparison"
1901
1902         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1903         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1904
1905         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1906                 pass "Trace match with $total event $event_name"
1907         else
1908                 fail "Trace match"
1909                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1910         fi
1911 }
1912
1913 function validate_trace
1914 {
1915         local event_name=$1
1916         local trace_path=$2
1917
1918         bail_out_if_no_babeltrace
1919
1920         OLDIFS=$IFS
1921         IFS=","
1922         for i in $event_name; do
1923                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1924                 if [ "$traced" -ne 0 ]; then
1925                         pass "Validate trace for event $i, $traced events"
1926                 else
1927                         fail "Validate trace for event $i"
1928                         diag "Found $traced occurrences of $i"
1929                 fi
1930         done
1931         ret=$?
1932         IFS=$OLDIFS
1933         return $ret
1934 }
1935
1936 function validate_trace_count
1937 {
1938         local event_name=$1
1939         local trace_path=$2
1940         local expected_count=$3
1941
1942         bail_out_if_no_babeltrace
1943
1944         cnt=0
1945         OLDIFS=$IFS
1946         IFS=","
1947         for i in $event_name; do
1948                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1949                 if [ "$traced" -ne 0 ]; then
1950                         pass "Validate trace for event $i, $traced events"
1951                 else
1952                         fail "Validate trace for event $i"
1953                         diag "Found $traced occurrences of $i"
1954                 fi
1955                 cnt=$(($cnt + $traced))
1956         done
1957         IFS=$OLDIFS
1958         test $cnt -eq $expected_count
1959         ok $? "Read a total of $cnt events, expected $expected_count"
1960 }
1961
1962 function validate_trace_count_range_incl_min_excl_max
1963 {
1964         local event_name=$1
1965         local trace_path=$2
1966         local expected_min=$3
1967         local expected_max=$4
1968
1969         bail_out_if_no_babeltrace
1970
1971         cnt=0
1972         OLDIFS=$IFS
1973         IFS=","
1974         for i in $event_name; do
1975                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1976                 if [ "$traced" -ge $expected_min ]; then
1977                         pass "Validate trace for event $i, $traced events"
1978                 else
1979                         fail "Validate trace for event $i"
1980                         diag "Found $traced occurrences of $i"
1981                 fi
1982                 cnt=$(($cnt + $traced))
1983         done
1984         IFS=$OLDIFS
1985         test $cnt -lt $expected_max
1986         ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
1987 }
1988
1989 function trace_first_line
1990 {
1991         local trace_path=$1
1992
1993         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1994 }
1995
1996 function validate_trace_exp()
1997 {
1998         local event_exp=$1
1999         local trace_path=$2
2000
2001         bail_out_if_no_babeltrace
2002
2003         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
2004         if [ "$traced" -ne 0 ]; then
2005                 pass "Validate trace for expression '${event_exp}', $traced events"
2006         else
2007                 fail "Validate trace for expression '${event_exp}'"
2008                 diag "Found $traced occurrences of '${event_exp}'"
2009         fi
2010         ret=$?
2011         return $ret
2012 }
2013
2014 function validate_trace_only_exp()
2015 {
2016         local event_exp=$1
2017         local trace_path=$2
2018
2019         bail_out_if_no_babeltrace
2020
2021         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
2022         local total=$($BABELTRACE_BIN $trace_path | wc -l)
2023
2024         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
2025                 pass "Trace match with $total for expression '${event_exp}'"
2026         else
2027                 fail "Trace match"
2028                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
2029         fi
2030         ret=$?
2031         return $ret
2032 }
2033
2034 function validate_trace_empty()
2035 {
2036         local trace_path=$1
2037
2038         bail_out_if_no_babeltrace
2039
2040         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
2041         ret=$?
2042         if [ $ret -ne 0 ]; then
2043                 fail "Failed to parse trace"
2044                 return $ret
2045         fi
2046
2047         traced=$(echo -n "$events" | wc -l)
2048         if [ "$traced" -eq 0 ]; then
2049                 pass "Validate empty trace"
2050         else
2051                 fail "Validate empty trace"
2052                 diag "Found $traced events in trace"
2053         fi
2054         ret=$?
2055         return $ret
2056 }
2057
2058 function validate_directory_empty ()
2059 {
2060         local trace_path="$1"
2061
2062         # Do not double quote `$trace_path` below as we want wildcards to be
2063         # expanded.
2064         files="$(ls -A $trace_path)"
2065         ret=$?
2066         if [ $ret -ne 0 ]; then
2067                 fail "Failed to list content of directory \"$trace_path\""
2068                 return $ret
2069         fi
2070
2071         nb_files="$(echo -n "$files" | wc -l)"
2072         ok $nb_files "Directory \"$trace_path\" is empty"
2073 }
2074
2075 function validate_trace_session_ust_empty()
2076 {
2077         validate_directory_empty "$1"/ust
2078 }
2079
2080 function validate_trace_session_kernel_empty()
2081 {
2082         validate_trace_empty "$1"/kernel
2083 }
2084
2085 function regenerate_metadata ()
2086 {
2087         local expected_to_fail=$1
2088         local sess_name=$2
2089
2090         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2091                 regenerate metadata -s $sess_name
2092         ret=$?
2093         if [[ $expected_to_fail -eq "1" ]]; then
2094                 test "$ret" -ne "0"
2095                 ok $? "Expected fail on regenerate metadata $sess_name"
2096         else
2097                 ok $ret "Metadata regenerate $sess_name"
2098         fi
2099 }
2100
2101 function regenerate_metadata_ok ()
2102 {
2103         regenerate_metadata 0 "$@"
2104 }
2105
2106 function regenerate_metadata_fail ()
2107 {
2108         regenerate_metadata 1 "$@"
2109 }
2110
2111 function regenerate_statedump ()
2112 {
2113         local expected_to_fail=$1
2114         local sess_name=$2
2115
2116         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2117                 regenerate statedump -s $sess_name
2118         ret=$?
2119         if [[ $expected_to_fail -eq "1" ]]; then
2120                 test "$ret" -ne "0"
2121                 ok $? "Expected fail on regenerate statedump $sess_name"
2122         else
2123                 ok $ret "Statedump regenerate $sess_name"
2124         fi
2125 }
2126
2127 function regenerate_statedump_ok ()
2128 {
2129         regenerate_statedump 0 "$@"
2130 }
2131
2132 function regenerate_statedump_fail ()
2133 {
2134         regenerate_statedump 1 "$@"
2135 }
2136
2137 function rotate_session ()
2138 {
2139         local expected_to_fail=$1
2140         local sess_name=$2
2141
2142         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2143                 rotate $sess_name
2144         ret=$?
2145         if [[ $expected_to_fail -eq "1" ]]; then
2146                 test "$ret" -ne "0"
2147                 ok $? "Expected fail on rotate session $sess_name"
2148         else
2149                 ok $ret "Rotate session $sess_name"
2150         fi
2151 }
2152
2153 function rotate_session_ok ()
2154 {
2155         rotate_session 0 "$@"
2156 }
2157
2158 function rotate_session_fail ()
2159 {
2160         rotate_session 1 "$@"
2161 }
2162
2163 function destructive_tests_enabled ()
2164 {
2165         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2166                 return 0
2167         else
2168                 return 1
2169         fi
2170 }
2171
2172 function lttng_enable_rotation_timer ()
2173 {
2174         local expected_to_fail=$1
2175         local sess_name=$2
2176         local period=$3
2177
2178         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2179                 enable-rotation -s $sess_name --timer $period
2180         ret=$?
2181         if [[ $expected_to_fail -eq "1" ]]; then
2182                 test "$ret" -ne "0"
2183                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2184         else
2185                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2186         fi
2187 }
2188
2189 function lttng_enable_rotation_timer_ok ()
2190 {
2191         lttng_enable_rotation_timer 0 $@
2192 }
2193
2194 function lttng_enable_rotation_timer_fail ()
2195 {
2196         lttng_enable_rotation_timer 1 $@
2197 }
2198
2199 function lttng_enable_rotation_size ()
2200 {
2201         local expected_to_fail=$1
2202         local sess_name=$2
2203         local size=$3
2204
2205         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2206                 enable-rotation -s $sess_name --size $size
2207         ret=$?
2208         if [[ $expected_to_fail -eq "1" ]]; then
2209                 test "$ret" -ne "0"
2210                 ok $? "Expected fail on rotate session $sess_name"
2211         else
2212                 ok $ret "Rotate session $sess_name"
2213         fi
2214 }
2215
2216 function lttng_enable_rotation_size_ok ()
2217 {
2218         lttng_enable_rotation_size 0 $@
2219 }
2220
2221 function lttng_enable_rotation_size_fail ()
2222 {
2223         lttng_enable_rotation_size 1 $@
2224 }
2225
2226 function lttng_clear_session ()
2227 {
2228         local expected_to_fail=$1
2229         local sess_name=$2
2230
2231         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2232                 clear $sess_name
2233         ret=$?
2234         if [[ $expected_to_fail -eq "1" ]]; then
2235                 test "$ret" -ne "0"
2236                 ok $? "Expected fail on clear session $sess_name"
2237         else
2238                 ok $ret "Clear session $sess_name"
2239         fi
2240 }
2241
2242 function lttng_clear_session_ok ()
2243 {
2244         lttng_clear_session 0 $@
2245 }
2246
2247 function lttng_clear_session_fail ()
2248 {
2249         lttng_clear_session 1 $@
2250 }
2251
2252 function lttng_clear_all ()
2253 {
2254         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2255                 clear --all
2256         ok $? "Clear all lttng sessions"
2257 }
2258
2259 function lttng_add_trigger()
2260 {
2261         local expected_to_fail="$1"
2262         local trigger_name="$2"
2263         shift 2
2264         local args=("$@")
2265
2266         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2267         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2268         ret=$?
2269         if [[ $expected_to_fail -eq "1" ]]; then
2270                 test "$ret" -ne "0"
2271                 ok $? "Add trigger $trigger_name failed as expected"
2272         else
2273                 ok $ret "Add trigger $trigger_name"
2274         fi
2275 }
2276
2277 function lttng_remove_trigger()
2278 {
2279         local expected_to_fail="$1"
2280         local trigger_name="$2"
2281         shift 2
2282
2283         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2284         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2285         ret=$?
2286         if [[ $expected_to_fail -eq "1" ]]; then
2287                 test "$ret" -ne "0"
2288                 ok $? "Remove trigger $trigger_name failed as expected"
2289         else
2290                 ok $ret "Remove trigger $trigger_name"
2291         fi
2292 }
2293
2294 function lttng_add_trigger_ok()
2295 {
2296         lttng_add_trigger 0 "$@"
2297 }
2298
2299 function lttng_add_trigger_fail()
2300 {
2301         lttng_add_trigger 1 "$@"
2302 }
2303
2304 function lttng_remove_trigger_ok()
2305 {
2306         lttng_remove_trigger 0 "$@"
2307 }
2308
2309 function list_triggers_matches_ok ()
2310 {
2311         local tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2312         local tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2313
2314         local test_name="$1"
2315         local expected_stdout_file="$2"
2316
2317         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2318
2319         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2320         ok $? "${test_name}: exit code is 0"
2321
2322         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2323         ok $? "${test_name}: expected stdout"
2324
2325         diff -u /dev/null "${tmp_stderr}"
2326         ok $? "${test_name}: expected stderr"
2327
2328         rm -f "${tmp_stdout}"
2329         rm -f "${tmp_stderr}"
2330 }
2331
2332 function list_triggers_matches_mi_ok ()
2333 {
2334         local tmp_stdout
2335         local tmp_stdout_raw
2336         local tmp_stderr
2337
2338         local test_name="$1"
2339         local expected_stdout_file="$2"
2340
2341         tmp_stdout_raw=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2342         tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2343         tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2344
2345         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2346
2347         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2348         ok $? "${test_name}: exit code is 0"
2349
2350         # Pretty-fy xml before further test.
2351         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2352
2353         $MI_VALIDATE "${tmp_stdout}"
2354         ok $? "list-trigger mi is valid"
2355
2356         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2357         ok $? "${test_name}: expected stdout"
2358
2359         diff -u /dev/null "${tmp_stderr}"
2360         ok $? "${test_name}: expected stderr"
2361
2362         rm -f "${tmp_stdout}"
2363         rm -f "${tmp_stdout_raw}"
2364         rm -f "${tmp_stderr}"
2365 }
2366
2367 function validate_path_pattern ()
2368 {
2369         local message=$1
2370         local pattern=$2
2371         # Base path is only used in error case and is used to list the content
2372         # of the base path.
2373         local base_path=$3
2374
2375
2376         [ -f $pattern ]
2377         ret=$?
2378         ok $ret "$message"
2379
2380         if [ "$ret" -ne "0" ]; then
2381                 diag "Path pattern expected: $pattern"
2382                 # List the tracepath for more info. We use find as a recursive
2383                 # directory lister.
2384                 diag "The base path content:"
2385                 find "$base_path" -print
2386         fi
2387 }
2388
2389 function validate_trace_path_ust_uid ()
2390 {
2391         local trace_path=$1
2392         local session_name=$2
2393         local uid=$UID
2394         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2395
2396         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2397 }
2398
2399 function validate_trace_path_ust_uid_network ()
2400 {
2401         local trace_path=$1
2402         local session_name=$2
2403         local base_path=$3
2404         local uid=$UID
2405         local hostname=$HOSTNAME
2406         local pattern
2407         local ret
2408
2409         # If the session was given a network base path (e.g
2410         # 127.0.0.1/my/custom/path on creation, there is no session name
2411         # component to the path on the relayd side. Caller can simply not pass a
2412         # session name for this scenario.
2413         if [ -n "$session_name" ]; then
2414                 session_name="$session_name-$date_time_pattern"
2415                 if [ -n "$base_path" ]; then
2416                         fail "Session name and base path are mutually exclusive"
2417                         return
2418                 fi
2419         fi
2420
2421         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2422
2423         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2424 }
2425
2426 function validate_trace_path_ust_uid_snapshot_network ()
2427 {
2428         local trace_path=$1
2429         local session_name=$2
2430         local snapshot_name=$3
2431         local snapshot_number=$4
2432         local base_path=$5
2433         local hostname=$HOSTNAME
2434         local uid=$UID
2435         local pattern
2436         local ret
2437
2438         # If the session/output was given a network base path (e.g
2439         # 127.0.0.1/my/custom/path on creation, there is no session name
2440         # component to the path on the relayd side. Caller can simply not pass a
2441         # session name for this scenario.
2442         if [ -n "$session_name" ]; then
2443                 session_name="$session_name-$date_time_pattern"
2444                 if [ -n "$base_path" ]; then
2445                         fail "Session name and base path are mutually exclusive"
2446                         return
2447                 fi
2448         fi
2449
2450         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2451
2452         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2453 }
2454
2455 function validate_trace_path_ust_uid_snapshot ()
2456 {
2457         local trace_path=$1
2458         local session_name=$2
2459         local snapshot_name=$3
2460         local snapshot_number=$4
2461         local base_path=$5
2462         local uid=$UID
2463         local pattern
2464         local ret
2465
2466         # If the session/output was given a network base path (e.g
2467         # 127.0.0.1/my/custom/path) on creation, there is no session name
2468         # component to the path on the relayd side. Caller can simply not pass a
2469         # session name for this scenario.
2470         if [ -n "$session_name" ]; then
2471                 session_name="$session_name-$date_time_pattern"
2472                 if [ -n "$base_path" ]; then
2473                         fail "Session name and base path are mutually exclusive"
2474                         return
2475                 fi
2476         fi
2477
2478         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2479
2480         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2481 }
2482
2483 function validate_trace_path_ust_pid ()
2484 {
2485         local trace_path=$1
2486         local session_name=$2
2487         local app_string=$3
2488         local pid=$4
2489         local pattern
2490         local ret
2491
2492         # If the session was given a trace path on creation, there is no session
2493         # name component to the path. Caller can simply not pass a session name
2494         # for this scenario.
2495         if [ -n "$session_name" ]; then
2496                 session_name="$session_name-$date_time_pattern"
2497         fi
2498
2499         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2500
2501         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2502 }
2503
2504 function validate_trace_path_kernel ()
2505 {
2506         local trace_path=$1
2507         local session_name=$2
2508         local pattern
2509
2510         # If the session was given a trace path on creation, there is no session
2511         # name component to the path. Caller can simply not pass a session name
2512         # for this scenario.
2513         if [ -n "$session_name" ]; then
2514                 session_name="$session_name-$date_time_pattern"
2515         fi
2516
2517         pattern="$trace_path/$session_name/kernel/metadata"
2518
2519         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2520 }
2521
2522 function validate_trace_path_kernel_network ()
2523 {
2524         local trace_path=$1
2525         local session_name=$2
2526         local hostname=$HOSTNAME
2527         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2528
2529         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2530 }
2531
2532 function validate_trace_path_kernel_snapshot ()
2533 {
2534         local trace_path=$1
2535         local session_name=$2
2536         local snapshot_name=$3
2537         local snapshot_number=$4
2538         local base_path=$5
2539         local pattern
2540         local ret
2541
2542         # If the session/output was given a network base path (e.g
2543         # 127.0.0.1/my/custom/path on creation, there is no session name
2544         # component to the path on the relayd side. Caller can simply not pass a
2545         # session name for this scenario.
2546         if [ -n "$session_name" ]; then
2547                 session_name="$session_name-$date_time_pattern"
2548                 if [ -n "$base_path" ]; then
2549                         fail "Session name and base path are mutually exclusive"
2550                         return
2551                 fi
2552         fi
2553
2554         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2555
2556         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2557 }
2558
2559 function validate_trace_path_kernel_snapshot_network ()
2560 {
2561         local trace_path=$1
2562         local session_name=$2
2563         local snapshot_name=$3
2564         local snapshot_number=$4
2565         local base_path=$5
2566         local hostname=$HOSTNAME
2567         local pattern
2568         local ret
2569
2570         # If the session/output was given a network base path (e.g
2571         # 127.0.0.1/my/custom/path on creation, there is no session name
2572         # component to the path on the relayd side. Caller can simply not pass a
2573         # session name for this scenario.
2574         if [ -n "$session_name" ]; then
2575                 session_name="$session_name-$date_time_pattern"
2576                 if [ -n "$base_path" ]; then
2577                         fail "Session name and base path are mutually exclusive"
2578                         return
2579                 fi
2580         fi
2581
2582         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2583
2584         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2585 }