tests/utils/utils.sh

   1 # Copyright (C) - 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # This library is free software; you can redistribute it and/or modify it under
   4 # the terms of the GNU Lesser General Public License as published by the Free
   5 # Software Foundation; version 2.1 of the License.
   6 #
   7 # This library is distributed in the hope that it will be useful, but WITHOUT
   8 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   9 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  10 # details.
  11 #
  12 # You should have received a copy of the GNU Lesser General Public License
  13 # along with this library; if not, write to the Free Software Foundation, Inc.,
  14 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  15
  16 SESSIOND_BIN="lttng-sessiond"
  17 SESSIOND_MATCH=".*lttng-sess.*"
  18 RUNAS_BIN="lttng-runas"
  19 RUNAS_MATCH=".*lttng-runas.*"
  20 CONSUMERD_BIN="lttng-consumerd"
  21 CONSUMERD_MATCH=".*lttng-consumerd.*"
  22 RELAYD_BIN="lttng-relayd"
  23 RELAYD_MATCH=".*lttng-relayd.*"
  24 LTTNG_BIN="lttng"
  25 BABELTRACE_BIN="babeltrace"
  26 OUTPUT_DEST=/dev/null
  27 ERROR_OUTPUT_DEST=/dev/null
  28
  29 # Minimal kernel version supported for session daemon tests
  30 KERNEL_MAJOR_VERSION=2
  31 KERNEL_MINOR_VERSION=6
  32 KERNEL_PATCHLEVEL_VERSION=27
  33
  34 # We set the default UST register timeout and network and app socket timeout to
  35 # "wait forever", so that basic tests don't have to worry about hitting
  36 # timeouts on busy systems. Specialized tests should test those corner-cases.
  37 export LTTNG_UST_REGISTER_TIMEOUT=-1
  38 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  39 export LTTNG_APP_SOCKET_TIMEOUT=-1
  40
  41 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  42 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  43 # its own sessiond if none is running. It also ensures that 'lttng create'
  44 # fails when no sessiond is running.
  45 export LTTNG_SESSIOND_PATH="/bin/true"
  46
  47 source $TESTDIR/utils/tap/tap.sh
  48
  49 if [ -z $LTTNG_TEST_TEARDOWN_TIMEOUT ]; then
  50         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  51 fi
  52
  53 function full_cleanup ()
  54 {
  55         # Try to kill daemons gracefully
  56         stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  57         stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  58
  59         # If daemons are still present, forcibly kill them
  60         stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  61         stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  62         stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  63
  64         # Disable trap for SIGTERM since the following kill to the
  65         # pidgroup will be SIGTERM. Otherwise it loops.
  66         # The '-' before the pid number ($$) indicates 'kill' to signal the
  67         # whole process group.
  68         trap - SIGTERM && kill -- -$$
  69         exit 1
  70 }
  71
  72 function null_pipes ()
  73 {
  74         exec 0>/dev/null
  75         exec 1>/dev/null
  76         exec 2>/dev/null
  77 }
  78
  79 trap full_cleanup SIGINT SIGTERM
  80
  81 # perl prove closes its child pipes before giving it a chance to run its
  82 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
  83 # to allow those trap handlers to proceed.
  84
  85 trap null_pipes SIGPIPE
  86
  87 function print_ok ()
  88 {
  89         # Check if we are a terminal
  90         if [ -t 1 ]; then
  91                 echo -e "\e[1;32mOK\e[0m"
  92         else
  93                 echo -e "OK"
  94         fi
  95 }
  96
  97 function print_fail ()
  98 {
  99         # Check if we are a terminal
 100         if [ -t 1 ]; then
 101                 echo -e "\e[1;31mFAIL\e[0m"
 102         else
 103                 echo -e "FAIL"
 104         fi
 105 }
 106
 107 function print_test_banner ()
 108 {
 109         local desc="$1"
 110         diag "$desc"
 111 }
 112
 113 function validate_kernel_version ()
 114 {
 115         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 116         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 117                 return 0
 118         fi
 119         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 120                 return 0
 121         fi
 122         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 123                 return 0
 124         fi
 125         return 1
 126 }
 127
 128 # Generate a random string
 129 #  $1 = number of characters; defaults to 16
 130 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 131 function randstring()
 132 {
 133         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 134         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 135         echo
 136 }
 137
 138 # Return the number of _configured_ CPUs.
 139 function conf_proc_count()
 140 {
 141         getconf _NPROCESSORS_CONF
 142         if [ $? -ne 0 ]; then
 143                 diag "Failed to get the number of configured CPUs"
 144         fi
 145         echo
 146 }
 147
 148 # Check if base lttng-modules are present.
 149 # Bail out on failure
 150 function validate_lttng_modules_present ()
 151 {
 152         modprobe -n lttng-tracer 2>/dev/null
 153         if [ $? -ne 0  ]; then
 154                 BAIL_OUT "LTTng modules not detected."
 155         fi
 156 }
 157
 158 function enable_kernel_lttng_event
 159 {
 160         local withtap="$1"
 161         local expected_to_fail="$2"
 162         local sess_name="$3"
 163         local event_name="$4"
 164         local channel_name="$5"
 165
 166         if [ -z "$event_name" ]; then
 167                 # Enable all event if no event name specified
 168                 event_name="-a"
 169         fi
 170
 171         if [ -z "$channel_name" ]; then
 172                 # default channel if none specified
 173                 chan=""
 174         else
 175                 chan="-c $channel_name"
 176         fi
 177
 178         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 179         ret=$?
 180         if [[ $expected_to_fail -eq "1" ]]; then
 181                 test $ret -ne "0"
 182                 ret=$?
 183                 if [ $withtap -eq "1" ]; then
 184                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 185                 fi
 186         else
 187                 if [ $withtap -eq "1" ]; then
 188                         ok $ret "Enable kernel event $event_name for session $sess_name"
 189                 fi
 190         fi
 191 }
 192
 193 function enable_kernel_lttng_event_ok ()
 194 {
 195         enable_kernel_lttng_event 1 0 "$@"
 196 }
 197
 198 function enable_kernel_lttng_event_fail ()
 199 {
 200         enable_kernel_lttng_event 1 1 "$@"
 201 }
 202
 203 function enable_kernel_lttng_event_notap ()
 204 {
 205         enable_kernel_lttng_event 0 0 "$@"
 206 }
 207
 208 # Old interface
 209 function lttng_enable_kernel_event
 210 {
 211         enable_kernel_lttng_event_ok "$@"
 212 }
 213
 214 function lttng_enable_kernel_syscall()
 215 {
 216         local expected_to_fail=$1
 217         local sess_name=$2
 218         local syscall_name=$3
 219         local channel_name=$4
 220
 221         if [ -z $syscall_name ]; then
 222                 # Enable all event if no syscall name specified
 223                 syscall_name="-a"
 224         fi
 225
 226         if [ -z $channel_name ]; then
 227                 # default channel if none specified
 228                 chan=""
 229         else
 230                 chan="-c $channel_name"
 231         fi
 232
 233         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 234         ret=$?
 235         if [[ $expected_to_fail -eq "1" ]]; then
 236                 test $ret -ne "0"
 237                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 238         else
 239                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 240         fi
 241 }
 242
 243 function lttng_enable_kernel_syscall_ok()
 244 {
 245         lttng_enable_kernel_syscall 0 "$@"
 246 }
 247
 248 function lttng_enable_kernel_syscall_fail()
 249 {
 250         lttng_enable_kernel_syscall 1 "$@"
 251 }
 252
 253 function lttng_disable_kernel_syscall()
 254 {
 255         local expected_to_fail=$1
 256         local sess_name=$2
 257         local syscall_name=$3
 258         local channel_name=$4
 259
 260         if [ -z $syscall_name ]; then
 261                 # Enable all event if no syscall name specified
 262                 syscall_name="-a"
 263         fi
 264
 265         if [ -z $channel_name ]; then
 266                 # default channel if none specified
 267                 chan=""
 268         else
 269                 chan="-c $channel_name"
 270         fi
 271
 272         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 273
 274         ret=$?
 275         if [[ $expected_to_fail -eq "1" ]]; then
 276                 test $ret -ne "0"
 277                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 278         else
 279                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 280         fi
 281 }
 282
 283 function lttng_disable_kernel_syscall_ok()
 284 {
 285         lttng_disable_kernel_syscall 0 "$@"
 286 }
 287
 288 function lttng_disable_kernel_syscall_fail()
 289 {
 290         lttng_disable_kernel_syscall 1 "$@"
 291 }
 292
 293 function lttng_enable_kernel_userspace_probe_event ()
 294 {
 295         local expected_to_fail="$1"
 296         local sess_name="$2"
 297         local target="$3"
 298         local event_name="$4"
 299
 300         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 301         ret=$?
 302         if [[ $expected_to_fail -eq "1" ]]; then
 303                 test $ret -ne "0"
 304                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 305         else
 306                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 307         fi
 308 }
 309
 310 function lttng_enable_kernel_userspace_probe_event_fail ()
 311 {
 312         lttng_enable_kernel_userspace_probe_event 1 "$@"
 313 }
 314
 315 function lttng_enable_kernel_userspace_probe_event_ok ()
 316 {
 317         lttng_enable_kernel_userspace_probe_event 0 "$@"
 318 }
 319
 320 function disable_kernel_lttng_userspace_probe_event_ok ()
 321 {
 322         local sess_name="$1"
 323         local event_name="$2"
 324
 325         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 326         ok $? "Disable kernel event $target for session $sess_name"
 327 }
 328 function lttng_enable_kernel_channel()
 329 {
 330         local withtap=$1
 331         local expected_to_fail=$2
 332         local sess_name=$3
 333         local channel_name=$4
 334         local opt=$5
 335
 336         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 337         ret=$?
 338         if [[ $expected_to_fail -eq "1" ]]; then
 339                 test "$ret" -ne "0"
 340                 ret=$?
 341                 if [ $withtap -eq "1" ]; then
 342                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 343                 fi
 344         else
 345                 if [ $withtap -eq "1" ]; then
 346                         ok $ret "Enable channel $channel_name for session $sess_name"
 347                 fi
 348         fi
 349 }
 350
 351 function lttng_enable_kernel_channel_ok()
 352 {
 353         lttng_enable_kernel_channel 1 0 "$@"
 354 }
 355
 356 function lttng_enable_kernel_channel_fail()
 357 {
 358         lttng_enable_kernel_channel 1 1 "$@"
 359 }
 360
 361 function lttng_enable_kernel_channel_notap()
 362 {
 363         lttng_enable_kernel_channel 0 0 "$@"
 364 }
 365
 366 function enable_kernel_lttng_channel_ok()
 367 {
 368         lttng_enable_kernel_channel 1 0 "$@"
 369 }
 370
 371 function lttng_disable_kernel_channel()
 372 {
 373         local expected_to_fail=$1
 374         local sess_name=$2
 375         local channel_name=$3
 376
 377         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 378         ret=$?
 379         if [[ $expected_to_fail -eq "1" ]]; then
 380                 test "$ret" -ne "0"
 381                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 382         else
 383                 ok $ret "Disable channel $channel_name for session $sess_name"
 384         fi
 385 }
 386
 387 function lttng_disable_kernel_channel_ok()
 388 {
 389         lttng_disable_kernel_channel 0 "$@"
 390 }
 391
 392 function lttng_disable_kernel_channel_fail()
 393 {
 394         lttng_disable_kernel_channel 1 "$@"
 395 }
 396
 397 function start_lttng_relayd_opt()
 398 {
 399         local withtap=$1
 400         local opt=$2
 401
 402         DIR=$(readlink -f "$TESTDIR")
 403
 404         if [ -z "$(pgrep "$RELAYD_MATCH")" ]; then
 405                 # shellcheck disable=SC2086
 406                 "$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 407                 #"$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" $opt -vvv >>/tmp/relayd.log 2>&1 &
 408                 if [ $? -eq 1 ]; then
 409                         if [ "$withtap" -eq "1" ]; then
 410                                 fail "Start lttng-relayd (opt: $opt)"
 411                         fi
 412                         return 1
 413                 else
 414                         if [ "$withtap" -eq "1" ]; then
 415                                 pass "Start lttng-relayd (opt: $opt)"
 416                         fi
 417                 fi
 418         else
 419                 pass "Start lttng-relayd (opt: $opt)"
 420         fi
 421 }
 422
 423 function start_lttng_relayd()
 424 {
 425         start_lttng_relayd_opt 1 "$@"
 426 }
 427
 428 function start_lttng_relayd_notap()
 429 {
 430         start_lttng_relayd_opt 0 "$@"
 431 }
 432
 433 function stop_lttng_relayd_opt()
 434 {
 435         local withtap=$1
 436         local signal=$2
 437
 438         if [ -z "$signal" ]; then
 439                 signal="SIGTERM"
 440         fi
 441
 442         local timeout_s=$3
 443         local dtimeleft_s=
 444
 445         # Multiply time by 2 to simplify integer arithmetic
 446         if [ -n "$timeout_s" ]; then
 447                 dtimeleft_s=$((timeout_s * 2))
 448         fi
 449
 450         local retval=0
 451         local pids=
 452
 453         pids=$(pgrep "$RELAYD_MATCH")
 454         if [ -z "$pids" ]; then
 455                 if [ "$withtap" -eq "1" ]; then
 456                         pass "No relay daemon to kill"
 457                 fi
 458                 return 0
 459         fi
 460
 461         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 462
 463         # shellcheck disable=SC2086
 464         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 465                 retval=1
 466                 if [ "$withtap" -eq "1" ]; then
 467                         fail "Kill relay daemon"
 468                 fi
 469         else
 470                 out=1
 471                 while [ -n "$out" ]; do
 472                         out=$(pgrep "$RELAYD_MATCH")
 473                         if [ -n "$dtimeleft_s" ]; then
 474                                 if [ $dtimeleft_s -lt 0 ]; then
 475                                         out=
 476                                         retval=1
 477                                 fi
 478                                 dtimeleft_s=$((dtimeleft_s - 1))
 479                         fi
 480                         sleep 0.5
 481                 done
 482                 if [ "$withtap" -eq "1" ]; then
 483                         if [ "$retval" -eq "0" ]; then
 484                                 pass "Wait after kill relay daemon"
 485                         else
 486                                 fail "Wait after kill relay daemon"
 487                         fi
 488                 fi
 489         fi
 490         return $retval
 491 }
 492
 493 function stop_lttng_relayd()
 494 {
 495         stop_lttng_relayd_opt 1 "$@"
 496 }
 497
 498 function stop_lttng_relayd_notap()
 499 {
 500         stop_lttng_relayd_opt 0 "$@"
 501 }
 502
 503 #First arg: show tap output
 504 #Second argument: load path for automatic loading
 505 function start_lttng_sessiond_opt()
 506 {
 507         local withtap=$1
 508         local load_path=$2
 509
 510         local env_vars=""
 511         local consumerd=""
 512
 513         local long_bit_value=
 514         long_bit_value=$(getconf LONG_BIT)
 515
 516         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 517                 # Env variable requested no session daemon
 518                 return
 519         fi
 520
 521         DIR=$(readlink -f "$TESTDIR")
 522
 523         # Get long_bit value for 32/64 consumerd
 524         case "$long_bit_value" in
 525                 32)
 526                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 527                         ;;
 528                 64)
 529                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 530                         ;;
 531                 *)
 532                         return
 533                         ;;
 534         esac
 535
 536         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 537         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 538                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 539         fi
 540         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 541
 542         if ! validate_kernel_version; then
 543             fail "Start session daemon"
 544             BAIL_OUT "*** Kernel too old for session daemon tests ***"
 545         fi
 546
 547         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 548         export LTTNG_SESSION_CONFIG_XSD_PATH
 549
 550         if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
 551                 # Have a load path ?
 552                 if [ -n "$load_path" ]; then
 553                         # shellcheck disable=SC2086
 554                         env $env_vars --load "$load_path" --background "$consumerd"
 555                 else
 556                         # shellcheck disable=SC2086
 557                         env $env_vars --background "$consumerd"
 558                 fi
 559                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 560                 status=$?
 561                 if [ "$withtap" -eq "1" ]; then
 562                         ok $status "Start session daemon"
 563                 fi
 564         fi
 565 }
 566
 567 function start_lttng_sessiond()
 568 {
 569         start_lttng_sessiond_opt 1 "$@"
 570 }
 571
 572 function start_lttng_sessiond_notap()
 573 {
 574         start_lttng_sessiond_opt 0 "$@"
 575 }
 576
 577 function stop_lttng_sessiond_opt()
 578 {
 579         local withtap=$1
 580         local signal=$2
 581
 582         if [ -z "$signal" ]; then
 583                 signal=SIGTERM
 584         fi
 585
 586         local timeout_s=$3
 587         local dtimeleft_s=
 588
 589         # Multiply time by 2 to simplify integer arithmetic
 590         if [ -n "$timeout_s" ]; then
 591                 dtimeleft_s=$((timeout_s * 2))
 592         fi
 593
 594         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 595                 # Env variable requested no session daemon
 596                 return 0
 597         fi
 598
 599         local retval=0
 600
 601         local runas_pids=
 602         runas_pids=$(pgrep "$RUNAS_MATCH")
 603
 604         local pids=
 605         pids=$(pgrep "$SESSIOND_MATCH")
 606
 607         if [ -n "$runas_pids" ]; then
 608                 pids="$pids $runas_pids"
 609         fi
 610
 611         if [ -z "$pids" ]; then
 612                 if [ "$withtap" -eq "1" ]; then
 613                         pass "No session daemon to kill"
 614                 fi
 615                 return 0
 616         fi
 617
 618         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 619
 620         # shellcheck disable=SC2086
 621         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 622                 retval=1
 623                 if [ "$withtap" -eq "1" ]; then
 624                         fail "Kill sessions daemon"
 625                 fi
 626         else
 627                 out=1
 628                 while [ -n "$out" ]; do
 629                         out=$(pgrep "${SESSIOND_MATCH}")
 630                         if [ -n "$dtimeleft_s" ]; then
 631                                 if [ $dtimeleft_s -lt 0 ]; then
 632                                         out=
 633                                         retval=1
 634                                 fi
 635                                 dtimeleft_s=$((dtimeleft_s - 1))
 636                         fi
 637                         sleep 0.5
 638                 done
 639                 out=1
 640                 while [ -n "$out" ]; do
 641                         out=$(pgrep "$CONSUMERD_MATCH")
 642                         if [ -n "$dtimeleft_s" ]; then
 643                                 if [ $dtimeleft_s -lt 0 ]; then
 644                                         out=
 645                                         retval=1
 646                                 fi
 647                                 dtimeleft_s=$((dtimeleft_s - 1))
 648                         fi
 649                         sleep 0.5
 650                 done
 651
 652                 if [ "$withtap" -eq "1" ]; then
 653                         if [ "$retval" -eq "0" ]; then
 654                                 pass "Wait after kill session daemon"
 655                         else
 656                                 fail "Wait after kill session daemon"
 657                         fi
 658                 fi
 659         fi
 660         if [ "$signal" = "SIGKILL" ]; then
 661                 if [ "$(id -u)" -eq "0" ]; then
 662                         local modules=
 663                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 664
 665                         if [ -n "$modules" ]; then
 666                                 diag "Unloading all LTTng modules"
 667                                 modprobe -r "$modules"
 668                         fi
 669                 fi
 670         fi
 671
 672         return $retval
 673 }
 674
 675 function stop_lttng_sessiond()
 676 {
 677         stop_lttng_sessiond_opt 1 "$@"
 678 }
 679
 680 function stop_lttng_sessiond_notap()
 681 {
 682         stop_lttng_sessiond_opt 0 "$@"
 683 }
 684
 685 function sigstop_lttng_sessiond_opt()
 686 {
 687         local withtap=$1
 688         local signal=SIGSTOP
 689
 690         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 691                 # Env variable requested no session daemon
 692                 return
 693         fi
 694
 695         PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
 696
 697         if [ "$withtap" -eq "1" ]; then
 698                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
 699         fi
 700
 701         # shellcheck disable=SC2086
 702         if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 703                 if [ "$withtap" -eq "1" ]; then
 704                         fail "Sending SIGSTOP to session daemon"
 705                 fi
 706         else
 707                 out=1
 708                 while [ $out -ne 0 ]; do
 709                         pid="$(pgrep "$SESSIOND_MATCH")"
 710
 711                         # Wait until state becomes stopped for session
 712                         # daemon(s).
 713                         out=0
 714                         for sessiond_pid in $pid; do
 715                                 state="$(ps -p "$sessiond_pid" -o state= )"
 716                                 if [[ -n "$state" && "$state" != "T" ]]; then
 717                                         out=1
 718                                 fi
 719                         done
 720                         sleep 0.5
 721                 done
 722                 if [ "$withtap" -eq "1" ]; then
 723                         pass "Sending SIGSTOP to session daemon"
 724                 fi
 725         fi
 726 }
 727
 728 function sigstop_lttng_sessiond()
 729 {
 730         sigstop_lttng_sessiond_opt 1 "$@"
 731 }
 732
 733 function sigstop_lttng_sessiond_notap()
 734 {
 735         sigstop_lttng_sessiond_opt 0 "$@"
 736 }
 737
 738 function stop_lttng_consumerd_opt()
 739 {
 740         local withtap=$1
 741         local signal=$2
 742
 743         if [ -z "$signal" ]; then
 744                 signal=SIGTERM
 745         fi
 746
 747         local timeout_s=$3
 748         local dtimeleft_s=
 749
 750         # Multiply time by 2 to simplify integer arithmetic
 751         if [ -n "$timeout_s" ]; then
 752                 dtimeleft_s=$((timeout_s * 2))
 753         fi
 754
 755         local retval=0
 756
 757         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 758
 759         if [ -z "$PID_CONSUMERD" ]; then
 760                 if [ "$withtap" -eq "1" ]; then
 761                         pass "No consumer daemon to kill"
 762                 fi
 763                 return 0
 764         fi
 765
 766         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 767
 768         # shellcheck disable=SC2086
 769         if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 770                 retval=1
 771                 if [ "$withtap" -eq "1" ]; then
 772                         fail "Kill consumer daemon"
 773                 fi
 774         else
 775                 out=1
 776                 while [ $out -ne 0 ]; do
 777                         pid="$(pgrep "$CONSUMERD_MATCH")"
 778
 779                         # If consumerds are still present check their status.
 780                         # A zombie status qualifies the consumerd as *killed*
 781                         out=0
 782                         for consumer_pid in $pid; do
 783                                 state="$(ps -p "$consumer_pid" -o state= )"
 784                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 785                                         out=1
 786                                 fi
 787                         done
 788                         if [ -n "$dtimeleft_s" ]; then
 789                                 if [ $dtimeleft_s -lt 0 ]; then
 790                                         out=0
 791                                         retval=1
 792                                 fi
 793                                 dtimeleft_s=$((dtimeleft_s - 1))
 794                         fi
 795                         sleep 0.5
 796                 done
 797                 if [ "$withtap" -eq "1" ]; then
 798                         if [ "$retval" -eq "0" ]; then
 799                                 pass "Wait after kill consumer daemon"
 800                         else
 801                                 fail "Wait after kill consumer daemon"
 802                         fi
 803                 fi
 804         fi
 805
 806         return $retval
 807 }
 808
 809 function stop_lttng_consumerd()
 810 {
 811         stop_lttng_consumerd_opt 1 "$@"
 812 }
 813
 814 function stop_lttng_consumerd_notap()
 815 {
 816         stop_lttng_consumerd_opt 0 "$@"
 817 }
 818
 819 function sigstop_lttng_consumerd_opt()
 820 {
 821         local withtap=$1
 822         local signal=SIGSTOP
 823
 824         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 825
 826         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 827
 828         # shellcheck disable=SC2086
 829         kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 830         retval=$?
 831
 832         if [ $retval -eq 1 ]; then
 833                 if [ "$withtap" -eq "1" ]; then
 834                         fail "Sending SIGSTOP to consumer daemon"
 835                 fi
 836                 return 1
 837         else
 838                 out=1
 839                 while [ $out -ne 0 ]; do
 840                         pid="$(pgrep "$CONSUMERD_MATCH")"
 841
 842                         # Wait until state becomes stopped for all
 843                         # consumers.
 844                         out=0
 845                         for consumer_pid in $pid; do
 846                                 state="$(ps -p "$consumer_pid" -o state= )"
 847                                 if [[ -n "$state" && "$state" != "T" ]]; then
 848                                         out=1
 849                                 fi
 850                         done
 851                         sleep 0.5
 852                 done
 853                 if [ "$withtap" -eq "1" ]; then
 854                         pass "Sending SIGSTOP to consumer daemon"
 855                 fi
 856         fi
 857         return $retval
 858 }
 859
 860 function sigstop_lttng_consumerd()
 861 {
 862         sigstop_lttng_consumerd_opt 1 "$@"
 863 }
 864
 865 function sigstop_lttng_consumerd_notap()
 866 {
 867         sigstop_lttng_consumerd_opt 0 "$@"
 868 }
 869
 870 function list_lttng_with_opts ()
 871 {
 872         local opts=$1
 873         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 874         ok $? "Lttng-tool list command with option $opts"
 875 }
 876
 877 function create_lttng_session_no_output ()
 878 {
 879         local sess_name=$1
 880         local opts="${@:2}"
 881
 882         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 883         ok $? "Create session $sess_name in no-output mode"
 884 }
 885
 886 function create_lttng_session_uri () {
 887         local sess_name=$1
 888         local uri=$2
 889         local opts="${@:3}"
 890
 891         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 892         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
 893 }
 894
 895 function create_lttng_session ()
 896 {
 897         local withtap=$1
 898         local expected_to_fail=$2
 899         local sess_name=$3
 900         local trace_path=$4
 901         local opt=$5
 902
 903         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path $opt > $OUTPUT_DEST
 904         ret=$?
 905         if [ $expected_to_fail -eq "1" ]; then
 906                 test "$ret" -ne "0"
 907                 ret=$?
 908                 if [ $withtap -eq "1" ]; then
 909                         ok $ret "Create session $sess_name in $trace_path failed as expected"
 910                 fi
 911         else
 912                 if [ $withtap -eq "1" ]; then
 913                         ok $ret "Create session $sess_name in $trace_path"
 914                 fi
 915         fi
 916         return $ret
 917 }
 918
 919 function create_lttng_session_ok ()
 920 {
 921         create_lttng_session 1 0 "$@"
 922 }
 923
 924 function create_lttng_session_fail ()
 925 {
 926         create_lttng_session 1 1 "$@"
 927 }
 928
 929 function create_lttng_session_notap ()
 930 {
 931         create_lttng_session 0 0 "$@"
 932 }
 933
 934
 935 function enable_ust_lttng_channel ()
 936 {
 937         local withtap=$1
 938         local expected_to_fail=$2
 939         local sess_name=$3
 940         local channel_name=$4
 941         local opt=$5
 942
 943         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 944         ret=$?
 945         if [[ $expected_to_fail -eq "1" ]]; then
 946                 test "$ret" -ne "0"
 947                 ret=$?
 948                 if [ $withtap -eq "1" ]; then
 949                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 950                 fi
 951         else
 952                 if [ $withtap -eq "1" ]; then
 953                         ok $ret "Enable channel $channel_name for session $sess_name"
 954                 fi
 955         fi
 956         return $ret
 957 }
 958
 959 function enable_ust_lttng_channel_ok ()
 960 {
 961         enable_ust_lttng_channel 1 0 "$@"
 962 }
 963
 964 function enable_ust_lttng_channel_fail ()
 965 {
 966         enable_ust_lttng_channel 1 1 "$@"
 967 }
 968
 969 function enable_ust_lttng_channel_notap ()
 970 {
 971         enable_ust_lttng_channel 0 0 "$@"
 972 }
 973
 974 function disable_ust_lttng_channel()
 975 {
 976         local sess_name=$1
 977         local channel_name=$2
 978
 979         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 980         ok $? "Disable channel $channel_name for session $sess_name"
 981 }
 982
 983 function enable_lttng_mmap_overwrite_kernel_channel()
 984 {
 985         local sess_name=$1
 986         local channel_name=$2
 987
 988         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 989         ok $? "Enable channel $channel_name for session $sess_name"
 990 }
 991
 992 function enable_lttng_mmap_discard_small_kernel_channel()
 993 {
 994         local sess_name=$1
 995         local channel_name=$2
 996
 997         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 998         ok $? "Enable small discard channel $channel_name for session $sess_name"
 999 }
1000
1001 function enable_lttng_mmap_overwrite_small_kernel_channel()
1002 {
1003         local sess_name=$1
1004         local channel_name=$2
1005
1006         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1007         ok $? "Enable small discard channel $channel_name for session $sess_name"
1008 }
1009
1010 function enable_lttng_mmap_overwrite_ust_channel()
1011 {
1012         local sess_name=$1
1013         local channel_name=$2
1014
1015         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1016         ok $? "Enable channel $channel_name for session $sess_name"
1017 }
1018
1019 function enable_ust_lttng_event ()
1020 {
1021         local withtap=$1
1022         local expected_to_fail=$2
1023         local sess_name=$3
1024         local event_name="$4"
1025         local channel_name=$5
1026
1027         if [ -z $channel_name ]; then
1028                 # default channel if none specified
1029                 chan=""
1030         else
1031                 chan="-c $channel_name"
1032         fi
1033
1034         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1035         ret=$?
1036         if [[ $expected_to_fail -eq "1" ]]; then
1037                 test $ret -ne "0"
1038                 ret=$?
1039                 if [[ $withtap -eq "1" ]]; then
1040                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1041                 fi
1042         else
1043                 if [[ $withtap -eq "1" ]]; then
1044                         ok $ret "Enable ust event $event_name for session $sess_name"
1045                 fi
1046         fi
1047         return $ret
1048 }
1049
1050 function enable_ust_lttng_event_ok ()
1051 {
1052         enable_ust_lttng_event 1 0 "$@"
1053 }
1054
1055 function enable_ust_lttng_event_fail ()
1056 {
1057         enable_ust_lttng_event 1 1 "$@"
1058 }
1059
1060 function enable_ust_lttng_event_notap ()
1061 {
1062         enable_ust_lttng_event 0 0 "$@"
1063 }
1064
1065 function enable_jul_lttng_event()
1066 {
1067         sess_name=$1
1068         event_name="$2"
1069         channel_name=$3
1070
1071         if [ -z $channel_name ]; then
1072                 # default channel if none specified
1073                 chan=""
1074         else
1075                 chan="-c $channel_name"
1076         fi
1077
1078         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1079         ok $? "Enable JUL event $event_name for session $sess_name"
1080 }
1081
1082 function enable_jul_lttng_event_loglevel()
1083 {
1084         local sess_name=$1
1085         local event_name="$2"
1086         local loglevel=$3
1087         local channel_name=$4
1088
1089         if [ -z $channel_name ]; then
1090                 # default channel if none specified
1091                 chan=""
1092         else
1093                 chan="-c $channel_name"
1094         fi
1095
1096         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1097         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1098 }
1099
1100 function enable_log4j_lttng_event()
1101 {
1102         sess_name=$1
1103         event_name="$2"
1104         channel_name=$3
1105
1106         if [ -z $channel_name ]; then
1107                 # default channel if none specified
1108                 chan=""
1109         else
1110                 chan="-c $channel_name"
1111         fi
1112
1113         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1114         ok $? "Enable LOG4J event $event_name for session $sess_name"
1115 }
1116
1117 function enable_log4j_lttng_event_loglevel()
1118 {
1119         local sess_name=$1
1120         local event_name="$2"
1121         local loglevel=$3
1122         local channel_name=$4
1123
1124         if [ -z $channel_name ]; then
1125                 # default channel if none specified
1126                 chan=""
1127         else
1128                 chan="-c $channel_name"
1129         fi
1130
1131         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1132         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
1133 }
1134
1135 function enable_python_lttng_event()
1136 {
1137         sess_name=$1
1138         event_name="$2"
1139         channel_name=$3
1140
1141         if [ -z $channel_name ]; then
1142                 # default channel if none specified
1143                 chan=""
1144         else
1145                 chan="-c $channel_name"
1146         fi
1147
1148         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1149         ok $? "Enable Python event $event_name for session $sess_name"
1150 }
1151
1152 function enable_python_lttng_event_loglevel()
1153 {
1154         local sess_name=$1
1155         local event_name="$2"
1156         local loglevel=$3
1157         local channel_name=$4
1158
1159         if [ -z $channel_name ]; then
1160                 # default channel if none specified
1161                 chan=""
1162         else
1163                 chan="-c $channel_name"
1164         fi
1165
1166         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1167         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1168 }
1169
1170 function enable_ust_lttng_event_filter()
1171 {
1172         local sess_name="$1"
1173         local event_name="$2"
1174         local filter="$3"
1175
1176         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1177         ok $? "Enable event $event_name with filtering for session $sess_name"
1178 }
1179
1180 function enable_ust_lttng_event_loglevel()
1181 {
1182         local sess_name="$1"
1183         local event_name="$2"
1184         local loglevel="$3"
1185
1186         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1187         ok $? "Enable event $event_name with loglevel $loglevel"
1188 }
1189
1190 function enable_ust_lttng_event_loglevel_only()
1191 {
1192         local sess_name="$1"
1193         local event_name="$2"
1194         local loglevel="$3"
1195
1196         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1197         ok $? "Enable event $event_name with loglevel-only $loglevel"
1198 }
1199
1200 function disable_ust_lttng_event ()
1201 {
1202         local sess_name="$1"
1203         local event_name="$2"
1204         local channel_name="$3"
1205
1206         if [ -z $channel_name ]; then
1207                 # default channel if none specified
1208                 chan=""
1209         else
1210                 chan="-c $channel_name"
1211         fi
1212
1213         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1214         ok $? "Disable event $event_name for session $sess_name"
1215 }
1216
1217 function disable_jul_lttng_event ()
1218 {
1219         local sess_name="$1"
1220         local event_name="$2"
1221
1222         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1223         ok $? "Disable JUL event $event_name for session $sess_name"
1224 }
1225
1226 function disable_log4j_lttng_event ()
1227 {
1228         local sess_name="$1"
1229         local event_name="$2"
1230
1231         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1232         ok $? "Disable LOG4J event $event_name for session $sess_name"
1233 }
1234
1235 function disable_python_lttng_event ()
1236 {
1237         local sess_name="$1"
1238         local event_name="$2"
1239
1240         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1241         ok $? "Disable Python event $event_name for session $sess_name"
1242 }
1243
1244 function start_lttng_tracing_opt ()
1245 {
1246         local withtap=$1
1247         local expected_to_fail=$2
1248         local sess_name=$3
1249
1250         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1251         ret=$?
1252         if [[ $expected_to_fail -eq "1" ]]; then
1253                 test "$ret" -ne "0"
1254                 ret=$?
1255                 if [ $withtap -eq "1" ]; then
1256                         ok $? "Start tracing for session $sess_name failed as expected"
1257                 fi
1258         else
1259                 if [ $withtap -eq "1" ]; then
1260                         ok $ret "Start tracing for session $sess_name"
1261                 fi
1262         fi
1263 }
1264
1265 function start_lttng_tracing_ok ()
1266 {
1267         start_lttng_tracing_opt 1 0 "$@"
1268 }
1269
1270 function start_lttng_tracing_fail ()
1271 {
1272         start_lttng_tracing_opt 1 1 "$@"
1273 }
1274
1275 function start_lttng_tracing_notap ()
1276 {
1277         start_lttng_tracing_opt 0 1 "$@"
1278 }
1279
1280 function stop_lttng_tracing_opt ()
1281 {
1282         local withtap=$1
1283         local expected_to_fail=$2
1284         local sess_name=$3
1285
1286         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1287         ret=$?
1288         if [[ $expected_to_fail -eq "1" ]]; then
1289                 test "$ret" -ne "0"
1290                 ret=$?
1291                 if [ $withtap -eq "1" ]; then
1292                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1293                 fi
1294         else
1295                 if [ $withtap -eq "1" ]; then
1296                         ok $ret "Stop lttng tracing for session $sess_name"
1297                 fi
1298         fi
1299 }
1300
1301 function stop_lttng_tracing_ok ()
1302 {
1303         stop_lttng_tracing_opt 1 0 "$@"
1304 }
1305
1306 function stop_lttng_tracing_fail ()
1307 {
1308         stop_lttng_tracing_opt 1 1 "$@"
1309 }
1310
1311 function stop_lttng_tracing_notap ()
1312 {
1313         stop_lttng_tracing_opt 0 0 "$@"
1314 }
1315
1316 function destroy_lttng_session ()
1317 {
1318         local withtap=$1
1319         local expected_to_fail=$2
1320         local sess_name=$3
1321
1322         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1323         ret=$?
1324         if [[ $expected_to_fail -eq "1" ]]; then
1325                 test "$ret" -ne "0"
1326                 ret=$?
1327                 if [ $withtap -eq "1" ]; then
1328                         ok $ret "Destroy session $sess_name failed as expected"
1329                 fi
1330         else
1331                 if [ $withtap -eq "1" ]; then
1332                         ok $ret "Destroy session $sess_name"
1333                 fi
1334         fi
1335 }
1336
1337 function destroy_lttng_session_ok ()
1338 {
1339         destroy_lttng_session 1 0 "$@"
1340
1341 }
1342
1343 function destroy_lttng_session_fail ()
1344 {
1345         destroy_lttng_session 1 1 "$@"
1346 }
1347
1348 function destroy_lttng_session_notap ()
1349 {
1350         destroy_lttng_session 0 0 "$@"
1351 }
1352
1353 function destroy_lttng_sessions ()
1354 {
1355         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1356         ok $? "Destroy all lttng sessions"
1357 }
1358
1359 function lttng_snapshot_add_output ()
1360 {
1361         local expected_to_fail=$1
1362         local sess_name=$2
1363         local trace_path=$3
1364
1365         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1366         ret=$?
1367         if [[ $expected_to_fail -eq 1 ]]; then
1368                 test "$ret" -ne "0"
1369                 ok $? "Added snapshot output file://$trace_path failed as expected"
1370         else
1371                 ok $ret "Added snapshot output file://$trace_path"
1372         fi
1373 }
1374
1375 function lttng_snapshot_add_output_ok ()
1376 {
1377         lttng_snapshot_add_output 0 "$@"
1378 }
1379
1380 function lttng_snapshot_add_output_fail ()
1381 {
1382         lttng_snapshot_add_output 1 "$@"
1383 }
1384
1385 function lttng_snapshot_del_output ()
1386 {
1387         local expected_to_fail=$1
1388         local sess_name=$2
1389         local id=$3
1390
1391         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1392         ret=$?
1393         if [[ $expected_to_fail -eq "1" ]]; then
1394                 test "$ret" -ne "0"
1395                 ok $? "Deleted snapshot output id $id failed as expected"
1396         else
1397                 ok $ret "Deleted snapshot output id $id"
1398         fi
1399 }
1400
1401 function lttng_snapshot_del_output_ok ()
1402 {
1403         lttng_snapshot_del_output 0 "$@"
1404 }
1405
1406 function lttng_snapshot_del_output_fail ()
1407 {
1408         lttng_snapshot_del_output 1 "$@"
1409 }
1410
1411 function lttng_snapshot_record ()
1412 {
1413         local sess_name=$1
1414         local trace_path=$2
1415
1416         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1417         ok $? "Snapshot recorded"
1418 }
1419
1420 function lttng_snapshot_list ()
1421 {
1422         local sess_name=$1
1423         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1424         ok $? "Snapshot list"
1425 }
1426
1427 function lttng_save()
1428 {
1429         local sess_name=$1
1430         local opts=$2
1431
1432         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1433         ok $? "Session saved"
1434 }
1435
1436 function lttng_load()
1437 {
1438         local expected_to_fail=$1
1439         local opts=$2
1440
1441         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1442         ret=$?
1443         if [[ $expected_to_fail -eq "1" ]]; then
1444                 test $ret -ne "0"
1445                 ok $? "Load command failed as expected with opts: $opts"
1446         else
1447                 ok $ret "Load command with opts: $opts"
1448         fi
1449 }
1450
1451 function lttng_load_ok()
1452 {
1453         lttng_load 0 "$@"
1454 }
1455
1456 function lttng_load_fail()
1457 {
1458         lttng_load 1 "$@"
1459 }
1460
1461 function lttng_track()
1462 {
1463         local expected_to_fail=$1
1464         local opts=$2
1465         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1466         ret=$?
1467         if [[ $expected_to_fail -eq "1" ]]; then
1468                 test $ret -ne "0"
1469                 ok $? "Track command failed as expected with opts: $opts"
1470         else
1471                 ok $ret "Track command with opts: $opts"
1472         fi
1473 }
1474
1475 function lttng_track_ok()
1476 {
1477         lttng_track 0 "$@"
1478 }
1479
1480 function lttng_track_fail()
1481 {
1482         lttng_track 1 "$@"
1483 }
1484
1485 function lttng_untrack()
1486 {
1487         local expected_to_fail=$1
1488         local opts=$2
1489         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1490         ret=$?
1491         if [[ $expected_to_fail -eq "1" ]]; then
1492                 test $ret -ne "0"
1493                 ok $? "Untrack command failed as expected with opts: $opts"
1494         else
1495                 ok $ret "Untrack command with opts: $opts"
1496         fi
1497 }
1498
1499 function lttng_untrack_ok()
1500 {
1501         lttng_untrack 0 "$@"
1502 }
1503
1504 function lttng_untrack_fail()
1505 {
1506         lttng_untrack 1 "$@"
1507 }
1508
1509 function lttng_track_pid_ok()
1510 {
1511         PID=$1
1512         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1513         ok $? "Lttng track pid on the kernel domain"
1514 }
1515
1516 function lttng_untrack_kernel_all_ok()
1517 {
1518         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1519         ok $? "Lttng untrack all pid on the kernel domain"
1520 }
1521
1522 function lttng_add_context_list()
1523 {
1524         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1525         ret=$?
1526         ok $ret "Context listing"
1527 }
1528
1529 function add_context_lttng()
1530 {
1531         local expected_to_fail="$1"
1532         local domain="$2"
1533         local session_name="$3"
1534         local channel_name="$4"
1535         local type="$5"
1536
1537         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1538         ret=$?
1539         if [[ $expected_to_fail -eq "1" ]]; then
1540                 test $ret -ne "0"
1541                 ok $? "Add context command failed as expected for type: $type"
1542         else
1543                 ok $ret "Add context command for type: $type"
1544         fi
1545 }
1546
1547 function add_context_ust_ok()
1548 {
1549         add_context_lttng 0 -u "$@"
1550 }
1551
1552 function add_context_ust_fail()
1553 {
1554         add_context_lttng 1 -u "$@"
1555 }
1556
1557 function add_context_kernel_ok()
1558 {
1559         add_context_lttng 0 -k "$@"
1560 }
1561
1562 function add_context_kernel_fail()
1563 {
1564         add_context_lttng 1 -k "$@"
1565 }
1566
1567 function validate_metadata_event ()
1568 {
1569         local event_name=$1
1570         local nr_event_id=$2
1571         local trace_path=$3
1572
1573         local metadata_file=$(find $trace_path | grep metadata)
1574         local metadata_path=$(dirname $metadata_file)
1575
1576         which $BABELTRACE_BIN >/dev/null
1577         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1578
1579         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1580
1581         if [ "$count" -ne "$nr_event_id" ]; then
1582                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1583                 diag "$count matching event id found in metadata"
1584         else
1585                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1586         fi
1587
1588 }
1589
1590 function trace_matches ()
1591 {
1592         local event_name=$1
1593         local nr_iter=$2
1594         local trace_path=$3
1595
1596         which $BABELTRACE_BIN >/dev/null
1597         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1598
1599         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1600
1601         if [ "$count" -ne "$nr_iter" ]; then
1602                 fail "Trace match"
1603                 diag "$count matching events found in trace"
1604         else
1605                 pass "Trace match"
1606         fi
1607 }
1608
1609 function trace_match_only()
1610 {
1611         local event_name=$1
1612         local nr_iter=$2
1613         local trace_path=$3
1614
1615         which $BABELTRACE_BIN >/dev/null
1616         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1617
1618         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1619         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1620
1621         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1622                 pass "Trace match with $total event $event_name"
1623         else
1624                 fail "Trace match"
1625                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1626         fi
1627 }
1628
1629 function validate_trace
1630 {
1631         local event_name=$1
1632         local trace_path=$2
1633
1634         which $BABELTRACE_BIN >/dev/null
1635         if [ $? -ne 0 ]; then
1636             skip 0 "Babeltrace binary not found. Skipping trace validation"
1637         fi
1638
1639         OLDIFS=$IFS
1640         IFS=","
1641         for i in $event_name; do
1642                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1643                 if [ "$traced" -ne 0 ]; then
1644                         pass "Validate trace for event $i, $traced events"
1645                 else
1646                         fail "Validate trace for event $i"
1647                         diag "Found $traced occurences of $i"
1648                 fi
1649         done
1650         ret=$?
1651         IFS=$OLDIFS
1652         return $ret
1653 }
1654
1655 function validate_trace_count
1656 {
1657         local event_name=$1
1658         local trace_path=$2
1659         local expected_count=$3
1660
1661         which $BABELTRACE_BIN >/dev/null
1662         if [ $? -ne 0 ]; then
1663             skip 0 "Babeltrace binary not found. Skipping trace validation"
1664         fi
1665
1666         cnt=0
1667         OLDIFS=$IFS
1668         IFS=","
1669         for i in $event_name; do
1670                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1671                 if [ "$traced" -ne 0 ]; then
1672                         pass "Validate trace for event $i, $traced events"
1673                 else
1674                         fail "Validate trace for event $i"
1675                         diag "Found $traced occurences of $i"
1676                 fi
1677                 cnt=$(($cnt + $traced))
1678         done
1679         IFS=$OLDIFS
1680         test $cnt -eq $expected_count
1681         ok $? "Read a total of $cnt events, expected $expected_count"
1682 }
1683
1684 function trace_first_line
1685 {
1686         local trace_path=$1
1687
1688         which $BABELTRACE_BIN >/dev/null
1689         if [ $? -ne 0 ]; then
1690             skip 0 "Babeltrace binary not found. Skipping trace validation"
1691         fi
1692
1693         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1694 }
1695
1696 function validate_trace_exp()
1697 {
1698         local event_exp=$1
1699         local trace_path=$2
1700
1701         which $BABELTRACE_BIN >/dev/null
1702         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1703
1704         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
1705         if [ "$traced" -ne 0 ]; then
1706                 pass "Validate trace for expression '${event_exp}', $traced events"
1707         else
1708                 fail "Validate trace for expression '${event_exp}'"
1709                 diag "Found $traced occurences of '${event_exp}'"
1710         fi
1711         ret=$?
1712         return $ret
1713 }
1714
1715 function validate_trace_only_exp()
1716 {
1717         local event_exp=$1
1718         local trace_path=$2
1719
1720         which $BABELTRACE_BIN >/dev/null
1721         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1722
1723         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
1724         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1725
1726         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
1727                 pass "Trace match with $total for expression '${event_exp}'"
1728         else
1729                 fail "Trace match"
1730                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
1731         fi
1732         ret=$?
1733         return $ret
1734 }
1735
1736 function validate_trace_empty()
1737 {
1738         local trace_path=$1
1739
1740         which $BABELTRACE_BIN >/dev/null
1741         if [ $? -ne 0 ]; then
1742             skip 0 "Babeltrace binary not found. Skipping trace validation"
1743         fi
1744
1745         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
1746         ret=$?
1747         if [ $ret -ne 0 ]; then
1748                 fail "Failed to parse trace"
1749                 return $ret
1750         fi
1751
1752         traced=$(echo -n "$events" | wc -l)
1753         if [ "$traced" -eq 0 ]; then
1754                 pass "Validate empty trace"
1755         else
1756                 fail "Validate empty trace"
1757                 diag "Found $traced events in trace"
1758         fi
1759         ret=$?
1760         return $ret
1761 }
1762
1763 function regenerate_metadata ()
1764 {
1765         local expected_to_fail=$1
1766         local sess_name=$2
1767
1768         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1769         ret=$?
1770         if [[ $expected_to_fail -eq "1" ]]; then
1771                 test "$ret" -ne "0"
1772                 ok $? "Expected fail on regenerate metadata $sess_name"
1773         else
1774                 ok $ret "Metadata regenerate $sess_name"
1775         fi
1776 }
1777
1778 function regenerate_metadata_ok ()
1779 {
1780         regenerate_metadata 0 "$@"
1781 }
1782
1783 function regenerate_metadata_fail ()
1784 {
1785         regenerate_metadata 1 "$@"
1786 }
1787
1788 function regenerate_statedump ()
1789 {
1790         local expected_to_fail=$1
1791         local sess_name=$2
1792
1793         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1794         ret=$?
1795         if [[ $expected_to_fail -eq "1" ]]; then
1796                 test "$ret" -ne "0"
1797                 ok $? "Expected fail on regenerate statedump $sess_name"
1798         else
1799                 ok $ret "Metadata regenerate $sess_name"
1800         fi
1801 }
1802
1803 function regenerate_statedump_ok ()
1804 {
1805         regenerate_statedump 0 "$@"
1806 }
1807
1808 function regenerate_statedump_fail ()
1809 {
1810         regenerate_statedump 1 "$@"
1811 }
1812
1813 function rotate_session ()
1814 {
1815         local expected_to_fail=$1
1816         local sess_name=$2
1817
1818         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1819         ret=$?
1820         if [[ $expected_to_fail -eq "1" ]]; then
1821                 test "$ret" -ne "0"
1822                 ok $? "Expected fail on rotate session $sess_name"
1823         else
1824                 ok $ret "Rotate session $sess_name"
1825         fi
1826 }
1827
1828 function rotate_session_ok ()
1829 {
1830         rotate_session 0 "$@"
1831 }
1832
1833 function rotate_session_fail ()
1834 {
1835         rotate_session 1 "$@"
1836 }
1837
1838 function destructive_tests_enabled ()
1839 {
1840         if [ ${LTTNG_ENABLE_DESTRUCTIVE_TESTS} = "will-break-my-system" ]; then
1841                 return 0
1842         else
1843                 return 1
1844         fi
1845 }
1846
1847 function lttng_enable_rotation_timer ()
1848 {
1849         local expected_to_fail=$1
1850         local sess_name=$2
1851         local period=$3
1852
1853         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1854         ret=$?
1855         if [[ $expected_to_fail -eq "1" ]]; then
1856                 test "$ret" -ne "0"
1857                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
1858         else
1859                 ok $ret "Set periodic rotation ($period) of session $sess_name"
1860         fi
1861 }
1862
1863 function lttng_enable_rotation_timer_ok ()
1864 {
1865         lttng_enable_rotation_timer 0 $@
1866 }
1867
1868 function lttng_enable_rotation_timer_fail ()
1869 {
1870         lttng_enable_rotation_timer 1 $@
1871 }
1872
1873 function lttng_enable_rotation_size ()
1874 {
1875         local expected_to_fail=$1
1876         local sess_name=$2
1877         local size=$3
1878
1879         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1880         ret=$?
1881         if [[ $expected_to_fail -eq "1" ]]; then
1882                 test "$ret" -ne "0"
1883                 ok $? "Expected fail on rotate session $sess_name"
1884         else
1885                 ok $ret "Rotate session $sess_name"
1886         fi
1887 }
1888
1889 function lttng_enable_rotation_size_ok ()
1890 {
1891         lttng_enable_rotation_size 0 $@
1892 }
1893
1894 function lttng_enable_rotation_size_fail ()
1895 {
1896         lttng_enable_rotation_size 1 $@
1897 }