tests/regression/tools/health/test_health.sh

   1 # Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
   2 # Copyright (C) - 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   3 #
   4 # This program is free software; you can redistribute it and/or modify it
   5 # under the terms of the GNU General Public License, version 2 only, as
   6 # published by the Free Software Foundation.
   7 #
   8 # This program is distributed in the hope that it will be useful, but WITHOUT
   9 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 # more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along with
  14 # this program; if not, write to the Free Software Foundation, Inc., 51
  15 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 TESTDIR=${CURDIR}/../../..
  18 LTTNG_BIN="lttng"
  19 UST_EVENT_NAME="tp:tptest"
  20 KERNEL_EVENT_NAME="sched_switch"
  21 CHANNEL_NAME="testchan"
  22 HEALTH_CHECK_BIN="health_check"
  23 NUM_TESTS=90
  24 SLEEP_TIME=30
  25
  26 source $TESTDIR/utils/utils.sh
  27
  28 function lttng_create_session_uri
  29 {
  30         # Create session with default path
  31         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $SESSION_NAME \
  32                 -U net://localhost >/dev/null 2>&1
  33         ok $? "Create session with default path"
  34 }
  35
  36 function report_errors
  37 {
  38         test_thread_error_string="$1"
  39         test_relayd="$2"
  40         err_no_relayd_match="Error querying relayd health"
  41
  42         # Check for health errors
  43         # Include inability to contact relayd health as an expected
  44         # error, since this can happen whenever the relayd shutdown due
  45         # to an error in any thread.
  46         out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l)
  47         if [ $test_relayd -ne 0 ]; then
  48                 outerr=$(grep "${err_no_relayd_match}" ${STDERR_PATH} | wc -l)
  49         else
  50                 outerr=0
  51         fi
  52         if [ $out -eq 0 ] && [ $outerr -eq 0 ]; then
  53                 fail "Validation failure"
  54                 diag "Health returned:"
  55                 diag "stdout:"
  56                 file=${STDOUT_PATH}
  57                 while read line ; do
  58                     diag "$line"
  59                 done < ${file}
  60
  61                 diag "stderr:"
  62                 file=${STDERR_PATH}
  63                 while read line ; do
  64                     diag "$line"
  65                 done < ${file}
  66         else
  67                 pass "Validation OK"
  68         fi
  69 }
  70
  71 function test_health
  72 {
  73         test_suffix="$1"
  74         test_thread_name="$2"
  75         test_thread_error_string="$3"
  76         test_needs_root="$4"
  77         test_consumerd="$5"
  78         test_relayd="$6"
  79
  80         diag "Test health problem detection with ${test_thread_name}"
  81
  82         # Set the socket timeout to 5 so the health check detection
  83         # happens within 25 s
  84         export LTTNG_NETWORK_SOCKET_TIMEOUT=5
  85         export LTTNG_RELAYD_HEALTH="${HEALTH_PATH}/test-health"
  86
  87         # Activate testpoints
  88         export LTTNG_TESTPOINT_ENABLE=1
  89
  90         # Activate specific thread test
  91         export ${test_thread_name}_${test_suffix}=1
  92
  93         # Spawn sessiond with preload healthexit lib
  94         export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
  95
  96         diag "Start session daemon"
  97         start_lttng_sessiond
  98
  99         if [ ${test_consumerd} -eq 1 ]; then
 100                 create_lttng_session_no_output $SESSION_NAME
 101
 102                 diag "With UST consumer daemons"
 103                 enable_ust_lttng_event_ok $SESSION_NAME $UST_EVENT_NAME $CHANNEL_NAME
 104
 105                 skip $isroot "Root access is needed. Skipping kernel consumer health check test." "1" ||
 106                 {
 107                         diag "With kernel consumer daemon"
 108                         lttng_enable_kernel_event $SESSION_NAME $KERNEL_EVENT_NAME $CHANNEL_NAME
 109                 }
 110                 start_lttng_tracing_ok $SESSION_NAME
 111         fi
 112
 113         if [ ${test_relayd} -eq 1 ]; then
 114                 diag "With relay daemon"
 115                 RELAYD_ARGS="--relayd-path=${LTTNG_RELAYD_HEALTH}"
 116
 117                 start_lttng_relayd "-o $TRACE_PATH"
 118         else
 119                 RELAYD_ARGS=
 120         fi
 121
 122         # Check health status, not caring about result
 123         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 124                 > /dev/null
 125
 126         # Wait
 127         diag "Check after running for ${SLEEP_TIME} seconds"
 128         sleep ${SLEEP_TIME}
 129
 130         # Check health status
 131         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 132                 > ${STDOUT_PATH} 2> ${STDERR_PATH}
 133
 134
 135         if [ ${test_needs_root} -eq 1 ]; then
 136                 skip ${isroot} "Root access needed for test \"${test_thread_name}\"." "1" ||
 137                 {
 138                         report_errors "${test_thread_error_string}" "${test_relayd}"
 139                 }
 140         else
 141                 report_errors "${test_thread_error_string}" "${test_relayd}"
 142         fi
 143
 144         if [ ${test_relayd} -eq 1 ]; then
 145                 # We may fail to stop relayd here, and this is OK, since
 146                 # it may have been killed volountarily by testpoint.
 147                 stop_lttng_relayd_notap
 148         fi
 149         stop_lttng_sessiond
 150
 151         unset LTTNG_TESTPOINT_ENABLE
 152         unset ${test_thread_name}_${test_suffix}
 153         unset LD_PRELOAD
 154         unset LTTNG_NETWORK_SOCKET_TIMEOUT
 155         unset LTTNG_RELAYD_HEALTH
 156 }
 157
 158 plan_tests $NUM_TESTS
 159
 160 print_test_banner "$TEST_DESC"
 161
 162 if [ -f "$CURDIR/$SESSIOND_PRELOAD" ]; then
 163         foundobj=1
 164 else
 165         foundobj=0
 166 fi
 167
 168 skip $foundobj "No shared object generated. Skipping all tests." $NUM_TESTS && exit 0
 169
 170 THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS"
 171         "LTTNG_SESSIOND_THREAD_MANAGE_APPS"
 172         "LTTNG_SESSIOND_THREAD_REG_APPS"
 173         "LTTNG_SESSIOND_THREAD_HT_CLEANUP"
 174         "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY"
 175         "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH"
 176         "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL"
 177
 178         "LTTNG_CONSUMERD_THREAD_CHANNEL"
 179         "LTTNG_CONSUMERD_THREAD_METADATA"
 180         "LTTNG_CONSUMERD_THREAD_METADATA_TIMER"
 181
 182         "LTTNG_RELAYD_THREAD_DISPATCHER"
 183         "LTTNG_RELAYD_THREAD_WORKER"
 184         "LTTNG_RELAYD_THREAD_LISTENER"
 185         "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER"
 186         "LTTNG_RELAYD_THREAD_LIVE_WORKER"
 187         "LTTNG_RELAYD_THREAD_LIVE_LISTENER"
 188 )
 189
 190 ERROR_STRING=(
 191         "Thread \"Session daemon command\" is not responding in component \"sessiond\"."
 192         "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"."
 193         "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"."
 194         "Thread \"Session daemon hash table cleanup\" is not responding in component \"sessiond\"."
 195         "Thread \"Session daemon application notification manager\" is not responding in component \"sessiond\"."
 196         "Thread \"Session daemon application registration dispatcher\" is not responding in component \"sessiond\"."
 197         "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"."
 198
 199         "Thread \"Consumer daemon channel\" is not responding"
 200         "Thread \"Consumer daemon metadata\" is not responding"
 201         "Thread \"Consumer daemon metadata timer\" is not responding"
 202
 203         "Thread \"Relay daemon dispatcher\" is not responding in component \"relayd\"."
 204         "Thread \"Relay daemon worker\" is not responding in component \"relayd\"."
 205         "Thread \"Relay daemon listener\" is not responding in component \"relayd\"."
 206         "Thread \"Relay daemon live dispatcher\" is not responding in component \"relayd\"."
 207         "Thread \"Relay daemon live worker\" is not responding in component \"relayd\"."
 208         "Thread \"Relay daemon live listener\" is not responding in component \"relayd\"."
 209 )
 210
 211 # TODO
 212 # "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER"
 213 # "Thread \"Session daemon manage consumer\" is not responding in component \"sessiond\"."
 214
 215 # TODO: test kernel consumerd specifically in addition to UST consumerd
 216
 217 # TODO: need refactoring of consumerd teardown
 218 # "LTTNG_CONSUMERD_THREAD_SESSIOND"
 219 # "Thread \"Consumer daemon session daemon command manager\" is not responding"
 220
 221 # TODO: this thread is responsible for close a file descriptor that
 222 # triggers teardown of metadata thread. We should revisit teardown of
 223 # consumerd.
 224 # "LTTNG_CONSUMERD_THREAD_DATA"
 225 # "Thread \"Consumer daemon data\" is not responding"
 226
 227 NEEDS_ROOT=(
 228         0
 229         0
 230         0
 231         0
 232         0
 233         0
 234         1
 235
 236         0
 237         0
 238         0
 239
 240         0
 241         0
 242         0
 243         0
 244         0
 245         0
 246 )
 247
 248 TEST_CONSUMERD=(
 249         0
 250         0
 251         0
 252         0
 253         0
 254         0
 255         0
 256
 257         1
 258         1
 259         1
 260
 261         1
 262         1
 263         1
 264         1
 265         1
 266         1
 267 )
 268
 269 TEST_RELAYD=(
 270         0
 271         0
 272         0
 273         0
 274         0
 275         0
 276         0
 277
 278         0
 279         0
 280         0
 281
 282         1
 283         1
 284         1
 285         1
 286         1
 287         1
 288 )
 289
 290 STDOUT_PATH=$(mktemp)
 291 STDERR_PATH=$(mktemp)
 292 TRACE_PATH=$(mktemp -d)
 293 HEALTH_PATH=$(mktemp -d)
 294
 295 if [ "$(id -u)" == "0" ]; then
 296         isroot=1
 297 else
 298         isroot=0
 299 fi
 300
 301 THREAD_COUNT=${#THREAD[@]}
 302 i=0
 303 while [ "$i" -lt "$THREAD_COUNT" ]; do
 304         test_health "${TEST_SUFFIX}" \
 305                 "${THREAD[$i]}" \
 306                 "${ERROR_STRING[$i]}" \
 307                 "${NEEDS_ROOT[$i]}" \
 308                 "${TEST_CONSUMERD[$i]}" \
 309                 "${TEST_RELAYD[$i]}"
 310         let "i++"
 311 done
 312
 313 rm -rf ${HEALTH_PATH}
 314 rm -rf ${TRACE_PATH}
 315 rm -f ${STDOUT_PATH}
 316 rm -f ${STDERR_PATH}