tests/regression/tools/health/test_health.sh

   1 # Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
   2 # Copyright (C) - 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   3 #
   4 # This program is free software; you can redistribute it and/or modify it
   5 # under the terms of the GNU General Public License, version 2 only, as
   6 # published by the Free Software Foundation.
   7 #
   8 # This program is distributed in the hope that it will be useful, but WITHOUT
   9 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 # more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along with
  14 # this program; if not, write to the Free Software Foundation, Inc., 51
  15 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 TESTDIR=${CURDIR}/../../..
  18 LTTNG_BIN="lttng"
  19 UST_EVENT_NAME="tp:tptest"
  20 KERNEL_EVENT_NAME="sched_switch"
  21 CHANNEL_NAME="testchan"
  22 HEALTH_CHECK_BIN="health_check"
  23 NUM_TESTS=106
  24 SLEEP_TIME=30
  25
  26 source $TESTDIR/utils/utils.sh
  27
  28 function report_errors
  29 {
  30         test_thread_error_string="$1"
  31         test_relayd="$2"
  32         err_no_relayd_match="Error querying relayd health"
  33
  34         # Check for health errors
  35         # Include inability to contact relayd health as an expected
  36         # error, since this can happen whenever the relayd shutdown due
  37         # to an error in any thread.
  38         out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l)
  39         if [ $test_relayd -ne 0 ]; then
  40                 outerr=$(grep "${err_no_relayd_match}" ${STDERR_PATH} | wc -l)
  41         else
  42                 outerr=0
  43         fi
  44         if [ $out -eq 0 ] && [ $outerr -eq 0 ]; then
  45                 fail "Validation failure"
  46                 diag "Health returned:"
  47                 diag "stdout:"
  48                 file=${STDOUT_PATH}
  49                 while read line ; do
  50                     diag "$line"
  51                 done < ${file}
  52
  53                 diag "stderr:"
  54                 file=${STDERR_PATH}
  55                 while read line ; do
  56                     diag "$line"
  57                 done < ${file}
  58         else
  59                 pass "Validation OK"
  60         fi
  61 }
  62
  63 function test_health
  64 {
  65         test_suffix="$1"
  66         test_thread_name="$2"
  67         test_thread_error_string="$3"
  68         test_needs_root="$4"
  69         test_consumerd="$5"
  70         test_relayd="$6"
  71
  72         diag "Test health problem detection with ${test_thread_name}"
  73
  74         # Set the socket timeout to 5 so the health check detection
  75         # happens within 25 s
  76         export LTTNG_NETWORK_SOCKET_TIMEOUT=5
  77         export LTTNG_RELAYD_HEALTH="${HEALTH_PATH}/test-health"
  78
  79         # Activate testpoints
  80         export LTTNG_TESTPOINT_ENABLE=1
  81
  82         # Activate specific thread test
  83         export ${test_thread_name}_${test_suffix}=1
  84
  85         # Spawn sessiond with preload healthexit lib
  86         export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
  87
  88         diag "Start session daemon"
  89         start_lttng_sessiond
  90
  91         if [ ${test_consumerd} -eq 1 ]; then
  92                 create_lttng_session_no_output $SESSION_NAME
  93
  94                 diag "With UST consumer daemons"
  95                 enable_ust_lttng_event_ok $SESSION_NAME $UST_EVENT_NAME $CHANNEL_NAME
  96
  97                 skip $isroot "Root access is needed. Skipping kernel consumer health check test." "1" ||
  98                 {
  99                         diag "With kernel consumer daemon"
 100                         lttng_enable_kernel_event $SESSION_NAME $KERNEL_EVENT_NAME $CHANNEL_NAME
 101                 }
 102                 start_lttng_tracing_ok $SESSION_NAME
 103         fi
 104
 105         if [ ${test_relayd} -eq 1 ]; then
 106                 diag "With relay daemon"
 107                 RELAYD_ARGS="--relayd-path=${LTTNG_RELAYD_HEALTH}"
 108
 109                 start_lttng_relayd "-o $TRACE_PATH"
 110         else
 111                 RELAYD_ARGS=
 112         fi
 113
 114         # Check health status, not caring about result
 115         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 116                 > /dev/null
 117
 118         # Wait
 119         diag "Check after running for ${SLEEP_TIME} seconds"
 120         sleep ${SLEEP_TIME}
 121
 122         # Check health status
 123         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 124                 > ${STDOUT_PATH} 2> ${STDERR_PATH}
 125
 126
 127         if [ ${test_needs_root} -eq 1 ]; then
 128                 skip ${isroot} "Root access needed for test \"${test_thread_name}\"." "1" ||
 129                 {
 130                         report_errors "${test_thread_error_string}" "${test_relayd}"
 131                 }
 132         else
 133                 report_errors "${test_thread_error_string}" "${test_relayd}"
 134         fi
 135
 136         if [ ${test_relayd} -eq 1 ]; then
 137                 # We may fail to stop relayd here, and this is OK, since
 138                 # it may have been killed volountarily by testpoint.
 139                 stop_lttng_relayd_notap $KILL_SIGNAL
 140         fi
 141         stop_lttng_consumerd $KILL_SIGNAL
 142         stop_lttng_sessiond $KILL_SIGNAL
 143
 144         unset LTTNG_TESTPOINT_ENABLE
 145         unset ${test_thread_name}_${test_suffix}
 146         unset LD_PRELOAD
 147         unset LTTNG_NETWORK_SOCKET_TIMEOUT
 148         unset LTTNG_RELAYD_HEALTH
 149 }
 150
 151 plan_tests $NUM_TESTS
 152
 153 print_test_banner "$TEST_DESC"
 154
 155 if [ -f "$CURDIR/$SESSIOND_PRELOAD" ]; then
 156         foundobj=1
 157 else
 158         foundobj=0
 159 fi
 160
 161 skip $foundobj "No shared object generated. Skipping all tests." $NUM_TESTS && exit 0
 162
 163 THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS"
 164         "LTTNG_SESSIOND_THREAD_MANAGE_APPS"
 165         "LTTNG_SESSIOND_THREAD_REG_APPS"
 166         "LTTNG_SESSIOND_THREAD_HT_CLEANUP"
 167         "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY"
 168         "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH"
 169         "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL"
 170
 171         "LTTNG_CONSUMERD_THREAD_CHANNEL"
 172         "LTTNG_CONSUMERD_THREAD_METADATA"
 173         "LTTNG_CONSUMERD_THREAD_METADATA_TIMER"
 174
 175         "LTTNG_RELAYD_THREAD_DISPATCHER"
 176         "LTTNG_RELAYD_THREAD_WORKER"
 177         "LTTNG_RELAYD_THREAD_LISTENER"
 178         "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER"
 179         "LTTNG_RELAYD_THREAD_LIVE_WORKER"
 180         "LTTNG_RELAYD_THREAD_LIVE_LISTENER"
 181 )
 182
 183 ERROR_STRING=(
 184         "Thread \"Session daemon command\" is not responding in component \"sessiond\"."
 185         "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"."
 186         "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"."
 187         "Thread \"Session daemon hash table cleanup\" is not responding in component \"sessiond\"."
 188         "Thread \"Session daemon application notification manager\" is not responding in component \"sessiond\"."
 189         "Thread \"Session daemon application registration dispatcher\" is not responding in component \"sessiond\"."
 190         "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"."
 191
 192         "Thread \"Consumer daemon channel\" is not responding"
 193         "Thread \"Consumer daemon metadata\" is not responding"
 194         "Thread \"Consumer daemon metadata timer\" is not responding"
 195
 196         "Thread \"Relay daemon dispatcher\" is not responding in component \"relayd\"."
 197         "Thread \"Relay daemon worker\" is not responding in component \"relayd\"."
 198         "Thread \"Relay daemon listener\" is not responding in component \"relayd\"."
 199         "Thread \"Relay daemon live dispatcher\" is not responding in component \"relayd\"."
 200         "Thread \"Relay daemon live worker\" is not responding in component \"relayd\"."
 201         "Thread \"Relay daemon live listener\" is not responding in component \"relayd\"."
 202 )
 203
 204 # TODO
 205 # "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER"
 206 # "Thread \"Session daemon manage consumer\" is not responding in component \"sessiond\"."
 207
 208 # TODO: test kernel consumerd specifically in addition to UST consumerd
 209
 210 # TODO: need refactoring of consumerd teardown
 211 # "LTTNG_CONSUMERD_THREAD_SESSIOND"
 212 # "Thread \"Consumer daemon session daemon command manager\" is not responding"
 213
 214 # TODO: this thread is responsible for close a file descriptor that
 215 # triggers teardown of metadata thread. We should revisit teardown of
 216 # consumerd.
 217 # "LTTNG_CONSUMERD_THREAD_DATA"
 218 # "Thread \"Consumer daemon data\" is not responding"
 219
 220 NEEDS_ROOT=(
 221         0
 222         0
 223         0
 224         0
 225         0
 226         0
 227         1
 228
 229         0
 230         0
 231         0
 232
 233         0
 234         0
 235         0
 236         0
 237         0
 238         0
 239 )
 240
 241 TEST_CONSUMERD=(
 242         0
 243         0
 244         0
 245         0
 246         0
 247         0
 248         0
 249
 250         1
 251         1
 252         1
 253
 254         1
 255         1
 256         1
 257         1
 258         1
 259         1
 260 )
 261
 262 TEST_RELAYD=(
 263         0
 264         0
 265         0
 266         0
 267         0
 268         0
 269         0
 270
 271         0
 272         0
 273         0
 274
 275         1
 276         1
 277         1
 278         1
 279         1
 280         1
 281 )
 282
 283 STDOUT_PATH=$(mktemp)
 284 STDERR_PATH=$(mktemp)
 285 TRACE_PATH=$(mktemp -d)
 286 HEALTH_PATH=$(mktemp -d)
 287
 288 if [ "$(id -u)" == "0" ]; then
 289         isroot=1
 290 else
 291         isroot=0
 292 fi
 293
 294 THREAD_COUNT=${#THREAD[@]}
 295 i=0
 296 while [ "$i" -lt "$THREAD_COUNT" ]; do
 297         test_health "${TEST_SUFFIX}" \
 298                 "${THREAD[$i]}" \
 299                 "${ERROR_STRING[$i]}" \
 300                 "${NEEDS_ROOT[$i]}" \
 301                 "${TEST_CONSUMERD[$i]}" \
 302                 "${TEST_RELAYD[$i]}"
 303         let "i++"
 304 done
 305
 306 rm -rf ${HEALTH_PATH}
 307 rm -rf ${TRACE_PATH}
 308 rm -f ${STDOUT_PATH}
 309 rm -f ${STDERR_PATH}