tests/regression/tools/health/test_health.sh

   1 # Copyright (C) 2012 Christian Babeux <christian.babeux@efficios.com>
   2 # Copyright (C) 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   3 #
   4 # SPDX-License-Identifier: GPL-2.0-only
   5
   6 TESTDIR=${CURDIR}/../../..
   7 UST_EVENT_NAME="tp:tptest"
   8 KERNEL_EVENT_NAME="sched_switch"
   9 CHANNEL_NAME="testchan"
  10 HEALTH_CHECK_BIN="health_check"
  11 NUM_TESTS=96
  12 SLEEP_TIME=30
  13
  14 source $TESTDIR/utils/utils.sh
  15
  16 function report_errors
  17 {
  18         test_thread_error_string="$1"
  19         test_relayd="$2"
  20         err_no_relayd_match="Error querying relayd health"
  21
  22         # Check for health errors
  23         # Include inability to contact relayd health as an expected
  24         # error, since this can happen whenever the relayd shutdown due
  25         # to an error in any thread.
  26         out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l)
  27         if [ $test_relayd -ne 0 ]; then
  28                 outerr=$(grep "${err_no_relayd_match}" ${STDERR_PATH} | wc -l)
  29         else
  30                 outerr=0
  31         fi
  32         if [ $out -eq 0 ] && [ $outerr -eq 0 ]; then
  33                 fail "Validation failure"
  34                 diag "Health returned:"
  35                 diag "stdout:"
  36                 file=${STDOUT_PATH}
  37                 while read line ; do
  38                     diag "$line"
  39                 done < ${file}
  40
  41                 diag "stderr:"
  42                 file=${STDERR_PATH}
  43                 while read line ; do
  44                     diag "$line"
  45                 done < ${file}
  46         else
  47                 pass "Validation OK"
  48         fi
  49 }
  50
  51 function test_health
  52 {
  53         test_suffix="$1"
  54         test_thread_name="$2"
  55         test_thread_error_string="$3"
  56         test_needs_root="$4"
  57         test_consumerd="$5"
  58         test_relayd="$6"
  59
  60         diag "Test health problem detection with ${test_thread_name}"
  61
  62         # Set the socket timeout to 5 so the health check detection
  63         # happens within 25 s
  64         export LTTNG_NETWORK_SOCKET_TIMEOUT=5
  65         export LTTNG_RELAYD_HEALTH="${HEALTH_PATH}/test-health"
  66
  67         # Activate testpoints
  68         export LTTNG_TESTPOINT_ENABLE=1
  69
  70         # Activate specific thread test
  71         export ${test_thread_name}_${test_suffix}=1
  72
  73         # Spawn sessiond with preloaded testpoint override lib
  74         export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
  75
  76         diag "Start session daemon"
  77         start_lttng_sessiond
  78
  79         if [ ${test_consumerd} -eq 1 ]; then
  80                 create_lttng_session_no_output $SESSION_NAME
  81
  82                 diag "With UST consumer daemons"
  83                 enable_ust_lttng_event_ok $SESSION_NAME $UST_EVENT_NAME $CHANNEL_NAME
  84
  85                 check_skip_kernel_test "1" "Skipping kernel consumer health check test." ||
  86                 {
  87                         diag "With kernel consumer daemon"
  88                         lttng_enable_kernel_event $SESSION_NAME $KERNEL_EVENT_NAME $CHANNEL_NAME
  89                 }
  90                 start_lttng_tracing_ok $SESSION_NAME
  91         fi
  92
  93         if [ ${test_relayd} -eq 1 ]; then
  94                 diag "With relay daemon"
  95                 RELAYD_ARGS="--relayd-path=${LTTNG_RELAYD_HEALTH}"
  96
  97                 start_lttng_relayd "-o $TRACE_PATH"
  98         else
  99                 RELAYD_ARGS=
 100         fi
 101
 102         # Check health status, not caring about result
 103         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 104                 > /dev/null
 105
 106         # Wait
 107         diag "Check after running for ${SLEEP_TIME} seconds"
 108         sleep ${SLEEP_TIME}
 109
 110         # Check health status
 111         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 112                 > ${STDOUT_PATH} 2> ${STDERR_PATH}
 113
 114
 115         if [ ${test_needs_root} -eq 1 ]; then
 116                 check_skip_kernel_test "1" "Skipping \"${test_thread_name}\"." ||
 117                 {
 118                         report_errors "${test_thread_error_string}" "${test_relayd}"
 119                 }
 120         else
 121                 report_errors "${test_thread_error_string}" "${test_relayd}"
 122         fi
 123
 124         if [ ${test_relayd} -eq 1 ]; then
 125                 # We may fail to stop relayd here, and this is OK, since
 126                 # it may have been killed volountarily by testpoint.
 127                 stop_lttng_relayd_cleanup $KILL_SIGNAL
 128         fi
 129
 130         if [ ${test_consumerd} -eq 1 ]; then
 131                 stop_lttng_consumerd $KILL_SIGNAL
 132         fi
 133         stop_lttng_sessiond $KILL_SIGNAL
 134
 135         unset LTTNG_TESTPOINT_ENABLE
 136         unset ${test_thread_name}_${test_suffix}
 137         unset LD_PRELOAD
 138         unset LTTNG_NETWORK_SOCKET_TIMEOUT
 139         unset LTTNG_RELAYD_HEALTH
 140 }
 141
 142 plan_tests $NUM_TESTS
 143
 144 print_test_banner "$TEST_DESC"
 145
 146 if [ -f "$CURDIR/$SESSIOND_PRELOAD" ]; then
 147         foundobj=1
 148 else
 149         foundobj=0
 150 fi
 151
 152 skip $foundobj "No shared object generated. Skipping all tests." $NUM_TESTS && exit 0
 153
 154 THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS"
 155         "LTTNG_SESSIOND_THREAD_MANAGE_APPS"
 156         "LTTNG_SESSIOND_THREAD_REG_APPS"
 157         "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY"
 158         "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH"
 159         "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL"
 160
 161         "LTTNG_CONSUMERD_THREAD_CHANNEL"
 162         "LTTNG_CONSUMERD_THREAD_METADATA"
 163         "LTTNG_CONSUMERD_THREAD_METADATA_TIMER"
 164
 165         "LTTNG_RELAYD_THREAD_DISPATCHER"
 166         "LTTNG_RELAYD_THREAD_WORKER"
 167         "LTTNG_RELAYD_THREAD_LISTENER"
 168         "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER"
 169         "LTTNG_RELAYD_THREAD_LIVE_WORKER"
 170         "LTTNG_RELAYD_THREAD_LIVE_LISTENER"
 171 )
 172
 173 ERROR_STRING=(
 174         "Thread \"Session daemon command\" is not responding in component \"sessiond\"."
 175         "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"."
 176         "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"."
 177         "Thread \"Session daemon application notification manager\" is not responding in component \"sessiond\"."
 178         "Thread \"Session daemon application registration dispatcher\" is not responding in component \"sessiond\"."
 179         "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"."
 180
 181         "Thread \"Consumer daemon channel\" is not responding"
 182         "Thread \"Consumer daemon metadata\" is not responding"
 183         "Thread \"Consumer daemon metadata timer\" is not responding"
 184
 185         "Thread \"Relay daemon dispatcher\" is not responding in component \"relayd\"."
 186         "Thread \"Relay daemon worker\" is not responding in component \"relayd\"."
 187         "Thread \"Relay daemon listener\" is not responding in component \"relayd\"."
 188         "Thread \"Relay daemon live dispatcher\" is not responding in component \"relayd\"."
 189         "Thread \"Relay daemon live worker\" is not responding in component \"relayd\"."
 190         "Thread \"Relay daemon live listener\" is not responding in component \"relayd\"."
 191 )
 192
 193 # TODO
 194 # "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER"
 195 # "Thread \"Session daemon manage consumer\" is not responding in component \"sessiond\"."
 196
 197 # TODO: test kernel consumerd specifically in addition to UST consumerd
 198
 199 # TODO: need refactoring of consumerd teardown
 200 # "LTTNG_CONSUMERD_THREAD_SESSIOND"
 201 # "Thread \"Consumer daemon session daemon command manager\" is not responding"
 202
 203 # TODO: this thread is responsible for close a file descriptor that
 204 # triggers teardown of metadata thread. We should revisit teardown of
 205 # consumerd.
 206 # "LTTNG_CONSUMERD_THREAD_DATA"
 207 # "Thread \"Consumer daemon data\" is not responding"
 208
 209 NEEDS_ROOT=(
 210         0
 211         0
 212         0
 213         0
 214         0
 215         1
 216
 217         0
 218         0
 219         0
 220
 221         0
 222         0
 223         0
 224         0
 225         0
 226         0
 227 )
 228
 229 TEST_CONSUMERD=(
 230         0
 231         0
 232         0
 233         0
 234         0
 235         0
 236
 237         1
 238         1
 239         1
 240
 241         1
 242         1
 243         1
 244         1
 245         1
 246         1
 247 )
 248
 249 TEST_RELAYD=(
 250         0
 251         0
 252         0
 253         0
 254         0
 255         0
 256
 257         0
 258         0
 259         0
 260
 261         1
 262         1
 263         1
 264         1
 265         1
 266         1
 267 )
 268
 269 STDOUT_PATH=$(mktemp -t tmp.test_health_stdout_path.XXXXXX)
 270 STDERR_PATH=$(mktemp -t tmp.test_health_stderr_path.XXXXXX)
 271 TRACE_PATH=$(mktemp -d -t tmp.test_health_trace_path.XXXXXX)
 272 HEALTH_PATH=$(mktemp -d -t tmp.test_health_trace_path.XXXXXX)
 273
 274 THREAD_COUNT=${#THREAD[@]}
 275 i=0
 276 while [ "$i" -lt "$THREAD_COUNT" ]; do
 277         test_health "${TEST_SUFFIX}" \
 278                 "${THREAD[$i]}" \
 279                 "${ERROR_STRING[$i]}" \
 280                 "${NEEDS_ROOT[$i]}" \
 281                 "${TEST_CONSUMERD[$i]}" \
 282                 "${TEST_RELAYD[$i]}"
 283         let "i++"
 284 done
 285
 286 rm -rf ${HEALTH_PATH}
 287 rm -rf ${TRACE_PATH}
 288 rm -f ${STDOUT_PATH}
 289 rm -f ${STDERR_PATH}