tests/regression/tools/health/test_health.sh

   1 # Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
   2 # Copyright (C) - 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   3 #
   4 # This program is free software; you can redistribute it and/or modify it
   5 # under the terms of the GNU General Public License, version 2 only, as
   6 # published by the Free Software Foundation.
   7 #
   8 # This program is distributed in the hope that it will be useful, but WITHOUT
   9 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 # more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along with
  14 # this program; if not, write to the Free Software Foundation, Inc., 51
  15 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 TESTDIR=${CURDIR}/../../..
  18 LTTNG_BIN="lttng"
  19 UST_EVENT_NAME="tp:tptest"
  20 KERNEL_EVENT_NAME="sched_switch"
  21 CHANNEL_NAME="testchan"
  22 HEALTH_CHECK_BIN="health_check"
  23 NUM_TESTS=90
  24 SLEEP_TIME=30
  25
  26 source $TESTDIR/utils/utils.sh
  27
  28 if [ ! -f "$CURDIR/$SESSIOND_PRELOAD" ]; then
  29         BAIL_OUT "${CURDIR}/${SESSIOND_PRELOAD} is missing."
  30 fi
  31
  32 function lttng_create_session_uri
  33 {
  34         # Create session with default path
  35         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $SESSION_NAME \
  36                 -U net://localhost >/dev/null 2>&1
  37         ok $? "Create session with default path"
  38 }
  39
  40 function report_errors
  41 {
  42         test_thread_error_string="$1"
  43         test_relayd="$2"
  44         err_no_relayd_match="Error querying relayd health"
  45
  46         # Check for health errors
  47         # Include inability to contact relayd health as an expected
  48         # error, since this can happen whenever the relayd shutdown due
  49         # to an error in any thread.
  50         out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l)
  51         if [ $test_relayd -ne 0 ]; then
  52                 outerr=$(grep "${err_no_relayd_match}" ${STDERR_PATH} | wc -l)
  53         else
  54                 outerr=0
  55         fi
  56         if [ $out -eq 0 ] && [ $outerr -eq 0 ]; then
  57                 fail "Validation failure"
  58                 diag "Health returned:"
  59                 diag "stdout:"
  60                 file=${STDOUT_PATH}
  61                 while read line ; do
  62                     diag "$line"
  63                 done < ${file}
  64
  65                 diag "stderr:"
  66                 file=${STDERR_PATH}
  67                 while read line ; do
  68                     diag "$line"
  69                 done < ${file}
  70         else
  71                 pass "Validation OK"
  72         fi
  73 }
  74
  75 function test_health
  76 {
  77         test_suffix="$1"
  78         test_thread_name="$2"
  79         test_thread_error_string="$3"
  80         test_needs_root="$4"
  81         test_consumerd="$5"
  82         test_relayd="$6"
  83
  84         diag "Test health problem detection with ${test_thread_name}"
  85
  86         # Set the socket timeout to 5 so the health check detection
  87         # happens within 25 s
  88         export LTTNG_NETWORK_SOCKET_TIMEOUT=5
  89         export LTTNG_RELAYD_HEALTH="${HEALTH_PATH}/test-health"
  90
  91         # Activate testpoints
  92         export LTTNG_TESTPOINT_ENABLE=1
  93
  94         # Activate specific thread test
  95         export ${test_thread_name}_${test_suffix}=1
  96
  97         # Spawn sessiond with preload healthexit lib
  98         export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
  99
 100         diag "Start session daemon"
 101         start_lttng_sessiond
 102
 103         if [ ${test_consumerd} -eq 1 ]; then
 104                 create_lttng_session_no_output $SESSION_NAME
 105
 106                 diag "With UST consumer daemons"
 107                 enable_ust_lttng_event_ok $SESSION_NAME $UST_EVENT_NAME $CHANNEL_NAME
 108
 109                 skip $isroot "Root access is needed. Skipping kernel consumer health check test." "1" ||
 110                 {
 111                         diag "With kernel consumer daemon"
 112                         lttng_enable_kernel_event $SESSION_NAME $KERNEL_EVENT_NAME $CHANNEL_NAME
 113                 }
 114                 start_lttng_tracing_ok $SESSION_NAME
 115         fi
 116
 117         if [ ${test_relayd} -eq 1 ]; then
 118                 diag "With relay daemon"
 119                 RELAYD_ARGS="--relayd-path=${LTTNG_RELAYD_HEALTH}"
 120
 121                 start_lttng_relayd "-o $TRACE_PATH"
 122         else
 123                 RELAYD_ARGS=
 124         fi
 125
 126         # Check health status, not caring about result
 127         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 128                 > /dev/null
 129
 130         # Wait
 131         diag "Check after running for ${SLEEP_TIME} seconds"
 132         sleep ${SLEEP_TIME}
 133
 134         # Check health status
 135         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 136                 > ${STDOUT_PATH} 2> ${STDERR_PATH}
 137
 138
 139         if [ ${test_needs_root} -eq 1 ]; then
 140                 skip ${isroot} "Root access needed for test \"${test_thread_name}\"." "1" ||
 141                 {
 142                         report_errors "${test_thread_error_string}" "${test_relayd}"
 143                 }
 144         else
 145                 report_errors "${test_thread_error_string}" "${test_relayd}"
 146         fi
 147
 148         if [ ${test_relayd} -eq 1 ]; then
 149                 # We may fail to stop relayd here, and this is OK, since
 150                 # it may have been killed volountarily by testpoint.
 151                 stop_lttng_relayd_notap
 152         fi
 153         stop_lttng_sessiond
 154
 155         unset LTTNG_TESTPOINT_ENABLE
 156         unset ${test_thread_name}_${test_suffix}
 157         unset LD_PRELOAD
 158         unset LTTNG_NETWORK_SOCKET_TIMEOUT
 159         unset LTTNG_RELAYD_HEALTH
 160 }
 161
 162 plan_tests $NUM_TESTS
 163
 164 print_test_banner "$TEST_DESC"
 165
 166 THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS"
 167         "LTTNG_SESSIOND_THREAD_MANAGE_APPS"
 168         "LTTNG_SESSIOND_THREAD_REG_APPS"
 169         "LTTNG_SESSIOND_THREAD_HT_CLEANUP"
 170         "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY"
 171         "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH"
 172         "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL"
 173
 174         "LTTNG_CONSUMERD_THREAD_CHANNEL"
 175         "LTTNG_CONSUMERD_THREAD_METADATA"
 176         "LTTNG_CONSUMERD_THREAD_METADATA_TIMER"
 177
 178         "LTTNG_RELAYD_THREAD_DISPATCHER"
 179         "LTTNG_RELAYD_THREAD_WORKER"
 180         "LTTNG_RELAYD_THREAD_LISTENER"
 181         "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER"
 182         "LTTNG_RELAYD_THREAD_LIVE_WORKER"
 183         "LTTNG_RELAYD_THREAD_LIVE_LISTENER"
 184 )
 185
 186 ERROR_STRING=(
 187         "Thread \"Session daemon command\" is not responding in component \"sessiond\"."
 188         "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"."
 189         "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"."
 190         "Thread \"Session daemon hash table cleanup\" is not responding in component \"sessiond\"."
 191         "Thread \"Session daemon application notification manager\" is not responding in component \"sessiond\"."
 192         "Thread \"Session daemon application registration dispatcher\" is not responding in component \"sessiond\"."
 193         "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"."
 194
 195         "Thread \"Consumer daemon channel\" is not responding"
 196         "Thread \"Consumer daemon metadata\" is not responding"
 197         "Thread \"Consumer daemon metadata timer\" is not responding"
 198
 199         "Thread \"Relay daemon dispatcher\" is not responding in component \"relayd\"."
 200         "Thread \"Relay daemon worker\" is not responding in component \"relayd\"."
 201         "Thread \"Relay daemon listener\" is not responding in component \"relayd\"."
 202         "Thread \"Relay daemon live dispatcher\" is not responding in component \"relayd\"."
 203         "Thread \"Relay daemon live worker\" is not responding in component \"relayd\"."
 204         "Thread \"Relay daemon live listener\" is not responding in component \"relayd\"."
 205 )
 206
 207 # TODO
 208 # "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER"
 209 # "Thread \"Session daemon manage consumer\" is not responding in component \"sessiond\"."
 210
 211 # TODO: test kernel consumerd specifically in addition to UST consumerd
 212
 213 # TODO: need refactoring of consumerd teardown
 214 # "LTTNG_CONSUMERD_THREAD_SESSIOND"
 215 # "Thread \"Consumer daemon session daemon command manager\" is not responding"
 216
 217 # TODO: this thread is responsible for close a file descriptor that
 218 # triggers teardown of metadata thread. We should revisit teardown of
 219 # consumerd.
 220 # "LTTNG_CONSUMERD_THREAD_DATA"
 221 # "Thread \"Consumer daemon data\" is not responding"
 222
 223 NEEDS_ROOT=(
 224         0
 225         0
 226         0
 227         0
 228         0
 229         0
 230         1
 231
 232         0
 233         0
 234         0
 235
 236         0
 237         0
 238         0
 239         0
 240         0
 241         0
 242 )
 243
 244 TEST_CONSUMERD=(
 245         0
 246         0
 247         0
 248         0
 249         0
 250         0
 251         0
 252
 253         1
 254         1
 255         1
 256
 257         1
 258         1
 259         1
 260         1
 261         1
 262         1
 263 )
 264
 265 TEST_RELAYD=(
 266         0
 267         0
 268         0
 269         0
 270         0
 271         0
 272         0
 273
 274         0
 275         0
 276         0
 277
 278         1
 279         1
 280         1
 281         1
 282         1
 283         1
 284 )
 285
 286 STDOUT_PATH=$(mktemp)
 287 STDERR_PATH=$(mktemp)
 288 TRACE_PATH=$(mktemp -d)
 289 HEALTH_PATH=$(mktemp -d)
 290
 291 if [ "$(id -u)" == "0" ]; then
 292         isroot=1
 293 else
 294         isroot=0
 295 fi
 296
 297 THREAD_COUNT=${#THREAD[@]}
 298 i=0
 299 while [ "$i" -lt "$THREAD_COUNT" ]; do
 300         test_health "${TEST_SUFFIX}" \
 301                 "${THREAD[$i]}" \
 302                 "${ERROR_STRING[$i]}" \
 303                 "${NEEDS_ROOT[$i]}" \
 304                 "${TEST_CONSUMERD[$i]}" \
 305                 "${TEST_RELAYD[$i]}"
 306         let "i++"
 307 done
 308
 309 rm -rf ${HEALTH_PATH}
 310 rm -rf ${TRACE_PATH}
 311 rm -f ${STDOUT_PATH}
 312 rm -f ${STDERR_PATH}