tests/utils/tap/tap.c

   1 /*
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (C) 2004 Nik Clayton
   5  * Copyright (C) 2017 Jérémie Galarneau
   6  */
   7
   8 #include "../utils.h"
   9 #include "common/compat/time.hpp"
  10 #include "tap.h"
  11
  12 #include <assert.h>
  13 #include <ctype.h>
  14 #include <errno.h>
  15 #include <limits.h>
  16 #include <stdarg.h>
  17 #include <stdio.h>
  18 #include <stdlib.h>
  19 #include <string.h>
  20 #include <time.h>
  21
  22 static int no_plan = 0;
  23 static int skip_all = 0;
  24 static int have_plan = 0;
  25 static unsigned int test_count = 0; /* Number of tests that have been run */
  26 static unsigned int e_tests = 0; /* Expected number of tests to run */
  27 static unsigned int failures = 0; /* Number of tests that failed */
  28 static char *todo_msg = NULL;
  29 static const char *todo_msg_fixed = "libtap malloc issue";
  30 static int todo = 0;
  31 static int test_died = 0;
  32 static int time_tests = 1;
  33 struct timespec last_time;
  34
  35 /* Encapsulate the pthread code in a conditional.  In the absence of
  36    libpthread the code does nothing */
  37 #ifdef HAVE_LIBPTHREAD
  38 #include <pthread.h>
  39 static pthread_mutex_t M = PTHREAD_MUTEX_INITIALIZER;
  40 #define LOCK   pthread_mutex_lock(&M);
  41 #define UNLOCK pthread_mutex_unlock(&M);
  42 #else
  43 #define LOCK
  44 #define UNLOCK
  45 #endif
  46
  47 static void _expected_tests(unsigned int);
  48 static void _tap_init(void);
  49 static void _cleanup(void);
  50
  51 #ifdef __MINGW32__
  52 static inline void flockfile(FILE *filehandle)
  53 {
  54         return;
  55 }
  56
  57 static inline void funlockfile(FILE *filehandle)
  58 {
  59         return;
  60 }
  61 #endif
  62
  63 /*
  64  * Generate a test result.
  65  *
  66  * ok -- boolean, indicates whether or not the test passed.
  67  * test_name -- the name of the test, may be NULL
  68  * test_comment -- a comment to print afterwards, may be NULL
  69  */
  70 unsigned int _gen_result(
  71         int ok, const char *func, const char *file, unsigned int line, const char *test_name, ...)
  72 {
  73         va_list ap;
  74         char *local_test_name = NULL;
  75         char *c;
  76         int name_is_digits;
  77
  78         LOCK;
  79
  80         test_count++;
  81
  82         /* Start by taking the test name and performing any printf()
  83            expansions on it */
  84         if (test_name != NULL) {
  85                 va_start(ap, test_name);
  86                 if (vasprintf(&local_test_name, test_name, ap) == -1) {
  87                         local_test_name = NULL;
  88                 }
  89                 va_end(ap);
  90
  91                 /* Make sure the test name contains more than digits
  92                    and spaces.  Emit an error message and exit if it
  93                    does */
  94                 if (local_test_name) {
  95                         name_is_digits = 1;
  96                         for (c = local_test_name; *c != '\0'; c++) {
  97                                 if (!isdigit((unsigned char) *c) && !isspace((unsigned char) *c)) {
  98                                         name_is_digits = 0;
  99                                         break;
 100                                 }
 101                         }
 102
 103                         if (name_is_digits) {
 104                                 diag("    You named your test '%s'.  You shouldn't use numbers for your test names.",
 105                                      local_test_name);
 106                                 diag("    Very confusing.");
 107                         }
 108                 }
 109         }
 110
 111         if (!ok) {
 112                 printf("not ");
 113                 failures++;
 114         }
 115
 116         printf("ok %d", test_count);
 117
 118         if (test_name != NULL) {
 119                 printf(" - ");
 120
 121                 /* Print the test name, escaping any '#' characters it
 122                    might contain */
 123                 if (local_test_name != NULL) {
 124                         flockfile(stdout);
 125                         for (c = local_test_name; *c != '\0'; c++) {
 126                                 if (*c == '#')
 127                                         fputc('\\', stdout);
 128                                 fputc((int) *c, stdout);
 129                         }
 130                         funlockfile(stdout);
 131                 } else { /* vasprintf() failed, use a fixed message */
 132                         printf("%s", todo_msg_fixed);
 133                 }
 134         }
 135
 136         /* If we're in a todo_start() block then flag the test as being
 137            TODO.  todo_msg should contain the message to print at this
 138            point.  If it's NULL then asprintf() failed, and we should
 139            use the fixed message.
 140
 141            This is not counted as a failure, so decrement the counter if
 142            the test failed. */
 143         if (todo) {
 144                 printf(" # TODO %s", todo_msg ? todo_msg : todo_msg_fixed);
 145                 if (!ok)
 146                         failures--;
 147         }
 148
 149         printf("\n");
 150         _output_test_time();
 151
 152         if (!ok) {
 153                 if (getenv("HARNESS_ACTIVE") != NULL)
 154                         fputs("\n", stderr);
 155
 156                 diag("    Failed %stest (%s:%s() at line %d)",
 157                      todo ? "(TODO) " : "",
 158                      file,
 159                      func,
 160                      line);
 161         }
 162         free(local_test_name);
 163
 164         UNLOCK;
 165
 166         /* We only care (when testing) that ok is positive, but here we
 167            specifically only want to return 1 or 0 */
 168         return ok ? 1 : 0;
 169 }
 170
 171 /*
 172  * Initialise the TAP library.  Will only do so once, however many times it's
 173  * called.
 174  */
 175 void _tap_init(void)
 176 {
 177         static int run_once = 0;
 178
 179         if (!run_once) {
 180                 atexit(_cleanup);
 181
 182                 /* stdout needs to be unbuffered so that the output appears
 183                    in the same place relative to stderr output as it does
 184                    with Test::Harness */
 185                 setbuf(stdout, 0);
 186
 187                 /*
 188                  * Check if the LTTNG_TESTS_TAP_AUTOTIME environment variable
 189                  * is set and contains at least one byte.
 190                  */
 191                 const char *autotime_env = getenv("LTTNG_TESTS_TAP_AUTOTIME");
 192                 if (autotime_env != NULL && strnlen(autotime_env, 1)) {
 193                         int tap_autotime;
 194
 195                         /*
 196                          * Check if LTTNG_TESTS_TAP_AUTOTIME is '0', also check
 197                          * errno because strtol() can return '0' on error.
 198                          */
 199                         errno = 0;
 200                         tap_autotime = strtol(autotime_env, NULL, 10);
 201                         if (tap_autotime == 0 && errno == 0) {
 202                                 time_tests = 0;
 203                         }
 204                 }
 205
 206                 run_once = 1;
 207         }
 208         lttng_clock_gettime(CLOCK_MONOTONIC, &last_time);
 209 }
 210
 211 /*
 212  * Note that there's no plan.
 213  */
 214 int plan_no_plan(void)
 215 {
 216         LOCK;
 217
 218         _tap_init();
 219
 220         if (have_plan != 0) {
 221                 fprintf(stderr, "You tried to plan twice!\n");
 222                 test_died = 1;
 223                 UNLOCK;
 224                 exit(255);
 225         }
 226
 227         have_plan = 1;
 228         no_plan = 1;
 229
 230         UNLOCK;
 231
 232         return 1;
 233 }
 234
 235 /*
 236  * Note that the plan is to skip all tests
 237  */
 238 int plan_skip_all(const char *reason)
 239 {
 240         LOCK;
 241
 242         _tap_init();
 243
 244         skip_all = 1;
 245
 246         printf("1..0");
 247
 248         if (reason != NULL)
 249                 printf(" # Skip %s", reason);
 250
 251         printf("\n");
 252
 253         UNLOCK;
 254
 255         exit(0);
 256 }
 257
 258 /*
 259  * Note the number of tests that will be run.
 260  */
 261 int plan_tests(unsigned int tests)
 262 {
 263         LOCK;
 264
 265         _tap_init();
 266
 267         if (have_plan != 0) {
 268                 fprintf(stderr, "You tried to plan twice!\n");
 269                 test_died = 1;
 270                 UNLOCK;
 271                 exit(255);
 272         }
 273
 274         if (tests == 0) {
 275                 fprintf(stderr, "You said to run 0 tests!  You've got to run something.\n");
 276                 test_died = 1;
 277                 UNLOCK;
 278                 exit(255);
 279         }
 280
 281         have_plan = 1;
 282
 283         _expected_tests(tests);
 284
 285         UNLOCK;
 286
 287         return e_tests;
 288 }
 289
 290 unsigned int diag(const char *fmt, ...)
 291 {
 292         va_list ap;
 293
 294         fputs("# ", stderr);
 295
 296         va_start(ap, fmt);
 297         vfprintf(stderr, fmt, ap);
 298         va_end(ap);
 299
 300         fputs("\n", stderr);
 301
 302         return 0;
 303 }
 304
 305 void diag_multiline(const char *val)
 306 {
 307         size_t len, i, line_start_idx = 0;
 308
 309         assert(val);
 310         len = strlen(val);
 311
 312         for (i = 0; i < len; i++) {
 313                 int line_length;
 314
 315                 if (val[i] != '\n') {
 316                         continue;
 317                 }
 318
 319                 assert((i - line_start_idx + 1) <= INT_MAX);
 320                 line_length = i - line_start_idx + 1;
 321                 fprintf(stderr, "# %.*s", line_length, &val[line_start_idx]);
 322                 line_start_idx = i + 1;
 323         }
 324 }
 325
 326 void _expected_tests(unsigned int tests)
 327 {
 328         printf("1..%d\n", tests);
 329         e_tests = tests;
 330 }
 331
 332 int skip(unsigned int n, const char *fmt, ...)
 333 {
 334         va_list ap;
 335         char *skip_msg = NULL;
 336
 337         LOCK;
 338
 339         va_start(ap, fmt);
 340         if (vasprintf(&skip_msg, fmt, ap) == -1) {
 341                 skip_msg = NULL;
 342         }
 343         va_end(ap);
 344
 345         while (n-- > 0) {
 346                 test_count++;
 347                 printf("ok %d # skip %s\n",
 348                        test_count,
 349                        skip_msg != NULL ? skip_msg : "libtap():malloc() failed");
 350                 _output_test_time();
 351         }
 352
 353         free(skip_msg);
 354
 355         UNLOCK;
 356
 357         return 1;
 358 }
 359
 360 void _output_test_time(void)
 361 {
 362         struct timespec new_time;
 363         int64_t time_ns;
 364         if (time_tests) {
 365                 lttng_clock_gettime(CLOCK_MONOTONIC, &new_time);
 366                 time_ns = elapsed_time_ns(&last_time, &new_time);
 367                 printf("  ---\n    duration_ms: %ld.%ld\n  ...\n",
 368                        time_ns / 1000000,
 369                        time_ns % 1000000);
 370         }
 371         lttng_clock_gettime(CLOCK_MONOTONIC, &last_time);
 372 }
 373
 374 void todo_start(const char *fmt, ...)
 375 {
 376         va_list ap;
 377
 378         LOCK;
 379
 380         va_start(ap, fmt);
 381         if (vasprintf(&todo_msg, fmt, ap) == -1) {
 382                 todo_msg = NULL;
 383         }
 384         va_end(ap);
 385
 386         todo = 1;
 387
 388         UNLOCK;
 389 }
 390
 391 void todo_end(void)
 392 {
 393         LOCK;
 394
 395         todo = 0;
 396         free(todo_msg);
 397
 398         UNLOCK;
 399 }
 400
 401 int exit_status(void)
 402 {
 403         int r;
 404
 405         LOCK;
 406
 407         /* If there's no plan, just return the number of failures */
 408         if (no_plan || !have_plan) {
 409                 UNLOCK;
 410                 return failures;
 411         }
 412
 413         /* Ran too many tests?  Return the number of tests that were run
 414            that shouldn't have been */
 415         if (e_tests < test_count) {
 416                 r = test_count - e_tests;
 417                 UNLOCK;
 418                 return r;
 419         }
 420
 421         /* Return the number of tests that failed + the number of tests
 422            that weren't run */
 423         r = failures + e_tests - test_count;
 424         UNLOCK;
 425
 426         return r;
 427 }
 428
 429 /*
 430  * Cleanup at the end of the run, produce any final output that might be
 431  * required.
 432  */
 433 void _cleanup(void)
 434 {
 435         LOCK;
 436
 437         /* If plan_no_plan() wasn't called, and we don't have a plan,
 438            and we're not skipping everything, then something happened
 439            before we could produce any output */
 440         if (!no_plan && !have_plan && !skip_all) {
 441                 diag("Looks like your test died before it could output anything.");
 442                 UNLOCK;
 443                 return;
 444         }
 445
 446         if (test_died) {
 447                 diag("Looks like your test died just after %d.", test_count);
 448                 UNLOCK;
 449                 return;
 450         }
 451
 452         /* No plan provided, but now we know how many tests were run, and can
 453            print the header at the end */
 454         if (!skip_all && (no_plan || !have_plan)) {
 455                 printf("1..%d\n", test_count);
 456         }
 457
 458         if ((have_plan && !no_plan) && e_tests < test_count) {
 459                 diag("Looks like you planned %d %s but ran %d extra.",
 460                      e_tests,
 461                      e_tests == 1 ? "test" : "tests",
 462                      test_count - e_tests);
 463                 UNLOCK;
 464                 return;
 465         }
 466
 467         if ((have_plan || !no_plan) && e_tests > test_count) {
 468                 diag("Looks like you planned %d %s but only ran %d.",
 469                      e_tests,
 470                      e_tests == 1 ? "test" : "tests",
 471                      test_count);
 472                 UNLOCK;
 473                 return;
 474         }
 475
 476         if (failures)
 477                 diag("Looks like you failed %d %s of %d.",
 478                      failures,
 479                      failures == 1 ? "test" : "tests",
 480                      test_count);
 481
 482         UNLOCK;
 483 }