tests/utils/tap/tap.c

   1 /*
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (C) 2004 Nik Clayton
   5  * Copyright (C) 2017 Jérémie Galarneau
   6  */
   7
   8 #include "../utils.h"
   9 #include "common/compat/time.hpp"
  10 #include "tap.h"
  11
  12 #include <assert.h>
  13 #include <ctype.h>
  14 #include <limits.h>
  15 #include <stdarg.h>
  16 #include <stdio.h>
  17 #include <stdlib.h>
  18 #include <string.h>
  19 #include <time.h>
  20
  21 static int no_plan = 0;
  22 static int skip_all = 0;
  23 static int have_plan = 0;
  24 static unsigned int test_count = 0; /* Number of tests that have been run */
  25 static unsigned int e_tests = 0; /* Expected number of tests to run */
  26 static unsigned int failures = 0; /* Number of tests that failed */
  27 static char *todo_msg = NULL;
  28 static const char *todo_msg_fixed = "libtap malloc issue";
  29 static int todo = 0;
  30 static int test_died = 0;
  31 static int time_tests = 1;
  32 struct timespec last_time;
  33
  34 /* Encapsulate the pthread code in a conditional.  In the absence of
  35    libpthread the code does nothing */
  36 #ifdef HAVE_LIBPTHREAD
  37 #include <pthread.h>
  38 static pthread_mutex_t M = PTHREAD_MUTEX_INITIALIZER;
  39 #define LOCK   pthread_mutex_lock(&M);
  40 #define UNLOCK pthread_mutex_unlock(&M);
  41 #else
  42 #define LOCK
  43 #define UNLOCK
  44 #endif
  45
  46 static void _expected_tests(unsigned int);
  47 static void _tap_init(void);
  48 static void _cleanup(void);
  49
  50 #ifdef __MINGW32__
  51 static inline void flockfile(FILE *filehandle)
  52 {
  53         return;
  54 }
  55
  56 static inline void funlockfile(FILE *filehandle)
  57 {
  58         return;
  59 }
  60 #endif
  61
  62 /*
  63  * Generate a test result.
  64  *
  65  * ok -- boolean, indicates whether or not the test passed.
  66  * test_name -- the name of the test, may be NULL
  67  * test_comment -- a comment to print afterwards, may be NULL
  68  */
  69 unsigned int _gen_result(
  70         int ok, const char *func, const char *file, unsigned int line, const char *test_name, ...)
  71 {
  72         va_list ap;
  73         char *local_test_name = NULL;
  74         char *c;
  75         int name_is_digits;
  76
  77         LOCK;
  78
  79         test_count++;
  80
  81         /* Start by taking the test name and performing any printf()
  82            expansions on it */
  83         if (test_name != NULL) {
  84                 va_start(ap, test_name);
  85                 if (vasprintf(&local_test_name, test_name, ap) == -1) {
  86                         local_test_name = NULL;
  87                 }
  88                 va_end(ap);
  89
  90                 /* Make sure the test name contains more than digits
  91                    and spaces.  Emit an error message and exit if it
  92                    does */
  93                 if (local_test_name) {
  94                         name_is_digits = 1;
  95                         for (c = local_test_name; *c != '\0'; c++) {
  96                                 if (!isdigit((unsigned char) *c) && !isspace((unsigned char) *c)) {
  97                                         name_is_digits = 0;
  98                                         break;
  99                                 }
 100                         }
 101
 102                         if (name_is_digits) {
 103                                 diag("    You named your test '%s'.  You shouldn't use numbers for your test names.",
 104                                      local_test_name);
 105                                 diag("    Very confusing.");
 106                         }
 107                 }
 108         }
 109
 110         if (!ok) {
 111                 printf("not ");
 112                 failures++;
 113         }
 114
 115         printf("ok %d", test_count);
 116
 117         if (test_name != NULL) {
 118                 printf(" - ");
 119
 120                 /* Print the test name, escaping any '#' characters it
 121                    might contain */
 122                 if (local_test_name != NULL) {
 123                         flockfile(stdout);
 124                         for (c = local_test_name; *c != '\0'; c++) {
 125                                 if (*c == '#')
 126                                         fputc('\\', stdout);
 127                                 fputc((int) *c, stdout);
 128                         }
 129                         funlockfile(stdout);
 130                 } else { /* vasprintf() failed, use a fixed message */
 131                         printf("%s", todo_msg_fixed);
 132                 }
 133         }
 134
 135         /* If we're in a todo_start() block then flag the test as being
 136            TODO.  todo_msg should contain the message to print at this
 137            point.  If it's NULL then asprintf() failed, and we should
 138            use the fixed message.
 139
 140            This is not counted as a failure, so decrement the counter if
 141            the test failed. */
 142         if (todo) {
 143                 printf(" # TODO %s", todo_msg ? todo_msg : todo_msg_fixed);
 144                 if (!ok)
 145                         failures--;
 146         }
 147
 148         printf("\n");
 149         _output_test_time();
 150
 151         if (!ok) {
 152                 if (getenv("HARNESS_ACTIVE") != NULL)
 153                         fputs("\n", stderr);
 154
 155                 diag("    Failed %stest (%s:%s() at line %d)",
 156                      todo ? "(TODO) " : "",
 157                      file,
 158                      func,
 159                      line);
 160         }
 161         free(local_test_name);
 162
 163         UNLOCK;
 164
 165         /* We only care (when testing) that ok is positive, but here we
 166            specifically only want to return 1 or 0 */
 167         return ok ? 1 : 0;
 168 }
 169
 170 /*
 171  * Initialise the TAP library.  Will only do so once, however many times it's
 172  * called.
 173  */
 174 void _tap_init(void)
 175 {
 176         static int run_once = 0;
 177
 178         if (!run_once) {
 179                 atexit(_cleanup);
 180
 181                 /* stdout needs to be unbuffered so that the output appears
 182                    in the same place relative to stderr output as it does
 183                    with Test::Harness */
 184                 setbuf(stdout, 0);
 185
 186                 char *autotime_env = getenv("TAP_AUTOTIME");
 187                 if (autotime_env != NULL) {
 188                         time_tests = atoi(autotime_env);
 189                         if (time_tests != 0) {
 190                                 time_tests = 1;
 191                         }
 192                 }
 193
 194                 run_once = 1;
 195         }
 196         lttng_clock_gettime(CLOCK_MONOTONIC, &last_time);
 197 }
 198
 199 /*
 200  * Note that there's no plan.
 201  */
 202 int plan_no_plan(void)
 203 {
 204         LOCK;
 205
 206         _tap_init();
 207
 208         if (have_plan != 0) {
 209                 fprintf(stderr, "You tried to plan twice!\n");
 210                 test_died = 1;
 211                 UNLOCK;
 212                 exit(255);
 213         }
 214
 215         have_plan = 1;
 216         no_plan = 1;
 217
 218         UNLOCK;
 219
 220         return 1;
 221 }
 222
 223 /*
 224  * Note that the plan is to skip all tests
 225  */
 226 int plan_skip_all(const char *reason)
 227 {
 228         LOCK;
 229
 230         _tap_init();
 231
 232         skip_all = 1;
 233
 234         printf("1..0");
 235
 236         if (reason != NULL)
 237                 printf(" # Skip %s", reason);
 238
 239         printf("\n");
 240
 241         UNLOCK;
 242
 243         exit(0);
 244 }
 245
 246 /*
 247  * Note the number of tests that will be run.
 248  */
 249 int plan_tests(unsigned int tests)
 250 {
 251         LOCK;
 252
 253         _tap_init();
 254
 255         if (have_plan != 0) {
 256                 fprintf(stderr, "You tried to plan twice!\n");
 257                 test_died = 1;
 258                 UNLOCK;
 259                 exit(255);
 260         }
 261
 262         if (tests == 0) {
 263                 fprintf(stderr, "You said to run 0 tests!  You've got to run something.\n");
 264                 test_died = 1;
 265                 UNLOCK;
 266                 exit(255);
 267         }
 268
 269         have_plan = 1;
 270
 271         _expected_tests(tests);
 272
 273         UNLOCK;
 274
 275         return e_tests;
 276 }
 277
 278 unsigned int diag(const char *fmt, ...)
 279 {
 280         va_list ap;
 281
 282         fputs("# ", stderr);
 283
 284         va_start(ap, fmt);
 285         vfprintf(stderr, fmt, ap);
 286         va_end(ap);
 287
 288         fputs("\n", stderr);
 289
 290         return 0;
 291 }
 292
 293 void diag_multiline(const char *val)
 294 {
 295         size_t len, i, line_start_idx = 0;
 296
 297         assert(val);
 298         len = strlen(val);
 299
 300         for (i = 0; i < len; i++) {
 301                 int line_length;
 302
 303                 if (val[i] != '\n') {
 304                         continue;
 305                 }
 306
 307                 assert((i - line_start_idx + 1) <= INT_MAX);
 308                 line_length = i - line_start_idx + 1;
 309                 fprintf(stderr, "# %.*s", line_length, &val[line_start_idx]);
 310                 line_start_idx = i + 1;
 311         }
 312 }
 313
 314 void _expected_tests(unsigned int tests)
 315 {
 316         printf("1..%d\n", tests);
 317         e_tests = tests;
 318 }
 319
 320 int skip(unsigned int n, const char *fmt, ...)
 321 {
 322         va_list ap;
 323         char *skip_msg = NULL;
 324
 325         LOCK;
 326
 327         va_start(ap, fmt);
 328         if (vasprintf(&skip_msg, fmt, ap) == -1) {
 329                 skip_msg = NULL;
 330         }
 331         va_end(ap);
 332
 333         while (n-- > 0) {
 334                 test_count++;
 335                 printf("ok %d # skip %s\n",
 336                        test_count,
 337                        skip_msg != NULL ? skip_msg : "libtap():malloc() failed");
 338                 _output_test_time();
 339         }
 340
 341         free(skip_msg);
 342
 343         UNLOCK;
 344
 345         return 1;
 346 }
 347
 348 void _output_test_time(void)
 349 {
 350         struct timespec new_time;
 351         int64_t time_ns;
 352         if (time_tests) {
 353                 lttng_clock_gettime(CLOCK_MONOTONIC, &new_time);
 354                 time_ns = elapsed_time_ns(&last_time, &new_time);
 355                 printf("  ---\n    duration_ms: %ld.%ld\n  ...\n",
 356                        time_ns / 1000000,
 357                        time_ns % 1000000);
 358         }
 359         lttng_clock_gettime(CLOCK_MONOTONIC, &last_time);
 360 }
 361
 362 void todo_start(const char *fmt, ...)
 363 {
 364         va_list ap;
 365
 366         LOCK;
 367
 368         va_start(ap, fmt);
 369         if (vasprintf(&todo_msg, fmt, ap) == -1) {
 370                 todo_msg = NULL;
 371         }
 372         va_end(ap);
 373
 374         todo = 1;
 375
 376         UNLOCK;
 377 }
 378
 379 void todo_end(void)
 380 {
 381         LOCK;
 382
 383         todo = 0;
 384         free(todo_msg);
 385
 386         UNLOCK;
 387 }
 388
 389 int exit_status(void)
 390 {
 391         int r;
 392
 393         LOCK;
 394
 395         /* If there's no plan, just return the number of failures */
 396         if (no_plan || !have_plan) {
 397                 UNLOCK;
 398                 return failures;
 399         }
 400
 401         /* Ran too many tests?  Return the number of tests that were run
 402            that shouldn't have been */
 403         if (e_tests < test_count) {
 404                 r = test_count - e_tests;
 405                 UNLOCK;
 406                 return r;
 407         }
 408
 409         /* Return the number of tests that failed + the number of tests
 410            that weren't run */
 411         r = failures + e_tests - test_count;
 412         UNLOCK;
 413
 414         return r;
 415 }
 416
 417 /*
 418  * Cleanup at the end of the run, produce any final output that might be
 419  * required.
 420  */
 421 void _cleanup(void)
 422 {
 423         LOCK;
 424
 425         /* If plan_no_plan() wasn't called, and we don't have a plan,
 426            and we're not skipping everything, then something happened
 427            before we could produce any output */
 428         if (!no_plan && !have_plan && !skip_all) {
 429                 diag("Looks like your test died before it could output anything.");
 430                 UNLOCK;
 431                 return;
 432         }
 433
 434         if (test_died) {
 435                 diag("Looks like your test died just after %d.", test_count);
 436                 UNLOCK;
 437                 return;
 438         }
 439
 440         /* No plan provided, but now we know how many tests were run, and can
 441            print the header at the end */
 442         if (!skip_all && (no_plan || !have_plan)) {
 443                 printf("1..%d\n", test_count);
 444         }
 445
 446         if ((have_plan && !no_plan) && e_tests < test_count) {
 447                 diag("Looks like you planned %d %s but ran %d extra.",
 448                      e_tests,
 449                      e_tests == 1 ? "test" : "tests",
 450                      test_count - e_tests);
 451                 UNLOCK;
 452                 return;
 453         }
 454
 455         if ((have_plan || !no_plan) && e_tests > test_count) {
 456                 diag("Looks like you planned %d %s but only ran %d.",
 457                      e_tests,
 458                      e_tests == 1 ? "test" : "tests",
 459                      test_count);
 460                 UNLOCK;
 461                 return;
 462         }
 463
 464         if (failures)
 465                 diag("Looks like you failed %d %s of %d.",
 466                      failures,
 467                      failures == 1 ? "test" : "tests",
 468                      test_count);
 469
 470         UNLOCK;
 471 }