tests/utils/tap.c

   1 /*-
   2  * Copyright (c) 2004 Nik Clayton
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <ctype.h>
  28 #include <stdarg.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31
  32 #include "tap.h"
  33
  34 static int no_plan = 0;
  35 static int skip_all = 0;
  36 static int have_plan = 0;
  37 static unsigned int test_count = 0; /* Number of tests that have been run */
  38 static unsigned int e_tests = 0; /* Expected number of tests to run */
  39 static unsigned int failures = 0; /* Number of tests that failed */
  40 static char *todo_msg = NULL;
  41 static char *todo_msg_fixed = "libtap malloc issue";
  42 static int todo = 0;
  43 static int test_died = 0;
  44 static int tap_is_disabled = 0;
  45
  46 /* Encapsulate the pthread code in a conditional.  In the absence of
  47    libpthread the code does nothing */
  48 #ifdef HAVE_LIBPTHREAD
  49 #include <pthread.h>
  50 static pthread_mutex_t M = PTHREAD_MUTEX_INITIALIZER;
  51 # define LOCK pthread_mutex_lock(&M);
  52 # define UNLOCK pthread_mutex_unlock(&M);
  53 #else
  54 # define LOCK
  55 # define UNLOCK
  56 #endif
  57
  58 static void _expected_tests(unsigned int);
  59 static void _tap_init(void);
  60 static void _cleanup(void);
  61
  62 /*
  63  * Generate a test result.
  64  *
  65  * ok -- boolean, indicates whether or not the test passed.
  66  * test_name -- the name of the test, may be NULL
  67  * test_comment -- a comment to print afterwards, may be NULL
  68  */
  69 unsigned int
  70 _gen_result(int ok, const char *func, char *file, unsigned int line,
  71             char *test_name, ...)
  72 {
  73         va_list ap;
  74         char *local_test_name = NULL;
  75         char *c;
  76         int name_is_digits;
  77
  78         LOCK;
  79
  80         test_count++;
  81
  82         /* Start by taking the test name and performing any printf()
  83            expansions on it */
  84         if(test_name != NULL) {
  85                 va_start(ap, test_name);
  86                 if (vasprintf(&local_test_name, test_name, ap) == -1) {
  87                         local_test_name = NULL;
  88                 }
  89                 va_end(ap);
  90
  91                 /* Make sure the test name contains more than digits
  92                    and spaces.  Emit an error message and exit if it
  93                    does */
  94                 if(local_test_name) {
  95                         name_is_digits = 1;
  96                         for(c = local_test_name; *c != '\0'; c++) {
  97                                 if(!isdigit(*c) && !isspace(*c)) {
  98                                         name_is_digits = 0;
  99                                         break;
 100                                 }
 101                         }
 102
 103                         if(name_is_digits) {
 104                                 diag("    You named your test '%s'.  You shouldn't use numbers for your test names.", local_test_name);
 105                                 diag("    Very confusing.");
 106                         }
 107                 }
 108         }
 109
 110         if(!ok) {
 111                 printf("not ");
 112                 failures++;
 113         }
 114
 115         printf("ok %d", test_count);
 116
 117         if(test_name != NULL) {
 118                 printf(" - ");
 119
 120                 /* Print the test name, escaping any '#' characters it
 121                    might contain */
 122                 if(local_test_name != NULL) {
 123                         flockfile(stdout);
 124                         for(c = local_test_name; *c != '\0'; c++) {
 125                                 if(*c == '#')
 126                                         fputc('\\', stdout);
 127                                 fputc((int)*c, stdout);
 128                         }
 129                         funlockfile(stdout);
 130                 } else {        /* vasprintf() failed, use a fixed message */
 131                         printf("%s", todo_msg_fixed);
 132                 }
 133         }
 134
 135         /* If we're in a todo_start() block then flag the test as being
 136            TODO.  todo_msg should contain the message to print at this
 137            point.  If it's NULL then asprintf() failed, and we should
 138            use the fixed message.
 139
 140            This is not counted as a failure, so decrement the counter if
 141            the test failed. */
 142         if(todo) {
 143                 printf(" # TODO %s", todo_msg ? todo_msg : todo_msg_fixed);
 144                 if(!ok)
 145                         failures--;
 146         }
 147
 148         printf("\n");
 149
 150         if(!ok) {
 151                 if(getenv("HARNESS_ACTIVE") != NULL)
 152                         fputs("\n", stderr);
 153
 154                 diag("    Failed %stest (%s:%s() at line %d)",
 155                      todo ? "(TODO) " : "", file, func, line);
 156         }
 157         free(local_test_name);
 158
 159         UNLOCK;
 160
 161         /* We only care (when testing) that ok is positive, but here we
 162            specifically only want to return 1 or 0 */
 163         return ok ? 1 : 0;
 164 }
 165
 166 /*
 167  * Initialise the TAP library.  Will only do so once, however many times it's
 168  * called.
 169  */
 170 void
 171 _tap_init(void)
 172 {
 173         static int run_once = 0;
 174
 175         if(!run_once) {
 176                 atexit(_cleanup);
 177
 178                 /* stdout needs to be unbuffered so that the output appears
 179                    in the same place relative to stderr output as it does
 180                    with Test::Harness */
 181                 setbuf(stdout, 0);
 182                 run_once = 1;
 183         }
 184 }
 185
 186 /*
 187  * Note that there's no plan.
 188  */
 189 int
 190 plan_no_plan(void)
 191 {
 192
 193         LOCK;
 194
 195         _tap_init();
 196
 197         if(have_plan != 0) {
 198                 fprintf(stderr, "You tried to plan twice!\n");
 199                 test_died = 1;
 200                 UNLOCK;
 201                 exit(255);
 202         }
 203
 204         have_plan = 1;
 205         no_plan = 1;
 206
 207         UNLOCK;
 208
 209         return 1;
 210 }
 211
 212 /*
 213  * Note that the plan is to skip all tests
 214  */
 215 int
 216 plan_skip_all(char *reason)
 217 {
 218
 219         LOCK;
 220
 221         _tap_init();
 222
 223         skip_all = 1;
 224
 225         printf("1..0");
 226
 227         if(reason != NULL)
 228                 printf(" # Skip %s", reason);
 229
 230         printf("\n");
 231
 232         UNLOCK;
 233
 234         exit(0);
 235 }
 236
 237 /*
 238  * Note the number of tests that will be run.
 239  */
 240 int
 241 plan_tests(unsigned int tests)
 242 {
 243
 244         LOCK;
 245
 246         _tap_init();
 247
 248         if(have_plan != 0) {
 249                 fprintf(stderr, "You tried to plan twice!\n");
 250                 test_died = 1;
 251                 UNLOCK;
 252                 exit(255);
 253         }
 254
 255         if(tests == 0) {
 256                 fprintf(stderr, "You said to run 0 tests!  You've got to run something.\n");
 257                 test_died = 1;
 258                 UNLOCK;
 259                 exit(255);
 260         }
 261
 262         have_plan = 1;
 263
 264         _expected_tests(tests);
 265
 266         UNLOCK;
 267
 268         return e_tests;
 269 }
 270
 271 unsigned int
 272 diag(char *fmt, ...)
 273 {
 274         va_list ap;
 275
 276         fputs("# ", stderr);
 277
 278         va_start(ap, fmt);
 279         vfprintf(stderr, fmt, ap);
 280         va_end(ap);
 281
 282         fputs("\n", stderr);
 283
 284         return 0;
 285 }
 286
 287 unsigned int
 288 rdiag_start(void)
 289 {
 290         fputs("# ", stderr);
 291         return 0;
 292 }
 293
 294 unsigned int
 295 rdiag(char *fmt, ...)
 296 {
 297         va_list ap;
 298
 299         va_start(ap, fmt);
 300         vfprintf(stderr, fmt, ap);
 301         va_end(ap);
 302
 303         return 0;
 304 }
 305
 306 unsigned int
 307 rdiag_end(void)
 308 {
 309         fputs("\n", stderr);
 310         return 0;
 311 }
 312
 313 void
 314 _expected_tests(unsigned int tests)
 315 {
 316
 317         printf("1..%d\n", tests);
 318         e_tests = tests;
 319 }
 320
 321 int
 322 skip(unsigned int n, char *fmt, ...)
 323 {
 324         va_list ap;
 325         char *skip_msg = NULL;
 326
 327         LOCK;
 328
 329         va_start(ap, fmt);
 330         if (asprintf(&skip_msg, fmt, ap) == -1) {
 331                 skip_msg = NULL;
 332         }
 333         va_end(ap);
 334
 335         while(n-- > 0) {
 336                 test_count++;
 337                 printf("ok %d # skip %s\n", test_count,
 338                        skip_msg != NULL ?
 339                        skip_msg : "libtap():malloc() failed");
 340         }
 341
 342         free(skip_msg);
 343
 344         UNLOCK;
 345
 346         return 1;
 347 }
 348
 349 void
 350 todo_start(char *fmt, ...)
 351 {
 352         va_list ap;
 353
 354         LOCK;
 355
 356         va_start(ap, fmt);
 357         if (vasprintf(&todo_msg, fmt, ap) == -1) {
 358                 todo_msg = NULL;
 359         }
 360         va_end(ap);
 361
 362         todo = 1;
 363
 364         UNLOCK;
 365 }
 366
 367 void
 368 todo_end(void)
 369 {
 370
 371         LOCK;
 372
 373         todo = 0;
 374         free(todo_msg);
 375
 376         UNLOCK;
 377 }
 378
 379 int
 380 exit_status(void)
 381 {
 382         int r;
 383
 384         LOCK;
 385
 386         /* If there's no plan, just return the number of failures */
 387         if(no_plan || !have_plan) {
 388                 UNLOCK;
 389                 return failures;
 390         }
 391
 392         /* Ran too many tests?  Return the number of tests that were run
 393            that shouldn't have been */
 394         if(e_tests < test_count) {
 395                 r = test_count - e_tests;
 396                 UNLOCK;
 397                 return r;
 398         }
 399
 400         /* Return the number of tests that failed + the number of tests
 401            that weren't run */
 402         r = failures + e_tests - test_count;
 403         UNLOCK;
 404
 405         return r;
 406 }
 407
 408 /*
 409  * Cleanup at the end of the run, produce any final output that might be
 410  * required.
 411  */
 412 void
 413 _cleanup(void)
 414 {
 415
 416         LOCK;
 417
 418         if (tap_is_disabled) {
 419                 UNLOCK;
 420                 return;
 421         }
 422
 423         /* If plan_no_plan() wasn't called, and we don't have a plan,
 424            and we're not skipping everything, then something happened
 425            before we could produce any output */
 426         if(!no_plan && !have_plan && !skip_all) {
 427                 diag("Looks like your test died before it could output anything.");
 428                 UNLOCK;
 429                 return;
 430         }
 431
 432         if(test_died) {
 433                 diag("Looks like your test died just after %d.", test_count);
 434                 UNLOCK;
 435                 return;
 436         }
 437
 438
 439         /* No plan provided, but now we know how many tests were run, and can
 440            print the header at the end */
 441         if(!skip_all && (no_plan || !have_plan)) {
 442                 printf("1..%d\n", test_count);
 443         }
 444
 445         if((have_plan && !no_plan) && e_tests < test_count) {
 446                 diag("Looks like you planned %d %s but ran %d extra.",
 447                      e_tests, e_tests == 1 ? "test" : "tests", test_count - e_tests);
 448                 UNLOCK;
 449                 return;
 450         }
 451
 452         if((have_plan || !no_plan) && e_tests > test_count) {
 453                 diag("Looks like you planned %d %s but only ran %d.",
 454                      e_tests, e_tests == 1 ? "test" : "tests", test_count);
 455                 UNLOCK;
 456                 return;
 457         }
 458
 459         if(failures)
 460                 diag("Looks like you failed %d %s of %d.",
 461                      failures, failures == 1 ? "test" : "tests", test_count);
 462
 463         UNLOCK;
 464 }
 465
 466 /* Disable tap for this process. */
 467 void
 468 tap_disable(void)
 469 {
 470         LOCK;
 471         tap_is_disabled = 1;
 472         UNLOCK;
 473 }