tests: Automatically time TAP tests
[lttng-tools.git] / tests / utils / tap / tap.c
CommitLineData
9f4a25d3 1/*
9d16b343
MJ
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
9f4a25d3
SM
4 * Copyright (C) 2004 Nik Clayton
5 * Copyright (C) 2017 Jérémie Galarneau
86a96e6c
CB
6 */
7
2a69bf14
KS
8#include "../utils.h"
9#include "common/compat/time.hpp"
28f23191
JG
10#include "tap.h"
11
12#include <assert.h>
86a96e6c 13#include <ctype.h>
28f23191 14#include <limits.h>
86a96e6c
CB
15#include <stdarg.h>
16#include <stdio.h>
17#include <stdlib.h>
9f4a25d3 18#include <string.h>
2a69bf14 19#include <time.h>
86a96e6c
CB
20
21static int no_plan = 0;
22static int skip_all = 0;
23static int have_plan = 0;
24static unsigned int test_count = 0; /* Number of tests that have been run */
25static unsigned int e_tests = 0; /* Expected number of tests to run */
26static unsigned int failures = 0; /* Number of tests that failed */
27static char *todo_msg = NULL;
b53d4e59 28static const char *todo_msg_fixed = "libtap malloc issue";
86a96e6c
CB
29static int todo = 0;
30static int test_died = 0;
2a69bf14
KS
31static int time_tests = 1;
32struct timespec last_time;
86a96e6c
CB
33
34/* Encapsulate the pthread code in a conditional. In the absence of
35 libpthread the code does nothing */
36#ifdef HAVE_LIBPTHREAD
37#include <pthread.h>
38static pthread_mutex_t M = PTHREAD_MUTEX_INITIALIZER;
28f23191
JG
39#define LOCK pthread_mutex_lock(&M);
40#define UNLOCK pthread_mutex_unlock(&M);
86a96e6c 41#else
28f23191
JG
42#define LOCK
43#define UNLOCK
86a96e6c
CB
44#endif
45
46static void _expected_tests(unsigned int);
47static void _tap_init(void);
48static void _cleanup(void);
49
9f4a25d3 50#ifdef __MINGW32__
28f23191
JG
51static inline void flockfile(FILE *filehandle)
52{
53 return;
9f4a25d3
SM
54}
55
28f23191
JG
56static inline void funlockfile(FILE *filehandle)
57{
58 return;
9f4a25d3
SM
59}
60#endif
61
86a96e6c
CB
62/*
63 * Generate a test result.
64 *
65 * ok -- boolean, indicates whether or not the test passed.
66 * test_name -- the name of the test, may be NULL
67 * test_comment -- a comment to print afterwards, may be NULL
68 */
28f23191
JG
69unsigned int _gen_result(
70 int ok, const char *func, const char *file, unsigned int line, const char *test_name, ...)
86a96e6c
CB
71{
72 va_list ap;
73 char *local_test_name = NULL;
74 char *c;
75 int name_is_digits;
76
77 LOCK;
78
79 test_count++;
80
81 /* Start by taking the test name and performing any printf()
82 expansions on it */
28f23191 83 if (test_name != NULL) {
86a96e6c 84 va_start(ap, test_name);
f2068bce
JG
85 if (vasprintf(&local_test_name, test_name, ap) == -1) {
86 local_test_name = NULL;
87 }
86a96e6c
CB
88 va_end(ap);
89
90 /* Make sure the test name contains more than digits
91 and spaces. Emit an error message and exit if it
92 does */
28f23191 93 if (local_test_name) {
86a96e6c 94 name_is_digits = 1;
28f23191
JG
95 for (c = local_test_name; *c != '\0'; c++) {
96 if (!isdigit((unsigned char) *c) && !isspace((unsigned char) *c)) {
86a96e6c
CB
97 name_is_digits = 0;
98 break;
99 }
100 }
101
28f23191
JG
102 if (name_is_digits) {
103 diag(" You named your test '%s'. You shouldn't use numbers for your test names.",
104 local_test_name);
86a96e6c
CB
105 diag(" Very confusing.");
106 }
107 }
108 }
109
28f23191 110 if (!ok) {
86a96e6c
CB
111 printf("not ");
112 failures++;
113 }
114
115 printf("ok %d", test_count);
116
28f23191 117 if (test_name != NULL) {
86a96e6c
CB
118 printf(" - ");
119
120 /* Print the test name, escaping any '#' characters it
121 might contain */
28f23191 122 if (local_test_name != NULL) {
86a96e6c 123 flockfile(stdout);
28f23191
JG
124 for (c = local_test_name; *c != '\0'; c++) {
125 if (*c == '#')
86a96e6c 126 fputc('\\', stdout);
28f23191 127 fputc((int) *c, stdout);
86a96e6c
CB
128 }
129 funlockfile(stdout);
28f23191 130 } else { /* vasprintf() failed, use a fixed message */
86a96e6c
CB
131 printf("%s", todo_msg_fixed);
132 }
133 }
134
135 /* If we're in a todo_start() block then flag the test as being
136 TODO. todo_msg should contain the message to print at this
137 point. If it's NULL then asprintf() failed, and we should
138 use the fixed message.
139
140 This is not counted as a failure, so decrement the counter if
141 the test failed. */
28f23191 142 if (todo) {
86a96e6c 143 printf(" # TODO %s", todo_msg ? todo_msg : todo_msg_fixed);
28f23191 144 if (!ok)
86a96e6c
CB
145 failures--;
146 }
147
148 printf("\n");
2a69bf14 149 _output_test_time();
86a96e6c 150
28f23191
JG
151 if (!ok) {
152 if (getenv("HARNESS_ACTIVE") != NULL)
86a96e6c
CB
153 fputs("\n", stderr);
154
155 diag(" Failed %stest (%s:%s() at line %d)",
28f23191
JG
156 todo ? "(TODO) " : "",
157 file,
158 func,
159 line);
86a96e6c
CB
160 }
161 free(local_test_name);
162
163 UNLOCK;
164
165 /* We only care (when testing) that ok is positive, but here we
166 specifically only want to return 1 or 0 */
167 return ok ? 1 : 0;
168}
169
170/*
171 * Initialise the TAP library. Will only do so once, however many times it's
172 * called.
173 */
28f23191 174void _tap_init(void)
86a96e6c
CB
175{
176 static int run_once = 0;
177
28f23191 178 if (!run_once) {
86a96e6c
CB
179 atexit(_cleanup);
180
181 /* stdout needs to be unbuffered so that the output appears
182 in the same place relative to stderr output as it does
183 with Test::Harness */
184 setbuf(stdout, 0);
2a69bf14
KS
185
186 char *autotime_env = getenv("TAP_AUTOTIME");
187 if (autotime_env != NULL) {
188 time_tests = atoi(autotime_env);
189 if (time_tests != 0) {
190 time_tests = 1;
191 }
192 }
193
86a96e6c
CB
194 run_once = 1;
195 }
2a69bf14 196 lttng_clock_gettime(CLOCK_MONOTONIC, &last_time);
86a96e6c
CB
197}
198
199/*
200 * Note that there's no plan.
201 */
28f23191 202int plan_no_plan(void)
86a96e6c 203{
86a96e6c
CB
204 LOCK;
205
206 _tap_init();
207
28f23191 208 if (have_plan != 0) {
86a96e6c
CB
209 fprintf(stderr, "You tried to plan twice!\n");
210 test_died = 1;
211 UNLOCK;
212 exit(255);
213 }
214
215 have_plan = 1;
216 no_plan = 1;
217
218 UNLOCK;
219
220 return 1;
221}
222
223/*
224 * Note that the plan is to skip all tests
225 */
28f23191 226int plan_skip_all(const char *reason)
86a96e6c 227{
86a96e6c
CB
228 LOCK;
229
230 _tap_init();
231
232 skip_all = 1;
233
234 printf("1..0");
235
28f23191 236 if (reason != NULL)
86a96e6c
CB
237 printf(" # Skip %s", reason);
238
239 printf("\n");
240
241 UNLOCK;
242
243 exit(0);
244}
245
246/*
247 * Note the number of tests that will be run.
248 */
28f23191 249int plan_tests(unsigned int tests)
86a96e6c 250{
86a96e6c
CB
251 LOCK;
252
253 _tap_init();
254
28f23191 255 if (have_plan != 0) {
86a96e6c
CB
256 fprintf(stderr, "You tried to plan twice!\n");
257 test_died = 1;
258 UNLOCK;
259 exit(255);
260 }
261
28f23191 262 if (tests == 0) {
86a96e6c
CB
263 fprintf(stderr, "You said to run 0 tests! You've got to run something.\n");
264 test_died = 1;
265 UNLOCK;
266 exit(255);
267 }
268
269 have_plan = 1;
270
271 _expected_tests(tests);
272
273 UNLOCK;
274
275 return e_tests;
276}
277
28f23191 278unsigned int diag(const char *fmt, ...)
86a96e6c
CB
279{
280 va_list ap;
281
282 fputs("# ", stderr);
283
284 va_start(ap, fmt);
285 vfprintf(stderr, fmt, ap);
286 va_end(ap);
287
288 fputs("\n", stderr);
289
290 return 0;
291}
292
28f23191 293void diag_multiline(const char *val)
9f4a25d3
SM
294{
295 size_t len, i, line_start_idx = 0;
296
297 assert(val);
298 len = strlen(val);
299
300 for (i = 0; i < len; i++) {
301 int line_length;
302
303 if (val[i] != '\n') {
304 continue;
305 }
306
307 assert((i - line_start_idx + 1) <= INT_MAX);
308 line_length = i - line_start_idx + 1;
309 fprintf(stderr, "# %.*s", line_length, &val[line_start_idx]);
310 line_start_idx = i + 1;
311 }
312}
313
28f23191 314void _expected_tests(unsigned int tests)
86a96e6c 315{
86a96e6c
CB
316 printf("1..%d\n", tests);
317 e_tests = tests;
318}
319
28f23191 320int skip(unsigned int n, const char *fmt, ...)
86a96e6c
CB
321{
322 va_list ap;
c2788d69 323 char *skip_msg = NULL;
86a96e6c
CB
324
325 LOCK;
326
327 va_start(ap, fmt);
9f4a25d3 328 if (vasprintf(&skip_msg, fmt, ap) == -1) {
f2068bce
JG
329 skip_msg = NULL;
330 }
86a96e6c
CB
331 va_end(ap);
332
28f23191 333 while (n-- > 0) {
86a96e6c 334 test_count++;
28f23191
JG
335 printf("ok %d # skip %s\n",
336 test_count,
337 skip_msg != NULL ? skip_msg : "libtap():malloc() failed");
2a69bf14 338 _output_test_time();
86a96e6c
CB
339 }
340
341 free(skip_msg);
342
343 UNLOCK;
344
345 return 1;
346}
347
2a69bf14
KS
348void _output_test_time(void)
349{
350 struct timespec new_time;
351 int64_t time_ns;
352 if (time_tests) {
353 lttng_clock_gettime(CLOCK_MONOTONIC, &new_time);
354 time_ns = elapsed_time_ns(&last_time, &new_time);
355 printf(" ---\n duration_ms: %ld.%ld\n ...\n",
356 time_ns / 1000000,
357 time_ns % 1000000);
358 }
359 lttng_clock_gettime(CLOCK_MONOTONIC, &last_time);
360}
361
28f23191 362void todo_start(const char *fmt, ...)
86a96e6c
CB
363{
364 va_list ap;
365
366 LOCK;
367
368 va_start(ap, fmt);
f2068bce
JG
369 if (vasprintf(&todo_msg, fmt, ap) == -1) {
370 todo_msg = NULL;
371 }
86a96e6c
CB
372 va_end(ap);
373
374 todo = 1;
375
376 UNLOCK;
377}
378
28f23191 379void todo_end(void)
86a96e6c 380{
86a96e6c
CB
381 LOCK;
382
383 todo = 0;
384 free(todo_msg);
385
386 UNLOCK;
387}
388
28f23191 389int exit_status(void)
86a96e6c
CB
390{
391 int r;
392
393 LOCK;
394
395 /* If there's no plan, just return the number of failures */
28f23191 396 if (no_plan || !have_plan) {
86a96e6c
CB
397 UNLOCK;
398 return failures;
399 }
400
401 /* Ran too many tests? Return the number of tests that were run
402 that shouldn't have been */
28f23191 403 if (e_tests < test_count) {
86a96e6c
CB
404 r = test_count - e_tests;
405 UNLOCK;
406 return r;
407 }
408
409 /* Return the number of tests that failed + the number of tests
410 that weren't run */
411 r = failures + e_tests - test_count;
412 UNLOCK;
413
414 return r;
415}
416
417/*
418 * Cleanup at the end of the run, produce any final output that might be
419 * required.
420 */
28f23191 421void _cleanup(void)
86a96e6c 422{
86a96e6c
CB
423 LOCK;
424
425 /* If plan_no_plan() wasn't called, and we don't have a plan,
426 and we're not skipping everything, then something happened
427 before we could produce any output */
28f23191 428 if (!no_plan && !have_plan && !skip_all) {
86a96e6c
CB
429 diag("Looks like your test died before it could output anything.");
430 UNLOCK;
431 return;
432 }
433
28f23191 434 if (test_died) {
86a96e6c
CB
435 diag("Looks like your test died just after %d.", test_count);
436 UNLOCK;
437 return;
438 }
439
86a96e6c
CB
440 /* No plan provided, but now we know how many tests were run, and can
441 print the header at the end */
28f23191 442 if (!skip_all && (no_plan || !have_plan)) {
86a96e6c
CB
443 printf("1..%d\n", test_count);
444 }
445
28f23191 446 if ((have_plan && !no_plan) && e_tests < test_count) {
86a96e6c 447 diag("Looks like you planned %d %s but ran %d extra.",
28f23191
JG
448 e_tests,
449 e_tests == 1 ? "test" : "tests",
450 test_count - e_tests);
86a96e6c
CB
451 UNLOCK;
452 return;
453 }
454
28f23191 455 if ((have_plan || !no_plan) && e_tests > test_count) {
86a96e6c 456 diag("Looks like you planned %d %s but only ran %d.",
28f23191
JG
457 e_tests,
458 e_tests == 1 ? "test" : "tests",
459 test_count);
86a96e6c
CB
460 UNLOCK;
461 return;
462 }
463
28f23191 464 if (failures)
86a96e6c 465 diag("Looks like you failed %d %s of %d.",
28f23191
JG
466 failures,
467 failures == 1 ? "test" : "tests",
468 test_count);
86a96e6c
CB
469
470 UNLOCK;
471}
This page took 0.074547 seconds and 4 git commands to generate.