trunk/tests/kernel/test-wbias-rwlock.c

   1 /* test-wbias-rwlock.c
   2  *
   3  */
   4
   5 #include <linux/module.h>
   6 #include <linux/proc_fs.h>
   7 #include <linux/sched.h>
   8 #include <linux/timex.h>
   9 #include <linux/kthread.h>
  10 #include <linux/delay.h>
  11 #include <linux/hardirq.h>
  12 #include <linux/module.h>
  13 #include <linux/percpu.h>
  14 #include <linux/spinlock.h>
  15 #include <asm/ptrace.h>
  16 #include <linux/wbias-rwlock.h>
  17
  18 /* Test with no contention duration, in seconds */
  19 #define SINGLE_WRITER_TEST_DURATION 10
  20 #define SINGLE_READER_TEST_DURATION 10
  21 #define MULTIPLE_READERS_TEST_DURATION 10
  22
  23 /* Test duration, in seconds */
  24 #define TEST_DURATION 60
  25
  26 #define NR_VARS 100
  27 #define NR_WRITERS 2
  28 #define NR_TRYLOCK_WRITERS 1
  29 #define NR_NPREADERS 2
  30 #define NR_TRYLOCK_READERS 1
  31
  32 /*
  33  * 1 : test standard rwlock
  34  * 0 : test wbiasrwlock
  35  */
  36 #define TEST_STD_RWLOCK 0
  37
  38 /*
  39  * 1 : test with thread and interrupt readers.
  40  * 0 : test only with thread readers.
  41  */
  42 #define TEST_INTERRUPTS 1
  43
  44 #if (TEST_INTERRUPTS)
  45 #define NR_INTERRUPT_READERS 1
  46 #define NR_TRYLOCK_INTERRUPT_READERS 1
  47 #else
  48 #define NR_INTERRUPT_READERS 0
  49 #define NR_TRYLOCK_INTERRUPT_READERS 0
  50 #endif
  51
  52 /*
  53  * 1 : test with thread preemption readers.
  54  * 0 : test only with non-preemptable thread readers.
  55  */
  56 #define TEST_PREEMPT 1
  57
  58 #if (TEST_PREEMPT)
  59 #define NR_PREADERS 2
  60 #else
  61 #define NR_PREADERS 0
  62 #endif
  63
  64
  65 /*
  66  * Writer iteration delay, in us. 0 for busy loop. Caution : writers can
  67  * starve readers.
  68  */
  69 #define WRITER_DELAY 100
  70 #define TRYLOCK_WRITER_DELAY 1000
  71
  72 /*
  73  * Number of iterations after which a trylock writer fails.
  74  * -1 for infinite loop.
  75  */
  76 #define TRYLOCK_WRITERS_FAIL_ITER 100
  77
  78 /* Thread and interrupt reader delay, in ms */
  79 #define THREAD_READER_DELAY 0   /* busy loop */
  80 #define INTERRUPT_READER_DELAY 100
  81
  82 static int var[NR_VARS];
  83 static struct task_struct *preader_threads[NR_PREADERS];
  84 static struct task_struct *npreader_threads[NR_NPREADERS];
  85 static struct task_struct *trylock_reader_threads[NR_TRYLOCK_READERS];
  86 static struct task_struct *writer_threads[NR_WRITERS];
  87 static struct task_struct *trylock_writer_threads[NR_TRYLOCK_WRITERS];
  88 static struct task_struct *interrupt_reader[NR_INTERRUPT_READERS];
  89 static struct task_struct *trylock_interrupt_reader[NR_TRYLOCK_INTERRUPT_READERS];
  90
  91 #if (TEST_STD_RWLOCK)
  92
  93 static DEFINE_RWLOCK(std_rw_lock);
  94
  95 #define wrap_read_lock()        read_lock(&std_rw_lock)
  96 #define wrap_read_trylock()     read_trylock(&std_rw_lock)
  97 #define wrap_read_unlock()      read_unlock(&std_rw_lock)
  98
  99 #define wrap_read_lock_inatomic()       read_lock(&std_rw_lock)
 100 #define wrap_read_trylock_inatomic()    read_trylock(&std_rw_lock)
 101 #define wrap_read_unlock_inatomic()     read_unlock(&std_rw_lock)
 102
 103 #define wrap_read_lock_irq()    read_lock(&std_rw_lock)
 104 #define wrap_read_trylock_irq() read_trylock(&std_rw_lock)
 105 #define wrap_read_unlock_irq()  read_unlock(&std_rw_lock)
 106
 107 #if (TEST_INTERRUPTS)
 108 #define wrap_write_lock()       write_lock_irq(&std_rw_lock)
 109 #define wrap_write_unlock()     write_unlock_irq(&std_rw_lock)
 110 #else
 111 #define wrap_write_lock()       write_lock(&std_rw_lock)
 112 #define wrap_write_unlock()     write_unlock(&std_rw_lock)
 113 #endif
 114
 115 #else
 116
 117 static DEFINE_WBIAS_RWLOCK(wbiasrwlock);
 118
 119 #define wrap_read_lock()        wbias_read_lock(&wbiasrwlock)
 120 #define wrap_read_trylock()     wbias_read_trylock(&wbiasrwlock)
 121 #define wrap_read_unlock()      wbias_read_unlock(&wbiasrwlock)
 122
 123 #define wrap_read_lock_inatomic()       wbias_read_lock_inatomic(&wbiasrwlock)
 124 #define wrap_read_trylock_inatomic()    \
 125                 wbias_read_trylock_inatomic(&wbiasrwlock)
 126 #define wrap_read_unlock_inatomic()     \
 127                 wbias_read_unlock_inatomic(&wbiasrwlock)
 128
 129 #define wrap_read_lock_irq()    wbias_read_lock_irq(&wbiasrwlock)
 130 #define wrap_read_trylock_irq() wbias_read_trylock_irq(&wbiasrwlock)
 131 #define wrap_read_unlock_irq()  wbias_read_unlock_irq(&wbiasrwlock)
 132
 133 #if (TEST_INTERRUPTS)
 134 #define wrap_write_lock()       wbias_write_lock_irq(&wbiasrwlock)
 135 #define wrap_write_unlock()     wbias_write_unlock_irq(&wbiasrwlock)
 136 #define wrap_write_trylock_else_subscribe()     \
 137         wbias_write_trylock_irq_else_subscribe(&wbiasrwlock)
 138 #define wrap_write_trylock_subscribed()         \
 139         wbias_write_trylock_irq_subscribed(&wbiasrwlock)
 140 #define wrap_write_unsubscribe()                \
 141         wbias_write_unsubscribe_irq(&wbiasrwlock)
 142 #else
 143 #if (TEST_PREEMPT)
 144 #define wrap_write_lock()       wbias_write_lock(&wbiasrwlock)
 145 #define wrap_write_unlock()     wbias_write_unlock(&wbiasrwlock)
 146 #define wrap_write_trylock_else_subscribe()     \
 147         wbias_write_trylock_else_subscribe(&wbiasrwlock)
 148 #define wrap_write_trylock_subscribed()         \
 149         wbias_write_trylock_subscribed(&wbiasrwlock)
 150 #define wrap_write_unsubscribe()                \
 151         wbias_write_unsubscribe(&wbiasrwlock)
 152 #else
 153 #define wrap_write_lock()       wbias_write_lock_atomic(&wbiasrwlock)
 154 #define wrap_write_unlock()     wbias_write_unlock_atomic(&wbiasrwlock)
 155 #define wrap_write_trylock_else_subscribe()     \
 156         wbias_write_trylock_atomic_else_subscribe(&wbiasrwlock)
 157 #define wrap_write_trylock_subscribed()         \
 158         wbias_write_trylock_atomic_subscribed(&wbiasrwlock)
 159 #define wrap_write_unsubscribe()                \
 160         wbias_write_unsubscribe_atomic(&wbiasrwlock)
 161 #endif
 162 #endif
 163
 164 #endif
 165
 166 static cycles_t cycles_calibration_min,
 167         cycles_calibration_avg,
 168         cycles_calibration_max;
 169
 170 static inline cycles_t calibrate_cycles(cycles_t cycles)
 171 {
 172         return cycles - cycles_calibration_avg;
 173 }
 174
 175 struct proc_dir_entry *pentry = NULL;
 176
 177 static int p_or_np_reader_thread(const char *typename,
 178                 void *data, int preemptable)
 179 {
 180         int i;
 181         int prev, cur;
 182         unsigned long iter = 0;
 183         cycles_t time1, time2, delay;
 184         cycles_t ldelaymax = 0, ldelaymin = ULLONG_MAX, ldelayavg = 0;
 185         cycles_t udelaymax = 0, udelaymin = ULLONG_MAX, udelayavg = 0;
 186
 187         printk("%s/%lu runnning\n", typename, (unsigned long)data);
 188         do {
 189                 iter++;
 190                 if (!preemptable)
 191                         preempt_disable();
 192                 rdtsc_barrier();
 193                 time1 = get_cycles();
 194                 rdtsc_barrier();
 195
 196                 if (!preemptable)
 197                         wrap_read_lock_inatomic();
 198                 else
 199                         wrap_read_lock();
 200
 201                 rdtsc_barrier();
 202                 time2 = get_cycles();
 203                 rdtsc_barrier();
 204                 delay = time2 - time1;
 205                 ldelaymax = max(ldelaymax, delay);
 206                 ldelaymin = min(ldelaymin, delay);
 207                 ldelayavg += delay;
 208                 prev = var[0];
 209                 for (i = 1; i < NR_VARS; i++) {
 210                         cur = var[i];
 211                         if (cur != prev)
 212                                 printk(KERN_ALERT
 213                                 "Unequal cur %d/prev %d at i %d, iter %lu "
 214                                 "in thread\n", cur, prev, i, iter);
 215                 }
 216
 217                 rdtsc_barrier();
 218                 time1 = get_cycles();
 219                 rdtsc_barrier();
 220
 221                 if (!preemptable)
 222                         wrap_read_unlock_inatomic();
 223                 else
 224                         wrap_read_unlock();
 225                 rdtsc_barrier();
 226                 time2 = get_cycles();
 227                 rdtsc_barrier();
 228                 delay = time2 - time1;
 229                 udelaymax = max(udelaymax, delay);
 230                 udelaymin = min(udelaymin, delay);
 231                 udelayavg += delay;
 232
 233                 if (!preemptable)
 234                         preempt_enable();
 235
 236                 if (THREAD_READER_DELAY)
 237                         msleep(THREAD_READER_DELAY);
 238         } while (!kthread_should_stop());
 239         if (!iter) {
 240                 printk("%s/%lu iterations : %lu", typename,
 241                         (unsigned long)data, iter);
 242         } else {
 243                 ldelayavg /= iter;
 244                 udelayavg /= iter;
 245                 printk("%s/%lu iterations : %lu, "
 246                         "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 247                         typename,
 248                         (unsigned long)data, iter,
 249                         calibrate_cycles(ldelaymin),
 250                         calibrate_cycles(ldelayavg),
 251                         calibrate_cycles(ldelaymax));
 252                 printk("%s/%lu iterations : %lu, "
 253                         "unlock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 254                         typename,
 255                         (unsigned long)data, iter,
 256                         calibrate_cycles(udelaymin),
 257                         calibrate_cycles(udelayavg),
 258                         calibrate_cycles(udelaymax));
 259         }
 260         return 0;
 261 }
 262
 263 static int preader_thread(void *data)
 264 {
 265         return p_or_np_reader_thread("preader_thread", data, 1);
 266 }
 267
 268 static int npreader_thread(void *data)
 269 {
 270         return p_or_np_reader_thread("npreader_thread", data, 0);
 271 }
 272
 273 static int trylock_reader_thread(void *data)
 274 {
 275         int i;
 276         int prev, cur;
 277         unsigned long iter = 0, success_iter = 0;
 278
 279         printk("trylock_reader_thread/%lu runnning\n", (unsigned long)data);
 280         do {
 281                 while (!wrap_read_trylock())
 282                         iter++;
 283                 success_iter++;
 284                 prev = var[0];
 285                 for (i = 1; i < NR_VARS; i++) {
 286                         cur = var[i];
 287                         if (cur != prev)
 288                                 printk(KERN_ALERT
 289                                 "Unequal cur %d/prev %d at i %d, iter %lu "
 290                                 "in thread\n", cur, prev, i, iter);
 291                 }
 292                 wrap_read_unlock();
 293                 if (THREAD_READER_DELAY)
 294                         msleep(THREAD_READER_DELAY);
 295         } while (!kthread_should_stop());
 296         printk("trylock_reader_thread/%lu iterations : %lu, "
 297                 "successful iterations : %lu\n",
 298                 (unsigned long)data, iter, success_iter);
 299         return 0;
 300 }
 301
 302 DEFINE_PER_CPU(cycles_t, int_ldelaymin);
 303 DEFINE_PER_CPU(cycles_t, int_ldelayavg);
 304 DEFINE_PER_CPU(cycles_t, int_ldelaymax);
 305 DEFINE_PER_CPU(cycles_t, int_udelaymin);
 306 DEFINE_PER_CPU(cycles_t, int_udelayavg);
 307 DEFINE_PER_CPU(cycles_t, int_udelaymax);
 308 DEFINE_PER_CPU(cycles_t, int_ipi_nr);
 309
 310 static void interrupt_reader_ipi(void *data)
 311 {
 312         int i;
 313         int prev, cur;
 314         cycles_t time1, time2;
 315         cycles_t *ldelaymax, *ldelaymin, *ldelayavg, *ipi_nr, delay;
 316         cycles_t *udelaymax, *udelaymin, *udelayavg;
 317
 318         /*
 319          * Skip the ipi caller, not in irq context.
 320          */
 321         if (!in_irq())
 322                 return;
 323
 324         ldelaymax = &per_cpu(int_ldelaymax, smp_processor_id());
 325         ldelaymin = &per_cpu(int_ldelaymin, smp_processor_id());
 326         ldelayavg = &per_cpu(int_ldelayavg, smp_processor_id());
 327         udelaymax = &per_cpu(int_udelaymax, smp_processor_id());
 328         udelaymin = &per_cpu(int_udelaymin, smp_processor_id());
 329         udelayavg = &per_cpu(int_udelayavg, smp_processor_id());
 330         ipi_nr = &per_cpu(int_ipi_nr, smp_processor_id());
 331
 332         rdtsc_barrier();
 333         time1 = get_cycles();
 334         rdtsc_barrier();
 335
 336         wrap_read_lock_irq();
 337
 338         rdtsc_barrier();
 339         time2 = get_cycles();
 340         rdtsc_barrier();
 341         delay = time2 - time1;
 342         *ldelaymax = max(*ldelaymax, delay);
 343         *ldelaymin = min(*ldelaymin, delay);
 344         *ldelayavg += delay;
 345         (*ipi_nr)++;
 346         prev = var[0];
 347         for (i = 1; i < NR_VARS; i++) {
 348                 cur = var[i];
 349                 if (cur != prev)
 350                         printk(KERN_ALERT
 351                         "Unequal cur %d/prev %d at i %d in interrupt\n",
 352                                 cur, prev, i);
 353         }
 354         rdtsc_barrier();
 355         time1 = get_cycles();
 356         rdtsc_barrier();
 357         wrap_read_unlock_irq();
 358         time2 = get_cycles();
 359         rdtsc_barrier();
 360         delay = time2 - time1;
 361         *udelaymax = max(*udelaymax, delay);
 362         *udelaymin = min(*udelaymin, delay);
 363         *udelayavg += delay;
 364 }
 365
 366 DEFINE_PER_CPU(unsigned long, trylock_int_iter);
 367 DEFINE_PER_CPU(unsigned long, trylock_int_success);
 368
 369 static void trylock_interrupt_reader_ipi(void *data)
 370 {
 371         int i;
 372         int prev, cur;
 373
 374         /*
 375          * Skip the ipi caller, not in irq context.
 376          */
 377         if (!in_irq())
 378                 return;
 379
 380         per_cpu(trylock_int_iter, smp_processor_id())++;
 381         while (!wrap_read_trylock_irq())
 382                 per_cpu(trylock_int_iter, smp_processor_id())++;
 383         per_cpu(trylock_int_success, smp_processor_id())++;
 384         prev = var[0];
 385         for (i = 1; i < NR_VARS; i++) {
 386                 cur = var[i];
 387                 if (cur != prev)
 388                         printk(KERN_ALERT
 389                         "Unequal cur %d/prev %d at i %d in interrupt\n",
 390                                 cur, prev, i);
 391         }
 392         wrap_read_unlock_irq();
 393 }
 394
 395
 396 static int interrupt_reader_thread(void *data)
 397 {
 398         unsigned long iter = 0;
 399         int i;
 400
 401         for_each_online_cpu(i) {
 402                 per_cpu(int_ldelaymax, i) = 0;
 403                 per_cpu(int_ldelaymin, i) = ULLONG_MAX;
 404                 per_cpu(int_ldelayavg, i) = 0;
 405                 per_cpu(int_udelaymax, i) = 0;
 406                 per_cpu(int_udelaymin, i) = ULLONG_MAX;
 407                 per_cpu(int_udelayavg, i) = 0;
 408                 per_cpu(int_ipi_nr, i) = 0;
 409         }
 410         do {
 411                 iter++;
 412                 on_each_cpu(interrupt_reader_ipi, NULL, 0);
 413                 if (INTERRUPT_READER_DELAY)
 414                         msleep(INTERRUPT_READER_DELAY);
 415         } while (!kthread_should_stop());
 416         printk("interrupt_reader_thread/%lu iterations : %lu\n",
 417                         (unsigned long)data, iter);
 418         for_each_online_cpu(i) {
 419                 if (!per_cpu(int_ipi_nr, i))
 420                         continue;
 421                 per_cpu(int_ldelayavg, i) /= per_cpu(int_ipi_nr, i);
 422                 per_cpu(int_udelayavg, i) /= per_cpu(int_ipi_nr, i);
 423                 printk("interrupt readers on CPU %i, "
 424                         "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 425                         i,
 426                         calibrate_cycles(per_cpu(int_ldelaymin, i)),
 427                         calibrate_cycles(per_cpu(int_ldelayavg, i)),
 428                         calibrate_cycles(per_cpu(int_ldelaymax, i)));
 429                 printk("interrupt readers on CPU %i, "
 430                         "unlock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 431                         i,
 432                         calibrate_cycles(per_cpu(int_udelaymin, i)),
 433                         calibrate_cycles(per_cpu(int_udelayavg, i)),
 434                         calibrate_cycles(per_cpu(int_udelaymax, i)));
 435         }
 436         return 0;
 437 }
 438
 439 static int trylock_interrupt_reader_thread(void *data)
 440 {
 441         unsigned long iter = 0;
 442         int i;
 443
 444         do {
 445                 iter++;
 446                 on_each_cpu(trylock_interrupt_reader_ipi, NULL, 0);
 447                 if (INTERRUPT_READER_DELAY)
 448                         msleep(INTERRUPT_READER_DELAY);
 449         } while (!kthread_should_stop());
 450         printk("trylock_interrupt_reader_thread/%lu iterations : %lu\n",
 451                         (unsigned long)data, iter);
 452         for_each_online_cpu(i) {
 453                 printk("trylock interrupt readers on CPU %i, "
 454                         "iterations %lu, "
 455                         "successful iterations : %lu\n",
 456                         i, per_cpu(trylock_int_iter, i),
 457                         per_cpu(trylock_int_success, i));
 458                 per_cpu(trylock_int_iter, i) = 0;
 459                 per_cpu(trylock_int_success, i) = 0;
 460         }
 461         return 0;
 462 }
 463
 464 static int writer_thread(void *data)
 465 {
 466         int i;
 467         int new;
 468         unsigned long iter = 0;
 469         cycles_t time1, time2, delay;
 470         cycles_t ldelaymax = 0, ldelaymin = ULLONG_MAX, ldelayavg = 0;
 471         cycles_t udelaymax = 0, udelaymin = ULLONG_MAX, udelayavg = 0;
 472
 473         printk("writer_thread/%lu runnning\n", (unsigned long)data);
 474         do {
 475                 iter++;
 476                 //preempt_disable();    /* for get_cycles accuracy */
 477                 rdtsc_barrier();
 478                 time1 = get_cycles();
 479                 rdtsc_barrier();
 480
 481                 wrap_write_lock();
 482
 483                 rdtsc_barrier();
 484                 time2 = get_cycles();
 485                 rdtsc_barrier();
 486                 delay = time2 - time1;
 487                 ldelaymax = max(ldelaymax, delay);
 488                 ldelaymin = min(ldelaymin, delay);
 489                 ldelayavg += delay;
 490                 new = (int)get_cycles();
 491                 for (i = 0; i < NR_VARS; i++) {
 492                         var[i] = new;
 493                 }
 494
 495                 rdtsc_barrier();
 496                 time1 = get_cycles();
 497                 rdtsc_barrier();
 498
 499                 wrap_write_unlock();
 500
 501                 rdtsc_barrier();
 502                 time2 = get_cycles();
 503                 rdtsc_barrier();
 504                 delay = time2 - time1;
 505                 udelaymax = max(udelaymax, delay);
 506                 udelaymin = min(udelaymin, delay);
 507                 udelayavg += delay;
 508
 509                 //preempt_enable();     /* for get_cycles accuracy */
 510                 if (WRITER_DELAY > 0)
 511                         udelay(WRITER_DELAY);
 512         } while (!kthread_should_stop());
 513         ldelayavg /= iter;
 514         udelayavg /= iter;
 515         printk("writer_thread/%lu iterations : %lu, "
 516                 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 517                 (unsigned long)data, iter,
 518                 calibrate_cycles(ldelaymin),
 519                 calibrate_cycles(ldelayavg),
 520                 calibrate_cycles(ldelaymax));
 521         printk("writer_thread/%lu iterations : %lu, "
 522                 "unlock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 523                 (unsigned long)data, iter,
 524                 calibrate_cycles(udelaymin),
 525                 calibrate_cycles(udelayavg),
 526                 calibrate_cycles(udelaymax));
 527         return 0;
 528 }
 529
 530 #if (TEST_STD_RWLOCK)
 531 static int trylock_writer_thread(void *data)
 532 {
 533         int i;
 534         int new;
 535         unsigned long iter = 0, success = 0, fail = 0;
 536
 537         printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data);
 538         do {
 539 #if (TEST_INTERRUPTS)
 540                 /* std write trylock cannot disable interrupts. */
 541                 local_irq_disable();
 542 #endif
 543
 544 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
 545                 for (;;) {
 546                         iter++;
 547                         if (write_trylock(&std_rw_lock))
 548                                 goto locked;
 549                 }
 550 #else
 551                 for (i = 0; i < TRYLOCK_WRITERS_FAIL_ITER; i++) {
 552                         iter++;
 553                         if (write_trylock(&std_rw_lock))
 554                                 goto locked;
 555                 }
 556 #endif
 557                 fail++;
 558 #if (TEST_INTERRUPTS)
 559                 local_irq_enable();
 560 #endif
 561                 goto loop;
 562 locked:
 563                 success++;
 564                 new = (int)get_cycles();
 565                 for (i = 0; i < NR_VARS; i++) {
 566                         var[i] = new;
 567                 }
 568 #if (TEST_INTERRUPTS)
 569                 write_unlock_irq(&std_rw_lock);
 570 #else
 571                 write_unlock(&std_rw_lock);
 572 #endif
 573 loop:
 574                 if (TRYLOCK_WRITER_DELAY > 0)
 575                         udelay(TRYLOCK_WRITER_DELAY);
 576         } while (!kthread_should_stop());
 577         printk("trylock_writer_thread/%lu iterations : "
 578                 "[try,success,fail after %d try], "
 579                 "%lu,%lu,%lu\n",
 580                 (unsigned long)data, TRYLOCK_WRITERS_FAIL_ITER,
 581                 iter, success, fail);
 582         return 0;
 583 }
 584
 585 #else /* !TEST_STD_RWLOCK */
 586
 587 static int trylock_writer_thread(void *data)
 588 {
 589         int i;
 590         int new;
 591         unsigned long iter = 0, success = 0, fail = 0;
 592
 593         printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data);
 594         do {
 595                 iter++;
 596                 if (wrap_write_trylock_else_subscribe())
 597                         goto locked;
 598
 599 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
 600                 for (;;) {
 601                         iter++;
 602                         if (wrap_write_trylock_subscribed())
 603                                 goto locked;
 604                 }
 605 #else
 606                 for (i = 0; i < TRYLOCK_WRITERS_FAIL_ITER - 1; i++) {
 607                         iter++;
 608                         if (wrap_write_trylock_subscribed())
 609                                 goto locked;
 610                 }
 611 #endif
 612                 fail++;
 613                 wrap_write_unsubscribe();
 614                 goto loop;
 615 locked:
 616                 success++;
 617                 new = (int)get_cycles();
 618                 for (i = 0; i < NR_VARS; i++) {
 619                         var[i] = new;
 620                 }
 621                 wrap_write_unlock();
 622 loop:
 623                 if (TRYLOCK_WRITER_DELAY > 0)
 624                         udelay(TRYLOCK_WRITER_DELAY);
 625         } while (!kthread_should_stop());
 626         printk("trylock_writer_thread/%lu iterations : "
 627                 "[try,success,fail after %d try], "
 628                 "%lu,%lu,%lu\n",
 629                 (unsigned long)data, TRYLOCK_WRITERS_FAIL_ITER,
 630                 iter, success, fail);
 631         return 0;
 632 }
 633
 634 #endif  /* TEST_STD_RWLOCK */
 635
 636 static void wbias_rwlock_create(void)
 637 {
 638         unsigned long i;
 639
 640         for (i = 0; i < NR_PREADERS; i++) {
 641                 printk("starting preemptable reader thread %lu\n", i);
 642                 preader_threads[i] = kthread_run(preader_thread, (void *)i,
 643                         "wbiasrwlock_preader");
 644                 BUG_ON(!preader_threads[i]);
 645         }
 646
 647         for (i = 0; i < NR_NPREADERS; i++) {
 648                 printk("starting non-preemptable reader thread %lu\n", i);
 649                 npreader_threads[i] = kthread_run(npreader_thread, (void *)i,
 650                         "wbiasrwlock_npreader");
 651                 BUG_ON(!npreader_threads[i]);
 652         }
 653
 654         for (i = 0; i < NR_TRYLOCK_READERS; i++) {
 655                 printk("starting trylock reader thread %lu\n", i);
 656                 trylock_reader_threads[i] = kthread_run(trylock_reader_thread,
 657                         (void *)i, "wbiasrwlock_trylock_reader");
 658                 BUG_ON(!trylock_reader_threads[i]);
 659         }
 660         for (i = 0; i < NR_INTERRUPT_READERS; i++) {
 661                 printk("starting interrupt reader %lu\n", i);
 662                 interrupt_reader[i] = kthread_run(interrupt_reader_thread,
 663                         (void *)i,
 664                         "wbiasrwlock_interrupt_reader");
 665         }
 666         for (i = 0; i < NR_TRYLOCK_INTERRUPT_READERS; i++) {
 667                 printk("starting trylock interrupt reader %lu\n", i);
 668                 trylock_interrupt_reader[i] =
 669                         kthread_run(trylock_interrupt_reader_thread,
 670                         (void *)i, "wbiasrwlock_trylock_interrupt_reader");
 671         }
 672         for (i = 0; i < NR_WRITERS; i++) {
 673                 printk("starting writer thread %lu\n", i);
 674                 writer_threads[i] = kthread_run(writer_thread, (void *)i,
 675                         "wbiasrwlock_writer");
 676                 BUG_ON(!writer_threads[i]);
 677         }
 678         for (i = 0; i < NR_TRYLOCK_WRITERS; i++) {
 679                 printk("starting trylock writer thread %lu\n", i);
 680                 trylock_writer_threads[i] = kthread_run(trylock_writer_thread,
 681                         (void *)i, "wbiasrwlock_trylock_writer");
 682                 BUG_ON(!trylock_writer_threads[i]);
 683         }
 684 }
 685
 686 static void wbias_rwlock_stop(void)
 687 {
 688         unsigned long i;
 689
 690         for (i = 0; i < NR_WRITERS; i++)
 691                 kthread_stop(writer_threads[i]);
 692         for (i = 0; i < NR_TRYLOCK_WRITERS; i++)
 693                 kthread_stop(trylock_writer_threads[i]);
 694         for (i = 0; i < NR_NPREADERS; i++)
 695                 kthread_stop(npreader_threads[i]);
 696         for (i = 0; i < NR_PREADERS; i++)
 697                 kthread_stop(preader_threads[i]);
 698         for (i = 0; i < NR_TRYLOCK_READERS; i++)
 699                 kthread_stop(trylock_reader_threads[i]);
 700         for (i = 0; i < NR_INTERRUPT_READERS; i++)
 701                 kthread_stop(interrupt_reader[i]);
 702         for (i = 0; i < NR_TRYLOCK_INTERRUPT_READERS; i++)
 703                 kthread_stop(trylock_interrupt_reader[i]);
 704 }
 705
 706
 707 static void perform_test(const char *name, void (*callback)(void))
 708 {
 709         printk("%s\n", name);
 710         callback();
 711 }
 712
 713 static int my_open(struct inode *inode, struct file *file)
 714 {
 715         unsigned long i;
 716         cycles_t time1, time2, delay;
 717
 718         printk("** get_cycles calibration **\n");
 719         cycles_calibration_min = ULLONG_MAX;
 720         cycles_calibration_avg = 0;
 721         cycles_calibration_max = 0;
 722
 723         local_irq_disable();
 724         for (i = 0; i < 10; i++) {
 725                 rdtsc_barrier();
 726                 time1 = get_cycles();
 727                 rdtsc_barrier();
 728                 rdtsc_barrier();
 729                 time2 = get_cycles();
 730                 rdtsc_barrier();
 731                 delay = time2 - time1;
 732                 cycles_calibration_min = min(cycles_calibration_min, delay);
 733                 cycles_calibration_avg += delay;
 734                 cycles_calibration_max = max(cycles_calibration_max, delay);
 735         }
 736         cycles_calibration_avg /= 10;
 737         local_irq_enable();
 738
 739         printk("get_cycles takes [min,avg,max] %llu,%llu,%llu cycles, "
 740                 "results calibrated on avg\n",
 741                 cycles_calibration_min,
 742                 cycles_calibration_avg,
 743                 cycles_calibration_max);
 744         printk("\n");
 745
 746         printk("** Single writer test, no contention **\n");
 747         wbias_rwlock_profile_latency_reset();
 748         writer_threads[0] = kthread_run(writer_thread, (void *)0,
 749                 "wbiasrwlock_writer");
 750         BUG_ON(!writer_threads[0]);
 751         ssleep(SINGLE_WRITER_TEST_DURATION);
 752         kthread_stop(writer_threads[0]);
 753         printk("\n");
 754
 755         wbias_rwlock_profile_latency_print();
 756
 757         printk("** Single trylock writer test, no contention **\n");
 758         wbias_rwlock_profile_latency_reset();
 759         trylock_writer_threads[0] = kthread_run(trylock_writer_thread,
 760                 (void *)0,
 761                 "trylock_wbiasrwlock_writer");
 762         BUG_ON(!trylock_writer_threads[0]);
 763         ssleep(SINGLE_WRITER_TEST_DURATION);
 764         kthread_stop(trylock_writer_threads[0]);
 765         printk("\n");
 766
 767         wbias_rwlock_profile_latency_print();
 768
 769         printk("** Single preemptable reader test, no contention **\n");
 770         wbias_rwlock_profile_latency_reset();
 771         preader_threads[0] = kthread_run(preader_thread, (void *)0,
 772                 "wbiasrwlock_preader");
 773         BUG_ON(!preader_threads[0]);
 774         ssleep(SINGLE_READER_TEST_DURATION);
 775         kthread_stop(preader_threads[0]);
 776         printk("\n");
 777
 778         wbias_rwlock_profile_latency_print();
 779
 780 #if (TEST_PREEMPT)
 781         printk("** Single non-preemptable reader test, no contention **\n");
 782         wbias_rwlock_profile_latency_reset();
 783         npreader_threads[0] = kthread_run(npreader_thread, (void *)0,
 784                 "wbiasrwlock_npreader");
 785         BUG_ON(!npreader_threads[0]);
 786         ssleep(SINGLE_READER_TEST_DURATION);
 787         kthread_stop(npreader_threads[0]);
 788         printk("\n");
 789
 790         wbias_rwlock_profile_latency_print();
 791 #endif
 792
 793         printk("** Multiple p/non-p readers test, no contention **\n");
 794         wbias_rwlock_profile_latency_reset();
 795         for (i = 0; i < NR_PREADERS; i++) {
 796                 printk("starting preader thread %lu\n", i);
 797                 preader_threads[i] = kthread_run(preader_thread, (void *)i,
 798                         "wbiasrwlock_preader");
 799                 BUG_ON(!preader_threads[i]);
 800         }
 801         for (i = 0; i < NR_NPREADERS; i++) {
 802                 printk("starting npreader thread %lu\n", i);
 803                 npreader_threads[i] = kthread_run(npreader_thread, (void *)i,
 804                         "wbiasrwlock_npreader");
 805                 BUG_ON(!npreader_threads[i]);
 806         }
 807         ssleep(SINGLE_READER_TEST_DURATION);
 808         for (i = 0; i < NR_NPREADERS; i++)
 809                 kthread_stop(npreader_threads[i]);
 810         for (i = 0; i < NR_PREADERS; i++)
 811                 kthread_stop(preader_threads[i]);
 812         printk("\n");
 813
 814         wbias_rwlock_profile_latency_print();
 815
 816         printk("** High contention test **\n");
 817         wbias_rwlock_profile_latency_reset();
 818         perform_test("wbias-rwlock-create", wbias_rwlock_create);
 819         ssleep(TEST_DURATION);
 820         perform_test("wbias-rwlock-stop", wbias_rwlock_stop);
 821         printk("\n");
 822         wbias_rwlock_profile_latency_print();
 823
 824         return -EPERM;
 825 }
 826
 827
 828 static struct file_operations my_operations = {
 829         .open = my_open,
 830 };
 831
 832 int init_module(void)
 833 {
 834         pentry = create_proc_entry("testwbiasrwlock", 0444, NULL);
 835         if (pentry)
 836                 pentry->proc_fops = &my_operations;
 837
 838         printk("PTHREAD_ROFFSET  : %016lX\n", PTHREAD_ROFFSET);
 839         printk("PTHREAD_RMASK    : %016lX\n", PTHREAD_RMASK);
 840         printk("NPTHREAD_ROFFSET : %016lX\n", NPTHREAD_ROFFSET);
 841         printk("NPTHREAD_RMASK   : %016lX\n", NPTHREAD_RMASK);
 842         printk("SOFTIRQ_ROFFSET  : %016lX\n", SOFTIRQ_ROFFSET);
 843         printk("SOFTIRQ_RMASK    : %016lX\n", SOFTIRQ_RMASK);
 844         printk("HARDIRQ_ROFFSET  : %016lX\n", HARDIRQ_ROFFSET);
 845         printk("HARDIRQ_RMASK    : %016lX\n", HARDIRQ_RMASK);
 846         printk("PTHREAD_WOFFSET  : %016lX\n", PTHREAD_WOFFSET);
 847         printk("PTHREAD_WMASK    : %016lX\n", PTHREAD_WMASK);
 848         printk("NPTHREAD_WOFFSET : %016lX\n", NPTHREAD_WOFFSET);
 849         printk("NPTHREAD_WMASK   : %016lX\n", NPTHREAD_WMASK);
 850         printk("WRITER_MUTEX     : %016lX\n", WRITER_MUTEX);
 851         printk("SOFTIRQ_WMASK    : %016lX\n", SOFTIRQ_WMASK);
 852         printk("HARDIRQ_WMASK    : %016lX\n", HARDIRQ_WMASK);
 853         printk("WQ_MUTEX         : %016lX\n", WQ_MUTEX);
 854         printk("WQ_ACTIVE        : %016lX\n", WQ_ACTIVE);
 855
 856         return 0;
 857 }
 858
 859 void cleanup_module(void)
 860 {
 861         remove_proc_entry("testwbiasrwlock", NULL);
 862 }
 863
 864 MODULE_LICENSE("GPL");
 865 MODULE_AUTHOR("Mathieu Desnoyers");
 866 MODULE_DESCRIPTION("wbias rwlock test");