trunk/tests/kernel/test-wbias-rwlock.c

   1 /* test-wbias-rwlock.c
   2  *
   3  */
   4
   5 #include <linux/module.h>
   6 #include <linux/proc_fs.h>
   7 #include <linux/sched.h>
   8 #include <linux/timex.h>
   9 #include <linux/kthread.h>
  10 #include <linux/delay.h>
  11 #include <linux/hardirq.h>
  12 #include <linux/module.h>
  13 #include <linux/percpu.h>
  14 #include <linux/spinlock.h>
  15 #include <asm/ptrace.h>
  16 #include <linux/wbias-rwlock.h>
  17
  18 /* Test with no contention duration, in seconds */
  19 #define SINGLE_WRITER_TEST_DURATION 10
  20 #define SINGLE_READER_TEST_DURATION 10
  21 #define MULTIPLE_READERS_TEST_DURATION 10
  22
  23 /* Test duration, in seconds */
  24 #define TEST_DURATION 60
  25
  26 #define NR_VARS 100
  27 #define NR_WRITERS 2
  28 #define NR_TRYLOCK_WRITERS 2
  29 #define NR_NPREADERS 4
  30 #define NR_TRYLOCK_READERS 2
  31
  32 /*
  33  * 1 : test standard rwlock
  34  * 0 : test wbiasrwlock
  35  */
  36 #define TEST_STD_RWLOCK 0
  37
  38 /*
  39  * 1 : test with thread and interrupt readers.
  40  * 0 : test only with thread readers.
  41  */
  42 #define TEST_INTERRUPTS 1
  43
  44 #if (TEST_INTERRUPTS)
  45 #define NR_INTERRUPT_READERS 1
  46 #define NR_TRYLOCK_INTERRUPT_READERS 1
  47 #else
  48 #define NR_INTERRUPT_READERS 0
  49 #define NR_TRYLOCK_INTERRUPT_READERS 0
  50 #endif
  51
  52 /*
  53  * 1 : test with thread preemption readers.
  54  * 0 : test only with non-preemptable thread readers.
  55  */
  56 #define TEST_PREEMPT 1
  57
  58 #if (TEST_PREEMPT)
  59 #define NR_PREADERS 2
  60 #else
  61 #define NR_PREADERS 0
  62 #endif
  63
  64
  65 /*
  66  * Writer iteration delay, in us. 0 for busy loop. Caution : writers can
  67  * starve readers.
  68  */
  69 #define WRITER_DELAY 100
  70 #define TRYLOCK_WRITER_DELAY 1000
  71
  72 /*
  73  * Number of iterations after which a trylock writer fails.
  74  * -1 for infinite loop.
  75  */
  76 #define TRYLOCK_WRITERS_FAIL_ITER 100
  77
  78 /* Thread and interrupt reader delay, in ms */
  79 #define THREAD_READER_DELAY 0   /* busy loop */
  80 #define INTERRUPT_READER_DELAY 100
  81
  82 #ifdef CONFIG_PREEMPT
  83 #define yield_in_non_preempt()
  84 #else
  85 #define yield_in_non_preempt()  yield()
  86 #endif
  87
  88 static int var[NR_VARS];
  89 static struct task_struct *preader_threads[NR_PREADERS];
  90 static struct task_struct *npreader_threads[NR_NPREADERS];
  91 static struct task_struct *trylock_reader_threads[NR_TRYLOCK_READERS];
  92 static struct task_struct *writer_threads[NR_WRITERS];
  93 static struct task_struct *trylock_writer_threads[NR_TRYLOCK_WRITERS];
  94 static struct task_struct *interrupt_reader[NR_INTERRUPT_READERS];
  95 static struct task_struct *trylock_interrupt_reader[NR_TRYLOCK_INTERRUPT_READERS];
  96
  97 #if (TEST_STD_RWLOCK)
  98
  99 static DEFINE_RWLOCK(std_rw_lock);
 100
 101 #define wrap_read_lock()        read_lock(&std_rw_lock)
 102 #define wrap_read_trylock()     read_trylock(&std_rw_lock)
 103 #define wrap_read_unlock()      read_unlock(&std_rw_lock)
 104
 105 #define wrap_read_lock_inatomic()       read_lock(&std_rw_lock)
 106 #define wrap_read_trylock_inatomic()    read_trylock(&std_rw_lock)
 107
 108 #define wrap_read_lock_irq()    read_lock(&std_rw_lock)
 109 #define wrap_read_trylock_irq() read_trylock(&std_rw_lock)
 110
 111 #if (TEST_INTERRUPTS)
 112 #define wrap_write_lock()       write_lock_irq(&std_rw_lock)
 113 #define wrap_write_unlock()     write_unlock_irq(&std_rw_lock)
 114 #else
 115 #define wrap_write_lock()       write_lock(&std_rw_lock)
 116 #define wrap_write_unlock()     write_unlock(&std_rw_lock)
 117 #endif
 118
 119 #define wrap_write_trylock()    write_trylock(&std_rw_lock)
 120
 121 #else
 122
 123 #if (TEST_INTERRUPTS)
 124 #if (TEST_PREEMPT)
 125 #define WBIASRWLOCKMASK (WB_WPTHREAD | WB_RIRQ | WB_RNPTHREAD | WB_RPTHREAD)
 126 #else
 127 #define WBIASRWLOCKMASK (WB_WNPTHREAD | WB_RIRQ | WB_RNPTHREAD)
 128 #endif
 129 #else
 130 #if (TEST_PREEMPT)
 131 #define WBIASRWLOCKMASK (WB_WPTHREAD | WB_RNPTHREAD | WB_RPTHREAD)
 132 #else
 133 #define WBIASRWLOCKMASK (WB_WNPTHREAD | WB_RNPTHREAD)
 134 #endif
 135 #endif
 136 static DEFINE_WBIAS_RWLOCK(wbiasrwlock, WBIASRWLOCKMASK);
 137 CHECK_WBIAS_RWLOCK_MAP(wbiasrwlock, WBIASRWLOCKMASK);
 138
 139
 140 #if (TEST_PREEMPT)
 141 #define wrap_read_lock()        wbias_read_lock(&wbiasrwlock)
 142 #define wrap_read_trylock()     wbias_read_trylock(&wbiasrwlock)
 143 #else
 144 #define wrap_read_lock()        wbias_read_lock_inatomic(&wbiasrwlock)
 145 #define wrap_read_trylock()     wbias_read_trylock_inatomic(&wbiasrwlock)
 146 #endif
 147 #define wrap_read_unlock()      wbias_read_unlock(&wbiasrwlock)
 148
 149 #define wrap_read_lock_inatomic()       wbias_read_lock_inatomic(&wbiasrwlock)
 150 #define wrap_read_trylock_inatomic()            \
 151                 wbias_read_trylock_inatomic(&wbiasrwlock)
 152
 153 #define wrap_read_lock_irq()    wbias_read_lock_irq(&wbiasrwlock)
 154 #define wrap_read_trylock_irq() wbias_read_trylock_irq(&wbiasrwlock)
 155
 156 #define wrap_write_lock()                       \
 157         wbias_write_lock(&wbiasrwlock, WBIASRWLOCKMASK)
 158 #define wrap_write_unlock()                     \
 159         wbias_write_unlock(&wbiasrwlock, WBIASRWLOCKMASK)
 160 #define wrap_write_trylock()                    \
 161         wbias_write_trylock(&wbiasrwlock, WBIASRWLOCKMASK)
 162
 163 #endif
 164
 165 static cycles_t cycles_calibration_min,
 166         cycles_calibration_avg,
 167         cycles_calibration_max;
 168
 169 static inline cycles_t calibrate_cycles(cycles_t cycles)
 170 {
 171         return cycles - cycles_calibration_avg;
 172 }
 173
 174 struct proc_dir_entry *pentry = NULL;
 175
 176 static int p_or_np_reader_thread(const char *typename,
 177                 void *data, int preemptable)
 178 {
 179         int i;
 180         int prev, cur;
 181         unsigned long iter = 0;
 182         cycles_t time1, time2, delay;
 183         cycles_t ldelaymax = 0, ldelaymin = ULLONG_MAX, ldelayavg = 0;
 184         cycles_t udelaymax = 0, udelaymin = ULLONG_MAX, udelayavg = 0;
 185
 186         printk("%s/%lu runnning\n", typename, (unsigned long)data);
 187         do {
 188                 iter++;
 189                 if (!preemptable)
 190                         preempt_disable();
 191                 rdtsc_barrier();
 192                 time1 = get_cycles();
 193                 rdtsc_barrier();
 194
 195                 if (!preemptable)
 196                         wrap_read_lock_inatomic();
 197                 else
 198                         wrap_read_lock();
 199
 200                 rdtsc_barrier();
 201                 time2 = get_cycles();
 202                 rdtsc_barrier();
 203                 delay = time2 - time1;
 204                 ldelaymax = max(ldelaymax, delay);
 205                 ldelaymin = min(ldelaymin, delay);
 206                 ldelayavg += delay;
 207                 prev = var[0];
 208                 for (i = 1; i < NR_VARS; i++) {
 209                         cur = var[i];
 210                         if (cur != prev) {
 211                                 printk(KERN_ALERT
 212                                 "Unequal cur %d/prev %d at i %d, iter %lu "
 213                                 "in reader thread\n",
 214                                 cur, prev, i, iter);
 215                         }
 216                 }
 217
 218                 rdtsc_barrier();
 219                 time1 = get_cycles();
 220                 rdtsc_barrier();
 221
 222                 wrap_read_unlock();
 223
 224                 rdtsc_barrier();
 225                 time2 = get_cycles();
 226                 rdtsc_barrier();
 227                 delay = time2 - time1;
 228                 udelaymax = max(udelaymax, delay);
 229                 udelaymin = min(udelaymin, delay);
 230                 udelayavg += delay;
 231
 232                 if (!preemptable)
 233                         preempt_enable();
 234
 235                 if (THREAD_READER_DELAY)
 236                         msleep(THREAD_READER_DELAY);
 237                 yield_in_non_preempt();
 238         } while (!kthread_should_stop());
 239         if (!iter) {
 240                 printk("%s/%lu iterations : %lu", typename,
 241                         (unsigned long)data, iter);
 242         } else {
 243                 ldelayavg /= iter;
 244                 udelayavg /= iter;
 245                 printk("%s/%lu iterations : %lu, "
 246                         "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 247                         typename,
 248                         (unsigned long)data, iter,
 249                         calibrate_cycles(ldelaymin),
 250                         calibrate_cycles(ldelayavg),
 251                         calibrate_cycles(ldelaymax));
 252                 printk("%s/%lu iterations : %lu, "
 253                         "unlock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 254                         typename,
 255                         (unsigned long)data, iter,
 256                         calibrate_cycles(udelaymin),
 257                         calibrate_cycles(udelayavg),
 258                         calibrate_cycles(udelaymax));
 259         }
 260         return 0;
 261 }
 262
 263 static int preader_thread(void *data)
 264 {
 265         return p_or_np_reader_thread("preader_thread", data, 1);
 266 }
 267
 268 static int npreader_thread(void *data)
 269 {
 270         return p_or_np_reader_thread("npreader_thread", data, 0);
 271 }
 272
 273 static int trylock_reader_thread(void *data)
 274 {
 275         int i;
 276         int prev, cur;
 277         unsigned long iter = 0, success_iter = 0;
 278
 279         printk("trylock_reader_thread/%lu runnning\n", (unsigned long)data);
 280         do {
 281 #if (!TEST_PREEMPT)
 282                 preempt_disable();
 283 #endif
 284                 while (!wrap_read_trylock()) {
 285                         cpu_relax();
 286                         iter++;
 287                 }
 288                 success_iter++;
 289                 prev = var[0];
 290                 for (i = 1; i < NR_VARS; i++) {
 291                         cur = var[i];
 292                         if (cur != prev) {
 293                                 printk(KERN_ALERT
 294                                 "Unequal cur %d/prev %d at i %d, iter %lu "
 295                                 "in trylock reader thread\n",
 296                                 cur, prev, i, iter);
 297                         }
 298                 }
 299                 wrap_read_unlock();
 300 #if (!TEST_PREEMPT)
 301                 preempt_enable();
 302 #endif
 303                 if (THREAD_READER_DELAY)
 304                         msleep(THREAD_READER_DELAY);
 305                 yield_in_non_preempt();
 306         } while (!kthread_should_stop());
 307         printk("trylock_reader_thread/%lu iterations : %lu, "
 308                 "successful iterations : %lu\n",
 309                 (unsigned long)data, iter, success_iter);
 310         return 0;
 311 }
 312
 313 DEFINE_PER_CPU(cycles_t, int_ldelaymin);
 314 DEFINE_PER_CPU(cycles_t, int_ldelayavg);
 315 DEFINE_PER_CPU(cycles_t, int_ldelaymax);
 316 DEFINE_PER_CPU(cycles_t, int_udelaymin);
 317 DEFINE_PER_CPU(cycles_t, int_udelayavg);
 318 DEFINE_PER_CPU(cycles_t, int_udelaymax);
 319 DEFINE_PER_CPU(cycles_t, int_ipi_nr);
 320
 321 static void interrupt_reader_ipi(void *data)
 322 {
 323         int i;
 324         int prev, cur;
 325         cycles_t time1, time2;
 326         cycles_t *ldelaymax, *ldelaymin, *ldelayavg, *ipi_nr, delay;
 327         cycles_t *udelaymax, *udelaymin, *udelayavg;
 328
 329         /*
 330          * Skip the ipi caller, not in irq context.
 331          */
 332         if (!in_irq())
 333                 return;
 334
 335         ldelaymax = &per_cpu(int_ldelaymax, smp_processor_id());
 336         ldelaymin = &per_cpu(int_ldelaymin, smp_processor_id());
 337         ldelayavg = &per_cpu(int_ldelayavg, smp_processor_id());
 338         udelaymax = &per_cpu(int_udelaymax, smp_processor_id());
 339         udelaymin = &per_cpu(int_udelaymin, smp_processor_id());
 340         udelayavg = &per_cpu(int_udelayavg, smp_processor_id());
 341         ipi_nr = &per_cpu(int_ipi_nr, smp_processor_id());
 342
 343         rdtsc_barrier();
 344         time1 = get_cycles();
 345         rdtsc_barrier();
 346
 347         wrap_read_lock_irq();
 348
 349         rdtsc_barrier();
 350         time2 = get_cycles();
 351         rdtsc_barrier();
 352         delay = time2 - time1;
 353         *ldelaymax = max(*ldelaymax, delay);
 354         *ldelaymin = min(*ldelaymin, delay);
 355         *ldelayavg += delay;
 356         (*ipi_nr)++;
 357         prev = var[0];
 358         for (i = 1; i < NR_VARS; i++) {
 359                 cur = var[i];
 360                 if (cur != prev)
 361                         printk(KERN_ALERT
 362                         "Unequal cur %d/prev %d at i %d in interrupt\n",
 363                                 cur, prev, i);
 364         }
 365         rdtsc_barrier();
 366         time1 = get_cycles();
 367         rdtsc_barrier();
 368         wrap_read_unlock();
 369         time2 = get_cycles();
 370         rdtsc_barrier();
 371         delay = time2 - time1;
 372         *udelaymax = max(*udelaymax, delay);
 373         *udelaymin = min(*udelaymin, delay);
 374         *udelayavg += delay;
 375 }
 376
 377 DEFINE_PER_CPU(unsigned long, trylock_int_iter);
 378 DEFINE_PER_CPU(unsigned long, trylock_int_success);
 379
 380 static void trylock_interrupt_reader_ipi(void *data)
 381 {
 382         int i;
 383         int prev, cur;
 384
 385         /*
 386          * Skip the ipi caller, not in irq context.
 387          */
 388         if (!in_irq())
 389                 return;
 390
 391         per_cpu(trylock_int_iter, smp_processor_id())++;
 392         while (!wrap_read_trylock_irq())
 393                 per_cpu(trylock_int_iter, smp_processor_id())++;
 394         per_cpu(trylock_int_success, smp_processor_id())++;
 395         prev = var[0];
 396         for (i = 1; i < NR_VARS; i++) {
 397                 cur = var[i];
 398                 if (cur != prev)
 399                         printk(KERN_ALERT
 400                         "Unequal cur %d/prev %d at i %d in interrupt\n",
 401                                 cur, prev, i);
 402         }
 403         wrap_read_unlock();
 404 }
 405
 406
 407 static int interrupt_reader_thread(void *data)
 408 {
 409         unsigned long iter = 0;
 410         int i;
 411
 412         for_each_online_cpu(i) {
 413                 per_cpu(int_ldelaymax, i) = 0;
 414                 per_cpu(int_ldelaymin, i) = ULLONG_MAX;
 415                 per_cpu(int_ldelayavg, i) = 0;
 416                 per_cpu(int_udelaymax, i) = 0;
 417                 per_cpu(int_udelaymin, i) = ULLONG_MAX;
 418                 per_cpu(int_udelayavg, i) = 0;
 419                 per_cpu(int_ipi_nr, i) = 0;
 420         }
 421         do {
 422                 iter++;
 423                 on_each_cpu(interrupt_reader_ipi, NULL, 0);
 424                 if (INTERRUPT_READER_DELAY)
 425                         msleep(INTERRUPT_READER_DELAY);
 426                 yield_in_non_preempt();
 427         } while (!kthread_should_stop());
 428         printk("interrupt_reader_thread/%lu iterations : %lu\n",
 429                         (unsigned long)data, iter);
 430         for_each_online_cpu(i) {
 431                 if (!per_cpu(int_ipi_nr, i))
 432                         continue;
 433                 per_cpu(int_ldelayavg, i) /= per_cpu(int_ipi_nr, i);
 434                 per_cpu(int_udelayavg, i) /= per_cpu(int_ipi_nr, i);
 435                 printk("interrupt readers on CPU %i, "
 436                         "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 437                         i,
 438                         calibrate_cycles(per_cpu(int_ldelaymin, i)),
 439                         calibrate_cycles(per_cpu(int_ldelayavg, i)),
 440                         calibrate_cycles(per_cpu(int_ldelaymax, i)));
 441                 printk("interrupt readers on CPU %i, "
 442                         "unlock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 443                         i,
 444                         calibrate_cycles(per_cpu(int_udelaymin, i)),
 445                         calibrate_cycles(per_cpu(int_udelayavg, i)),
 446                         calibrate_cycles(per_cpu(int_udelaymax, i)));
 447         }
 448         return 0;
 449 }
 450
 451 static int trylock_interrupt_reader_thread(void *data)
 452 {
 453         unsigned long iter = 0;
 454         int i;
 455
 456         do {
 457                 iter++;
 458                 on_each_cpu(trylock_interrupt_reader_ipi, NULL, 0);
 459                 if (INTERRUPT_READER_DELAY)
 460                         msleep(INTERRUPT_READER_DELAY);
 461                 yield_in_non_preempt();
 462         } while (!kthread_should_stop());
 463         printk("trylock_interrupt_reader_thread/%lu iterations : %lu\n",
 464                         (unsigned long)data, iter);
 465         for_each_online_cpu(i) {
 466                 printk("trylock interrupt readers on CPU %i, "
 467                         "iterations %lu, "
 468                         "successful iterations : %lu\n",
 469                         i, per_cpu(trylock_int_iter, i),
 470                         per_cpu(trylock_int_success, i));
 471                 per_cpu(trylock_int_iter, i) = 0;
 472                 per_cpu(trylock_int_success, i) = 0;
 473         }
 474         return 0;
 475 }
 476
 477 static int writer_thread(void *data)
 478 {
 479         int i;
 480         int new, prev, cur;
 481         unsigned long iter = 0;
 482         cycles_t time1, time2, delay;
 483         cycles_t ldelaymax = 0, ldelaymin = ULLONG_MAX, ldelayavg = 0;
 484         cycles_t udelaymax = 0, udelaymin = ULLONG_MAX, udelayavg = 0;
 485
 486         printk("writer_thread/%lu runnning\n", (unsigned long)data);
 487         do {
 488                 iter++;
 489 #if (!TEST_PREEMPT)
 490                 preempt_disable();
 491 #endif
 492                 rdtsc_barrier();
 493                 time1 = get_cycles();
 494                 rdtsc_barrier();
 495
 496                 wrap_write_lock();
 497
 498                 rdtsc_barrier();
 499                 time2 = get_cycles();
 500                 rdtsc_barrier();
 501                 delay = time2 - time1;
 502                 ldelaymax = max(ldelaymax, delay);
 503                 ldelaymin = min(ldelaymin, delay);
 504                 ldelayavg += delay;
 505                 /*
 506                  * Read the previous values, check that they are coherent.
 507                  */
 508                 prev = var[0];
 509                 for (i = 1; i < NR_VARS; i++) {
 510                         cur = var[i];
 511                         if (cur != prev)
 512                                 printk(KERN_ALERT
 513                                 "Unequal cur %d/prev %d at i %d, iter %lu "
 514                                 "in writer thread\n",
 515                                 cur, prev, i, iter);
 516                 }
 517                 new = (int)get_cycles();
 518                 for (i = 0; i < NR_VARS; i++) {
 519                         var[i] = new;
 520                 }
 521
 522                 rdtsc_barrier();
 523                 time1 = get_cycles();
 524                 rdtsc_barrier();
 525
 526                 wrap_write_unlock();
 527
 528                 rdtsc_barrier();
 529                 time2 = get_cycles();
 530                 rdtsc_barrier();
 531                 delay = time2 - time1;
 532                 udelaymax = max(udelaymax, delay);
 533                 udelaymin = min(udelaymin, delay);
 534                 udelayavg += delay;
 535
 536 #if (!TEST_PREEMPT)
 537                 preempt_enable();
 538 #endif
 539                 if (WRITER_DELAY > 0)
 540                         udelay(WRITER_DELAY);
 541                 cpu_relax();    /*
 542                                  * make sure we don't busy-loop faster than
 543                                  * the lock busy-loop, it would cause reader and
 544                                  * writer starvation.
 545                                  */
 546                 yield_in_non_preempt();
 547         } while (!kthread_should_stop());
 548         ldelayavg /= iter;
 549         udelayavg /= iter;
 550         printk("writer_thread/%lu iterations : %lu, "
 551                 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 552                 (unsigned long)data, iter,
 553                 calibrate_cycles(ldelaymin),
 554                 calibrate_cycles(ldelayavg),
 555                 calibrate_cycles(ldelaymax));
 556         printk("writer_thread/%lu iterations : %lu, "
 557                 "unlock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 558                 (unsigned long)data, iter,
 559                 calibrate_cycles(udelaymin),
 560                 calibrate_cycles(udelayavg),
 561                 calibrate_cycles(udelaymax));
 562         return 0;
 563 }
 564
 565 static int trylock_writer_thread(void *data)
 566 {
 567         int i;
 568         int new;
 569         unsigned long iter = 0, success = 0, fail = 0;
 570
 571         printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data);
 572         do {
 573 #if ((!TEST_PREEMPT) && (!TEST_STD_RWLOCK))
 574                 preempt_disable();
 575 #endif
 576
 577 #if (TEST_STD_RWLOCK && TEST_INTERRUPTS)
 578                 /* std write trylock cannot disable interrupts. */
 579                 local_irq_disable();
 580 #endif
 581
 582 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
 583                 for (;;) {
 584                         iter++;
 585                         if (wrap_write_trylock())
 586                                 goto locked;
 587                         cpu_relax();
 588                 }
 589 #else
 590                 for (i = 0; i < TRYLOCK_WRITERS_FAIL_ITER; i++) {
 591                         iter++;
 592                         if (wrap_write_trylock())
 593                                 goto locked;
 594                         cpu_relax();
 595                 }
 596 #endif
 597                 fail++;
 598
 599 #if (TEST_STD_RWLOCK && TEST_INTERRUPTS)
 600                 local_irq_enable();
 601 #endif
 602
 603 #if ((!TEST_PREEMPT) && (!TEST_STD_RWLOCK))
 604                 preempt_enable();
 605 #endif
 606                 goto loop;
 607 locked:
 608                 success++;
 609                 new = (int)get_cycles();
 610                 for (i = 0; i < NR_VARS; i++) {
 611                         var[i] = new;
 612                 }
 613                 wrap_write_unlock();
 614 #if ((!TEST_PREEMPT) && (!TEST_STD_RWLOCK))
 615                 preempt_enable();
 616 #endif
 617 loop:
 618                 if (TRYLOCK_WRITER_DELAY > 0)
 619                         udelay(TRYLOCK_WRITER_DELAY);
 620                 cpu_relax();    /*
 621                                  * make sure we don't busy-loop faster than
 622                                  * the lock busy-loop, it would cause reader and
 623                                  * writer starvation.
 624                                  */
 625                 yield_in_non_preempt();
 626         } while (!kthread_should_stop());
 627         printk("trylock_writer_thread/%lu iterations : "
 628                 "[try,success,fail after %d try], "
 629                 "%lu,%lu,%lu\n",
 630                 (unsigned long)data, TRYLOCK_WRITERS_FAIL_ITER,
 631                 iter, success, fail);
 632         return 0;
 633 }
 634
 635 static void wbias_rwlock_create(void)
 636 {
 637         unsigned long i;
 638
 639         for (i = 0; i < NR_PREADERS; i++) {
 640                 printk("starting preemptable reader thread %lu\n", i);
 641                 preader_threads[i] = kthread_run(preader_thread, (void *)i,
 642                         "wbiasrwlock_preader");
 643                 BUG_ON(!preader_threads[i]);
 644         }
 645
 646         for (i = 0; i < NR_NPREADERS; i++) {
 647                 printk("starting non-preemptable reader thread %lu\n", i);
 648                 npreader_threads[i] = kthread_run(npreader_thread, (void *)i,
 649                         "wbiasrwlock_npreader");
 650                 BUG_ON(!npreader_threads[i]);
 651         }
 652
 653         for (i = 0; i < NR_TRYLOCK_READERS; i++) {
 654                 printk("starting trylock reader thread %lu\n", i);
 655                 trylock_reader_threads[i] = kthread_run(trylock_reader_thread,
 656                         (void *)i, "wbiasrwlock_trylock_reader");
 657                 BUG_ON(!trylock_reader_threads[i]);
 658         }
 659         for (i = 0; i < NR_INTERRUPT_READERS; i++) {
 660                 printk("starting interrupt reader %lu\n", i);
 661                 interrupt_reader[i] = kthread_run(interrupt_reader_thread,
 662                         (void *)i,
 663                         "wbiasrwlock_interrupt_reader");
 664         }
 665         for (i = 0; i < NR_TRYLOCK_INTERRUPT_READERS; i++) {
 666                 printk("starting trylock interrupt reader %lu\n", i);
 667                 trylock_interrupt_reader[i] =
 668                         kthread_run(trylock_interrupt_reader_thread,
 669                         (void *)i, "wbiasrwlock_trylock_interrupt_reader");
 670         }
 671         for (i = 0; i < NR_WRITERS; i++) {
 672                 printk("starting writer thread %lu\n", i);
 673                 writer_threads[i] = kthread_run(writer_thread, (void *)i,
 674                         "wbiasrwlock_writer");
 675                 BUG_ON(!writer_threads[i]);
 676         }
 677         for (i = 0; i < NR_TRYLOCK_WRITERS; i++) {
 678                 printk("starting trylock writer thread %lu\n", i);
 679                 trylock_writer_threads[i] = kthread_run(trylock_writer_thread,
 680                         (void *)i, "wbiasrwlock_trylock_writer");
 681                 BUG_ON(!trylock_writer_threads[i]);
 682         }
 683 }
 684
 685 static void wbias_rwlock_stop(void)
 686 {
 687         unsigned long i;
 688
 689         for (i = 0; i < NR_WRITERS; i++)
 690                 kthread_stop(writer_threads[i]);
 691         for (i = 0; i < NR_TRYLOCK_WRITERS; i++)
 692                 kthread_stop(trylock_writer_threads[i]);
 693         for (i = 0; i < NR_NPREADERS; i++)
 694                 kthread_stop(npreader_threads[i]);
 695         for (i = 0; i < NR_PREADERS; i++)
 696                 kthread_stop(preader_threads[i]);
 697         for (i = 0; i < NR_TRYLOCK_READERS; i++)
 698                 kthread_stop(trylock_reader_threads[i]);
 699         for (i = 0; i < NR_INTERRUPT_READERS; i++)
 700                 kthread_stop(interrupt_reader[i]);
 701         for (i = 0; i < NR_TRYLOCK_INTERRUPT_READERS; i++)
 702                 kthread_stop(trylock_interrupt_reader[i]);
 703 }
 704
 705
 706 static void perform_test(const char *name, void (*callback)(void))
 707 {
 708         printk("%s\n", name);
 709         callback();
 710 }
 711
 712 static int my_open(struct inode *inode, struct file *file)
 713 {
 714         unsigned long i;
 715         cycles_t time1, time2, delay;
 716
 717         printk("** get_cycles calibration **\n");
 718         cycles_calibration_min = ULLONG_MAX;
 719         cycles_calibration_avg = 0;
 720         cycles_calibration_max = 0;
 721
 722         local_irq_disable();
 723         for (i = 0; i < 10; i++) {
 724                 rdtsc_barrier();
 725                 time1 = get_cycles();
 726                 rdtsc_barrier();
 727                 rdtsc_barrier();
 728                 time2 = get_cycles();
 729                 rdtsc_barrier();
 730                 delay = time2 - time1;
 731                 cycles_calibration_min = min(cycles_calibration_min, delay);
 732                 cycles_calibration_avg += delay;
 733                 cycles_calibration_max = max(cycles_calibration_max, delay);
 734         }
 735         cycles_calibration_avg /= 10;
 736         local_irq_enable();
 737
 738         printk("get_cycles takes [min,avg,max] %llu,%llu,%llu cycles, "
 739                 "results calibrated on avg\n",
 740                 cycles_calibration_min,
 741                 cycles_calibration_avg,
 742                 cycles_calibration_max);
 743         printk("\n");
 744
 745 #if (NR_WRITERS)
 746         printk("** Single writer test, no contention **\n");
 747         wbias_rwlock_profile_latency_reset();
 748         writer_threads[0] = kthread_run(writer_thread, (void *)0,
 749                 "wbiasrwlock_writer");
 750         BUG_ON(!writer_threads[0]);
 751         ssleep(SINGLE_WRITER_TEST_DURATION);
 752         kthread_stop(writer_threads[0]);
 753         printk("\n");
 754
 755         wbias_rwlock_profile_latency_print();
 756 #endif
 757
 758 #if (NR_TRYLOCK_WRITERS)
 759         printk("** Single trylock writer test, no contention **\n");
 760         wbias_rwlock_profile_latency_reset();
 761         trylock_writer_threads[0] = kthread_run(trylock_writer_thread,
 762                 (void *)0,
 763                 "trylock_wbiasrwlock_writer");
 764         BUG_ON(!trylock_writer_threads[0]);
 765         ssleep(SINGLE_WRITER_TEST_DURATION);
 766         kthread_stop(trylock_writer_threads[0]);
 767         printk("\n");
 768
 769         wbias_rwlock_profile_latency_print();
 770 #endif
 771
 772 #if (TEST_PREEMPT)
 773         printk("** Single preemptable reader test, no contention **\n");
 774         wbias_rwlock_profile_latency_reset();
 775         preader_threads[0] = kthread_run(preader_thread, (void *)0,
 776                 "wbiasrwlock_preader");
 777         BUG_ON(!preader_threads[0]);
 778         ssleep(SINGLE_READER_TEST_DURATION);
 779         kthread_stop(preader_threads[0]);
 780         printk("\n");
 781
 782         wbias_rwlock_profile_latency_print();
 783 #endif
 784
 785         printk("** Single non-preemptable reader test, no contention **\n");
 786         wbias_rwlock_profile_latency_reset();
 787         npreader_threads[0] = kthread_run(npreader_thread, (void *)0,
 788                 "wbiasrwlock_npreader");
 789         BUG_ON(!npreader_threads[0]);
 790         ssleep(SINGLE_READER_TEST_DURATION);
 791         kthread_stop(npreader_threads[0]);
 792         printk("\n");
 793
 794         wbias_rwlock_profile_latency_print();
 795
 796         printk("** Multiple p/non-p readers test, no contention **\n");
 797         wbias_rwlock_profile_latency_reset();
 798 #if (TEST_PREEMPT)
 799         for (i = 0; i < NR_PREADERS; i++) {
 800                 printk("starting preader thread %lu\n", i);
 801                 preader_threads[i] = kthread_run(preader_thread, (void *)i,
 802                         "wbiasrwlock_preader");
 803                 BUG_ON(!preader_threads[i]);
 804         }
 805 #endif
 806         for (i = 0; i < NR_NPREADERS; i++) {
 807                 printk("starting npreader thread %lu\n", i);
 808                 npreader_threads[i] = kthread_run(npreader_thread, (void *)i,
 809                         "wbiasrwlock_npreader");
 810                 BUG_ON(!npreader_threads[i]);
 811         }
 812         ssleep(SINGLE_READER_TEST_DURATION);
 813         for (i = 0; i < NR_NPREADERS; i++)
 814                 kthread_stop(npreader_threads[i]);
 815 #if (TEST_PREEMPT)
 816         for (i = 0; i < NR_PREADERS; i++)
 817                 kthread_stop(preader_threads[i]);
 818 #endif
 819         printk("\n");
 820
 821         wbias_rwlock_profile_latency_print();
 822
 823         printk("** High contention test **\n");
 824         wbias_rwlock_profile_latency_reset();
 825         perform_test("wbias-rwlock-create", wbias_rwlock_create);
 826         ssleep(TEST_DURATION);
 827         perform_test("wbias-rwlock-stop", wbias_rwlock_stop);
 828         printk("\n");
 829         wbias_rwlock_profile_latency_print();
 830
 831         return -EPERM;
 832 }
 833
 834
 835 static struct file_operations my_operations = {
 836         .open = my_open,
 837 };
 838
 839 int init_module(void)
 840 {
 841         pentry = create_proc_entry("testwbiasrwlock", 0444, NULL);
 842         if (pentry)
 843                 pentry->proc_fops = &my_operations;
 844
 845         printk("UC_READER_MASK   :         %08X\n", UC_READER_MASK);
 846         printk("UC_HARDIRQ_R_MASK:         %08X\n", UC_HARDIRQ_READER_MASK);
 847         printk("UC_SOFTIRQ_R_MASK:         %08X\n", UC_SOFTIRQ_READER_MASK);
 848         printk("UC_NPTHREA_R_MASK:         %08X\n", UC_NPTHREAD_READER_MASK);
 849         printk("UC_PTHREAD_R_MASK:         %08X\n", UC_PTHREAD_READER_MASK);
 850         printk("UC_WRITER        :         %08X\n", UC_WRITER);
 851         printk("UC_SLOW_WRITER   :         %08X\n", UC_SLOW_WRITER);
 852         printk("UC_WQ_ACTIVE     :         %08X\n", UC_WQ_ACTIVE);
 853         printk("WS_MASK          :         %08X\n", WS_MASK);
 854         printk("WS_WQ_MUTEX      :         %08X\n", WS_WQ_MUTEX);
 855         printk("WS_COUNT_MUTEX   :         %08X\n", WS_COUNT_MUTEX);
 856         printk("WS_LOCK_MUTEX    :         %08X\n", WS_LOCK_MUTEX);
 857         printk("CTX_RMASK        : %016lX\n", CTX_RMASK);
 858         printk("CTX_WMASK        : %016lX\n", CTX_WMASK);
 859
 860         return 0;
 861 }
 862
 863 void cleanup_module(void)
 864 {
 865         remove_proc_entry("testwbiasrwlock", NULL);
 866 }
 867
 868 MODULE_LICENSE("GPL");
 869 MODULE_AUTHOR("Mathieu Desnoyers");
 870 MODULE_DESCRIPTION("wbias rwlock test");