tests/kernel/test-psrwlock.c

   1 /*
   2  * test-psrwlock.c
   3  */
   4
   5 #include <linux/module.h>
   6 #include <linux/proc_fs.h>
   7 #include <linux/sched.h>
   8 #include <linux/timex.h>
   9 #include <linux/kthread.h>
  10 #include <linux/delay.h>
  11 #include <linux/hardirq.h>
  12 #include <linux/module.h>
  13 #include <linux/percpu.h>
  14 #include <linux/spinlock.h>
  15 #include <asm/ptrace.h>
  16 #include <linux/psrwlock.h>
  17
  18 /* Test with no contention duration, in seconds */
  19 #define SINGLE_WRITER_TEST_DURATION 10
  20 #define SINGLE_READER_TEST_DURATION 10
  21 #define MULTIPLE_READERS_TEST_DURATION 10
  22
  23 /* Test duration, in seconds */
  24 #define TEST_DURATION 60
  25
  26 #define NR_VARS 100
  27 #define NR_WRITERS 2
  28 #define NR_TRYLOCK_WRITERS 1
  29 #define NR_NPREADERS 2
  30 #define NR_TRYLOCK_READERS 1
  31
  32 /*
  33  * 1 : test standard rwlock
  34  * 0 : test psrwlock
  35  */
  36 #define TEST_STD_RWLOCK 0
  37
  38 /*
  39  * 1 : test with thread and interrupt readers.
  40  * 0 : test only with thread readers.
  41  */
  42 #define TEST_INTERRUPTS 1
  43
  44 #if (TEST_INTERRUPTS)
  45 #define NR_INTERRUPT_READERS 1
  46 #define NR_TRYLOCK_INTERRUPT_READERS 1
  47 #else
  48 #define NR_INTERRUPT_READERS 0
  49 #define NR_TRYLOCK_INTERRUPT_READERS 0
  50 #endif
  51
  52 /*
  53  * 1 : test with thread preemption readers.
  54  * 0 : test only with non-preemptable thread readers.
  55  */
  56 #define TEST_PREEMPT 1
  57
  58 #if (TEST_PREEMPT)
  59 #define NR_PREADERS 2
  60 #else
  61 #define NR_PREADERS 0
  62 #endif
  63
  64
  65 /*
  66  * Writer iteration delay, in us. 0 for busy loop. Caution : writers can
  67  * starve readers.
  68  */
  69 #define WRITER_DELAY 100
  70 #define TRYLOCK_WRITER_DELAY 1000
  71
  72 /*
  73  * Number of iterations after which a trylock writer fails.
  74  * -1 for infinite loop.
  75  */
  76 #define TRYLOCK_WRITERS_FAIL_ITER 100
  77
  78 /* Thread and interrupt reader delay, in ms */
  79 #define THREAD_READER_DELAY 0   /* busy loop */
  80 #define INTERRUPT_READER_DELAY 100
  81
  82 #ifdef CONFIG_PREEMPT
  83 #define yield_in_non_preempt()
  84 #else
  85 #define yield_in_non_preempt()  yield()
  86 #endif
  87
  88 static int var[NR_VARS];
  89 static struct task_struct *preader_threads[NR_PREADERS];
  90 static struct task_struct *npreader_threads[NR_NPREADERS];
  91 static struct task_struct *trylock_reader_threads[NR_TRYLOCK_READERS];
  92 static struct task_struct *writer_threads[NR_WRITERS];
  93 static struct task_struct *trylock_writer_threads[NR_TRYLOCK_WRITERS];
  94 static struct task_struct *interrupt_reader[NR_INTERRUPT_READERS];
  95 static struct task_struct *trylock_interrupt_reader[NR_TRYLOCK_INTERRUPT_READERS];
  96
  97 #if (TEST_STD_RWLOCK)
  98
  99 static DEFINE_RWLOCK(std_rw_lock);
 100
 101 #define wrap_read_lock()        read_lock(&std_rw_lock)
 102 #define wrap_read_trylock()     read_trylock(&std_rw_lock)
 103 #define wrap_read_unlock()      read_unlock(&std_rw_lock)
 104
 105 #define wrap_read_lock_inatomic()       read_lock(&std_rw_lock)
 106 #define wrap_read_trylock_inatomic()    read_trylock(&std_rw_lock)
 107
 108 #define wrap_read_lock_irq()    read_lock(&std_rw_lock)
 109 #define wrap_read_trylock_irq() read_trylock(&std_rw_lock)
 110
 111 #if (TEST_INTERRUPTS)
 112 #define wrap_write_lock()       write_lock_irq(&std_rw_lock)
 113 #define wrap_write_unlock()     write_unlock_irq(&std_rw_lock)
 114 #else
 115 #define wrap_write_lock()       write_lock(&std_rw_lock)
 116 #define wrap_write_unlock()     write_unlock(&std_rw_lock)
 117 #endif
 118
 119 #define wrap_write_trylock()    write_trylock(&std_rw_lock)
 120
 121 #else
 122
 123 #if (TEST_INTERRUPTS)
 124 #if (TEST_PREEMPT)
 125 #define PSRWLOCKWCTX PSRW_PRIO_P
 126 #define PSRWLOCKRCTX (PSR_IRQ | PSR_NPTHREAD | PSR_PTHREAD)
 127 #else
 128 #define PSRWLOCKWCTX PSRW_PRIO_NP
 129 #define PSRWLOCKRCTX (PSR_IRQ | PSR_NPTHREAD)
 130 #endif
 131 #else
 132 #if (TEST_PREEMPT)
 133 #define PSRWLOCKWCTX PSRW_PRIO_P
 134 #define PSRWLOCKRCTX (PSR_NPTHREAD | PSR_PTHREAD)
 135 #else
 136 #define PSRWLOCKWCTX PSRW_PRIO_NP
 137 #define PSRWLOCKRCTX (PSR_NPTHREAD)
 138 #endif
 139 #endif
 140
 141 static DEFINE_PSRWLOCK(psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX);
 142 CHECK_PSRWLOCK_MAP(psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX);
 143
 144
 145 #if (TEST_PREEMPT)
 146 #define wrap_read_lock()                        \
 147         psread_lock(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 148 #define wrap_read_trylock()                     \
 149         psread_trylock(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 150 #else
 151 #define wrap_read_lock()                        \
 152         psread_lock_inatomic(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 153 #define wrap_read_trylock()                     \
 154         psread_trylock_inatomic(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 155 #endif
 156 #define wrap_read_unlock()                      \
 157         psread_unlock(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 158
 159 #define wrap_read_lock_inatomic()               \
 160         psread_lock_inatomic(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 161 #define wrap_read_trylock_inatomic()            \
 162         psread_trylock_inatomic(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 163
 164 #define wrap_read_lock_irq()                    \
 165         psread_lock_irq(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 166 #define wrap_read_trylock_irq()                 \
 167         psread_trylock_irq(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 168
 169 #define wrap_write_lock()                       \
 170         pswrite_lock(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 171 #define wrap_write_unlock()                     \
 172         pswrite_unlock(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 173 #define wrap_write_trylock()                    \
 174         pswrite_trylock(&psrwlock, PSRWLOCKWCTX, PSRWLOCKRCTX)
 175
 176 #endif
 177
 178 static cycles_t cycles_calibration_min,
 179         cycles_calibration_avg,
 180         cycles_calibration_max;
 181
 182 static inline cycles_t calibrate_cycles(cycles_t cycles)
 183 {
 184         return cycles - cycles_calibration_avg;
 185 }
 186
 187 struct proc_dir_entry *pentry = NULL;
 188
 189 static int p_or_np_reader_thread(const char *typename,
 190                 void *data, int preemptable)
 191 {
 192         int i;
 193         int prev, cur;
 194         unsigned long iter = 0;
 195         cycles_t time1, time2, delay;
 196         cycles_t ldelaymax = 0, ldelaymin = ULLONG_MAX, ldelayavg = 0;
 197         cycles_t udelaymax = 0, udelaymin = ULLONG_MAX, udelayavg = 0;
 198
 199         printk("%s/%lu runnning\n", typename, (unsigned long)data);
 200         do {
 201                 iter++;
 202                 if (!preemptable)
 203                         preempt_disable();
 204                 rdtsc_barrier();
 205                 time1 = get_cycles();
 206                 rdtsc_barrier();
 207
 208                 if (!preemptable)
 209                         wrap_read_lock_inatomic();
 210                 else
 211                         wrap_read_lock();
 212
 213                 rdtsc_barrier();
 214                 time2 = get_cycles();
 215                 rdtsc_barrier();
 216                 delay = time2 - time1;
 217                 ldelaymax = max(ldelaymax, delay);
 218                 ldelaymin = min(ldelaymin, delay);
 219                 ldelayavg += delay;
 220                 prev = var[0];
 221                 for (i = 1; i < NR_VARS; i++) {
 222                         cur = var[i];
 223                         if (cur != prev) {
 224                                 printk(KERN_ALERT
 225                                 "Unequal cur %d/prev %d at i %d, iter %lu "
 226                                 "in reader thread\n",
 227                                 cur, prev, i, iter);
 228                         }
 229                 }
 230
 231                 rdtsc_barrier();
 232                 time1 = get_cycles();
 233                 rdtsc_barrier();
 234
 235                 wrap_read_unlock();
 236
 237                 rdtsc_barrier();
 238                 time2 = get_cycles();
 239                 rdtsc_barrier();
 240                 delay = time2 - time1;
 241                 udelaymax = max(udelaymax, delay);
 242                 udelaymin = min(udelaymin, delay);
 243                 udelayavg += delay;
 244
 245                 if (!preemptable)
 246                         preempt_enable();
 247
 248                 if (THREAD_READER_DELAY)
 249                         msleep(THREAD_READER_DELAY);
 250                 yield_in_non_preempt();
 251         } while (!kthread_should_stop());
 252         if (!iter) {
 253                 printk("%s/%lu iterations : %lu", typename,
 254                         (unsigned long)data, iter);
 255         } else {
 256                 ldelayavg /= iter;
 257                 udelayavg /= iter;
 258                 printk("%s/%lu iterations : %lu, "
 259                         "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 260                         typename,
 261                         (unsigned long)data, iter,
 262                         calibrate_cycles(ldelaymin),
 263                         calibrate_cycles(ldelayavg),
 264                         calibrate_cycles(ldelaymax));
 265                 printk("%s/%lu iterations : %lu, "
 266                         "unlock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 267                         typename,
 268                         (unsigned long)data, iter,
 269                         calibrate_cycles(udelaymin),
 270                         calibrate_cycles(udelayavg),
 271                         calibrate_cycles(udelaymax));
 272         }
 273         return 0;
 274 }
 275
 276 static int preader_thread(void *data)
 277 {
 278         return p_or_np_reader_thread("preader_thread", data, 1);
 279 }
 280
 281 static int npreader_thread(void *data)
 282 {
 283         return p_or_np_reader_thread("npreader_thread", data, 0);
 284 }
 285
 286 static int trylock_reader_thread(void *data)
 287 {
 288         int i;
 289         int prev, cur;
 290         unsigned long iter = 0, success_iter = 0;
 291
 292         printk("trylock_reader_thread/%lu runnning\n", (unsigned long)data);
 293         do {
 294 #if (!TEST_PREEMPT)
 295                 preempt_disable();
 296 #endif
 297                 while (!wrap_read_trylock()) {
 298                         cpu_relax();
 299                         iter++;
 300                 }
 301                 success_iter++;
 302                 prev = var[0];
 303                 for (i = 1; i < NR_VARS; i++) {
 304                         cur = var[i];
 305                         if (cur != prev) {
 306                                 printk(KERN_ALERT
 307                                 "Unequal cur %d/prev %d at i %d, iter %lu "
 308                                 "in trylock reader thread\n",
 309                                 cur, prev, i, iter);
 310                         }
 311                 }
 312                 wrap_read_unlock();
 313 #if (!TEST_PREEMPT)
 314                 preempt_enable();
 315 #endif
 316                 if (THREAD_READER_DELAY)
 317                         msleep(THREAD_READER_DELAY);
 318                 yield_in_non_preempt();
 319         } while (!kthread_should_stop());
 320         printk("trylock_reader_thread/%lu iterations : %lu, "
 321                 "successful iterations : %lu\n",
 322                 (unsigned long)data, iter + success_iter, success_iter);
 323         return 0;
 324 }
 325
 326 DEFINE_PER_CPU(cycles_t, int_ldelaymin);
 327 DEFINE_PER_CPU(cycles_t, int_ldelayavg);
 328 DEFINE_PER_CPU(cycles_t, int_ldelaymax);
 329 DEFINE_PER_CPU(cycles_t, int_udelaymin);
 330 DEFINE_PER_CPU(cycles_t, int_udelayavg);
 331 DEFINE_PER_CPU(cycles_t, int_udelaymax);
 332 DEFINE_PER_CPU(cycles_t, int_ipi_nr);
 333
 334 static void interrupt_reader_ipi(void *data)
 335 {
 336         int i;
 337         int prev, cur;
 338         cycles_t time1, time2;
 339         cycles_t *ldelaymax, *ldelaymin, *ldelayavg, *ipi_nr, delay;
 340         cycles_t *udelaymax, *udelaymin, *udelayavg;
 341
 342         /*
 343          * Skip the ipi caller, not in irq context.
 344          */
 345         if (!in_irq())
 346                 return;
 347
 348         ldelaymax = &per_cpu(int_ldelaymax, smp_processor_id());
 349         ldelaymin = &per_cpu(int_ldelaymin, smp_processor_id());
 350         ldelayavg = &per_cpu(int_ldelayavg, smp_processor_id());
 351         udelaymax = &per_cpu(int_udelaymax, smp_processor_id());
 352         udelaymin = &per_cpu(int_udelaymin, smp_processor_id());
 353         udelayavg = &per_cpu(int_udelayavg, smp_processor_id());
 354         ipi_nr = &per_cpu(int_ipi_nr, smp_processor_id());
 355
 356         rdtsc_barrier();
 357         time1 = get_cycles();
 358         rdtsc_barrier();
 359
 360         wrap_read_lock_irq();
 361
 362         rdtsc_barrier();
 363         time2 = get_cycles();
 364         rdtsc_barrier();
 365         delay = time2 - time1;
 366         *ldelaymax = max(*ldelaymax, delay);
 367         *ldelaymin = min(*ldelaymin, delay);
 368         *ldelayavg += delay;
 369         (*ipi_nr)++;
 370         prev = var[0];
 371         for (i = 1; i < NR_VARS; i++) {
 372                 cur = var[i];
 373                 if (cur != prev)
 374                         printk(KERN_ALERT
 375                         "Unequal cur %d/prev %d at i %d in interrupt\n",
 376                                 cur, prev, i);
 377         }
 378         rdtsc_barrier();
 379         time1 = get_cycles();
 380         rdtsc_barrier();
 381         wrap_read_unlock();
 382         time2 = get_cycles();
 383         rdtsc_barrier();
 384         delay = time2 - time1;
 385         *udelaymax = max(*udelaymax, delay);
 386         *udelaymin = min(*udelaymin, delay);
 387         *udelayavg += delay;
 388 }
 389
 390 DEFINE_PER_CPU(unsigned long, trylock_int_iter);
 391 DEFINE_PER_CPU(unsigned long, trylock_int_success);
 392
 393 static void trylock_interrupt_reader_ipi(void *data)
 394 {
 395         int i;
 396         int prev, cur;
 397
 398         /*
 399          * Skip the ipi caller, not in irq context.
 400          */
 401         if (!in_irq())
 402                 return;
 403
 404         per_cpu(trylock_int_iter, smp_processor_id())++;
 405         while (!wrap_read_trylock_irq())
 406                 per_cpu(trylock_int_iter, smp_processor_id())++;
 407         per_cpu(trylock_int_success, smp_processor_id())++;
 408         prev = var[0];
 409         for (i = 1; i < NR_VARS; i++) {
 410                 cur = var[i];
 411                 if (cur != prev)
 412                         printk(KERN_ALERT
 413                         "Unequal cur %d/prev %d at i %d in interrupt\n",
 414                                 cur, prev, i);
 415         }
 416         wrap_read_unlock();
 417 }
 418
 419
 420 static int interrupt_reader_thread(void *data)
 421 {
 422         unsigned long iter = 0;
 423         int i;
 424
 425         for_each_online_cpu(i) {
 426                 per_cpu(int_ldelaymax, i) = 0;
 427                 per_cpu(int_ldelaymin, i) = ULLONG_MAX;
 428                 per_cpu(int_ldelayavg, i) = 0;
 429                 per_cpu(int_udelaymax, i) = 0;
 430                 per_cpu(int_udelaymin, i) = ULLONG_MAX;
 431                 per_cpu(int_udelayavg, i) = 0;
 432                 per_cpu(int_ipi_nr, i) = 0;
 433         }
 434         do {
 435                 iter++;
 436                 on_each_cpu(interrupt_reader_ipi, NULL, 0);
 437                 if (INTERRUPT_READER_DELAY)
 438                         msleep(INTERRUPT_READER_DELAY);
 439                 yield_in_non_preempt();
 440         } while (!kthread_should_stop());
 441         printk("interrupt_reader_thread/%lu iterations : %lu\n",
 442                         (unsigned long)data, iter);
 443         for_each_online_cpu(i) {
 444                 if (!per_cpu(int_ipi_nr, i))
 445                         continue;
 446                 per_cpu(int_ldelayavg, i) /= per_cpu(int_ipi_nr, i);
 447                 per_cpu(int_udelayavg, i) /= per_cpu(int_ipi_nr, i);
 448                 printk("interrupt readers on CPU %i, "
 449                         "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 450                         i,
 451                         calibrate_cycles(per_cpu(int_ldelaymin, i)),
 452                         calibrate_cycles(per_cpu(int_ldelayavg, i)),
 453                         calibrate_cycles(per_cpu(int_ldelaymax, i)));
 454                 printk("interrupt readers on CPU %i, "
 455                         "unlock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 456                         i,
 457                         calibrate_cycles(per_cpu(int_udelaymin, i)),
 458                         calibrate_cycles(per_cpu(int_udelayavg, i)),
 459                         calibrate_cycles(per_cpu(int_udelaymax, i)));
 460         }
 461         return 0;
 462 }
 463
 464 static int trylock_interrupt_reader_thread(void *data)
 465 {
 466         unsigned long iter = 0;
 467         int i;
 468
 469         do {
 470                 iter++;
 471                 on_each_cpu(trylock_interrupt_reader_ipi, NULL, 0);
 472                 if (INTERRUPT_READER_DELAY)
 473                         msleep(INTERRUPT_READER_DELAY);
 474                 yield_in_non_preempt();
 475         } while (!kthread_should_stop());
 476         printk("trylock_interrupt_reader_thread/%lu iterations : %lu\n",
 477                         (unsigned long)data, iter);
 478         for_each_online_cpu(i) {
 479                 printk("trylock interrupt readers on CPU %i, "
 480                         "iterations %lu, "
 481                         "successful iterations : %lu\n",
 482                         i, per_cpu(trylock_int_iter, i),
 483                         per_cpu(trylock_int_success, i));
 484                 per_cpu(trylock_int_iter, i) = 0;
 485                 per_cpu(trylock_int_success, i) = 0;
 486         }
 487         return 0;
 488 }
 489
 490 static int writer_thread(void *data)
 491 {
 492         int i;
 493         int new, prev, cur;
 494         unsigned long iter = 0;
 495         cycles_t time1, time2, delay;
 496         cycles_t ldelaymax = 0, ldelaymin = ULLONG_MAX, ldelayavg = 0;
 497         cycles_t udelaymax = 0, udelaymin = ULLONG_MAX, udelayavg = 0;
 498
 499         printk("writer_thread/%lu runnning\n", (unsigned long)data);
 500         do {
 501                 iter++;
 502 #if (!TEST_PREEMPT)
 503                 preempt_disable();
 504 #endif
 505                 rdtsc_barrier();
 506                 time1 = get_cycles();
 507                 rdtsc_barrier();
 508
 509                 wrap_write_lock();
 510
 511                 rdtsc_barrier();
 512                 time2 = get_cycles();
 513                 rdtsc_barrier();
 514                 delay = time2 - time1;
 515                 ldelaymax = max(ldelaymax, delay);
 516                 ldelaymin = min(ldelaymin, delay);
 517                 ldelayavg += delay;
 518                 /*
 519                  * Read the previous values, check that they are coherent.
 520                  */
 521                 prev = var[0];
 522                 for (i = 1; i < NR_VARS; i++) {
 523                         cur = var[i];
 524                         if (cur != prev)
 525                                 printk(KERN_ALERT
 526                                 "Unequal cur %d/prev %d at i %d, iter %lu "
 527                                 "in writer thread\n",
 528                                 cur, prev, i, iter);
 529                 }
 530                 new = (int)get_cycles();
 531                 for (i = 0; i < NR_VARS; i++) {
 532                         var[i] = new;
 533                 }
 534
 535                 rdtsc_barrier();
 536                 time1 = get_cycles();
 537                 rdtsc_barrier();
 538
 539                 wrap_write_unlock();
 540
 541                 rdtsc_barrier();
 542                 time2 = get_cycles();
 543                 rdtsc_barrier();
 544                 delay = time2 - time1;
 545                 udelaymax = max(udelaymax, delay);
 546                 udelaymin = min(udelaymin, delay);
 547                 udelayavg += delay;
 548
 549 #if (!TEST_PREEMPT)
 550                 preempt_enable();
 551 #endif
 552                 if (WRITER_DELAY > 0)
 553                         udelay(WRITER_DELAY);
 554                 cpu_relax();    /*
 555                                  * make sure we don't busy-loop faster than
 556                                  * the lock busy-loop, it would cause reader and
 557                                  * writer starvation.
 558                                  */
 559                 yield_in_non_preempt();
 560         } while (!kthread_should_stop());
 561         ldelayavg /= iter;
 562         udelayavg /= iter;
 563         printk("writer_thread/%lu iterations : %lu, "
 564                 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 565                 (unsigned long)data, iter,
 566                 calibrate_cycles(ldelaymin),
 567                 calibrate_cycles(ldelayavg),
 568                 calibrate_cycles(ldelaymax));
 569         printk("writer_thread/%lu iterations : %lu, "
 570                 "unlock delay [min,avg,max] %llu,%llu,%llu cycles\n",
 571                 (unsigned long)data, iter,
 572                 calibrate_cycles(udelaymin),
 573                 calibrate_cycles(udelayavg),
 574                 calibrate_cycles(udelaymax));
 575         return 0;
 576 }
 577
 578 static int trylock_writer_thread(void *data)
 579 {
 580         int i;
 581         int new;
 582         unsigned long iter = 0, success = 0, fail = 0;
 583
 584         printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data);
 585         do {
 586 #if ((!TEST_PREEMPT) && (!TEST_STD_RWLOCK))
 587                 preempt_disable();
 588 #endif
 589
 590 #if (TEST_STD_RWLOCK && TEST_INTERRUPTS)
 591                 /* std write trylock cannot disable interrupts. */
 592                 local_irq_disable();
 593 #endif
 594
 595 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
 596                 for (;;) {
 597                         iter++;
 598                         if (wrap_write_trylock())
 599                                 goto locked;
 600                         cpu_relax();
 601                 }
 602 #else
 603                 for (i = 0; i < TRYLOCK_WRITERS_FAIL_ITER; i++) {
 604                         iter++;
 605                         if (wrap_write_trylock())
 606                                 goto locked;
 607                         cpu_relax();
 608                 }
 609 #endif
 610                 fail++;
 611
 612 #if (TEST_STD_RWLOCK && TEST_INTERRUPTS)
 613                 local_irq_enable();
 614 #endif
 615
 616 #if ((!TEST_PREEMPT) && (!TEST_STD_RWLOCK))
 617                 preempt_enable();
 618 #endif
 619                 goto loop;
 620 locked:
 621                 success++;
 622                 new = (int)get_cycles();
 623                 for (i = 0; i < NR_VARS; i++) {
 624                         var[i] = new;
 625                 }
 626                 wrap_write_unlock();
 627 #if ((!TEST_PREEMPT) && (!TEST_STD_RWLOCK))
 628                 preempt_enable();
 629 #endif
 630 loop:
 631                 if (TRYLOCK_WRITER_DELAY > 0)
 632                         udelay(TRYLOCK_WRITER_DELAY);
 633                 cpu_relax();    /*
 634                                  * make sure we don't busy-loop faster than
 635                                  * the lock busy-loop, it would cause reader and
 636                                  * writer starvation.
 637                                  */
 638                 yield_in_non_preempt();
 639         } while (!kthread_should_stop());
 640         printk("trylock_writer_thread/%lu iterations : "
 641                 "[try,success,fail after %d try], "
 642                 "%lu,%lu,%lu\n",
 643                 (unsigned long)data, TRYLOCK_WRITERS_FAIL_ITER,
 644                 iter, success, fail);
 645         return 0;
 646 }
 647
 648 static void psrwlock_create(void)
 649 {
 650         unsigned long i;
 651
 652         for (i = 0; i < NR_PREADERS; i++) {
 653                 printk("starting preemptable reader thread %lu\n", i);
 654                 preader_threads[i] = kthread_run(preader_thread, (void *)i,
 655                         "psrwlock_preader");
 656                 BUG_ON(!preader_threads[i]);
 657         }
 658
 659         for (i = 0; i < NR_NPREADERS; i++) {
 660                 printk("starting non-preemptable reader thread %lu\n", i);
 661                 npreader_threads[i] = kthread_run(npreader_thread, (void *)i,
 662                         "psrwlock_npreader");
 663                 BUG_ON(!npreader_threads[i]);
 664         }
 665
 666         for (i = 0; i < NR_TRYLOCK_READERS; i++) {
 667                 printk("starting trylock reader thread %lu\n", i);
 668                 trylock_reader_threads[i] = kthread_run(trylock_reader_thread,
 669                         (void *)i, "psrwlock_trylock_reader");
 670                 BUG_ON(!trylock_reader_threads[i]);
 671         }
 672         for (i = 0; i < NR_INTERRUPT_READERS; i++) {
 673                 printk("starting interrupt reader %lu\n", i);
 674                 interrupt_reader[i] = kthread_run(interrupt_reader_thread,
 675                         (void *)i,
 676                         "psrwlock_interrupt_reader");
 677         }
 678         for (i = 0; i < NR_TRYLOCK_INTERRUPT_READERS; i++) {
 679                 printk("starting trylock interrupt reader %lu\n", i);
 680                 trylock_interrupt_reader[i] =
 681                         kthread_run(trylock_interrupt_reader_thread,
 682                         (void *)i, "psrwlock_trylock_interrupt_reader");
 683         }
 684         for (i = 0; i < NR_WRITERS; i++) {
 685                 printk("starting writer thread %lu\n", i);
 686                 writer_threads[i] = kthread_run(writer_thread, (void *)i,
 687                         "psrwlock_writer");
 688                 BUG_ON(!writer_threads[i]);
 689         }
 690         for (i = 0; i < NR_TRYLOCK_WRITERS; i++) {
 691                 printk("starting trylock writer thread %lu\n", i);
 692                 trylock_writer_threads[i] = kthread_run(trylock_writer_thread,
 693                         (void *)i, "psrwlock_trylock_writer");
 694                 BUG_ON(!trylock_writer_threads[i]);
 695         }
 696 }
 697
 698 static void psrwlock_stop(void)
 699 {
 700         unsigned long i;
 701
 702         for (i = 0; i < NR_WRITERS; i++)
 703                 kthread_stop(writer_threads[i]);
 704         for (i = 0; i < NR_TRYLOCK_WRITERS; i++)
 705                 kthread_stop(trylock_writer_threads[i]);
 706         for (i = 0; i < NR_NPREADERS; i++)
 707                 kthread_stop(npreader_threads[i]);
 708         for (i = 0; i < NR_PREADERS; i++)
 709                 kthread_stop(preader_threads[i]);
 710         for (i = 0; i < NR_TRYLOCK_READERS; i++)
 711                 kthread_stop(trylock_reader_threads[i]);
 712         for (i = 0; i < NR_INTERRUPT_READERS; i++)
 713                 kthread_stop(interrupt_reader[i]);
 714         for (i = 0; i < NR_TRYLOCK_INTERRUPT_READERS; i++)
 715                 kthread_stop(trylock_interrupt_reader[i]);
 716 }
 717
 718
 719 static void perform_test(const char *name, void (*callback)(void))
 720 {
 721         printk("%s\n", name);
 722         callback();
 723 }
 724
 725 static int my_open(struct inode *inode, struct file *file)
 726 {
 727         unsigned long i;
 728         cycles_t time1, time2, delay;
 729
 730         printk("** get_cycles calibration **\n");
 731         cycles_calibration_min = ULLONG_MAX;
 732         cycles_calibration_avg = 0;
 733         cycles_calibration_max = 0;
 734
 735         local_irq_disable();
 736         for (i = 0; i < 10; i++) {
 737                 rdtsc_barrier();
 738                 time1 = get_cycles();
 739                 rdtsc_barrier();
 740                 rdtsc_barrier();
 741                 time2 = get_cycles();
 742                 rdtsc_barrier();
 743                 delay = time2 - time1;
 744                 cycles_calibration_min = min(cycles_calibration_min, delay);
 745                 cycles_calibration_avg += delay;
 746                 cycles_calibration_max = max(cycles_calibration_max, delay);
 747         }
 748         cycles_calibration_avg /= 10;
 749         local_irq_enable();
 750
 751         printk("get_cycles takes [min,avg,max] %llu,%llu,%llu cycles, "
 752                 "results calibrated on avg\n",
 753                 cycles_calibration_min,
 754                 cycles_calibration_avg,
 755                 cycles_calibration_max);
 756         printk("\n");
 757
 758 #if (NR_WRITERS)
 759         printk("** Single writer test, no contention **\n");
 760         psrwlock_profile_latency_reset();
 761         writer_threads[0] = kthread_run(writer_thread, (void *)0,
 762                 "psrwlock_writer");
 763         BUG_ON(!writer_threads[0]);
 764         ssleep(SINGLE_WRITER_TEST_DURATION);
 765         kthread_stop(writer_threads[0]);
 766         printk("\n");
 767
 768         psrwlock_profile_latency_print();
 769 #endif
 770
 771 #if (NR_TRYLOCK_WRITERS)
 772         printk("** Single trylock writer test, no contention **\n");
 773         psrwlock_profile_latency_reset();
 774         trylock_writer_threads[0] = kthread_run(trylock_writer_thread,
 775                 (void *)0,
 776                 "trylock_psrwlock_writer");
 777         BUG_ON(!trylock_writer_threads[0]);
 778         ssleep(SINGLE_WRITER_TEST_DURATION);
 779         kthread_stop(trylock_writer_threads[0]);
 780         printk("\n");
 781
 782         psrwlock_profile_latency_print();
 783 #endif
 784
 785 #if (TEST_PREEMPT)
 786         printk("** Single preemptable reader test, no contention **\n");
 787         psrwlock_profile_latency_reset();
 788         preader_threads[0] = kthread_run(preader_thread, (void *)0,
 789                 "psrwlock_preader");
 790         BUG_ON(!preader_threads[0]);
 791         ssleep(SINGLE_READER_TEST_DURATION);
 792         kthread_stop(preader_threads[0]);
 793         printk("\n");
 794
 795         psrwlock_profile_latency_print();
 796 #endif
 797
 798         printk("** Single non-preemptable reader test, no contention **\n");
 799         psrwlock_profile_latency_reset();
 800         npreader_threads[0] = kthread_run(npreader_thread, (void *)0,
 801                 "psrwlock_npreader");
 802         BUG_ON(!npreader_threads[0]);
 803         ssleep(SINGLE_READER_TEST_DURATION);
 804         kthread_stop(npreader_threads[0]);
 805         printk("\n");
 806
 807         psrwlock_profile_latency_print();
 808
 809         printk("** Multiple p/non-p readers test, no contention **\n");
 810         psrwlock_profile_latency_reset();
 811 #if (TEST_PREEMPT)
 812         for (i = 0; i < NR_PREADERS; i++) {
 813                 printk("starting preader thread %lu\n", i);
 814                 preader_threads[i] = kthread_run(preader_thread, (void *)i,
 815                         "psrwlock_preader");
 816                 BUG_ON(!preader_threads[i]);
 817         }
 818 #endif
 819         for (i = 0; i < NR_NPREADERS; i++) {
 820                 printk("starting npreader thread %lu\n", i);
 821                 npreader_threads[i] = kthread_run(npreader_thread, (void *)i,
 822                         "psrwlock_npreader");
 823                 BUG_ON(!npreader_threads[i]);
 824         }
 825         ssleep(SINGLE_READER_TEST_DURATION);
 826         for (i = 0; i < NR_NPREADERS; i++)
 827                 kthread_stop(npreader_threads[i]);
 828 #if (TEST_PREEMPT)
 829         for (i = 0; i < NR_PREADERS; i++)
 830                 kthread_stop(preader_threads[i]);
 831 #endif
 832         printk("\n");
 833
 834         psrwlock_profile_latency_print();
 835
 836         printk("** High contention test **\n");
 837         psrwlock_profile_latency_reset();
 838         perform_test("psrwlock-create", psrwlock_create);
 839         ssleep(TEST_DURATION);
 840         perform_test("psrwlock-stop", psrwlock_stop);
 841         printk("\n");
 842         psrwlock_profile_latency_print();
 843
 844         return -EPERM;
 845 }
 846
 847
 848 static struct file_operations my_operations = {
 849         .open = my_open,
 850 };
 851
 852 int init_module(void)
 853 {
 854         pentry = create_proc_entry("testpsrwlock", 0444, NULL);
 855         if (pentry)
 856                 pentry->proc_fops = &my_operations;
 857
 858         printk("UC_READER_MASK   :         %08X\n", UC_READER_MASK);
 859         printk("UC_HARDIRQ_R_MASK:         %08X\n", UC_HARDIRQ_READER_MASK);
 860         printk("UC_SOFTIRQ_R_MASK:         %08X\n", UC_SOFTIRQ_READER_MASK);
 861         printk("UC_NPTHREA_R_MASK:         %08X\n", UC_NPTHREAD_READER_MASK);
 862         printk("UC_PTHREAD_R_MASK:         %08X\n", UC_PTHREAD_READER_MASK);
 863         printk("UC_WRITER        :         %08X\n", UC_WRITER);
 864         printk("UC_SLOW_WRITER   :         %08X\n", UC_SLOW_WRITER);
 865         printk("UC_WQ_ACTIVE     :         %08X\n", UC_WQ_ACTIVE);
 866         printk("WS_MASK          :         %08X\n", WS_MASK);
 867         printk("WS_WQ_MUTEX      :         %08X\n", WS_WQ_MUTEX);
 868         printk("WS_COUNT_MUTEX   :         %08X\n", WS_COUNT_MUTEX);
 869         printk("WS_LOCK_MUTEX    :         %08X\n", WS_LOCK_MUTEX);
 870         printk("CTX_RMASK        : %016lX\n", CTX_RMASK);
 871         printk("CTX_WMASK        : %016lX\n", CTX_WMASK);
 872
 873         return 0;
 874 }
 875
 876 void cleanup_module(void)
 877 {
 878         remove_proc_entry("testpsrwlock", NULL);
 879 }
 880
 881 MODULE_LICENSE("GPL");
 882 MODULE_AUTHOR("Mathieu Desnoyers");
 883 MODULE_DESCRIPTION("psrwlock test");