lttv/lttv/sync/event_matching_tcp.c

   1 /* This file is part of the Linux Trace Toolkit viewer
   2  * Copyright (C) 2009 Benjamin Poirier <benjamin.poirier@polymtl.ca>
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License Version 2 as
   6  * published by the Free Software Foundation;
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston,
  16  * MA 02111-1307, USA.
  17  */
  18
  19 #ifdef HAVE_CONFIG_H
  20 #include <config.h>
  21 #endif
  22
  23 #include <errno.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <unistd.h>
  27
  28 #include "event_analysis.h"
  29 #include "sync_chain.h"
  30
  31 #include "event_matching_tcp.h"
  32
  33
  34 #ifndef g_info
  35 #define g_info(format...) g_log (G_LOG_DOMAIN, G_LOG_LEVEL_INFO, format)
  36 #endif
  37
  38
  39 // Functions common to all matching modules
  40 static void initMatchingTCP(SyncState* const syncState);
  41 static void destroyMatchingTCP(SyncState* const syncState);
  42
  43 static void matchEventTCP(SyncState* const syncState, Event* const event);
  44 static GArray* finalizeMatchingTCP(SyncState* const syncState);
  45 static void printMatchingStatsTCP(SyncState* const syncState);
  46 static void writeMatchingGraphsPlotsTCP(SyncState* const syncState, const
  47         unsigned int i, const unsigned int j);
  48
  49 // Functions specific to this module
  50 static void registerMatchingTCP() __attribute__((constructor (101)));
  51
  52 static void matchEvents(SyncState* const syncState, Event* const event,
  53         GHashTable* const unMatchedList, GHashTable* const
  54         unMatchedOppositeList, const size_t fieldOffset, const size_t
  55         oppositeFieldOffset);
  56 static void partialDestroyMatchingTCP(SyncState* const syncState);
  57
  58 static bool isAck(const Message* const message);
  59 static bool needsAck(const Message* const message);
  60 static void buildReversedConnectionKey(ConnectionKey* const
  61         reversedConnectionKey, const ConnectionKey* const connectionKey);
  62
  63 static void openGraphDataFiles(SyncState* const syncState);
  64 static void closeGraphDataFiles(SyncState* const syncState);
  65 static void writeMessagePoint(FILE* stream, const Message* const message);
  66
  67
  68 static MatchingModule matchingModuleTCP = {
  69         .name= "TCP",
  70         .canMatch[TCP]= true,
  71         .canMatch[UDP]= false,
  72         .initMatching= &initMatchingTCP,
  73         .destroyMatching= &destroyMatchingTCP,
  74         .matchEvent= &matchEventTCP,
  75         .finalizeMatching= &finalizeMatchingTCP,
  76         .printMatchingStats= &printMatchingStatsTCP,
  77         .graphFunctions= {
  78                 .writeTraceTracePlots= &writeMatchingGraphsPlotsTCP,
  79         }
  80 };
  81
  82
  83 /*
  84  * Matching module registering function
  85  */
  86 static void registerMatchingTCP()
  87 {
  88         g_queue_push_tail(&matchingModules, &matchingModuleTCP);
  89 }
  90
  91
  92 /*
  93  * Matching init function
  94  *
  95  * This function is called at the beginning of a synchronization run for a set
  96  * of traces.
  97  *
  98  * Allocate the matching specific data structures
  99  *
 100  * Args:
 101  *   syncState     container for synchronization data.
 102  *                 This function allocates these matchingData members:
 103  *                 unMatchedInE
 104  *                 unMatchedOutE
 105  *                 unAcked
 106  *                 stats
 107  */
 108 static void initMatchingTCP(SyncState* const syncState)
 109 {
 110         MatchingDataTCP* matchingData;
 111
 112         matchingData= malloc(sizeof(MatchingDataTCP));
 113         syncState->matchingData= matchingData;
 114
 115         matchingData->unMatchedInE= g_hash_table_new_full(&ghfSegmentKeyHash,
 116                 &gefSegmentKeyEqual, NULL, &gdnDestroyEvent);
 117         matchingData->unMatchedOutE= g_hash_table_new_full(&ghfSegmentKeyHash,
 118                 &gefSegmentKeyEqual, NULL, &gdnDestroyEvent);
 119         matchingData->unAcked= g_hash_table_new_full(&ghfConnectionKeyHash,
 120                 &gefConnectionKeyEqual, &gdnConnectionKeyDestroy,
 121                 &gdnTCPSegmentListDestroy);
 122
 123         if (syncState->stats)
 124         {
 125                 unsigned int i;
 126
 127                 matchingData->stats= calloc(1, sizeof(MatchingStatsTCP));
 128                 matchingData->stats->totMessageArray= malloc(syncState->traceNb *
 129                         sizeof(unsigned int*));
 130                 for (i= 0; i < syncState->traceNb; i++)
 131                 {
 132                         matchingData->stats->totMessageArray[i]=
 133                                 calloc(syncState->traceNb, sizeof(unsigned int));
 134                 }
 135         }
 136         else
 137         {
 138                 matchingData->stats= NULL;
 139         }
 140
 141         if (syncState->graphsStream)
 142         {
 143                 openGraphDataFiles(syncState);
 144         }
 145         else
 146         {
 147                 matchingData->messagePoints= NULL;
 148         }
 149 }
 150
 151
 152 /*
 153  * Matching destroy function
 154  *
 155  * Free the matching specific data structures
 156  *
 157  * Args:
 158  *   syncState     container for synchronization data.
 159  *                 This function deallocates these matchingData members:
 160  *                 stats
 161  */
 162 static void destroyMatchingTCP(SyncState* const syncState)
 163 {
 164         MatchingDataTCP* matchingData;
 165
 166         matchingData= (MatchingDataTCP*) syncState->matchingData;
 167
 168         if (matchingData == NULL)
 169         {
 170                 return;
 171         }
 172
 173         partialDestroyMatchingTCP(syncState);
 174
 175         if (syncState->stats)
 176         {
 177                 unsigned int i;
 178
 179                 for (i= 0; i < syncState->traceNb; i++)
 180                 {
 181                         free(matchingData->stats->totMessageArray[i]);
 182                 }
 183                 free(matchingData->stats->totMessageArray);
 184                 free(matchingData->stats);
 185         }
 186
 187         free(syncState->matchingData);
 188         syncState->matchingData= NULL;
 189 }
 190
 191
 192 /*
 193  * Free some of the matching specific data structures
 194  *
 195  * This function can be called right after the events have been processed to
 196  * free some data structures that are not needed for finalization.
 197  *
 198  * Args:
 199  *   syncState     container for synchronization data.
 200  *                 This function deallocates these matchingData members:
 201  *                 unMatchedInE
 202  *                 unMatchedOut
 203  *                 unAcked
 204  */
 205 static void partialDestroyMatchingTCP(SyncState* const syncState)
 206 {
 207         MatchingDataTCP* matchingData;
 208
 209         matchingData= (MatchingDataTCP*) syncState->matchingData;
 210
 211         if (matchingData == NULL || matchingData->unMatchedInE == NULL)
 212         {
 213                 return;
 214         }
 215
 216         g_hash_table_destroy(matchingData->unMatchedInE);
 217         matchingData->unMatchedInE= NULL;
 218         g_hash_table_destroy(matchingData->unMatchedOutE);
 219         g_hash_table_destroy(matchingData->unAcked);
 220
 221         if (syncState->graphsStream && matchingData->messagePoints)
 222         {
 223                 closeGraphDataFiles(syncState);
 224         }
 225 }
 226
 227
 228 /*
 229  * Try to match one event from a trace with the corresponding event from
 230  * another trace.
 231  *
 232  * Args:
 233  *   syncState     container for synchronization data.
 234  *   event         new event to match
 235  */
 236 static void matchEventTCP(SyncState* const syncState, Event* const event)
 237 {
 238         MatchingDataTCP* matchingData;
 239
 240         g_assert(event->type == TCP);
 241
 242         matchingData= (MatchingDataTCP*) syncState->matchingData;
 243
 244         if (event->event.tcpEvent->direction == IN)
 245         {
 246                 matchEvents(syncState, event, matchingData->unMatchedInE,
 247                         matchingData->unMatchedOutE, offsetof(Message, inE),
 248                         offsetof(Message, outE));
 249         }
 250         else
 251         {
 252                 matchEvents(syncState, event, matchingData->unMatchedOutE,
 253                         matchingData->unMatchedInE, offsetof(Message, outE),
 254                         offsetof(Message, inE));
 255         }
 256 }
 257
 258
 259 /*
 260  * Call the partial matching destroyer and Obtain the factors from downstream
 261  *
 262  * Args:
 263  *   syncState     container for synchronization data.
 264  *
 265  * Returns:
 266  *   Factors[traceNb] synchronization factors for each trace
 267  */
 268 static GArray* finalizeMatchingTCP(SyncState* const syncState)
 269 {
 270         partialDestroyMatchingTCP(syncState);
 271
 272         return syncState->analysisModule->finalizeAnalysis(syncState);
 273 }
 274
 275
 276 /*
 277  * Print statistics related to matching. Must be called after
 278  * finalizeMatching.
 279  *
 280  * Args:
 281  *   syncState     container for synchronization data.
 282  */
 283 static void printMatchingStatsTCP(SyncState* const syncState)
 284 {
 285         unsigned int i, j;
 286         MatchingDataTCP* matchingData;
 287
 288         if (!syncState->stats)
 289         {
 290                 return;
 291         }
 292
 293         matchingData= (MatchingDataTCP*) syncState->matchingData;
 294
 295         printf("TCP matching stats:\n");
 296         printf("\ttotal input and output events matched together to form a packet: %u\n",
 297                 matchingData->stats->totPacket);
 298
 299         printf("\tMessage traffic:\n");
 300
 301         for (i= 0; i < syncState->traceNb; i++)
 302         {
 303                 for (j= i + 1; j < syncState->traceNb; j++)
 304                 {
 305                         printf("\t\t%3d - %-3d: sent %-10u received %-10u\n", i, j,
 306                                 matchingData->stats->totMessageArray[j][i],
 307                                 matchingData->stats->totMessageArray[i][j]);
 308                 }
 309         }
 310
 311         if (syncState->analysisModule->analyzeExchange != NULL)
 312         {
 313                 printf("\ttotal packets identified needing an acknowledge: %u\n",
 314                         matchingData->stats->totPacketNeedAck);
 315                 printf("\ttotal exchanges (four events matched together): %u\n",
 316                         matchingData->stats->totExchangeEffective);
 317                 printf("\ttotal synchronization exchanges: %u\n",
 318                         matchingData->stats->totExchangeSync);
 319         }
 320 }
 321
 322
 323 /*
 324  * Implementation of a packet matching algorithm for TCP
 325  *
 326  * Args:
 327  *   event:        new event to match
 328  *   unMatchedList: list of unmatched events of the same type (send or
 329  *                 receive) as event
 330  *   unMatchedOppositeList: list of unmatched events of the opposite type of
 331  *                 event
 332  *   fieldOffset:  offset of the Event field in the Message struct for the
 333  *                 field of the type of event
 334  *   oppositeFieldOffset: offset of the Event field in the Message struct
 335  *                 for the field of the opposite type of event
 336  */
 337 static void matchEvents(SyncState* const syncState, Event* const event,
 338         GHashTable* const unMatchedList, GHashTable* const unMatchedOppositeList,
 339         const size_t fieldOffset, const size_t oppositeFieldOffset)
 340 {
 341         Event* companionEvent;
 342         Message* packet;
 343         MatchingDataTCP* matchingData;
 344         GQueue* conUnAcked;
 345
 346         matchingData= (MatchingDataTCP*) syncState->matchingData;
 347
 348         companionEvent= g_hash_table_lookup(unMatchedOppositeList, event->event.tcpEvent->segmentKey);
 349         if (companionEvent != NULL)
 350         {
 351                 g_debug("Found matching companion event, ");
 352
 353                 // If it's there, remove it and create a Message
 354                 g_hash_table_steal(unMatchedOppositeList, event->event.tcpEvent->segmentKey);
 355                 packet= malloc(sizeof(Message));
 356                 *((Event**) ((void*) packet + fieldOffset))= event;
 357                 *((Event**) ((void*) packet + oppositeFieldOffset))= companionEvent;
 358                 packet->print= &printTCPSegment;
 359                 // Both events can now share the same segmentKey
 360                 free(packet->outE->event.tcpEvent->segmentKey);
 361                 packet->outE->event.tcpEvent->segmentKey= packet->inE->event.tcpEvent->segmentKey;
 362
 363                 if (syncState->stats)
 364                 {
 365                         matchingData->stats->totPacket++;
 366                         matchingData->stats->totMessageArray[packet->inE->traceNum][packet->outE->traceNum]++;
 367                 }
 368
 369                 // Discard loopback traffic
 370                 if (packet->inE->traceNum == packet->outE->traceNum)
 371                 {
 372                         destroyTCPSegment(packet);
 373                         return;
 374                 }
 375
 376                 if (syncState->graphsStream)
 377                 {
 378                         writeMessagePoint(matchingData->messagePoints[packet->inE->traceNum][packet->outE->traceNum],
 379                                 packet);
 380                 }
 381
 382                 if (syncState->analysisModule->analyzeMessage != NULL)
 383                 {
 384                         syncState->analysisModule->analyzeMessage(syncState, packet);
 385                 }
 386
 387                 // We can skip the rest of the algorithm if the analysis module is not
 388                 // interested in exchanges
 389                 if (syncState->analysisModule->analyzeExchange == NULL)
 390                 {
 391                         destroyTCPSegment(packet);
 392                         return;
 393                 }
 394
 395                 // If this packet acknowleges some data ...
 396                 if (isAck(packet))
 397                 {
 398                         ConnectionKey oppositeConnectionKey;
 399
 400                         buildReversedConnectionKey(&oppositeConnectionKey,
 401                                 &event->event.tcpEvent->segmentKey->connectionKey);
 402                         conUnAcked= g_hash_table_lookup(matchingData->unAcked,
 403                                 &oppositeConnectionKey);
 404                         if (conUnAcked != NULL)
 405                         {
 406                                 Message* ackedPacket;
 407                                 GList* result;
 408                                 Exchange* exchange;
 409
 410                                 exchange= NULL;
 411
 412                                 result= g_queue_find_custom(conUnAcked, packet, &gcfTCPSegmentAckCompare);
 413
 414                                 while (result != NULL)
 415                                 {
 416                                         // Remove the acknowledged packet from the unAcked list
 417                                         // and keep it for later offset calculations
 418                                         g_debug("Found matching unAcked packet, ");
 419
 420                                         ackedPacket= (Message*) result->data;
 421                                         g_queue_delete_link(conUnAcked, result);
 422
 423                                         if (syncState->stats)
 424                                         {
 425                                                 matchingData->stats->totExchangeEffective++;
 426                                         }
 427
 428                                         if (exchange == NULL)
 429                                         {
 430                                                 exchange= malloc(sizeof(Exchange));
 431                                                 exchange->message= packet;
 432                                                 exchange->acks= g_queue_new();
 433                                         }
 434
 435                                         g_queue_push_tail(exchange->acks, ackedPacket);
 436
 437                                         result= g_queue_find_custom(conUnAcked, packet,
 438                                                 &gcfTCPSegmentAckCompare);
 439                                 }
 440
 441                                 // It might be possible to do an offset calculation
 442                                 if (exchange != NULL)
 443                                 {
 444                                         ackedPacket= g_queue_peek_tail(exchange->acks);
 445                                         if (ackedPacket->outE->traceNum != packet->inE->traceNum
 446                                                 || ackedPacket->inE->traceNum !=
 447                                                 packet->outE->traceNum || packet->inE->traceNum ==
 448                                                 packet->outE->traceNum)
 449                                         {
 450                                                 ackedPacket->print(ackedPacket);
 451                                                 packet->print(packet);
 452                                                 g_error("Disorganized exchange encountered during "
 453                                                         "synchronization");
 454                                         }
 455                                         else
 456                                         {
 457                                                 if (syncState->stats)
 458                                                 {
 459                                                         matchingData->stats->totExchangeSync++;
 460                                                 }
 461
 462                                                 syncState->analysisModule->analyzeExchange(syncState,
 463                                                         exchange);
 464                                         }
 465
 466                                         exchange->message= NULL;
 467                                         destroyTCPExchange(exchange);
 468                                 }
 469                         }
 470                 }
 471
 472                 if (needsAck(packet))
 473                 {
 474                         if (syncState->stats)
 475                         {
 476                                 matchingData->stats->totPacketNeedAck++;
 477                         }
 478
 479                         // If this packet will generate an ack, add it to the unAcked list
 480                         g_debug("Adding to unAcked, ");
 481                         conUnAcked= g_hash_table_lookup(matchingData->unAcked,
 482                                 &event->event.tcpEvent->segmentKey->connectionKey);
 483                         if (conUnAcked == NULL)
 484                         {
 485                                 ConnectionKey* connectionKey;
 486
 487                                 connectionKey= malloc(sizeof(ConnectionKey));
 488                                 memcpy(connectionKey, &event->event.tcpEvent->segmentKey->connectionKey,
 489                                         sizeof(ConnectionKey));
 490                                 g_hash_table_insert(matchingData->unAcked, connectionKey,
 491                                         conUnAcked= g_queue_new());
 492                         }
 493                         g_queue_push_tail(conUnAcked, packet);
 494                 }
 495                 else
 496                 {
 497                         destroyTCPSegment(packet);
 498                 }
 499         }
 500         else
 501         {
 502                 // If there's no corresponding event, add the event to the unmatched
 503                 // list for this type of event
 504                 g_debug("Adding to unmatched event list, ");
 505                 g_hash_table_replace(unMatchedList, event->event.tcpEvent->segmentKey, event);
 506         }
 507 }
 508
 509
 510 /*
 511  * Check if a packet is an acknowledge
 512  *
 513  * Args:
 514  *   packet        TCP Message
 515  *
 516  * Returns:
 517  *   true if it is,
 518  *   false otherwise
 519  */
 520 static bool isAck(const Message* const packet)
 521 {
 522         if (packet->inE->event.tcpEvent->segmentKey->ack == 1)
 523         {
 524                 return true;
 525         }
 526         else
 527         {
 528                 return false;
 529         }
 530 }
 531
 532
 533 /*
 534  * Check if a packet will increment the sequence number, thus needing an
 535  * acknowledge
 536  *
 537  * Args:
 538  *   packet        TCP Message
 539  *
 540  * Returns:
 541  *   true if the packet will need an acknowledge
 542  *   false otherwise
 543  */
 544 static bool needsAck(const Message* const packet)
 545 {
 546         if (packet->inE->event.tcpEvent->segmentKey->syn || packet->inE->event.tcpEvent->segmentKey->fin ||
 547                 packet->inE->event.tcpEvent->segmentKey->tot_len - packet->inE->event.tcpEvent->segmentKey->ihl * 4 -
 548                 packet->inE->event.tcpEvent->segmentKey->doff * 4 > 0)
 549         {
 550                 return true;
 551         }
 552         else
 553         {
 554                 return false;
 555         }
 556 }
 557
 558
 559 /*
 560  * Populate a connection key structure for the opposite direction of a
 561  * connection
 562  *
 563  * Args:
 564  *   reversedConnectionKey the result, must be pre-allocated
 565  *   connectionKey the connection key to reverse
 566  */
 567 static void buildReversedConnectionKey(ConnectionKey* const
 568         reversedConnectionKey, const ConnectionKey* const connectionKey)
 569 {
 570         reversedConnectionKey->saddr= connectionKey->daddr;
 571         reversedConnectionKey->daddr= connectionKey->saddr;
 572         reversedConnectionKey->source= connectionKey->dest;
 573         reversedConnectionKey->dest= connectionKey->source;
 574 }
 575
 576
 577 /*
 578  * Create and open files used to store message points to genereate
 579  * graphs. Allocate and populate array to store file pointers.
 580  *
 581  * Args:
 582  *   syncState:    container for synchronization data
 583  */
 584 static void openGraphDataFiles(SyncState* const syncState)
 585 {
 586         unsigned int i, j;
 587         int retval;
 588         char* cwd;
 589         char name[29];
 590         MatchingDataTCP* matchingData;
 591
 592         matchingData= (MatchingDataTCP*) syncState->matchingData;
 593
 594         cwd= changeToGraphDir(syncState->graphsDir);
 595
 596         matchingData->messagePoints= malloc(syncState->traceNb * sizeof(FILE**));
 597         for (i= 0; i < syncState->traceNb; i++)
 598         {
 599                 matchingData->messagePoints[i]= malloc(syncState->traceNb *
 600                         sizeof(FILE*));
 601                 for (j= 0; j < syncState->traceNb; j++)
 602                 {
 603                         if (i != j)
 604                         {
 605                                 retval= snprintf(name, sizeof(name),
 606                                         "matching_tcp-%03u_to_%03u.data", j, i);
 607                                 if (retval > sizeof(name) - 1)
 608                                 {
 609                                         name[sizeof(name) - 1]= '\0';
 610                                 }
 611                                 if ((matchingData->messagePoints[i][j]= fopen(name, "w")) ==
 612                                         NULL)
 613                                 {
 614                                         g_error(strerror(errno));
 615                                 }
 616                         }
 617                 }
 618         }
 619
 620         retval= chdir(cwd);
 621         if (retval == -1)
 622         {
 623                 g_error(strerror(errno));
 624         }
 625         free(cwd);
 626 }
 627
 628
 629 /*
 630  * Write a message point to a file used to generate graphs
 631  *
 632  * Args:
 633  *   stream:       FILE*, file pointer where to write the point
 634  *   message:      message for which to write the point
 635  */
 636 static void writeMessagePoint(FILE* stream, const Message* const message)
 637 {
 638         uint64_t x, y;
 639
 640         if (message->inE->traceNum < message->outE->traceNum)
 641         {
 642                 // CA is inE->traceNum
 643                 x= message->inE->cpuTime;
 644                 y= message->outE->cpuTime;
 645         }
 646         else
 647         {
 648                 // CA is outE->traceNum
 649                 x= message->outE->cpuTime;
 650                 y= message->inE->cpuTime;
 651         }
 652
 653         fprintf(stream, "%20llu %20llu\n", x, y);
 654 }
 655
 656
 657 /*
 658  * Close files used to store convex hull points to genereate graphs.
 659  * Deallocate array to store file pointers.
 660  *
 661  * Args:
 662  *   syncState:    container for synchronization data
 663  */
 664 static void closeGraphDataFiles(SyncState* const syncState)
 665 {
 666         unsigned int i, j;
 667         MatchingDataTCP* matchingData;
 668         int retval;
 669
 670         matchingData= (MatchingDataTCP*) syncState->matchingData;
 671
 672         if (matchingData->messagePoints == NULL)
 673         {
 674                 return;
 675         }
 676
 677         for (i= 0; i < syncState->traceNb; i++)
 678         {
 679                 for (j= 0; j < syncState->traceNb; j++)
 680                 {
 681                         if (i != j)
 682                         {
 683                                 retval= fclose(matchingData->messagePoints[i][j]);
 684                                 if (retval != 0)
 685                                 {
 686                                         g_error(strerror(errno));
 687                                 }
 688                         }
 689                 }
 690                 free(matchingData->messagePoints[i]);
 691         }
 692         free(matchingData->messagePoints);
 693
 694         matchingData->messagePoints= NULL;
 695 }
 696
 697
 698 /*
 699  * Write the matching-specific graph lines in the gnuplot script.
 700  *
 701  * Args:
 702  *   syncState:    container for synchronization data
 703  *   i:            first trace number
 704  *   j:            second trace number, garanteed to be larger than i
 705  */
 706 static void writeMatchingGraphsPlotsTCP(SyncState* const syncState, const
 707         unsigned int i, const unsigned int j)
 708 {
 709         fprintf(syncState->graphsStream,
 710                 "\t\"matching_tcp-%1$03d_to_%2$03d.data\" "
 711                         "title \"Sent messages\" with points linetype 4 "
 712                         "linecolor rgb \"#98fc66\" pointtype 9 pointsize 2, \\\n"
 713                 "\t\"matching_tcp-%2$03d_to_%1$03d.data\" "
 714                         "title \"Received messages\" with points linetype 4 "
 715                         "linecolor rgb \"#6699cc\" pointtype 11 pointsize 2, \\\n", i, j);
 716 }