lttv/lttv/sync/event_matching_tcp.c

   1 /* This file is part of the Linux Trace Toolkit viewer
   2  * Copyright (C) 2009 Benjamin Poirier <benjamin.poirier@polymtl.ca>
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License Version 2 as
   6  * published by the Free Software Foundation;
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston,
  16  * MA 02111-1307, USA.
  17  */
  18
  19 #ifdef HAVE_CONFIG_H
  20 #include <config.h>
  21 #endif
  22
  23 #include <errno.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <unistd.h>
  27
  28 #include "event_analysis.h"
  29 #include "sync_chain.h"
  30
  31 #include "event_matching_tcp.h"
  32
  33
  34 #ifndef g_info
  35 #define g_info(format...) g_log (G_LOG_DOMAIN, G_LOG_LEVEL_INFO, format)
  36 #endif
  37
  38
  39 // Functions common to all matching modules
  40 static void initMatchingTCP(SyncState* const syncState);
  41 static void destroyMatchingTCP(SyncState* const syncState);
  42
  43 static void matchEventTCP(SyncState* const syncState, Event* const event);
  44 static GArray* finalizeMatchingTCP(SyncState* const syncState);
  45 static void printMatchingStatsTCP(SyncState* const syncState);
  46 static void writeMatchingGraphsPlotsTCP(FILE* stream, SyncState* const
  47         syncState, const unsigned int i, const unsigned int j);
  48
  49 // Functions specific to this module
  50 static void registerMatchingTCP() __attribute__((constructor (101)));
  51
  52 static void matchEvents(SyncState* const syncState, Event* const event,
  53         GHashTable* const unMatchedList, GHashTable* const
  54         unMatchedOppositeList, const size_t fieldOffset, const size_t
  55         oppositeFieldOffset);
  56 static void partialDestroyMatchingTCP(SyncState* const syncState);
  57
  58 static bool isAck(const Message* const message);
  59 static bool needsAck(const Message* const message);
  60 static void buildReversedConnectionKey(ConnectionKey* const
  61         reversedConnectionKey, const ConnectionKey* const connectionKey);
  62
  63 static void openGraphDataFiles(SyncState* const syncState);
  64 static void closeGraphDataFiles(SyncState* const syncState);
  65 static void writeMessagePoint(FILE* stream, const Message* const message);
  66
  67
  68 static MatchingModule matchingModuleTCP = {
  69         .name= "TCP",
  70         .canMatch[TCP]= true,
  71         .canMatch[UDP]= false,
  72         .initMatching= &initMatchingTCP,
  73         .destroyMatching= &destroyMatchingTCP,
  74         .matchEvent= &matchEventTCP,
  75         .finalizeMatching= &finalizeMatchingTCP,
  76         .printMatchingStats= &printMatchingStatsTCP,
  77         .writeMatchingGraphsPlots= &writeMatchingGraphsPlotsTCP,
  78         .writeMatchingGraphsOptions= NULL,
  79 };
  80
  81
  82 /*
  83  * Matching module registering function
  84  */
  85 static void registerMatchingTCP()
  86 {
  87         g_queue_push_tail(&matchingModules, &matchingModuleTCP);
  88 }
  89
  90
  91 /*
  92  * Matching init function
  93  *
  94  * This function is called at the beginning of a synchronization run for a set
  95  * of traces.
  96  *
  97  * Allocate the matching specific data structures
  98  *
  99  * Args:
 100  *   syncState     container for synchronization data.
 101  *                 This function allocates these matchingData members:
 102  *                 unMatchedInE
 103  *                 unMatchedOutE
 104  *                 unAcked
 105  *                 stats
 106  */
 107 static void initMatchingTCP(SyncState* const syncState)
 108 {
 109         MatchingDataTCP* matchingData;
 110
 111         matchingData= malloc(sizeof(MatchingDataTCP));
 112         syncState->matchingData= matchingData;
 113
 114         matchingData->unMatchedInE= g_hash_table_new_full(&ghfSegmentKeyHash,
 115                 &gefSegmentKeyEqual, NULL, &gdnDestroyEvent);
 116         matchingData->unMatchedOutE= g_hash_table_new_full(&ghfSegmentKeyHash,
 117                 &gefSegmentKeyEqual, NULL, &gdnDestroyEvent);
 118         matchingData->unAcked= g_hash_table_new_full(&ghfConnectionKeyHash,
 119                 &gefConnectionKeyEqual, &gdnConnectionKeyDestroy,
 120                 &gdnTCPSegmentListDestroy);
 121
 122         if (syncState->stats)
 123         {
 124                 unsigned int i;
 125
 126                 matchingData->stats= calloc(1, sizeof(MatchingStatsTCP));
 127                 matchingData->stats->totMessageArray= malloc(syncState->traceNb *
 128                         sizeof(unsigned int*));
 129                 for (i= 0; i < syncState->traceNb; i++)
 130                 {
 131                         matchingData->stats->totMessageArray[i]=
 132                                 calloc(syncState->traceNb, sizeof(unsigned int));
 133                 }
 134         }
 135         else
 136         {
 137                 matchingData->stats= NULL;
 138         }
 139
 140         if (syncState->graphs)
 141         {
 142                 openGraphDataFiles(syncState);
 143         }
 144         else
 145         {
 146                 matchingData->messagePoints= NULL;
 147         }
 148 }
 149
 150
 151 /*
 152  * Matching destroy function
 153  *
 154  * Free the matching specific data structures
 155  *
 156  * Args:
 157  *   syncState     container for synchronization data.
 158  *                 This function deallocates these matchingData members:
 159  *                 stats
 160  */
 161 static void destroyMatchingTCP(SyncState* const syncState)
 162 {
 163         MatchingDataTCP* matchingData;
 164
 165         matchingData= (MatchingDataTCP*) syncState->matchingData;
 166
 167         if (matchingData == NULL)
 168         {
 169                 return;
 170         }
 171
 172         partialDestroyMatchingTCP(syncState);
 173
 174         if (syncState->stats)
 175         {
 176                 unsigned int i;
 177
 178                 for (i= 0; i < syncState->traceNb; i++)
 179                 {
 180                         free(matchingData->stats->totMessageArray[i]);
 181                 }
 182                 free(matchingData->stats->totMessageArray);
 183                 free(matchingData->stats);
 184         }
 185
 186         free(syncState->matchingData);
 187         syncState->matchingData= NULL;
 188 }
 189
 190
 191 /*
 192  * Free some of the matching specific data structures
 193  *
 194  * This function can be called right after the events have been processed to
 195  * free some data structures that are not needed for finalization.
 196  *
 197  * Args:
 198  *   syncState     container for synchronization data.
 199  *                 This function deallocates these matchingData members:
 200  *                 unMatchedInE
 201  *                 unMatchedOut
 202  *                 unAcked
 203  */
 204 static void partialDestroyMatchingTCP(SyncState* const syncState)
 205 {
 206         MatchingDataTCP* matchingData;
 207
 208         matchingData= (MatchingDataTCP*) syncState->matchingData;
 209
 210         if (matchingData == NULL || matchingData->unMatchedInE == NULL)
 211         {
 212                 return;
 213         }
 214
 215         g_hash_table_destroy(matchingData->unMatchedInE);
 216         matchingData->unMatchedInE= NULL;
 217         g_hash_table_destroy(matchingData->unMatchedOutE);
 218         g_hash_table_destroy(matchingData->unAcked);
 219
 220         if (syncState->graphs && matchingData->messagePoints)
 221         {
 222                 closeGraphDataFiles(syncState);
 223         }
 224 }
 225
 226
 227 /*
 228  * Try to match one event from a trace with the corresponding event from
 229  * another trace.
 230  *
 231  * Args:
 232  *   syncState     container for synchronization data.
 233  *   event         new event to match
 234  */
 235 static void matchEventTCP(SyncState* const syncState, Event* const event)
 236 {
 237         MatchingDataTCP* matchingData;
 238
 239         g_assert(event->type == TCP);
 240
 241         matchingData= (MatchingDataTCP*) syncState->matchingData;
 242
 243         if (event->event.tcpEvent->direction == IN)
 244         {
 245                 matchEvents(syncState, event, matchingData->unMatchedInE,
 246                         matchingData->unMatchedOutE, offsetof(Message, inE),
 247                         offsetof(Message, outE));
 248         }
 249         else
 250         {
 251                 matchEvents(syncState, event, matchingData->unMatchedOutE,
 252                         matchingData->unMatchedInE, offsetof(Message, outE),
 253                         offsetof(Message, inE));
 254         }
 255 }
 256
 257
 258 /*
 259  * Call the partial matching destroyer and Obtain the factors from downstream
 260  *
 261  * Args:
 262  *   syncState     container for synchronization data.
 263  *
 264  * Returns:
 265  *   Factors[traceNb] synchronization factors for each trace
 266  */
 267 static GArray* finalizeMatchingTCP(SyncState* const syncState)
 268 {
 269         partialDestroyMatchingTCP(syncState);
 270
 271         return syncState->analysisModule->finalizeAnalysis(syncState);
 272 }
 273
 274
 275 /*
 276  * Print statistics related to matching. Must be called after
 277  * finalizeMatching.
 278  *
 279  * Args:
 280  *   syncState     container for synchronization data.
 281  */
 282 static void printMatchingStatsTCP(SyncState* const syncState)
 283 {
 284         unsigned int i, j;
 285         MatchingDataTCP* matchingData;
 286
 287         if (!syncState->stats)
 288         {
 289                 return;
 290         }
 291
 292         matchingData= (MatchingDataTCP*) syncState->matchingData;
 293
 294         printf("TCP matching stats:\n");
 295         printf("\ttotal input and output events matched together to form a packet: %u\n",
 296                 matchingData->stats->totPacket);
 297
 298         printf("\tMessage traffic:\n");
 299
 300         for (i= 0; i < syncState->traceNb; i++)
 301         {
 302                 for (j= i + 1; j < syncState->traceNb; j++)
 303                 {
 304                         printf("\t\t%3d - %-3d: sent %-10u received %-10u\n", i, j,
 305                                 matchingData->stats->totMessageArray[j][i],
 306                                 matchingData->stats->totMessageArray[i][j]);
 307                 }
 308         }
 309
 310         if (syncState->analysisModule->analyzeExchange != NULL)
 311         {
 312                 printf("\ttotal packets identified needing an acknowledge: %u\n",
 313                         matchingData->stats->totPacketNeedAck);
 314                 printf("\ttotal exchanges (four events matched together): %u\n",
 315                         matchingData->stats->totExchangeEffective);
 316                 printf("\ttotal synchronization exchanges: %u\n",
 317                         matchingData->stats->totExchangeSync);
 318         }
 319 }
 320
 321
 322 /*
 323  * Implementation of a packet matching algorithm for TCP
 324  *
 325  * Args:
 326  *   event:        new event to match
 327  *   unMatchedList: list of unmatched events of the same type (send or
 328  *                 receive) as event
 329  *   unMatchedOppositeList: list of unmatched events of the opposite type of
 330  *                 event
 331  *   fieldOffset:  offset of the Event field in the Message struct for the
 332  *                 field of the type of event
 333  *   oppositeFieldOffset: offset of the Event field in the Message struct
 334  *                 for the field of the opposite type of event
 335  */
 336 static void matchEvents(SyncState* const syncState, Event* const event,
 337         GHashTable* const unMatchedList, GHashTable* const unMatchedOppositeList,
 338         const size_t fieldOffset, const size_t oppositeFieldOffset)
 339 {
 340         Event* companionEvent;
 341         Message* packet;
 342         MatchingDataTCP* matchingData;
 343         GQueue* conUnAcked;
 344
 345         matchingData= (MatchingDataTCP*) syncState->matchingData;
 346
 347         companionEvent= g_hash_table_lookup(unMatchedOppositeList, event->event.tcpEvent->segmentKey);
 348         if (companionEvent != NULL)
 349         {
 350                 g_debug("Found matching companion event, ");
 351
 352                 // If it's there, remove it and create a Message
 353                 g_hash_table_steal(unMatchedOppositeList, event->event.tcpEvent->segmentKey);
 354                 packet= malloc(sizeof(Message));
 355                 *((Event**) ((void*) packet + fieldOffset))= event;
 356                 *((Event**) ((void*) packet + oppositeFieldOffset))= companionEvent;
 357                 packet->print= &printTCPSegment;
 358                 // Both events can now share the same segmentKey
 359                 free(packet->outE->event.tcpEvent->segmentKey);
 360                 packet->outE->event.tcpEvent->segmentKey= packet->inE->event.tcpEvent->segmentKey;
 361
 362                 if (syncState->stats)
 363                 {
 364                         matchingData->stats->totPacket++;
 365                         matchingData->stats->totMessageArray[packet->inE->traceNum][packet->outE->traceNum]++;
 366                 }
 367
 368                 // Discard loopback traffic
 369                 if (packet->inE->traceNum == packet->outE->traceNum)
 370                 {
 371                         destroyTCPSegment(packet);
 372                         return;
 373                 }
 374
 375                 if (syncState->graphs)
 376                 {
 377                         writeMessagePoint(matchingData->messagePoints[packet->inE->traceNum][packet->outE->traceNum],
 378                                 packet);
 379                 }
 380
 381                 if (syncState->analysisModule->analyzeMessage != NULL)
 382                 {
 383                         syncState->analysisModule->analyzeMessage(syncState, packet);
 384                 }
 385
 386                 // We can skip the rest of the algorithm if the analysis module is not
 387                 // interested in exchanges
 388                 if (syncState->analysisModule->analyzeExchange == NULL)
 389                 {
 390                         destroyTCPSegment(packet);
 391                         return;
 392                 }
 393
 394                 // If this packet acknowleges some data ...
 395                 if (isAck(packet))
 396                 {
 397                         ConnectionKey oppositeConnectionKey;
 398
 399                         buildReversedConnectionKey(&oppositeConnectionKey,
 400                                 &event->event.tcpEvent->segmentKey->connectionKey);
 401                         conUnAcked= g_hash_table_lookup(matchingData->unAcked,
 402                                 &oppositeConnectionKey);
 403                         if (conUnAcked != NULL)
 404                         {
 405                                 Message* ackedPacket;
 406                                 GList* result;
 407                                 Exchange* exchange;
 408
 409                                 exchange= NULL;
 410
 411                                 result= g_queue_find_custom(conUnAcked, packet, &gcfTCPSegmentAckCompare);
 412
 413                                 while (result != NULL)
 414                                 {
 415                                         // Remove the acknowledged packet from the unAcked list
 416                                         // and keep it for later offset calculations
 417                                         g_debug("Found matching unAcked packet, ");
 418
 419                                         ackedPacket= (Message*) result->data;
 420                                         g_queue_delete_link(conUnAcked, result);
 421
 422                                         if (syncState->stats)
 423                                         {
 424                                                 matchingData->stats->totExchangeEffective++;
 425                                         }
 426
 427                                         if (exchange == NULL)
 428                                         {
 429                                                 exchange= malloc(sizeof(Exchange));
 430                                                 exchange->message= packet;
 431                                                 exchange->acks= g_queue_new();
 432                                         }
 433
 434                                         g_queue_push_tail(exchange->acks, ackedPacket);
 435
 436                                         result= g_queue_find_custom(conUnAcked, packet,
 437                                                 &gcfTCPSegmentAckCompare);
 438                                 }
 439
 440                                 // It might be possible to do an offset calculation
 441                                 if (exchange != NULL)
 442                                 {
 443                                         ackedPacket= g_queue_peek_tail(exchange->acks);
 444                                         if (ackedPacket->outE->traceNum != packet->inE->traceNum
 445                                                 || ackedPacket->inE->traceNum !=
 446                                                 packet->outE->traceNum || packet->inE->traceNum ==
 447                                                 packet->outE->traceNum)
 448                                         {
 449                                                 ackedPacket->print(ackedPacket);
 450                                                 packet->print(packet);
 451                                                 g_error("Disorganized exchange encountered during "
 452                                                         "synchronization");
 453                                         }
 454                                         else
 455                                         {
 456                                                 if (syncState->stats)
 457                                                 {
 458                                                         matchingData->stats->totExchangeSync++;
 459                                                 }
 460
 461                                                 syncState->analysisModule->analyzeExchange(syncState,
 462                                                         exchange);
 463                                         }
 464
 465                                         exchange->message= NULL;
 466                                         destroyTCPExchange(exchange);
 467                                 }
 468                         }
 469                 }
 470
 471                 if (needsAck(packet))
 472                 {
 473                         if (syncState->stats)
 474                         {
 475                                 matchingData->stats->totPacketNeedAck++;
 476                         }
 477
 478                         // If this packet will generate an ack, add it to the unAcked list
 479                         g_debug("Adding to unAcked, ");
 480                         conUnAcked= g_hash_table_lookup(matchingData->unAcked,
 481                                 &event->event.tcpEvent->segmentKey->connectionKey);
 482                         if (conUnAcked == NULL)
 483                         {
 484                                 ConnectionKey* connectionKey;
 485
 486                                 connectionKey= malloc(sizeof(ConnectionKey));
 487                                 memcpy(connectionKey, &event->event.tcpEvent->segmentKey->connectionKey,
 488                                         sizeof(ConnectionKey));
 489                                 g_hash_table_insert(matchingData->unAcked, connectionKey,
 490                                         conUnAcked= g_queue_new());
 491                         }
 492                         g_queue_push_tail(conUnAcked, packet);
 493                 }
 494                 else
 495                 {
 496                         destroyTCPSegment(packet);
 497                 }
 498         }
 499         else
 500         {
 501                 // If there's no corresponding event, add the event to the unmatched
 502                 // list for this type of event
 503                 g_debug("Adding to unmatched event list, ");
 504                 g_hash_table_replace(unMatchedList, event->event.tcpEvent->segmentKey, event);
 505         }
 506 }
 507
 508
 509 /*
 510  * Check if a packet is an acknowledge
 511  *
 512  * Args:
 513  *   packet        TCP Message
 514  *
 515  * Returns:
 516  *   true if it is,
 517  *   false otherwise
 518  */
 519 static bool isAck(const Message* const packet)
 520 {
 521         if (packet->inE->event.tcpEvent->segmentKey->ack == 1)
 522         {
 523                 return true;
 524         }
 525         else
 526         {
 527                 return false;
 528         }
 529 }
 530
 531
 532 /*
 533  * Check if a packet will increment the sequence number, thus needing an
 534  * acknowledge
 535  *
 536  * Args:
 537  *   packet        TCP Message
 538  *
 539  * Returns:
 540  *   true if the packet will need an acknowledge
 541  *   false otherwise
 542  */
 543 static bool needsAck(const Message* const packet)
 544 {
 545         if (packet->inE->event.tcpEvent->segmentKey->syn || packet->inE->event.tcpEvent->segmentKey->fin ||
 546                 packet->inE->event.tcpEvent->segmentKey->tot_len - packet->inE->event.tcpEvent->segmentKey->ihl * 4 -
 547                 packet->inE->event.tcpEvent->segmentKey->doff * 4 > 0)
 548         {
 549                 return true;
 550         }
 551         else
 552         {
 553                 return false;
 554         }
 555 }
 556
 557
 558 /*
 559  * Populate a connection key structure for the opposite direction of a
 560  * connection
 561  *
 562  * Args:
 563  *   reversedConnectionKey the result, must be pre-allocated
 564  *   connectionKey the connection key to reverse
 565  */
 566 static void buildReversedConnectionKey(ConnectionKey* const
 567         reversedConnectionKey, const ConnectionKey* const connectionKey)
 568 {
 569         reversedConnectionKey->saddr= connectionKey->daddr;
 570         reversedConnectionKey->daddr= connectionKey->saddr;
 571         reversedConnectionKey->source= connectionKey->dest;
 572         reversedConnectionKey->dest= connectionKey->source;
 573 }
 574
 575
 576 /*
 577  * Create and open files used to store message points to genereate
 578  * graphs. Allocate and populate array to store file pointers.
 579  *
 580  * Args:
 581  *   syncState:    container for synchronization data
 582  */
 583 static void openGraphDataFiles(SyncState* const syncState)
 584 {
 585         unsigned int i, j;
 586         int retval;
 587         char* cwd;
 588         char name[29];
 589         MatchingDataTCP* matchingData;
 590
 591         matchingData= (MatchingDataTCP*) syncState->matchingData;
 592
 593         cwd= changeToGraphDir(syncState->graphs);
 594
 595         matchingData->messagePoints= malloc(syncState->traceNb * sizeof(FILE**));
 596         for (i= 0; i < syncState->traceNb; i++)
 597         {
 598                 matchingData->messagePoints[i]= malloc(syncState->traceNb *
 599                         sizeof(FILE*));
 600                 for (j= 0; j < syncState->traceNb; j++)
 601                 {
 602                         if (i != j)
 603                         {
 604                                 retval= snprintf(name, sizeof(name),
 605                                         "matching_tcp-%03u_to_%03u.data", j, i);
 606                                 if (retval > sizeof(name) - 1)
 607                                 {
 608                                         name[sizeof(name) - 1]= '\0';
 609                                 }
 610                                 if ((matchingData->messagePoints[i][j]= fopen(name, "w")) ==
 611                                         NULL)
 612                                 {
 613                                         g_error(strerror(errno));
 614                                 }
 615                         }
 616                 }
 617         }
 618
 619         retval= chdir(cwd);
 620         if (retval == -1)
 621         {
 622                 g_error(strerror(errno));
 623         }
 624         free(cwd);
 625 }
 626
 627
 628 /*
 629  * Write a message point to a file used to generate graphs
 630  *
 631  * Args:
 632  *   stream:       FILE*, file pointer where to write the point
 633  *   message:       message for which to write the point
 634  */
 635 static void writeMessagePoint(FILE* stream, const Message* const message)
 636 {
 637         LttCycleCount x, y;
 638
 639         if (message->inE->traceNum < message->outE->traceNum)
 640         {
 641                 // CA is inE->traceNum
 642                 x= message->inE->cpuTime;
 643                 y= message->outE->cpuTime;
 644         }
 645         else
 646         {
 647                 // CA is outE->traceNum
 648                 x= message->outE->cpuTime;
 649                 y= message->inE->cpuTime;
 650         }
 651
 652         fprintf(stream, "%20llu %20llu\n", x, y);
 653 }
 654
 655
 656 /*
 657  * Close files used to store convex hull points to genereate graphs.
 658  * Deallocate array to store file pointers.
 659  *
 660  * Args:
 661  *   syncState:    container for synchronization data
 662  */
 663 static void closeGraphDataFiles(SyncState* const syncState)
 664 {
 665         unsigned int i, j;
 666         MatchingDataTCP* matchingData;
 667         int retval;
 668
 669         matchingData= (MatchingDataTCP*) syncState->matchingData;
 670
 671         if (matchingData->messagePoints == NULL)
 672         {
 673                 return;
 674         }
 675
 676         for (i= 0; i < syncState->traceNb; i++)
 677         {
 678                 for (j= 0; j < syncState->traceNb; j++)
 679                 {
 680                         if (i != j)
 681                         {
 682                                 retval= fclose(matchingData->messagePoints[i][j]);
 683                                 if (retval != 0)
 684                                 {
 685                                         g_error(strerror(errno));
 686                                 }
 687                         }
 688                 }
 689                 free(matchingData->messagePoints[i]);
 690         }
 691         free(matchingData->messagePoints);
 692
 693         matchingData->messagePoints= NULL;
 694 }
 695
 696
 697 /*
 698  * Write the matching-specific graph lines in the gnuplot script.
 699  *
 700  * Args:
 701  *   stream:       stream where to write the data
 702  *   syncState:    container for synchronization data
 703  *   i:            first trace number
 704  *   j:            second trace number, garanteed to be larger than i
 705  */
 706 static void writeMatchingGraphsPlotsTCP(FILE* stream, SyncState* const
 707         syncState, const unsigned int i, const unsigned int j)
 708 {
 709         fprintf(stream,
 710                 "\t\"matching_tcp-%1$03d_to_%2$03d.data\" "
 711                         "title \"Sent messages\" with points linetype 4 "
 712                         "linecolor rgb \"#98fc66\" pointtype 9 pointsize 2, \\\n"
 713                 "\t\"matching_tcp-%2$03d_to_%1$03d.data\" "
 714                         "title \"Received messages\" with points linetype 4 "
 715                         "linecolor rgb \"#6699cc\" pointtype 11 pointsize 2, \\\n", i, j);
 716 }