gprofng/libcollector/synctrace.c

   1 /* Copyright (C) 2021-2023 Free Software Foundation, Inc.
   2    Contributed by Oracle.
   3
   4    This file is part of GNU Binutils.
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3, or (at your option)
   9    any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, write to the Free Software
  18    Foundation, 51 Franklin Street - Fifth Floor, Boston,
  19    MA 02110-1301, USA.  */
  20
  21 /*
  22  *      Synchronization events
  23  */
  24 #include "config.h"
  25 #include <alloca.h>
  26 #include <dlfcn.h>
  27 #include <unistd.h>
  28 #include <semaphore.h>          /* sem_wait() */
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <sys/param.h>
  32 #include <pthread.h>
  33
  34 #include "gp-defs.h"
  35 #include "collector.h"
  36 #include "gp-experiment.h"
  37 #include "data_pckts.h"
  38 #include "tsd.h"
  39 #include "cc_libcollector.h"
  40
  41 /* define the packet that will be written out */
  42 typedef struct Sync_packet
  43 { /* Synchronization delay tracing packet */
  44   Common_packet comm;
  45   hrtime_t requested;       /* time of synchronization request */
  46   Vaddr_type objp;          /* vaddr of synchronization object */
  47 } Sync_packet;
  48
  49 static int open_experiment (const char *);
  50 static int start_data_collection (void);
  51 static int stop_data_collection (void);
  52 static int close_experiment (void);
  53 static int detach_experiment (void);
  54 static int init_thread_intf ();
  55 static int sync_calibrate ();
  56
  57 static ModuleInterface module_interface ={
  58   SP_SYNCTRACE_FILE,        /* description */
  59   NULL,                     /* initInterface */
  60   open_experiment,          /* openExperiment */
  61   start_data_collection,    /* startDataCollection */
  62   stop_data_collection,     /* stopDataCollection */
  63   close_experiment,         /* closeExperiment */
  64   detach_experiment         /* detachExperiment (fork child) */
  65 };
  66
  67 static CollectorInterface *collector_interface = NULL;
  68 static int sync_mode = 0;
  69 static long sync_scope = 0;
  70 static int sync_native = 0;
  71 static int sync_java = 0;
  72 static CollectorModule sync_hndl = COLLECTOR_MODULE_ERR;
  73 static unsigned sync_key = COLLECTOR_TSD_INVALID_KEY;
  74 static long sync_threshold = -1; /* calibrate the value */
  75 static int init_thread_intf_started = 0;
  76 static int init_thread_intf_finished = 0;
  77
  78 #define CHCK_NREENTRANCE(x)     (!sync_native || !sync_mode || ((x) = collector_interface->getKey( sync_key )) == NULL || (*(x) != 0))
  79 #define RECHCK_NREENTRANCE(x)   (!sync_native || !sync_mode || ((x) = collector_interface->getKey( sync_key )) == NULL || (*(x) == 0))
  80 #define CHCK_JREENTRANCE(x)     (!sync_java || !sync_mode || ((x) = collector_interface->getKey( sync_key )) == NULL || (*(x) != 0))
  81 #define RECHCK_JREENTRANCE(x)   (!sync_java || !sync_mode || ((x) = collector_interface->getKey( sync_key )) == NULL || (*(x) == 0))
  82 #define PUSH_REENTRANCE(x)      ((*(x))++)
  83 #define POP_REENTRANCE(x)       ((*(x))--)
  84 #define gethrtime       collector_interface->getHiResTime
  85
  86 /*
  87  * In most cases, the functions which require interposition are implemented as
  88  * weak symbols corresponding to an associated internal function named with a
  89  * leading underscore: e.g., mutex_lock() is simply an alias for _mutex_lock().
  90  * For the wait functions, however, the published version (used by applications)
  91  * is distinct from the internal version (used by system libraries), i.e.,
  92  * cond_wait() is an alias for _cond_wait_cancel() rather than _cond_wait().
  93  */
  94 static long int (*__real_strtol)(const char *nptr, char **endptr, int base) = NULL;
  95 static int (*__real_fprintf) (FILE *stream, const char *format, ...) = NULL;
  96 static void (*__real___collector_jprofile_enable_synctrace) (void) = NULL;
  97 static int (*__real_pthread_mutex_lock) (pthread_mutex_t *mutex) = NULL;
  98 static int (*__real_pthread_mutex_lock_2_17) (pthread_mutex_t *mutex) = NULL;
  99 static int (*__real_pthread_mutex_lock_2_2_5) (pthread_mutex_t *mutex) = NULL;
 100 static int (*__real_pthread_mutex_lock_2_0) (pthread_mutex_t *mutex) = NULL;
 101 static int (*__real_pthread_mutex_unlock) (pthread_mutex_t *mutex) = NULL;
 102 static int (*__real_pthread_cond_wait) (pthread_cond_t *restrict cond,
 103                                 pthread_mutex_t *restrict mutex) = NULL;
 104 static int (*__real_pthread_cond_timedwait) (pthread_cond_t *restrict cond,
 105                         pthread_mutex_t *restrict mutex,
 106                         const struct timespec *restrict abstime) = NULL;
 107 static int (*__real_pthread_join) (pthread_t thread, void **retval) = NULL;
 108 static int (*__real_pthread_join_2_34) (pthread_t thread, void **retval) = NULL;
 109 static int (*__real_pthread_join_2_17) (pthread_t thread, void **retval) = NULL;
 110 static int (*__real_pthread_join_2_2_5) (pthread_t thread, void **retval) = NULL;
 111 static int (*__real_pthread_join_2_0) (pthread_t thread, void **retval) = NULL;
 112 static int (*__real_sem_wait) (sem_t *sem) = NULL;
 113 static int (*__real_sem_wait_2_34) (sem_t *sem) = NULL;
 114 static int (*__real_sem_wait_2_17) (sem_t *sem) = NULL;
 115 static int (*__real_sem_wait_2_2_5) (sem_t *sem) = NULL;
 116 static int (*__real_sem_wait_2_1) (sem_t *sem) = NULL;
 117 static int (*__real_sem_wait_2_0) (sem_t *sem) = NULL;
 118 static int (*__real_pthread_cond_wait_2_17) (pthread_cond_t *restrict cond,
 119                                 pthread_mutex_t *restrict mutex) = NULL;
 120 static int (*__real_pthread_cond_wait_2_3_2) (pthread_cond_t *restrict cond,
 121                                 pthread_mutex_t *restrict mutex) = NULL;
 122 static int (*__real_pthread_cond_wait_2_2_5) (pthread_cond_t *restrict cond,
 123                                 pthread_mutex_t *restrict mutex) = NULL;
 124 static int (*__real_pthread_cond_wait_2_0) (pthread_cond_t *restrict cond,
 125                                 pthread_mutex_t *restrict mutex) = NULL;
 126 static int (*__real_pthread_cond_timedwait_2_17) (pthread_cond_t *restrict cond,
 127                         pthread_mutex_t *restrict mutex,
 128                         const struct timespec *restrict abstime) = NULL;
 129 static int (*__real_pthread_cond_timedwait_2_3_2) (pthread_cond_t *restrict cond,
 130                         pthread_mutex_t *restrict mutex,
 131                         const struct timespec *restrict abstime) = NULL;
 132 static int (*__real_pthread_cond_timedwait_2_2_5) (pthread_cond_t *restrict cond,
 133                         pthread_mutex_t *restrict mutex,
 134                         const struct timespec *restrict abstime) = NULL;
 135 static int (*__real_pthread_cond_timedwait_2_0) (pthread_cond_t *restrict cond,
 136                         pthread_mutex_t *restrict mutex,
 137                         const struct timespec *restrict abstime) = NULL;
 138
 139
 140 static void
 141 collector_memset (void *s, int c, size_t n)
 142 {
 143   unsigned char *s1 = s;
 144   while (n--)
 145     *s1++ = (unsigned char) c;
 146 }
 147
 148 void
 149 __collector_module_init (CollectorInterface *_collector_interface)
 150 {
 151   if (_collector_interface == NULL)
 152     return;
 153   collector_interface = _collector_interface;
 154   TprintfT (0, "synctrace: __collector_module_init\n");
 155   sync_hndl = collector_interface->registerModule (&module_interface);
 156
 157   /* Initialize next module */
 158   ModuleInitFunc next_init = (ModuleInitFunc) dlsym (RTLD_NEXT, "__collector_module_init");
 159   if (next_init != NULL)
 160     next_init (_collector_interface);
 161 }
 162
 163 static int
 164 open_experiment (const char *exp)
 165 {
 166   long thresh = 0;
 167   if (init_thread_intf_finished == 0)
 168     init_thread_intf ();
 169   if (collector_interface == NULL)
 170     {
 171       Tprintf (0, "synctrace: collector_interface is null.\n");
 172       return COL_ERROR_SYNCINIT;
 173     }
 174   if (sync_hndl == COLLECTOR_MODULE_ERR)
 175     {
 176       Tprintf (0, "synctrace: handle create failed.\n");
 177       collector_interface->writeLog ("<event kind=\"%s\" id=\"%d\">data handle not created</event>\n",
 178                                      SP_JCMD_CERROR, COL_ERROR_SYNCINIT);
 179       return COL_ERROR_SYNCINIT;
 180     }
 181   TprintfT (0, "synctrace: open_experiment %s\n", exp);
 182
 183   char *params = (char *) collector_interface->getParams ();
 184   while (params)
 185     {
 186       if ((params[0] == 's') && (params[1] == ':'))
 187         {
 188           char *ptr = params + 2;
 189           Tprintf (DBG_LT1, "synctrace: open_experiment s: parameter = %s\n", ptr);
 190           while (*ptr != ',' && *ptr != ';')
 191             ptr++;
 192           sync_scope = 0;
 193           if (*ptr == ',')
 194             {
 195               sync_scope = CALL_REAL (strtol) (ptr + 1, NULL, 0);
 196               switch (sync_scope)
 197                 {
 198                 case 1:
 199                   sync_java = 0;
 200                   sync_native = 1;
 201                   break;
 202                 case 2:
 203                   sync_java = 1;
 204                   sync_native = 0;
 205                   break;
 206                 default:
 207                 case 3:
 208                   sync_native = 1;
 209                   sync_java = 1;
 210                   break;
 211                 }
 212               Tprintf (0, "\tsynctrace: sync_scope found as %ld\n", sync_scope);
 213             }
 214           else
 215             {
 216               /* the old-style descriptor, without scope */
 217               /* if there was no comma, use the old default */
 218               sync_scope = 3;
 219               sync_java = 1;
 220               sync_native = 1;
 221               Tprintf (0, "\tsynctrace: sync_scope not found set to %ld\n", sync_scope);
 222             }
 223           if (__real___collector_jprofile_enable_synctrace == NULL)
 224             sync_java = 0;
 225           thresh = CALL_REAL (strtol)(params + 2, NULL, 0);
 226           break; /* from the loop to find the "s:thresh,scope" entry */
 227         }
 228       else
 229         params++;
 230     }
 231   if (params == NULL)  /* Sync data collection not specified */
 232     return COL_ERROR_SYNCINIT;
 233   if (thresh < 0)  /* calibrate the threshold, keep it as a negative number */
 234     thresh = -sync_calibrate ();
 235
 236   sync_key = collector_interface->createKey (sizeof ( int), NULL, NULL);
 237   if (sync_key == (unsigned) - 1)
 238     {
 239       Tprintf (0, "synctrace: TSD key create failed.\n");
 240       collector_interface->writeLog ("<event kind=\"%s\" id=\"%d\">TSD key not created</event>\n",
 241                                      SP_JCMD_CERROR, COL_ERROR_SYNCINIT);
 242       return COL_ERROR_SYNCINIT;
 243     }
 244   /* if Java synctrace was requested, tell the jprofile module */
 245   if (sync_java)
 246     {
 247       TprintfT (0, "synctrace: enabling Java synctrace\n");
 248       CALL_REAL (__collector_jprofile_enable_synctrace)();
 249     }
 250   collector_interface->writeLog ("<profile name=\"%s\" threshold=\"%ld\" scope=\"%ld\">\n",
 251                                  SP_JCMD_SYNCTRACE, thresh, sync_scope);
 252   collector_interface->writeLog ("  <profdata fname=\"%s\"/>\n",
 253                                  module_interface.description);
 254   /* Record Sync_packet description */
 255   Sync_packet *pp = NULL;
 256   collector_interface->writeLog ("  <profpckt kind=\"%d\" uname=\"Synchronization tracing data\">\n", SYNC_PCKT);
 257   collector_interface->writeLog ("    <field name=\"LWPID\" uname=\"Lightweight process id\" offset=\"%d\" type=\"%s\"/>\n",
 258                                  &pp->comm.lwp_id, sizeof (pp->comm.lwp_id) == 4 ? "INT32" : "INT64");
 259   collector_interface->writeLog ("    <field name=\"THRID\" uname=\"Thread number\" offset=\"%d\" type=\"%s\"/>\n",
 260                                  &pp->comm.thr_id, sizeof (pp->comm.thr_id) == 4 ? "INT32" : "INT64");
 261   collector_interface->writeLog ("    <field name=\"CPUID\" uname=\"CPU id\" offset=\"%d\" type=\"%s\"/>\n",
 262                                  &pp->comm.cpu_id, sizeof (pp->comm.cpu_id) == 4 ? "INT32" : "INT64");
 263   collector_interface->writeLog ("    <field name=\"TSTAMP\" uname=\"High resolution timestamp\" offset=\"%d\" type=\"%s\"/>\n",
 264                                  &pp->comm.tstamp, sizeof (pp->comm.tstamp) == 4 ? "INT32" : "INT64");
 265   collector_interface->writeLog ("    <field name=\"FRINFO\" offset=\"%d\" type=\"%s\"/>\n",
 266                                  &pp->comm.frinfo, sizeof (pp->comm.frinfo) == 4 ? "INT32" : "INT64");
 267   collector_interface->writeLog ("    <field name=\"SRQST\" uname=\"Synchronization start time\" offset=\"%d\" type=\"%s\"/>\n",
 268                                  &pp->requested, sizeof (pp->requested) == 4 ? "INT32" : "INT64");
 269   collector_interface->writeLog ("    <field name=\"SOBJ\" uname=\"Synchronization object address\" offset=\"%d\" type=\"%s\"/>\n",
 270                                  &pp->objp, sizeof (pp->objp) == 4 ? "INT32" : "INT64");
 271   collector_interface->writeLog ("  </profpckt>\n");
 272   collector_interface->writeLog ("</profile>\n");
 273
 274   /* Convert threshold from microsec to nanosec */
 275   sync_threshold = (thresh > 0 ? thresh : -thresh) * 1000;
 276   TprintfT (0, "synctrace: open_experiment complete %ld\n", sync_threshold);
 277   return COL_ERROR_NONE;
 278 }
 279
 280 static int
 281 start_data_collection (void)
 282 {
 283   sync_mode = 1;
 284   TprintfT (0, "synctrace: start_data_collection\n");
 285   return 0;
 286 }
 287
 288 static int
 289 stop_data_collection (void)
 290 {
 291   sync_mode = 0;
 292   TprintfT (0, "synctrace: stop_data_collection\n");
 293   return 0;
 294 }
 295
 296 static int
 297 close_experiment (void)
 298 {
 299   sync_mode = 0;
 300   sync_threshold = -1;
 301   sync_key = COLLECTOR_TSD_INVALID_KEY;
 302   TprintfT (0, "synctrace: close_experiment\n");
 303   return 0;
 304 }
 305
 306 /* fork child.  Clean up state but don't write to experiment */
 307 static int
 308 detach_experiment (void)
 309 {
 310   sync_mode = 0;
 311   sync_threshold = -1;
 312   sync_key = COLLECTOR_TSD_INVALID_KEY;
 313   TprintfT (0, "synctrace: detach_experiment\n");
 314   return 0;
 315 }
 316
 317 #define NUM_ITER    100     /* number of iterations in calibration */
 318 #define NUM_WARMUP    3     /* number of warm up iterations */
 319
 320 static int
 321 sync_calibrate ()
 322 {
 323   pthread_mutex_t mt = PTHREAD_MUTEX_INITIALIZER;
 324   hrtime_t bt, at, delta;
 325   hrtime_t avg, max, min;
 326   int i;
 327   int ret;
 328   avg = (hrtime_t) 0;
 329   min = max = (hrtime_t) 0;
 330   for (i = 0; i < NUM_ITER + NUM_WARMUP; i++)
 331     {
 332       /* Here we simulate a real call */
 333       bt = gethrtime ();
 334       ret = CALL_REAL (pthread_mutex_lock)(&mt);
 335       at = gethrtime ();
 336       CALL_REAL (pthread_mutex_unlock)(&mt);
 337       if (i < NUM_WARMUP)   /* skip these iterations */
 338         continue;
 339       /* add the time of this one */
 340       delta = at - bt;
 341       avg += delta;
 342       if (min == 0)
 343         min = delta;
 344       if (delta < min)
 345         min = delta;
 346       if (delta > max)
 347         max = delta;
 348     }
 349   /* compute average time */
 350   avg = avg / NUM_ITER;
 351
 352   /* pretty simple, let's see how it works */
 353   if (max < 6 * avg)
 354     max = 6 * avg;
 355   /* round up to the nearest microsecond */
 356   ret = (int) ((max + 999) / 1000);
 357   return ret;
 358 }
 359
 360 static int
 361 init_pthread_mutex_lock (void *dlflag)
 362 {
 363   __real_pthread_mutex_lock_2_17 = dlvsym (dlflag, "pthread_mutex_lock", "GLIBC_2.17");
 364   __real_pthread_mutex_lock_2_2_5 = dlvsym (dlflag, "pthread_mutex_lock", "GLIBC_2.2.5");
 365   __real_pthread_mutex_lock_2_0 = dlvsym (dlflag, "pthread_mutex_lock", "GLIBC_2.0");
 366   if (__real_pthread_mutex_lock_2_17)
 367     __real_pthread_mutex_lock = __real_pthread_mutex_lock_2_17;
 368   else if (__real_pthread_mutex_lock_2_2_5)
 369     __real_pthread_mutex_lock = __real_pthread_mutex_lock_2_2_5;
 370   else if (__real_pthread_mutex_lock_2_0)
 371     __real_pthread_mutex_lock = __real_pthread_mutex_lock_2_0;
 372   else
 373     __real_pthread_mutex_lock = dlsym (dlflag, "pthread_mutex_lock");
 374   return __real_pthread_mutex_lock ? 1 : 0;
 375 }
 376
 377 static int
 378 init_thread_intf ()
 379 {
 380   void *dlflag = RTLD_NEXT;
 381   int err = 0;
 382   /* if we detect recursion/reentrance, SEGV so we can get a stack */
 383   init_thread_intf_started++;
 384   if (!init_thread_intf_finished && init_thread_intf_started >= 3)
 385     {
 386       /* pull the plug if recursion occurs... */
 387       abort ();
 388     }
 389   /* lookup fprint to print fatal error message */
 390   void *ptr = dlsym (RTLD_DEFAULT, "fprintf");
 391   if (ptr)
 392     {
 393       __real_fprintf = (void *) ptr;
 394     }
 395   else
 396     {
 397       abort ();
 398     }
 399
 400   /* find the __collector_jprofile_enable_synctrace routine in jprofile module */
 401   ptr = dlsym (RTLD_DEFAULT, "__collector_jprofile_enable_synctrace");
 402   if (ptr)
 403     __real___collector_jprofile_enable_synctrace = (void *) ptr;
 404   else
 405     {
 406 #if defined(GPROFNG_JAVA_PROFILING)
 407       CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT __collector_jprofile_enable_synctrace\n");
 408       err = COL_ERROR_SYNCINIT;
 409 #endif
 410       sync_java = 0;
 411     }
 412
 413   dlflag = RTLD_NEXT;
 414   if (init_pthread_mutex_lock (dlflag) == 0)
 415     {
 416       /* We are probably dlopened after libthread/libc,
 417        * try to search in the previously loaded objects
 418        */
 419       dlflag = RTLD_DEFAULT;
 420       if (init_pthread_mutex_lock (dlflag) == 0)
 421         {
 422           CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_mutex_lock\n");
 423           err = COL_ERROR_SYNCINIT;
 424         }
 425     }
 426
 427   if ((ptr = dlvsym (dlflag, "pthread_mutex_unlock", "GLIBC_2.17")) != NULL)
 428     __real_pthread_mutex_unlock = ptr;
 429   else if ((ptr = dlvsym (dlflag, "pthread_mutex_unlock", "GLIBC_2.2.5")) != NULL)
 430     __real_pthread_mutex_unlock = ptr;
 431   else if ((ptr = dlvsym (dlflag, "pthread_mutex_unlock", "GLIBC_2.0")) != NULL)
 432     __real_pthread_mutex_unlock = ptr;
 433   else
 434     __real_pthread_mutex_unlock = dlsym (dlflag, "pthread_mutex_unlock");
 435   if (__real_pthread_mutex_unlock == NULL)
 436     {
 437       CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_mutex_unlock\n");
 438       err = COL_ERROR_SYNCINIT;
 439     }
 440
 441   __real_pthread_cond_wait_2_17 = dlvsym (dlflag, "pthread_cond_wait", "GLIBC_2.17");
 442   __real_pthread_cond_wait_2_3_2 = dlvsym (dlflag, "pthread_cond_wait", "GLIBC_2.3.2");
 443   __real_pthread_cond_wait_2_2_5 = dlvsym (dlflag, "pthread_cond_wait", "GLIBC_2.2.5");
 444   __real_pthread_cond_wait_2_0 = dlvsym (dlflag, "pthread_cond_wait", "GLIBC_2.0");
 445   if (__real_pthread_cond_wait_2_17)
 446     __real_pthread_cond_wait = __real_pthread_cond_wait_2_17;
 447   else if (__real_pthread_cond_wait_2_3_2)
 448     __real_pthread_cond_wait = __real_pthread_cond_wait_2_3_2;
 449   else if (__real_pthread_cond_wait_2_2_5)
 450     __real_pthread_cond_wait = __real_pthread_cond_wait_2_2_5;
 451   else if (__real_pthread_cond_wait_2_0)
 452     __real_pthread_cond_wait = __real_pthread_cond_wait_2_0;
 453   else
 454     __real_pthread_cond_wait = dlsym (dlflag, "pthread_cond_wait");
 455   if (__real_pthread_cond_wait == NULL)
 456     {
 457       CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_cond_wait\n");
 458       err = COL_ERROR_SYNCINIT;
 459     }
 460
 461   __real_pthread_cond_timedwait_2_17 = dlvsym (dlflag, "pthread_cond_timedwait", "GLIBC_2.17");
 462   __real_pthread_cond_timedwait_2_3_2 = dlvsym (dlflag, "pthread_cond_timedwait", "GLIBC_2.3.2");
 463   __real_pthread_cond_timedwait_2_2_5 = dlvsym (dlflag, "pthread_cond_timedwait", "GLIBC_2.2.5");
 464   __real_pthread_cond_timedwait_2_0 = dlvsym (dlflag, "pthread_cond_timedwait", "GLIBC_2.0");
 465   if (__real_pthread_cond_timedwait_2_17)
 466     __real_pthread_cond_timedwait = __real_pthread_cond_timedwait_2_17;
 467   else if (__real_pthread_cond_timedwait_2_3_2)
 468     __real_pthread_cond_timedwait = __real_pthread_cond_timedwait_2_3_2;
 469   else if (__real_pthread_cond_timedwait_2_2_5)
 470     __real_pthread_cond_timedwait = __real_pthread_cond_timedwait_2_2_5;
 471   else if (__real_pthread_cond_timedwait_2_0)
 472     __real_pthread_cond_timedwait = __real_pthread_cond_timedwait_2_0;
 473   else
 474     __real_pthread_cond_timedwait = dlsym (dlflag, "pthread_cond_timedwait");
 475   if (__real_pthread_cond_timedwait == NULL)
 476     {
 477       CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_cond_timedwait\n");
 478       err = COL_ERROR_SYNCINIT;
 479     }
 480
 481   __real_pthread_join_2_34 = dlvsym (dlflag, "pthread_join", "GLIBC_2.34");
 482   __real_pthread_join_2_17 = dlvsym (dlflag, "pthread_join", "GLIBC_2.17");
 483   __real_pthread_join_2_2_5 = dlvsym (dlflag, "pthread_join", "GLIBC_2.2.5");
 484   __real_pthread_join_2_0 = dlvsym (dlflag, "pthread_join", "GLIBC_2.0");
 485   if (__real_pthread_join_2_34)
 486     __real_pthread_join = __real_pthread_join_2_34;
 487   else if (__real_pthread_join_2_17)
 488     __real_pthread_join = __real_pthread_join_2_17;
 489   else if (__real_pthread_join_2_2_5)
 490     __real_pthread_join = __real_pthread_join_2_2_5;
 491   else if (__real_pthread_join_2_0)
 492     __real_pthread_join = __real_pthread_join_2_0;
 493   else
 494     __real_pthread_join = dlsym (dlflag, "pthread_join");
 495   if (__real_pthread_join == NULL)
 496     {
 497       CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT pthread_join\n");
 498       err = COL_ERROR_SYNCINIT;
 499     }
 500
 501   __real_sem_wait_2_34 = dlvsym (dlflag, "sem_wait", "GLIBC_2.34");
 502   __real_sem_wait_2_17 = dlvsym (dlflag, "sem_wait", "GLIBC_2.17");
 503   __real_sem_wait_2_2_5 = dlvsym (dlflag, "sem_wait", "GLIBC_2.2.5");
 504   __real_sem_wait_2_1 = dlvsym (dlflag, "sem_wait", "GLIBC_2.1");
 505   __real_sem_wait_2_0 = dlvsym (dlflag, "sem_wait", "GLIBC_2.0");
 506   if (__real_sem_wait_2_34)
 507     __real_sem_wait = __real_sem_wait_2_34;
 508   else if (__real_sem_wait_2_17)
 509     __real_sem_wait = __real_sem_wait_2_17;
 510   else if (__real_sem_wait_2_2_5)
 511     __real_sem_wait = __real_sem_wait_2_2_5;
 512   else if (__real_sem_wait_2_1)
 513     __real_sem_wait = __real_sem_wait_2_1;
 514   else if (__real_sem_wait_2_0)
 515     __real_sem_wait = __real_sem_wait_2_0;
 516   else
 517     __real_sem_wait = dlsym (dlflag, "sem_wait");
 518   if (__real_sem_wait == NULL)
 519     {
 520       CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT sem_wait\n");
 521       err = COL_ERROR_SYNCINIT;
 522     }
 523
 524
 525   ptr = dlsym (dlflag, "strtol");
 526   if (ptr)
 527     __real_strtol = (void *) ptr;
 528   else
 529     {
 530       CALL_REAL (fprintf)(stderr, "synctrace_init COL_ERROR_SYNCINIT strtol\n");
 531       err = COL_ERROR_SYNCINIT;
 532     }
 533   init_thread_intf_finished++;
 534   TprintfT (0, "synctrace init_thread_intf complete\n");
 535   return err;
 536 }
 537
 538 /* These next two routines are used from jprofile to record Java synctrace data */
 539 void
 540 __collector_jsync_begin ()
 541 {
 542   int *guard;
 543   if (CHCK_JREENTRANCE (guard))
 544     {
 545       Tprintf (DBG_LT1, "__collector_jsync_begin: skipped\n");
 546       return;
 547     }
 548   Tprintf (DBG_LT1, "__collector_jsync_begin: start event\n");
 549   PUSH_REENTRANCE (guard);
 550 }
 551
 552 void
 553 __collector_jsync_end (hrtime_t reqt, void *object)
 554 {
 555   int *guard;
 556   if (RECHCK_JREENTRANCE (guard))
 557     {
 558       Tprintf (DBG_LT1, "__collector_jsync_end: skipped\n");
 559       return;
 560     }
 561   hrtime_t grnt = gethrtime ();
 562   if (grnt - reqt >= sync_threshold)
 563     {
 564       Sync_packet spacket;
 565       collector_memset (&spacket, 0, sizeof (Sync_packet));
 566       spacket.comm.tsize = sizeof (Sync_packet);
 567       spacket.comm.tstamp = grnt;
 568       spacket.requested = reqt;
 569       spacket.objp = (intptr_t) object;
 570       spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
 571                         spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
 572       collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
 573     }
 574   Tprintf (DBG_LT1, "__collector_jsync_begin: end event\n");
 575   POP_REENTRANCE (guard);
 576 }
 577 /*-------------------------------------------------------- pthread_mutex_lock */
 578 static int
 579 gprofng_pthread_mutex_lock (int (real_func) (pthread_mutex_t *),
 580                             pthread_mutex_t *mp)
 581 {
 582   int *guard;
 583   if (CHCK_NREENTRANCE (guard))
 584     return (real_func) (mp);
 585   PUSH_REENTRANCE (guard);
 586   hrtime_t reqt = gethrtime ();
 587   int ret = (real_func) (mp);
 588   if (RECHCK_NREENTRANCE (guard))
 589     {
 590       POP_REENTRANCE (guard);
 591       return ret;
 592     }
 593   hrtime_t grnt = gethrtime ();
 594   if (grnt - reqt >= sync_threshold)
 595     {
 596       Sync_packet spacket;
 597       collector_memset (&spacket, 0, sizeof (Sync_packet));
 598       spacket.comm.tsize = sizeof (Sync_packet);
 599       spacket.comm.tstamp = grnt;
 600       spacket.requested = reqt;
 601       spacket.objp = (intptr_t) mp;
 602       spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
 603                         spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
 604       collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
 605     }
 606   POP_REENTRANCE (guard);
 607   return ret;
 608 }
 609
 610 #define DCL_PTHREAD_MUTEX_LOCK(dcl_f) \
 611   int dcl_f (pthread_mutex_t *mp) \
 612   { \
 613     if (__real_pthread_mutex_lock == NULL) \
 614       init_thread_intf (); \
 615     return gprofng_pthread_mutex_lock (__real_pthread_mutex_lock, mp); \
 616   }
 617
 618 DCL_FUNC_VER (DCL_PTHREAD_MUTEX_LOCK, pthread_mutex_lock_2_17, pthread_mutex_lock@GLIBC_2.17)
 619 DCL_FUNC_VER (DCL_PTHREAD_MUTEX_LOCK, pthread_mutex_lock_2_2_5, pthread_mutex_lock@GLIBC_2.2.5)
 620 DCL_FUNC_VER (DCL_PTHREAD_MUTEX_LOCK, pthread_mutex_lock_2_0, pthread_mutex_lock@GLIBC_2.0)
 621 DCL_PTHREAD_MUTEX_LOCK (pthread_mutex_lock)
 622
 623 /*------------------------------------------------------------- pthread_cond_wait */
 624 static int
 625 gprofng_pthread_cond_wait (int(real_func) (pthread_cond_t *, pthread_mutex_t *),
 626                            pthread_cond_t *cond, pthread_mutex_t *mutex)
 627 {
 628   int *guard;
 629   if (CHCK_NREENTRANCE (guard))
 630     return (real_func) (cond, mutex);
 631   PUSH_REENTRANCE (guard);
 632   hrtime_t reqt = gethrtime ();
 633   int ret = -1;
 634   ret = (real_func) (cond, mutex);
 635   if (RECHCK_NREENTRANCE (guard))
 636     {
 637       POP_REENTRANCE (guard);
 638       return ret;
 639     }
 640   hrtime_t grnt = gethrtime ();
 641   if (grnt - reqt >= sync_threshold)
 642     {
 643       Sync_packet spacket;
 644       collector_memset (&spacket, 0, sizeof (Sync_packet));
 645       spacket.comm.tsize = sizeof (Sync_packet);
 646       spacket.comm.tstamp = grnt;
 647       spacket.requested = reqt;
 648       spacket.objp = (intptr_t) mutex;
 649       spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
 650                         spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
 651       collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
 652     }
 653   POP_REENTRANCE (guard);
 654   return ret;
 655 }
 656
 657 #define DCL_PTHREAD_COND_WAIT(dcl_f) \
 658   int dcl_f (pthread_cond_t *cond, pthread_mutex_t *mutex) \
 659   { \
 660     if (__real_pthread_cond_wait == NULL) \
 661       init_thread_intf (); \
 662     return gprofng_pthread_cond_wait (__real_pthread_cond_wait, cond, mutex); \
 663   }
 664
 665 DCL_FUNC_VER (DCL_PTHREAD_COND_WAIT, pthread_cond_wait_2_17, pthread_cond_wait@GLIBC_2.17)
 666 DCL_FUNC_VER (DCL_PTHREAD_COND_WAIT, pthread_cond_wait_2_3_2, pthread_cond_wait@GLIBC_2.3.2)
 667 DCL_FUNC_VER (DCL_PTHREAD_COND_WAIT, pthread_cond_wait_2_2_5, pthread_cond_wait@GLIBC_2.2.5)
 668 DCL_FUNC_VER (DCL_PTHREAD_COND_WAIT, pthread_cond_wait_2_0, pthread_cond_wait@GLIBC_2.0)
 669 DCL_PTHREAD_COND_WAIT (pthread_cond_wait)
 670
 671 /*---------------------------------------------------- pthread_cond_timedwait */
 672 static int
 673 gprofng_pthread_cond_timedwait (int(real_func) (pthread_cond_t *,
 674                                     pthread_mutex_t*, const struct timespec *),
 675                                 pthread_cond_t *cond, pthread_mutex_t *mutex,
 676                                 const struct timespec *abstime)
 677 {
 678   int *guard;
 679   if (CHCK_NREENTRANCE (guard))
 680     return (real_func) (cond, mutex, abstime);
 681   PUSH_REENTRANCE (guard);
 682   hrtime_t reqt = gethrtime ();
 683   int ret = -1;
 684   ret = (real_func) (cond, mutex, abstime);
 685   if (RECHCK_NREENTRANCE (guard))
 686     {
 687       POP_REENTRANCE (guard);
 688       return ret;
 689     }
 690   hrtime_t grnt = gethrtime ();
 691   if (grnt - reqt >= sync_threshold)
 692     {
 693       Sync_packet spacket;
 694       collector_memset (&spacket, 0, sizeof (Sync_packet));
 695       spacket.comm.tsize = sizeof (Sync_packet);
 696       spacket.comm.tstamp = grnt;
 697       spacket.requested = reqt;
 698       spacket.objp = (intptr_t) mutex;
 699       spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
 700                         spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
 701       collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
 702     }
 703   POP_REENTRANCE (guard);
 704   return ret;
 705 }
 706
 707 #define DCL_PTHREAD_COND_TIMEDWAIT(dcl_f) \
 708   int dcl_f (pthread_cond_t *cond, pthread_mutex_t *mutex, \
 709              const struct timespec *abstime) \
 710   { \
 711     if (__real_pthread_cond_timedwait == NULL) \
 712       init_thread_intf (); \
 713     return gprofng_pthread_cond_timedwait (__real_pthread_cond_timedwait, cond, mutex, abstime); \
 714   }
 715
 716 DCL_FUNC_VER (DCL_PTHREAD_COND_TIMEDWAIT, pthread_cond_timedwait_2_17, pthread_cond_timedwait@GLIBC_2.17)
 717 DCL_FUNC_VER (DCL_PTHREAD_COND_TIMEDWAIT, pthread_cond_timedwait_2_3_2, pthread_cond_timedwait@GLIBC_2.3.2)
 718 DCL_FUNC_VER (DCL_PTHREAD_COND_TIMEDWAIT, pthread_cond_timedwait_2_2_5, pthread_cond_timedwait@GLIBC_2.2.5)
 719 DCL_FUNC_VER (DCL_PTHREAD_COND_TIMEDWAIT, pthread_cond_timedwait_2_0, pthread_cond_timedwait@GLIBC_2.0)
 720 DCL_PTHREAD_COND_TIMEDWAIT (pthread_cond_timedwait)
 721
 722
 723 /*------------------------------------------------------------- pthread_join */
 724 static int
 725 gprofng_pthread_join (int(real_func) (pthread_t, void **),
 726                       pthread_t target_thread, void **status)
 727 {
 728   int *guard;
 729   if (CHCK_NREENTRANCE (guard))
 730     return real_func (target_thread, status);
 731   PUSH_REENTRANCE (guard);
 732   hrtime_t reqt = gethrtime ();
 733   int ret = real_func(target_thread, status);
 734   if (RECHCK_NREENTRANCE (guard))
 735     {
 736       POP_REENTRANCE (guard);
 737       return ret;
 738     }
 739   hrtime_t grnt = gethrtime ();
 740   if (grnt - reqt >= sync_threshold)
 741     {
 742       Sync_packet spacket;
 743       collector_memset (&spacket, 0, sizeof (Sync_packet));
 744       spacket.comm.tsize = sizeof (Sync_packet);
 745       spacket.comm.tstamp = grnt;
 746       spacket.requested = reqt;
 747       spacket.objp = (Vaddr_type) target_thread;
 748       spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
 749                         spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
 750       collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
 751     }
 752   POP_REENTRANCE (guard);
 753   return ret;
 754 }
 755
 756 #define DCL_PTHREAD_JOIN(dcl_f) \
 757   int dcl_f (pthread_t target_thread, void **status) \
 758   { \
 759     if (__real_pthread_join == NULL) \
 760       init_thread_intf (); \
 761     return gprofng_pthread_join (__real_pthread_join, target_thread, status); \
 762   }
 763
 764 DCL_FUNC_VER (DCL_PTHREAD_JOIN, pthread_join_2_34, pthread_join@GLIBC_2.34)
 765 DCL_FUNC_VER (DCL_PTHREAD_JOIN, pthread_join_2_17, pthread_join@GLIBC_2.17)
 766 DCL_FUNC_VER (DCL_PTHREAD_JOIN, pthread_join_2_2_5, pthread_join@GLIBC_2.2.5)
 767 DCL_FUNC_VER (DCL_PTHREAD_JOIN, pthread_join_2_0, pthread_join@GLIBC_2.0)
 768 DCL_PTHREAD_JOIN (pthread_join)
 769
 770 /*------------------------------------------------------------- sem_wait */
 771 static int
 772 gprofng_sem_wait (int (real_func) (sem_t *), sem_t *sp)
 773 {
 774   int *guard;
 775   if (CHCK_NREENTRANCE (guard))
 776     return real_func (sp);
 777   PUSH_REENTRANCE (guard);
 778   hrtime_t reqt = gethrtime ();
 779   int ret = -1;
 780   ret = real_func (sp);
 781   if (RECHCK_NREENTRANCE (guard))
 782     {
 783       POP_REENTRANCE (guard);
 784       return ret;
 785     }
 786   hrtime_t grnt = gethrtime ();
 787   if (grnt - reqt >= sync_threshold)
 788     {
 789       Sync_packet spacket;
 790       collector_memset (&spacket, 0, sizeof (Sync_packet));
 791       spacket.comm.tsize = sizeof (Sync_packet);
 792       spacket.comm.tstamp = grnt;
 793       spacket.requested = reqt;
 794       spacket.objp = (intptr_t) sp;
 795       spacket.comm.frinfo = collector_interface->getFrameInfo (sync_hndl,
 796                         spacket.comm.tstamp, FRINFO_FROM_STACK_ARG, &spacket);
 797       collector_interface->writeDataRecord (sync_hndl, (Common_packet*) &spacket);
 798     }
 799   POP_REENTRANCE (guard);
 800   return ret;
 801 }
 802
 803 #define DCL_SEM_WAIT(dcl_f) \
 804   int dcl_f (sem_t *sp) \
 805   { \
 806     if (__real_sem_wait == NULL) \
 807       init_thread_intf (); \
 808     return gprofng_sem_wait (__real_sem_wait, sp); \
 809   }
 810
 811 DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_34, sem_wait@GLIBC_2.34)
 812 DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_17, sem_wait@GLIBC_2.17)
 813 DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_2_5, sem_wait@GLIBC_2.2.5)
 814 DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_0, sem_wait@GLIBC_2.0)
 815 DCL_FUNC_VER (DCL_SEM_WAIT, sem_wait_2_1, sem_wait@GLIBC_2.1)
 816 DCL_SEM_WAIT (sem_wait)