/* main.c - Main program for tic backends */ /* see copyright.txt for usage terms */ #define TI_NO_SRCPOS #define TIC_INTERNAL_DEFINE /* turns extern's to not externs */ #include #include #include #include #include #ifndef USE_GC_NONE #undef TRUE #undef FALSE #include /* DATASTART */ #endif #if defined(COMM_AM2) && defined(HAVE_ENAMESERVER_H) #include #endif #include "native-utils.h" int usingAMSPMD = 0; /* true for workers using the AMUDP/AMMPI SPMD job startup API */ static char *progName; int politep = 0; void free_resources(void); /* return log2 if exact, else return -1 */ int getLog(int i) { int exp = 0; if(i <= 0) return -1; for(exp=0; (i&1)==0; i>>=1) exp++; return i==1 ? exp : -1; } #ifdef COMM_GASNET void setupNames() {} /* nothing to do */ #elif defined(COMM_AM2) #ifdef HAVE_ENAMESERVER_H #define NS_Safe(func) { if (func != 0) {fprintf(stderr, "ERROR: %s failed\n", #func); exit(1);}} void setupNames() { char inName[2048]; byte *result; int result_len; int i; int translations; en_t *global_names = (en_t *) ti_malloc(sizeof(en_t)*COMM_GetBParallelDegree()); TIC_AM_Safe(AM_MaxNumTranslations(&translations)); if (translations < COMM_GetBParallelDegree()) { fprintf(stderr, "ERROR: AM_MaxNumTranslations (%d) less than BOXES (%d).\n", translations, COMM_GetBParallelDegree()); exit(-1); } TIC_AM_Safe(AM_GetNumTranslations(TIC_AM_ENDPOINT, &translations)); if (translations < COMM_GetBParallelDegree()) { TIC_AM_Safe(AM_SetNumTranslations(TIC_AM_ENDPOINT, COMM_GetBParallelDegree())); } /* Insert our global endpoint name into the name server */ if (PROCS == 1) { TIC_AM_Safe(AM_Map(TIC_AM_ENDPOINT, 0, TIC_AM_ENDPOINTNAME, TIC_TAG)); } else { #ifdef GLUNIX NS_Safe(NameServer_Init()); #endif sprintf(TIC_AM_NODEDESC, "tic mill progname: %s global pid: %d mybox: %d", progName, COMM_GetMyGuid(), COMM_GetMyBoxNumber()); NS_Safe (NameServer_Bind_Version_Flags (TIC_AM_NODEDESC, (unsigned char*)&TIC_AM_ENDPOINTNAME, sizeof (en_t), ENS_VERSION_NOT_THERE, ENS_FLAG_DESTROY_BINDING_ON_CONNECTION_CLOSE)); #ifdef GLUNIX if (!Glib_Barrier()) { perror("global barrier"); exit(-1); } #endif /* Glunix */ for (i = 0; i < COMM_GetBParallelDegree(); i++) { if (COMM_GetMyBoxNumber() != i) { sprintf(inName, "tic mill progname: %s global pid: %d mybox: %d", progName, COMM_GetMyGuid(), i); do { NameServer_Lookup(inName, &result, &result_len); } while (result_len == 0); if (result == NULL) { fprintf(stderr, "ERROR: result Null for lookup %s\n", inName); exit(-1); } if (result_len != sizeof(en_t)) { fprintf(stderr, "ERROR: result_len(%d)!=sizeof(en_t)(%d) for lookup %s\n", result_len, sizeof(en_t), inName); exit(-1); } memcpy(&global_names[i], result, sizeof(en_t)); TIC_AM_Safe(AM_Map(TIC_AM_ENDPOINT, i, global_names[i], TIC_TAG)); } } } } #elif defined(COMM_AMLAPI) int setupNames() { int i; int counter = 0; /* Array to store en_t address from all tasks */ void **global_addr = (void**) ti_malloc(sizeof(void*)*COMM_GetBParallelDegree()); /* Buffer of en_t address */ en_t *buffer = (en_t *) ti_malloc(sizeof(en_t)); int* buffer_test = (int *) ti_malloc (sizeof (int)); /* Origin and Target Counter with address */ void **tgt_addr = (void**) ti_malloc(sizeof(void*)*COMM_GetBParallelDegree()); lapi_cntr_t lapi_cntr; /* Origin counter */ lapi_handle_t *lapi_hndl = AMLAPI_GetLAPIHandle (); /* Get My LAPI endpoint */ int translations; TIC_AM_Safe(AM_MaxNumTranslations(&translations)); if (translations < COMM_GetBParallelDegree()) { fprintf(stderr, "ERROR: AM_MaxNumTranslations (%d) less than BOXES (%d).\n", translations, COMM_GetBParallelDegree()); exit(-1); } TIC_AM_Safe(AM_GetNumTranslations(TIC_AM_ENDPOINT, &translations)); if (translations < COMM_GetBParallelDegree()) { TIC_AM_Safe(AM_SetNumTranslations(TIC_AM_ENDPOINT, COMM_GetBParallelDegree())); } /* Initialize counters to be zero at the start */ LAPI_Setcntr(*lapi_hndl, &lapi_cntr, 0); /* Exchange buffer address and target counter of every task */ LAPI_Address_init (*lapi_hndl, &TIC_AM_ENDPOINTNAME, global_addr); LAPI_Gfence(*lapi_hndl); /* Global barrier to sync before starting */ for (i = 0; i < COMM_GetBParallelDegree(); i++) { if (COMM_GetMyBoxNumber() != i) { LAPI_Get(*lapi_hndl, i, sizeof(en_t), (void*) global_addr[i], (void*) buffer, NULL, &lapi_cntr); while( counter == 0 ) LAPI_Getcntr(*lapi_hndl, &lapi_cntr, &counter); LAPI_Setcntr(*lapi_hndl, &lapi_cntr, 0); counter = 0; #if 0 printf ("%i thinks %i's endpoint name is: %i/%i and it's own endpoint name is %i/%i \n", AMLAPI_MyProc (), i, buffer->node, buffer->endpoint_id, TIC_AM_ENDPOINTNAME.node, TIC_AM_ENDPOINTNAME.endpoint_id); #endif TIC_AM_Safe(AM_Map(TIC_AM_ENDPOINT, i, *buffer, TIC_TAG)); } LAPI_Gfence(*lapi_hndl); /* Global barrier to sync before starting */ } } #else void setupNames() { fprintf(stderr, "ERROR: tried to access a global endpoint name server, but Tic was compiled with no nameserver\n"); fflush(stderr); exit(1); } #endif #endif /* COMM_AM2 */ #ifdef COMM_AM2 void setupAM() { #ifdef COMM_GASNET /* nothing to do - (GASNet handler registration happens in setup_global_layer) */ #else uintptr_t maxSegLength; int maxHandlers; int i; if (!usingAMSPMD) { TIC_AM_Safe(AM_Init()); TIC_AM_Safe(AM_AllocateBundle(AM_SEQ, &TIC_AM_BUNDLE)); TIC_AM_Safe(AM_AllocateEndpoint(TIC_AM_BUNDLE, &TIC_AM_ENDPOINT, &TIC_AM_ENDPOINTNAME)); } TIC_AM_Safe(AM_MaxSegLength(&maxSegLength)); #ifdef USE_GC_NONE TIC_AM_Safe(AM_SetSeg(TIC_AM_ENDPOINT, (void *)NULL, maxSegLength)); TIC_AM_SEGOFFSET = NULL; #else TIC_AM_Safe(AM_SetSeg(TIC_AM_ENDPOINT, (void *)DATASTART, maxSegLength)); TIC_AM_SEGOFFSET = DATASTART; #endif /* TIC_AM_Safe(AM_SetEventMask(TIC_AM_BUNDLE, AM_NOEVENTS)); */ /* Setup Handlers */ if ((maxHandlers = AM_MaxNumHandlers()) < tic_numhandlers) { fprintf(stderr, "ERROR: Only %d handler table entries available; need %d\n", maxHandlers, tic_numhandlers); exit(1); } for (i = 0; i < tic_numhandlers; i++) { TIC_AM_Safe(AM_SetHandler(TIC_AM_ENDPOINT, tic_handler_list[i].handler_index, tic_handler_list[i].func)); } if (!usingAMSPMD) TIC_AM_Safe(AM_SetTag(TIC_AM_ENDPOINT, TIC_TAG)); #endif tic_amsize_init(); } #endif /* COMM_AM2 */ int parseNumNodes(int *pargc, char ***pargv) { /* called by setup_global_layer to parse and remove the number of nodes * argument for an implicit startup master */ int num_nodes; int i; if (*pargc < 2) { fprintf(stderr, "Tic: Missing parallel degree\n"); fprintf(stderr, "Tic: Specify degree as first argument or use rexec/glurun/tcrun to start job\n"); fprintf (stderr,"Tic: Usage '%s {program arguments}'\n", (*pargv)[0]); exit(-1); } /* * argv[1] is number of nodes; argv[0] is program name; argv is * list of arguments including program name and number of nodes. * We need to remove argv[1] when the argument array is passed * to the tic_main(). */ num_nodes = atoi((*pargv)[1]); if (num_nodes < 1) { fprintf (stderr, "Tic: Invalid number of nodes: %s\n", (*pargv)[1]); fprintf (stderr, "Tic: Usage '%s {program arguments}'\n", (*pargv)[0]); exit (1); } #ifdef GLUNIX { int i = Glib_GetNumAvailableNodes(); if (i < num_nodes) { fprintf (stderr, "Tic: Error - Only %d nodes available (requested %d).\n", i, num_nodes); exit (1); } } #endif /* remove the num_nodes argument */ for (i = 1; i < (*pargc)-1; i++) { (*pargv)[i] = (*pargv)[i+1]; } (*pargv)[(*pargc)-1] = NULL; (*pargc)--; if (!getenv("TI_BACKEND_SILENT")) { printf("Tic: Spawning %s on %d nodes\n", (*pargv)[0], num_nodes); fflush(stdout); } return num_nodes; } extern julong gpid; #ifdef COMM_AMUDP #if defined(TI_CSPAWN_CMD) TI_IDENT(AMUDP_DEFAULT_SPAWNFN_IDENT_STRING, "$TitaniumDefaultSpawnFunction: C $"); TI_IDENT(AMUDP_DEFAULT_CSPAWNCMD_IDENT_STRING, "$TitaniumCSpawnCommand: " TI_CSPAWN_CMD " $"); #elif defined(REXEC) TI_IDENT(AMUDP_DEFAULT_SPAWNFN_IDENT_STRING, "$TitaniumDefaultSpawnFunction: R $"); #elif defined(GLUNIX) TI_IDENT(AMUDP_DEFAULT_SPAWNFN_IDENT_STRING, "$TitaniumDefaultSpawnFunction: G $"); #else /* AMUDP implicit ssh startup */ TI_IDENT(AMUDP_DEFAULT_SPAWNFN_IDENT_STRING, "$TitaniumDefaultSpawnFunction: S $"); #endif #endif void tic_killmyprocess(int exitcode) { /* wrapper for _exit() that does the "right thing" to immediately kill this process */ #if defined(MEMORY_SHARED) && defined(HAVE_PTHREAD_KILL_OTHER_THREADS_NP) /* on LinuxThreads we need to explicitly kill other threads before calling _exit() */ pthread_kill_other_threads_np(); #endif _exit(exitcode); /* use _exit to bypass atexit handlers */ abort(); /* never reach here */ } void setup_global_layer(int *pargc, char ***pargv) { /* this function is responsible for performing "implicit" job startup. * That is, spawning a parallel job when the application was executed * directly without the use of a job spawner like glurun, rexec, tcrun, etc. * It also takes advantage of AMUDP's flexible job startup API where appropriate * The command-line format for all implicit startups is: * {program args...} */ #if defined(GLUNIX) || defined(COMM_AMUDP) || defined(COMM_AMMPI) || defined(COMM_GASNET) /* we only support implicit startup for these job spawning systems */ #else return; #endif #ifdef COMM_GASNET #ifndef GASNET_SEGMENT_EVERYTHING #error GASNet backend compiled without GASNET_SEGMENT_EVERYTHING GASNet configuration #endif { /* GASNet startup may be explicit or implicit (not defined) */ int retval = gasnet_attach(tic_handler_list, tic_numhandlers, (uintptr_t)-1, 0); if (retval != GASNET_OK) { fprintf(stderr, "ERROR: calling gasnet_attach: %i\n", retval); abort(); } } #endif #ifdef COMM_AMMPI {/* AMMPI startup is always explicit, but we'll handle it here */ int networkdepth=0; #if 0 { int initialized = 0; char systemErrDesc[MPI_MAX_ERROR_STRING+10]; int len = MPI_MAX_ERROR_STRING; int errval; /* Init MPI, if necessary (this may result in job spawn, depending on the MPI implementation) */ if ((errval=MPI_Initialized(&initialized)) != MPI_SUCCESS) { if (MPI_Error_string(errval, systemErrDesc, &len) != MPI_SUCCESS || len == 0) strcpy(systemErrDesc, "(no description available)"); fprintf(stderr,"Error calling MPI_Initialized(): %s\n", systemErrDesc); fflush(stderr); abort(); } if (!initialized) { if ((errval=MPI_Init(pargc, pargv)) != MPI_SUCCESS) { if (MPI_Error_string(errval, systemErrDesc, &len) != MPI_SUCCESS || len == 0) strcpy(systemErrDesc, "(no description available)"); fprintf(stderr,"Error calling MPI_Initialized(): %s\n", systemErrDesc); fflush(stderr); abort(); } } } #endif AMMPI_VerboseErrors = 1; /* get detailed info about any failures */ AMMPI_SPMDkillmyprocess = &tic_killmyprocess; if (getenv("TI_BACKEND_SILENT")) { AMMPI_SilentMode = 1; /* suppress diagnostic output */ } if (getenv("TI_NETWORKDEPTH")) networkdepth = atoi(getenv("TI_NETWORKDEPTH")); if (networkdepth < 1) networkdepth = 20; /* a reasonable default for many systems */ /* startup this worker */ { int rank; TIC_AM_Safe(AMMPI_SPMDStartup(pargc, pargv, networkdepth, &gpid, &TIC_AM_BUNDLE, &TIC_AM_ENDPOINT)); rank = AMMPI_SPMDMyProc(); TIC_AM_Safe(AM_GetTranslationName(TIC_AM_ENDPOINT, rank, &TIC_AM_ENDPOINTNAME)); /* lookup our name */ usingAMSPMD = 1; /* signal that we've completed endpoint initialization */ return; } } #endif /* generic initialization */ #ifdef GLUNIX if (!(Glib_Initialize() != 0)) { fprintf(stderr, "Global layer initialization failed.\n"); fflush(stderr); exit(-1); } #endif /* Glunix */ #ifdef COMM_AMUDP AMUDP_VerboseErrors = 1; /* get detailed info about any failures */ if (getenv("TI_BACKEND_SILENT")) { AMUDP_SilentMode = 1; /* suppress diagnostic output */ } if (AMUDP_SPMDIsWorker(*pargv)) { /* we're an implicit worker using AMUDP SPMD API */ int rank; TIC_AM_Safe(AMUDP_SPMDStartup(pargc, pargv, 0, 0, NULL, &gpid, &TIC_AM_BUNDLE, &TIC_AM_ENDPOINT)); rank = AMUDP_SPMDMyProc(); TIC_AM_Safe(AM_GetTranslationName(TIC_AM_ENDPOINT, rank, &TIC_AM_ENDPOINTNAME)); /* lookup our name */ usingAMSPMD = 1; /* signal that we've completed endpoint initialization */ if (getenvMaster("TI_BACKEND_SILENT")) { AMUDP_SilentMode = 1; /* suppress diagnostic output */ } AMUDP_SPMDkillmyprocess = &tic_killmyprocess; } else { /* decide whether we're an implicit master */ int isExplicitWorker = 0; char spawnfn = '\0'; #ifdef REXEC isExplicitWorker = (int)getenv("REXEC_PAR_DEGREE"); spawnfn = 'R'; #elif defined(GLUNIX) isExplicitWorker = !Glib_AmIStartup(); spawnfn = 'G'; #else /* AMUDP implicit ssh startup */ isExplicitWorker = 0; /* we don't support explicit ssh startup */ spawnfn = 'S'; #endif if (!isExplicitWorker) { /* we are an implicit master */ amudp_spawnfn_t fp = NULL; int i; int networkdepth = 0; int num_nodes = parseNumNodes(pargc, pargv); if (getenv("TI_SPAWNFN")) /* user has overridden default implicit job spawner */ spawnfn = *getenv("TI_SPAWNFN"); if (getenv("TI_NETWORKDEPTH")) networkdepth = atoi(getenv("TI_NETWORKDEPTH")); if (networkdepth < 1) networkdepth = 20; /* a reasonable default for many systems */ for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) { if (toupper(spawnfn) == toupper(AMUDP_Spawnfn_Desc[i].abbrev)) { fp = AMUDP_Spawnfn_Desc[i].fnptr; break; } } if (!fp) { fprintf (stderr, "Tic: Invalid spawn function specified in TI_SPAWNFN\n"); fprintf (stderr, "Tic: The following mechanisms are available:\n"); for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) { fprintf(stderr, " '%c' %s\n", toupper(AMUDP_Spawnfn_Desc[i].abbrev), AMUDP_Spawnfn_Desc[i].desc); } exit (1); } TIC_AM_Safe(AMUDP_SPMDStartup(pargc, pargv, num_nodes, networkdepth, fp, &gpid, NULL, NULL)); abort(); /* master startup should never return */ } } #else /* not COMM_AMUDP */ #ifdef GLUNIX if (Glib_AmIStartup()) { int num_nodes = parseNumNodes(pargc, pargv); /* Spawn only returns if it failed */ if (Glib_Spawn(num_nodes, (*pargv)[0], *pargv)) { perror("Global spawn failed"); exit(-1); } abort(); } #endif /* Glunix */ #endif } #define TIENV_FILENAME ".tienv" /* read and parse the .tienv file in the executable directory for optional * environment initialization (required for some distributed spawners that don't properly * propagate environment variables to the slave processes). Input is program pathname. */ void readEnvVarFile(char *progpathname) { char filename[255]; char line[1024]; int linenum = 0; char *p; FILE *fp; strcpy(filename, progpathname); /* look for file in executable directory */ if (strchr(filename,'/')) p = strrchr(filename,'/'); else p = strrchr(filename,'\\'); if (p) *(p+1) = '\0'; else filename[0] = '\0'; strcat(filename, TIENV_FILENAME); fp = fopen(filename, "r"); if (!fp) return; while (fgets(line, 1024, fp)) { char *p = line; char *pend = NULL; char delim; char name[255]; char value[1024]; int sawequal=0; linenum++; while (*p && isspace(*p)) p++; if (!*p || *p == '#') continue; /* # starts a comment */ if (*p == '"') { p++; pend = p + strcspn(p, "\""); } /* get name */ else pend = p + strcspn(p, "\t ="); strncpy(name, p, pend-p); name[pend-p] = '\0'; if (*pend && *pend != '=') p = pend+1; else p = pend; while (*p && isspace(*p)) p++; if (*p == '=') { sawequal=1; p++; } while (*p && isspace(*p)) p++; if (*p == '"') { p++; pend = p + strcspn(p, "\""); } /* get name */ else pend = p + strcspn(p, "\t "); strncpy(value, p, pend-p); value[pend-p] = '\0'; if (!*name || !sawequal) { fprintf(stderr,"Tic: Ignoring illegal environment initializer at %s:%i.\n" "Tic: format is one variable per line: name = value\n" "Tic: where name and value may be \"quoted\" and name must be non-empty.\n", filename, linenum); } else { if (setenv(name, value, 1)) perror("setenv: in readEnvVarFile()"); } } if (ferror(fp)) perror("fgets()"); if (!getenv("TI_BACKEND_SILENT")) printf("Tic: Reading environment variable settings from %s\n", filename); } void free_resources () { #ifdef COMM_GASNET gasnet_exit(0); #elif defined(COMM_AM2) if (usingAMSPMD) { /* add a pause to help ensure all final output has reached the console */ fflush(stdout); fflush(stderr); tic_sched_yield(); sleep(1); tic_sched_yield(); fflush(stdout); fflush(stderr); #if defined(COMM_AMUDP) AMUDP_SPMDExit(0); #elif defined(COMM_AMMPI) AMMPI_SPMDExit(0); #endif } else if (BOXES > 1) { /* need to prevent more than one thread from running this */ static ti_lock_t cleanup_lock = ti_lock_decl_initializer; static int cleanup_complete = 0; ti_lock(&cleanup_lock); if (!cleanup_complete) { TIC_AM_Safe(AM_FreeEndpoint (TIC_AM_ENDPOINT)); TIC_AM_Safe(AM_FreeBundle (TIC_AM_BUNDLE)); TIC_AM_Safe(AM_Terminate()); #ifdef HAVE_ENAMESERVER_H NS_Safe((NameServer_Finish(),0)); /* NameServer_Finish() has a void return */ #endif cleanup_complete = 1; } ti_unlock(&cleanup_lock); } #endif /* AMII */ } extern void printExitStats(int sglobal); static int init_complete = 0; void free_resource_handler (int sig) { if (init_complete) printExitStats(0); free_resources (); #ifdef GLUNIX /* Glunix needs this to prevent orphaned processes */ Glib_Signal(Glib_GetMyNpid(), VNN_ALL, SIGTERM); #endif /* often unsafe to call exit from signal handler - call _exit() instead */ _exit (0); } tic_thread_key_t tic_thread_key; #ifdef AIX /* DOB: work-around for missing symbols we don't use when linking statically on AIX */ int getpass_auto() { abort(); } int max_pw_passlen() { abort(); } int crypt_r() { abort(); } int salt_len() { abort(); } int encrypted_pw_passlen() { abort(); } int max_history_size() { abort(); } int _valid_crypted() { abort(); } #endif Process TIC_PROCS; Box TIC_BOXES; Box TIC_MYBOX; #ifdef MEMORY_SHARED Process TIC_MYBOXPROCS; #else Process TIC_MYPROC; #endif void memory_init(); void stats_init(void); extern int COMM_init_box_const_complete; void init_box_consts() { /* This process is a node in a parallel job. */ TIC_PROCS = COMM_GetParallelDegree(); TIC_BOXES = COMM_GetBParallelDegree(); TIC_MYBOX = COMM_GetMyBoxNumber(); #ifdef MEMORY_SHARED TIC_MYBOXPROCS = COMM_GetMyBParallelDegree(); #else TIC_MYPROC = myProcessNumber(); #endif COMM_init_box_const_complete = 1; com_init(); store_init(); memory_init(); #if defined(USE_DISTRIBUTED_GC) && !defined(USE_GC_NONE) dgc_init(); #endif #ifdef COMM_AM2 gather_init(); #endif stats_init(); tic_amlock_init(); tic_init_staticdata_translation(); COMM_Init_Gpid(); } extern char *RUNTIME_MODEL; char *RUNTIME_MODEL = TIC_BACKEND_NAME; /* these strings are read from the executable by ident and tcrun */ TI_IDENT(BackendIdent, "$TitaniumBackend: " TIC_BACKEND_NAME " $"); TI_IDENT(ConfigureArgs, "$TitaniumConfigureArgs: " TI_CONFIGURE_ARGS " $"); extern void region_init(void); void *tic_main(int argc, char **argv) { ti_main(argc, argv); return 0; } void *thread_go(void *procinfo) { processInfo *procInfo = (processInfo *)procinfo; int result = tic_thread_set_key_value(tic_thread_key, (void *)procInfo); if (result != 0) { printf("error in tic thread set key value: %d\n", result); return NULL; } #if defined(__alpha) || defined(_CRAYT3E) || defined(__crayx1) /* DOB: Setup a trap handler for SIGFPE (floating point exception) to keep it from crashing us The Alpha CPU delivers a floating point exception leading to SIGFPE whenever a floating point exception occurs, and it's up to the OS to clean things up - but it can only properly do so if we compiled the code with -ieee (Compaq C), which makes all FP ops restartable but also lowers performance (more info, see man ieee(3)). If we compile with -ieee, the OS ensures we get all the right IEEE FP results, even with SIGFPE ignored (but we take a performance hit). Without -ieee, the SIGFPE is usually fatal because the results are not well-defined. However, experiments (on a 21264A) indicate that if you ignore the signal, exceptional operations still correctly produce NaN's and Infinity's, and the only thing we lose is denormalized IEEE FP results instead become zeroes (which we can live with by default, and users who care can add -ieee to the C flags). Cray X1 has the same issue, although there appears to be no way to get the correct IEEE behavior - "cc -h fp0" ensure we get the signal for NaN's and divbyzero (rather that just for divbyzero, which is the default), but there doesn't appear to be a mode to cause the proper fixup to happen on signal. Ignoring the signal seems to result in Inf and NaN being correctly generated, but it seems the hardware never creates a denormalized FP number (at least on scalar CPU) Furthermore, even with the FPE signal ignored, we still get ugly warnings to stdout. Cray sucks. */ if (signal(SIGFPE, SIG_IGN) == SIG_ERR) { perror("signal(SIGFPE, SIG_IGN)"); abort(); } #endif #ifdef DEBUG_THREADS fprintf(stderr,"Processor starting MYPROC=%i PROCS=%i MYBOX=%i BOXES=%i MYBOXPROC=%i MYBOXPROCS=%i\n", MYPROC, PROCS, MYBOX, BOXES, MYBOXPROC, MYBOXPROCS); fflush(stderr); #endif ti_set_srcpos("_STARTUP",0); /* tell trace to ignore startup communication on each thread */ { char hostname[MAXHOSTNAMELEN]; gethostname(hostname, sizeof(hostname)); ti_trace_printf(( "GASNET_TRACE_MAGIC: I am thread %d of %d, on node %d of %d (%s) in job ", (int)MYPROC, (int)PROCS, (int)MYBOX, (int)BOXES, hostname, (unsigned long long)COMM_GetMyGuid64())); } local_barrier(); /* wait for all local threads */ region_init(); monitor_startup(); ti_set_srcpos("_STARTUP",0); barrier(); init_complete = 1; ti_main (procInfo->box->argc, procInfo->box->argv); ti_set_srcpos("_FINAL_BARRIER",0); barrier(); printExitStats(1); return NULL; } #ifdef PTHREAD #ifdef PTHREAD_STACK_MIN #define TIC_PTHREAD_STACK_MIN PTHREAD_STACK_MIN #else #define TIC_PTHREAD_STACK_MIN GASNETT_PAGESIZE #endif #ifndef TIC_STACK_DEFAULT #define TIC_STACK_DEFAULT (2*1024*1024) /* 2MB */ #endif static intptr_t __tic_threadstacksz = -1; int __tic_create_pthread(void *(*thread_func)(void *), void *arg, tic_thread_t *threadptr) { if (__tic_threadstacksz == -1) { /* find a suitable stack size for pthreads we're creating, with optional user intervention */ const char *ss = getenvMaster("TI_STACK_SIZE"); pthread_attr_t attr; __tic_errcheckZ(pthread_attr_init(&attr)); __tic_threadstacksz = TIC_STACK_DEFAULT; if (ss) { const char *p = ss; for ( ; *p ; p++) if (!isdigit(*p) && !isspace(*p)) { fprintf(stderr, "ERROR: TI_STACK_SIZE must be an integer >= %i\n", TIC_PTHREAD_STACK_MIN); abort(); } __tic_threadstacksz = atoi(ss); } __tic_threadstacksz = MAX(__tic_threadstacksz, TIC_PTHREAD_STACK_MIN); while (pthread_attr_setstacksize(&attr, __tic_threadstacksz) == EINVAL) { if (__tic_threadstacksz <= TIC_PTHREAD_STACK_MIN) { ti_trace_printf(("pthread stack size search failed")); __tic_threadstacksz = 0; break; } else __tic_threadstacksz /= 2; } if (__tic_threadstacksz) ti_trace_printf(("pthread stack size set to %lu", (unsigned long)__tic_threadstacksz)); } { pthread_attr_t attr; int attrinit_knownerr = 0, setscope_knownerr = 0; #if defined(CYGWIN) /* buggy Cygwin pthreads lib intermittently generates EBUSY on pthread_attr_init, for no apparent reason */ attrinit_knownerr = EBUSY; #endif #if defined(CYGWIN) || defined(FREEBSD) /* these apparently do not support PTHREAD_SCOPE_SYSTEM contention scope */ setscope_knownerr = ENOTSUP; #endif #if defined(IRIX) /* this failure occurs if process lacks the CAP_SCHED_MGT capability */ setscope_knownerr = EPERM; #endif __tic_errcheckZopt(pthread_attr_init(&attr), attrinit_knownerr); __tic_errcheckZopt(pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM), setscope_knownerr); if (__tic_threadstacksz) { __tic_errcheckZ(pthread_attr_setstacksize(&attr, __tic_threadstacksz)); } #ifdef HAVE_PTHREAD_ATTR_SETGUARDSIZE /* set a guard page, if supported (ignore errors) */ pthread_attr_setguardsize(&attr, GASNETT_PAGESIZE); #endif return __tic_errcheckZ(pthread_create(threadptr, &attr, thread_func, arg)); } } #endif void spawn_threads() { int i; processInfo *thisproc; tic_thread_key_init(&tic_thread_key); barrier_init(); /* this must happen on the primordial thread before thread spawn */ #ifdef MEMORY_DISTRIBUTED /* need an external barrier here to allow titanium barrier to be bootstrapped on all nodes */ #if defined(COMM_AMUDP) if (usingAMSPMD) AMUDP_SPMDBarrier(); else #elif defined(COMM_AMMPI) if (usingAMSPMD) AMMPI_SPMDBarrier(); else #endif { #if defined(GLUNIX) if (!Glib_Barrier()) { perror("global barrier"); exit(-1); } #elif defined( COMM_AMLAPI ) lapi_handle_t *lapi_hndl = AMLAPI_GetLAPIHandle (); /* Get My LAPI endpoint */ LAPI_Gfence(*lapi_hndl); /* Global barrier to sync before starting */ #elif defined(COMM_GASNET) gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); gasnet_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS); #else abort(); /* we need something else here - barrier() is no good */ barrier(); /* this is not strictly correct and may cause problems */ #endif /* Glunix */ } #endif /* this thread will eventually act as thread 0 for this box */ thisproc = COMM_GetHisBoxProcess(0); tic_thread_set_key_value(tic_thread_key, thisproc); thisproc->threadID = tic_thread_self(); init_box_consts(); /* called on proc 0 before other threads spawned */ /* spawn other threads */ for (i = 1; i < MYBOXPROCS; i++) { int result; thisproc = COMM_GetHisBoxProcess(i); if (tic_create_thread(thread_go, (void *)thisproc, &thisproc->threadID)) abort(); #ifdef DEBUG_THREADS printf("created thread 0x%08x\n",(int) thisproc->threadID); #endif } thisproc = COMM_GetHisBoxProcess(0); thread_go(thisproc); /* run process 0 */ return; } void join_threads() { int i; for (i = 1; i < MYBOXPROCS; i++) { int result; processInfo *thisproc = COMM_GetHisBoxProcess(i); #ifdef DEBUG_THREADS printf("joining thread 0x%08x\n",(int) thisproc->threadID); #endif if (tic_thread_join(thisproc->threadID, NULL)) abort(); } } static volatile int tic_frozen = 1; static int _frozen() { /* ensure a full stack frame */ return *&tic_frozen; } void freezeForDebugger(char *hostname) { printf("frozen for debugger: host %s, pid %d\n", hostname, getpid()); fflush(stdout); while (_frozen()) tic_sched_yield(); } int real_main (int argc, char** argv, char** envp); int main (int argc, char** argv, char** envp) { int dummy = 0; ti_gc_stack_init(&dummy); /* called to set the cold stack ptr for the primordial thread */ return real_main(argc, argv, envp); } int real_main (int argc, char** argv, char** envp) { int countBoxMem, countProcMem; char *boxMem, *procMem; char hostname[MAXHOSTNAMELEN] = "(unknown)"; tic_maxrlimit(); /* max out process resource limits */ progName = argv[0]; readEnvVarFile(progName); /* must precede gasnet_init to get tracing options */ #ifdef COMM_GASNET { /* GASNet init must precede inspection of environment and cmd-line args */ int retval = gasnet_init(&argc, &argv); if (retval != GASNET_OK) { fprintf(stderr, "ERROR: calling gasnet_init: %i\n", retval); abort(); } } #endif /* do these *very* early, in case library routines allocate memory */ ti_gc_mem_init(); /* init the GC for the main thread - not necessary for dynamically created threads, should only be run on the primordial thread */ ti_set_srcpos("_STARTUP",0); /* tell trace to ignore startup communication on primordial thread */ setup_global_layer(&argc, &argv); /* must come before any worker-related code, as this may be the startup master node */ signal (SIGQUIT, free_resource_handler); #ifndef COMM_GASNET signal (SIGHUP, free_resource_handler); signal (SIGINT, free_resource_handler); signal (SIGTERM, free_resource_handler); #endif countBoxMem = COMM_Init_One(); boxMem = (char *) ti_malloc(countBoxMem); memset(boxMem, 0, countBoxMem); gethostname(hostname, sizeof(hostname)); if (getenvMaster("TI_FREEZE")) freezeForDebugger(hostname); countProcMem = COMM_Init_Two(boxMem); procMem = (char *) ti_malloc(countProcMem); memset(procMem, 0, countProcMem); COMM_Init_Three(procMem); #if defined(MEMORY_DISTRIBUTED) || defined(MEMORY_SHARED) if (!getenvMaster("TI_BACKEND_SILENT")) { if (COMM_GetMyBParallelDegree() == 1) { printf("Tic: %s is OS process %d of %d (Ti proc %i of %i)\n", hostname, COMM_GetMyBoxNumber(), COMM_GetBParallelDegree(), COMM_GetHisBoxProcess(0)->processNumber, COMM_GetParallelDegree()); } else { printf("Tic: %s is OS process %d of %d (Ti procs %i..%i of %i)\n", hostname, COMM_GetMyBoxNumber(), COMM_GetBParallelDegree(), COMM_GetHisBoxProcess(0)->processNumber, COMM_GetHisBoxProcess(0)->processNumber + COMM_GetMyBParallelDegree() - 1, COMM_GetParallelDegree()); } fflush(stdout); } #endif if (COMM_GetBParallelDegree() > 1) { #ifdef COMM_AM2 setupAM(); #else /* COMM_AM2 */ fprintf(stderr, "Error: Trying to use more than one box without a known communications layer.\n"); return 2; #endif /* COMM_AM2 */ } COMM_Init_Four(argc, argv); if (COMM_GetBParallelDegree() > 1) { #ifdef COMM_AM2 if (usingAMSPMD) { #if defined(COMM_AMUDP) TIC_AM_Safe(AMUDP_SPMDBarrier()); /* need a barrier after handler setup */ #elif defined(COMM_AMMPI) TIC_AM_Safe(AMMPI_SPMDBarrier()); /* need a barrier after handler setup */ #endif } else setupNames(); #if defined(COMM_AMUDP) || defined(COMM_AMMPI) /* AMUDP/AMMPI requires us to specify the number of peers and the desired network depth */ if (!usingAMSPMD) { /* SPMD startup already takes care of this */ int networkdepth = 0; if (getenv("TI_NETWORKDEPTH")) networkdepth = atoi(getenv("TI_NETWORKDEPTH")); if (networkdepth < 1) networkdepth = 20; /* a reasonable default for many systems */ TIC_AM_Safe(AM_SetExpectedResources(TIC_AM_ENDPOINT, COMM_GetBParallelDegree(), networkdepth)); } #endif #else /* COMM_AM2 */ fprintf(stderr, "Error: Trying to use more than one box without a known communications layer.\n"); return 2; #endif /* COMM_AM2 */ } tic_thread_set_concurrency(COMM_GetMyBParallelDegree()); spawn_threads(); join_threads(); fflush(stdout); fflush(stderr); #ifdef GLUNIX /* don't free AM endpoints until we're sure everyone is ready (AM-based barrier() not good enough) */ if (!Glib_Barrier()) { perror("global barrier"); exit(-1); } #endif /* Glunix */ free_resources (); return 0; /* if Titanium program returned from void main, that means return 0 */ }