#ifdef not
static char rcsid[] = "$Id: ompsm_thread.c,v 1.31 2002/10/09 02:03:49 a-hasega Exp $";
#endif
/* 
 * $RWC_Release: Omni-1.6 $
 * $RWC_Copyright:
 *  Omni Compiler Software Version 1.5-1.6
 *  Copyright (C) 2002 PC Cluster Consortium
 *  
 *  This software is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License version
 *  2.1 published by the Free Software Foundation.
 *  
 *  Omni Compiler Software Version 1.0-1.4
 *  Copyright (C) 1999, 2000, 2001.
 *   Tsukuba Research Center, Real World Computing Partnership, Japan.
 *  
 *  Please check the Copyright and License information in the files named
 *  COPYRIGHT and LICENSE under the top  directory of the Omni Compiler
 *  Software release kit.
 *  
 *  
 *  $
 */

/* 
 * RWC OpenMP C compiler Run-time libraries 
 *  for shmem interface 
 */
#include "ompsm.h"
#include <unistd.h>
#include <stdlib.h>

int _ompc_debug_flag = FALSE;	/* debug output control */
int _ompc_n_proc = N_PROC_DEFAULT;	/* number of PE */

/* root of global data */
struct _global_		*__G__;

struct ompsm_thread	tp_top;
struct ompsm_thread	*_ompsm_thread = &tp_top;

int _ompc_n_node;
int _ompc_node_id;
int _ompc_in_parallel		= 0;	/* parallel nest level */
int _ompc_serialized_count	= 0;	/* parallel_if(false) count */

extern void _ompc_set_runtime_schedule(char *s);

volatile int _ompc_nested;	/* nested enable/disable */
volatile int _ompc_dynamic;	/* dynamic enable/disable */
volatile int _ompc_max_threads;	/* max number of thread */
int _ompc_num_threads;

/* prototype */
static void _ompsm_init_env(void);
static void _ompsm_finalize(void);
void _ompc_fatal(char *msg);
void _ompsm_estimate_shmem_size(void);
void _ompsm_init_global_data(void);
static int	getvalue (char *str, int *v);


/* 
 * initialize library
 */
void
_ompsm_init(int argc,char *argv[])
{
  _ompc_init(argc,argv);
}


void
_ompc_init(int argc,char *argv[])
{
  _ompsm_init_env ();

  if (_ompc_debug_flag) {
    fprintf(stderr, "_ompc_init ...\n");
    fflush(stderr);
  }

  _ompsm_estimate_shmem_size ();

  /* initialize shmem runtime */
  OMPSM_INIT (argc,argv);

  atexit (_ompsm_finalize);
  _ompsm_allocate_global_data ();

#ifdef USE_UNIX_SHMEM
  _ompc_atomic_lock_p = OMPSM_ALLOC_LOCK();
  _ompc_ordered_lock_p = OMPSM_ALLOC_LOCK();
  __G__->atomic_lock = _ompc_atomic_lock_p;
  __G__->ordered_lock = _ompc_ordered_lock_p;
#endif
  __G__->debug_flag = _ompc_debug_flag;
  __G__->runtime_sched_kind = _ompc_runtime_sched_kind;
  __G__->runtime_chunk_size = _ompc_runtime_chunk_size;
    
#ifdef USE_SCASH
  _ompsm_scash_start_slaves();
#endif    

  OMPSM_BARRIER (); /* sync with slave */

  _ompsm_init_global_data ();

  if(_ompc_debug_flag) {
    fprintf(stderr, "init end(Master)\n");
  }

}


static void 
_ompsm_init_env ()
{
  char  *cp;
  int	val, rv;
#ifndef USE_SCASH
  char	buff[BUFSIZ];
#endif

  cp = getenv("OMPC_DEBUG");
  if(cp != NULL){
    _ompc_debug_flag = TRUE;
    fprintf(stderr,"debug flag on ...\n");
  }

  cp = getenv("OMPC_NUM_PROCS");   /* processor */
  if ( cp != NULL ) {
#ifdef USE_SCASH    
    _ompc_fatal("Omni on SCASH do not support OMPC_NUM_PROCS environment valuse.\nif you want change threads, use scrun --nodes option.\n");
#else
    sscanf(cp, "%d", &val);
    if (val <= 0  ||  MAX_PROC < val) {
      sprintf (buff,
	       "bad OMPC_NUM_PROCS(shuld be set 0 < OMPC_NUM_PROCS<= %d)",
	       MAX_PROC);
      _ompc_fatal (buff);
    }
    _ompc_n_proc = val;
#endif
  }

  _ompc_nested = 0;	/* nested enable/disable */
  _ompc_dynamic = 0;	/* dynamic enable/disable */
  _ompc_max_threads = _ompc_n_proc;	/* max number of thread, default */

  cp = getenv("OMP_SCHEDULE");
  if(cp != NULL) {
    _ompc_set_runtime_schedule(cp);
  }

  cp = getenv("OMP_DYNAMIC");
  if(cp != NULL && (strcmp(cp,"TRUE") == 0 || strcmp(cp,"true") == 0)) {
    _ompc_dynamic = 1;
  }

  cp = getenv("OMP_NESTED");
  if(cp != NULL && (strcmp(cp,"TRUE") == 0 || strcmp(cp,"true") == 0)) {
    _ompc_nested = 1;
  }

  cp = getenv("OMP_NUM_THREADS");	/* a number of team member */
  if ( cp == NULL ){
    _ompc_num_threads = _ompc_n_proc;
  } else {
#ifdef USE_SCASH    
    _ompc_fatal("Omni on SCASH do not support OMP_NUM_THREADS environment valuse.\nif you want change threads, use scrun --nodes option.\n");
#else
    sscanf(cp, "%d", &val);
    if (val <= 0  ||  MAX_PROC < val) {
      sprintf (buff,
	       "bad OMPC_NUM_THREADS(shuld be set 0 < OMPC_NUM_THREADS <= %d)",
	       MAX_PROC);
      _ompc_fatal (buff);
    }
    _ompc_num_threads = val;
    if (_ompc_num_threads > _ompc_max_threads) {
      _ompc_max_threads = _ompc_num_threads;
    }
#endif
  }
#ifdef USE_SCASH
  cp = getenv("OMPC_SCASH_NO_DIST");
  if (cp != NULL) {
    _ompc_scash_dist_flag = FALSE;
    if(_ompc_debug_flag) {
      fprintf(stderr,"SCASH distribution disabled\n");
    }
  }

  if ((cp = getenv ("OMNI_SCASH_ARGS_SIZE")) != NULL) {
    rv = getvalue (cp, &val);
    if (rv != 0  ||  val <=0) {
      _ompc_fatal ("bad OMNI_SCASH_ARGS_SIZE(<=0)");
    }
    _scash_args_size = val;
  }

  /* obsolute, OMNI_SCASH_HEAP_SIZE change to OMPC_SHMEM_SIZE */
  if ((cp = getenv ("OMNI_SCASH_HEAP_SIZE")) != NULL) {
    rv = getvalue (cp, &val);
    if (rv != 0  ||  val <=0) {
      _ompc_fatal ("bad OMNI_SCASH_HEAP_SIZE(<=0)");
    }
    _ompsm_free_shmem_size = val;
  }
#endif

#if defined(USE_UNIX_SHMEM) || defined(USE_SCASH)
  if ((cp = getenv ("OMPC_SHMEM_HEAP")) != NULL) {
    rv = getvalue (cp, &val);
    if (rv != 0  ||  val <=0) {
      _ompc_fatal ("bad OMPC_SHMEM_HEAP(<=0)");
    }
    _ompsm_free_shmem_size = val;
  }
#endif
}


static int
getvalue (char *str, int *v)
{
  int	base;
  char	*ep;


  while (isspace(*str)) {
    str ++;
  }
  
  if (*str == '0') {
    if (*(str+1) == 'x') {
      str += 2;
      base = 16;
    } else {
      str += 1;
      base = 8;
    }
  } else {
    base = 10;
  }

  *v = strtol (str, &ep, base);
  switch (*ep) {
  case 'M':  case 'm':
    *v = *v * 1024 * 1024;
    ep += 1;
    break;
  case 'K':  case 'k':
    *v = *v * 1024;
    ep += 1;
    break;
  }

  while (isspace(*ep)) {
    ep += 1;
  }
  if (*ep != (char)NULL) {
    return 1;  /* error */
  }

  return 0;
}


/* finalize */
void 
_ompsm_finalize()
{
  OMPSM_LIB_IN ();
  if(_ompc_debug_flag) {
    fprintf(stderr, "_ompsm_finalize is called\n");
  }
  OMPSM_FINALIZE ();
  OMPSM_LIB_OUT ();
}


void 
_ompc_fatal(char * msg)
{
  fprintf(stderr, "OMPSM FATAL: %s\n", msg);
  exit(-1);
}


void
_ompsm_slave_main()
{

  if(_ompc_debug_flag) {
    fprintf(stderr, "id=%d:slave init ...\n",_ompc_node_id);
  }

  OMPSM_BARRIER ();		      /* sync with master */
  _ompc_debug_flag = __G__->debug_flag;
  _ompc_runtime_sched_kind =__G__->runtime_sched_kind;
  _ompc_runtime_chunk_size =__G__->runtime_chunk_size;
#ifdef USE_UNIX_SHMEM
  _ompc_atomic_lock_p =__G__->atomic_lock;
  _ompc_ordered_lock_p =__G__->ordered_lock;
#endif
  atexit (_ompsm_finalize);

  if(_ompc_debug_flag) {
    fprintf(stderr, "id=%d:slave start ...\n",_ompc_node_id);
  }
  _ompsm_init_global_data ();

  _ompc_in_parallel = 1;	      /* always in parallel */


  OMPSM_LIB_IN ();

  for (;;) {
    OMPSM_SLAVE_BEGIN();
    if(_ompc_debug_flag) {
      fprintf(stderr, "slave run id=%d : call %x args = %d\n",
	      _ompc_node_id, (unsigned int)(__G__->thd_func), __G__->thd_narg);
      if (__G__->thd_narg) {
	int	i;
	fprintf (stderr, " ( ");
	for (i=0; i<__G__->thd_narg; i++) {
	  fprintf (stderr, "%08x ", (unsigned int)__G__->thd_args[i]);
	}
	fprintf (stderr, ")");
      }
      fprintf (stderr, "\n");
    }
    if(__G__->thd_func == (cfunc)NULL) {
      break;
    }

    OMPSM_LIB_OUT ();

    (*(__G__->thd_func))(__G__->thd_args);

    OMPSM_LIB_IN ();

    OMPSM_SLAVE_END();
    if(_ompc_debug_flag) {
      fprintf(stderr, "slave stop id=%d...\n",_ompc_node_id);
    }
  }

  OMPSM_LIB_OUT ();
}


/* called from compiled code. */
void
_ompc_do_parallel_if(int cond, cfunc f, void **args, int narg)
{
  struct ompsm_thread	*prev;
  int			i;

  OMPSM_LIB_IN ();
  prev = _ompsm_thread;
  _ompsm_thread = (struct ompsm_thread *) malloc (sizeof(struct ompsm_thread));
  if (_ompsm_thread == NULL) {
    _ompc_fatal ("can not allocate memory");
  }

  if (cond && ((NESTING_COUNT) == 0)) {
    _ompc_in_parallel += 1;
    if (_ompc_debug_flag) {
      int i;
      fprintf (stderr, "do_parallel : call %x, args = %d", (unsigned int)f, narg);
      if (narg) {
	fprintf (stderr, " ( ");
	for (i=0; i<narg; i++) {
	  fprintf (stderr, "%08x ", (unsigned int)args[i]);
	}
	fprintf (stderr, ")");
      }
      fprintf (stderr, "\n");
    }
    __G__->thd_func = f;
    __G__->thd_narg = narg;
    if(narg != 0){
      if (PARALLEL_MAX_ARG < narg) {
	_ompc_fatal ("ompc_do_parallel have too many args...");
      }
      for(i = 0; i < narg; i++) {
	__G__->thd_args[i] = args[i];
      }
    } 
    OMPSM_MASTER_BEGIN();
    OMPSM_LIB_OUT ();
    (*f)(args);
    OMPSM_LIB_IN ();
    OMPSM_MASTER_END();
    _ompc_in_parallel -= 1;

  } else {
    if (cond) {
      _ompc_in_parallel += 1;
    } else {
      _ompc_serialized_count += 1;
    }
    OMPSM_LIB_OUT ();
    (*f)(args);
    OMPSM_LIB_IN ();
    if (cond) {
      _ompc_in_parallel -= 1;
    } else {
      _ompc_serialized_count -= 1;
    }
  }

  free (_ompsm_thread);
  _ompsm_thread = prev;
  OMPSM_LIB_OUT ();
}


void 
_ompc_do_parallel(cfunc f, void **args, int narg)
{
  _ompc_do_parallel_if (1, f, args, narg);
}


void *
_ompc_alloc_shared(void *vaddr, int size)
{
  void *p;

  if ((IN_PARALLEL) || (_ompc_n_node == 1)) {
    return vaddr;
  } else {
    if(OMPSM_IS_SHARED(vaddr)) {
      return vaddr;
    }

    OMPSM_LIB_IN ();
    p = (void *)OMPSM_ALLOCA(size);
    bcopy(vaddr,p,size);
    OMPSM_LIB_OUT ();

    return p;
  }
}


void 
_ompc_free_shared(void *p,void *vaddr,int size)
{
  if ((NOT_IN_PARALLEL) && (_ompc_n_node != 1)) {
    if(OMPSM_IS_SHARED(vaddr)) {
      return;
    }

    OMPSM_LIB_IN ();
    bcopy(p,vaddr,size);
    OMPSM_FREEA(p,size);
    OMPSM_LIB_OUT ();
  }
}


void
_ompc_terminate (int exitcode)
{
  OMPSM_TERMINATE (exitcode);
}