#!/bin/csh

onintr interrupted

############################################################################
# Determine the default NWChem executable.  
# .  if NWCHEM_EXECUTABLE is defined use that if it exists
# .  else if NWCHEM_TOP is defined use the version in that tree if it exists
# .  else use the build by Rick
############################################################################

if ($?NWCHEM_EXECUTABLE) then
  set NWCHEM = $NWCHEM_EXECUTABLE
  if (-e $NWCHEM) goto got_default_executable
endif
if ($?NWCHEM_TOP && $?NWCHEM_TARGET) then
  set NWCHEM = $NWCHEM_TOP/bin/$NWCHEM_TARGET/nwchem
  if (-e $NWCHEM) goto got_default_executable
endif
set NWCHEM = ~nwchem/bin/nwchem
if (! -e $NWCHEM) set NWCHEM = "no-default-available"
got_default_executable:

############################################################################
# Determine the full path to the current directory.  This must
# constructed to be machine independent so refer it to the login directory
# using tilde for brevity of output (hence the control of globbing and
# avoiding double quotes when testing for existence.
############################################################################

pushd $HOME >& /dev/null
set PWDHOME = `pwd`
popd >& /dev/null
set noglob
set FULLPATH = `pwd | sed -e "s,$PWDHOME,~,"`
unset noglob

############################################################################
# Other miscellaneous defaults
############################################################################

set TIME = "1:00:00"
set NPROC = 1
set NTASK = 1
set NNODE = 1
set QUEUE = batch
set MDON = "n"
set HOST = "SP"
#set ACCOUNT = " "
set ACCOUNT = $MP_ACCOUNT
#if ($?ACCOUNT_NO) then
#  set ACCOUNT = $ACCOUNT_NO
#endif

############################################################################
# If there are no arguments then prompt for data, otherwise
# process the input line arguments
############################################################################

if ($#argv == 0) then

# Get INPUT, OUTPUT, NPROC, TIME, QUEUE

  set INTERACTIVE = 1
  echo " "
  echo "NWChem interactive load leveller submission"
  echo "-------------------------------------------"
  echo " "

  again3:
  echo -n "What type of IBM are you submitting too? (SP/SMP): "
  set HOST = "$<"
  if ("$HOST" != "SP" && "$HOST" != "SMP") then
     echo "Invalid input $HOST, try again"
     goto again3
  endif
  echo ""
  again2:
  set DEFAULTINPUT = "$FULLPATH/nwchem.nw"
  echo -n "Input file name ($DEFAULTINPUT): "
  set INPUT = "$<"
  if ("$INPUT" == "") set INPUT = "$DEFAULTINPUT"
# See if we need to prepend the full path
  set TEST = "$FULLPATH/$INPUT"
  if (-f $TEST) then
     set INPUT = "$TEST"
  else if (-f $TEST.nw) then
     set INPUT = "$TEST.nw"
  endif
  if (! -f $INPUT) then
    echo "  unable to locate input file $INPUT"
    goto again2
  endif

# Construct an output filename that does not already exist

  set noglob
  set OUTPUT = `dirname $INPUT`/`basename $INPUT .nw`.out
  unset noglob
  @ i = 1
  while (-e $OUTPUT) 
    echo " $OUTPUT already exists ... trying again"
    set noglob
    set OUTPUT = `dirname $INPUT`/`basename $INPUT .nw`.$i.out
    unset noglob
    @ i ++
  end

  echo -n "Full path for output ($OUTPUT): "
  set TEST = "$<"
  if ("$TEST" != "") set OUTPUT = "$TEST"

  if ("$HOST" == "SP") then
     echo -n "Number of processors (1): "
     set NPROC = "$<"
     if ("$NPROC" == "") set NPROC = 1
  else
     echo -n "Number of SMP nodes (1): "
     set NNODE = "$<"
     if ("$NNODE" == "") set NNODE = 1
     echo -n "Number of processors per node (1): "
     set NTASK = "$<"
     if ("$NTASK" == "") set NTASK = 1
  endif

  echo -n "Time limit (1:00:00): "
  set TIME = "$<"
  if ("$TIME" == "") set TIME = "1:00:00"

  echo -n "Account to charge: ("$MP_ACCOUNT")"
  set ACCOUNT = "$<"
  if ("$ACCOUNT" == "") set ACCOUNT = $MP_ACCOUNT
  
  echo -n "Load Leveller queue ($QUEUE): "
  set TEST = "$<"
  if ("$TEST" != "") set QUEUE = "$TEST"

  echo -n "Are you doing a molecular dynamics run? (y/n): "
  set TEST = "$<"
  if ("$TEST" != "y" && "$TEST" != "n") then
    echo "Invalid response: assuming a non molecular dynamics run"
    set MDON = "n"
  else
    set MDON = "$TEST"
  endif

  if ("$MDON" == "y") then
    echo ""
    echo "During the molecular dynamics run, this script will copy all"
    echo "*.pdb, *.top, and *.rst files in the current directory to the "
    echo "/scratch/$USER directory of node 0.  After completion, all files "
    echo "except the nwchem executable will be copied back to the current"
    echo "directory. "
    echo ""
    echo -n "Do you want the files copied from and to /scratch? (y/n): "
    set MDON = "$<"
    echo ""
  endif

  again1:
  echo -n "NWChem executable ($NWCHEM): "
  set TEST = "$<"
  if ("$TEST" == "") then
    # default
  else if (-f $FULLPATH/$TEST) then
    set NWCHEM = "$FULLPATH/$TEST"
  else if (-f $TEST) then
    set NWCHEM = "$TEST"
  else
    echo "  cannot locate $TEST"
    goto again1
  endif
  if (! -f $NWCHEM) then
    echo "  cannot locate $NWCHEM"
    goto again1
  endif

else

############################################################################
# Non-interactive ... llnw input [-accnt accnt_no] [-nprocs nproc] [-tasks ntask] [-nnodes nnode] [-queue queue] [-time time] [-mdon y/n]
############################################################################

  if ("$1" == "-help" || "$1" == "help") then
    goto usage
  endif

  if (! -e $NWCHEM) then
    echo " NWChem executable '$NWCHEM' not found.  Try setting "
    echo " the default executable with (for csh)"
    echo "   setenv NWCHEM_EXECUTABLE full-path-for-nwchem"
    exit 1
  endif

  set INPUT = "$1"
  if ("$INPUT" == "") set INPUT = "nwchem.nw"
  set TEST = "$FULLPATH/$INPUT"
  if (-f $TEST) then
     set INPUT = "$TEST"
  else if (-f $TEST.nw) then
     set INPUT = "$TEST.nw"
  endif
  if (! -f $INPUT) then
    echo "  Unable to locate input file $INPUT"
    exit 1
  endif

  set noglob
  set OUTPUT = `dirname $INPUT`/`basename $INPUT .nw`.out
  unset noglob
  @ i = 1
  while (-e $OUTPUT) 
    echo " $OUTPUT already exists ... trying again"
    set noglob
    set OUTPUT = `dirname $INPUT`/`basename $INPUT .nw`.$i.out
    unset noglob
    @ i ++
  end

  shift

# Process remaining optional arguments

  while ($#argv) 
    set key = "$1"
    shift
    if (! $#argv) then
      goto usage
    endif
    set value = "$1"
    shift

    if ("$key" == "-time") then
      set TIME = "$value"
    else if ("$key" == "-queue") then
      set QUEUE = "$value"
    else if ("$key" == "-nprocs") then
      set NPROC = "$value"
      set HOST = "SP"
    else if ("$key" == "-tasks") then
      set NTASK = "$value"
      set HOST = "SMP"
    else if ("$key" == "-nnodes") then
      set NNODE = "$value"
      set HOST = "SMP"
    else if ("$key" == "-accnt") then
      set ACCOUNT = "$value"
    else if ("$key" == "-mdon") then
      set MDON = "$value"
    else
      echo " unknown option: $key"
      goto usage
    endif
  end
endif

############################################################################
# Finally know everything.  Determine name of file for the job and error
# output, print info out and confirm before submission if interactive
############################################################################

set noglob
set LLERR = `dirname $OUTPUT`/`basename $OUTPUT .out`.err
set LLJOB = `dirname $OUTPUT`/`basename $OUTPUT .out`.job
unset noglob

echo " "
if ("$HOST" == "SP") then
echo " NPROC      = $NPROC"
else
echo " NTASK      = $NTASK"
echo " NNODE      = $NNODE"
endif
echo " TIME       = $TIME"
echo " QUEUE      = $QUEUE"
echo " INPUT      = $INPUT"
echo " OUTPUT     = $OUTPUT"
echo " ERROR      = $LLERR"
echo " JOB        = $LLJOB"
echo " NWCHEM     = $NWCHEM"
echo " ACCOUNT    = $ACCOUNT"
echo " MDON       = $MDON"
echo " "

if ($?INTERACTIVE) then
  echo -n "OK? (yes): "
  set TEST = "$<"
  if ("$TEST" == "" || "$TEST" == "y" || "$TEST" == "yes") then
    # OK
  else
    echo "  job submission aborted"
    exit 1
  endif
endif

# Remove ~ since poe will not expand it
set THEINPUT = `echo $INPUT | sed -e "s,~,$PWDHOME,"`

# set scratchdir
set SCRATCHDIR = /scratch
if ("$HOST" == "SMP") then
   set SCRATCHDIR = /scratch/$USER
endif

############################################################################
# Put the job into the file $LLJOB so the user can refer to it
############################################################################

cat <<EOF1 > $LLJOB
#!/bin/csh -x
# @ job_type         =    parallel
# @ class            =    $QUEUE
# @ environment      =    COPY_ALL; MP_INFOLEVEL=3; MP_PULSE=0; MP_SINGLE_THREAD=yes; MP_WAIT_MODE=yield; MP_EUILIB=us; MP_CPU_USAGE=unique;AIXTHREAD_SCOPE=S;RT_GRQ=ON;restart=no
# @ input            =    /dev/null
# @ output           =    $OUTPUT
# @ error            =    $LLERR
# @ wall_clock_limit =    $TIME
# @ cpu_limit        =    $TIME
# @ account_no       =    $ACCOUNT
# @ notification     =    complete
EOF1
if ("$HOST" == "SP") then
cat <<EOFt1 >> $LLJOB
# @ network.lapi     = css0,not_shared,US
# @ min_processors   =    $NPROC
# @ max_processors   =    $NPROC
EOFt1
else
cat <<EOFt2 >> $LLJOB
# @ network.lapi     = css0,shared,US
# @ node             = $NNODE
# @ tasks_per_node   = $NTASK
EOFt2
endif

if (-e $HOME/.llnw.config) then
  cat $HOME/.llnw.config >> $LLJOB
endif

cat <<EOF3 >> $LLJOB
# @ queue
#
# This gives good MPI performance on few nodes but hangs on many nodes
#MP_WAIT_MODE=yield;

############################################################################
# Work in the local scratch directory --- faster and more reliable
############################################################################

cd /scratch

EOF3
if ("$HOST" == "SMP") then
cat <<EOF3a >> $LLJOB
if ( ! -d $SCRATCHDIR ) then
  mkdir -p $SCRATCHDIR
endif

EOF3a
cat <<EOF3b >> $LLJOB
############################################################################
# Set up environment variables for the POE collective commands
############################################################################

echo " "
echo "List of current environment variables:"
env 
echo " "
echo "List of the libraries being used by the executable"
dump -H $NWCHEM

# 
# This is to help in getting the refund information automatically
#
echo "refund:UserID = $USER"
echo "refund:Job ID = \$LOADL_STEP_ID"
EOF3b
if ("$HOST" == "SP") then
cat <<EOFt3 >> $LLJOB
echo "refund:Number of processors = $NPROC"
EOFt3
else
cat <<EOFt4 >> $LLJOB
echo "refund:Number of nodes = $NNODE"
echo "refund:Number of tasks_per_node = $NTASK"
EOFt4
endif
cat <<EOF4 >> $LLJOB
echo "refund:Amount of time requested = $TIME"

setenv WCOLL /scratch/wcoll.\$\$

# This is convoluted to avoid exceeding limit on comand line length
env LOADL_PROCESSOR_LIST | \\
		sed -e 'y/ /\\n/' > \$WCOLL
echo " "
echo Processor list
echo " "
sed 's/^/refund:/' \$WCOLL >> \$WCOLL.2
cat \$WCOLL.2
echo " "

EOF4
if ("$MDON" == "y") then
cat <<EOFt5 >> $LLJOB
if (-e $PWDHOME/*.top) cp $PWDHOME/*.top $SCRATCHDIR    
if (-e $PWDHOME/*.pdb) cp $PWDHOME/*.pdb $SCRATCHDIR    
if (-e $PWDHOME/*.rst) cp $PWDHOME/*.rst $SCRATCHDIR    
EOFt5
endif
if ("$HOST" == "SP") then
cat <<EOFt6 >> $LLJOB
cp $NWCHEM $SCRATCHDIR/nwchem
EOFt6
else
cat <<EOFt7 >> $LLJOB
mcp $NWCHEM $SCRATCHDIR/nwchem -nodes $NNODE -euilib ip
EOFt7
endif

cat <<EOF5 >> $LLJOB

############################################################################
# Actually run the job
############################################################################

cat << EOFcat > poe.commands
EOF5
endif
echo "zzzzzzzzzzzzzzzzzzzzzzzz"
if ("$HOST" == "SP") then
cat <<EOFt8 >> $LLJOB
/u2/nwchem/bin/copyexe $SCRATCHDIR/nwchem
$SCRATCHDIR/nwchem $THEINPUT
EOFt8
else
cat <<EOFt9 >> $LLJOB
$SCRATCHDIR/nwchem $THEINPUT
EOFt9
endif
cat <<EOF6 >> $LLJOB
quit
EOFcat

echo -n "refund:"
date
time poe -newjob yes -cmdfile poe.commands
echo -n "refund:"
date
EOF6

if ("$MDON" == "y") then
cat <<EOF7 >> $LLJOB

############################################################################
# Copy MD files back
############################################################################

cp $SCRATCHDIR/*.* $PWDHOME/. 

EOF7
endif
cat <<EOF >> $LLJOB
############################################################################
# End of the job script
############################################################################
EOF

############################################################################
# Submit the job to load leveller and display brief queue information
############################################################################

llsubmit $LLJOB

#if ($status != 0) then
#  echo "  llsubmit failed"
#  exit 1
#endif

llq 

############################################################################
# Touch the output file so that subsequent job submissions
# will not attempt to use the same name
############################################################################

touch $OUTPUT

############################################################################
# Normal shell script exit here
############################################################################

exit 0


############################################################################
# If interrupted jump here
############################################################################

interrupted:
	echo " interrupted"
	exit 1

############################################################################
# Print usage informatio
############################################################################

usage:
    echo " llnw usage: "
    echo "   1) use with no arguments for interactive prompts"
    echo "   2) llnw filename [-accnt accnt_no] [-nprocs nproc] [-tasks ntask] [-nnodes nnode]"
    echo "                    [-queue queue] [-time time] [-mdon y/n]"
    echo "   3) llnw -help ... prints this information"  
    echo " "
    echo "   Input filenames may be specified without any trailing .nw and"
    echo "   either relative to the current directory or with an absolute path."
    echo " "
    exit 1
