--- /dev/null
+#!/usr/bin/env python
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+# Steve Reinhardt
+# Ali Saidi
+
+import os, os.path, shutil, signal, socket, sys, time
+from os import environ as env
+from os.path import join as joinpath, expanduser
+
+class rsync:
+ def __init__(self):
+ self.sudo = False
+ self.rsync = 'rsync'
+ self.compress = False
+ self.archive = True
+ self.delete = False
+ self.options = ''
+
+ def do(self, src, dst):
+ args = []
+ if self.sudo:
+ args.append('sudo')
+
+ args.append(self.rsync)
+ if (self.archive):
+ args.append('-a')
+ if (self.compress):
+ args.append('-z')
+ if (self.delete):
+ args.append('--delete')
+ if len(self.options):
+ args.append(self.options)
+ args.append(src)
+ args.append(dst)
+
+ return os.spawnvp(os.P_WAIT, args[0], args)
+
+def cleandir(dir):
+ for root, dirs, files in os.walk(dir, False):
+ for name in files:
+ os.remove(joinpath(root, name))
+ for name in dirs:
+ os.rmdir(joinpath(root, name))
+
+def date():
+ return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime())
+
+def remfile(file):
+ if os.path.isfile(file):
+ os.unlink(file)
+
+def readval(filename):
+ file = open(filename, 'r')
+ value = file.readline().strip()
+ file.close()
+ return value
+
+if __name__ == '__main__':
+ rootdir = env.setdefault('ROOTDIR', os.getcwd())
+ jobid = env['PBS_JOBID']
+ jobname = env['PBS_JOBNAME']
+ jobdir = joinpath(rootdir, jobname)
+ basedir = joinpath(rootdir, 'Base')
+ user = env['USER']
+
+ env['POOLJOB'] = 'True'
+ env['OUTPUT_DIR'] = jobdir
+ env['JOBFILE'] = joinpath(basedir, 'test.py')
+ env['JOBNAME'] = jobname
+
+ def echofile(filename, string):
+ try:
+ f = file(joinpath(jobdir, filename), 'w')
+ print >>f, string
+ f.flush()
+ f.close()
+ except IOError,e:
+ sys.exit(e)
+
+ if os.path.isdir("/work"):
+ workbase = "/work"
+ else:
+ workbase = "/tmp/"
+
+ workdir = joinpath(workbase, '%s.%s' % (user, jobid))
+
+ os.umask(0022)
+
+ echofile('.start', date())
+ echofile('.jobid', jobid)
+ echofile('.host', socket.gethostname())
+
+ if os.path.isdir(workdir):
+ cleandir(workdir)
+ else:
+ os.mkdir(workdir)
+
+ if os.path.isdir('/z/dist'):
+ sync = rsync()
+ sync.delete = True
+ sync.sudo = True
+ sync.do('poolfs::dist/m5/', '/z/dist/m5/')
+
+ try:
+ os.chdir(workdir)
+ except OSError,e:
+ sys.exit(e)
+
+ os.symlink(joinpath(jobdir, 'output'), 'status.out')
+
+ args = [ joinpath(basedir, 'm5'), joinpath(basedir, 'run.mpy') ]
+ if not len(args):
+ sys.exit("no arguments")
+
+ print 'starting job... %s' % date()
+ print ' '.join(args)
+ print
+ sys.stdout.flush()
+
+ childpid = os.fork()
+ if not childpid:
+ # Execute command
+ sys.stdin.close()
+ fd = os.open(joinpath(jobdir, "output"),
+ os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
+ os.dup2(fd, sys.stdout.fileno())
+ os.dup2(fd, sys.stderr.fileno())
+ os.execvp(args[0], args)
+
+ def handler(signum, frame):
+ if childpid != 0:
+ os.kill(childpid, signum)
+
+ signal.signal(signal.SIGHUP, handler)
+ signal.signal(signal.SIGINT, handler)
+ signal.signal(signal.SIGQUIT, handler)
+ signal.signal(signal.SIGTERM, handler)
+ signal.signal(signal.SIGSTOP, handler)
+ signal.signal(signal.SIGCONT, handler)
+ signal.signal(signal.SIGUSR1, handler)
+ signal.signal(signal.SIGUSR2, handler)
+
+ done = 0
+ while not done:
+ try:
+ thepid,ec = os.waitpid(childpid, 0)
+ if ec:
+ print 'Exit code ', ec
+ echofile('.failure', date())
+ else:
+ echofile('.success', date())
+ done = 1
+ except OSError:
+ pass
+
+ print '\njob complete... %s' % date()
+ echofile('.stop', date())
--- /dev/null
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+
+from os.path import expanduser
+def crossproduct(options):
+ number = len(options)
+ indexes = [ 0 ] * number
+ maxes = [ len(opt) for opt in options ]
+ def next():
+ for i in xrange(number - 1, -1, -1):
+ indexes[i] += 1
+ if indexes[i] < maxes[i]:
+ return False
+
+ indexes[i] = 0
+ return True
+
+ done = False
+ while not done:
+ result = []
+ for i in xrange(number):
+ result.append(options[i][indexes[i]])
+ yield result
+ done = next()
+
+class JobFile(object):
+ def __init__(self, file):
+ self.data = {}
+ execfile(expanduser(file), self.data)
+ self.options = self.data['options']
+ self.environment = self.data['environment']
+ self.jobinfo = {}
+ self.jobs = []
+ for job in crossproduct(self.options):
+ jobname = '.'.join([ id[0] for id in job ])
+ self.jobs.append(jobname)
+ list = []
+ for info in job:
+ for item in info[1:]:
+ list.append(item)
+ self.jobinfo[jobname] = list
+
+ def env(self, jobname):
+ env = {}
+ for key,val in self.jobinfo[jobname]:
+ env[key] = val
+
+ for key,val in self.environment:
+ env[key] = val
+ return env
+
+ def printinfo(self, jobname):
+ print '%s:' % jobname
+ for key,val in self.jobinfo[jobname]:
+ print ' %s = %s' % (key, val)
+
+ for key,val in self.environment:
+ print ' %s = %s' % (key, val)
--- /dev/null
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Nathan Binkert
+
+import os, re, sys
+
+def ssh(host, script, tty = False, user = ''):
+ args = [ 'ssh', '-x' ]
+ if user:
+ args.append('-l' + user)
+ if tty:
+ args.append('-t')
+ args.append(host)
+ args.append(script)
+
+ return os.spawnvp(os.P_WAIT, args[0], args)
+
+class qsub:
+ def __init__(self):
+ self.hold = False
+ self.join = False
+ self.keep_stdout = False
+ self.keep_stderr = False
+ self.node_type = ''
+ self.mail_abort = False
+ self.mail_begin = False
+ self.mail_end = False
+ self.name = ''
+ self.stdout = ''
+ self.priority = 0
+ self.queue = ''
+ self.pbshost = ''
+ self.qsub = 'qsub'
+ self.env = {}
+ self.onlyecho = False
+ self.verbose = False
+
+ def do(self, script, ):
+ args = [self.qsub]
+
+ if self.env:
+ arg = '-v'
+ arg += ','.join([ '%s=%s' % i for i in self.env.iteritems() ])
+ args.append(arg)
+
+ if self.hold:
+ args.append('-h')
+
+ if len(self.stdout):
+ args.append('-olocalhost:' + self.stdout)
+
+ if self.keep_stdout and self.keep_stderr:
+ args.append('-koe')
+ elif self.keep_stdout:
+ args.append('-ko')
+ elif self.keep_stderr:
+ args.append('-ke')
+ else:
+ args.append('-kn')
+
+ if self.join:
+ args.append('-joe')
+
+ if len(self.node_type):
+ args.append('-lnodes=' + self.node_type)
+
+ if self.mail_abort or self.mail_begin or self.mail_end:
+ flags = ''
+ if self.mail_abort:
+ flags.append('a')
+ if self.mail_begin:
+ flags.append('b')
+ if self.mail_end:
+ flags.append('e')
+ if len(flags):
+ args.append('-m ' + flags)
+
+ if len(self.name):
+ args.append("-N%s" % self.name)
+
+ if self.priority != 0:
+ args.append('-p' + self.priority)
+
+ if len(self.queue):
+ args.append('-q' + self.queue)
+
+ args.append(script)
+
+ if self.verbose or self.onlyecho:
+ print >>sys.stderr, 'PBS Command: ', ' '.join(args)
+
+ if self.onlyecho:
+ return 0
+
+ print >>sys.stderr, 'PBS Jobid: ',
+
+ ec = os.spawnvp(os.P_WAIT, args[0], args)
+
+ if ec != 0 and len(self.pbshost):
+ ec = ssh(self.pbshost, ' '.join(args))
+
+ return ec
--- /dev/null
+#!/usr/bin/env python
+# Copyright (c) 2005 The Regents of The University of Michigan
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Ali Saidi
+# Nathan Binkert
+
+import os, os.path, re, sys
+from os import environ as env, listdir
+from os.path import basename, isdir, isfile, islink, join as joinpath
+from filecmp import cmp as filecmp
+from shutil import copyfile
+
+progname = basename(sys.argv[0])
+usage = """\
+Usage:
+ %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp>
+ -c clean directory if job can be run
+ -e only echo pbs command info, don't actually send the job
+ -f force the job to run regardless of state
+ -q <queue> submit job to the named queue
+ -v be verbose
+
+ %(progname)s -l [-v] <regexp>
+ -l list job names, don't submit
+ -v be verbose (list job parameters)
+
+ %(progname)s -h
+ -h display this help
+""" % locals()
+
+try:
+ import getopt
+ opts, args = getopt.getopt(sys.argv[1:], '-cefhlq:v')
+except getopt.GetoptError:
+ sys.exit(usage)
+
+clean = False
+onlyecho = False
+exprs = []
+force = False
+listonly = False
+queue = ''
+verbose = False
+for o,a in opts:
+ if o == '-c':
+ clean = True
+ if o == '-e':
+ onlyecho = True
+ if o == '-f':
+ force = True
+ if o == '-h':
+ print usage
+ sys.exit(0)
+ if o == '-l':
+ listonly = True
+ if o == '-q':
+ queue = a
+ if o == '-v':
+ verbose = True
+
+for arg in args:
+ exprs.append(re.compile(arg))
+
+if not listonly and not onlyecho and isdir('Link'):
+ print 'Checking for outdated files in Link directory'
+ entries = listdir('Link')
+ for entry in entries:
+ link = joinpath('Link', entry)
+ if not islink(link):
+ continue
+
+ base = joinpath('Base', entry)
+ if not isfile(base) or not filecmp(link, base):
+ print '%s is different than source %s...copying' % (base, link)
+ copyfile(link, base)
+
+import job, jobfile, pbs
+
+test = jobfile.JobFile(joinpath('Base', 'test.py'))
+
+joblist = []
+for jobname in test.jobs:
+ if not exprs:
+ joblist.append(jobname)
+ continue
+
+ for expr in exprs:
+ if expr.match(jobname):
+ joblist.append(jobname)
+ break
+
+if listonly:
+ if verbose:
+ for jobname in joblist:
+ test.printinfo(jobname)
+ else:
+ for jobname in joblist:
+ print jobname
+ sys.exit(0)
+
+if not onlyecho:
+ jl = []
+ for jobname in joblist:
+ if os.path.exists(jobname):
+ if not force:
+ if os.path.isfile(joinpath(jobname, '.success')):
+ continue
+
+ if os.path.isfile(joinpath(jobname, '.start')) and \
+ not os.path.isfile(joinpath(jobname, '.stop')):
+ continue
+
+ if not clean:
+ sys.exit('job directory not clean!')
+
+ job.cleandir(jobname)
+ else:
+ os.mkdir(jobname)
+ jl.append(jobname)
+ joblist = jl
+
+rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd())
+for jobname in joblist:
+ jobdir = joinpath(rootdir, jobname)
+
+ if not onlyecho and not os.path.isdir(jobdir):
+ sys.exit('%s is not a directory. Cannot build job' % jobdir)
+
+ print >>sys.stderr, 'Job name: %s' % jobname
+ print >>sys.stderr, 'Job directory: %s' % jobdir
+
+ qsub = pbs.qsub()
+ qsub.pbshost = 'simpool.eecs.umich.edu'
+ qsub.stdout = joinpath(jobdir, 'jobout')
+ qsub.name = jobname
+ qsub.join = True
+ qsub.node_type = 'FAST'
+ qsub.onlyecho = onlyecho
+ qsub.env['ROOTDIR'] = rootdir
+ qsub.verbose = verbose
+ if len(queue):
+ qsub.queue = queue
+
+ qsub.do(joinpath('Base', 'job.py'))
+ print >>sys.stderr, ''
projects / gem5.git / commitdiff