util/pbs/jobfile.py

   1 # Copyright (c) 2005-2006 The Regents of The University of Michigan
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 #
  27 # Authors: Nathan Binkert
  28
  29 import sys
  30
  31 class ternary(object):
  32     def __new__(cls, *args):
  33         if len(args) > 1:
  34             raise TypeError, \
  35                   '%s() takes at most 1 argument (%d given)' % \
  36                   (cls.__name__, len(args))
  37
  38         if args:
  39             if not isinstance(args[0], (bool, ternary)):
  40                 raise TypeError, \
  41                       '%s() argument must be True, False, or Any' % \
  42                       cls.__name__
  43             return args[0]
  44         return super(ternary, cls).__new__(cls)
  45
  46     def __bool__(self):
  47         return True
  48
  49     def __neg__(self):
  50         return self
  51
  52     def __eq__(self, other):
  53         return True
  54
  55     def __ne__(self, other):
  56         return False
  57
  58     def __str__(self):
  59         return 'Any'
  60
  61     def __repr__(self):
  62         return 'Any'
  63
  64 Any = ternary()
  65
  66 class Flags(dict):
  67     def __init__(self, *args, **kwargs):
  68         super(Flags, self).__init__()
  69         self.update(*args, **kwargs)
  70
  71     def __getattr__(self, attr):
  72         return self[attr]
  73
  74     def __setattr__(self, attr, value):
  75         self[attr] = value
  76
  77     def __setitem__(self, item, value):
  78         return super(Flags, self).__setitem__(item, ternary(value))
  79
  80     def __getitem__(self, item):
  81         if item not in self:
  82             return False
  83         return super(Flags, self).__getitem__(item)
  84
  85     def update(self, *args, **kwargs):
  86         for arg in args:
  87             if isinstance(arg, Flags):
  88                 super(Flags, self).update(arg)
  89             elif isinstance(arg, dict):
  90                 for key,val in kwargs.iteritems():
  91                     self[key] = val
  92             else:
  93                 raise AttributeError, \
  94                       'flags not of type %s or %s, but %s' % \
  95                       (Flags, dict, type(arg))
  96
  97         for key,val in kwargs.iteritems():
  98             self[key] = val
  99
 100     def match(self, *args, **kwargs):
 101         match = Flags(*args, **kwargs)
 102
 103         for key,value in match.iteritems():
 104             if self[key] != value:
 105                 return False
 106
 107         return True
 108
 109 def crossproduct(items):
 110     if not isinstance(items, (list, tuple)):
 111         raise AttributeError, 'crossproduct works only on sequences'
 112
 113     if not items:
 114         yield None
 115         return
 116
 117     current = items[0]
 118     remainder = items[1:]
 119
 120     if not hasattr(current, '__iter__'):
 121         current = [ current ]
 122
 123     for item in current:
 124         for rem in crossproduct(remainder):
 125             data = [ item ]
 126             if rem:
 127                 data += rem
 128             yield data
 129
 130 def flatten(items):
 131     if not isinstance(items, (list, tuple)):
 132         yield items
 133         return
 134
 135     for item in items:
 136         for flat in flatten(item):
 137             yield flat
 138
 139 class Data(object):
 140     def __init__(self, name, desc, **kwargs):
 141         self.name = name
 142         self.desc = desc
 143         self.system = None
 144         self.flags = Flags()
 145         self.env = {}
 146         for k,v in kwargs.iteritems():
 147             setattr(self, k, v)
 148
 149     def update(self, obj):
 150         if not isinstance(obj, Data):
 151             raise AttributeError, "can only update from Data object"
 152
 153         self.env.update(obj.env)
 154         self.flags.update(obj.flags)
 155         if obj.system:
 156             if self.system and self.system != obj.system:
 157                 raise AttributeError, \
 158                       "conflicting values for system: '%s'/'%s'" % \
 159                       (self.system, obj.system)
 160             self.system = obj.system
 161
 162     def printinfo(self):
 163         if self.name:
 164             print 'name: %s' % self.name
 165         if self.desc:
 166             print 'desc: %s' % self.desc
 167         if self.system:
 168             print 'system: %s' % self.system
 169
 170     def printverbose(self):
 171         print 'flags:'
 172         keys = self.flags.keys()
 173         keys.sort()
 174         for key in keys:
 175             print '    %s = %s' % (key, self.flags[key])
 176         print 'env:'
 177         keys = self.env.keys()
 178         keys.sort()
 179         for key in keys:
 180             print '    %s = %s' % (key, self.env[key])
 181         print
 182
 183     def __str__(self):
 184         return self.name
 185
 186 class Job(Data):
 187     def __init__(self, options):
 188         super(Job, self).__init__('', '')
 189         self.setoptions(options)
 190
 191         self.checkpoint = False
 192         opts = []
 193         for opt in options:
 194             cpt = opt.group.checkpoint
 195             if not cpt:
 196                 self.checkpoint = True
 197                 continue
 198             if isinstance(cpt, Option):
 199                 opt = cpt.clone(suboptions=False)
 200             else:
 201                 opt = opt.clone(suboptions=False)
 202
 203             opts.append(opt)
 204
 205         if not opts:
 206             self.checkpoint = False
 207
 208         if self.checkpoint:
 209             self.checkpoint = Job(opts)
 210
 211     def clone(self):
 212         return Job(self.options)
 213
 214     def __getattribute__(self, attr):
 215         if attr == 'name':
 216             names = [ ]
 217             for opt in self.options:
 218                 if opt.name:
 219                     names.append(opt.name)
 220             return ':'.join(names)
 221
 222         if attr == 'desc':
 223             descs = [ ]
 224             for opt in self.options:
 225                 if opt.desc:
 226                     descs.append(opt.desc)
 227             return ', '.join(descs)
 228
 229         return super(Job, self).__getattribute__(attr)
 230
 231     def setoptions(self, options):
 232         config = options[0].config
 233         for opt in options:
 234             if opt.config != config:
 235                 raise AttributeError, \
 236                       "All options are not from the same Configuration"
 237
 238         self.config = config
 239         self.groups = [ opt.group for opt in options ]
 240         self.options = options
 241
 242         self.update(self.config)
 243         for group in self.groups:
 244             self.update(group)
 245
 246         for option in self.options:
 247             self.update(option)
 248             if option._suboption:
 249                 self.update(option._suboption)
 250
 251     def printinfo(self):
 252         super(Job, self).printinfo()
 253         if self.checkpoint:
 254             print 'checkpoint: %s' % self.checkpoint.name
 255         print 'config: %s' % self.config.name
 256         print 'groups: %s' % [ g.name for g in self.groups ]
 257         print 'options: %s' % [ o.name for o in self.options ]
 258         super(Job, self).printverbose()
 259
 260 class SubOption(Data):
 261     def __init__(self, name, desc, **kwargs):
 262         super(SubOption, self).__init__(name, desc, **kwargs)
 263         self.number = None
 264
 265 class Option(Data):
 266     def __init__(self, name, desc, **kwargs):
 267         super(Option, self).__init__(name, desc, **kwargs)
 268         self._suboptions = []
 269         self._suboption = None
 270         self.number = None
 271
 272     def __getattribute__(self, attr):
 273         if attr == 'name':
 274             name = self.__dict__[attr]
 275             if self._suboption is not None:
 276                 name = '%s:%s' % (name, self._suboption.name)
 277             return name
 278
 279         if attr == 'desc':
 280             desc = [ self.__dict__[attr] ]
 281             if self._suboption is not None and self._suboption.desc:
 282                 desc.append(self._suboption.desc)
 283             return ', '.join(desc)
 284
 285
 286         return super(Option, self).__getattribute__(attr)
 287
 288     def suboption(self, name, desc, **kwargs):
 289         subo = SubOption(name, desc, **kwargs)
 290         subo.config = self.config
 291         subo.group = self.group
 292         subo.option = self
 293         subo.number = len(self._suboptions)
 294         self._suboptions.append(subo)
 295         return subo
 296
 297     def clone(self, suboptions=True):
 298         option = Option(self.__dict__['name'], self.__dict__['desc'])
 299         option.update(self)
 300         option.group = self.group
 301         option.config = self.config
 302         option.number = self.number
 303         if suboptions:
 304             option._suboptions.extend(self._suboptions)
 305             option._suboption = self._suboption
 306         return option
 307
 308     def subopts(self):
 309         if not self._suboptions:
 310             return [ self ]
 311
 312         subopts = []
 313         for subo in self._suboptions:
 314             option = self.clone()
 315             option._suboption = subo
 316             subopts.append(option)
 317
 318         return subopts
 319
 320     def printinfo(self):
 321         super(Option, self).printinfo()
 322         print 'config: %s' % self.config.name
 323         super(Option, self).printverbose()
 324
 325 class Group(Data):
 326     def __init__(self, name, desc, **kwargs):
 327         super(Group, self).__init__(name, desc, **kwargs)
 328         self._options = []
 329         self.checkpoint = False
 330         self.number = None
 331
 332     def option(self, name, desc, **kwargs):
 333         opt = Option(name, desc, **kwargs)
 334         opt.config = self.config
 335         opt.group = self
 336         opt.number = len(self._options)
 337         self._options.append(opt)
 338         return opt
 339
 340     def options(self):
 341         return self._options
 342
 343     def subopts(self):
 344         subopts = []
 345         for opt in self._options:
 346             for subo in opt.subopts():
 347                 subopts.append(subo)
 348         return subopts
 349
 350     def printinfo(self):
 351         super(Group, self).printinfo()
 352         print 'config: %s' % self.config.name
 353         print 'options: %s' % [ o.name for o in self._options ]
 354         super(Group, self).printverbose()
 355
 356 class Configuration(Data):
 357     def __init__(self, name, desc, **kwargs):
 358         super(Configuration, self).__init__(name, desc, **kwargs)
 359         self._groups = []
 360         self._posfilters = []
 361         self._negfilters = []
 362
 363     def group(self, name, desc, **kwargs):
 364         grp = Group(name, desc, **kwargs)
 365         grp.config = self
 366         grp.number = len(self._groups)
 367         self._groups.append(grp)
 368         return grp
 369
 370     def groups(self, flags=Flags(), sign=True):
 371         if not flags:
 372             return self._groups
 373
 374         return [ grp for grp in self._groups if sign ^ grp.flags.match(flags) ]
 375
 376     def checkchildren(self, kids):
 377         for kid in kids:
 378             if kid.config != self:
 379                 raise AttributeError, "child from the wrong configuration"
 380
 381     def sortgroups(self, groups):
 382         groups = [ (grp.number, grp) for grp in groups ]
 383         groups.sort()
 384         return [ grp[1] for grp in groups ]
 385
 386     def options(self, groups = None, checkpoint = False):
 387         if groups is None:
 388             groups = self._groups
 389         self.checkchildren(groups)
 390         groups = self.sortgroups(groups)
 391         if checkpoint:
 392             groups = [ grp for grp in groups if grp.checkpoint ]
 393             optgroups = [ g.options() for g in groups ]
 394         else:
 395             optgroups = [ g.subopts() for g in groups ]
 396         for options in crossproduct(optgroups):
 397             for opt in options:
 398                 cpt = opt.group.checkpoint
 399                 if not isinstance(cpt, bool) and cpt != opt:
 400                     if checkpoint:
 401                         break
 402                     else:
 403                         yield options
 404             else:
 405                 if checkpoint:
 406                     yield options
 407
 408     def addfilter(self, filt, pos=True):
 409         import re
 410         filt = re.compile(filt)
 411         if pos:
 412             self._posfilters.append(filt)
 413         else:
 414             self._negfilters.append(filt)
 415
 416     def jobfilter(self, job):
 417         for filt in self._negfilters:
 418             if filt.match(job.name):
 419                 return False
 420
 421         if not self._posfilters:
 422             return True
 423
 424         for filt in self._posfilters:
 425             if filt.match(job.name):
 426                 return True
 427
 428         return False
 429
 430     def checkpoints(self, groups = None):
 431         for options in self.options(groups, True):
 432             job = Job(options)
 433             if self.jobfilter(job):
 434                 yield job
 435
 436     def jobs(self, groups = None):
 437         for options in self.options(groups, False):
 438             job = Job(options)
 439             if self.jobfilter(job):
 440                 yield job
 441
 442     def alljobs(self, groups = None):
 443         for options in self.options(groups, True):
 444             yield Job(options)
 445         for options in self.options(groups, False):
 446             yield Job(options)
 447
 448     def find(self, jobname):
 449         for job in self.alljobs():
 450             if job.name == jobname:
 451                 return job
 452         else:
 453             raise AttributeError, "job '%s' not found" % jobname
 454
 455     def job(self, options):
 456         self.checkchildren(options)
 457         options = [ (opt.group.number, opt) for opt in options ]
 458         options.sort()
 459         options = [ opt[1] for opt in options ]
 460         job = Job(options)
 461         return job
 462
 463     def printinfo(self):
 464         super(Configuration, self).printinfo()
 465         print 'groups: %s' % [ g.name for g in self._grouips ]
 466         super(Configuration, self).printverbose()
 467
 468 def JobFile(jobfile):
 469     from os.path import expanduser, isfile, join as joinpath
 470     filename = expanduser(jobfile)
 471
 472     # Can't find filename in the current path, search sys.path
 473     if not isfile(filename):
 474         for path in sys.path:
 475             testname = joinpath(path, filename)
 476             if isfile(testname):
 477                 filename = testname
 478                 break
 479         else:
 480             raise AttributeError, \
 481                   "Could not find file '%s'" % jobfile
 482
 483     data = {}
 484     execfile(filename, data)
 485     if 'conf' not in data:
 486         raise ImportError, 'cannot import name conf from %s' % jobfile
 487     conf = data['conf']
 488     import jobfile
 489     if not isinstance(conf, Configuration):
 490         raise AttributeError, \
 491               'conf in jobfile: %s (%s) is not type %s' % \
 492               (jobfile, type(conf), Configuration)
 493     return conf
 494
 495 if __name__ == '__main__':
 496     from jobfile import *
 497     import sys
 498
 499     usage = 'Usage: %s [-b] [-c] [-v] <jobfile>' % sys.argv[0]
 500
 501     try:
 502         import getopt
 503         opts, args = getopt.getopt(sys.argv[1:], '-bcv')
 504     except getopt.GetoptError:
 505         sys.exit(usage)
 506
 507     if len(args) != 1:
 508         raise AttributeError, usage
 509
 510     both = False
 511     checkpoint = False
 512     verbose = False
 513     for opt,arg in opts:
 514         if opt == '-b':
 515             both = True
 516             checkpoint = True
 517         if opt == '-c':
 518             checkpoint = True
 519         if opt == '-v':
 520             verbose = True
 521
 522     jobfile = args[0]
 523     conf = JobFile(jobfile)
 524
 525     if both:
 526         gen = conf.alljobs()
 527     elif checkpoint:
 528         gen = conf.checkpoints()
 529     else:
 530         gen = conf.jobs()
 531
 532     for job in gen:
 533         if not verbose:
 534             cpt = ''
 535             if job.checkpoint:
 536                 cpt = job.checkpoint.name
 537             print job.name, cpt
 538         else:
 539             job.printinfo()