util/pbs/jobfile.py

   1 # Copyright (c) 2005 The Regents of The University of Michigan
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 #
  27 # Authors: Nathan Binkert
  28
  29 import sys
  30
  31 class ternary(object):
  32     def __new__(cls, *args):
  33         if len(args) > 1:
  34             raise TypeError, \
  35                   '%s() takes at most 1 argument (%d given)' % \
  36                   (cls.__name__, len(args))
  37
  38         if args:
  39             if not isinstance(args[0], (bool, ternary)):
  40                 raise TypeError, \
  41                       '%s() argument must be True, False, or Any' % \
  42                       cls.__name__
  43             return args[0]
  44         return super(ternary, cls).__new__(cls)
  45
  46     def __bool__(self):
  47         return True
  48
  49     def __neg__(self):
  50         return self
  51
  52     def __eq__(self, other):
  53         return True
  54
  55     def __ne__(self, other):
  56         return False
  57
  58     def __str__(self):
  59         return 'Any'
  60
  61     def __repr__(self):
  62         return 'Any'
  63
  64 Any = ternary()
  65
  66 class Flags(dict):
  67     def __init__(self, *args, **kwargs):
  68         super(Flags, self).__init__()
  69         self.update(*args, **kwargs)
  70
  71     def __getattr__(self, attr):
  72         return self[attr]
  73
  74     def __setattr__(self, attr, value):
  75         self[attr] = value
  76
  77     def __setitem__(self, item, value):
  78         return super(Flags, self).__setitem__(item, ternary(value))
  79
  80     def __getitem__(self, item):
  81         if item not in self:
  82             return False
  83         return super(Flags, self).__getitem__(item)
  84
  85     def update(self, *args, **kwargs):
  86         for arg in args:
  87             if isinstance(arg, Flags):
  88                 super(Flags, self).update(arg)
  89             elif isinstance(arg, dict):
  90                 for key,val in kwargs.iteritems():
  91                     self[key] = val
  92             else:
  93                 raise AttributeError, \
  94                       'flags not of type %s or %s, but %s' % \
  95                       (Flags, dict, type(arg))
  96
  97         for key,val in kwargs.iteritems():
  98             self[key] = val
  99
 100     def match(self, *args, **kwargs):
 101         match = Flags(*args, **kwargs)
 102
 103         for key,value in match.iteritems():
 104             if self[key] != value:
 105                 return False
 106
 107         return True
 108
 109 def crossproduct(items):
 110     if not isinstance(items, (list, tuple)):
 111         raise AttributeError, 'crossproduct works only on sequences'
 112
 113     if not items:
 114         yield None
 115         return
 116
 117     current = items[0]
 118     remainder = items[1:]
 119
 120     if not hasattr(current, '__iter__'):
 121         current = [ current ]
 122
 123     for item in current:
 124         for rem in crossproduct(remainder):
 125             data = [ item ]
 126             if rem:
 127                 data += rem
 128             yield data
 129
 130 def flatten(items):
 131     if not isinstance(items, (list, tuple)):
 132         yield items
 133         return
 134
 135     for item in items:
 136         for flat in flatten(item):
 137             yield flat
 138
 139 class Data(object):
 140     def __init__(self, name, desc, **kwargs):
 141         self.name = name
 142         self.desc = desc
 143         self.system = None
 144         self.flags = Flags()
 145         self.env = {}
 146         for k,v in kwargs.iteritems():
 147             setattr(self, k, v)
 148
 149     def update(self, obj):
 150         if not isinstance(obj, Data):
 151             raise AttributeError, "can only update from Data object"
 152
 153         self.env.update(obj.env)
 154         self.flags.update(obj.flags)
 155         if obj.system:
 156             if self.system and self.system != obj.system:
 157                 raise AttributeError, \
 158                       "conflicting values for system: '%s'/'%s'" % \
 159                       (self.system, obj.system)
 160             self.system = obj.system
 161
 162     def printinfo(self):
 163         if self.name:
 164             print 'name: %s' % self.name
 165         if self.desc:
 166             print 'desc: %s' % self.desc
 167         if self.system:
 168             print 'system: %s' % self.system
 169
 170     def printverbose(self):
 171         print 'flags:'
 172         keys = self.flags.keys()
 173         keys.sort()
 174         for key in keys:
 175             print '    %s = %s' % (key, self.flags[key])
 176         print 'env:'
 177         keys = self.env.keys()
 178         keys.sort()
 179         for key in keys:
 180             print '    %s = %s' % (key, self.env[key])
 181         print
 182
 183     def __str__(self):
 184         return self.name
 185
 186 class Job(Data):
 187     def __init__(self, options):
 188         super(Job, self).__init__('', '')
 189         self.setoptions(options)
 190
 191         self.checkpoint = False
 192         opts = []
 193         for opt in options:
 194             cpt = opt.group.checkpoint
 195             if not cpt:
 196                 self.checkpoint = True
 197                 continue
 198             if isinstance(cpt, Option):
 199                 opt = cpt.clone(suboptions=False)
 200             else:
 201                 opt = opt.clone(suboptions=False)
 202
 203             opts.append(opt)
 204
 205         if not opts:
 206             self.checkpoint = False
 207
 208         if self.checkpoint:
 209             self.checkpoint = Job(opts)
 210
 211     def clone(self):
 212         return Job(self.options)
 213
 214     def __getattribute__(self, attr):
 215         if attr == 'name':
 216             names = [ ]
 217             for opt in self.options:
 218                 if opt.name:
 219                     names.append(opt.name)
 220             return ':'.join(names)
 221
 222         if attr == 'desc':
 223             descs = [ ]
 224             for opt in self.options:
 225                 if opt.desc:
 226                     descs.append(opt.desc)
 227             return ', '.join(descs)
 228
 229         return super(Job, self).__getattribute__(attr)
 230
 231     def setoptions(self, options):
 232         config = options[0].config
 233         for opt in options:
 234             if opt.config != config:
 235                 raise AttributeError, \
 236                       "All options are not from the same Configuration"
 237
 238         self.config = config
 239         self.groups = [ opt.group for opt in options ]
 240         self.options = options
 241
 242         self.update(self.config)
 243         for group in self.groups:
 244             self.update(group)
 245
 246         for option in self.options:
 247             self.update(option)
 248             if option._suboption:
 249                 self.update(option._suboption)
 250
 251     def printinfo(self):
 252         super(Job, self).printinfo()
 253         if self.checkpoint:
 254             print 'checkpoint: %s' % self.checkpoint.name
 255         print 'config: %s' % self.config.name
 256         print 'groups: %s' % [ g.name for g in self.groups ]
 257         print 'options: %s' % [ o.name for o in self.options ]
 258         super(Job, self).printverbose()
 259
 260 class SubOption(Data):
 261     def __init__(self, name, desc, **kwargs):
 262         super(SubOption, self).__init__(name, desc, **kwargs)
 263         self.number = None
 264
 265 class Option(Data):
 266     def __init__(self, name, desc, **kwargs):
 267         super(Option, self).__init__(name, desc, **kwargs)
 268         self._suboptions = []
 269         self._suboption = None
 270         self.number = None
 271
 272     def __getattribute__(self, attr):
 273         if attr == 'name':
 274             name = self.__dict__[attr]
 275             if self._suboption is not None:
 276                 name = '%s:%s' % (name, self._suboption.name)
 277             return name
 278
 279         if attr == 'desc':
 280             desc = self.__dict__[attr]
 281             if self._suboption is not None:
 282                 desc = '%s, %s' % (desc, self._suboption.desc)
 283             return desc
 284
 285         return super(Option, self).__getattribute__(attr)
 286
 287     def suboption(self, name, desc, **kwargs):
 288         subo = SubOption(name, desc, **kwargs)
 289         subo.config = self.config
 290         subo.group = self.group
 291         subo.option = self
 292         subo.number = len(self._suboptions)
 293         self._suboptions.append(subo)
 294         return subo
 295
 296     def clone(self, suboptions=True):
 297         option = Option(self.__dict__['name'], self.__dict__['desc'])
 298         option.update(self)
 299         option.group = self.group
 300         option.config = self.config
 301         option.number = self.number
 302         if suboptions:
 303             option._suboptions.extend(self._suboptions)
 304             option._suboption = self._suboption
 305         return option
 306
 307     def subopts(self):
 308         if not self._suboptions:
 309             return [ self ]
 310
 311         subopts = []
 312         for subo in self._suboptions:
 313             option = self.clone()
 314             option._suboption = subo
 315             subopts.append(option)
 316
 317         return subopts
 318
 319     def printinfo(self):
 320         super(Option, self).printinfo()
 321         print 'config: %s' % self.config.name
 322         super(Option, self).printverbose()
 323
 324 class Group(Data):
 325     def __init__(self, name, desc, **kwargs):
 326         super(Group, self).__init__(name, desc, **kwargs)
 327         self._options = []
 328         self.checkpoint = False
 329         self.number = None
 330
 331     def option(self, name, desc, **kwargs):
 332         opt = Option(name, desc, **kwargs)
 333         opt.config = self.config
 334         opt.group = self
 335         opt.number = len(self._options)
 336         self._options.append(opt)
 337         return opt
 338
 339     def options(self):
 340         return self._options
 341
 342     def subopts(self):
 343         subopts = []
 344         for opt in self._options:
 345             for subo in opt.subopts():
 346                 subopts.append(subo)
 347         return subopts
 348
 349     def printinfo(self):
 350         super(Group, self).printinfo()
 351         print 'config: %s' % self.config.name
 352         print 'options: %s' % [ o.name for o in self._options ]
 353         super(Group, self).printverbose()
 354
 355 class Configuration(Data):
 356     def __init__(self, name, desc, **kwargs):
 357         super(Configuration, self).__init__(name, desc, **kwargs)
 358         self._groups = []
 359
 360     def group(self, name, desc, **kwargs):
 361         grp = Group(name, desc, **kwargs)
 362         grp.config = self
 363         grp.number = len(self._groups)
 364         self._groups.append(grp)
 365         return grp
 366
 367     def groups(self, flags=Flags(), sign=True):
 368         if not flags:
 369             return self._groups
 370
 371         return [ grp for grp in self._groups if sign ^ grp.flags.match(flags) ]
 372
 373     def checkchildren(self, kids):
 374         for kid in kids:
 375             if kid.config != self:
 376                 raise AttributeError, "child from the wrong configuration"
 377
 378     def sortgroups(self, groups):
 379         groups = [ (grp.number, grp) for grp in groups ]
 380         groups.sort()
 381         return [ grp[1] for grp in groups ]
 382
 383     def options(self, groups = None, checkpoint = False):
 384         if groups is None:
 385             groups = self._groups
 386         self.checkchildren(groups)
 387         groups = self.sortgroups(groups)
 388         if checkpoint:
 389             groups = [ grp for grp in groups if grp.checkpoint ]
 390             optgroups = [ g.options() for g in groups ]
 391         else:
 392             optgroups = [ g.subopts() for g in groups ]
 393         for options in crossproduct(optgroups):
 394             for opt in options:
 395                 cpt = opt.group.checkpoint
 396                 if not isinstance(cpt, bool) and cpt != opt:
 397                     if checkpoint:
 398                         break
 399                     else:
 400                         yield options
 401             else:
 402                 if checkpoint:
 403                     yield options
 404
 405     def checkpoints(self, groups = None):
 406         for options in self.options(groups, True):
 407             yield Job(options)
 408
 409     def jobs(self, groups = None):
 410         for options in self.options(groups, False):
 411             yield Job(options)
 412
 413     def alljobs(self, groups = None):
 414         for options in self.options(groups, True):
 415             yield Job(options)
 416         for options in self.options(groups, False):
 417             yield Job(options)
 418
 419     def find(self, jobname):
 420         for job in self.alljobs():
 421             if job.name == jobname:
 422                 return job
 423         else:
 424             raise AttributeError, "job '%s' not found" % jobname
 425
 426     def job(self, options):
 427         self.checkchildren(options)
 428         options = [ (opt.group.number, opt) for opt in options ]
 429         options.sort()
 430         options = [ opt[1] for opt in options ]
 431         job = Job(options)
 432         return job
 433
 434     def printinfo(self):
 435         super(Configuration, self).printinfo()
 436         print 'groups: %s' % [ g.name for g in self._grouips ]
 437         super(Configuration, self).printverbose()
 438
 439 def JobFile(jobfile):
 440     from os.path import expanduser, isfile, join as joinpath
 441     filename = expanduser(jobfile)
 442
 443     # Can't find filename in the current path, search sys.path
 444     if not isfile(filename):
 445         for path in sys.path:
 446             testname = joinpath(path, filename)
 447             if isfile(testname):
 448                 filename = testname
 449                 break
 450         else:
 451             raise AttributeError, \
 452                   "Could not find file '%s'" % jobfile
 453
 454     data = {}
 455     execfile(filename, data)
 456     if 'conf' not in data:
 457         raise ImportError, 'cannot import name conf from %s' % jobfile
 458     conf = data['conf']
 459     import jobfile
 460     if not isinstance(conf, Configuration):
 461         raise AttributeError, \
 462               'conf in jobfile: %s (%s) is not type %s' % \
 463               (jobfile, type(conf), Configuration)
 464     return conf
 465
 466 if __name__ == '__main__':
 467     from jobfile import *
 468     import sys
 469
 470     usage = 'Usage: %s [-b] [-c] [-v] <jobfile>' % sys.argv[0]
 471
 472     try:
 473         import getopt
 474         opts, args = getopt.getopt(sys.argv[1:], '-bcv')
 475     except getopt.GetoptError:
 476         sys.exit(usage)
 477
 478     if len(args) != 1:
 479         raise AttributeError, usage
 480
 481     both = False
 482     checkpoint = False
 483     verbose = False
 484     for opt,arg in opts:
 485         if opt == '-b':
 486             both = True
 487             checkpoint = True
 488         if opt == '-c':
 489             checkpoint = True
 490         if opt == '-v':
 491             verbose = True
 492
 493     jobfile = args[0]
 494     conf = JobFile(jobfile)
 495
 496     if both:
 497         gen = conf.alljobs()
 498     elif checkpoint:
 499         gen = conf.checkpoints()
 500     else:
 501         gen = conf.jobs()
 502
 503     for job in gen:
 504         if not verbose:
 505             cpt = ''
 506             if job.checkpoint:
 507                 cpt = job.checkpoint.name
 508             print job.name, cpt
 509         else:
 510             job.printinfo()