util/file_types.py

   1 import os
   2
   3 # lanuage type for each file extension
   4 lang_types = {
   5     '.c'     : "C",
   6     '.cl'    : "C",
   7     '.h'     : "C",
   8     '.cc'    : "C++",
   9     '.hh'    : "C++",
  10     '.cxx'   : "C++",
  11     '.hxx'   : "C++",
  12     '.cpp'   : "C++",
  13     '.hpp'   : "C++",
  14     '.C'     : "C++",
  15     '.H'     : "C++",
  16     '.i'     : "swig",
  17     '.py'    : "python",
  18     '.pl'    : "perl",
  19     '.pm'    : "perl",
  20     '.s'     : "asm",
  21     '.S'     : "asm",
  22     '.l'     : "lex",
  23     '.ll'    : "lex",
  24     '.y'     : "yacc",
  25     '.yy'    : "yacc",
  26     '.isa'   : "isa",
  27     '.sh'    : "shell",
  28     '.slicc' : "slicc",
  29     '.sm'    : "slicc",
  30     '.awk'   : "awk",
  31     '.el'    : "lisp",
  32     '.txt'   : "text",
  33     '.tex'   : "tex",
  34     '.mk'    : "make",
  35     }
  36
  37 # languages based on file prefix
  38 lang_prefixes = (
  39     ('SCons',    'scons'),
  40     ('Make',     'make'),
  41     ('make',     'make'),
  42     ('Doxyfile', 'doxygen'),
  43     )
  44
  45 # languages based on #! line of first file
  46 hash_bang = (
  47     ('python', 'python'),
  48     ('perl',   'perl'),
  49     ('sh',     'shell'),
  50     )
  51
  52 # the list of all languages that we detect
  53 all_languages = frozenset(lang_types.itervalues())
  54 all_languages |= frozenset(lang for start,lang in lang_prefixes)
  55 all_languages |= frozenset(lang for start,lang in hash_bang)
  56
  57 def lang_type(filename, firstline=None, openok=True):
  58     '''identify the language of a given filename and potentially the
  59     firstline of the file.  If the firstline of the file is not
  60     provided and openok is True, open the file and read the first line
  61     if necessary'''
  62
  63     basename = os.path.basename(filename)
  64     name,extension = os.path.splitext(basename)
  65
  66     # first try to detect language based on file extension
  67     try:
  68         return lang_types[extension]
  69     except KeyError:
  70         pass
  71
  72     # now try to detect language based on file prefix
  73     for start,lang in lang_prefixes:
  74         if basename.startswith(start):
  75             return lang
  76
  77     # if a first line was not provided but the file is ok to open,
  78     # grab the first line of the file.
  79     if firstline is None and openok:
  80         handle = file(filename, 'r')
  81         firstline = handle.readline()
  82         handle.close()
  83
  84     # try to detect language based on #! in first line
  85     if firstline and firstline.startswith('#!'):
  86         for string,lang in hash_bang:
  87             if firstline.find(string) > 0:
  88                 return lang
  89
  90     # sorry, we couldn't detect the language
  91     return None
  92
  93 # directories and files to ignore by default
  94 default_dir_ignore = frozenset(('.hg', '.svn', 'build', 'ext'))
  95 default_file_ignore = frozenset(('parsetab.py', ))
  96
  97 def find_files(base, languages=all_languages,
  98                dir_ignore=default_dir_ignore,
  99                file_ignore=default_file_ignore):
 100     '''find all files in a directory and its subdirectories based on a
 101     set of languages, ignore directories specified in dir_ignore and
 102     files specified in file_ignore'''
 103     if base[-1] != '/':
 104         base += '/'
 105
 106     def update_dirs(dirs):
 107         '''strip the ignored directories out of the provided list'''
 108         index = len(dirs) - 1
 109         for i,d in enumerate(reversed(dirs)):
 110             if d in dir_ignore:
 111                 del dirs[index - i]
 112
 113     # walk over base
 114     for root,dirs,files in os.walk(base):
 115         root = root.replace(base, '', 1)
 116
 117         # strip ignored directories from the list
 118         update_dirs(dirs)
 119
 120         for filename in files:
 121             if filename in file_ignore:
 122                 # skip ignored files
 123                 continue
 124
 125             # try to figure out the language of the specified file
 126             fullpath = os.path.join(base, root, filename)
 127             language = lang_type(fullpath)
 128
 129             # if the file is one of the langauges that we want return
 130             # its name and the language
 131             if language in languages:
 132                 yield fullpath, language
 133
 134 def update_file(dst, src, language, mutator):
 135     '''update a file of the specified language with the provided
 136     mutator generator.  If inplace is provided, update the file in
 137     place and return the handle to the updated file.  If inplace is
 138     false, write the updated file to cStringIO'''
 139
 140     # if the source and destination are the same, we're updating in place
 141     inplace = dst == src
 142
 143     if isinstance(src, str):
 144         # if a filename was provided, open the file
 145         if inplace:
 146             mode = 'r+'
 147         else:
 148             mode = 'r'
 149         src = file(src, mode)
 150
 151     orig_lines = []
 152
 153     # grab all of the lines of the file and strip them of their line ending
 154     old_lines = list(line.rstrip('\r\n') for line in src.xreadlines())
 155     new_lines = list(mutator(old_lines, src.name, language))
 156
 157     for line in src.xreadlines():
 158         line = line
 159
 160     if inplace:
 161         # if we're updating in place and the file hasn't changed, do nothing
 162         if old_lines == new_lines:
 163             return
 164
 165         # otherwise, truncate the file and seek to the beginning.
 166         dst = src
 167         dst.truncate(0)
 168         dst.seek(0)
 169     elif isinstance(dst, str):
 170         # if we're not updating in place and a destination file name
 171         # was provided, create a file object
 172         dst = file(dst, 'w')
 173
 174     for line in new_lines:
 175         dst.write(line)
 176         dst.write('\n')