util/file_types.py

   1 import os
   2
   3 # lanuage type for each file extension
   4 lang_types = {
   5     '.c'     : "C",
   6     '.h'     : "C",
   7     '.cc'    : "C++",
   8     '.hh'    : "C++",
   9     '.cxx'   : "C++",
  10     '.hxx'   : "C++",
  11     '.cpp'   : "C++",
  12     '.hpp'   : "C++",
  13     '.C'     : "C++",
  14     '.H'     : "C++",
  15     '.i'     : "swig",
  16     '.py'    : "python",
  17     '.pl'    : "perl",
  18     '.pm'    : "perl",
  19     '.s'     : "asm",
  20     '.S'     : "asm",
  21     '.l'     : "lex",
  22     '.ll'    : "lex",
  23     '.y'     : "yacc",
  24     '.yy'    : "yacc",
  25     '.isa'   : "isa",
  26     '.sh'    : "shell",
  27     '.slicc' : "slicc",
  28     '.sm'    : "slicc",
  29     '.awk'   : "awk",
  30     '.el'    : "lisp",
  31     '.txt'   : "text",
  32     '.tex'   : "tex",
  33     }
  34
  35 # languages based on file prefix
  36 lang_prefixes = (
  37     ('SCons',    'scons'),
  38     ('Make',     'make'),
  39     ('make',     'make'),
  40     ('Doxyfile', 'doxygen'),
  41     )
  42
  43 # languages based on #! line of first file
  44 hash_bang = (
  45     ('python', 'python'),
  46     ('perl',   'perl'),
  47     ('sh',     'shell'),
  48     )
  49
  50 # the list of all languages that we detect
  51 all_languages = frozenset(lang_types.itervalues())
  52 all_languages |= frozenset(lang for start,lang in lang_prefixes)
  53 all_languages |= frozenset(lang for start,lang in hash_bang)
  54
  55 def lang_type(filename, firstline=None, openok=True):
  56     '''identify the language of a given filename and potentially the
  57     firstline of the file.  If the firstline of the file is not
  58     provided and openok is True, open the file and read the first line
  59     if necessary'''
  60
  61     basename = os.path.basename(filename)
  62     name,extension = os.path.splitext(basename)
  63
  64     # first try to detect language based on file extension
  65     try:
  66         return lang_types[extension]
  67     except KeyError:
  68         pass
  69
  70     # now try to detect language based on file prefix
  71     for start,lang in lang_prefixes:
  72         if basename.startswith(start):
  73             return lang
  74
  75     # if a first line was not provided but the file is ok to open,
  76     # grab the first line of the file.
  77     if firstline is None and openok:
  78         handle = file(filename, 'r')
  79         firstline = handle.readline()
  80         handle.close()
  81
  82     # try to detect language based on #! in first line
  83     if firstline and firstline.startswith('#!'):
  84         for string,lang in hash_bang:
  85             if firstline.find(string) > 0:
  86                 return lang
  87
  88     # sorry, we couldn't detect the language
  89     return None
  90
  91 # directories and files to ignore by default
  92 default_dir_ignore = frozenset(('.hg', '.svn', 'build', 'ext'))
  93 default_file_ignore = frozenset(('parsetab.py', ))
  94
  95 def find_files(base, languages=all_languages,
  96                dir_ignore=default_dir_ignore,
  97                file_ignore=default_file_ignore):
  98     '''find all files in a directory and its subdirectories based on a
  99     set of languages, ignore directories specified in dir_ignore and
 100     files specified in file_ignore'''
 101     if base[-1] != '/':
 102         base += '/'
 103
 104     def update_dirs(dirs):
 105         '''strip the ignored directories out of the provided list'''
 106         index = len(dirs) - 1
 107         for i,d in enumerate(reversed(dirs)):
 108             if d in dir_ignore:
 109                 del dirs[index - i]
 110
 111     # walk over base
 112     for root,dirs,files in os.walk(base):
 113         root = root.replace(base, '', 1)
 114
 115         # strip ignored directories from the list
 116         update_dirs(dirs)
 117
 118         for filename in files:
 119             if filename in file_ignore:
 120                 # skip ignored files
 121                 continue
 122
 123             # try to figure out the language of the specified file
 124             fullpath = os.path.join(base, root, filename)
 125             language = lang_type(fullpath)
 126
 127             # if the file is one of the langauges that we want return
 128             # its name and the language
 129             if language in languages:
 130                 yield fullpath, language
 131
 132 def update_file(dst, src, language, mutator):
 133     '''update a file of the specified language with the provided
 134     mutator generator.  If inplace is provided, update the file in
 135     place and return the handle to the updated file.  If inplace is
 136     false, write the updated file to cStringIO'''
 137
 138     # if the source and destination are the same, we're updating in place
 139     inplace = dst == src
 140
 141     if isinstance(src, str):
 142         # if a filename was provided, open the file
 143         if inplace:
 144             mode = 'r+'
 145         else:
 146             mode = 'r'
 147         src = file(src, mode)
 148
 149     orig_lines = []
 150
 151     # grab all of the lines of the file and strip them of their line ending
 152     old_lines = list(line.rstrip('\r\n') for line in src.xreadlines())
 153     new_lines = list(mutator(old_lines, src.name, language))
 154
 155     for line in src.xreadlines():
 156         line = line
 157
 158     if inplace:
 159         # if we're updating in place and the file hasn't changed, do nothing
 160         if old_lines == new_lines:
 161             return
 162
 163         # otherwise, truncate the file and seek to the beginning.
 164         dst = src
 165         dst.truncate(0)
 166         dst.seek(0)
 167     elif isinstance(dst, str):
 168         # if we're not updating in place and a destination file name
 169         # was provided, create a file object
 170         dst = file(dst, 'w')
 171
 172     for line in new_lines:
 173         dst.write(line)
 174         dst.write('\n')