contrib/mklog.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) 2020 Free Software Foundation, Inc.
   4 #
   5 # This file is part of GCC.
   6 #
   7 # GCC is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3, or (at your option)
  10 # any later version.
  11 #
  12 # GCC is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with GCC; see the file COPYING.  If not, write to
  19 # the Free Software Foundation, 51 Franklin Street, Fifth Floor,
  20 # Boston, MA 02110-1301, USA.
  21
  22 # This script parses a .diff file generated with 'diff -up' or 'diff -cp'
  23 # and adds a skeleton ChangeLog file to the file. It does not try to be
  24 # too smart when parsing function names, but it produces a reasonable
  25 # approximation.
  26 #
  27 # Author: Martin Liska <mliska@suse.cz>
  28
  29 import argparse
  30 import os
  31 import re
  32 import sys
  33 from itertools import takewhile
  34
  35 import requests
  36
  37 from unidiff import PatchSet
  38
  39 pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<pr>PR [a-z+-]+\/[0-9]+)')
  40 dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<dr>DR [0-9]+)')
  41 dg_regex = re.compile(r'{\s+dg-(error|warning)')
  42 identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)')
  43 comment_regex = re.compile(r'^\/\*')
  44 struct_regex = re.compile(r'^(class|struct|union|enum)\s+'
  45                           r'(GTY\(.*\)\s+)?([a-zA-Z0-9_]+)')
  46 macro_regex = re.compile(r'#\s*(define|undef)\s+([a-zA-Z0-9_]+)')
  47 super_macro_regex = re.compile(r'^DEF[A-Z0-9_]+\s*\(([a-zA-Z0-9_]+)')
  48 fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]')
  49 template_and_param_regex = re.compile(r'<[^<>]*>')
  50 bugzilla_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/bug?id=%s&' \
  51                'include_fields=summary'
  52
  53 function_extensions = set(['.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def'])
  54
  55 help_message = """\
  56 Generate ChangeLog template for PATCH.
  57 PATCH must be generated using diff(1)'s -up or -cp options
  58 (or their equivalent in git).
  59 """
  60
  61 script_folder = os.path.realpath(__file__)
  62 gcc_root = os.path.dirname(os.path.dirname(script_folder))
  63
  64
  65 def find_changelog(path):
  66     folder = os.path.split(path)[0]
  67     while True:
  68         if os.path.exists(os.path.join(gcc_root, folder, 'ChangeLog')):
  69             return folder
  70         folder = os.path.dirname(folder)
  71         if folder == '':
  72             return folder
  73     raise AssertionError()
  74
  75
  76 def extract_function_name(line):
  77     if comment_regex.match(line):
  78         return None
  79     m = struct_regex.search(line)
  80     if m:
  81         # Struct declaration
  82         return m.group(1) + ' ' + m.group(3)
  83     m = macro_regex.search(line)
  84     if m:
  85         # Macro definition
  86         return m.group(2)
  87     m = super_macro_regex.search(line)
  88     if m:
  89         # Supermacro
  90         return m.group(1)
  91     m = fn_regex.search(line)
  92     if m:
  93         # Discard template and function parameters.
  94         fn = m.group(1)
  95         fn = re.sub(template_and_param_regex, '', fn)
  96         return fn.rstrip()
  97     return None
  98
  99
 100 def try_add_function(functions, line):
 101     fn = extract_function_name(line)
 102     if fn and fn not in functions:
 103         functions.append(fn)
 104     return bool(fn)
 105
 106
 107 def sort_changelog_files(changed_file):
 108     return (changed_file.is_added_file, changed_file.is_removed_file)
 109
 110
 111 def get_pr_titles(prs):
 112     output = ''
 113     for pr in prs:
 114         id = pr.split('/')[-1]
 115         r = requests.get(bugzilla_url % id)
 116         bugs = r.json()['bugs']
 117         if len(bugs) == 1:
 118             output += '%s - %s\n' % (pr, bugs[0]['summary'])
 119             print(output)
 120     if output:
 121         output += '\n'
 122     return output
 123
 124
 125 def generate_changelog(data, no_functions=False, fill_pr_titles=False):
 126     changelogs = {}
 127     changelog_list = []
 128     prs = []
 129     out = ''
 130     diff = PatchSet(data)
 131
 132     for file in diff:
 133         changelog = find_changelog(file.path)
 134         if changelog not in changelogs:
 135             changelogs[changelog] = []
 136             changelog_list.append(changelog)
 137         changelogs[changelog].append(file)
 138
 139         # Extract PR entries from newly added tests
 140         if 'testsuite' in file.path and file.is_added_file:
 141             # Only search first ten lines as later lines may
 142             # contains commented code which a note that it
 143             # has not been tested due to a certain PR or DR.
 144             for line in list(file)[0][0:10]:
 145                 m = pr_regex.search(line.value)
 146                 if m:
 147                     pr = m.group('pr')
 148                     if pr not in prs:
 149                         prs.append(pr)
 150                 else:
 151                     m = dr_regex.search(line.value)
 152                     if m:
 153                         dr = m.group('dr')
 154                         if dr not in prs:
 155                             prs.append(dr)
 156                     elif dg_regex.search(line.value):
 157                         # Found dg-warning/dg-error line
 158                         break
 159
 160     if fill_pr_titles:
 161         out += get_pr_titles(prs)
 162
 163     # sort ChangeLog so that 'testsuite' is at the end
 164     for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x):
 165         files = changelogs[changelog]
 166         out += '%s:\n' % os.path.join(changelog, 'ChangeLog')
 167         out += '\n'
 168         for pr in prs:
 169             out += '\t%s\n' % pr
 170         # new and deleted files should be at the end
 171         for file in sorted(files, key=sort_changelog_files):
 172             assert file.path.startswith(changelog)
 173             in_tests = 'testsuite' in changelog or 'testsuite' in file.path
 174             relative_path = file.path[len(changelog):].lstrip('/')
 175             functions = []
 176             if file.is_added_file:
 177                 msg = 'New test' if in_tests else 'New file'
 178                 out += '\t* %s: %s.\n' % (relative_path, msg)
 179             elif file.is_removed_file:
 180                 out += '\t* %s: Removed.\n' % (relative_path)
 181             elif hasattr(file, 'is_rename') and file.is_rename:
 182                 out += '\t* %s: Moved to...\n' % (relative_path)
 183                 new_path = file.target_file[2:]
 184                 # A file can be theoretically moved to a location that
 185                 # belongs to a different ChangeLog.  Let user fix it.
 186                 if new_path.startswith(changelog):
 187                     new_path = new_path[len(changelog):].lstrip('/')
 188                 out += '\t* %s: ...here.\n' % (new_path)
 189             else:
 190                 if not no_functions:
 191                     for hunk in file:
 192                         # Do not add function names for testsuite files
 193                         extension = os.path.splitext(relative_path)[1]
 194                         if not in_tests and extension in function_extensions:
 195                             last_fn = None
 196                             modified_visited = False
 197                             success = False
 198                             for line in hunk:
 199                                 m = identifier_regex.match(line.value)
 200                                 if line.is_added or line.is_removed:
 201                                     if not line.value.strip():
 202                                         continue
 203                                     modified_visited = True
 204                                     if m and try_add_function(functions,
 205                                                               m.group(1)):
 206                                         last_fn = None
 207                                         success = True
 208                                 elif line.is_context:
 209                                     if last_fn and modified_visited:
 210                                         try_add_function(functions, last_fn)
 211                                         last_fn = None
 212                                         modified_visited = False
 213                                         success = True
 214                                     elif m:
 215                                         last_fn = m.group(1)
 216                                         modified_visited = False
 217                             if not success:
 218                                 try_add_function(functions,
 219                                                  hunk.section_header)
 220                 if functions:
 221                     out += '\t* %s (%s):\n' % (relative_path, functions[0])
 222                     for fn in functions[1:]:
 223                         out += '\t(%s):\n' % fn
 224                 else:
 225                     out += '\t* %s:\n' % relative_path
 226         out += '\n'
 227     return out
 228
 229
 230 if __name__ == '__main__':
 231     parser = argparse.ArgumentParser(description=help_message)
 232     parser.add_argument('input', nargs='?',
 233                         help='Patch file (or missing, read standard input)')
 234     parser.add_argument('-s', '--no-functions', action='store_true',
 235                         help='Do not generate function names in ChangeLogs')
 236     parser.add_argument('-p', '--fill-up-bug-titles', action='store_true',
 237                         help='Download title of mentioned PRs')
 238     parser.add_argument('-c', '--changelog',
 239                         help='Append the ChangeLog to a git commit message '
 240                              'file')
 241     args = parser.parse_args()
 242     if args.input == '-':
 243         args.input = None
 244
 245     input = open(args.input) if args.input else sys.stdin
 246     data = input.read()
 247     output = generate_changelog(data, args.no_functions,
 248                                 args.fill_up_bug_titles)
 249     if args.changelog:
 250         lines = open(args.changelog).read().split('\n')
 251         start = list(takewhile(lambda l: not l.startswith('#'), lines))
 252         end = lines[len(start):]
 253         with open(args.changelog, 'w') as f:
 254             if start:
 255                 # appent empty line
 256                 if start[-1] != '':
 257                     start.append('')
 258             else:
 259                 # append 2 empty lines
 260                 start = 2 * ['']
 261             f.write('\n'.join(start))
 262             f.write('\n')
 263             f.write(output)
 264             f.write('\n'.join(end))
 265     else:
 266         print(output, end='')