From 00243d9a640a14413c4e051447e09741a543deae Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Fri, 15 May 2020 00:44:07 +0200 Subject: [PATCH] New mklog script. contrib/ChangeLog: 2020-05-15 Martin Liska * gcc-git-customization.sh: Add alias.gcc-mklog new hook. * mklog.py: New file. * test_mklog.py: New file. --- contrib/ChangeLog | 7 + contrib/gcc-git-customization.sh | 2 + contrib/mklog.py | 223 ++++++++++++++++++++ contrib/test_mklog.py | 345 +++++++++++++++++++++++++++++++ 4 files changed, 577 insertions(+) create mode 100755 contrib/mklog.py create mode 100755 contrib/test_mklog.py diff --git a/contrib/ChangeLog b/contrib/ChangeLog index 50650a8650d..85f067507f1 100644 --- a/contrib/ChangeLog +++ b/contrib/ChangeLog @@ -1,3 +1,10 @@ +2020-05-19 Martin Liska + + * gcc-git-customization.sh: Add + alias.gcc-mklog new hook. + * mklog.py: New file. + * test_mklog.py: New file. + 2020-05-19 Martin Liska * legacy/mklog: Moved from mklog. diff --git a/contrib/gcc-git-customization.sh b/contrib/gcc-git-customization.sh index ce293d1fe42..91d378ba32a 100755 --- a/contrib/gcc-git-customization.sh +++ b/contrib/gcc-git-customization.sh @@ -27,6 +27,8 @@ git config alias.gcc-undescr \!"f() { o=\$(git config --get gcc-config.upstream) git config alias.gcc-verify '!f() { "`git rev-parse --show-toplevel`/contrib/gcc-changelog/git_check_commit.py" $@; } ; f' +git config alias.gcc-mklog '!f() { "`git rev-parse --show-toplevel`/contrib/mklog.py" $@; } ; f' + # Make diff on MD files use "(define" as a function marker. # Use this in conjunction with a .gitattributes file containing # *.md diff=md diff --git a/contrib/mklog.py b/contrib/mklog.py new file mode 100755 index 00000000000..cc3f937c253 --- /dev/null +++ b/contrib/mklog.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020 Free Software Foundation, Inc. +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# GCC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + +# This script parses a .diff file generated with 'diff -up' or 'diff -cp' +# and adds a skeleton ChangeLog file to the file. It does not try to be +# too smart when parsing function names, but it produces a reasonable +# approximation. +# +# Author: Martin Liska + +import argparse +import bs4 +import os +import re +import requests +import sys + +from unidiff import PatchSet + +pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?PPR [a-z+-]+\/[0-9]+)') +identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)') +comment_regex = re.compile(r'^\/\*') +struct_regex = re.compile(r'^((class|struct|union|enum)\s+[a-zA-Z0-9_]+)') +macro_regex = re.compile(r'#\s*(define|undef)\s+([a-zA-Z0-9_]+)') +super_macro_regex = re.compile(r'^DEF[A-Z0-9_]+\s*\(([a-zA-Z0-9_]+)') +fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]') +template_and_param_regex = re.compile(r'<[^<>]*>') + +function_extensions = set(['.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def']) + +help_message = """\ +Generate ChangeLog template for PATCH. +PATCH must be generated using diff(1)'s -up or -cp options +(or their equivalent in git). +""" + +script_folder = os.path.realpath(__file__) +gcc_root = os.path.dirname(os.path.dirname(script_folder)) + + +def find_changelog(path): + folder = os.path.split(path)[0] + while True: + if os.path.exists(os.path.join(gcc_root, folder, 'ChangeLog')): + return folder + folder = os.path.dirname(folder) + if folder == '': + return folder + raise AssertionError() + + +def extract_function_name(line): + if comment_regex.match(line): + return None + m = struct_regex.search(line) + if m: + # Struct declaration + return m.group(1) + m = macro_regex.search(line) + if m: + # Macro definition + return m.group(2) + m = super_macro_regex.search(line) + if m: + # Supermacro + return m.group(1) + m = fn_regex.search(line) + if m: + # Discard template and function parameters. + fn = m.group(1) + fn = re.sub(template_and_param_regex, '', fn) + return fn.rstrip() + return None + + +def try_add_function(functions, line): + fn = extract_function_name(line) + if fn and fn not in functions: + functions.append(fn) + return bool(fn) + + +def sort_changelog_files(changed_file): + return (changed_file.is_added_file, changed_file.is_removed_file) + + +def get_pr_titles(prs): + if not prs: + return '' + + output = '' + for pr in prs: + id = pr.split('/')[-1] + r = requests.get('https://gcc.gnu.org/PR%s' % id) + html = bs4.BeautifulSoup(r.text, features='lxml') + title = html.title.text + title = title[title.find('–') + 1:].strip() + output += '%s - %s\n' % (pr, title) + output += '\n' + return output + +def generate_changelog(data, no_functions=False, fill_pr_titles=False): + changelogs = {} + changelog_list = [] + prs = [] + out = '' + diff = PatchSet(data) + + for file in diff: + changelog = find_changelog(file.path) + if changelog not in changelogs: + changelogs[changelog] = [] + changelog_list.append(changelog) + changelogs[changelog].append(file) + + # Extract PR entries from newly added tests + if 'testsuite' in file.path and file.is_added_file: + for line in list(file)[0]: + m = pr_regex.search(line.value) + if m: + pr = m.group('pr') + if pr not in prs: + prs.append(pr) + else: + break + + if fill_pr_titles: + out += get_pr_titles(prs) + + # sort ChangeLog so that 'testsuite' is at the end + for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x): + files = changelogs[changelog] + out += '%s:\n' % os.path.join(changelog, 'ChangeLog') + out += '\n' + for pr in prs: + out += '\t%s\n' % pr + # new and deleted files should be at the end + for file in sorted(files, key=sort_changelog_files): + assert file.path.startswith(changelog) + in_tests = 'testsuite' in changelog or 'testsuite' in file.path + relative_path = file.path[len(changelog):].lstrip('/') + functions = [] + if file.is_added_file: + msg = 'New test' if in_tests else 'New file' + out += '\t* %s: %s.\n' % (relative_path, msg) + elif file.is_removed_file: + out += '\t* %s: Removed.\n' % (relative_path) + else: + if not no_functions: + for hunk in file: + # Do not add function names for testsuite files + extension = os.path.splitext(relative_path)[1] + if not in_tests and extension in function_extensions: + last_fn = None + modified_visited = False + success = False + for line in hunk: + m = identifier_regex.match(line.value) + if line.is_added or line.is_removed: + if not line.value.strip(): + continue + modified_visited = True + if m and try_add_function(functions, + m.group(1)): + last_fn = None + success = True + elif line.is_context: + if last_fn and modified_visited: + try_add_function(functions, last_fn) + last_fn = None + modified_visited = False + success = True + elif m: + last_fn = m.group(1) + modified_visited = False + if not success: + try_add_function(functions, + hunk.section_header) + if functions: + out += '\t* %s (%s):\n' % (relative_path, functions[0]) + for fn in functions[1:]: + out += '\t(%s):\n' % fn + else: + out += '\t* %s:\n' % relative_path + out += '\n' + return out + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=help_message) + parser.add_argument('input', nargs='?', + help='Patch file (or missing, read standard input)') + parser.add_argument('-s', '--no-functions', action='store_true', + help='Do not generate function names in ChangeLogs') + parser.add_argument('-p', '--fill-up-bug-titles', action='store_true', + help='Download title of mentioned PRs') + args = parser.parse_args() + if args.input == '-': + args.input = None + + input = open(args.input) if args.input else sys.stdin + data = input.read() + output = generate_changelog(data, args.no_functions, + args.fill_up_bug_titles) + print(output, end='') diff --git a/contrib/test_mklog.py b/contrib/test_mklog.py new file mode 100755 index 00000000000..ca7b9e79d95 --- /dev/null +++ b/contrib/test_mklog.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020 Free Software Foundation, Inc. +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# GCC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + +# This script parses a .diff file generated with 'diff -up' or 'diff -cp' +# and adds a skeleton ChangeLog file to the file. It does not try to be +# too smart when parsing function names, but it produces a reasonable +# approximation. +# +# Author: Martin Liska + +import unittest + +from mklog import generate_changelog + +PATCH1 = '''\ +diff --git a/gcc/config/riscv/riscv.h b/gcc/config/riscv/riscv.h +index 567c23380fe..e6209ede9d6 100644 +--- a/gcc/config/riscv/riscv.h ++++ b/gcc/config/riscv/riscv.h +@@ -920,6 +920,7 @@ extern unsigned riscv_stack_boundary; + #define SHIFT_RS1 15 + #define SHIFT_IMM 20 + #define IMM_BITS 12 ++#define C_S_BITS 5 + #define C_SxSP_BITS 6 + + #define IMM_REACH (1LL << IMM_BITS) +@@ -929,6 +930,10 @@ extern unsigned riscv_stack_boundary; + #define SWSP_REACH (4LL << C_SxSP_BITS) + #define SDSP_REACH (8LL << C_SxSP_BITS) + ++/* This is the maximum value that can be represented in a compressed load/store ++ offset (an unsigned 5-bit value scaled by 4). */ ++#define CSW_MAX_OFFSET ((4LL << C_S_BITS) - 1) & ~3 ++ + /* Called from RISCV_REORG, this is defined in riscv-sr.c. */ + + extern void riscv_remove_unneeded_save_restore_calls (void); + +''' + +EXPECTED1 = '''\ +gcc/ChangeLog: + + * config/riscv/riscv.h (C_S_BITS): + (CSW_MAX_OFFSET): + +''' + +PATCH2 = '''\ +diff --git a/gcc/targhooks.h b/gcc/targhooks.h +index 9704d23f1db..b572a36e8cf 100644 +--- a/gcc/targhooks.h ++++ b/gcc/targhooks.h +@@ -120,7 +120,7 @@ extern bool default_empty_mask_is_expensive (unsigned); + extern void *default_init_cost (class loop *); + extern unsigned default_add_stmt_cost (class vec_info *, void *, int, + enum vect_cost_for_stmt, +- class _stmt_vec_info *, int, ++ class _stmt_vec_info *, tree, int, + enum vect_cost_model_location); + extern void default_finish_cost (void *, unsigned *, unsigned *, unsigned *); + extern void default_destroy_cost_data (void *); +@@ -186,6 +186,7 @@ extern tree default_emutls_var_init (tree, tree, tree); + extern unsigned int default_hard_regno_nregs (unsigned int, machine_mode); + extern bool default_hard_regno_scratch_ok (unsigned int); + extern bool default_mode_dependent_address_p (const_rtx, addr_space_t); ++extern bool default_new_address_profitable_p (rtx, rtx_insn *, rtx); + extern bool default_target_option_valid_attribute_p (tree, tree, tree, int); + extern bool default_target_option_pragma_parse (tree, tree); + extern bool default_target_can_inline_p (tree, tree); + +''' + +EXPECTED2 = '''\ +gcc/ChangeLog: + + * targhooks.h (default_add_stmt_cost): + (default_new_address_profitable_p): + +''' + +PATCH3 = '''\ +diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h +index 2b1e33f94ae..7f47402f9b9 100644 +--- a/libcpp/include/cpplib.h ++++ b/libcpp/include/cpplib.h +@@ -173,7 +173,7 @@ enum c_lang {CLK_GNUC89 = 0, CLK_GNUC99, CLK_GNUC11, CLK_GNUC17, CLK_GNUC2X, + CLK_STDC2X, + CLK_GNUCXX, CLK_CXX98, CLK_GNUCXX11, CLK_CXX11, + CLK_GNUCXX14, CLK_CXX14, CLK_GNUCXX17, CLK_CXX17, +- CLK_GNUCXX2A, CLK_CXX2A, CLK_ASM}; ++ CLK_GNUCXX20, CLK_CXX20, CLK_ASM}; + + /* Payload of a NUMBER, STRING, CHAR or COMMENT token. */ + struct GTY(()) cpp_string { +@@ -484,7 +484,7 @@ struct cpp_options + /* Nonzero for C2X decimal floating-point constants. */ + unsigned char dfp_constants; + +- /* Nonzero for C++2a __VA_OPT__ feature. */ ++ /* Nonzero for C++20 __VA_OPT__ feature. */ + unsigned char va_opt; + + /* Nonzero for the '::' token. */ + +''' + +EXPECTED3 = '''\ +libcpp/ChangeLog: + + * include/cpplib.h (enum c_lang): + (struct cpp_options): + +''' + +EXPECTED3B = '''\ +libcpp/ChangeLog: + + * include/cpplib.h: + +''' + +PATCH4 = '''\ +diff --git a/gcc/ipa-icf.c b/gcc/ipa-icf.c +index aab79492357..f0df1002488 100644 +--- a/gcc/ipa-icf.c ++++ b/gcc/ipa-icf.c +@@ -1,5 +1,7 @@ + + ++ ++ + /* Interprocedural Identical Code Folding pass + Copyright (C) 2014-2020 Free Software Foundation, Inc. + +diff --git a/gcc/testsuite/gcc.dg/pr32374.c b/gcc/testsuite/gcc.dg/pr32374.c +deleted file mode 100644 +index de15d559f5b..00000000000 +--- a/gcc/testsuite/gcc.dg/pr32374.c ++++ /dev/null +@@ -1,20 +0,0 @@ +-/* { dg-do compile } */ +-/* { dg-options "-O2" } */ +- +-extern int *stderr; +- +-void f (int *, const char *, ...); +- +-void g (const char *conf_name) +-{ +- typedef struct +- { +- const char *label; +- const int value; +- } Section; +- +- const Section sections[2] = { {"", 0}, {"", 1} }; +- +- f (stderr, "", "", conf_name, 0, sections[0]); +- f (stderr, "", "", conf_name, 0, sections[0]); +-} +diff --git a/gcc/testsuite/gcc.dg/pr40209.c b/gcc/testsuite/gcc.dg/pr40209.c +index 4e77df5c2e6..c23d69d1f1b 100644 +--- a/gcc/testsuite/gcc.dg/pr40209.c ++++ b/gcc/testsuite/gcc.dg/pr40209.c +@@ -1,6 +1,8 @@ + /* { dg-do compile } */ + /* { dg-options "-O2 -fprofile-use -fopt-info -Wno-missing-profile" } */ + ++ ++ + void process(const char *s); + + struct BaseHolder { +diff --git a/gcc/testsuite/gcc.dg/pr50209.c b/gcc/testsuite/gcc.dg/pr50209.c +new file mode 100644 +index 00000000000..b28b04f6431 +--- /dev/null ++++ b/gcc/testsuite/gcc.dg/pr50209.c +@@ -0,0 +1,3 @@ ++ ++ ++ +diff --git a/gcc/testsuite/gcc.dg/pr63567-1.c b/gcc/testsuite/gcc.dg/pr63567-1.c +index 97da171563e..00c5ecc11fa 100644 +--- a/gcc/testsuite/gcc.dg/pr63567-1.c ++++ b/gcc/testsuite/gcc.dg/pr63567-1.c +@@ -1,3 +1,4 @@ ++ + /* PR c/63567 */ + /* { dg-do compile } */ + /* { dg-options "" } */ +diff --git a/gcc/varasm.c b/gcc/varasm.c +index f062e48071f..fd3c7ca8cf3 100644 +--- a/gcc/varasm.c ++++ b/gcc/varasm.c +@@ -1,3 +1,5 @@ ++ ++ + /* Output variables, constants and external declarations, for GNU compiler. + Copyright (C) 1987-2020 Free Software Foundation, Inc. + +diff --git a/libssp/gets-chk.c b/libssp/gets-chk.c +index 4ad78c1f77b..6687b368038 100644 +--- a/libssp/gets-chk.c ++++ b/libssp/gets-chk.c +@@ -32,6 +32,8 @@ see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + + ++ ++ + #include "config.h" + #include + #include +''' + +EXPECTED4 = '''\ +gcc/ChangeLog: + + * ipa-icf.c: + * varasm.c: + +libssp/ChangeLog: + + * gets-chk.c: + +gcc/testsuite/ChangeLog: + + * gcc.dg/pr40209.c: + * gcc.dg/pr63567-1.c: + * gcc.dg/pr32374.c: Removed. + * gcc.dg/pr50209.c: New test. + +''' + +PATCH5 = '''\ +diff --git a/gcc/testsuite/gcc.target/i386/pr95046-6.c b/gcc/testsuite/gcc.target/i386/pr95046-6.c +new file mode 100644 +index 00000000000..dcc8999c446 +--- /dev/null ++++ b/gcc/testsuite/gcc.target/i386/pr95046-6.c +@@ -0,0 +1,44 @@ ++/* PR target/95046 */ ++/* { dg-do compile { target { ! ia32 } } } */ ++/* { dg-options "-O3 -mavx512vl" } */ ++ ++ ++double r[2]; ++int s[2]; ++unsigned int u[2]; ++ ++void ++test_float (void) ++{ ++ for (int i = 0; i < 2; i++) ++ r[i] = s[i]; ++} ++ ++/* { dg-final { scan-assembler "\tvcvtdq2pd" } } */ ++ ++void ++test_ufloat (void) ++{ ++ for (int i = 0; i < 2; i++) ++ r[i] = u[i]; ++} ++ ++/* { dg-final { scan-assembler "\tvcvtudq2pd" } } */ ++ ++void ++test_fix (void) ++{ ++ for (int i = 0; i < 2; i++) ++ s[i] = r[i]; ++} ++ ++/* { dg-final { scan-assembler "\tvcvttpd2dqx" } } */ ++ ++void ++test_ufix (void) ++{ ++ for (int i = 0; i < 2; i++) ++ u[i] = r[i]; ++} ++ ++/* { dg-final { scan-assembler "\tvcvttpd2udqx" } } */ +-- +2.26.2 + +''' + +EXPECTED5 = '''\ +PR target/95046 - Vectorize V2SFmode operations + +gcc/testsuite/ChangeLog: + + PR target/95046 + * gcc.target/i386/pr95046-6.c: New test. + +''' + +class TestMklog(unittest.TestCase): + def test_macro_definition(self): + changelog = generate_changelog(PATCH1) + assert changelog == EXPECTED1 + + def test_changed_argument(self): + changelog = generate_changelog(PATCH2) + assert changelog == EXPECTED2 + + def test_enum_and_struct(self): + changelog = generate_changelog(PATCH3) + assert changelog == EXPECTED3 + + def test_no_function(self): + changelog = generate_changelog(PATCH3, True) + assert changelog == EXPECTED3B + + def test_sorting(self): + changelog = generate_changelog(PATCH4) + assert changelog == EXPECTED4 + + def test_pr_bugzilla_download(self): + changelog = generate_changelog(PATCH5, fill_pr_titles=True) + assert changelog == EXPECTED5 -- 2.30.2