From 0c72d3e0f097bfb5b3f38aee96ea45a16d0341cf Mon Sep 17 00:00:00 2001 From: Jacob Lifshay Date: Tue, 21 Sep 2021 21:24:56 -0700 Subject: [PATCH] add columns to csv for bug budgets, as well as totals submitted/paid --- src/budget_sync/budget_graph.py | 43 +++++-- src/budget_sync/main.py | 155 ++++++++++++----------- src/budget_sync/util.py | 2 +- src/budget_sync/write_budget_markdown.py | 2 +- 4 files changed, 121 insertions(+), 81 deletions(-) diff --git a/src/budget_sync/budget_graph.py b/src/budget_sync/budget_graph.py index c9319f9..e2824ee 100644 --- a/src/budget_sync/budget_graph.py +++ b/src/budget_sync/budget_graph.py @@ -16,6 +16,7 @@ from collections import OrderedDict import collections + class OrderedSet(collections.MutableSet): def __init__(self, iterable=None): @@ -350,6 +351,22 @@ class Node: retval[key] = Payment._from_toml(self, key, value) return retval + @cached_property + def submitted_excluding_subtasks(self) -> Money: + retval = Money() + for payment in self.payments.values(): + if payment.submitted is not None or payment.paid is not None: + retval += payment.amount + return retval + + @cached_property + def paid_excluding_subtasks(self) -> Money: + retval = Money() + for payment in self.payments.values(): + if payment.paid is not None: + retval += payment.amount + return retval + @property def parent(self) -> Optional["Node"]: if self.parent_id is not None: @@ -547,6 +564,7 @@ class BudgetGraphIncorrectRootForMilestone(BudgetGraphError): class BudgetGraph: nodes: Dict[int, Node] + milestone_payments: Dict[Milestone, List[Payment]] def __init__(self, bugs: Iterable[Bug], config: Config): self.nodes = OrderedDict() @@ -559,7 +577,7 @@ class BudgetGraph: node.parent.immediate_children.add(node) self.milestone_payments = OrderedDict() # useful debug prints - #for bug in bugs: + # for bug in bugs: # node = self.nodes[bug.id] # print ("bug added", bug.id, node, node.parent.immediate_children) @@ -621,7 +639,7 @@ class BudgetGraph: subtasks_total += child.fixed_budget_including_subtasks childlist.append(child.bug.id) # useful debug prints - #print ("subtask total", node.bug.id, root.bug.id, subtasks_total, + # print ("subtask total", node.bug.id, root.bug.id, subtasks_total, # childlist) payees_total = Money(0) @@ -637,9 +655,9 @@ class BudgetGraph: previous_payment = payee_payments.get(payment.payee) if previous_payment is not None: # NOT AN ERROR - print ("NOT AN ERROR", BudgetGraphDuplicatePayeesForTask( - node.bug.id, root.bug.id, - previous_payment[-1].payee_key, payment.payee_key)) + print("NOT AN ERROR", BudgetGraphDuplicatePayeesForTask( + node.bug.id, root.bug.id, + previous_payment[-1].payee_key, payment.payee_key)) payee_payments[payment.payee].append(payment) else: payee_payments[payment.payee] = [payment] @@ -782,6 +800,15 @@ class BudgetGraph: retval[node.assignee].append(node) return retval + @cached_property + def assigned_nodes_for_milestones(self) -> Dict[Milestone, List[Node]]: + retval = {milestone: [] + for milestone in self.config.milestones.values()} + for node in self.nodes.values(): + if node.milestone is not None: + retval[node.milestone].append(node) + return retval + @cached_property def payments(self) -> Dict[Person, Dict[Milestone, List[Payment]]]: retval = OrderedDict() @@ -789,7 +816,7 @@ class BudgetGraph: milestone_payments = OrderedDict() for milestone in self.config.milestones.values(): milestone_payments[milestone] = [] # per-person payments - self.milestone_payments[milestone] = [] # global payments + self.milestone_payments[milestone] = [] # global payments retval[person] = milestone_payments for node in self.nodes.values(): if node.milestone is not None: @@ -799,10 +826,10 @@ class BudgetGraph: self.milestone_payments[node.milestone].append(payment) return retval - def get_milestone_people(self): + def get_milestone_people(self) -> Dict[Milestone, OrderedSet]: """get a list of people associated with each milestone """ - payments = list(self.payments) # just activate the payments + payments = list(self.payments) # just activate the payments retval = OrderedDict() for milestone in self.milestone_payments.keys(): retval[milestone] = OrderedSet() diff --git a/src/budget_sync/main.py b/src/budget_sync/main.py index 3e51357..8d47c69 100644 --- a/src/budget_sync/main.py +++ b/src/budget_sync/main.py @@ -29,35 +29,8 @@ mdwn_people_template = """\ """ -def main(): - parser = argparse.ArgumentParser( - description="Check for errors in " - "Libre-SOC's style of budget tracking in Bugzilla.") - parser.add_argument( - "-c", "--config", type=argparse.FileType('r'), - required=True, help="The path to the configuration TOML file", - dest="config", metavar="") - parser.add_argument( - "-o", "--output-dir", type=Path, default=None, - help="The path to the output directory, will be created if it " - "doesn't exist", - dest="output_dir", metavar="") - args = parser.parse_args() - try: - with args.config as config_file: - config = Config.from_file(config_file) - except (IOError, ConfigParseError) as e: - logging.error("Failed to parse config file: %s", e) - return - logging.info("Using Bugzilla instance at %s", config.bugzilla_url) - bz = Bugzilla(config.bugzilla_url) - logging.debug("Connected to Bugzilla") - budget_graph = BudgetGraph(all_bugs(bz), config) - for error in budget_graph.get_errors(): - logging.error("%s", error) - if args.output_dir is not None: - write_budget_markdown(budget_graph, args.output_dir) - +def write_budget_csv(budget_graph: BudgetGraph, + output_dir: Path): # quick hack to display total payment amounts per-milestone for milestone, payments in budget_graph.milestone_payments.items(): print(milestone) @@ -92,14 +65,16 @@ def main(): milestone_csvs = {} milestone_headings = {} all_people = OrderedDict() - # pre-initialise the CSV lists (to avoid overwrite) - for milestone, payments in budget_graph.milestone_payments.items(): + for milestone, nodes in budget_graph.assigned_nodes_for_milestones.items(): milestone_csvs[milestone] = {} # rows in the CSV file - - for milestone, payments in budget_graph.milestone_payments.items(): - # first get the list of people, then create some columns people = milestones_people[milestone] - headings = ['bug_id'] + headings = ['bug_id', + 'budget_excluding_subtasks', + 'budget_including_subtasks', + 'fixed_budget_excluding_subtasks', + 'fixed_budget_including_subtasks', + 'submitted_excluding_subtasks', + 'paid_excluding_subtasks'] for person in people: name = str(person).replace(" ", "_") all_people[person] = person @@ -108,46 +83,84 @@ def main(): headings.append(name+"_req") headings.append(name+"_paid") milestone_headings[milestone] = headings - # now we go through the whole "payments" thing again... - for payment in payments: - row = milestone_csvs[milestone].get(payment.node.bug.id, None) - if row is None: - row = {'bug_id': payment.node.bug.id} - - short_name = str(payment.payee.output_markdown_file) - name = short_name.replace(".mdwn", "") - - row[name] = str(payment.amount) - if payment.submitted is not None: - requested = str(payment.submitted) - else: - requested = "" - if payment.paid is not None: - paid = str(payment.paid) - else: - paid = "" - row[name+"_req"] = requested - row[name+"_paid"] = paid + for node in nodes: + # skip uninteresting nodes + if len(node.payments) == 0 \ + and node.budget_excluding_subtasks == 0 \ + and node.budget_including_subtasks == 0: + continue + row = {'bug_id': node.bug.id, + 'budget_excluding_subtasks': str(node.budget_excluding_subtasks), + 'budget_including_subtasks': str(node.budget_including_subtasks), + 'fixed_budget_excluding_subtasks': str(node.fixed_budget_excluding_subtasks), + 'fixed_budget_including_subtasks': str(node.fixed_budget_including_subtasks), + 'submitted_excluding_subtasks': str(node.submitted_excluding_subtasks), + 'paid_excluding_subtasks': str(node.paid_excluding_subtasks)} + for payment in node.payments.values(): + short_name = str(payment.payee.output_markdown_file) + name = short_name.replace(".mdwn", "") + + row[name] = str(payment.amount) + if payment.submitted is not None: + requested = str(payment.submitted) + else: + requested = "" + if payment.paid is not None: + paid = str(payment.paid) + else: + paid = "" + row[name+"_req"] = requested + row[name+"_paid"] = paid print(row) - milestone_csvs[milestone][payment.node.bug.id] = row + milestone_csvs[milestone][node.bug.id] = row + + with open(output_dir.joinpath("csvs.mdwn"), "w") as f: + # write out the people pages + # TODO, has to be done by the markdown page name + # f.write("# People\n\n") + # for name, person in all_people.items(): + # fname = output_dir.joinpath(f"{name}.csv") + # f.write(mdwn_people_template % (person, fname)) + # and the CSV files + for milestone, rows in milestone_csvs.items(): + ident = milestone.identifier + header = milestone_headings[milestone] + fname = output_dir.joinpath(f"{ident}.csv") + rows = rows.values() # turn into list + write_csv(fname, rows, header) + f.write(mdwn_csv_template % (ident, fname)) + +def main(): + parser = argparse.ArgumentParser( + description="Check for errors in " + "Libre-SOC's style of budget tracking in Bugzilla.") + parser.add_argument( + "-c", "--config", type=argparse.FileType('r'), + required=True, help="The path to the configuration TOML file", + dest="config", metavar="") + parser.add_argument( + "-o", "--output-dir", type=Path, default=None, + help="The path to the output directory, will be created if it " + "doesn't exist", + dest="output_dir", metavar="") + args = parser.parse_args() + try: + with args.config as config_file: + config = Config.from_file(config_file) + except (IOError, ConfigParseError) as e: + logging.error("Failed to parse config file: %s", e) + return + logging.info("Using Bugzilla instance at %s", config.bugzilla_url) + bz = Bugzilla(config.bugzilla_url) + logging.debug("Connected to Bugzilla") + budget_graph = BudgetGraph(all_bugs(bz), config) + for error in budget_graph.get_errors(): + logging.error("%s", error) if args.output_dir is not None: - with open("%s/csvs.mdwn" % args.output_dir, "w") as f: - # write out the people pages - # TODO, has to be done by the markdown page name - # f.write("# People\n\n") - # for name, person in all_people.items(): - # fname = "%s/%s" % (args.output_dir, name) - # f.write(mdwn_people_template % (person, fname)) - # and the CSV files - for milestone, rows in milestone_csvs.items(): - ident = milestone.identifier - header = milestone_headings[milestone] - fname = "%s/%s.csv" % (args.output_dir, ident) - rows = rows.values() # turn into list - write_csv(fname, rows, header) - f.write(mdwn_csv_template % (ident, fname)) + write_budget_markdown(budget_graph, args.output_dir) + write_budget_csv(budget_graph, args.output_dir) if __name__ == "__main__": diff --git a/src/budget_sync/util.py b/src/budget_sync/util.py index 342f685..ddcb145 100644 --- a/src/budget_sync/util.py +++ b/src/budget_sync/util.py @@ -37,7 +37,7 @@ def all_bugs(bz: Bugzilla) -> Iterator[Bug]: while True: bugs = bz.getbugs(list(range(chunk_start, chunk_start + chunk_size))) chunk_start += chunk_size - print ("bugs loaded", len(bugs), chunk_start) + print("bugs loaded", len(bugs), chunk_start) if len(bugs) == 0: return yield from bugs diff --git a/src/budget_sync/write_budget_markdown.py b/src/budget_sync/write_budget_markdown.py index 624e639..c0092d2 100644 --- a/src/budget_sync/write_budget_markdown.py +++ b/src/budget_sync/write_budget_markdown.py @@ -147,7 +147,7 @@ def _markdown_for_person(person: Person, else: assert payee_state == PayeeState.Paid display_status_header = f"## Paid by NLNet" - display_status_header="\n%s\n" % display_status_header + display_status_header = "\n%s\n" % display_status_header for milestone, payments_list in payments_dict.items(): milestone_header = f"\n### {milestone.identifier}\n" for payment in payments_list: -- 2.30.2