From 1677f851d34271d1c01c3070cf491116c26b85bd Mon Sep 17 00:00:00 2001 From: Jacob Lifshay Date: Thu, 7 Jul 2022 00:33:09 -0700 Subject: [PATCH] implement calculating subtotals for MoU Milestones for subsets of bugs --- src/budget_sync/budget_graph.py | 4 +- src/budget_sync/main.py | 52 +++++++- .../test/test_write_budget_markdown.py | 125 ++++++++++++++++-- src/budget_sync/write_budget_markdown.py | 88 +++++++++--- 4 files changed, 235 insertions(+), 34 deletions(-) diff --git a/src/budget_sync/budget_graph.py b/src/budget_sync/budget_graph.py index b3d457c..7066306 100644 --- a/src/budget_sync/budget_graph.py +++ b/src/budget_sync/budget_graph.py @@ -494,7 +494,9 @@ class Node: yield node def __eq__(self, other): - return self.bug.id == other.bug.id + if isinstance(other, Node): + return self.bug.id == other.bug.id + return NotImplemented def __hash__(self): return self.bug.id diff --git a/src/budget_sync/main.py b/src/budget_sync/main.py index c7abc43..1e1c61c 100644 --- a/src/budget_sync/main.py +++ b/src/budget_sync/main.py @@ -1,14 +1,18 @@ -from typing import Dict, List +import os +import re +import sys +from typing import Optional +from budget_sync.ordered_set import OrderedSet from budget_sync.write_budget_csv import write_budget_csv from bugzilla import Bugzilla import logging import argparse from pathlib import Path -from budget_sync.money import Money from budget_sync.util import all_bugs -from budget_sync.config import Config, ConfigParseError, Milestone -from budget_sync.budget_graph import BudgetGraph, BudgetGraphBaseError, PaymentSummary -from budget_sync.write_budget_markdown import write_budget_markdown +from budget_sync.config import Config, ConfigParseError +from budget_sync.budget_graph import BudgetGraph, PaymentSummary +from budget_sync.write_budget_markdown import (write_budget_markdown, + markdown_for_person) def main(): @@ -24,8 +28,16 @@ def main(): help="The path to the output directory, will be created if it " "doesn't exist", dest="output_dir", metavar="") - parser.add_argument('--username', help="Log in with this username") - parser.add_argument('--password', help="Log in with this password") + parser.add_argument('--subset', + help="write the output for this subset of bugs", + metavar=",,...") + parser.add_argument('--subset-person', + help="write the output for this person", + dest="person") + parser.add_argument('--username', + help="Log in with this Bugzilla username") + parser.add_argument('--password', + help="Log in with this Bugzilla password") args = parser.parse_args() try: with args.config as config_file: @@ -42,12 +54,38 @@ def main(): budget_graph = BudgetGraph(all_bugs(bz), config) for error in budget_graph.get_errors(): logging.error("%s", error) + if args.person or args.subset: + if not args.person: + logging.fatal("must use --subset-person with --subset option") + sys.exit(1) + print_markdown_for_person(budget_graph, config, + args.person, args.subset) + return if args.output_dir is not None: write_budget_markdown(budget_graph, args.output_dir) write_budget_csv(budget_graph, args.output_dir) summarize_milestones(budget_graph) +def print_markdown_for_person(budget_graph: BudgetGraph, config: Config, + person_str: str, subset_str: Optional[str]): + person = config.all_names.get(person_str) + if person is None: + logging.fatal("--subset-person: unknown person: %s", person_str) + sys.exit(1) + nodes_subset = None + if subset_str: + nodes_subset = OrderedSet() + for bug_id in re.split(r"[\s,]+", subset_str): + try: + node = budget_graph.nodes[int(bug_id)] + except (ValueError, KeyError): + logging.fatal("--subset: unknown bug: %s", bug_id) + sys.exit(1) + nodes_subset.add(node) + print(markdown_for_person(budget_graph, person, nodes_subset)) + + def print_budget_then_children(indent, nodes, bug_id): """recursive indented printout of budgets """ diff --git a/src/budget_sync/test/test_write_budget_markdown.py b/src/budget_sync/test/test_write_budget_markdown.py index 1698767..e26e92a 100644 --- a/src/budget_sync/test/test_write_budget_markdown.py +++ b/src/budget_sync/test/test_write_budget_markdown.py @@ -1,11 +1,12 @@ import unittest from budget_sync.config import Config +from budget_sync.ordered_set import OrderedSet from budget_sync.test.mock_bug import MockBug from budget_sync.test.mock_path import MockFilesystem, MockPath, DIR from budget_sync.test.test_mock_path import make_filesystem_and_report_if_error from budget_sync.budget_graph import BudgetGraph from budget_sync.write_budget_markdown import ( - write_budget_markdown, DisplayStatus, markdown_escape) + write_budget_markdown, DisplayStatus, markdown_escape, markdown_for_person) from budget_sync.util import BugStatus @@ -70,7 +71,7 @@ class TestWriteBudgetMarkdown(unittest.TestCase): cf_total_budget="0", cf_nlnet_milestone=None, cf_payees_list="", - summary="", + summary="summary1", assigned_to="person1@example.com"), ], config) self.assertEqual([], budget_graph.get_errors()) @@ -121,7 +122,7 @@ class TestWriteBudgetMarkdown(unittest.TestCase): cf_total_budget="1000", cf_nlnet_milestone="milestone 1", cf_payees_list="", - summary="", + summary="summary1", assigned_to="person1@example.com"), MockBug(bug_id=2, cf_budget_parent=1, @@ -129,7 +130,7 @@ class TestWriteBudgetMarkdown(unittest.TestCase): cf_total_budget="300", cf_nlnet_milestone="milestone 1", cf_payees_list="person2 = 100", - summary="", + summary="summary2", assigned_to="person1@example.com"), MockBug(bug_id=3, cf_budget_parent=2, @@ -137,7 +138,7 @@ class TestWriteBudgetMarkdown(unittest.TestCase): cf_total_budget="200", cf_nlnet_milestone="milestone 1", cf_payees_list="person1 = 100", - summary="", + summary="summary3", assigned_to="person1@example.com"), MockBug(bug_id=4, cf_budget_parent=3, @@ -145,7 +146,7 @@ class TestWriteBudgetMarkdown(unittest.TestCase): cf_total_budget="100", cf_nlnet_milestone="milestone 1", cf_payees_list="person2 = 100", - summary="", + summary="summary4", assigned_to="person1@example.com"), ], config) self.assertEqual([], budget_graph.get_errors()) @@ -171,10 +172,16 @@ class TestWriteBudgetMarkdown(unittest.TestCase): b'### milestone 1\n' b'\n' b'* [Bug #3](https://bugzilla.example.com/show_bug.cgi?id=3):\n' - b' \n' + b' summary3\n' b' * €100 which is the total amount\n' b' * this task is part of MoU Milestone\n' b' [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2)\n' + b'\n' + b'#### MoU Milestone subtotals for not yet submitted payments\n' + b'\n' + b'* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n' + b' summary2\n' + b' * subtotal €100 out of total including subtasks of €300\n' ), '/output_dir/person2.mdwn': ( b'\n' @@ -192,16 +199,116 @@ class TestWriteBudgetMarkdown(unittest.TestCase): b'### milestone 1\n' b'\n' b'* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n' - b' \n' + b' summary2\n' b' * €100 which is the total amount\n' b' * this task is a MoU Milestone\n' b'* [Bug #4](https://bugzilla.example.com/show_bug.cgi?id=4):\n' - b' \n' + b' summary4\n' b' * €100 which is the total amount\n' b' * this task is part of MoU Milestone\n' b' [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2)\n' + b'\n' + b'#### MoU Milestone subtotals for not yet submitted payments\n' + b'\n' + b'* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n' + b' summary2\n' + b' * subtotal €200 out of total including subtasks of €300\n' ), }, filesystem) + + def test_markdown_for_person(self): + config = Config.from_str( + """ + bugzilla_url = "https://bugzilla.example.com/" + [milestones] + "milestone 1" = { canonical_bug_id = 1 } + [people."person1"] + email = "person1@example.com" + full_name = "Person One" + [people."person2"] + full_name = "Person Two" + """) + budget_graph = BudgetGraph([ + MockBug(bug_id=1, + cf_budget_parent=None, + cf_budget="600", + cf_total_budget="1000", + cf_nlnet_milestone="milestone 1", + cf_payees_list="", + summary="summary1", + assigned_to="person1@example.com"), + MockBug(bug_id=2, + cf_budget_parent=1, + cf_budget="100", + cf_total_budget="400", + cf_nlnet_milestone="milestone 1", + cf_payees_list="person2 = 100", + summary="summary2", + assigned_to="person1@example.com"), + MockBug(bug_id=3, + cf_budget_parent=2, + cf_budget="100", + cf_total_budget="300", + cf_nlnet_milestone="milestone 1", + cf_payees_list="person1 = 100", + summary="summary3", + assigned_to="person1@example.com"), + MockBug(bug_id=4, + cf_budget_parent=3, + cf_budget="100", + cf_total_budget="100", + cf_nlnet_milestone="milestone 1", + cf_payees_list="person2 = 100", + summary="summary4", + assigned_to="person1@example.com"), + MockBug(bug_id=5, + cf_budget_parent=3, + cf_budget="100", + cf_total_budget="100", + cf_nlnet_milestone="milestone 1", + cf_payees_list="person2 = 100", + summary="summary4", + assigned_to="person1@example.com"), + ], config) + self.assertEqual([], budget_graph.get_errors()) + person = config.all_names["person2"] + nodes_subset = OrderedSet([budget_graph.nodes[2], + budget_graph.nodes[3], + budget_graph.nodes[4]]) + expected = [ + '\n', + '\n', + '# Person Two (person2)\n', + '\n', + '\n', + '\n', + '# Status Tracking\n', + '\n', + '\n', + '## Payment not yet submitted\n', + '\n', + '\n', + '### milestone 1\n', + '\n', + '* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n', + ' summary2\n', + ' * €100 which is the total amount\n', + ' * this task is a MoU Milestone\n', + '* [Bug #4](https://bugzilla.example.com/show_bug.cgi?id=4):\n', + ' summary4\n', + ' * €100 which is the total amount\n', + ' * this task is part of MoU Milestone\n', + ' [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2)\n', + '\n', + '#### MoU Milestone subtotals for not yet submitted payments\n', + '\n', + '* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n', + ' summary2\n', + ' * subtotal €200 out of total including subtasks of €400\n', + ] + self.assertEqual(markdown_for_person( + budget_graph, person, nodes_subset).splitlines(keepends=True), + expected) # TODO: add more test cases diff --git a/src/budget_sync/write_budget_markdown.py b/src/budget_sync/write_budget_markdown.py index 427f97f..ce2e246 100644 --- a/src/budget_sync/write_budget_markdown.py +++ b/src/budget_sync/write_budget_markdown.py @@ -1,9 +1,12 @@ +from collections import defaultdict from pathlib import Path from typing import Dict, List, Any, Optional from io import StringIO import enum -from budget_sync.budget_graph import BudgetGraph, Node, Payment, PayeeState -from budget_sync.config import Person, Milestone, Config +from budget_sync.budget_graph import BudgetGraph, Node, Payment, PayeeState, PaymentSummary +from budget_sync.config import Person, Milestone +from budget_sync.money import Money +from budget_sync.ordered_set import OrderedSet from budget_sync.util import BugStatus @@ -68,14 +71,22 @@ class MarkdownWriter: self.last_headers, headers) assert headers == self.last_headers - def write_node(self, - headers: List[str], - node: Node, - payment: Optional[Payment]): + def write_node_header(self, + headers: List[str], + node: Optional[Node]): self.write_headers(headers) + if node is None: + print("* None", file=self.buffer) + return summary = markdown_escape(node.bug.summary) print(f"* [Bug #{node.bug.id}]({node.bug_url}):\n {summary}", file=self.buffer) + + def write_node(self, + headers: List[str], + node: Node, + payment: Optional[Payment]): + self.write_node_header(headers, node) if payment is not None: if node.fixed_budget_excluding_subtasks \ != node.budget_excluding_subtasks: @@ -113,7 +124,11 @@ class MarkdownWriter: def _markdown_for_person(person: Person, payments_dict: Dict[Milestone, List[Payment]], - assigned_nodes: List[Node]) -> str: + assigned_nodes: List[Node], + nodes_subset: Optional[OrderedSet[Node]] = None, + ) -> str: + def node_included(node: Node) -> bool: + return nodes_subset is None or node in nodes_subset writer = MarkdownWriter() print(f"", file=writer.buffer) writer.write_headers([f"\n# {person.full_name} ({person.identifier})\n"]) @@ -129,6 +144,8 @@ def _markdown_for_person(person: Person, def write_display_status_chunk(display_status: DisplayStatus): display_status_header = f"\n## {display_status.value}\n" for node in displayed_nodes_dict[display_status]: + if not node_included(node): + continue if display_status == DisplayStatus.Completed: payment_found = False for payment in node.payments.values(): @@ -153,21 +170,48 @@ def _markdown_for_person(person: Person, for payee_state in PayeeState: if payee_state == PayeeState.NotYetSubmitted: - display_status_header = f"## Payment not yet submitted" + display_status_header = "\n## Payment not yet submitted\n" + subtotals_header = ("\n#### MoU Milestone subtotals for not " + "yet submitted payments\n") elif payee_state == PayeeState.Submitted: - display_status_header = f"## Submitted to NLNet but not yet paid" + display_status_header = ("\n## Submitted to NLNet but " + "not yet paid\n") + subtotals_header = ("\n#### MoU Milestone subtotals for " + "submitted but not yet paid payments\n") else: assert payee_state == PayeeState.Paid - display_status_header = f"## Paid by NLNet" - display_status_header = "\n%s\n" % display_status_header + display_status_header = "\n## Paid by NLNet\n" + subtotals_header = ("\n#### MoU Milestone subtotals for paid " + "payments\n") for milestone, payments_list in payments_dict.items(): milestone_header = f"\n### {milestone.identifier}\n" + mou_subtotals: Dict[Optional[Node], Money] = defaultdict(Money) + headers = [status_tracking_header, + display_status_header, + milestone_header] for payment in payments_list: - if payment.state == payee_state: - writer.write_node(headers=[status_tracking_header, - display_status_header, - milestone_header], + node = payment.node + if payment.state == payee_state and node_included(node): + mou_subtotals[node.closest_bug_in_mou] += payment.amount + writer.write_node(headers=headers, node=payment.node, payment=payment) + headers.append(subtotals_header) + for node, subtotal in mou_subtotals.items(): + writer.write_node_header(headers, node) + if node is None: + budget = "" + elif node.fixed_budget_including_subtasks \ + != node.budget_including_subtasks: + budget = (" out of total including subtasks of " + f"€{node.fixed_budget_including_subtasks}" + " (budget is fixed from amount appearing in " + "bug report, which is " + f"€{node.budget_including_subtasks})") + else: + budget = (" out of total including subtasks of " + f"€{node.fixed_budget_including_subtasks}") + print(f" * subtotal €{subtotal}{budget}", + file=writer.buffer) # write_display_status_chunk(DisplayStatus.NotYetStarted) @@ -175,11 +219,21 @@ def _markdown_for_person(person: Person, def write_budget_markdown(budget_graph: BudgetGraph, - output_dir: Path): + output_dir: Path, + nodes_subset: Optional[OrderedSet[Node]] = None): output_dir.mkdir(parents=True, exist_ok=True) for person, payments_dict in budget_graph.payments.items(): markdown = _markdown_for_person(person, payments_dict, - budget_graph.assigned_nodes[person]) + budget_graph.assigned_nodes[person], + nodes_subset) output_file = output_dir.joinpath(person.output_markdown_file) output_file.write_text(markdown, encoding="utf-8") + + +def markdown_for_person(budget_graph: BudgetGraph, person: Person, + nodes_subset: Optional[OrderedSet[Node]] = None, + ) -> str: + return _markdown_for_person(person, budget_graph.payments[person], + budget_graph.assigned_nodes[person], + nodes_subset) -- 2.30.2