add columns to csv for bug budgets, as well as totals submitted/paid

diff --git a/src/budget_sync/budget_graph.py b/src/budget_sync/budget_graph.py
index c9319f9f6ea405c51daeef6fdc1a093bdeae45ce..e2824ee30ea4dc70589a610a1cdb74df46692fe0 100644 (file)
--- a/src/budget_sync/budget_graph.py
+++ b/src/budget_sync/budget_graph.py
@@ -16,6 +16,7 @@ from collections import OrderedDict
 
 import collections
 
+
 class OrderedSet(collections.MutableSet):
 
     def __init__(self, iterable=None):
@@ -350,6 +351,22 @@ class Node:
             retval[key] = Payment._from_toml(self, key, value)
         return retval
 
+    @cached_property
+    def submitted_excluding_subtasks(self) -> Money:
+        retval = Money()
+        for payment in self.payments.values():
+            if payment.submitted is not None or payment.paid is not None:
+                retval += payment.amount
+        return retval
+
+    @cached_property
+    def paid_excluding_subtasks(self) -> Money:
+        retval = Money()
+        for payment in self.payments.values():
+            if payment.paid is not None:
+                retval += payment.amount
+        return retval
+
     @property
     def parent(self) -> Optional["Node"]:
         if self.parent_id is not None:
@@ -547,6 +564,7 @@ class BudgetGraphIncorrectRootForMilestone(BudgetGraphError):
 
 class BudgetGraph:
     nodes: Dict[int, Node]
+    milestone_payments: Dict[Milestone, List[Payment]]
 
     def __init__(self, bugs: Iterable[Bug], config: Config):
         self.nodes = OrderedDict()
@@ -559,7 +577,7 @@ class BudgetGraph:
             node.parent.immediate_children.add(node)
         self.milestone_payments = OrderedDict()
         # useful debug prints
-        #for bug in bugs:
+        # for bug in bugs:
         #    node = self.nodes[bug.id]
         #    print ("bug added", bug.id, node, node.parent.immediate_children)
 
@@ -621,7 +639,7 @@ class BudgetGraph:
             subtasks_total += child.fixed_budget_including_subtasks
             childlist.append(child.bug.id)
         # useful debug prints
-        #print ("subtask total", node.bug.id, root.bug.id, subtasks_total,
+        # print ("subtask total", node.bug.id, root.bug.id, subtasks_total,
         #                        childlist)
 
         payees_total = Money(0)
@@ -637,9 +655,9 @@ class BudgetGraph:
                 previous_payment = payee_payments.get(payment.payee)
                 if previous_payment is not None:
                     # NOT AN ERROR
-                    print ("NOT AN ERROR", BudgetGraphDuplicatePayeesForTask(
-                           node.bug.id, root.bug.id,
-                           previous_payment[-1].payee_key, payment.payee_key))
+                    print("NOT AN ERROR", BudgetGraphDuplicatePayeesForTask(
+                        node.bug.id, root.bug.id,
+                        previous_payment[-1].payee_key, payment.payee_key))
                     payee_payments[payment.payee].append(payment)
                 else:
                     payee_payments[payment.payee] = [payment]
@@ -782,6 +800,15 @@ class BudgetGraph:
             retval[node.assignee].append(node)
         return retval
 
+    @cached_property
+    def assigned_nodes_for_milestones(self) -> Dict[Milestone, List[Node]]:
+        retval = {milestone: []
+                  for milestone in self.config.milestones.values()}
+        for node in self.nodes.values():
+            if node.milestone is not None:
+                retval[node.milestone].append(node)
+        return retval
+
     @cached_property
     def payments(self) -> Dict[Person, Dict[Milestone, List[Payment]]]:
         retval = OrderedDict()
@@ -789,7 +816,7 @@ class BudgetGraph:
             milestone_payments = OrderedDict()
             for milestone in self.config.milestones.values():
                 milestone_payments[milestone] = []      # per-person payments
-                self.milestone_payments[milestone] = [] # global payments
+                self.milestone_payments[milestone] = []  # global payments
             retval[person] = milestone_payments
         for node in self.nodes.values():
             if node.milestone is not None:
@@ -799,10 +826,10 @@ class BudgetGraph:
                     self.milestone_payments[node.milestone].append(payment)
         return retval
 
-    def get_milestone_people(self):
+    def get_milestone_people(self) -> Dict[Milestone, OrderedSet]:
         """get a list of people associated with each milestone
         """
-        payments = list(self.payments) # just activate the payments
+        payments = list(self.payments)  # just activate the payments
         retval = OrderedDict()
         for milestone in self.milestone_payments.keys():
             retval[milestone] = OrderedSet()

diff --git a/src/budget_sync/main.py b/src/budget_sync/main.py
index 3e5135729468eef5161f7867ac6334d7321d7802..8d47c69dc770dc80037c020e294fbe6d67458fd2 100644 (file)
--- a/src/budget_sync/main.py
+++ b/src/budget_sync/main.py
@@ -29,35 +29,8 @@ mdwn_people_template = """\
 """
 
 
-def main():
-    parser = argparse.ArgumentParser(
-        description="Check for errors in "
-        "Libre-SOC's style of budget tracking in Bugzilla.")
-    parser.add_argument(
-        "-c", "--config", type=argparse.FileType('r'),
-        required=True, help="The path to the configuration TOML file",
-        dest="config", metavar="<path/to/budget-sync-config.toml>")
-    parser.add_argument(
-        "-o", "--output-dir", type=Path, default=None,
-        help="The path to the output directory, will be created if it "
-        "doesn't exist",
-        dest="output_dir", metavar="<path/to/output/dir>")
-    args = parser.parse_args()
-    try:
-        with args.config as config_file:
-            config = Config.from_file(config_file)
-    except (IOError, ConfigParseError) as e:
-        logging.error("Failed to parse config file: %s", e)
-        return
-    logging.info("Using Bugzilla instance at %s", config.bugzilla_url)
-    bz = Bugzilla(config.bugzilla_url)
-    logging.debug("Connected to Bugzilla")
-    budget_graph = BudgetGraph(all_bugs(bz), config)
-    for error in budget_graph.get_errors():
-        logging.error("%s", error)
-    if args.output_dir is not None:
-        write_budget_markdown(budget_graph, args.output_dir)
-
+def write_budget_csv(budget_graph: BudgetGraph,
+                     output_dir: Path):
     # quick hack to display total payment amounts per-milestone
     for milestone, payments in budget_graph.milestone_payments.items():
         print(milestone)
@@ -92,14 +65,16 @@ def main():
     milestone_csvs = {}
     milestone_headings = {}
     all_people = OrderedDict()
-    # pre-initialise the CSV lists (to avoid overwrite)
-    for milestone, payments in budget_graph.milestone_payments.items():
+    for milestone, nodes in budget_graph.assigned_nodes_for_milestones.items():
         milestone_csvs[milestone] = {}  # rows in the CSV file
-
-    for milestone, payments in budget_graph.milestone_payments.items():
-        # first get the list of people, then create some columns
         people = milestones_people[milestone]
-        headings = ['bug_id']
+        headings = ['bug_id',
+                    'budget_excluding_subtasks',
+                    'budget_including_subtasks',
+                    'fixed_budget_excluding_subtasks',
+                    'fixed_budget_including_subtasks',
+                    'submitted_excluding_subtasks',
+                    'paid_excluding_subtasks']
         for person in people:
             name = str(person).replace(" ", "_")
             all_people[person] = person
@@ -108,46 +83,84 @@ def main():
             headings.append(name+"_req")
             headings.append(name+"_paid")
         milestone_headings[milestone] = headings
-        # now we go through the whole "payments" thing again...
-        for payment in payments:
-            row = milestone_csvs[milestone].get(payment.node.bug.id, None)
-            if row is None:
-                row = {'bug_id': payment.node.bug.id}
-
-            short_name = str(payment.payee.output_markdown_file)
-            name = short_name.replace(".mdwn", "")
-
-            row[name] = str(payment.amount)
-            if payment.submitted is not None:
-                requested = str(payment.submitted)
-            else:
-                requested = ""
-            if payment.paid is not None:
-                paid = str(payment.paid)
-            else:
-                paid = ""
-            row[name+"_req"] = requested
-            row[name+"_paid"] = paid
+        for node in nodes:
+            # skip uninteresting nodes
+            if len(node.payments) == 0 \
+                    and node.budget_excluding_subtasks == 0 \
+                    and node.budget_including_subtasks == 0:
+                continue
+            row = {'bug_id': node.bug.id,
+                   'budget_excluding_subtasks': str(node.budget_excluding_subtasks),
+                   'budget_including_subtasks': str(node.budget_including_subtasks),
+                   'fixed_budget_excluding_subtasks': str(node.fixed_budget_excluding_subtasks),
+                   'fixed_budget_including_subtasks': str(node.fixed_budget_including_subtasks),
+                   'submitted_excluding_subtasks': str(node.submitted_excluding_subtasks),
+                   'paid_excluding_subtasks': str(node.paid_excluding_subtasks)}
+            for payment in node.payments.values():
+                short_name = str(payment.payee.output_markdown_file)
+                name = short_name.replace(".mdwn", "")
+
+                row[name] = str(payment.amount)
+                if payment.submitted is not None:
+                    requested = str(payment.submitted)
+                else:
+                    requested = ""
+                if payment.paid is not None:
+                    paid = str(payment.paid)
+                else:
+                    paid = ""
+                row[name+"_req"] = requested
+                row[name+"_paid"] = paid
 
             print(row)
-            milestone_csvs[milestone][payment.node.bug.id] = row
+            milestone_csvs[milestone][node.bug.id] = row
+
+    with open(output_dir.joinpath("csvs.mdwn"), "w") as f:
+        # write out the people pages
+        # TODO, has to be done by the markdown page name
+        # f.write("# People\n\n")
+        # for name, person in all_people.items():
+        #    fname = output_dir.joinpath(f"{name}.csv")
+        #    f.write(mdwn_people_template % (person, fname))
+        # and the CSV files
+        for milestone, rows in milestone_csvs.items():
+            ident = milestone.identifier
+            header = milestone_headings[milestone]
+            fname = output_dir.joinpath(f"{ident}.csv")
+            rows = rows.values()  # turn into list
+            write_csv(fname, rows, header)
+            f.write(mdwn_csv_template % (ident, fname))
 
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Check for errors in "
+        "Libre-SOC's style of budget tracking in Bugzilla.")
+    parser.add_argument(
+        "-c", "--config", type=argparse.FileType('r'),
+        required=True, help="The path to the configuration TOML file",
+        dest="config", metavar="<path/to/budget-sync-config.toml>")
+    parser.add_argument(
+        "-o", "--output-dir", type=Path, default=None,
+        help="The path to the output directory, will be created if it "
+        "doesn't exist",
+        dest="output_dir", metavar="<path/to/output/dir>")
+    args = parser.parse_args()
+    try:
+        with args.config as config_file:
+            config = Config.from_file(config_file)
+    except (IOError, ConfigParseError) as e:
+        logging.error("Failed to parse config file: %s", e)
+        return
+    logging.info("Using Bugzilla instance at %s", config.bugzilla_url)
+    bz = Bugzilla(config.bugzilla_url)
+    logging.debug("Connected to Bugzilla")
+    budget_graph = BudgetGraph(all_bugs(bz), config)
+    for error in budget_graph.get_errors():
+        logging.error("%s", error)
     if args.output_dir is not None:
-        with open("%s/csvs.mdwn" % args.output_dir, "w") as f:
-            # write out the people pages
-            # TODO, has to be done by the markdown page name
-            # f.write("# People\n\n")
-            # for name, person in all_people.items():
-            #    fname = "%s/%s" % (args.output_dir, name)
-            #    f.write(mdwn_people_template % (person, fname))
-            # and the CSV files
-            for milestone, rows in milestone_csvs.items():
-                ident = milestone.identifier
-                header = milestone_headings[milestone]
-                fname = "%s/%s.csv" % (args.output_dir, ident)
-                rows = rows.values()  # turn into list
-                write_csv(fname, rows, header)
-                f.write(mdwn_csv_template % (ident, fname))
+        write_budget_markdown(budget_graph, args.output_dir)
+        write_budget_csv(budget_graph, args.output_dir)
 
 
 if __name__ == "__main__":

diff --git a/src/budget_sync/util.py b/src/budget_sync/util.py
index 342f685e9ed4f822cfcadc5424f6c05be6f98844..ddcb145c58fcb96804f46e7d8b360f7f3562b98d 100644 (file)
--- a/src/budget_sync/util.py
+++ b/src/budget_sync/util.py
@@ -37,7 +37,7 @@ def all_bugs(bz: Bugzilla) -> Iterator[Bug]:
     while True:
         bugs = bz.getbugs(list(range(chunk_start, chunk_start + chunk_size)))
         chunk_start += chunk_size
-        print ("bugs loaded", len(bugs), chunk_start)
+        print("bugs loaded", len(bugs), chunk_start)
         if len(bugs) == 0:
             return
         yield from bugs

diff --git a/src/budget_sync/write_budget_markdown.py b/src/budget_sync/write_budget_markdown.py
index 624e639ce3ab43935aa8f84564dee531b9040e71..c0092d2215b69f3106c78b66f247e284f7ed022e 100644 (file)
--- a/src/budget_sync/write_budget_markdown.py
+++ b/src/budget_sync/write_budget_markdown.py
@@ -147,7 +147,7 @@ def _markdown_for_person(person: Person,
         else:
             assert payee_state == PayeeState.Paid
             display_status_header = f"## Paid by NLNet"
-        display_status_header="\n%s\n" % display_status_header
+        display_status_header = "\n%s\n" % display_status_header
         for milestone, payments_list in payments_dict.items():
             milestone_header = f"\n### {milestone.identifier}\n"
             for payment in payments_list: