src/budget_sync/budget_graph.py

   1 from bugzilla.bug import Bug
   2 from bugzilla import Bugzilla
   3 from typing import Set, Dict, Iterable, Optional, List, Union, Any
   4 from budget_sync.util import all_bugs
   5 from budget_sync.money import Money
   6 from functools import cached_property
   7 import toml
   8 import sys
   9 import enum
  10 from collections import deque
  11 from datetime import date, time, datetime
  12
  13
  14 class BudgetGraphBaseError(Exception):
  15     pass
  16
  17
  18 class BudgetGraphParseError(BudgetGraphBaseError):
  19     def __init__(self, bug_id: int):
  20         self.bug_id = bug_id
  21
  22
  23 class BudgetGraphPayeesParseError(BudgetGraphParseError):
  24     def __init__(self, bug_id: int, msg: str):
  25         super().__init__(bug_id)
  26         self.msg = msg
  27
  28     def __str__(self):
  29         return f"Failed to parse cf_payees_list field of bug #{self.bug_id}: {self.msg}"
  30
  31
  32 class BudgetGraphLoopError(BudgetGraphBaseError):
  33     def __init__(self, bug_ids: List[int]):
  34         self.bug_ids = bug_ids
  35
  36     def __str__(self):
  37         retval = f"Detected Loop in Budget Graph: #{self.bug_ids[-1]} -> "
  38         retval += " -> ".join((f"#{i}" for i in self.bug_ids))
  39         return retval
  40
  41
  42 class _NodeSimpleReprWrapper:
  43     def __init__(self, node: "Node"):
  44         self.node = node
  45
  46     def __repr__(self):
  47         return f"#{self.node.bug.id}"
  48
  49     def __lt__(self, other):
  50         # for list.sort()
  51         return self.node.bug.id < other.node.bug.id
  52
  53
  54 class PayeeState(enum.Enum):
  55     NotYetSubmitted = "not yet submitted"
  56     Submitted = "submitted"
  57     Paid = "paid"
  58
  59
  60 _Date = Union[date, datetime]
  61
  62
  63 def _parse_money_from_toml(value: Any) -> Money:
  64     if not isinstance(value, (int, str)):
  65         msg = f"monetary amount is not a string or integer " \
  66             f"(to use fractional amounts such as 123.45, write " \
  67             f"\"123.45\"): {value!r}"
  68         raise ValueError(msg)
  69     return Money(value)
  70
  71
  72 def _parse_date_time_or_none_from_toml(value: Any) -> Optional[_Date]:
  73     if value is None or isinstance(value, (date, datetime)):
  74         return value
  75     elif isinstance(value, time):
  76         msg = f"just a time of day by itself is not enough," \
  77             f" a date must be included: {str(value)}"
  78         raise ValueError(msg)
  79     elif isinstance(value, bool):
  80         msg = f"invalid date: {str(value).lower()}"
  81         raise ValueError(msg)
  82     elif isinstance(value, (str, int, float)):
  83         msg = f"invalid date: {value!r}"
  84         raise ValueError(msg)
  85     else:
  86         msg = f"invalid date"
  87         raise ValueError(msg)
  88
  89
  90 class Payment:
  91     def __init__(self,
  92                  node: "Node",
  93                  payee_key: str,
  94                  amount: Money,
  95                  paid: Optional[_Date],
  96                  submitted: Optional[_Date]):
  97         self.node = node
  98         self.payee_key = payee_key
  99         self.amount = amount
 100         self.paid = paid
 101         self.submitted = submitted
 102
 103     @property
 104     def state(self):
 105         if self.paid is not None:
 106             return PayeeState.Paid
 107         if self.submitted is not None:
 108             return PayeeState.Submitted
 109         return PayeeState.NotYetSubmitted
 110
 111     @staticmethod
 112     def from_toml(node: "Node", payee_key: str, toml_value: Any):
 113         paid = None
 114         submitted = None
 115         known_keys = ("paid", "submitted", "amount")
 116         if isinstance(toml_value, dict):
 117             try:
 118                 amount = toml_value['amount']
 119             except KeyError:
 120                 msg = f"value for key {payee_key!r} is missing the " \
 121                     f"`amount` field which is required"
 122                 raise BudgetGraphPayeesParseError(node.bug.id, msg) \
 123                     .with_traceback(sys.exc_info()[2])
 124             for k, v in toml_value.items():
 125                 if k in ("paid", "submitted"):
 126                     try:
 127                         parsed_value = _parse_date_time_or_none_from_toml(v)
 128                     except ValueError as e:
 129                         msg = f"failed to parse `{k}` field for" \
 130                             f" key {payee_key!r}: {e}"
 131                         raise BudgetGraphPayeesParseError(
 132                             node.bug.id, msg) \
 133                             .with_traceback(sys.exc_info()[2])
 134                     if k == "paid":
 135                         paid = parsed_value
 136                     else:
 137                         assert k == "submitted"
 138                         submitted = parsed_value
 139                 if k not in known_keys:
 140                     msg = f"value for key {payee_key!r} has an unknown" \
 141                         f" field: `{k}`"
 142                     raise BudgetGraphPayeesParseError(node.bug.id, msg) \
 143                         .with_traceback(sys.exc_info()[2])
 144             try:
 145                 paid = _parse_date_time_or_none_from_toml(
 146                     toml_value.get('paid'))
 147             except ValueError as e:
 148                 msg = f"failed to parse `paid` field for" \
 149                     f" key {payee_key!r}: {e}"
 150                 raise BudgetGraphPayeesParseError(
 151                     node.bug.id, msg) \
 152                     .with_traceback(sys.exc_info()[2])
 153             try:
 154                 submitted = _parse_date_time_or_none_from_toml(
 155                     toml_value.get('submitted'))
 156             except ValueError as e:
 157                 msg = f"failed to parse `submitted` field for" \
 158                     f" key {payee_key!r}: {e}"
 159                 raise BudgetGraphPayeesParseError(
 160                     node.bug.id, msg) \
 161                     .with_traceback(sys.exc_info()[2])
 162         elif isinstance(toml_value, (int, str, float)):
 163             # float included for better error messages
 164             amount = toml_value
 165         else:
 166             msg = f"value for key {payee_key!r} is invalid -- it should " \
 167                 f"either be a monetary value or a table"
 168             raise BudgetGraphPayeesParseError(node.bug.id, msg)
 169         try:
 170             amount = _parse_money_from_toml(amount)
 171         except ValueError as e:
 172             msg = f"failed to parse monetary amount for key {payee_key!r}: {e}"
 173             raise BudgetGraphPayeesParseError(
 174                 node.bug.id, msg) \
 175                 .with_traceback(sys.exc_info()[2])
 176         return Payment(node=node, payee_key=payee_key, amount=amount,
 177                        paid=paid, submitted=submitted)
 178
 179     def __repr__(self):
 180         return (f"Payment(node={_NodeSimpleReprWrapper(self.node)}, "
 181                 f"payee_key={self.payee_key!r}, "
 182                 f"amount={self.amount}, "
 183                 f"state={self.state.name}, "
 184                 f"paid={str(self.paid)}, "
 185                 f"submitted={str(self.submitted)})")
 186
 187
 188 class Node:
 189     graph: "BudgetGraph"
 190     bug: Bug
 191     parent_id: Optional[int]
 192     immediate_children: Set["Node"]
 193     budget_excluding_subtasks: Money
 194     budget_including_subtasks: Money
 195     fixed_budget_excluding_subtasks: Money
 196     fixed_budget_including_subtasks: Money
 197     nlnet_milestone: Optional[str]
 198
 199     def __init__(self, graph: "BudgetGraph", bug: Bug):
 200         self.graph = graph
 201         self.bug = bug
 202         self.parent_id = getattr(bug, "cf_budget_parent", None)
 203         self.immediate_children = set()
 204         self.budget_excluding_subtasks = Money.from_str(bug.cf_budget)
 205         self.fixed_budget_excluding_subtasks = self.budget_excluding_subtasks
 206         self.budget_including_subtasks = Money.from_str(bug.cf_total_budget)
 207         self.fixed_budget_including_subtasks = self.budget_including_subtasks
 208         self.nlnet_milestone = bug.cf_nlnet_milestone
 209         if self.nlnet_milestone == "---":
 210             self.nlnet_milestone = None
 211
 212     @cached_property
 213     def payments(self) -> Dict[str, Payment]:
 214         try:
 215             parsed = toml.loads(self.bug.cf_payees_list)
 216         except toml.TomlDecodeError as e:
 217             new_err = BudgetGraphPayeesParseError(
 218                 self.bug.id, f"TOML parse error: {e}")
 219             raise new_err.with_traceback(sys.exc_info()[2])
 220         retval = {}
 221         for key, value in parsed.items():
 222             if not isinstance(key, str):
 223                 raise BudgetGraphPayeesParseError(
 224                     self.bug.id, f"key is not a string: {key!r}")
 225             retval[key] = Payment.from_toml(self, key, value)
 226         return retval
 227
 228     @property
 229     def parent(self) -> Optional["Node"]:
 230         if self.parent_id is not None:
 231             return self.graph.nodes[self.parent_id]
 232         return None
 233
 234     def parents(self) -> Iterable["Node"]:
 235         parent = self.parent
 236         while parent is not None:
 237             yield parent
 238             parent = parent.parent
 239
 240     def _raise_loop_error(self):
 241         bug_ids = []
 242         for parent in self.parents():
 243             bug_ids.append(parent.bug.id)
 244             if parent == self:
 245                 break
 246         raise BudgetGraphLoopError(bug_ids)
 247
 248     @cached_property
 249     def root(self) -> "Node":
 250         # also checks for loop errors
 251         retval = self
 252         for parent in self.parents():
 253             retval = parent
 254             if parent == self:
 255                 self._raise_loop_error()
 256         return retval
 257
 258     def children(self) -> Iterable["Node"]:
 259         def visitor(node: Node) -> Iterable[Node]:
 260             for i in node.immediate_children:
 261                 yield i
 262                 yield from visitor(i)
 263         return visitor(self)
 264
 265     def children_breadth_first(self) -> Iterable["Node"]:
 266         q = deque(self.immediate_children)
 267         while True:
 268             try:
 269                 node = q.popleft()
 270             except IndexError:
 271                 return
 272             q.extend(node.immediate_children)
 273             yield node
 274
 275     def __eq__(self, other):
 276         return self.bug.id == other.bug.id
 277
 278     def __ne__(self, other):
 279         return self.bug.id != other.bug.id
 280
 281     def __hash__(self):
 282         return self.bug.id
 283
 284     def __repr__(self):
 285         try:
 286             root = _NodeSimpleReprWrapper(self.root)
 287         except BudgetGraphLoopError:
 288             root = "<loop error>"
 289         immediate_children = []
 290         for i in self.immediate_children:
 291             immediate_children.append(_NodeSimpleReprWrapper(i))
 292         immediate_children.sort()
 293         parent = f"#{self.parent_id}" if self.parent_id is not None else None
 294         payments = list(self.payments.values())
 295         return (f"Node(graph=..., "
 296                 f"id={_NodeSimpleReprWrapper(self)}, "
 297                 f"root={root}, "
 298                 f"parent={parent}, "
 299                 f"budget_excluding_subtasks={self.budget_excluding_subtasks}, "
 300                 f"budget_including_subtasks={self.budget_including_subtasks}, "
 301                 f"fixed_budget_excluding_subtasks={self.fixed_budget_excluding_subtasks}, "
 302                 f"fixed_budget_including_subtasks={self.fixed_budget_including_subtasks}, "
 303                 f"nlnet_milestone={self.nlnet_milestone!r}, "
 304                 f"immediate_children={immediate_children!r}, "
 305                 f"payments={payments!r}")
 306
 307
 308 class BudgetGraphError(BudgetGraphBaseError):
 309     def __init__(self, bug_id: int, root_bug_id: int):
 310         self.bug_id = bug_id
 311         self.root_bug_id = root_bug_id
 312
 313
 314 class BudgetGraphMoneyWithNoMilestone(BudgetGraphError):
 315     def __str__(self):
 316         return (f"Bug assigned money but without"
 317                 f" any assigned milestone: #{self.bug_id}")
 318
 319
 320 class BudgetGraphMilestoneMismatch(BudgetGraphError):
 321     def __str__(self):
 322         return (f"Bug's assigned milestone doesn't match the milestone "
 323                 f"assigned to the root bug: descendant bug"
 324                 f" #{self.bug_id}, root bug"
 325                 f" #{self.root_bug_id}")
 326
 327
 328 class BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(BudgetGraphError):
 329     def __init__(self, bug_id: int, root_bug_id: int,
 330                  expected_budget_excluding_subtasks: Money):
 331         super().__init__(bug_id, root_bug_id)
 332         self.expected_budget_excluding_subtasks = \
 333             expected_budget_excluding_subtasks
 334
 335     def __str__(self):
 336         return (f"Budget assigned to task excluding subtasks "
 337                 f"(cf_budget field) doesn't match calculated value: "
 338                 f"bug #{self.bug_id}, calculated value"
 339                 f" {self.expected_budget_excluding_subtasks}")
 340
 341
 342 class BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(BudgetGraphError):
 343     def __init__(self, bug_id: int, root_bug_id: int,
 344                  expected_budget_including_subtasks: Money):
 345         super().__init__(bug_id, root_bug_id)
 346         self.expected_budget_including_subtasks = \
 347             expected_budget_including_subtasks
 348
 349     def __str__(self):
 350         return (f"Budget assigned to task including subtasks "
 351                 f"(cf_total_budget field) doesn't match calculated value: "
 352                 f"bug #{self.bug_id}, calculated value"
 353                 f" {self.expected_budget_including_subtasks}")
 354
 355
 356 class BudgetGraphNegativeMoney(BudgetGraphError):
 357     def __str__(self):
 358         return (f"Budget assigned to task is less than zero: "
 359                 f"bug #{self.bug_id}")
 360
 361
 362 class BudgetGraphPayeesMoneyMismatch(BudgetGraphError):
 363     def __init__(self, bug_id: int, root_bug_id: int, payees_total: Money,
 364                  expected_payees_total: Money):
 365         super().__init__(bug_id, root_bug_id)
 366         self.payees_total = payees_total
 367         self.expected_payees_total = expected_payees_total
 368
 369     def __str__(self):
 370         return (f"Total budget assigned to payees (cf_payees_list) doesn't "
 371                 f"match expected value: bug #{self.bug_id}, calculated total "
 372                 f"{self.payees_total}, expected value "
 373                 f"{self.expected_payees_total}")
 374
 375
 376 class BudgetGraphNegativePayeeMoney(BudgetGraphError):
 377     def __init__(self, bug_id: int, root_bug_id: int, payee_key: str):
 378         super().__init__(bug_id, root_bug_id)
 379         self.payee_key = payee_key
 380
 381     def __str__(self):
 382         return (f"Budget assigned to payee for task is less than zero: "
 383                 f"bug #{self.bug_id}, payee {self.payee_key!r}")
 384
 385
 386 class BudgetGraph:
 387     nodes: Dict[int, Node]
 388
 389     def __init__(self, bugs: Iterable[Bug]):
 390         self.nodes = {}
 391         for bug in bugs:
 392             self.nodes[bug.id] = Node(self, bug)
 393         for node in self.nodes.values():
 394             if node.parent is None:
 395                 continue
 396             node.parent.immediate_children.add(node)
 397
 398     @cached_property
 399     def roots(self) -> Set[Node]:
 400         roots = set()
 401         for node in self.nodes.values():
 402             # calling .root also checks for loop errors
 403             root = node.root
 404             roots.add(root)
 405         return roots
 406
 407     def _get_node_errors(self, root: Node, node: Node,
 408                          errors: List[BudgetGraphBaseError]):
 409         if node.nlnet_milestone is None:
 410             if node.budget_including_subtasks != 0 \
 411                     or node.budget_excluding_subtasks != 0:
 412                 errors.append(BudgetGraphMoneyWithNoMilestone(
 413                     node.bug.id, root.bug.id))
 414
 415         if node.nlnet_milestone != root.nlnet_milestone:
 416             errors.append(BudgetGraphMilestoneMismatch(
 417                 node.bug.id, root.bug.id))
 418
 419         if node.budget_excluding_subtasks < 0 \
 420                 or node.budget_including_subtasks < 0:
 421             errors.append(BudgetGraphNegativeMoney(
 422                 node.bug.id, root.bug.id))
 423
 424         subtasks_total = Money(0)
 425         for child in node.immediate_children:
 426             subtasks_total += child.fixed_budget_including_subtasks
 427
 428         payees_total = Money(0)
 429         for payment in node.payments.values():
 430             if payment.amount < 0:
 431                 errors.append(BudgetGraphNegativePayeeMoney(
 432                     node.bug.id, root.bug.id, payment.payee_key))
 433             payees_total += payment.amount
 434
 435         def set_including_from_excluding_and_error():
 436             node.fixed_budget_including_subtasks = \
 437                 node.budget_excluding_subtasks + subtasks_total
 438             errors.append(
 439                 BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(
 440                     node.bug.id, root.bug.id,
 441                     node.fixed_budget_including_subtasks))
 442
 443         def set_including_from_payees_and_error():
 444             node.fixed_budget_including_subtasks = \
 445                 payees_total + subtasks_total
 446             errors.append(
 447                 BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(
 448                     node.bug.id, root.bug.id,
 449                     node.fixed_budget_including_subtasks))
 450
 451         def set_excluding_from_including_and_error():
 452             node.fixed_budget_excluding_subtasks = \
 453                 node.budget_including_subtasks - subtasks_total
 454             errors.append(
 455                 BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(
 456                     node.bug.id, root.bug.id,
 457                     node.fixed_budget_excluding_subtasks))
 458
 459         def set_excluding_from_payees_and_error():
 460             node.fixed_budget_excluding_subtasks = \
 461                 payees_total
 462             errors.append(
 463                 BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(
 464                     node.bug.id, root.bug.id,
 465                     node.fixed_budget_excluding_subtasks))
 466
 467         def set_payees_from_including_and_error():
 468             fixed_payees_total = \
 469                 node.budget_including_subtasks - subtasks_total
 470             errors.append(BudgetGraphPayeesMoneyMismatch(
 471                 node.bug.id, root.bug.id, payees_total, fixed_payees_total))
 472
 473         def set_payees_from_excluding_and_error():
 474             fixed_payees_total = \
 475                 node.budget_excluding_subtasks
 476             errors.append(BudgetGraphPayeesMoneyMismatch(
 477                 node.bug.id, root.bug.id, payees_total, fixed_payees_total))
 478
 479         payees_matches_including = \
 480             node.budget_including_subtasks - subtasks_total == payees_total
 481         payees_matches_excluding = \
 482             node.budget_excluding_subtasks == payees_total
 483         including_matches_excluding = \
 484             node.budget_including_subtasks - subtasks_total \
 485             == node.budget_excluding_subtasks
 486
 487         if payees_matches_including \
 488                 and payees_matches_excluding \
 489                 and including_matches_excluding:
 490             pass  # no error
 491         elif payees_matches_including:
 492             # can't have 2 match without all 3 matching
 493             assert not payees_matches_excluding
 494             assert not including_matches_excluding
 495             if node.budget_including_subtasks == 0 and len(node.payments) == 0:
 496                 set_including_from_excluding_and_error()
 497             else:
 498                 set_excluding_from_including_and_error()
 499         elif payees_matches_excluding:
 500             # can't have 2 match without all 3 matching
 501             assert not payees_matches_including
 502             assert not including_matches_excluding
 503             if node.budget_excluding_subtasks == 0 and len(node.payments) == 0:
 504                 if node.budget_including_subtasks == 0:
 505                     set_including_from_excluding_and_error()
 506                 else:
 507                     set_excluding_from_including_and_error()
 508             else:
 509                 set_including_from_excluding_and_error()
 510         elif including_matches_excluding:
 511             # can't have 2 match without all 3 matching
 512             assert not payees_matches_including
 513             assert not payees_matches_excluding
 514             if len(node.payments) == 0:
 515                 pass  # no error -- payees is just not set
 516             elif node.budget_excluding_subtasks == 0 \
 517                     and node.budget_including_subtasks == 0:
 518                 set_excluding_from_payees_and_error()
 519                 set_including_from_payees_and_error()
 520             else:
 521                 set_payees_from_excluding_and_error()
 522         else:
 523             # nothing matches
 524             if len(node.payments) == 0:
 525                 # payees unset -- don't need to set payees
 526                 if node.budget_including_subtasks == 0:
 527                     set_including_from_excluding_and_error()
 528                 else:
 529                     set_excluding_from_including_and_error()
 530             elif node.budget_excluding_subtasks == 0 \
 531                     and node.budget_including_subtasks == 0:
 532                 set_excluding_from_payees_and_error()
 533                 set_including_from_payees_and_error()
 534             elif node.budget_excluding_subtasks == 0:
 535                 set_excluding_from_including_and_error()
 536                 set_payees_from_including_and_error()
 537             elif node.budget_including_subtasks == 0:
 538                 set_including_from_excluding_and_error()
 539                 set_payees_from_excluding_and_error()
 540             else:
 541                 set_including_from_excluding_and_error()
 542                 set_payees_from_excluding_and_error()
 543
 544     def get_errors(self) -> List[BudgetGraphBaseError]:
 545         errors = []
 546         try:
 547             roots = self.roots
 548         except BudgetGraphBaseError as e:
 549             errors.append(e)
 550             return errors
 551
 552         for root in roots:
 553             try:
 554                 for child in reversed(list(root.children_breadth_first())):
 555                     try:
 556                         self._get_node_errors(root, child, errors)
 557                     except BudgetGraphBaseError as e:
 558                         errors.append(e)
 559                 self._get_node_errors(root, root, errors)
 560             except BudgetGraphBaseError as e:
 561                 errors.append(e)
 562         return errors
 563
 564     @cached_property
 565     def payee_keys(self) -> Set[str]:
 566         retval = set()
 567         for node in self.nodes.values():
 568             for payee_key in node.payments.keys():
 569                 retval.add(payee_key)
 570         return retval