src/budget_sync/budget_graph.py

   1 from bugzilla.bug import Bug
   2 from bugzilla import Bugzilla
   3 from typing import Set, Dict, Iterable, Optional, List, Union, Any
   4 from budget_sync.util import all_bugs
   5 from budget_sync.money import Money
   6 from functools import cached_property
   7 import toml
   8 import sys
   9 import enum
  10 from collections import deque
  11 from datetime import date, time, datetime
  12
  13
  14 class BudgetGraphBaseError(Exception):
  15     pass
  16
  17
  18 class BudgetGraphParseError(BudgetGraphBaseError):
  19     def __init__(self, bug_id: int):
  20         self.bug_id = bug_id
  21
  22
  23 class BudgetGraphPayeesParseError(BudgetGraphParseError):
  24     def __init__(self, bug_id: int, msg: str):
  25         super().__init__(bug_id)
  26         self.msg = msg
  27
  28     def __str__(self):
  29         return f"Failed to parse cf_payees_list field of bug #{self.bug_id}: {self.msg}"
  30
  31
  32 class BudgetGraphLoopError(BudgetGraphBaseError):
  33     def __init__(self, bug_ids: List[int]):
  34         self.bug_ids = bug_ids
  35
  36     def __str__(self):
  37         retval = f"Detected Loop in Budget Graph: #{self.bug_ids[-1]} -> "
  38         retval += " -> ".join((f"#{i}" for i in self.bug_ids))
  39         return retval
  40
  41
  42 class _NodeSimpleReprWrapper:
  43     def __init__(self, node: "Node"):
  44         self.node = node
  45
  46     def __repr__(self):
  47         return f"#{self.node.bug.id}"
  48
  49     def __lt__(self, other):
  50         # for list.sort()
  51         return self.node.bug.id < other.node.bug.id
  52
  53
  54 class PayeeState(enum.Enum):
  55     NotYetSubmitted = "not yet submitted"
  56     Submitted = "submitted"
  57     Paid = "paid"
  58
  59
  60 _Date = Union[date, datetime]
  61
  62
  63 def _parse_money_from_toml(value: Any) -> Money:
  64     if not isinstance(value, (int, str)):
  65         msg = f"monetary amount is not a string or integer " \
  66             f"(to use fractional amounts such as 123.45, write " \
  67             f"\"123.45\"): {value!r}"
  68         raise ValueError(msg)
  69     return Money(value)
  70
  71
  72 def _parse_date_time_or_none_from_toml(value: Any) -> Optional[_Date]:
  73     if value is None or isinstance(value, (date, datetime)):
  74         return value
  75     elif isinstance(value, time):
  76         msg = f"just a time of day by itself is not enough," \
  77             f" a date must be included: {str(value)}"
  78         raise ValueError(msg)
  79     elif isinstance(value, bool):
  80         msg = f"invalid date: {str(value).lower()}"
  81         raise ValueError(msg)
  82     elif isinstance(value, (str, int, float)):
  83         msg = f"invalid date: {value!r}"
  84         raise ValueError(msg)
  85     else:
  86         msg = f"invalid date"
  87         raise ValueError(msg)
  88
  89
  90 class Payment:
  91     def __init__(self,
  92                  node: "Node",
  93                  payee_key: str,
  94                  amount: Money,
  95                  paid: Optional[_Date],
  96                  submitted: Optional[_Date]):
  97         self.node = node
  98         self.payee_key = payee_key
  99         self.amount = amount
 100         self.paid = paid
 101         self.submitted = submitted
 102
 103     @property
 104     def state(self):
 105         if self.paid is not None:
 106             return PayeeState.Paid
 107         if self.submitted is not None:
 108             return PayeeState.Submitted
 109         return PayeeState.NotYetSubmitted
 110
 111     @staticmethod
 112     def _from_toml(node: "Node", payee_key: str, toml_value: Any) -> "Payment":
 113         paid = None
 114         submitted = None
 115         known_keys = ("paid", "submitted", "amount")
 116         if isinstance(toml_value, dict):
 117             try:
 118                 amount = toml_value['amount']
 119             except KeyError:
 120                 msg = f"value for key {payee_key!r} is missing the " \
 121                     f"`amount` field which is required"
 122                 raise BudgetGraphPayeesParseError(node.bug.id, msg) \
 123                     .with_traceback(sys.exc_info()[2])
 124             for k, v in toml_value.items():
 125                 if k in ("paid", "submitted"):
 126                     try:
 127                         parsed_value = _parse_date_time_or_none_from_toml(v)
 128                     except ValueError as e:
 129                         msg = f"failed to parse `{k}` field for" \
 130                             f" key {payee_key!r}: {e}"
 131                         raise BudgetGraphPayeesParseError(
 132                             node.bug.id, msg) \
 133                             .with_traceback(sys.exc_info()[2])
 134                     if k == "paid":
 135                         paid = parsed_value
 136                     else:
 137                         assert k == "submitted"
 138                         submitted = parsed_value
 139                 if k not in known_keys:
 140                     msg = f"value for key {payee_key!r} has an unknown" \
 141                         f" field: `{k}`"
 142                     raise BudgetGraphPayeesParseError(node.bug.id, msg) \
 143                         .with_traceback(sys.exc_info()[2])
 144             try:
 145                 paid = _parse_date_time_or_none_from_toml(
 146                     toml_value.get('paid'))
 147             except ValueError as e:
 148                 msg = f"failed to parse `paid` field for" \
 149                     f" key {payee_key!r}: {e}"
 150                 raise BudgetGraphPayeesParseError(
 151                     node.bug.id, msg) \
 152                     .with_traceback(sys.exc_info()[2])
 153             try:
 154                 submitted = _parse_date_time_or_none_from_toml(
 155                     toml_value.get('submitted'))
 156             except ValueError as e:
 157                 msg = f"failed to parse `submitted` field for" \
 158                     f" key {payee_key!r}: {e}"
 159                 raise BudgetGraphPayeesParseError(
 160                     node.bug.id, msg) \
 161                     .with_traceback(sys.exc_info()[2])
 162         elif isinstance(toml_value, (int, str, float)):
 163             # float included for better error messages
 164             amount = toml_value
 165         else:
 166             msg = f"value for key {payee_key!r} is invalid -- it should " \
 167                 f"either be a monetary value or a table"
 168             raise BudgetGraphPayeesParseError(node.bug.id, msg)
 169         try:
 170             amount = _parse_money_from_toml(amount)
 171         except ValueError as e:
 172             msg = f"failed to parse monetary amount for key {payee_key!r}: {e}"
 173             raise BudgetGraphPayeesParseError(
 174                 node.bug.id, msg) \
 175                 .with_traceback(sys.exc_info()[2])
 176         return Payment(node=node, payee_key=payee_key, amount=amount,
 177                        paid=paid, submitted=submitted)
 178
 179     def __repr__(self):
 180         return (f"Payment(node={_NodeSimpleReprWrapper(self.node)}, "
 181                 f"payee_key={self.payee_key!r}, "
 182                 f"amount={self.amount}, "
 183                 f"state={self.state.name}, "
 184                 f"paid={str(self.paid)}, "
 185                 f"submitted={str(self.submitted)})")
 186
 187
 188 class Node:
 189     graph: "BudgetGraph"
 190     bug: Bug
 191     parent_id: Optional[int]
 192     immediate_children: Set["Node"]
 193     budget_excluding_subtasks: Money
 194     budget_including_subtasks: Money
 195     fixed_budget_excluding_subtasks: Money
 196     fixed_budget_including_subtasks: Money
 197     nlnet_milestone: Optional[str]
 198
 199     def __init__(self, graph: "BudgetGraph", bug: Bug):
 200         self.graph = graph
 201         self.bug = bug
 202         self.parent_id = getattr(bug, "cf_budget_parent", None)
 203         self.immediate_children = set()
 204         self.budget_excluding_subtasks = Money.from_str(bug.cf_budget)
 205         self.fixed_budget_excluding_subtasks = self.budget_excluding_subtasks
 206         self.budget_including_subtasks = Money.from_str(bug.cf_total_budget)
 207         self.fixed_budget_including_subtasks = self.budget_including_subtasks
 208         self.nlnet_milestone = bug.cf_nlnet_milestone
 209         if self.nlnet_milestone == "---":
 210             self.nlnet_milestone = None
 211
 212     @cached_property
 213     def payments(self) -> Dict[str, Payment]:
 214         try:
 215             parsed = toml.loads(self.bug.cf_payees_list)
 216         except toml.TomlDecodeError as e:
 217             new_err = BudgetGraphPayeesParseError(
 218                 self.bug.id, f"TOML parse error: {e}")
 219             raise new_err.with_traceback(sys.exc_info()[2])
 220         retval = {}
 221         for key, value in parsed.items():
 222             if not isinstance(key, str):
 223                 raise BudgetGraphPayeesParseError(
 224                     self.bug.id, f"key is not a string: {key!r}")
 225             retval[key] = Payment._from_toml(self, key, value)
 226         return retval
 227
 228     @property
 229     def parent(self) -> Optional["Node"]:
 230         if self.parent_id is not None:
 231             return self.graph.nodes[self.parent_id]
 232         return None
 233
 234     def parents(self) -> Iterable["Node"]:
 235         parent = self.parent
 236         while parent is not None:
 237             yield parent
 238             parent = parent.parent
 239
 240     def _raise_loop_error(self):
 241         bug_ids = []
 242         for parent in self.parents():
 243             bug_ids.append(parent.bug.id)
 244             if parent == self:
 245                 break
 246         raise BudgetGraphLoopError(bug_ids)
 247
 248     @cached_property
 249     def root(self) -> "Node":
 250         # also checks for loop errors
 251         retval = self
 252         for parent in self.parents():
 253             retval = parent
 254             if parent == self:
 255                 self._raise_loop_error()
 256         return retval
 257
 258     def children(self) -> Iterable["Node"]:
 259         def visitor(node: Node) -> Iterable[Node]:
 260             for i in node.immediate_children:
 261                 yield i
 262                 yield from visitor(i)
 263         return visitor(self)
 264
 265     def children_breadth_first(self) -> Iterable["Node"]:
 266         q = deque(self.immediate_children)
 267         while True:
 268             try:
 269                 node = q.popleft()
 270             except IndexError:
 271                 return
 272             q.extend(node.immediate_children)
 273             yield node
 274
 275     def __eq__(self, other):
 276         return self.bug.id == other.bug.id
 277
 278     def __hash__(self):
 279         return self.bug.id
 280
 281     def __repr__(self):
 282         try:
 283             root = _NodeSimpleReprWrapper(self.root)
 284         except BudgetGraphLoopError:
 285             root = "<loop error>"
 286         immediate_children = []
 287         for i in self.immediate_children:
 288             immediate_children.append(_NodeSimpleReprWrapper(i))
 289         immediate_children.sort()
 290         parent = f"#{self.parent_id}" if self.parent_id is not None else None
 291         payments = list(self.payments.values())
 292         return (f"Node(graph=..., "
 293                 f"id={_NodeSimpleReprWrapper(self)}, "
 294                 f"root={root}, "
 295                 f"parent={parent}, "
 296                 f"budget_excluding_subtasks={self.budget_excluding_subtasks}, "
 297                 f"budget_including_subtasks={self.budget_including_subtasks}, "
 298                 f"fixed_budget_excluding_subtasks={self.fixed_budget_excluding_subtasks}, "
 299                 f"fixed_budget_including_subtasks={self.fixed_budget_including_subtasks}, "
 300                 f"nlnet_milestone={self.nlnet_milestone!r}, "
 301                 f"immediate_children={immediate_children!r}, "
 302                 f"payments={payments!r}")
 303
 304
 305 class BudgetGraphError(BudgetGraphBaseError):
 306     def __init__(self, bug_id: int, root_bug_id: int):
 307         self.bug_id = bug_id
 308         self.root_bug_id = root_bug_id
 309
 310
 311 class BudgetGraphMoneyWithNoMilestone(BudgetGraphError):
 312     def __str__(self):
 313         return (f"Bug assigned money but without"
 314                 f" any assigned milestone: #{self.bug_id}")
 315
 316
 317 class BudgetGraphMilestoneMismatch(BudgetGraphError):
 318     def __str__(self):
 319         return (f"Bug's assigned milestone doesn't match the milestone "
 320                 f"assigned to the root bug: descendant bug"
 321                 f" #{self.bug_id}, root bug"
 322                 f" #{self.root_bug_id}")
 323
 324
 325 class BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(BudgetGraphError):
 326     def __init__(self, bug_id: int, root_bug_id: int,
 327                  expected_budget_excluding_subtasks: Money):
 328         super().__init__(bug_id, root_bug_id)
 329         self.expected_budget_excluding_subtasks = \
 330             expected_budget_excluding_subtasks
 331
 332     def __str__(self):
 333         return (f"Budget assigned to task excluding subtasks "
 334                 f"(cf_budget field) doesn't match calculated value: "
 335                 f"bug #{self.bug_id}, calculated value"
 336                 f" {self.expected_budget_excluding_subtasks}")
 337
 338
 339 class BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(BudgetGraphError):
 340     def __init__(self, bug_id: int, root_bug_id: int,
 341                  expected_budget_including_subtasks: Money):
 342         super().__init__(bug_id, root_bug_id)
 343         self.expected_budget_including_subtasks = \
 344             expected_budget_including_subtasks
 345
 346     def __str__(self):
 347         return (f"Budget assigned to task including subtasks "
 348                 f"(cf_total_budget field) doesn't match calculated value: "
 349                 f"bug #{self.bug_id}, calculated value"
 350                 f" {self.expected_budget_including_subtasks}")
 351
 352
 353 class BudgetGraphNegativeMoney(BudgetGraphError):
 354     def __str__(self):
 355         return (f"Budget assigned to task is less than zero: "
 356                 f"bug #{self.bug_id}")
 357
 358
 359 class BudgetGraphPayeesMoneyMismatch(BudgetGraphError):
 360     def __init__(self, bug_id: int, root_bug_id: int, payees_total: Money,
 361                  expected_payees_total: Money):
 362         super().__init__(bug_id, root_bug_id)
 363         self.payees_total = payees_total
 364         self.expected_payees_total = expected_payees_total
 365
 366     def __str__(self):
 367         return (f"Total budget assigned to payees (cf_payees_list) doesn't "
 368                 f"match expected value: bug #{self.bug_id}, calculated total "
 369                 f"{self.payees_total}, expected value "
 370                 f"{self.expected_payees_total}")
 371
 372
 373 class BudgetGraphNegativePayeeMoney(BudgetGraphError):
 374     def __init__(self, bug_id: int, root_bug_id: int, payee_key: str):
 375         super().__init__(bug_id, root_bug_id)
 376         self.payee_key = payee_key
 377
 378     def __str__(self):
 379         return (f"Budget assigned to payee for task is less than zero: "
 380                 f"bug #{self.bug_id}, payee {self.payee_key!r}")
 381
 382
 383 class BudgetGraph:
 384     nodes: Dict[int, Node]
 385
 386     def __init__(self, bugs: Iterable[Bug]):
 387         self.nodes = {}
 388         for bug in bugs:
 389             self.nodes[bug.id] = Node(self, bug)
 390         for node in self.nodes.values():
 391             if node.parent is None:
 392                 continue
 393             node.parent.immediate_children.add(node)
 394
 395     @cached_property
 396     def roots(self) -> Set[Node]:
 397         roots = set()
 398         for node in self.nodes.values():
 399             # calling .root also checks for loop errors
 400             root = node.root
 401             roots.add(root)
 402         return roots
 403
 404     def _get_node_errors(self, root: Node, node: Node,
 405                          errors: List[BudgetGraphBaseError]):
 406         if node.nlnet_milestone is None:
 407             if node.budget_including_subtasks != 0 \
 408                     or node.budget_excluding_subtasks != 0:
 409                 errors.append(BudgetGraphMoneyWithNoMilestone(
 410                     node.bug.id, root.bug.id))
 411
 412         if node.nlnet_milestone != root.nlnet_milestone:
 413             errors.append(BudgetGraphMilestoneMismatch(
 414                 node.bug.id, root.bug.id))
 415
 416         if node.budget_excluding_subtasks < 0 \
 417                 or node.budget_including_subtasks < 0:
 418             errors.append(BudgetGraphNegativeMoney(
 419                 node.bug.id, root.bug.id))
 420
 421         subtasks_total = Money(0)
 422         for child in node.immediate_children:
 423             subtasks_total += child.fixed_budget_including_subtasks
 424
 425         payees_total = Money(0)
 426         for payment in node.payments.values():
 427             if payment.amount < 0:
 428                 errors.append(BudgetGraphNegativePayeeMoney(
 429                     node.bug.id, root.bug.id, payment.payee_key))
 430             payees_total += payment.amount
 431
 432         def set_including_from_excluding_and_error():
 433             node.fixed_budget_including_subtasks = \
 434                 node.budget_excluding_subtasks + subtasks_total
 435             errors.append(
 436                 BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(
 437                     node.bug.id, root.bug.id,
 438                     node.fixed_budget_including_subtasks))
 439
 440         def set_including_from_payees_and_error():
 441             node.fixed_budget_including_subtasks = \
 442                 payees_total + subtasks_total
 443             errors.append(
 444                 BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(
 445                     node.bug.id, root.bug.id,
 446                     node.fixed_budget_including_subtasks))
 447
 448         def set_excluding_from_including_and_error():
 449             node.fixed_budget_excluding_subtasks = \
 450                 node.budget_including_subtasks - subtasks_total
 451             errors.append(
 452                 BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(
 453                     node.bug.id, root.bug.id,
 454                     node.fixed_budget_excluding_subtasks))
 455
 456         def set_excluding_from_payees_and_error():
 457             node.fixed_budget_excluding_subtasks = \
 458                 payees_total
 459             errors.append(
 460                 BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(
 461                     node.bug.id, root.bug.id,
 462                     node.fixed_budget_excluding_subtasks))
 463
 464         def set_payees_from_including_and_error():
 465             fixed_payees_total = \
 466                 node.budget_including_subtasks - subtasks_total
 467             errors.append(BudgetGraphPayeesMoneyMismatch(
 468                 node.bug.id, root.bug.id, payees_total, fixed_payees_total))
 469
 470         def set_payees_from_excluding_and_error():
 471             fixed_payees_total = \
 472                 node.budget_excluding_subtasks
 473             errors.append(BudgetGraphPayeesMoneyMismatch(
 474                 node.bug.id, root.bug.id, payees_total, fixed_payees_total))
 475
 476         payees_matches_including = \
 477             node.budget_including_subtasks - subtasks_total == payees_total
 478         payees_matches_excluding = \
 479             node.budget_excluding_subtasks == payees_total
 480         including_matches_excluding = \
 481             node.budget_including_subtasks - subtasks_total \
 482             == node.budget_excluding_subtasks
 483
 484         if payees_matches_including \
 485                 and payees_matches_excluding \
 486                 and including_matches_excluding:
 487             pass  # no error
 488         elif payees_matches_including:
 489             # can't have 2 match without all 3 matching
 490             assert not payees_matches_excluding
 491             assert not including_matches_excluding
 492             if node.budget_including_subtasks == 0 and len(node.payments) == 0:
 493                 set_including_from_excluding_and_error()
 494             else:
 495                 set_excluding_from_including_and_error()
 496         elif payees_matches_excluding:
 497             # can't have 2 match without all 3 matching
 498             assert not payees_matches_including
 499             assert not including_matches_excluding
 500             if node.budget_excluding_subtasks == 0 and len(node.payments) == 0:
 501                 if node.budget_including_subtasks == 0:
 502                     set_including_from_excluding_and_error()
 503                 else:
 504                     set_excluding_from_including_and_error()
 505             else:
 506                 set_including_from_excluding_and_error()
 507         elif including_matches_excluding:
 508             # can't have 2 match without all 3 matching
 509             assert not payees_matches_including
 510             assert not payees_matches_excluding
 511             if len(node.payments) == 0:
 512                 pass  # no error -- payees is just not set
 513             elif node.budget_excluding_subtasks == 0 \
 514                     and node.budget_including_subtasks == 0:
 515                 set_excluding_from_payees_and_error()
 516                 set_including_from_payees_and_error()
 517             else:
 518                 set_payees_from_excluding_and_error()
 519         else:
 520             # nothing matches
 521             if len(node.payments) == 0:
 522                 # payees unset -- don't need to set payees
 523                 if node.budget_including_subtasks == 0:
 524                     set_including_from_excluding_and_error()
 525                 else:
 526                     set_excluding_from_including_and_error()
 527             elif node.budget_excluding_subtasks == 0 \
 528                     and node.budget_including_subtasks == 0:
 529                 set_excluding_from_payees_and_error()
 530                 set_including_from_payees_and_error()
 531             elif node.budget_excluding_subtasks == 0:
 532                 set_excluding_from_including_and_error()
 533                 set_payees_from_including_and_error()
 534             elif node.budget_including_subtasks == 0:
 535                 set_including_from_excluding_and_error()
 536                 set_payees_from_excluding_and_error()
 537             else:
 538                 set_including_from_excluding_and_error()
 539                 set_payees_from_excluding_and_error()
 540
 541     def get_errors(self) -> List[BudgetGraphBaseError]:
 542         errors = []
 543         try:
 544             roots = self.roots
 545         except BudgetGraphBaseError as e:
 546             errors.append(e)
 547             return errors
 548
 549         for root in roots:
 550             try:
 551                 for child in reversed(list(root.children_breadth_first())):
 552                     try:
 553                         self._get_node_errors(root, child, errors)
 554                     except BudgetGraphBaseError as e:
 555                         errors.append(e)
 556                 self._get_node_errors(root, root, errors)
 557             except BudgetGraphBaseError as e:
 558                 errors.append(e)
 559         return errors
 560
 561     @cached_property
 562     def payee_keys(self) -> Set[str]:
 563         retval = set()
 564         for node in self.nodes.values():
 565             for payee_key in node.payments.keys():
 566                 retval.add(payee_key)
 567         return retval