working on implementing write_budget_markdown
[utils.git] / src / budget_sync / budget_graph.py
1 from bugzilla.bug import Bug
2 from bugzilla import Bugzilla
3 from typing import Set, Dict, Iterable, Optional, List, Union, Any
4 from budget_sync.util import all_bugs
5 from budget_sync.money import Money
6 from budget_sync.config import Config, Person, Milestone
7 from functools import cached_property
8 import toml
9 import sys
10 import enum
11 from collections import deque
12 from datetime import date, time, datetime
13
14
15 class BudgetGraphBaseError(Exception):
16 pass
17
18
19 class BudgetGraphParseError(BudgetGraphBaseError):
20 def __init__(self, bug_id: int):
21 self.bug_id = bug_id
22
23
24 class BudgetGraphPayeesParseError(BudgetGraphParseError):
25 def __init__(self, bug_id: int, msg: str):
26 super().__init__(bug_id)
27 self.msg = msg
28
29 def __str__(self):
30 return f"Failed to parse cf_payees_list field of " \
31 f"bug #{self.bug_id}: {self.msg}"
32
33
34 class BudgetGraphLoopError(BudgetGraphBaseError):
35 def __init__(self, bug_ids: List[int]):
36 self.bug_ids = bug_ids
37
38 def __str__(self):
39 retval = f"Detected Loop in Budget Graph: #{self.bug_ids[-1]} -> "
40 retval += " -> ".join((f"#{i}" for i in self.bug_ids))
41 return retval
42
43
44 class _NodeSimpleReprWrapper:
45 def __init__(self, node: "Node"):
46 self.node = node
47
48 def __repr__(self):
49 return f"#{self.node.bug.id}"
50
51 def __lt__(self, other):
52 # for list.sort()
53 return self.node.bug.id < other.node.bug.id
54
55
56 class PayeeState(enum.Enum):
57 NotYetSubmitted = "not yet submitted"
58 Submitted = "submitted"
59 Paid = "paid"
60
61
62 _Date = Union[date, datetime]
63
64
65 def _parse_money_from_toml(value: Any) -> Money:
66 if not isinstance(value, (int, str)):
67 msg = f"monetary amount is not a string or integer " \
68 f"(to use fractional amounts such as 123.45, write " \
69 f"\"123.45\"): {value!r}"
70 raise ValueError(msg)
71 return Money(value)
72
73
74 def _parse_date_time_or_none_from_toml(value: Any) -> Optional[_Date]:
75 if value is None or isinstance(value, (date, datetime)):
76 return value
77 elif isinstance(value, time):
78 msg = f"just a time of day by itself is not enough," \
79 f" a date must be included: {str(value)}"
80 raise ValueError(msg)
81 elif isinstance(value, bool):
82 msg = f"invalid date: {str(value).lower()}"
83 raise ValueError(msg)
84 elif isinstance(value, (str, int, float)):
85 msg = f"invalid date: {value!r}"
86 raise ValueError(msg)
87 else:
88 msg = f"invalid date"
89 raise ValueError(msg)
90
91
92 class Payment:
93 def __init__(self,
94 node: "Node",
95 payee_key: str,
96 amount: Money,
97 paid: Optional[_Date],
98 submitted: Optional[_Date]):
99 self.node = node
100 self.payee_key = payee_key
101 self.amount = amount
102 self.paid = paid
103 self.submitted = submitted
104
105 @cached_property
106 def payee(self) -> Person:
107 try:
108 return self.node.graph.config.all_names[self.payee_key]
109 except KeyError:
110 msg = f"unknown payee name: {self.payee_key!r} is not the name " \
111 f"or an alias of any known person"
112 raise BudgetGraphPayeesParseError(self.node.bug.id, msg) \
113 .with_traceback(sys.exc_info()[2])
114
115 @property
116 def state(self):
117 if self.paid is not None:
118 return PayeeState.Paid
119 if self.submitted is not None:
120 return PayeeState.Submitted
121 return PayeeState.NotYetSubmitted
122
123 @staticmethod
124 def _from_toml(node: "Node", payee_key: str, toml_value: Any) -> "Payment":
125 paid = None
126 submitted = None
127 known_keys = ("paid", "submitted", "amount")
128 if isinstance(toml_value, dict):
129 try:
130 amount = toml_value['amount']
131 except KeyError:
132 msg = f"value for key {payee_key!r} is missing the " \
133 f"`amount` field which is required"
134 raise BudgetGraphPayeesParseError(node.bug.id, msg) \
135 .with_traceback(sys.exc_info()[2])
136 for k, v in toml_value.items():
137 if k in ("paid", "submitted"):
138 try:
139 parsed_value = _parse_date_time_or_none_from_toml(v)
140 except ValueError as e:
141 msg = f"failed to parse `{k}` field for" \
142 f" key {payee_key!r}: {e}"
143 raise BudgetGraphPayeesParseError(
144 node.bug.id, msg) \
145 .with_traceback(sys.exc_info()[2])
146 if k == "paid":
147 paid = parsed_value
148 else:
149 assert k == "submitted"
150 submitted = parsed_value
151 if k not in known_keys:
152 msg = f"value for key {payee_key!r} has an unknown" \
153 f" field: `{k}`"
154 raise BudgetGraphPayeesParseError(node.bug.id, msg) \
155 .with_traceback(sys.exc_info()[2])
156 try:
157 paid = _parse_date_time_or_none_from_toml(
158 toml_value.get('paid'))
159 except ValueError as e:
160 msg = f"failed to parse `paid` field for" \
161 f" key {payee_key!r}: {e}"
162 raise BudgetGraphPayeesParseError(
163 node.bug.id, msg) \
164 .with_traceback(sys.exc_info()[2])
165 try:
166 submitted = _parse_date_time_or_none_from_toml(
167 toml_value.get('submitted'))
168 except ValueError as e:
169 msg = f"failed to parse `submitted` field for" \
170 f" key {payee_key!r}: {e}"
171 raise BudgetGraphPayeesParseError(
172 node.bug.id, msg) \
173 .with_traceback(sys.exc_info()[2])
174 elif isinstance(toml_value, (int, str, float)):
175 # float included for better error messages
176 amount = toml_value
177 else:
178 msg = f"value for key {payee_key!r} is invalid -- it should " \
179 f"either be a monetary value or a table"
180 raise BudgetGraphPayeesParseError(node.bug.id, msg)
181 try:
182 amount = _parse_money_from_toml(amount)
183 except ValueError as e:
184 msg = f"failed to parse monetary amount for key {payee_key!r}: {e}"
185 raise BudgetGraphPayeesParseError(
186 node.bug.id, msg) \
187 .with_traceback(sys.exc_info()[2])
188 return Payment(node=node, payee_key=payee_key, amount=amount,
189 paid=paid, submitted=submitted)
190
191 def __repr__(self):
192 try:
193 payee = f"Person<{self.payee.identifier!r}>"
194 except BudgetGraphBaseError:
195 payee = "<unknown person>"
196 return (f"Payment(node={_NodeSimpleReprWrapper(self.node)}, "
197 f"payee={payee}, "
198 f"payee_key={self.payee_key!r}, "
199 f"amount={self.amount}, "
200 f"state={self.state.name}, "
201 f"paid={str(self.paid)}, "
202 f"submitted={str(self.submitted)})")
203
204
205 class BudgetGraphUnknownMilestone(BudgetGraphParseError):
206 def __init__(self, bug_id: int, milestone_str: str):
207 super().__init__(bug_id)
208 self.milestone_str = milestone_str
209
210 def __str__(self):
211 return f"failed to parse cf_nlnet_milestone field of bug " \
212 f"#{self.bug_id}: unknown milestone: {self.milestone_str!r}"
213
214
215 class Node:
216 graph: "BudgetGraph"
217 bug: Bug
218 parent_id: Optional[int]
219 immediate_children: Set["Node"]
220 budget_excluding_subtasks: Money
221 budget_including_subtasks: Money
222 fixed_budget_excluding_subtasks: Money
223 fixed_budget_including_subtasks: Money
224 milestone_str: Optional[str]
225
226 def __init__(self, graph: "BudgetGraph", bug: Bug):
227 self.graph = graph
228 self.bug = bug
229 self.parent_id = getattr(bug, "cf_budget_parent", None)
230 self.immediate_children = set()
231 self.budget_excluding_subtasks = Money.from_str(bug.cf_budget)
232 self.fixed_budget_excluding_subtasks = self.budget_excluding_subtasks
233 self.budget_including_subtasks = Money.from_str(bug.cf_total_budget)
234 self.fixed_budget_including_subtasks = self.budget_including_subtasks
235 self.milestone_str = bug.cf_nlnet_milestone
236 if self.milestone_str == "---":
237 self.milestone_str = None
238
239 @cached_property
240 def bug_url(self) -> str:
241 return f"{self.graph.config.bugzilla_url_stripped}/show_bug.cgi?" \
242 f"id={self.bug.id}"
243
244 @cached_property
245 def milestone(self) -> Optional[Milestone]:
246 try:
247 if self.milestone_str is not None:
248 return self.graph.config.milestones[self.milestone_str]
249 return None
250 except KeyError:
251 new_err = BudgetGraphUnknownMilestone(
252 self.bug.id, self.milestone_str)
253 raise new_err.with_traceback(sys.exc_info()[2])
254
255 @cached_property
256 def payments(self) -> Dict[str, Payment]:
257 try:
258 parsed = toml.loads(self.bug.cf_payees_list)
259 except toml.TomlDecodeError as e:
260 new_err = BudgetGraphPayeesParseError(
261 self.bug.id, f"TOML parse error: {e}")
262 raise new_err.with_traceback(sys.exc_info()[2])
263 retval = {}
264 for key, value in parsed.items():
265 if not isinstance(key, str):
266 raise BudgetGraphPayeesParseError(
267 self.bug.id, f"key is not a string: {key!r}")
268 retval[key] = Payment._from_toml(self, key, value)
269 return retval
270
271 @property
272 def parent(self) -> Optional["Node"]:
273 if self.parent_id is not None:
274 return self.graph.nodes[self.parent_id]
275 return None
276
277 def parents(self) -> Iterable["Node"]:
278 parent = self.parent
279 while parent is not None:
280 yield parent
281 parent = parent.parent
282
283 def _raise_loop_error(self):
284 bug_ids = []
285 for parent in self.parents():
286 bug_ids.append(parent.bug.id)
287 if parent == self:
288 break
289 raise BudgetGraphLoopError(bug_ids)
290
291 @cached_property
292 def root(self) -> "Node":
293 # also checks for loop errors
294 retval = self
295 for parent in self.parents():
296 retval = parent
297 if parent == self:
298 self._raise_loop_error()
299 return retval
300
301 def children(self) -> Iterable["Node"]:
302 def visitor(node: Node) -> Iterable[Node]:
303 for i in node.immediate_children:
304 yield i
305 yield from visitor(i)
306 return visitor(self)
307
308 def children_breadth_first(self) -> Iterable["Node"]:
309 q = deque(self.immediate_children)
310 while True:
311 try:
312 node = q.popleft()
313 except IndexError:
314 return
315 q.extend(node.immediate_children)
316 yield node
317
318 def __eq__(self, other):
319 return self.bug.id == other.bug.id
320
321 def __hash__(self):
322 return self.bug.id
323
324 def __repr__(self):
325 try:
326 root = _NodeSimpleReprWrapper(self.root)
327 except BudgetGraphLoopError:
328 root = "<loop error>"
329 try:
330 milestone = repr(self.milestone)
331 except BudgetGraphBaseError:
332 milestone = "<unknown milestone>"
333 immediate_children = []
334 for i in self.immediate_children:
335 immediate_children.append(_NodeSimpleReprWrapper(i))
336 immediate_children.sort()
337 parent = f"#{self.parent_id}" if self.parent_id is not None else None
338 payments = list(self.payments.values())
339 return (f"Node(graph=..., "
340 f"id={_NodeSimpleReprWrapper(self)}, "
341 f"root={root}, "
342 f"parent={parent}, "
343 f"budget_excluding_subtasks={self.budget_excluding_subtasks}, "
344 f"budget_including_subtasks={self.budget_including_subtasks}, "
345 f"fixed_budget_excluding_subtasks={self.fixed_budget_excluding_subtasks}, "
346 f"fixed_budget_including_subtasks={self.fixed_budget_including_subtasks}, "
347 f"milestone_str={self.milestone_str!r}, "
348 f"milestone={milestone}, "
349 f"immediate_children={immediate_children!r}, "
350 f"payments={payments!r}")
351
352
353 class BudgetGraphError(BudgetGraphBaseError):
354 def __init__(self, bug_id: int, root_bug_id: int):
355 self.bug_id = bug_id
356 self.root_bug_id = root_bug_id
357
358
359 class BudgetGraphMoneyWithNoMilestone(BudgetGraphError):
360 def __str__(self):
361 return (f"Bug assigned money but without"
362 f" any assigned milestone: #{self.bug_id}")
363
364
365 class BudgetGraphMilestoneMismatch(BudgetGraphError):
366 def __str__(self):
367 return (f"Bug's assigned milestone doesn't match the milestone "
368 f"assigned to the root bug: descendant bug"
369 f" #{self.bug_id}, root bug"
370 f" #{self.root_bug_id}")
371
372
373 class BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(BudgetGraphError):
374 def __init__(self, bug_id: int, root_bug_id: int,
375 expected_budget_excluding_subtasks: Money):
376 super().__init__(bug_id, root_bug_id)
377 self.expected_budget_excluding_subtasks = \
378 expected_budget_excluding_subtasks
379
380 def __str__(self):
381 return (f"Budget assigned to task excluding subtasks "
382 f"(cf_budget field) doesn't match calculated value: "
383 f"bug #{self.bug_id}, calculated value"
384 f" {self.expected_budget_excluding_subtasks}")
385
386
387 class BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(BudgetGraphError):
388 def __init__(self, bug_id: int, root_bug_id: int,
389 expected_budget_including_subtasks: Money):
390 super().__init__(bug_id, root_bug_id)
391 self.expected_budget_including_subtasks = \
392 expected_budget_including_subtasks
393
394 def __str__(self):
395 return (f"Budget assigned to task including subtasks "
396 f"(cf_total_budget field) doesn't match calculated value: "
397 f"bug #{self.bug_id}, calculated value"
398 f" {self.expected_budget_including_subtasks}")
399
400
401 class BudgetGraphNegativeMoney(BudgetGraphError):
402 def __str__(self):
403 return (f"Budget assigned to task is less than zero: "
404 f"bug #{self.bug_id}")
405
406
407 class BudgetGraphPayeesMoneyMismatch(BudgetGraphError):
408 def __init__(self, bug_id: int, root_bug_id: int, payees_total: Money,
409 expected_payees_total: Money):
410 super().__init__(bug_id, root_bug_id)
411 self.payees_total = payees_total
412 self.expected_payees_total = expected_payees_total
413
414 def __str__(self):
415 return (f"Total budget assigned to payees (cf_payees_list) doesn't "
416 f"match expected value: bug #{self.bug_id}, calculated total "
417 f"{self.payees_total}, expected value "
418 f"{self.expected_payees_total}")
419
420
421 class BudgetGraphNegativePayeeMoney(BudgetGraphError):
422 def __init__(self, bug_id: int, root_bug_id: int, payee_key: str):
423 super().__init__(bug_id, root_bug_id)
424 self.payee_key = payee_key
425
426 def __str__(self):
427 return (f"Budget assigned to payee for task is less than zero: "
428 f"bug #{self.bug_id}, payee {self.payee_key!r}")
429
430
431 class BudgetGraphDuplicatePayeesForTask(BudgetGraphError):
432 def __init__(self, bug_id: int, root_bug_id: int, payee1_key: str, payee2_key: str):
433 super().__init__(bug_id, root_bug_id)
434 self.payee1_key = payee1_key
435 self.payee2_key = payee2_key
436
437 def __str__(self):
438 return (f"Budget assigned to multiple aliases of the same person in "
439 f"a single task: bug #{self.bug_id}, budget assigned to both "
440 f"{self.payee1_key!r} and {self.payee2_key!r}")
441
442
443 class BudgetGraphIncorrectRootForMilestone(BudgetGraphError):
444 def __init__(self, bug_id: int, milestone: str, milestone_canonical_bug_id: int):
445 super().__init__(bug_id, bug_id)
446 self.milestone = milestone
447 self.milestone_canonical_bug_id = milestone_canonical_bug_id
448
449 def __str__(self):
450 return (f"Bug #{self.bug_id} is not the canonical root bug for "
451 f"assigned milestone {self.milestone!r} but has no parent "
452 f"bug set: the milestone's canonical root bug is "
453 f"#{self.milestone_canonical_bug_id}")
454
455
456 class BudgetGraph:
457 nodes: Dict[int, Node]
458
459 def __init__(self, bugs: Iterable[Bug], config: Config):
460 self.nodes = {}
461 self.config = config
462 for bug in bugs:
463 self.nodes[bug.id] = Node(self, bug)
464 for node in self.nodes.values():
465 if node.parent is None:
466 continue
467 node.parent.immediate_children.add(node)
468
469 @cached_property
470 def roots(self) -> Set[Node]:
471 roots = set()
472 for node in self.nodes.values():
473 # calling .root also checks for loop errors
474 root = node.root
475 roots.add(root)
476 return roots
477
478 def _get_node_errors(self, root: Node, node: Node,
479 errors: List[BudgetGraphBaseError]):
480 if node.milestone_str is None:
481 if node.budget_including_subtasks != 0 \
482 or node.budget_excluding_subtasks != 0:
483 errors.append(BudgetGraphMoneyWithNoMilestone(
484 node.bug.id, root.bug.id))
485
486 try:
487 # check for milestone errors
488 node.milestone
489 if root == node and node.milestone is not None \
490 and node.milestone.canonical_bug_id != node.bug.id:
491 if node.budget_including_subtasks != 0 \
492 or node.budget_excluding_subtasks != 0:
493 errors.append(BudgetGraphIncorrectRootForMilestone(
494 node.bug.id, node.milestone.identifier,
495 node.milestone.canonical_bug_id
496 ))
497 except BudgetGraphBaseError as e:
498 errors.append(e)
499
500 if node.milestone_str != root.milestone_str:
501 errors.append(BudgetGraphMilestoneMismatch(
502 node.bug.id, root.bug.id))
503
504 if node.budget_excluding_subtasks < 0 \
505 or node.budget_including_subtasks < 0:
506 errors.append(BudgetGraphNegativeMoney(
507 node.bug.id, root.bug.id))
508
509 subtasks_total = Money(0)
510 for child in node.immediate_children:
511 subtasks_total += child.fixed_budget_including_subtasks
512
513 payees_total = Money(0)
514 payee_payments = {}
515 for payment in node.payments.values():
516 if payment.amount < 0:
517 errors.append(BudgetGraphNegativePayeeMoney(
518 node.bug.id, root.bug.id, payment.payee_key))
519 payees_total += payment.amount
520 try:
521 # check for payee errors
522 payment.payee
523 previous_payment = payee_payments.get(payment.payee)
524 if previous_payment is not None:
525 errors.append(BudgetGraphDuplicatePayeesForTask(
526 node.bug.id, root.bug.id,
527 previous_payment.payee_key, payment.payee_key
528 ))
529 payee_payments[payment.payee] = payment
530 except BudgetGraphBaseError as e:
531 errors.append(e)
532
533 def set_including_from_excluding_and_error():
534 node.fixed_budget_including_subtasks = \
535 node.budget_excluding_subtasks + subtasks_total
536 errors.append(
537 BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(
538 node.bug.id, root.bug.id,
539 node.fixed_budget_including_subtasks))
540
541 def set_including_from_payees_and_error():
542 node.fixed_budget_including_subtasks = \
543 payees_total + subtasks_total
544 errors.append(
545 BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(
546 node.bug.id, root.bug.id,
547 node.fixed_budget_including_subtasks))
548
549 def set_excluding_from_including_and_error():
550 node.fixed_budget_excluding_subtasks = \
551 node.budget_including_subtasks - subtasks_total
552 errors.append(
553 BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(
554 node.bug.id, root.bug.id,
555 node.fixed_budget_excluding_subtasks))
556
557 def set_excluding_from_payees_and_error():
558 node.fixed_budget_excluding_subtasks = \
559 payees_total
560 errors.append(
561 BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(
562 node.bug.id, root.bug.id,
563 node.fixed_budget_excluding_subtasks))
564
565 def set_payees_from_including_and_error():
566 fixed_payees_total = \
567 node.budget_including_subtasks - subtasks_total
568 errors.append(BudgetGraphPayeesMoneyMismatch(
569 node.bug.id, root.bug.id, payees_total, fixed_payees_total))
570
571 def set_payees_from_excluding_and_error():
572 fixed_payees_total = \
573 node.budget_excluding_subtasks
574 errors.append(BudgetGraphPayeesMoneyMismatch(
575 node.bug.id, root.bug.id, payees_total, fixed_payees_total))
576
577 payees_matches_including = \
578 node.budget_including_subtasks - subtasks_total == payees_total
579 payees_matches_excluding = \
580 node.budget_excluding_subtasks == payees_total
581 including_matches_excluding = \
582 node.budget_including_subtasks - subtasks_total \
583 == node.budget_excluding_subtasks
584
585 if payees_matches_including \
586 and payees_matches_excluding \
587 and including_matches_excluding:
588 pass # no error
589 elif payees_matches_including:
590 # can't have 2 match without all 3 matching
591 assert not payees_matches_excluding
592 assert not including_matches_excluding
593 if node.budget_including_subtasks == 0 and len(node.payments) == 0:
594 set_including_from_excluding_and_error()
595 else:
596 set_excluding_from_including_and_error()
597 elif payees_matches_excluding:
598 # can't have 2 match without all 3 matching
599 assert not payees_matches_including
600 assert not including_matches_excluding
601 if node.budget_excluding_subtasks == 0 and len(node.payments) == 0:
602 if node.budget_including_subtasks == 0:
603 set_including_from_excluding_and_error()
604 else:
605 set_excluding_from_including_and_error()
606 else:
607 set_including_from_excluding_and_error()
608 elif including_matches_excluding:
609 # can't have 2 match without all 3 matching
610 assert not payees_matches_including
611 assert not payees_matches_excluding
612 if len(node.payments) == 0:
613 pass # no error -- payees is just not set
614 elif node.budget_excluding_subtasks == 0 \
615 and node.budget_including_subtasks == 0:
616 set_excluding_from_payees_and_error()
617 set_including_from_payees_and_error()
618 else:
619 set_payees_from_excluding_and_error()
620 else:
621 # nothing matches
622 if len(node.payments) == 0:
623 # payees unset -- don't need to set payees
624 if node.budget_including_subtasks == 0:
625 set_including_from_excluding_and_error()
626 else:
627 set_excluding_from_including_and_error()
628 elif node.budget_excluding_subtasks == 0 \
629 and node.budget_including_subtasks == 0:
630 set_excluding_from_payees_and_error()
631 set_including_from_payees_and_error()
632 elif node.budget_excluding_subtasks == 0:
633 set_excluding_from_including_and_error()
634 set_payees_from_including_and_error()
635 elif node.budget_including_subtasks == 0:
636 set_including_from_excluding_and_error()
637 set_payees_from_excluding_and_error()
638 else:
639 set_including_from_excluding_and_error()
640 set_payees_from_excluding_and_error()
641
642 def get_errors(self) -> List[BudgetGraphBaseError]:
643 errors = []
644 try:
645 roots = self.roots
646 except BudgetGraphBaseError as e:
647 errors.append(e)
648 return errors
649
650 for root in roots:
651 try:
652 for child in reversed(list(root.children_breadth_first())):
653 try:
654 self._get_node_errors(root, child, errors)
655 except BudgetGraphBaseError as e:
656 errors.append(e)
657 self._get_node_errors(root, root, errors)
658 except BudgetGraphBaseError as e:
659 errors.append(e)
660 return errors
661
662 @cached_property
663 def payments(self) -> Dict[Person, Dict[Milestone, List[Payment]]]:
664 retval = {}
665 for person in self.config.people.values():
666 milestone_payments = {}
667 for milestone in self.config.milestones.values():
668 milestone_payments[milestone] = []
669 retval[person] = milestone_payments
670 for node in self.nodes.values():
671 if node.milestone is not None:
672 for payment in node.payments.values():
673 retval[payment.payee][node.milestone].append(payment)
674 return retval