budget-sync is now working after adding code to use a config file
[utils.git] / src / budget_sync / budget_graph.py
1 from bugzilla.bug import Bug
2 from bugzilla import Bugzilla
3 from typing import Set, Dict, Iterable, Optional, List, Union, Any
4 from budget_sync.util import all_bugs
5 from budget_sync.money import Money
6 from budget_sync.config import Config, Person, Milestone
7 from functools import cached_property
8 import toml
9 import sys
10 import enum
11 from collections import deque
12 from datetime import date, time, datetime
13
14
15 class BudgetGraphBaseError(Exception):
16 pass
17
18
19 class BudgetGraphParseError(BudgetGraphBaseError):
20 def __init__(self, bug_id: int):
21 self.bug_id = bug_id
22
23
24 class BudgetGraphPayeesParseError(BudgetGraphParseError):
25 def __init__(self, bug_id: int, msg: str):
26 super().__init__(bug_id)
27 self.msg = msg
28
29 def __str__(self):
30 return f"Failed to parse cf_payees_list field of " \
31 f"bug #{self.bug_id}: {self.msg}"
32
33
34 class BudgetGraphLoopError(BudgetGraphBaseError):
35 def __init__(self, bug_ids: List[int]):
36 self.bug_ids = bug_ids
37
38 def __str__(self):
39 retval = f"Detected Loop in Budget Graph: #{self.bug_ids[-1]} -> "
40 retval += " -> ".join((f"#{i}" for i in self.bug_ids))
41 return retval
42
43
44 class _NodeSimpleReprWrapper:
45 def __init__(self, node: "Node"):
46 self.node = node
47
48 def __repr__(self):
49 return f"#{self.node.bug.id}"
50
51 def __lt__(self, other):
52 # for list.sort()
53 return self.node.bug.id < other.node.bug.id
54
55
56 class PayeeState(enum.Enum):
57 NotYetSubmitted = "not yet submitted"
58 Submitted = "submitted"
59 Paid = "paid"
60
61
62 _Date = Union[date, datetime]
63
64
65 def _parse_money_from_toml(value: Any) -> Money:
66 if not isinstance(value, (int, str)):
67 msg = f"monetary amount is not a string or integer " \
68 f"(to use fractional amounts such as 123.45, write " \
69 f"\"123.45\"): {value!r}"
70 raise ValueError(msg)
71 return Money(value)
72
73
74 def _parse_date_time_or_none_from_toml(value: Any) -> Optional[_Date]:
75 if value is None or isinstance(value, (date, datetime)):
76 return value
77 elif isinstance(value, time):
78 msg = f"just a time of day by itself is not enough," \
79 f" a date must be included: {str(value)}"
80 raise ValueError(msg)
81 elif isinstance(value, bool):
82 msg = f"invalid date: {str(value).lower()}"
83 raise ValueError(msg)
84 elif isinstance(value, (str, int, float)):
85 msg = f"invalid date: {value!r}"
86 raise ValueError(msg)
87 else:
88 msg = f"invalid date"
89 raise ValueError(msg)
90
91
92 class Payment:
93 def __init__(self,
94 node: "Node",
95 payee_key: str,
96 amount: Money,
97 paid: Optional[_Date],
98 submitted: Optional[_Date]):
99 self.node = node
100 self.payee_key = payee_key
101 self.amount = amount
102 self.paid = paid
103 self.submitted = submitted
104
105 @cached_property
106 def payee(self) -> Person:
107 try:
108 return self.node.graph.config.all_names[self.payee_key]
109 except KeyError:
110 msg = f"unknown payee name: {self.payee_key!r} is not the name " \
111 f"or an alias of any known person"
112 raise BudgetGraphPayeesParseError(self.node.bug.id, msg) \
113 .with_traceback(sys.exc_info()[2])
114
115 @property
116 def state(self):
117 if self.paid is not None:
118 return PayeeState.Paid
119 if self.submitted is not None:
120 return PayeeState.Submitted
121 return PayeeState.NotYetSubmitted
122
123 @staticmethod
124 def _from_toml(node: "Node", payee_key: str, toml_value: Any) -> "Payment":
125 paid = None
126 submitted = None
127 known_keys = ("paid", "submitted", "amount")
128 if isinstance(toml_value, dict):
129 try:
130 amount = toml_value['amount']
131 except KeyError:
132 msg = f"value for key {payee_key!r} is missing the " \
133 f"`amount` field which is required"
134 raise BudgetGraphPayeesParseError(node.bug.id, msg) \
135 .with_traceback(sys.exc_info()[2])
136 for k, v in toml_value.items():
137 if k in ("paid", "submitted"):
138 try:
139 parsed_value = _parse_date_time_or_none_from_toml(v)
140 except ValueError as e:
141 msg = f"failed to parse `{k}` field for" \
142 f" key {payee_key!r}: {e}"
143 raise BudgetGraphPayeesParseError(
144 node.bug.id, msg) \
145 .with_traceback(sys.exc_info()[2])
146 if k == "paid":
147 paid = parsed_value
148 else:
149 assert k == "submitted"
150 submitted = parsed_value
151 if k not in known_keys:
152 msg = f"value for key {payee_key!r} has an unknown" \
153 f" field: `{k}`"
154 raise BudgetGraphPayeesParseError(node.bug.id, msg) \
155 .with_traceback(sys.exc_info()[2])
156 try:
157 paid = _parse_date_time_or_none_from_toml(
158 toml_value.get('paid'))
159 except ValueError as e:
160 msg = f"failed to parse `paid` field for" \
161 f" key {payee_key!r}: {e}"
162 raise BudgetGraphPayeesParseError(
163 node.bug.id, msg) \
164 .with_traceback(sys.exc_info()[2])
165 try:
166 submitted = _parse_date_time_or_none_from_toml(
167 toml_value.get('submitted'))
168 except ValueError as e:
169 msg = f"failed to parse `submitted` field for" \
170 f" key {payee_key!r}: {e}"
171 raise BudgetGraphPayeesParseError(
172 node.bug.id, msg) \
173 .with_traceback(sys.exc_info()[2])
174 elif isinstance(toml_value, (int, str, float)):
175 # float included for better error messages
176 amount = toml_value
177 else:
178 msg = f"value for key {payee_key!r} is invalid -- it should " \
179 f"either be a monetary value or a table"
180 raise BudgetGraphPayeesParseError(node.bug.id, msg)
181 try:
182 amount = _parse_money_from_toml(amount)
183 except ValueError as e:
184 msg = f"failed to parse monetary amount for key {payee_key!r}: {e}"
185 raise BudgetGraphPayeesParseError(
186 node.bug.id, msg) \
187 .with_traceback(sys.exc_info()[2])
188 return Payment(node=node, payee_key=payee_key, amount=amount,
189 paid=paid, submitted=submitted)
190
191 def __repr__(self):
192 try:
193 payee = f"Person<{self.payee.identifier!r}>"
194 except BudgetGraphBaseError:
195 payee = "<unknown person>"
196 return (f"Payment(node={_NodeSimpleReprWrapper(self.node)}, "
197 f"payee={payee}, "
198 f"payee_key={self.payee_key!r}, "
199 f"amount={self.amount}, "
200 f"state={self.state.name}, "
201 f"paid={str(self.paid)}, "
202 f"submitted={str(self.submitted)})")
203
204
205 class BudgetGraphUnknownMilestone(BudgetGraphParseError):
206 def __init__(self, bug_id: int, milestone_str: str):
207 super().__init__(bug_id)
208 self.milestone_str = milestone_str
209
210 def __str__(self):
211 return f"failed to parse cf_nlnet_milestone field of bug " \
212 f"#{self.bug_id}: unknown milestone: {self.milestone_str!r}"
213
214
215 class Node:
216 graph: "BudgetGraph"
217 bug: Bug
218 parent_id: Optional[int]
219 immediate_children: Set["Node"]
220 budget_excluding_subtasks: Money
221 budget_including_subtasks: Money
222 fixed_budget_excluding_subtasks: Money
223 fixed_budget_including_subtasks: Money
224 milestone_str: Optional[str]
225
226 def __init__(self, graph: "BudgetGraph", bug: Bug):
227 self.graph = graph
228 self.bug = bug
229 self.parent_id = getattr(bug, "cf_budget_parent", None)
230 self.immediate_children = set()
231 self.budget_excluding_subtasks = Money.from_str(bug.cf_budget)
232 self.fixed_budget_excluding_subtasks = self.budget_excluding_subtasks
233 self.budget_including_subtasks = Money.from_str(bug.cf_total_budget)
234 self.fixed_budget_including_subtasks = self.budget_including_subtasks
235 self.milestone_str = bug.cf_nlnet_milestone
236 if self.milestone_str == "---":
237 self.milestone_str = None
238
239 @cached_property
240 def milestone(self) -> Optional[Milestone]:
241 try:
242 if self.milestone_str is not None:
243 return self.graph.config.milestones[self.milestone_str]
244 return None
245 except KeyError:
246 new_err = BudgetGraphUnknownMilestone(
247 self.bug.id, self.milestone_str)
248 raise new_err.with_traceback(sys.exc_info()[2])
249
250 @cached_property
251 def payments(self) -> Dict[str, Payment]:
252 try:
253 parsed = toml.loads(self.bug.cf_payees_list)
254 except toml.TomlDecodeError as e:
255 new_err = BudgetGraphPayeesParseError(
256 self.bug.id, f"TOML parse error: {e}")
257 raise new_err.with_traceback(sys.exc_info()[2])
258 retval = {}
259 for key, value in parsed.items():
260 if not isinstance(key, str):
261 raise BudgetGraphPayeesParseError(
262 self.bug.id, f"key is not a string: {key!r}")
263 retval[key] = Payment._from_toml(self, key, value)
264 return retval
265
266 @property
267 def parent(self) -> Optional["Node"]:
268 if self.parent_id is not None:
269 return self.graph.nodes[self.parent_id]
270 return None
271
272 def parents(self) -> Iterable["Node"]:
273 parent = self.parent
274 while parent is not None:
275 yield parent
276 parent = parent.parent
277
278 def _raise_loop_error(self):
279 bug_ids = []
280 for parent in self.parents():
281 bug_ids.append(parent.bug.id)
282 if parent == self:
283 break
284 raise BudgetGraphLoopError(bug_ids)
285
286 @cached_property
287 def root(self) -> "Node":
288 # also checks for loop errors
289 retval = self
290 for parent in self.parents():
291 retval = parent
292 if parent == self:
293 self._raise_loop_error()
294 return retval
295
296 def children(self) -> Iterable["Node"]:
297 def visitor(node: Node) -> Iterable[Node]:
298 for i in node.immediate_children:
299 yield i
300 yield from visitor(i)
301 return visitor(self)
302
303 def children_breadth_first(self) -> Iterable["Node"]:
304 q = deque(self.immediate_children)
305 while True:
306 try:
307 node = q.popleft()
308 except IndexError:
309 return
310 q.extend(node.immediate_children)
311 yield node
312
313 def __eq__(self, other):
314 return self.bug.id == other.bug.id
315
316 def __hash__(self):
317 return self.bug.id
318
319 def __repr__(self):
320 try:
321 root = _NodeSimpleReprWrapper(self.root)
322 except BudgetGraphLoopError:
323 root = "<loop error>"
324 try:
325 milestone = repr(self.milestone)
326 except BudgetGraphBaseError:
327 milestone = "<unknown milestone>"
328 immediate_children = []
329 for i in self.immediate_children:
330 immediate_children.append(_NodeSimpleReprWrapper(i))
331 immediate_children.sort()
332 parent = f"#{self.parent_id}" if self.parent_id is not None else None
333 payments = list(self.payments.values())
334 return (f"Node(graph=..., "
335 f"id={_NodeSimpleReprWrapper(self)}, "
336 f"root={root}, "
337 f"parent={parent}, "
338 f"budget_excluding_subtasks={self.budget_excluding_subtasks}, "
339 f"budget_including_subtasks={self.budget_including_subtasks}, "
340 f"fixed_budget_excluding_subtasks={self.fixed_budget_excluding_subtasks}, "
341 f"fixed_budget_including_subtasks={self.fixed_budget_including_subtasks}, "
342 f"milestone_str={self.milestone_str!r}, "
343 f"milestone={milestone}, "
344 f"immediate_children={immediate_children!r}, "
345 f"payments={payments!r}")
346
347
348 class BudgetGraphError(BudgetGraphBaseError):
349 def __init__(self, bug_id: int, root_bug_id: int):
350 self.bug_id = bug_id
351 self.root_bug_id = root_bug_id
352
353
354 class BudgetGraphMoneyWithNoMilestone(BudgetGraphError):
355 def __str__(self):
356 return (f"Bug assigned money but without"
357 f" any assigned milestone: #{self.bug_id}")
358
359
360 class BudgetGraphMilestoneMismatch(BudgetGraphError):
361 def __str__(self):
362 return (f"Bug's assigned milestone doesn't match the milestone "
363 f"assigned to the root bug: descendant bug"
364 f" #{self.bug_id}, root bug"
365 f" #{self.root_bug_id}")
366
367
368 class BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(BudgetGraphError):
369 def __init__(self, bug_id: int, root_bug_id: int,
370 expected_budget_excluding_subtasks: Money):
371 super().__init__(bug_id, root_bug_id)
372 self.expected_budget_excluding_subtasks = \
373 expected_budget_excluding_subtasks
374
375 def __str__(self):
376 return (f"Budget assigned to task excluding subtasks "
377 f"(cf_budget field) doesn't match calculated value: "
378 f"bug #{self.bug_id}, calculated value"
379 f" {self.expected_budget_excluding_subtasks}")
380
381
382 class BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(BudgetGraphError):
383 def __init__(self, bug_id: int, root_bug_id: int,
384 expected_budget_including_subtasks: Money):
385 super().__init__(bug_id, root_bug_id)
386 self.expected_budget_including_subtasks = \
387 expected_budget_including_subtasks
388
389 def __str__(self):
390 return (f"Budget assigned to task including subtasks "
391 f"(cf_total_budget field) doesn't match calculated value: "
392 f"bug #{self.bug_id}, calculated value"
393 f" {self.expected_budget_including_subtasks}")
394
395
396 class BudgetGraphNegativeMoney(BudgetGraphError):
397 def __str__(self):
398 return (f"Budget assigned to task is less than zero: "
399 f"bug #{self.bug_id}")
400
401
402 class BudgetGraphPayeesMoneyMismatch(BudgetGraphError):
403 def __init__(self, bug_id: int, root_bug_id: int, payees_total: Money,
404 expected_payees_total: Money):
405 super().__init__(bug_id, root_bug_id)
406 self.payees_total = payees_total
407 self.expected_payees_total = expected_payees_total
408
409 def __str__(self):
410 return (f"Total budget assigned to payees (cf_payees_list) doesn't "
411 f"match expected value: bug #{self.bug_id}, calculated total "
412 f"{self.payees_total}, expected value "
413 f"{self.expected_payees_total}")
414
415
416 class BudgetGraphNegativePayeeMoney(BudgetGraphError):
417 def __init__(self, bug_id: int, root_bug_id: int, payee_key: str):
418 super().__init__(bug_id, root_bug_id)
419 self.payee_key = payee_key
420
421 def __str__(self):
422 return (f"Budget assigned to payee for task is less than zero: "
423 f"bug #{self.bug_id}, payee {self.payee_key!r}")
424
425
426 class BudgetGraphDuplicatePayeesForTask(BudgetGraphError):
427 def __init__(self, bug_id: int, root_bug_id: int, payee1_key: str, payee2_key: str):
428 super().__init__(bug_id, root_bug_id)
429 self.payee1_key = payee1_key
430 self.payee2_key = payee2_key
431
432 def __str__(self):
433 return (f"Budget assigned to multiple aliases of the same person in "
434 f"a single task: bug #{self.bug_id}, budget assigned to both "
435 f"{self.payee1_key!r} and {self.payee2_key!r}")
436
437
438 class BudgetGraphIncorrectRootForMilestone(BudgetGraphError):
439 def __init__(self, bug_id: int, milestone: str, milestone_canonical_bug_id: int):
440 super().__init__(bug_id, bug_id)
441 self.milestone = milestone
442 self.milestone_canonical_bug_id = milestone_canonical_bug_id
443
444 def __str__(self):
445 return (f"Bug #{self.bug_id} is not the canonical root bug for "
446 f"assigned milestone {self.milestone!r} but has no parent "
447 f"bug set: the milestone's canonical root bug is "
448 f"#{self.milestone_canonical_bug_id}")
449
450
451 class BudgetGraph:
452 nodes: Dict[int, Node]
453
454 def __init__(self, bugs: Iterable[Bug], config: Config):
455 self.nodes = {}
456 self.config = config
457 for bug in bugs:
458 self.nodes[bug.id] = Node(self, bug)
459 for node in self.nodes.values():
460 if node.parent is None:
461 continue
462 node.parent.immediate_children.add(node)
463
464 @cached_property
465 def roots(self) -> Set[Node]:
466 roots = set()
467 for node in self.nodes.values():
468 # calling .root also checks for loop errors
469 root = node.root
470 roots.add(root)
471 return roots
472
473 def _get_node_errors(self, root: Node, node: Node,
474 errors: List[BudgetGraphBaseError]):
475 if node.milestone_str is None:
476 if node.budget_including_subtasks != 0 \
477 or node.budget_excluding_subtasks != 0:
478 errors.append(BudgetGraphMoneyWithNoMilestone(
479 node.bug.id, root.bug.id))
480
481 try:
482 # check for milestone errors
483 node.milestone
484 if root == node and node.milestone is not None \
485 and node.milestone.canonical_bug_id != node.bug.id:
486 if node.budget_including_subtasks != 0 \
487 or node.budget_excluding_subtasks != 0:
488 errors.append(BudgetGraphIncorrectRootForMilestone(
489 node.bug.id, node.milestone.identifier,
490 node.milestone.canonical_bug_id
491 ))
492 except BudgetGraphBaseError as e:
493 errors.append(e)
494
495 if node.milestone_str != root.milestone_str:
496 errors.append(BudgetGraphMilestoneMismatch(
497 node.bug.id, root.bug.id))
498
499 if node.budget_excluding_subtasks < 0 \
500 or node.budget_including_subtasks < 0:
501 errors.append(BudgetGraphNegativeMoney(
502 node.bug.id, root.bug.id))
503
504 subtasks_total = Money(0)
505 for child in node.immediate_children:
506 subtasks_total += child.fixed_budget_including_subtasks
507
508 payees_total = Money(0)
509 payee_payments = {}
510 for payment in node.payments.values():
511 if payment.amount < 0:
512 errors.append(BudgetGraphNegativePayeeMoney(
513 node.bug.id, root.bug.id, payment.payee_key))
514 payees_total += payment.amount
515 try:
516 # check for payee errors
517 payment.payee
518 previous_payment = payee_payments.get(payment.payee)
519 if previous_payment is not None:
520 errors.append(BudgetGraphDuplicatePayeesForTask(
521 node.bug.id, root.bug.id,
522 previous_payment.payee_key, payment.payee_key
523 ))
524 payee_payments[payment.payee] = payment
525 except BudgetGraphBaseError as e:
526 errors.append(e)
527
528 def set_including_from_excluding_and_error():
529 node.fixed_budget_including_subtasks = \
530 node.budget_excluding_subtasks + subtasks_total
531 errors.append(
532 BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(
533 node.bug.id, root.bug.id,
534 node.fixed_budget_including_subtasks))
535
536 def set_including_from_payees_and_error():
537 node.fixed_budget_including_subtasks = \
538 payees_total + subtasks_total
539 errors.append(
540 BudgetGraphMoneyMismatchForBudgetIncludingSubtasks(
541 node.bug.id, root.bug.id,
542 node.fixed_budget_including_subtasks))
543
544 def set_excluding_from_including_and_error():
545 node.fixed_budget_excluding_subtasks = \
546 node.budget_including_subtasks - subtasks_total
547 errors.append(
548 BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(
549 node.bug.id, root.bug.id,
550 node.fixed_budget_excluding_subtasks))
551
552 def set_excluding_from_payees_and_error():
553 node.fixed_budget_excluding_subtasks = \
554 payees_total
555 errors.append(
556 BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(
557 node.bug.id, root.bug.id,
558 node.fixed_budget_excluding_subtasks))
559
560 def set_payees_from_including_and_error():
561 fixed_payees_total = \
562 node.budget_including_subtasks - subtasks_total
563 errors.append(BudgetGraphPayeesMoneyMismatch(
564 node.bug.id, root.bug.id, payees_total, fixed_payees_total))
565
566 def set_payees_from_excluding_and_error():
567 fixed_payees_total = \
568 node.budget_excluding_subtasks
569 errors.append(BudgetGraphPayeesMoneyMismatch(
570 node.bug.id, root.bug.id, payees_total, fixed_payees_total))
571
572 payees_matches_including = \
573 node.budget_including_subtasks - subtasks_total == payees_total
574 payees_matches_excluding = \
575 node.budget_excluding_subtasks == payees_total
576 including_matches_excluding = \
577 node.budget_including_subtasks - subtasks_total \
578 == node.budget_excluding_subtasks
579
580 if payees_matches_including \
581 and payees_matches_excluding \
582 and including_matches_excluding:
583 pass # no error
584 elif payees_matches_including:
585 # can't have 2 match without all 3 matching
586 assert not payees_matches_excluding
587 assert not including_matches_excluding
588 if node.budget_including_subtasks == 0 and len(node.payments) == 0:
589 set_including_from_excluding_and_error()
590 else:
591 set_excluding_from_including_and_error()
592 elif payees_matches_excluding:
593 # can't have 2 match without all 3 matching
594 assert not payees_matches_including
595 assert not including_matches_excluding
596 if node.budget_excluding_subtasks == 0 and len(node.payments) == 0:
597 if node.budget_including_subtasks == 0:
598 set_including_from_excluding_and_error()
599 else:
600 set_excluding_from_including_and_error()
601 else:
602 set_including_from_excluding_and_error()
603 elif including_matches_excluding:
604 # can't have 2 match without all 3 matching
605 assert not payees_matches_including
606 assert not payees_matches_excluding
607 if len(node.payments) == 0:
608 pass # no error -- payees is just not set
609 elif node.budget_excluding_subtasks == 0 \
610 and node.budget_including_subtasks == 0:
611 set_excluding_from_payees_and_error()
612 set_including_from_payees_and_error()
613 else:
614 set_payees_from_excluding_and_error()
615 else:
616 # nothing matches
617 if len(node.payments) == 0:
618 # payees unset -- don't need to set payees
619 if node.budget_including_subtasks == 0:
620 set_including_from_excluding_and_error()
621 else:
622 set_excluding_from_including_and_error()
623 elif node.budget_excluding_subtasks == 0 \
624 and node.budget_including_subtasks == 0:
625 set_excluding_from_payees_and_error()
626 set_including_from_payees_and_error()
627 elif node.budget_excluding_subtasks == 0:
628 set_excluding_from_including_and_error()
629 set_payees_from_including_and_error()
630 elif node.budget_including_subtasks == 0:
631 set_including_from_excluding_and_error()
632 set_payees_from_excluding_and_error()
633 else:
634 set_including_from_excluding_and_error()
635 set_payees_from_excluding_and_error()
636
637 def get_errors(self) -> List[BudgetGraphBaseError]:
638 errors = []
639 try:
640 roots = self.roots
641 except BudgetGraphBaseError as e:
642 errors.append(e)
643 return errors
644
645 for root in roots:
646 try:
647 for child in reversed(list(root.children_breadth_first())):
648 try:
649 self._get_node_errors(root, child, errors)
650 except BudgetGraphBaseError as e:
651 errors.append(e)
652 self._get_node_errors(root, root, errors)
653 except BudgetGraphBaseError as e:
654 errors.append(e)
655 return errors
656
657 @cached_property
658 def payments(self) -> Dict[Person, Dict[Milestone, List[Payment]]]:
659 retval = {}
660 for person in self.config.people.values():
661 milestone_payments = {}
662 for milestone in self.config.milestones.values():
663 milestone_payments[milestone] = []
664 retval[person] = milestone_payments
665 for node in self.nodes.values():
666 if node.milestone is not None:
667 for payment in node.payments.values():
668 retval[payment.payee][node.milestone].append(payment)
669 return retval