--- /dev/null
+[run]
+branch = True
+include =
+ src/*
+omit =
+ src/budget_sync/test/*
+
+[report]
+exclude_lines =
+ :nocov:
+partial_branches =
+ :nobr:
/.vscode
*.pyc
/output
-.coverage
\ No newline at end of file
+/task_db
+.coverage
+cov.xml
+/dump
--- /dev/null
+image: debian:10
+
+build:
+ stage: build
+ before_script:
+ - apt-get update
+ # one package per line to simplify sorting, git diff, etc.
+ - >-
+ apt-get -y install --no-install-recommends
+ python3-pip
+ python3-setuptools
+ - export PATH="$HOME/.local/bin:$PATH"
+ - python3 -m pip install --user pytest-xdist
+ script:
+ - python3 setup.py develop
+ - pytest -v -n auto
--- /dev/null
+.PHANTOM: build upload clean
+
+build:
+ mkdir -p task_db
+ budget-sync -c budget-sync-config.toml -o task_db/mdwn
+
+justupload:
+ rsync -HPavz --no-perms --no-group --no-times -e 'ssh -p 922' --delete task_db/* \
+ lkcl@libre-soc.org:/var/www/libresoc-nlnet/task_db
+ ssh lkcl@libre-soc.org update-ikiwiki.sh
+
+upload: build justupload
+
+clean:
+ rm -rf task_db
https://git.libre-soc.org/?p=utils.git;a=summary
clone with:
- git clone 'git://git.libre-soc.org/utils.git'
+ git clone 'https://git.libre-soc.org/git/utils.git'
-install with "python3 setup.py develop" as root (or "python3 setup.py
-develop --user" and remember to add ~/.local/bin to $PATH)
+as root install with
+
+ python3 setup.py develop
+
+or as user, just remember to add ~/.local/bin to $PATH)
+
+ python3 setup.py develop --user
run as:
+
nohup budget-sync -c budget-sync-config.toml -o mdwn
+to create JSON output with comment #0 for each MoU task:
+
+ nohup budget-sync -c budget-sync-config.toml -o mdwn --comments
+
then examine the nohup.out and also the individual markdown files,
-or compile to html first:
+or compile to html first (after apt-get install python-markdown):
+
markdown_py -f lkcl.html lkcl.mdwn
+
+to perform multiple *DIRECT* database updates automatically, use the
+following command:
+
+ python3 ./src/budget_sync/update.py -c budget-sync-config.toml \
+ --user {name_in_TOML_field} \
+ --username {BUGZILLA_EMAIL} \
+ --password '{BUGZILLA_PASSWORD}' \
+ --bug NNN,MMM,OOO \
+ --submitted=YYYY-MM-DD or --paid=YYYY-MM-DD
+
+**PLEASE EXERCISE EXTREME CAUTION WHEN USING THIS COMMAND**
+
bugzilla_url = "https://bugs.libre-soc.org"
-[people."Jacob R. Lifshay"]
+[people."2022-08-071"]
+email = "2022-08-071@nlnet.nl"
+aliases = []
+full_name = "NLnet 2022-08-071 cavatools"
+
+[people."2022-02-051"]
+email = "2022-02-051@nlnet.nl"
+aliases = []
+full_name = "NLnet 2022-02-051 OPF"
+
+[people."2022-08E"]
+email = "2022-08E@nlnet.nl"
+aliases = []
+full_name = "NLnet 2022-08E"
+
+[people."2022-02A"]
+email = "2022-02A@nlnet.nl"
+aliases = []
+full_name = "NLnet 2022-02A"
+
+[people."james"]
+email = "james.lewis@redsemiconductor.com"
+aliases = []
+full_name = "James Lewis"
+
+[people."djac"]
+email = "djac@calderwoodhan.com"
+aliases = []
+full_name = "David Calderwood"
+
+[people."shriya"]
+email = "shriya.sharma@redsemiconductor.com"
+aliases = []
+full_name = "Shriya Sharma"
+
+[people."sadoon"]
+email = "sadoon@albader.co"
+aliases = []
+full_name = "Sadoon Albader"
+
+[people."programmerjake"]
email = "programmerjake@gmail.com"
-aliases = ["programmerjake", "jacob", "Jacob", "Jacob Lifshay"]
-output_markdown_file = "programmerjake.mdwn"
+aliases = ["jacob", "Jacob", "Jacob Lifshay", "fpga-fund"]
+full_name = "Jacob R. Lifshay"
+
+[people."dan"]
+email = "danleighton@gmail.com"
+aliases = [ "danleighton",
+ ]
+full_name = "Dan Leighton"
-[people."Luke Kenneth Casson Leighton"]
+[people."red"]
+email = "lkcl@libre-soc.org"
+aliases = [ "lip6_donated_nlnet",
+ "lip6_donated",
+ ]
+full_name = "RED Semiconductor Ltd"
+
+[people."lkcl"]
email = "lkcl@lkcl.net"
-aliases = ["lkcl", "luke", "Luke", "Luke Leighton"]
-output_markdown_file = "lkcl.mdwn"
+aliases = ["donated", "donated_samuel", "luke", "Luke", "Luke Leighton"]
+full_name = "Luke Kenneth Casson Leighton"
+
+[people."ghostmansd"]
+email = "ghostmansd@gmail.com"
+aliases = ["ghostmansd", "dmitry", "ghostman"]
+full_name = "Dmitry Selyutin"
+
+[people."NLnet_2019_10P"]
+email = "2019-10P@nlnet.nl"
+aliases = []
+full_name = "NLnet 2019-10P"
+
+[people."markos"]
+email = "konstantinos@vectorcamp.gr"
+aliases = ['konstantinos']
+full_name = "Konstantinos Margaritis"
-[people."Samuel A. Falvo II"]
+[people."vectorcamp"]
+email = "info@vectorcamp.gr"
+aliases = []
+full_name = "Vectorcamp"
+
+[people."tpearson"]
+email = "tpearson@raptorengineering.com"
+aliases = []
+full_name = "Tim Pearson"
+
+[people."andrey"]
+email = "andy.miroshnikov@gmail.com"
+aliases = ["octavius"]
+full_name = "Andrey Miroshnikov"
+
+[people."mikolaj"]
+email = "wielgusmikolaj@gmail.com"
+aliases = ["mikolajw"]
+full_name = "Mikolaj"
+
+[people."asics_ws"]
+email = "rudi@asics.ws"
+aliases = ["rudi"]
+full_name = "Rudi (ASICs.ws)"
+
+[people."3mdeb_maciej"]
+email = "maciej.pijanowski@3mdeb.com"
+aliases = ["maciej", "maciej_3mdeb"]
+full_name = "Maciej (3mdeb)"
+
+[people."lxo"]
+email = "oliva@libre-soc.org"
+aliases = ["oliva", "aoliva"]
+full_name = "Alexandre Oliva"
+
+[people."awygle"]
+email = "awygle@gmail.com"
+aliases = ["mail@awygle.com"]
+full_name = "Andrew Wygle"
+
+[people."Samuel_A_Falvo_II"]
email = "kc5tja@arrl.net"
aliases = ["kc5tja", "samuel", "Samuel", "Samuel Falvo II", "sam.falvo"]
-output_markdown_file = "Samuel_A_Falvo_II.mdwn"
+full_name = "Samuel A. Falvo II"
-[people."Vivek Pandya"]
+[people."vivek_pandya"]
email = "vivekvpandya@gmail.com"
aliases = ["vivekvpandya", "vivek pandya", "vivek", "Vivek"]
-output_markdown_file = "vivek_pandya.mdwn"
+full_name = "Vivek Pandya"
-[people."Florent Kermarrec"]
+[people."florent_kermarrec"]
email = "florent@enjoy-digital.fr"
aliases = ["florent", "Florent"]
-output_markdown_file = "florent_kermarrec.mdwn"
+full_name = "Florent Kermarrec"
-[people."Michael Nolan"]
+[people."michael_nolan"]
email = "mtnolan2640@gmail.com"
-aliases = ["donated", "mnolan", "michael", "Michael", "mtnolan", "mtnolan2640"]
-output_markdown_file = "michael_nolan.mdwn"
+aliases = ["mnolan", "michael", "Michael", "mtnolan", "mtnolan2640"]
+full_name = "Michael Nolan"
-[people."Alain D D Williams"]
+[people."addw"]
email = "addw@phcomp.co.uk"
-aliases = ["alain", "Alain", "Alain Williams", "addw"]
-output_markdown_file = "addw.mdwn"
+aliases = ["Alain", "Alain Williams", "alain"]
+full_name = "Alain D D Williams"
-[people."Jock Tanner"]
+[people."jock_tanner"]
email = "tanner.of.kha@gmail.com"
aliases = ["jock"]
-output_markdown_file = "jock_tanner.mdwn"
+full_name = "Jock Tanner"
-[people."R Veera Kumar"]
-email = "vklr@vkten.in"
-aliases = ["Veera", "veera", "Veera Kumar"]
-output_markdown_file = "veera.mdwn"
+[people."veera"]
+email = "veerakumar.r@gmail.com"
+aliases = ["vklr", "Veera Kumar", "Veera"]
+full_name = "R Veera Kumar"
-[people."Jean-Paul Chaput"]
+[people."jean-paul_chaput"]
email = "Jean-Paul.Chaput@lip6.fr"
-aliases = ["lip6", "Jean Paul Chaput", "Jean-Paul", "jean-paul", "jean paul"]
-output_markdown_file = "jean-paul_chaput.mdwn"
+aliases = [
+ "lip6",
+ "Jean Paul Chaput",
+ "Jean-Paul",
+ "jean-paul",
+ "jean paul",
+]
+full_name = "Jean-Paul Chaput"
-[people."Staf Verhaegen"]
+[people."staf_verhaegen"]
email = "staf@fibraservi.eu"
-aliases = ["Staf", "staf"]
-output_markdown_file = "staf_verhaegen.mdwn"
+aliases = ["staf", "Staf", "chips4makers"]
+full_name = "Staf Verhaegen"
-[people."Lauri Kasanen"]
+[people."lauri_kasanen"]
email = "cand@gmx.com"
-aliases = ["Lauri", "lauri"]
-output_markdown_file = "lauri_kasanen.mdwn"
+aliases = ["lauri", "Lauri"]
+full_name = "Lauri Kasanen"
-[people."Yehowshua Immanuel"]
+[people."yehowshua_immanuel"]
email = "yimmanuel3@gatech.edu"
-aliases = ["Yehowshua", "yehowshua"]
-output_markdown_file = "yehowshua_immanuel.mdwn"
+aliases = ["yehowshua", "Yehowshua"]
+full_name = "Yehowshua Immanuel"
[people."whitequark"]
email = "whitequark@whitequark.org"
aliases = []
-output_markdown_file = "whitequark.mdwn"
+full_name = "whitequark"
-[people."Tobias Platen"]
+[people."tplaten"]
email = "libre-soc@platen-software.de"
-aliases = ["Tobias", "tobias", "tplaten"]
-output_markdown_file = "tplaten.mdwn"
+aliases = ["tobias", "Tobias"]
+full_name = "Tobias Platen"
-[people."Cole Poirier"]
+[people."cole"]
email = "colepoirier@gmail.com"
-aliases = ["Cole", "cole", "colepoirier"]
-output_markdown_file = "cole.mdwn"
+aliases = ["Cole", "colepoirier"]
+full_name = "Cole Poirier"
-[people."Aleksandar Kostovic"]
+[people."aleksandar_kostovic"]
email = "alexandar.kostovic@gmail.com"
aliases = ["alexandar", "aleksandar"]
-output_markdown_file = "aleksandar_kostovic.mdwn"
+full_name = "Aleksandar Kostovic"
-[people."Cesar Strauss"]
+[people."cesar_strauss"]
email = "cestrauss@gmail.com"
-aliases = ["Cesar", "cesar", "cestrauss"]
-output_markdown_file = "cesar_strauss.mdwn"
+aliases = ["cesar", "Cesar", "cestrauss"]
+full_name = "Cesar Strauss"
-[people."Dimitri Galayko"]
+[people."dimitri_galayko"]
email = "dimitri.galayko@lip6.fr"
+aliases = ["dimitri", "dimitry"]
+full_name = "Dimitri Galayko"
+
+[people."marie_minerve"]
+email = "marie-minerve.louerat@lip6.fr"
+aliases = ["marie"]
+full_name = "Marie-Minerve Louerat"
+
+[people."drchat"]
+email = "me@justinm.one"
+aliases = ["DrChat"]
+full_name = "Justin Moore"
+
+[people."rwilbur"]
+email = "richard.wilbur@gmail.com"
+aliases = []
+full_name = "Richard Wilbur"
+
+[people."hendrik"]
+email = "hendrik@topoi.pooq.com"
+aliases = []
+full_name = "Hendrik Boom"
+
+[people."klehman"]
+email = "klehman9@comcast.net"
+aliases = []
+full_name = "Kyle Lehman"
+
+[people."toshaan"]
+email = "toshaan@vantosh.com"
+aliases = []
+full_name = "Toshaan Bharvani"
+
+[people."vantosh"]
+email = "info@vantosh.com"
aliases = []
-output_markdown_file = "dimitri_galayko.mdwn"
+full_name = "Vantosh bv"
[milestones]
-"NLnet.2019.02" = { canonical_bug_id = 191 }
+"NLnet.2019.02.012" = { canonical_bug_id = 191 }
"NLnet.2019.10.Cells" = { canonical_bug_id = 153 }
-"NLNet.2019.10.Formal" = { canonical_bug_id = 158 }
-"NLNet.2019.10.Standards" = { canonical_bug_id = 174 }
-"NLNet.2019.10.Wishbone" = { canonical_bug_id = 175 }
-"NLNet.2019.Coriolis2" = { canonical_bug_id = 138 }
-"NLNet.2019.Video" = { canonical_bug_id = 137 }
-"NLNet.2019.Vulkan" = { canonical_bug_id = 140 }
+"NLNet.2019.10.032.Formal" = { canonical_bug_id = 158 }
+"NLNet.2019.10.046.Standards" = { canonical_bug_id = 174 }
+"NLNet.2019.10.043.Wishbone" = { canonical_bug_id = 175 }
+"NLNet.2019.02.029.Coriolis2" = { canonical_bug_id = 138 }
+"NLNet.2019.10.031.Video" = { canonical_bug_id = 137 }
+"NLNet.2019.10.042.Vulkan" = { canonical_bug_id = 140 }
+"NLnet.2021.02A.052.CryptoRouter" = { canonical_bug_id = 589 }
+"NGI.POINTER.Gigabit.ASIC" = { canonical_bug_id = 690 }
+"NLnet.2021-08-049.coriolis2" = { canonical_bug_id = 748 }
+"NLnet.2021-08-071.cavatools" = { canonical_bug_id = 939 }
+"NLnet.2022-08-051.OPF" = { canonical_bug_id = 952 }
+"NLnet.2022-08-107.ongoing" = { canonical_bug_id = 961 }
+"NLnet.2023-12-059.expansion" = { canonical_bug_id = 1211 }
+"NLnet.2023-12-121.binutils" = { canonical_bug_id = 1212 }
+"NGI.SEARCH" = { canonical_bug_id = 1097 }
+"new binutils" = { canonical_bug_id = 1259 }
"Future" = { canonical_bug_id = 487 }
--- /dev/null
+# list of allowed signers for use with signing git commits using ssh keys.
+# links explaining how to set it up (GitLab *not* necessary to use):
+# https://docs.gitlab.com/ee/user/project/repository/ssh_signed_commits/#configure-git-to-sign-commits-with-your-ssh-key
+# https://docs.gitlab.com/ee/user/project/repository/ssh_signed_commits/#verify-commits-locally
+#
+# add to your git config, replacing the path with the path to this file:
+# git config gpg.ssh.allowedSignersFile "$(realpath ../path/to/utils.git/git-ssh-allowed-signers)"
+#
+# file format:
+# https://man7.org/linux/man-pages/man1/ssh-keygen.1.html#ALLOWED_SIGNERS
+#
+# add your ssh public keys one per line, don't forget namespaces="git"
+#
+programmerjake@gmail.com namespaces="git" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6PKJf11y3Zy092zLoJ4g0WVTQ8/e4cPki0XN8+iDwlaVL9klb3sB9PZYakD4AZyPQ5MxWVt1QnR2qXgnQ4laGlyw5ho1BM7zBpbUD08aCfTiBO1Fms52ci3+2sTDkStHJ62PaPa4IuezTBoHkR1M1FUVcOUWAAIpH7BHoP4Ayb0uUgAf8AxZtOcEZa+w0EwDOB0DRPFYWE2vu2q/aJHLvQHwGikeg5bi+o0odam+lkg0ShedBzSWqBP/Mxn65p+xQ2q0FViRZWBpyOhEN4hi2kF/2L/yinbQVuhIU5C1VVb5cjqClD0YA+Ln/DPhXcvw2hzSKxZKlDHKa7Gv3os/H jacob@jacob-desktop
+programmerjake@gmail.com namespaces="git" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCVhwvXwttucaxlfGEGCahhlfLbbe0yBJD48z9r34R2Yq+y0/c02xgn59TiO+OrRd/HPbGpKta0mHN4wnI8tCO7B/iAEK6ECHrSlYSHUdgs15RRb4C+z49nEmVgo9L7RP6SPzLdhmXAadwivfCXNzX7nAYuTM9toRyhD/2Tn/LpGaW5w2R4+4xtWA1DksHRpkRmX5QWrwiE2FgcEwQBHZz/tVBgK49zoWBHlR4ak+gMK/45W0H/jCPP5wLKkF//I1Ov7sE+QEDFH+5qaxCh3u/ww+n/v6291xwzctSdrTikuqyyQ+1Y/wQFiqsQBgii3121ZesSIUAl/hmqXKpAgk4J jacob@jacob-HP-ProBook-6470b
install_requires = [
"python-bugzilla",
"toml",
+ "cached-property",
]
setup(
+from budget_sync.ordered_set import OrderedSet
from bugzilla.bug import Bug
-from typing import Set, Dict, Iterable, Optional, List, Union, Any
-from budget_sync.util import BugStatus
+from typing import Callable, Set, Dict, Iterable, Optional, List, Tuple, Union, Any
+from budget_sync.util import BugStatus, PrettyPrinter
from budget_sync.money import Money
from budget_sync.config import Config, Person, Milestone
-from functools import cached_property
import toml
import sys
import enum
from collections import deque
from datetime import date, time, datetime
+try:
+ from functools import cached_property
+except ImportError: # :nocov:
+ # compatibility with python < 3.8
+ from cached_property import cached_property # :nocov:
class BudgetGraphBaseError(Exception):
f"submitted={str(self.submitted)})")
+@enum.unique
+class PaymentSummaryState(enum.Enum):
+ Submitted = PayeeState.Submitted
+ Paid = PayeeState.Paid
+ NotYetSubmitted = PayeeState.NotYetSubmitted
+ Inconsistent = None
+
+
+class PaymentSummary:
+ total_submitted: Money
+ """includes amount paid"""
+
+ def __init__(self, payments: Iterable[Payment]):
+ self.payments = tuple(payments)
+ self.total = Money(0)
+ self.total_paid = Money(0)
+ self.total_submitted = Money(0)
+ self.submitted_date = None
+ self.paid_date = None
+ self.not_submitted = []
+ summary_state = None
+ for payment in self.payments:
+ if summary_state is None:
+ summary_state = PaymentSummaryState(payment.state)
+ self.submitted_date = payment.submitted
+ self.paid_date = payment.paid
+ elif summary_state != PaymentSummaryState(payment.state) \
+ or self.submitted_date != payment.submitted \
+ or self.paid_date != payment.paid:
+ summary_state = PaymentSummaryState.Inconsistent
+ self.paid_date = None
+ self.submitted_date = None
+ self.total += payment.amount
+ if payment.state is PayeeState.Submitted:
+ self.total_submitted += payment.amount
+ elif payment.state is PayeeState.Paid:
+ self.total_submitted += payment.amount
+ self.total_paid += payment.amount
+ else:
+ assert payment.state is PayeeState.NotYetSubmitted
+ self.not_submitted.append(payment.node.bug.id)
+ if summary_state is None:
+ self.state = PaymentSummaryState.NotYetSubmitted
+ else:
+ self.state = summary_state
+
+ def get_not_submitted(self):
+ return self.not_submitted
+
+ def __repr__(self) -> str:
+ return (f"PaymentSummary(total={self.total}, "
+ f"total_paid={self.total_paid}, "
+ f"total_submitted={self.total_submitted}, "
+ f"submitted_date={self.submitted_date}, "
+ f"paid_date={self.paid_date}, "
+ f"state={self.state}, "
+ f"payments={self.payments})")
+
+ def __pretty_print__(self, pp: PrettyPrinter):
+ with pp.type_pp("PaymentSummary") as tpp:
+ tpp.field("total", self.total)
+ tpp.field("total_submitted", self.total_submitted)
+ tpp.field("submitted_date", self.submitted_date)
+ tpp.field("paid_date", self.paid_date)
+ tpp.field("state", self.state)
+ tpp.field("payments", self.payments)
+
+
class BudgetGraphUnknownMilestone(BudgetGraphParseError):
def __init__(self, bug_id: int, milestone_str: str):
super().__init__(bug_id)
graph: "BudgetGraph"
bug: Bug
parent_id: Optional[int]
- immediate_children: Set["Node"]
+ immediate_children: OrderedSet["Node"]
budget_excluding_subtasks: Money
budget_including_subtasks: Money
fixed_budget_excluding_subtasks: Money
self.graph = graph
self.bug = bug
self.parent_id = getattr(bug, "cf_budget_parent", None)
- self.immediate_children = set()
+ self.immediate_children = OrderedSet()
self.budget_excluding_subtasks = Money.from_str(bug.cf_budget)
self.fixed_budget_excluding_subtasks = self.budget_excluding_subtasks
self.budget_including_subtasks = Money.from_str(bug.cf_total_budget)
@cached_property
def milestone(self) -> Optional[Milestone]:
- try:
- if self.milestone_str is not None:
- return self.graph.config.milestones[self.milestone_str]
+ if self.milestone_str is None:
return None
+ try:
+ return self.graph.config.milestones[self.milestone_str]
except KeyError:
new_err = BudgetGraphUnknownMilestone(
self.bug.id, self.milestone_str)
retval[key] = Payment._from_toml(self, key, value)
return retval
+ @cached_property
+ def resolved_payments(self) -> Dict[Person, List[Payment]]:
+ retval: Dict[Person, List[Payment]] = {}
+ for payment in self.payments.values():
+ if payment.payee not in retval:
+ retval[payment.payee] = []
+ retval[payment.payee].append(payment)
+ return retval
+
+ @cached_property
+ def payment_summaries(self) -> Dict[Person, PaymentSummary]:
+ return {person: PaymentSummary(payments)
+ for person, payments in self.resolved_payments.items()}
+
+ @cached_property
+ def submitted_excluding_subtasks(self) -> Money:
+ retval = Money()
+ for payment in self.payments.values():
+ if payment.submitted is not None or payment.paid is not None:
+ retval += payment.amount
+ return retval
+
+ @cached_property
+ def paid_excluding_subtasks(self) -> Money:
+ retval = Money()
+ for payment in self.payments.values():
+ if payment.paid is not None:
+ retval += payment.amount
+ return retval
+
+ @cached_property
+ def submitted_including_subtasks(self) -> Money:
+ retval = self.submitted_excluding_subtasks
+ for i in self.immediate_children:
+ retval += i.submitted_including_subtasks
+ return retval
+
+ @cached_property
+ def paid_including_subtasks(self) -> Money:
+ retval = self.paid_excluding_subtasks
+ for i in self.immediate_children:
+ retval += i.paid_including_subtasks
+ return retval
+
@property
def parent(self) -> Optional["Node"]:
if self.parent_id is not None:
self._raise_loop_error()
return retval
+ @cached_property
+ def is_in_nlnet_mou(self):
+ """returns true if this bugreport is an immediate child of a top-level
+ milestone. it does *not* return true for the top-level bug itself
+ because only the immediate children comprise the MoU.
+ """
+ try:
+ if self.parent is not None and self.milestone is not None:
+ return self.parent.bug.id == self.milestone.canonical_bug_id
+ except BudgetGraphBaseError:
+ pass
+ return False
+
+ @cached_property
+ def closest_bug_in_mou(self) -> Optional["Node"]:
+ """returns the closest bug that is in a NLNet MoU, searching only in
+ this bug and parents.
+ """
+ if self.is_in_nlnet_mou:
+ return self
+ for parent in self.parents():
+ if parent.is_in_nlnet_mou:
+ return parent
+ return None
+
def children(self) -> Iterable["Node"]:
def visitor(node: Node) -> Iterable[Node]:
for i in node.immediate_children:
yield node
def __eq__(self, other):
- return self.bug.id == other.bug.id
+ if isinstance(other, Node):
+ return self.bug.id == other.bug.id
+ return NotImplemented
def __hash__(self):
return self.bug.id
+ def __pretty_print__(self, pp: PrettyPrinter):
+ with pp.type_pp("Node") as tpp:
+ tpp.field("graph", ...)
+ tpp.field("id", _NodeSimpleReprWrapper(self))
+ tpp.try_field("root",
+ lambda: _NodeSimpleReprWrapper(self.root),
+ BudgetGraphLoopError)
+ parent = f"#{self.parent_id}" if self.parent_id is not None else None
+ tpp.field("parent", parent)
+ tpp.field("budget_excluding_subtasks",
+ self.budget_excluding_subtasks)
+ tpp.field("budget_including_subtasks",
+ self.budget_including_subtasks)
+ tpp.field("fixed_budget_excluding_subtasks",
+ self.fixed_budget_excluding_subtasks)
+ tpp.field("fixed_budget_including_subtasks",
+ self.fixed_budget_including_subtasks)
+ tpp.field("milestone_str", self.milestone_str)
+ tpp.field("is_in_nlnet_mou", self.is_in_nlnet_mou)
+ tpp.try_field("milestone", lambda: self.milestone,
+ BudgetGraphBaseError)
+ immediate_children = [_NodeSimpleReprWrapper(i)
+ for i in self.immediate_children]
+ tpp.field("immediate_children", immediate_children)
+ tpp.try_field("payments",
+ lambda: list(self.payments.values()),
+ BudgetGraphBaseError)
+ try:
+ status = repr(self.status)
+ except BudgetGraphBaseError:
+ status = f"<unknown status: {self.bug.status!r}>"
+ tpp.field("status", status)
+ try:
+ assignee = f"Person<{self.assignee.identifier!r}>"
+ except BudgetGraphBaseError:
+ assignee = f"<unknown assignee: {self.bug.assigned_to!r}>"
+ tpp.field("assignee", assignee)
+ tpp.try_field("resolved_payments",
+ lambda: self.resolved_payments,
+ BudgetGraphBaseError)
+ tpp.try_field("payment_summaries",
+ lambda: self.payment_summaries,
+ BudgetGraphBaseError)
+
def __repr__(self):
try:
root = _NodeSimpleReprWrapper(self.root)
immediate_children.sort()
parent = f"#{self.parent_id}" if self.parent_id is not None else None
payments = list(self.payments.values())
+ resolved_payments = self.resolved_payments
+ payment_summaries = self.payment_summaries
return (f"Node(graph=..., "
f"id={_NodeSimpleReprWrapper(self)}, "
f"root={root}, "
f"fixed_budget_excluding_subtasks={self.fixed_budget_excluding_subtasks}, "
f"fixed_budget_including_subtasks={self.fixed_budget_including_subtasks}, "
f"milestone_str={self.milestone_str!r}, "
+ f"is_in_nlnet_mou={self.is_in_nlnet_mou!r}, "
f"milestone={milestone}, "
f"immediate_children={immediate_children!r}, "
f"payments={payments!r}, "
f"status={status}, "
- f"assignee={assignee})")
+ f"assignee={assignee}, "
+ f"resolved_payments={resolved_payments!r}, "
+ f"payment_summaries={payment_summaries!r})")
class BudgetGraphError(BudgetGraphBaseError):
f"bug #{self.bug_id}, payee {self.payee_key!r}")
-class BudgetGraphDuplicatePayeesForTask(BudgetGraphError):
- def __init__(self, bug_id: int, root_bug_id: int, payee1_key: str, payee2_key: str):
- super().__init__(bug_id, root_bug_id)
- self.payee1_key = payee1_key
- self.payee2_key = payee2_key
-
- def __str__(self):
- return (f"Budget assigned to multiple aliases of the same person in "
- f"a single task: bug #{self.bug_id}, budget assigned to both "
- f"{self.payee1_key!r} and {self.payee2_key!r}")
-
-
class BudgetGraphIncorrectRootForMilestone(BudgetGraphError):
def __init__(self, bug_id: int, milestone: str, milestone_canonical_bug_id: int):
super().__init__(bug_id, bug_id)
if node.parent is None:
continue
node.parent.immediate_children.add(node)
+ # useful debug prints
+ # for bug in bugs:
+ # node = self.nodes[bug.id]
+ # print ("bug added", bug.id, node, node.parent.immediate_children)
@cached_property
- def roots(self) -> Set[Node]:
- roots = set()
+ def roots(self) -> OrderedSet[Node]:
+ roots = OrderedSet()
for node in self.nodes.values():
# calling .root also checks for loop errors
root = node.root
try:
# check for milestone errors
node.milestone
- if root == node and node.milestone is not None \
- and node.milestone.canonical_bug_id != node.bug.id:
- if node.budget_including_subtasks != 0 \
- or node.budget_excluding_subtasks != 0:
- errors.append(BudgetGraphIncorrectRootForMilestone(
- node.bug.id, node.milestone.identifier,
- node.milestone.canonical_bug_id
- ))
+ if root == node and node.milestone is not None:
+ if node.milestone.canonical_bug_id != node.bug.id:
+ if node.budget_including_subtasks != 0 \
+ or node.budget_excluding_subtasks != 0:
+ errors.append(BudgetGraphIncorrectRootForMilestone(
+ node.bug.id, node.milestone.identifier,
+ node.milestone.canonical_bug_id
+ ))
except BudgetGraphBaseError as e:
errors.append(e)
errors.append(BudgetGraphNegativeMoney(
node.bug.id, root.bug.id))
+ childlist = []
subtasks_total = Money(0)
for child in node.immediate_children:
subtasks_total += child.fixed_budget_including_subtasks
+ childlist.append(child.bug.id)
+ # useful debug prints
+ # print ("subtask total", node.bug.id, root.bug.id, subtasks_total,
+ # childlist)
payees_total = Money(0)
- payee_payments = {}
+ payee_payments: Dict[Person, List[Payment]] = {}
for payment in node.payments.values():
if payment.amount < 0:
errors.append(BudgetGraphNegativePayeeMoney(
payment.payee
previous_payment = payee_payments.get(payment.payee)
if previous_payment is not None:
- errors.append(BudgetGraphDuplicatePayeesForTask(
- node.bug.id, root.bug.id,
- previous_payment.payee_key, payment.payee_key
- ))
- payee_payments[payment.payee] = payment
+ payee_payments[payment.payee].append(payment)
+ else:
+ payee_payments[payment.payee] = [payment]
except BudgetGraphBaseError as e:
errors.append(e)
node.fixed_budget_including_subtasks))
def set_excluding_from_including_and_error():
- node.fixed_budget_excluding_subtasks = \
- node.budget_including_subtasks - subtasks_total
+ v = node.budget_including_subtasks - subtasks_total
+ if v < 0:
+ return set_including_from_excluding_and_error()
+ node.fixed_budget_excluding_subtasks = v
errors.append(
BudgetGraphMoneyMismatchForBudgetExcludingSubtasks(
node.bug.id, root.bug.id,
@cached_property
def assigned_nodes(self) -> Dict[Person, List[Node]]:
+ retval: Dict[Person, List[Node]]
retval = {person: [] for person in self.config.people.values()}
for node in self.nodes.values():
retval[node.assignee].append(node)
return retval
+ @cached_property
+ def assigned_nodes_for_milestones(self) -> Dict[Milestone, List[Node]]:
+ retval: Dict[Milestone, List[Node]]
+ retval = {milestone: []
+ for milestone in self.config.milestones.values()}
+ for node in self.nodes.values():
+ if node.milestone is not None:
+ retval[node.milestone].append(node)
+ return retval
+
+ @cached_property
+ def milestone_payments(self) -> Dict[Milestone, List[Payment]]:
+ retval: Dict[Milestone, List[Payment]] = {
+ milestone: [] for milestone in self.config.milestones.values()
+ }
+ for node in self.nodes.values():
+ if node.milestone is not None:
+ retval[node.milestone].extend(node.payments.values())
+ return retval
+
@cached_property
def payments(self) -> Dict[Person, Dict[Milestone, List[Payment]]]:
- retval = {}
- for person in self.config.people.values():
- milestone_payments = {}
- for milestone in self.config.milestones.values():
- milestone_payments[milestone] = []
- retval[person] = milestone_payments
+ retval: Dict[Person, Dict[Milestone, List[Payment]]] = {
+ person: {
+ milestone: []
+ for milestone in self.config.milestones.values()
+ }
+ for person in self.config.people.values()
+ }
for node in self.nodes.values():
if node.milestone is not None:
for payment in node.payments.values():
retval[payment.payee][node.milestone].append(payment)
return retval
+ @cached_property
+ def milestone_people(self) -> Dict[Milestone, OrderedSet[Person]]:
+ """get a list of people associated with each milestone
+ """
+ payments = list(self.payments) # just activate the payments
+ retval = {}
+ for milestone in self.milestone_payments.keys():
+ retval[milestone] = OrderedSet()
+ for milestone, payments in self.milestone_payments.items():
+ for payment in payments:
+ retval[milestone].add(payment.payee)
+ return retval
+
+ def __pretty_print__(self, pp: PrettyPrinter):
+ with pp.type_pp("BudgetGraph") as tpp:
+ tpp.field("nodes", self.nodes)
+ tpp.try_field("roots",
+ lambda: [_NodeSimpleReprWrapper(i)
+ for i in self.roots],
+ BudgetGraphBaseError)
+ tpp.try_field("assigned_nodes",
+ lambda: {
+ person: [
+ _NodeSimpleReprWrapper(node)
+ for node in nodes
+ ]
+ for person, nodes in self.assigned_nodes.items()
+ },
+ BudgetGraphBaseError)
+ tpp.try_field("assigned_nodes_for_milestones",
+ lambda: {
+ milestone: [
+ _NodeSimpleReprWrapper(node)
+ for node in nodes
+ ]
+ for milestone, nodes in self.assigned_nodes_for_milestones.items()
+ },
+ BudgetGraphBaseError)
+ tpp.try_field("payments",
+ lambda: self.payments, BudgetGraphBaseError)
+ tpp.try_field("milestone_people",
+ lambda: self.milestone_people,
+ BudgetGraphBaseError)
+
def __repr__(self):
nodes = [*self.nodes.values()]
+
+ def repr_or_failed(f: Callable[[], Any]) -> str:
+ try:
+ return repr(f())
+ except BudgetGraphBaseError:
+ return "<failed>"
+
try:
roots = [_NodeSimpleReprWrapper(i) for i in self.roots]
roots.sort()
roots_str = repr(roots)
except BudgetGraphBaseError:
roots_str = "<failed>"
- return f"BudgetGraph{{nodes={nodes!r}, roots={roots}}}"
+ assigned_nodes = repr_or_failed(lambda: {
+ person: [
+ _NodeSimpleReprWrapper(node)
+ for node in nodes
+ ]
+ for person, nodes in self.assigned_nodes.items()
+ })
+ assigned_nodes_for_milestones = repr_or_failed(lambda: {
+ milestone: [
+ _NodeSimpleReprWrapper(node)
+ for node in nodes
+ ]
+ for milestone, nodes in self.assigned_nodes_for_milestones.items()
+ })
+ milestone_payments = repr_or_failed(lambda: self.milestone_payments)
+ payments = repr_or_failed(lambda: self.payments)
+ milestone_people = repr_or_failed(lambda: self.milestone_people)
+ return (f"BudgetGraph{{nodes={nodes!r}, "
+ f"roots={roots}, "
+ f"assigned_nodes={assigned_nodes}, "
+ f"assigned_nodes_for_milestones={assigned_nodes_for_milestones}, "
+ f"milestone_payments={milestone_payments}, "
+ f"payments={payments}, "
+ f"milestone_people={milestone_people}}}")
+from budget_sync.ordered_set import OrderedSet
+from budget_sync.util import PrettyPrinter
import toml
import sys
-from typing import Set, Dict, Any, Optional
-from functools import cached_property
+from typing import Mapping, Set, Dict, Any, Optional
+try:
+ from functools import cached_property
+except ImportError:
+ # compatability with python < 3.8
+ from cached_property import cached_property
class ConfigParseError(Exception):
class Person:
- aliases: Set[str]
+ aliases: OrderedSet[str]
email: Optional[str]
def __init__(self, config: "Config", identifier: str,
- output_markdown_file: str,
- aliases: Optional[Set[str]] = None,
+ full_name: str,
+ aliases: Optional[OrderedSet[str]] = None,
email: Optional[str] = None):
self.config = config
self.identifier = identifier
- self.output_markdown_file = output_markdown_file
if aliases is None:
- aliases = set()
+ aliases = OrderedSet()
+ else:
+ assert isinstance(aliases, OrderedSet)
self.aliases = aliases
self.email = email
+ self.full_name = full_name
@cached_property
- def all_names(self) -> Set[str]:
- retval = self.aliases.copy()
+ def all_names(self) -> OrderedSet[str]:
+ retval = OrderedSet(self.aliases)
retval.add(self.identifier)
+ retval.add(self.full_name)
if self.email is not None:
retval.add(self.email)
return retval
+ @cached_property
+ def output_markdown_file(self) -> str:
+ return self.identifier + '.mdwn'
+
def __eq__(self, other):
return self.identifier == other.identifier
def __hash__(self):
return hash(self.identifier)
+ def __pretty_print__(self, pp: PrettyPrinter):
+ with pp.type_pp("Person") as tpp:
+ tpp.field("config", ...)
+ tpp.field("identifier", self.identifier)
+
def __repr__(self):
return (f"Person(config=..., identifier={self.identifier!r}, "
- f"output_markdown_file={self.output_markdown_file!r}, "
+ f"full_name={self.full_name!r}, "
f"aliases={self.aliases!r}, email={self.email!r})")
for name in person.all_names:
other_person = retval.get(name)
if other_person is not None and other_person is not person:
- alias_or_email = "alias"
+ alias_or_email_or_full_name = "alias"
if name == person.email:
- alias_or_email = "email"
+ alias_or_email_or_full_name = "email"
+ if name == person.full_name:
+ alias_or_email_or_full_name = "full_name"
if name in self.people:
raise ConfigParseError(
- f"{alias_or_email} is not allowed to be the same "
+ f"{alias_or_email_or_full_name} is not allowed "
+ f"to be the same "
f"as any person's identifier: in person entry for "
f"{person.identifier!r}: {name!r} is also the "
f"identifier for person"
f" {other_person.identifier!r}")
raise ConfigParseError(
- f"{alias_or_email} is not allowed to be the same as "
- f"another person's alias or email: in person entry "
+ f"{alias_or_email_or_full_name} is not allowed "
+ f"to be the same as "
+ f"another person's alias, email, or full_name: "
+ f"in person entry "
f"for {person.identifier!r}: {name!r} is also an alias"
- f" or email for person {other_person.identifier!r}")
+ f", email, or full_name for person "
+ f"{other_person.identifier!r}")
retval[name] = person
return retval
def canonical_bug_ids(self) -> Dict[int, Milestone]:
# also checks for any bug id clashes and raises
# ConfigParseError if any are detected
- retval = {}
+ retval: Dict[int, Milestone] = {}
for milestone in self.milestones.values():
other_milestone = retval.get(milestone.canonical_bug_id)
if other_milestone is not None:
if not isinstance(value, dict):
raise ConfigParseError(
f"person entry for {identifier!r} must be a table")
- aliases = set()
+ aliases = OrderedSet()
email = None
- output_markdown_file = None
+ full_name = None
for k, v in value.items():
assert isinstance(k, str)
if k == "aliases":
f"`email` field in person entry for {identifier!r} "
f"must be a string")
email = v
- elif k == "output_markdown_file":
+ elif k == "full_name":
if not isinstance(v, str):
raise ConfigParseError(
- f"`output_markdown_file` field in person entry for "
+ f"`full_name` field in person entry for "
f"{identifier!r} must be a string")
- output_markdown_file = v
+ full_name = v
else:
raise ConfigParseError(
f"unknown field in person entry for {identifier!r}: `{k}`")
- if output_markdown_file is None:
- raise ConfigParseError(f"`output_markdown_file` field is missing in "
+ if full_name is None:
+ raise ConfigParseError(f"`full_name` field is missing in "
f"person entry for {identifier!r}")
return Person(config=self, identifier=identifier,
- output_markdown_file=output_markdown_file,
+ full_name=full_name,
aliases=aliases, email=email)
def _parse_people(self, people: Any):
self.canonical_bug_ids
@staticmethod
- def _from_toml(parsed_toml: Dict[str, Any]) -> "Config":
+ def _from_toml(parsed_toml: Mapping[str, Any]) -> "Config":
people = None
bugzilla_url = None
milestones = None
--- /dev/null
+from budget_sync.util import all_bugs
+from budget_sync.config import Config, ConfigParseError
+from bugzilla import Bugzilla
+import logging
+import argparse
+from pathlib import Path
+import json
+from xmlrpc.client import DateTime
+
+
+def _encode_json(obj):
+ if isinstance(obj, DateTime):
+ return str(obj)
+ raise TypeError(type(obj))
+
+
+def main():
+ logging.basicConfig(level=logging.INFO)
+ parser = argparse.ArgumentParser(
+ description="Dump all bugzilla data accessible from API")
+ parser.add_argument(
+ "-c", "--config", type=argparse.FileType('r'),
+ required=True, help="The path to the configuration TOML file",
+ dest="config", metavar="<path/to/budget-sync-config.toml>")
+ parser.add_argument(
+ "-o", "--output-dir", type=Path, required=True,
+ help="The path to the output directory, will be created if it "
+ "doesn't exist",
+ dest="output_dir", metavar="<path/to/output/dir>")
+ args = parser.parse_args()
+ try:
+ with args.config as config_file:
+ config = Config.from_file(config_file)
+ except (IOError, ConfigParseError) as e:
+ logging.error("Failed to parse config file: %s", e)
+ return
+ logging.info("Using Bugzilla instance at %s", config.bugzilla_url)
+ bz = Bugzilla(config.bugzilla_url)
+ output_dir = args.output_dir
+ output_dir.mkdir(parents=True, exist_ok=True)
+ for bug in all_bugs(bz):
+ bug_json = {}
+ bug_json_path = output_dir / ("bug-%d.json" % (bug.id,))
+ bug_json["data"] = bug.get_raw_data()
+ bug_json["history"] = bug.get_history_raw()
+ bug_json["comments"] = bug.getcomments()
+ attachments = [a.copy() for a in bug.get_attachments()]
+ bug_json["attachments"] = attachments
+ for a in attachments:
+ data = a.pop("data", None)
+ if data is None:
+ continue
+ file_name = "attachment-%d.dat" % (a['id'],)
+ a['data'] = file_name
+ attachment_path = output_dir / file_name
+ attachment_path.write_bytes(data.data)
+ logging.info("Wrote %s", attachment_path)
+ bug_json = json.dumps(bug_json, indent=4, default=_encode_json)
+ bug_json_path.write_text(bug_json, encoding="utf-8")
+ logging.info("Wrote %s", bug_json_path)
+
+
+if __name__ == "__main__":
+ main()
+import os
+import re
+import sys
+import json
+from typing import Optional
+from budget_sync.ordered_set import OrderedSet
+from budget_sync.write_budget_csv import write_budget_csv
from bugzilla import Bugzilla
import logging
import argparse
from pathlib import Path
-from budget_sync.util import all_bugs
+from budget_sync.util import all_bugs, tty_out
from budget_sync.config import Config, ConfigParseError
-from budget_sync.budget_graph import BudgetGraph, BudgetGraphBaseError
-from budget_sync.write_budget_markdown import write_budget_markdown
+from budget_sync.budget_graph import BudgetGraph, PaymentSummary
+from budget_sync.write_budget_markdown import (write_budget_markdown,
+ markdown_for_person)
+def spc(s):
+ return s # not needed when using <pre> instead of <tt>
+ return s.replace(" ", " ")
+
+bugurl = "https://bugs.libre-soc.org/show_bug.cgi?id="
def main():
parser = argparse.ArgumentParser(
help="The path to the output directory, will be created if it "
"doesn't exist",
dest="output_dir", metavar="<path/to/output/dir>")
+ parser.add_argument('--subset',
+ help="write the output for this subset of bugs",
+ metavar="<bug-id>,<bug-id>,...")
+ parser.add_argument('--subset-person',
+ help="write the output for this person",
+ dest="person")
+ parser.add_argument('--username',
+ help="Log in with this Bugzilla username")
+ parser.add_argument('--password',
+ help="Log in with this Bugzilla password")
+ parser.add_argument('--comments', action='store_true',
+ help="Put JSON into comments")
+ parser.add_argument('--detail', action='store_true',
+ help="Add detail to report (links description people)")
args = parser.parse_args()
try:
with args.config as config_file:
logging.error("Failed to parse config file: %s", e)
return
logging.info("Using Bugzilla instance at %s", config.bugzilla_url)
- bz = Bugzilla(config.bugzilla_url)
- logging.debug("Connected to Bugzilla")
- budget_graph = BudgetGraph(all_bugs(bz), config)
- for error in budget_graph.get_errors():
- logging.error("%s", error)
- if args.output_dir is not None:
- write_budget_markdown(budget_graph, args.output_dir)
+ # download all bugs and create a summary report
+ reportdir = "./" if args.output_dir is None else str(args.output_dir)
+ reportdir = "/".join(reportdir.split("/")[:-1]) # strip /mdwn
+ reportname = reportdir + "/report.mdwn"
+ with open(reportname, "w") as f:
+ print("<pre>", file=f) # for using the output as markdown
+ bz = Bugzilla(config.bugzilla_url)
+ if args.username:
+ logging.debug("logging in...")
+ bz.interactive_login(args.username, args.password)
+ logging.debug("Connected to Bugzilla")
+ budget_graph = BudgetGraph(all_bugs(bz), config)
+ for error in budget_graph.get_errors():
+ logging.error("%s", error)
+ if args.person or args.subset:
+ if not args.person:
+ logging.fatal("must use --subset-person with --subset option")
+ sys.exit(1)
+ print_markdown_for_person(f, budget_graph, config,
+ args.person, args.subset)
+ return
+ if args.output_dir is not None:
+ write_budget_markdown(budget_graph, args.output_dir)
+ write_budget_csv(budget_graph, args.output_dir)
+ summarize_milestones(f, budget_graph, detail=args.detail)
+ print("</pre>", file=f) # for using the output as markdown
+ # now create the JSON milestone files for putting into NLnet RFP system
+ json_milestones(budget_graph, args.comments, args.output_dir)
+
+
+def print_markdown_for_person(f, budget_graph, config, person_str, subset_str):
+ person = config.all_names.get(person_str)
+ if person is None:
+ logging.fatal("--subset-person: unknown person: %s", person_str)
+ sys.exit(1)
+ nodes_subset = None
+ if subset_str:
+ nodes_subset = OrderedSet()
+ for bug_id in re.split(r"[\s,]+", subset_str):
+ try:
+ node = budget_graph.nodes[int(bug_id)]
+ except (ValueError, KeyError):
+ logging.fatal("--subset: unknown bug: %s", bug_id)
+ sys.exit(1)
+ nodes_subset.add(node)
+ print(markdown_for_person(budget_graph, person, nodes_subset), file=f)
+
+
+def print_budget_then_children(f, indent, nodes, bug_id, detail=False):
+ """recursive indented printout of budgets
+ """
+
+ bug = nodes[bug_id]
+ b_incl = str(bug.fixed_budget_including_subtasks)
+ b_excl = str(bug.fixed_budget_excluding_subtasks)
+ s_incl = str(bug.submitted_including_subtasks)
+ p_incl = str(bug.paid_including_subtasks)
+ if b_incl == s_incl and b_incl == p_incl:
+ descr = "(s+p)"
+ elif b_incl == s_incl:
+ descr = "(s) p %s" % p_incl
+ elif b_incl == p_incl:
+ descr = "(p) s %s" % s_incl
+ elif s_incl == p_incl:
+ descr = " s,p %s" % (p_incl)
+ else:
+ descr = "s %s p %s" % (s_incl, p_incl)
+ excl_desc = " "
+ if b_incl != b_excl:
+ excl_desc = "excltasks %6s" % b_excl
+ bugid = spc("%5d" % bug.bug.id)
+ buglink = "<a href='%s%d'>%s</a>" % (bugurl, bug.bug.id, bugid)
+ print(spc("bug #URL %s budget %6s %s %s" %
+ (' | ' * indent,
+ b_incl,
+ excl_desc,
+ descr
+ )).replace("URL", buglink), file=f)
+ if detail:
+ print(spc(" %s | (%s)" %
+ (' | ' * indent,
+ bug.bug.summary[:40]
+ )), file=f)
+ # print(repr(bug))
+
+ for child in bug.immediate_children:
+ if (str(child.budget_including_subtasks) == "0" and
+ str(child.budget_excluding_subtasks) == "0"):
+ continue
+ print_budget_then_children(f, indent+1, nodes, child.bug.id, detail)
+
+
+def summarize_milestones(f, budget_graph, detail=False):
+ for milestone, payments in budget_graph.milestone_payments.items():
+ summary = PaymentSummary(payments)
+ print(f"{milestone.identifier}", file=f)
+ print(f" {summary.total} submitted: "
+ f"{summary.total_submitted} paid: {summary.total_paid}",
+ file=f)
+ not_submitted = summary.get_not_submitted()
+ if not_submitted:
+ print("not submitted %s" % not_submitted, file=f)
+
+ # and one to display people
+ for person in budget_graph.milestone_people[milestone]:
+ ident = "<a href='../mdwn/%s/'>%s</a>"
+ ident %= (person.identifier, person.identifier)
+ print(spc(" %-30s - %s" % (ident, person.full_name)), file=f)
+ print("", file=f)
+
+ # now do trees
+ for milestone, payments in budget_graph.milestone_payments.items():
+ print("%s %d" % (milestone.identifier, milestone.canonical_bug_id),
+ file=f)
+ print_budget_then_children(f, 0, budget_graph.nodes,
+ milestone.canonical_bug_id, detail)
+ print("", file=f)
+
+
+def json_milestones(budget_graph: BudgetGraph, add_comments: bool,
+ output_dir: Path):
+ """reports milestones as json format
+ """
+ bug_comments_map = {}
+ if add_comments:
+ need_set = set()
+ bugzilla = None
+ for nodes in budget_graph.assigned_nodes_for_milestones.values():
+ for node in nodes:
+ need_set.add(node.bug.id)
+ bugzilla = node.bug.bugzilla
+ need_list = sorted(need_set)
+ total = len(need_list)
+ with tty_out() as term:
+ step = 100
+ i = 0
+ while i < total:
+ cur_need = need_list[i:i + step]
+ stop = i + len(cur_need)
+ print("loading comments %d:%d of %d" % (i, stop, total),
+ flush=True, file=term)
+ comments = bugzilla.get_comments(cur_need)['bugs']
+ if len(comments) < len(cur_need) and len(cur_need) > 1:
+ step = max(1, step // 2)
+ print("failed, trying smaller step of %d" % step,
+ flush=True, file=term)
+ continue
+ bug_comments_map.update(comments)
+ i += len(cur_need)
+ for milestone, payments in budget_graph.milestone_payments.items():
+ summary = PaymentSummary(payments)
+ # and one to display people
+ ppl = []
+ for person in budget_graph.milestone_people[milestone]:
+ p = {'name': person.full_name, 'email': person.email}
+ ppl.append(p)
+
+ tasks = []
+ canonical = budget_graph.nodes[milestone.canonical_bug_id]
+ for child in canonical.immediate_children:
+ milestones = []
+ # include the task itself as a milestone
+ for st in list(child.children()) + [child]:
+ amount = st.fixed_budget_excluding_subtasks.int()
+ if amount == 0: # skip anything at zero
+ continue
+ # if "task itself" then put the milestone as "wrapup"
+ if st.bug == child.bug:
+ description = 'wrapup'
+ intro = []
+ else:
+ # otherwise create a description and get comment #0
+ description = "%d %s" % (st.bug.id, st.bug.summary)
+ # add parent and MoU top-level
+ parent_id = st.parent.bug.id
+ if parent_id != child.bug.id:
+ description += "\n(Sub-sub-task of %d)" % parent_id
+ task = {'description': description,
+ 'amount': amount,
+ }
+ #mou_bug = st.closest_bug_in_mou
+ # if mou_bug is not None:
+ # task['mou_task'] = mou_bug.bug.id
+ milestones.append(task)
+ # create MoU task: get comment #0
+ intro = []
+ comment = "%s\n " % child.bug_url
+ if add_comments:
+ comment += "\n"
+ comments = bug_comments_map[str(child.bug.id)]['comments']
+ lines = comments[0]['text'].splitlines()
+ for i, line in enumerate(lines):
+ # look for a line with only 2 or more `-` as the
+ # standard way in markdown of having a "break" (<hl />)
+ # this truncates the comment so that the RFP database
+ # has only the "summary description" but the rest may
+ # be used for "TODO" lists
+ l = line.strip()
+ if len(l) >= 2 and l == "-" * len(l):
+ lines[i:] = []
+ break
+ comment += "\n".join(lines)
+ intro.append(comment)
+ # print (description, intro)
+ # sys.stdout.flush()
+ task = {'title': "%d %s" % (child.bug.id, child.bug.summary),
+ 'intro': intro,
+ 'amount': child.fixed_budget_including_subtasks.int(),
+ 'url': "{{ %s }} " % child.bug_url,
+ 'milestones': milestones
+ }
+ tasks.append(task)
+
+ d = {'participants': ppl,
+ 'preamble': '',
+ 'type': 'Group',
+ 'url': canonical.bug_url,
+ 'plan': {'intro': [''],
+ 'tasks': tasks,
+ 'rfp_secret': '',
+ }
+ }
+
+ output_file = output_dir / f"report.{milestone.identifier}.json"
+ output_file.write_text(json.dumps(d, indent=2), encoding="utf-8")
if __name__ == "__main__":
cents = -cents
return Money(cents=cents)
+ def int(self):
+ return int(str(self))
+
def __str__(self):
retval = "-" if self.cents < 0 else ""
retval += str(abs(self.cents) // CENTS_PER_EURO)
--- /dev/null
+from typing import Any, Dict, Iterable, Iterator, MutableSet, Optional, TypeVar
+
+__all__ = ['OrderedSet']
+_T_co = TypeVar('_T_co')
+
+
+class OrderedSet(MutableSet[_T_co]):
+ __map: Dict[_T_co, None]
+
+ def __init__(self, iterable: Iterable[_T_co] = ()):
+ self.__map = {i: None for i in iterable}
+
+ def __len__(self) -> int:
+ return len(self.__map)
+
+ def __contains__(self, key: Any) -> bool:
+ return key in self.__map
+
+ def add(self, key: _T_co):
+ self.__map[key] = None
+
+ def discard(self, key: Any):
+ self.__map.pop(key, None)
+
+ def __iter__(self) -> Iterator[_T_co]:
+ return iter(self.__map.keys())
+
+ def __repr__(self) -> str:
+ if len(self) == 0:
+ return "OrderedSet()"
+ return f"OrderedSet({list(self)!r})"
from posixpath import normpath
from enum import Enum
import errno
-from functools import cached_property
+try:
+ from functools import cached_property
+except ImportError:
+ # compatability with python < 3.8
+ from cached_property import cached_property
class MockDir(Enum):
BudgetGraphNegativeMoney, BudgetGraphMilestoneMismatch,
BudgetGraphNegativePayeeMoney, BudgetGraphPayeesParseError,
BudgetGraphPayeesMoneyMismatch, BudgetGraphUnknownMilestone,
- BudgetGraphDuplicatePayeesForTask, BudgetGraphIncorrectRootForMilestone,
+ BudgetGraphIncorrectRootForMilestone,
BudgetGraphUnknownStatus, BudgetGraphUnknownAssignee)
from budget_sync.money import Money
from budget_sync.util import BugStatus
"'milestone 1' but has no parent bug set: the milestone's "
"canonical root bug is #1")
- def test_budget_graph_duplicate_payees_for_task(self):
- self.assertEqual(str(BudgetGraphDuplicatePayeesForTask(
- 2, 1, "alias1", "alias2")),
- "Budget assigned to multiple aliases of the same person in a "
- "single task: bug #2, budget assigned to both 'alias1' "
- "and 'alias2'")
-
def test_budget_graph_loop_error(self):
self.assertEqual(str(BudgetGraphLoopError([1, 2, 3, 4, 5])),
"Detected Loop in Budget Graph: #5 -> #1 "
bugzilla_url = "https://bugzilla.example.com/"
[people."person1"]
aliases = ["person1_alias1", "alias1"]
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
[people."person2"]
email = "person2@example.com"
aliases = ["person1_alias2", "alias2", "person 2"]
- output_markdown_file = "person2.mdwn"
+ full_name = "Person Two"
[people."person3"]
email = "user@example.com"
- output_markdown_file = "person3.mdwn"
+ full_name = "Person Three"
[milestones]
"milestone 1" = { canonical_bug_id = 1 }
"milestone 2" = { canonical_bug_id = 2 }
"BudgetGraph{nodes=[Node(graph=..., id=#1, root=#1, parent=None, "
"budget_excluding_subtasks=10, budget_including_subtasks=20, "
"fixed_budget_excluding_subtasks=10, "
- "fixed_budget_including_subtasks=20, "
- "milestone_str='milestone 1', milestone=Milestone(config=..., "
- "identifier='milestone 1', canonical_bug_id=1), "
- "immediate_children=[#2], payments=[], "
- "status=BugStatus.CONFIRMED, assignee=Person<'person3'>), "
- "Node(graph=..., id=#2, root=#1, "
- "parent=#1, budget_excluding_subtasks=10, "
+ "fixed_budget_including_subtasks=20, milestone_str='milestone "
+ "1', is_in_nlnet_mou=False, "
+ "milestone=Milestone(config=..., identifier='milestone 1', "
+ "canonical_bug_id=1), immediate_children=[#2], payments=[], "
+ "status=BugStatus.CONFIRMED, assignee=Person<'person3'>, "
+ "resolved_payments={}, payment_summaries={}), Node(graph=..., "
+ "id=#2, root=#1, parent=#1, budget_excluding_subtasks=10, "
"budget_including_subtasks=10, "
"fixed_budget_excluding_subtasks=10, "
- "fixed_budget_including_subtasks=10, "
- "milestone_str='milestone 1', milestone=Milestone(config=..., "
- "identifier='milestone 1', canonical_bug_id=1), "
- "immediate_children=[], payments=[], "
- "status=BugStatus.CONFIRMED, assignee=Person<'person3'>)], "
- "roots=[#1]}")
+ "fixed_budget_including_subtasks=10, milestone_str='milestone "
+ "1', is_in_nlnet_mou=True, "
+ "milestone=Milestone(config=..., identifier='milestone 1', "
+ "canonical_bug_id=1), immediate_children=[], payments=[], "
+ "status=BugStatus.CONFIRMED, assignee=Person<'person3'>, "
+ "resolved_payments={}, payment_summaries={})], roots=[#1], "
+ "assigned_nodes={Person(config=..., identifier='person1', "
+ "full_name='Person One', "
+ "aliases=OrderedSet(['person1_alias1', 'alias1']), email=None): "
+ "[], Person(config=..., identifier='person2', "
+ "full_name='Person Two', "
+ "aliases=OrderedSet(['person1_alias2', 'alias2', 'person 2']), "
+ "email='person2@example.com'): [], Person(config=..., "
+ "identifier='person3', full_name='Person Three', "
+ "aliases=OrderedSet(), email='user@example.com'): [#1, #2]}, "
+ "assigned_nodes_for_milestones={Milestone(config=..., "
+ "identifier='milestone 1', canonical_bug_id=1): [#1, #2], "
+ "Milestone(config=..., identifier='milestone 2', "
+ "canonical_bug_id=2): []}, "
+ "milestone_payments={Milestone(config=..., identifier='milestone "
+ "1', canonical_bug_id=1): [], Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): []}, "
+ "payments={Person(config=..., identifier='person1', "
+ "full_name='Person One', "
+ "aliases=OrderedSet(['person1_alias1', 'alias1']), email=None): "
+ "{Milestone(config=..., identifier='milestone 1', "
+ "canonical_bug_id=1): [], Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): []}, "
+ "Person(config=..., identifier='person2', "
+ "full_name='Person Two', "
+ "aliases=OrderedSet(['person1_alias2', 'alias2', 'person 2']), "
+ "email='person2@example.com'): {Milestone(config=..., "
+ "identifier='milestone 1', canonical_bug_id=1): [], "
+ "Milestone(config=..., identifier='milestone 2', "
+ "canonical_bug_id=2): []}, Person(config=..., "
+ "identifier='person3', full_name='Person Three', "
+ "aliases=OrderedSet(), email='user@example.com'): "
+ "{Milestone(config=..., identifier='milestone 1', "
+ "canonical_bug_id=1): [], Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): []}}, "
+ "milestone_people={Milestone(config=..., identifier='milestone "
+ "1', canonical_bug_id=1): OrderedSet(), Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): OrderedSet()}}")
bg = BudgetGraph([MockBug(bug_id=1, status="blah",
assigned_to="unknown@example.com")],
EXAMPLE_CONFIG)
"budget_excluding_subtasks=0, budget_including_subtasks=0, "
"fixed_budget_excluding_subtasks=0, "
"fixed_budget_including_subtasks=0, milestone_str=None, "
+ "is_in_nlnet_mou=False, "
"milestone=None, immediate_children=[], payments=[], "
- "status=<unknown status: 'blah'>, "
- "assignee=<unknown assignee: 'unknown@example.com'>)], "
- "roots=[#1]}")
+ "status=<unknown status: 'blah'>, assignee=<unknown assignee: "
+ "'unknown@example.com'>, resolved_payments={}, "
+ "payment_summaries={})], roots=[#1], assigned_nodes=<failed>, "
+ "assigned_nodes_for_milestones={Milestone(config=..., "
+ "identifier='milestone 1', canonical_bug_id=1): [], "
+ "Milestone(config=..., identifier='milestone 2', "
+ "canonical_bug_id=2): []}, "
+ "milestone_payments={Milestone(config=..., "
+ "identifier='milestone 1', canonical_bug_id=1): [], "
+ "Milestone(config=..., identifier='milestone 2', "
+ "canonical_bug_id=2): []}, payments={Person(config=..., "
+ "identifier='person1', full_name='Person One', "
+ "aliases=OrderedSet(['person1_alias1', 'alias1']), email=None): "
+ "{Milestone(config=..., identifier='milestone 1', "
+ "canonical_bug_id=1): [], Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): []}, "
+ "Person(config=..., identifier='person2', "
+ "full_name='Person Two', "
+ "aliases=OrderedSet(['person1_alias2', 'alias2', "
+ "'person 2']), email='person2@example.com'): "
+ "{Milestone(config=..., identifier='milestone 1', "
+ "canonical_bug_id=1): [], Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): []}, "
+ "Person(config=..., identifier='person3', "
+ "full_name='Person Three', aliases=OrderedSet(), "
+ "email='user@example.com'): {Milestone(config=..., "
+ "identifier='milestone 1', canonical_bug_id=1): [], "
+ "Milestone(config=..., identifier='milestone 2', "
+ "canonical_bug_id=2): []}}, "
+ "milestone_people={Milestone(config=..., identifier='milestone "
+ "1', canonical_bug_id=1): OrderedSet(), Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): OrderedSet()}}")
+ bg = BudgetGraph([MockBug(bug_id=1, status="blah",
+ assigned_to="unknown@example.com",
+ cf_payees_list="""\
+person1 = {paid=2020-03-15,amount=5}
+alias1 = {paid=2020-03-15,amount=10}
+person2 = {submitted=2020-03-15,amount=15}
+alias2 = {paid=2020-03-16,amount=23}
+""")],
+ EXAMPLE_CONFIG)
+ self.assertEqual(
+ repr(bg),
+ "BudgetGraph{nodes=[Node(graph=..., id=#1, root=#1, parent=None, "
+ "budget_excluding_subtasks=0, budget_including_subtasks=0, "
+ "fixed_budget_excluding_subtasks=0, "
+ "fixed_budget_including_subtasks=0, milestone_str=None, "
+ "is_in_nlnet_mou=False, "
+ "milestone=None, immediate_children=[], "
+ "payments=[Payment(node=#1, payee=Person<'person1'>, "
+ "payee_key='person1', amount=5, state=Paid, paid=2020-03-15, "
+ "submitted=None), Payment(node=#1, payee=Person<'person1'>, "
+ "payee_key='alias1', amount=10, state=Paid, paid=2020-03-15, "
+ "submitted=None), Payment(node=#1, payee=Person<'person2'>, "
+ "payee_key='person2', amount=15, state=Submitted, paid=None, "
+ "submitted=2020-03-15), Payment(node=#1, "
+ "payee=Person<'person2'>, payee_key='alias2', amount=23, "
+ "state=Paid, paid=2020-03-16, submitted=None)], status=<unknown "
+ "status: 'blah'>, assignee=<unknown assignee: "
+ "'unknown@example.com'>, resolved_payments={Person(config=..., "
+ "identifier='person1', full_name='Person One', "
+ "aliases=OrderedSet(['person1_alias1', 'alias1']), email=None): "
+ "[Payment(node=#1, payee=Person<'person1'>, payee_key='person1', "
+ "amount=5, state=Paid, paid=2020-03-15, submitted=None), "
+ "Payment(node=#1, payee=Person<'person1'>, payee_key='alias1', "
+ "amount=10, state=Paid, paid=2020-03-15, submitted=None)], "
+ "Person(config=..., identifier='person2', "
+ "full_name='Person Two', "
+ "aliases=OrderedSet(['person1_alias2', 'alias2', 'person 2']), "
+ "email='person2@example.com'): [Payment(node=#1, "
+ "payee=Person<'person2'>, payee_key='person2', amount=15, "
+ "state=Submitted, paid=None, submitted=2020-03-15), "
+ "Payment(node=#1, payee=Person<'person2'>, payee_key='alias2', "
+ "amount=23, state=Paid, paid=2020-03-16, submitted=None)]}, "
+ "payment_summaries={Person(config=..., identifier='person1', "
+ "full_name='Person One', "
+ "aliases=OrderedSet(['person1_alias1', 'alias1']), email=None): "
+ "PaymentSummary(total=15, total_paid=15, total_submitted=15, "
+ "submitted_date=None, paid_date=2020-03-15, "
+ "state=PaymentSummaryState.Paid, payments=(Payment(node=#1, "
+ "payee=Person<'person1'>, payee_key='person1', amount=5, "
+ "state=Paid, paid=2020-03-15, submitted=None), Payment(node=#1, "
+ "payee=Person<'person1'>, payee_key='alias1', amount=10, "
+ "state=Paid, paid=2020-03-15, submitted=None))), "
+ "Person(config=..., identifier='person2', "
+ "full_name='Person Two', "
+ "aliases=OrderedSet(['person1_alias2', 'alias2', 'person 2']), "
+ "email='person2@example.com'): PaymentSummary(total=38, "
+ "total_paid=23, total_submitted=38, submitted_date=None, "
+ "paid_date=None, state=PaymentSummaryState.Inconsistent, "
+ "payments=(Payment(node=#1, payee=Person<'person2'>, "
+ "payee_key='person2', amount=15, state=Submitted, paid=None, "
+ "submitted=2020-03-15), Payment(node=#1, "
+ "payee=Person<'person2'>, payee_key='alias2', amount=23, "
+ "state=Paid, paid=2020-03-16, submitted=None)))})], roots=[#1], "
+ "assigned_nodes=<failed>, "
+ "assigned_nodes_for_milestones={Milestone(config=..., "
+ "identifier='milestone 1', canonical_bug_id=1): [], "
+ "Milestone(config=..., identifier='milestone 2', "
+ "canonical_bug_id=2): []}, "
+ "milestone_payments={Milestone(config=..., identifier='milestone "
+ "1', canonical_bug_id=1): [], Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): []}, "
+ "payments={Person(config=..., identifier='person1', "
+ "full_name='Person One', "
+ "aliases=OrderedSet(['person1_alias1', 'alias1']), email=None): "
+ "{Milestone(config=..., identifier='milestone 1', "
+ "canonical_bug_id=1): [], Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): []}, "
+ "Person(config=..., identifier='person2', "
+ "full_name='Person Two', "
+ "aliases=OrderedSet(['person1_alias2', 'alias2', 'person 2']), "
+ "email='person2@example.com'): {Milestone(config=..., "
+ "identifier='milestone 1', canonical_bug_id=1): [], "
+ "Milestone(config=..., identifier='milestone 2', "
+ "canonical_bug_id=2): []}, Person(config=..., "
+ "identifier='person3', full_name='Person Three', "
+ "aliases=OrderedSet(), email='user@example.com'): "
+ "{Milestone(config=..., identifier='milestone 1', "
+ "canonical_bug_id=1): [], Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): []}}, "
+ "milestone_people={Milestone(config=..., identifier='milestone "
+ "1', canonical_bug_id=1): OrderedSet(), Milestone(config=..., "
+ "identifier='milestone 2', canonical_bug_id=2): OrderedSet()}}")
def test_empty(self):
bg = BudgetGraph([], EXAMPLE_CONFIG)
cf_budget=child_budget,
cf_total_budget=child_budget,
cf_nlnet_milestone="milestone 1",
- cf_payees_list="",
summary=""),
], EXAMPLE_CONFIG)
node1: Node = bg.nodes[1]
errors = bg.get_errors()
self.assertErrorTypesMatches(errors, [
BudgetGraphNegativeMoney,
- BudgetGraphMoneyMismatchForBudgetExcludingSubtasks])
+ BudgetGraphMoneyMismatchForBudgetIncludingSubtasks])
self.assertEqual(errors[0].bug_id, 1)
self.assertEqual(errors[0].root_bug_id, 1)
self.assertEqual(errors[1].bug_id, 1)
self.assertEqual(errors[1].root_bug_id, 1)
- self.assertEqual(errors[1].expected_budget_excluding_subtasks, -10)
+ self.assertEqual(errors[1].expected_budget_including_subtasks, 0)
bg = BudgetGraph([
MockBug(bug_id=1,
cf_budget_parent=None,
summary=""),
], EXAMPLE_CONFIG)
errors = bg.get_errors()
- self.assertErrorTypesMatches(errors,
- [BudgetGraphDuplicatePayeesForTask])
- self.assertEqual(errors[0].bug_id, 1)
- self.assertEqual(errors[0].root_bug_id, 1)
- self.assertEqual(errors[0].payee1_key, "person1")
- self.assertEqual(errors[0].payee2_key, "alias1")
+ self.assertErrorTypesMatches(errors, [])
+ person1 = EXAMPLE_CONFIG.people["person1"]
+ person2 = EXAMPLE_CONFIG.people["person2"]
+ person3 = EXAMPLE_CONFIG.people["person3"]
+ milestone1 = EXAMPLE_CONFIG.milestones["milestone 1"]
+ milestone2 = EXAMPLE_CONFIG.milestones["milestone 2"]
+ node1: Node = bg.nodes[1]
+ node1_payment_person1 = node1.payments["person1"]
+ node1_payment_alias1 = node1.payments["alias1"]
+ self.assertEqual(bg.payments, {
+ person1: {
+ milestone1: [node1_payment_person1, node1_payment_alias1],
+ milestone2: [],
+ },
+ person2: {milestone1: [], milestone2: []},
+ person3: {milestone1: [], milestone2: []},
+ })
+ self.assertEqual(
+ repr(node1.payment_summaries),
+ "{Person(config=..., identifier='person1', "
+ "full_name='Person One', "
+ "aliases=OrderedSet(['person1_alias1', 'alias1']), email=None): "
+ "PaymentSummary(total=10, total_paid=0, total_submitted=0, "
+ "submitted_date=None, paid_date=None, "
+ "state=PaymentSummaryState.NotYetSubmitted, "
+ "payments=(Payment(node=#1, payee=Person<'person1'>, "
+ "payee_key='person1', amount=5, state=NotYetSubmitted, "
+ "paid=None, submitted=None), Payment(node=#1, "
+ "payee=Person<'person1'>, payee_key='alias1', amount=5, "
+ "state=NotYetSubmitted, paid=None, submitted=None)))}")
def test_incorrect_root_for_milestone(self):
bg = BudgetGraph([
self.assertEqual(bg.nodes[1].assignee,
EXAMPLE_CONFIG.people["person2"])
+ def test_closest_bug_in_mou(self):
+ bg = BudgetGraph([
+ MockBug(bug_id=1, cf_nlnet_milestone="milestone 1"),
+ MockBug(bug_id=2, cf_budget_parent=1,
+ cf_nlnet_milestone="milestone 1"),
+ MockBug(bug_id=3, cf_budget_parent=2,
+ cf_nlnet_milestone="milestone 1"),
+ MockBug(bug_id=4, cf_budget_parent=1,
+ cf_nlnet_milestone="milestone 1"),
+ MockBug(bug_id=5, cf_budget_parent=4,
+ cf_nlnet_milestone="milestone 1"),
+ MockBug(bug_id=6),
+ MockBug(bug_id=7, cf_nlnet_milestone="bad milestone"),
+ MockBug(bug_id=8, cf_budget_parent=7,
+ cf_nlnet_milestone="bad milestone"),
+ ], EXAMPLE_CONFIG)
+ errors = bg.get_errors()
+ self.assertErrorTypesMatches(errors, [BudgetGraphUnknownMilestone,
+ BudgetGraphUnknownMilestone])
+ self.assertEqual(bg.nodes[1].closest_bug_in_mou, None)
+ self.assertEqual(bg.nodes[2].closest_bug_in_mou, bg.nodes[2])
+ self.assertEqual(bg.nodes[3].closest_bug_in_mou, bg.nodes[2])
+ self.assertEqual(bg.nodes[4].closest_bug_in_mou, bg.nodes[4])
+ self.assertEqual(bg.nodes[5].closest_bug_in_mou, bg.nodes[4])
+ self.assertEqual(bg.nodes[6].closest_bug_in_mou, None)
+ self.assertEqual(bg.nodes[7].closest_bug_in_mou, None)
+ self.assertEqual(bg.nodes[8].closest_bug_in_mou, None)
+
if __name__ == "__main__":
unittest.main()
class TestConfig(unittest.TestCase):
+ maxDiff = None
+
def test_config_parsing(self):
def check_error(text: str, expected_error_text: str):
with self.assertRaises(ConfigParseError) as e:
bugzilla_url = ""
[people."person1"]
""",
- "`output_markdown_file` field is missing in person entry for "
+ "`full_name` field is missing in person entry for "
"'person1'")
check_error(
"""
bugzilla_url = ""
[people."person1"]
- output_markdown_file = 1
+ full_name = 1
""",
- "`output_markdown_file` field in person entry for 'person1' must "
+ "`full_name` field in person entry for 'person1' must "
"be a string")
check(
"""
[milestones]
[people."person1"]
aliases = ["a"]
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
[people."person2"]
aliases = ["b"]
- output_markdown_file = "person2.mdwn"
+ full_name = "Person Two"
""",
"Config(bugzilla_url='', people={"
"'person1': Person(config=..., identifier='person1', "
- "output_markdown_file='person1.mdwn', "
- "aliases={'a'}, email=None), "
+ "full_name='Person One', "
+ "aliases=OrderedSet(['a']), email=None), "
"'person2': Person(config=..., identifier='person2', "
- "output_markdown_file='person2.mdwn', "
- "aliases={'b'}, email=None)}, milestones={})")
+ "full_name='Person Two', "
+ "aliases=OrderedSet(['b']), email=None)}, milestones={})")
check_error(
"""
bugzilla_url = ""
[people."person1"]
email = 123
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
""",
"`email` field in person entry for 'person1' must be a string")
check(
[milestones]
[people."person1"]
email = "email@example.com"
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
""",
"Config(bugzilla_url='', people={"
"'person1': Person(config=..., identifier='person1', "
- "output_markdown_file='person1.mdwn', "
- "aliases=set(), email='email@example.com')}, milestones={})")
+ "full_name='Person One', "
+ "aliases=OrderedSet(), email='email@example.com')}, "
+ "milestones={})")
check_error(
"""
bugzilla_url = ""
[people."person1"]
blah = 123
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
""",
"unknown field in person entry for 'person1': `blah`")
check_error(
bugzilla_url = ""
[milestones]
[people."person1"]
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
[people."person2"]
aliases = ["person1"]
- output_markdown_file = "person2.mdwn"
+ full_name = "Person Two"
""",
"alias is not allowed to be the same as any person's identifier: "
"in person entry for 'person2': 'person1' is also the identifier "
bugzilla_url = ""
[milestones]
[people."person1"]
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
aliases = ["a"]
[people."person2"]
aliases = ["a"]
- output_markdown_file = "person2.mdwn"
+ full_name = "Person Two"
""",
- "alias is not allowed to be the same as another person's alias or "
- "email: in person entry for 'person2': 'a' is also an alias or "
- "email for person 'person1'")
+ "alias is not allowed to be the same as another person's alias, "
+ "email, or full_name: in person entry for 'person2': 'a' is also an alias, "
+ "email, or full_name for person 'person1'")
check_error(
"""
bugzilla_url = ""
[milestones]
[people."person1"]
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
aliases = ["abc@example.com"]
[people."person2"]
email = "abc@example.com"
- output_markdown_file = "person2.mdwn"
+ full_name = "Person Two"
+ """,
+ "email is not allowed to be the same as another person's alias, "
+ "email, or full_name: in person entry for 'person2': 'abc@example.com' is also "
+ "an alias, email, or full_name for person 'person1'")
+ check_error(
+ """
+ bugzilla_url = ""
+ [milestones]
+ [people."person1"]
+ full_name = "Person One"
+ aliases = ["Person Two"]
+ [people."person2"]
+ email = "abc@example.com"
+ full_name = "Person Two"
""",
- "email is not allowed to be the same as another person's alias or "
- "email: in person entry for 'person2': 'abc@example.com' is also "
- "an alias or email for person 'person1'")
+ "full_name is not allowed to be the same as another person's alias, "
+ "email, or full_name: in person entry for 'person2': 'Person Two' is also "
+ "an alias, email, or full_name for person 'person1'")
check_error(
"""
bugzilla_url = ""
[milestones]
[people."person2"]
email = "abc@example.com"
- output_markdown_file = "person2.mdwn"
+ full_name = "Person Two"
[people."person1"]
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
aliases = ["abc@example.com"]
""",
- "alias is not allowed to be the same as another person's alias or "
- "email: in person entry for 'person1': 'abc@example.com' is also "
- "an alias or email for person 'person2'")
+ "alias is not allowed to be the same as another person's alias, "
+ "email, or full_name: in person entry for 'person1': 'abc@example.com' is also "
+ "an alias, email, or full_name for person 'person2'")
check_error(
"""
bugzilla_url = ""
[milestones]
[people."person1"]
aliases = ["person1_alias1", "alias1"]
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
[people."person2"]
aliases = ["person2_alias2", "alias2"]
- output_markdown_file = "person2.mdwn"
+ full_name = "Person Two"
""")
person1 = config.people['person1']
person2 = config.people['person2']
self.assertEqual(config.all_names,
{
'person1': person1,
+ 'Person One': person1,
'person1_alias1': person1,
'alias1': person1,
'person2': person2,
+ 'Person Two': person2,
'person2_alias2': person2,
'alias2': person2,
})
[people."person1"]
email = "person1@example.com"
aliases = ["alias1"]
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
[milestones]
"Milestone 1" = { canonical_bug_id = 123 }
""")),
"Config(bugzilla_url='https://bugzilla.example.com/', "
"people={'person1': Person(config=..., identifier='person1', "
- "output_markdown_file='person1.mdwn', "
- "aliases={'alias1'}, email='person1@example.com')}, "
+ "full_name='Person One', "
+ "aliases=OrderedSet(['alias1']), email='person1@example.com')}, "
"milestones={'Milestone 1': Milestone(config=..., "
"identifier='Milestone 1', canonical_bug_id=123)})")
--- /dev/null
+import unittest
+from budget_sync.ordered_set import OrderedSet
+
+
+class TestOrderedSet(unittest.TestCase):
+ def test_repr(self):
+ self.assertEqual(repr(OrderedSet()), "OrderedSet()")
+ self.assertEqual(repr(OrderedSet((1,))), "OrderedSet([1])")
+ self.assertEqual(repr(OrderedSet((1, 2))), "OrderedSet([1, 2])")
+ self.assertEqual(repr(OrderedSet((2, 1))), "OrderedSet([2, 1])")
+ self.assertEqual(repr(OrderedSet((2, 2))), "OrderedSet([2])")
+
+ def test_len(self):
+ self.assertEqual(len(OrderedSet()), 0)
+ self.assertEqual(len(OrderedSet((1,))), 1)
+ self.assertEqual(len(OrderedSet((1, 2))), 2)
+ self.assertEqual(len(OrderedSet((2, 1))), 2)
+ self.assertEqual(len(OrderedSet((2, 2))), 1)
+
+ def test_contains(self):
+ self.assertFalse(0 in OrderedSet())
+ self.assertFalse(1 in OrderedSet())
+ self.assertTrue(0 in OrderedSet([0]))
+ self.assertFalse(1 in OrderedSet([0]))
+ self.assertTrue(0 in OrderedSet([0, 1]))
+ self.assertTrue(1 in OrderedSet([0, 1]))
+ self.assertTrue(0 in OrderedSet([1, 0]))
+ self.assertTrue(1 in OrderedSet([1, 0]))
+
+ def test_add(self):
+ s = OrderedSet()
+ self.assertEqual(repr(s), "OrderedSet()")
+ s.add(1)
+ self.assertEqual(repr(s), "OrderedSet([1])")
+ s.add(2)
+ self.assertEqual(repr(s), "OrderedSet([1, 2])")
+ s.add(2)
+ self.assertEqual(repr(s), "OrderedSet([1, 2])")
+ s.add(1)
+ self.assertEqual(repr(s), "OrderedSet([1, 2])")
+ s.add(0)
+ self.assertEqual(repr(s), "OrderedSet([1, 2, 0])")
+
+ def test_discard(self):
+ s = OrderedSet()
+ s.discard(1)
+ self.assertEqual(repr(s), "OrderedSet()")
+ s = OrderedSet([1])
+ s.discard(1)
+ self.assertEqual(repr(s), "OrderedSet()")
+ s = OrderedSet([1, 2, 3])
+ s.discard(1)
+ self.assertEqual(repr(s), "OrderedSet([2, 3])")
+ s = OrderedSet([3, 2, 1])
+ s.discard(1)
+ self.assertEqual(repr(s), "OrderedSet([3, 2])")
+ s = OrderedSet([3, 2, 1])
+ s.discard(None)
+ self.assertEqual(repr(s), "OrderedSet([3, 2, 1])")
+
+ def test_iter(self):
+ self.assertEqual(list(OrderedSet()), [])
+ self.assertEqual(list(OrderedSet((1,))), [1])
+ self.assertEqual(list(OrderedSet((1, 2))), [1, 2])
+ self.assertEqual(list(OrderedSet((2, 1))), [2, 1])
+ self.assertEqual(list(OrderedSet((2, 2))), [2])
+
+
+if __name__ == "__main__":
+ unittest.main()
--- /dev/null
+from budget_sync.budget_graph import BudgetGraph
+from budget_sync.config import Config
+from budget_sync.test.mock_path import DIR, MockPath
+from budget_sync.test.test_mock_path import make_filesystem_and_report_if_error
+from budget_sync.util import pretty_print
+from budget_sync.write_budget_csv import write_budget_csv
+from budget_sync.test.mock_bug import MockBug
+import unittest
+
+
+class TestWriteBudgetMarkdown(unittest.TestCase):
+ maxDiff = None
+
+ def test(self):
+ config = Config.from_str(
+ """
+ bugzilla_url = "https://bugzilla.example.com/"
+ [people."person1"]
+ aliases = ["person1_alias1", "alias1"]
+ full_name = "Person One"
+ [people."person2"]
+ email = "person2@example.com"
+ aliases = ["person1_alias2", "alias2", "person 2"]
+ full_name = "Person Two"
+ [people."person3"]
+ email = "user@example.com"
+ full_name = "Person Three"
+ [milestones]
+ "milestone 1" = { canonical_bug_id = 1 }
+ "milestone 2" = { canonical_bug_id = 2 }
+ """)
+ budget_graph = BudgetGraph([
+ MockBug(bug_id=1,
+ cf_budget_parent=None,
+ cf_budget="1000",
+ cf_total_budget="1000",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="""
+ person1 = 123
+ alias1 = 456
+ person2 = {amount=421,paid=2020-01-01}
+ """,
+ summary="",
+ assigned_to="person2@example.com"),
+ MockBug(bug_id=2,
+ cf_budget_parent=None,
+ cf_budget="0",
+ cf_total_budget="0",
+ cf_nlnet_milestone="milestone 2",
+ cf_payees_list="",
+ summary="",
+ assigned_to="person2@example.com"),
+ ], config)
+ self.assertEqual([], budget_graph.get_errors())
+ # pretty_print(budget_graph)
+ with make_filesystem_and_report_if_error(self) as filesystem:
+ output_dir = MockPath("/output_dir/", filesystem=filesystem)
+ write_budget_csv(budget_graph, output_dir)
+ self.assertEqual(filesystem.files, {
+ '/': DIR,
+ '/output_dir': DIR,
+ '/output_dir/csvs.mdwn': b"""\
+# milestone 1
+
+[[!table format=csv file="/output_dir/milestone 1.csv"]]
+# milestone 2
+
+[[!table format=csv file="/output_dir/milestone 2.csv"]]""",
+ '/output_dir/milestone 1.csv': b"""\
+bug_id,excl_subtasks,inc_subtasks,fixed_excl_subtasks,fixed_inc_subtasks,req_excl_subtasks,paid_excl_subtasks,person1 (planned amt),person1 (req amt),person1 (req date),person1 (paid amt),person1 (paid date),person2 (planned amt),person2 (req amt),person2 (req date),person2 (paid amt),person2 (paid date)
+1,1000,1000,1000,1000,421,421,579,0,,0,,421,421,,421,2020-01-01
+""",
+ '/output_dir/milestone 2.csv': b"""\
+bug_id,excl_subtasks,inc_subtasks,fixed_excl_subtasks,fixed_inc_subtasks,req_excl_subtasks,paid_excl_subtasks
+"""
+ })
+ # TODO: add more test cases
+
+
+if __name__ == "__main__":
+ unittest.main()
import unittest
from budget_sync.config import Config
+from budget_sync.ordered_set import OrderedSet
from budget_sync.test.mock_bug import MockBug
-from budget_sync.test.mock_path import MockPath, DIR
+from budget_sync.test.mock_path import MockFilesystem, MockPath, DIR
from budget_sync.test.test_mock_path import make_filesystem_and_report_if_error
from budget_sync.budget_graph import BudgetGraph
from budget_sync.write_budget_markdown import (
- write_budget_markdown, DisplayStatus, markdown_escape)
+ write_budget_markdown, DisplayStatus, markdown_escape, markdown_for_person)
from budget_sync.util import BugStatus
self.assertEqual(markdown_escape("abc * def_k < &k"),
r"abc \* def\_k < &k")
+ def format_files_dict(self, files):
+ assert isinstance(files, dict)
+ files_list: "list[str]" = []
+ for path, contents in files.items():
+ assert isinstance(path, str)
+ if contents is DIR:
+ files_list.append(f" {path!r}: DIR,")
+ continue
+ assert isinstance(contents, bytes)
+ lines: "list[str]" = []
+ for line in contents.splitlines(keepends=True):
+ lines.append(f" {line!r}")
+ if len(lines) == 0:
+ files_list.append(f" {path!r}: b'',")
+ else:
+ lines_str = '\n'.join(lines)
+ files_list.append(f" {path!r}: (\n{lines_str}\n ),")
+ if len(files_list) == 0:
+ return "{}"
+ return "{\n" + "\n".join(files_list) + "\n}"
+
+ def assertFiles(self, expected_files, filesystem: MockFilesystem):
+ files = filesystem.files
+ self.assertIsInstance(expected_files, dict)
+ if files == expected_files:
+ return
+ files_str = self.format_files_dict(files)
+ expected_files_str = self.format_files_dict(expected_files)
+ self.assertEqual(
+ files, expected_files,
+ msg=f"\nfiles:\n{files_str}\nexpected:\n{expected_files_str}")
+
def test(self):
config = Config.from_str(
"""
[milestones]
[people."person1"]
email = "person1@example.com"
- output_markdown_file = "person1.mdwn"
+ full_name = "Person One"
[people."person2"]
- output_markdown_file = "person2.mdwn"
+ full_name = "Person Two"
""")
budget_graph = BudgetGraph([
MockBug(bug_id=1,
cf_total_budget="0",
cf_nlnet_milestone=None,
cf_payees_list="",
- summary="",
+ summary="summary1",
+ assigned_to="person1@example.com"),
+ ], config)
+ self.assertEqual([], budget_graph.get_errors())
+ with make_filesystem_and_report_if_error(self) as filesystem:
+ output_dir = MockPath("/output_dir/", filesystem=filesystem)
+ write_budget_markdown(budget_graph, output_dir)
+ self.assertFiles({
+ '/': DIR,
+ '/output_dir': DIR,
+ '/output_dir/person1.mdwn': (
+ b'<!-- autogenerated by budget-sync -->\n'
+ b'\n'
+ b'# Person One (person1)\n'
+ b'\n'
+ b'\n'
+ b'\n'
+ b'# Status Tracking\n'
+ b'\n'
+ ),
+ '/output_dir/person2.mdwn': (
+ b'<!-- autogenerated by budget-sync -->\n'
+ b'\n'
+ b'# Person Two (person2)\n'
+ b'\n'
+ b'\n'
+ b'\n'
+ b'# Status Tracking\n'
+ b'\n'
+ ),
+ }, filesystem)
+
+ def test2(self):
+ config = Config.from_str(
+ """
+ bugzilla_url = "https://bugzilla.example.com/"
+ [milestones]
+ "milestone 1" = { canonical_bug_id = 1 }
+ [people."person1"]
+ email = "person1@example.com"
+ full_name = "Person One"
+ [people."person2"]
+ full_name = "Person Two"
+ """)
+ budget_graph = BudgetGraph([
+ MockBug(bug_id=1,
+ cf_budget_parent=None,
+ cf_budget="700",
+ cf_total_budget="1000",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="",
+ summary="summary1",
+ assigned_to="person1@example.com"),
+ MockBug(bug_id=2,
+ cf_budget_parent=1,
+ cf_budget="100",
+ cf_total_budget="300",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="person2 = 100",
+ summary="summary2",
+ assigned_to="person1@example.com"),
+ MockBug(bug_id=3,
+ cf_budget_parent=2,
+ cf_budget="100",
+ cf_total_budget="200",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="person1 = 100",
+ summary="summary3",
+ assigned_to="person1@example.com"),
+ MockBug(bug_id=4,
+ cf_budget_parent=3,
+ cf_budget="100",
+ cf_total_budget="100",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="person2 = 100",
+ summary="summary4",
assigned_to="person1@example.com"),
], config)
self.assertEqual([], budget_graph.get_errors())
with make_filesystem_and_report_if_error(self) as filesystem:
output_dir = MockPath("/output_dir/", filesystem=filesystem)
write_budget_markdown(budget_graph, output_dir)
- self.assertEqual({
- "/": DIR,
- "/output_dir": DIR,
- '/output_dir/person1.mdwn': b'<!-- autogenerated by '
- b'budget-sync -->\n# person1\n\n# Status Tracking\n',
- '/output_dir/person2.mdwn': b'<!-- autogenerated by '
- b'budget-sync -->\n# person2\n\n# Status Tracking\n',
- }, filesystem.files)
+ self.assertFiles({
+ '/': DIR,
+ '/output_dir': DIR,
+ '/output_dir/person1.mdwn': (
+ b'<!-- autogenerated by budget-sync -->\n'
+ b'\n'
+ b'# Person One (person1)\n'
+ b'\n'
+ b'\n'
+ b'\n'
+ b'# Status Tracking\n'
+ b'\n'
+ b'\n'
+ b'## Payment not yet submitted\n'
+ b'\n'
+ b'\n'
+ b'### milestone 1\n'
+ b'\n'
+ b'* [Bug #3](https://bugzilla.example.com/show_bug.cgi?id=3):\n'
+ b' summary3\n'
+ b' * €100 which is the total amount\n'
+ b' * this task is part of MoU Milestone\n'
+ b' [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2)\n'
+ b'\n'
+ b'MoU Milestone subtotals for not yet submitted payments\n'
+ b'\n'
+ b'* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n'
+ b' summary2\n'
+ b' * subtotal €100 out of total including subtasks of €300\n'
+ ),
+ '/output_dir/person2.mdwn': (
+ b'<!-- autogenerated by budget-sync -->\n'
+ b'\n'
+ b'# Person Two (person2)\n'
+ b'\n'
+ b'\n'
+ b'\n'
+ b'# Status Tracking\n'
+ b'\n'
+ b'\n'
+ b'## Payment not yet submitted\n'
+ b'\n'
+ b'\n'
+ b'### milestone 1\n'
+ b'\n'
+ b'* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n'
+ b' summary2\n'
+ b' * €100 which is the total amount\n'
+ b' * this task is a MoU Milestone\n'
+ b'* [Bug #4](https://bugzilla.example.com/show_bug.cgi?id=4):\n'
+ b' summary4\n'
+ b' * €100 which is the total amount\n'
+ b' * this task is part of MoU Milestone\n'
+ b' [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2)\n'
+ b'\n'
+ b'MoU Milestone subtotals for not yet submitted payments\n'
+ b'\n'
+ b'* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n'
+ b' summary2\n'
+ b' * subtotal €200 out of total including subtasks of €300\n'
+ ),
+ }, filesystem)
+
+ def test_markdown_for_person(self):
+ config = Config.from_str(
+ """
+ bugzilla_url = "https://bugzilla.example.com/"
+ [milestones]
+ "milestone 1" = { canonical_bug_id = 1 }
+ [people."person1"]
+ email = "person1@example.com"
+ full_name = "Person One"
+ [people."person2"]
+ full_name = "Person Two"
+ """)
+ budget_graph = BudgetGraph([
+ MockBug(bug_id=1,
+ cf_budget_parent=None,
+ cf_budget="600",
+ cf_total_budget="1000",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="",
+ summary="summary1",
+ assigned_to="person1@example.com"),
+ MockBug(bug_id=2,
+ cf_budget_parent=1,
+ cf_budget="100",
+ cf_total_budget="400",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="person2 = 100",
+ summary="summary2",
+ assigned_to="person1@example.com"),
+ MockBug(bug_id=3,
+ cf_budget_parent=2,
+ cf_budget="100",
+ cf_total_budget="300",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="person1 = 100",
+ summary="summary3",
+ assigned_to="person1@example.com"),
+ MockBug(bug_id=4,
+ cf_budget_parent=3,
+ cf_budget="100",
+ cf_total_budget="100",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="person2 = 100",
+ summary="summary4",
+ assigned_to="person1@example.com"),
+ MockBug(bug_id=5,
+ cf_budget_parent=3,
+ cf_budget="100",
+ cf_total_budget="100",
+ cf_nlnet_milestone="milestone 1",
+ cf_payees_list="person2 = 100",
+ summary="summary4",
+ assigned_to="person1@example.com"),
+ ], config)
+ self.assertEqual([], budget_graph.get_errors())
+ person = config.all_names["person2"]
+ nodes_subset = OrderedSet([budget_graph.nodes[2],
+ budget_graph.nodes[3],
+ budget_graph.nodes[4]])
+ expected = [
+ '<!-- autogenerated by budget-sync -->\n',
+ '\n',
+ '# Person Two (person2)\n',
+ '\n',
+ '\n',
+ '\n',
+ '# Status Tracking\n',
+ '\n',
+ '\n',
+ '## Payment not yet submitted\n',
+ '\n',
+ '\n',
+ '### milestone 1\n',
+ '\n',
+ '* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n',
+ ' summary2\n',
+ ' * €100 which is the total amount\n',
+ ' * this task is a MoU Milestone\n',
+ '* [Bug #4](https://bugzilla.example.com/show_bug.cgi?id=4):\n',
+ ' summary4\n',
+ ' * €100 which is the total amount\n',
+ ' * this task is part of MoU Milestone\n',
+ ' [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2)\n',
+ '\n',
+ 'MoU Milestone subtotals for not yet submitted payments\n',
+ '\n',
+ '* [Bug #2](https://bugzilla.example.com/show_bug.cgi?id=2):\n',
+ ' summary2\n',
+ ' * subtotal €200 out of total including subtasks of €400\n',
+ ]
+ self.assertEqual(markdown_for_person(
+ budget_graph, person, nodes_subset).splitlines(keepends=True),
+ expected)
# TODO: add more test cases
--- /dev/null
+import toml
+from budget_sync.write_budget_csv import write_budget_csv
+from bugzilla import Bugzilla
+import logging
+import argparse
+from pathlib import Path
+from budget_sync.config import Config, ConfigParseError, Milestone
+from budget_sync.budget_graph import (BudgetGraph, BudgetGraphBaseError,
+ PaymentSummary)
+from budget_sync.write_budget_markdown import write_budget_markdown
+from datetime import datetime, date
+
+logging.basicConfig(level=logging.INFO)
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Check for errors in "
+ "Libre-SOC's style of budget tracking in Bugzilla.")
+ parser.add_argument(
+ "-c", "--config", type=argparse.FileType('r'),
+ required=True, help="The path to the configuration TOML file",
+ dest="config", metavar="<path/to/budget-sync-config.toml>")
+ parser.add_argument(
+ "-o", "--output-dir", type=Path, default=None,
+ help="The path to the output directory, will be created if it "
+ "doesn't exist",
+ dest="output_dir", metavar="<path/to/output/dir>")
+ parser.add_argument('--username', help="Log in with this username")
+ parser.add_argument('--password', help="Log in with this password")
+ parser.add_argument('--bug', help="bug number")
+ parser.add_argument('--user', help="set payee user")
+ parser.add_argument('--paid', help="set paid date")
+ parser.add_argument('--submitted', help="set submitted date")
+ args = parser.parse_args()
+ try:
+ with args.config as config_file:
+ config = Config.from_file(config_file)
+ except (IOError, ConfigParseError) as e:
+ logging.error("Failed to parse config file: %s", e)
+ return
+ logging.info("Using Bugzilla instance at %s", config.bugzilla_url)
+ bz = Bugzilla(config.bugzilla_url)
+ if args.username:
+ logging.debug("logging in...")
+ bz.interactive_login(args.username, args.password)
+ logging.debug("Connected to Bugzilla")
+ bugs = str(args.bug).split(",")
+ buglist = bz.getbugs(bugs)
+ logging.info("got bugs %s" % args.bug)
+ for bug in buglist:
+ print("payees", bug.id)
+ print(" "+"\n ".join(bug.cf_payees_list.split('\n')))
+
+ parsed_toml = toml.loads(bug.cf_payees_list)
+
+ payee = parsed_toml[args.user]
+ if isinstance(payee, int):
+ payee = {'amount': payee}
+
+ modified = False
+
+ if args.submitted and 'submitted' not in payee:
+ modified = True
+ d = datetime.strptime(args.submitted, "%Y-%m-%d")
+ payee['submitted'] = date(d.year, d.month, d.day)
+
+ if args.paid and 'paid' not in payee:
+ modified = True
+ d = datetime.strptime(args.paid, "%Y-%m-%d")
+ payee['paid'] = date(d.year, d.month, d.day)
+
+ # skip over not modified
+ if not modified:
+ continue
+
+ parsed_toml[args.user] = payee
+
+ encoder = toml.encoder.TomlPreserveInlineDictEncoder()
+ ttxt = toml.dumps(parsed_toml, encoder=encoder)
+ print(ttxt)
+
+ #update = bz.build_update(cf_payees_list=ttxt)
+ bz.update_bugs([bug.id], {'cf_payees_list': ttxt})
+
+
+if __name__ == "__main__":
+ main()
+from contextlib import contextmanager
+from budget_sync.ordered_set import OrderedSet
from bugzilla import Bugzilla
from bugzilla.bug import Bug
-from typing import Iterator, Union
+from typing import Any, Callable, Dict, Iterator, List, Type, Union
from enum import Enum
+from io import StringIO
+import os
class BugStatus(Enum):
raise
+@contextmanager
+def tty_out():
+ try:
+ if hasattr(os, "ctermid"):
+ term = open(os.ctermid(), "wt", encoding="utf-8")
+ elif os.name == "nt":
+ term = open("CONOUT$", "wt", encoding="utf-8")
+ else:
+ term = None
+ except OSError:
+ # no terminal available
+ term = None
+ try: # can't use `with` since it doesn't work with None
+ yield term
+ finally:
+ if term is not None:
+ term.close()
+
+
def all_bugs(bz: Bugzilla) -> Iterator[Bug]:
chunk_start = 1
chunk_size = 100
- while True:
- bugs = bz.getbugs(list(range(chunk_start, chunk_start + chunk_size)))
- chunk_start += chunk_size
- yield from bugs
- if len(bugs) < chunk_size:
+ with tty_out() as term:
+ while True:
+ bugs = list(range(chunk_start, chunk_start + chunk_size))
+ bugs = bz.getbugs(bugs)
+ chunk_start += chunk_size
+ # progress indicator, should go to terminal
+ if term is not None:
+ print("bugs loaded", len(bugs), chunk_start, flush=True,
+ file=term)
+ if len(bugs) == 0:
+ return
+ yield from bugs
+
+
+class SequencePrettyPrinter:
+ def __init__(self,
+ pretty_printer: "PrettyPrinter",
+ start_delimiter: str = '[\n',
+ end_delimiter: str = ']',
+ item_separator: str = ',\n'):
+ self.__pretty_printer = pretty_printer
+ self.__start_delimiter = start_delimiter
+ self.__end_delimiter = end_delimiter
+ self.__item_separator = item_separator
+
+ def __enter__(self):
+ self.__pretty_printer.write_raw_str(self.__start_delimiter)
+ self.__pretty_printer.adjust_indent(1)
+ return self
+
+ def item(self, value: Any):
+ self.__pretty_printer.write(value)
+ self.__pretty_printer.write_raw_str(self.__item_separator)
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.__pretty_printer.adjust_indent(-1)
+ self.__pretty_printer.write_raw_str(self.__end_delimiter)
+
+
+class MappingPrettyPrinter:
+ def __init__(self,
+ pretty_printer: "PrettyPrinter",
+ start_delimiter: str = '[\n',
+ end_delimiter: str = ']',
+ key_value_separator: str = ': ',
+ item_separator: str = ',\n'):
+ self.__pretty_printer = pretty_printer
+ self.__start_delimiter = start_delimiter
+ self.__end_delimiter = end_delimiter
+ self.__key_value_separator = key_value_separator
+ self.__item_separator = item_separator
+
+ def __enter__(self):
+ self.__pretty_printer.write_raw_str(self.__start_delimiter)
+ self.__pretty_printer.adjust_indent(1)
+ return self
+
+ def item(self, key: Any, value: Any):
+ self.__pretty_printer.write(key)
+ self.__pretty_printer.write_raw_str(self.__key_value_separator)
+ self.__pretty_printer.write(value)
+ self.__pretty_printer.write_raw_str(self.__item_separator)
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.__pretty_printer.adjust_indent(-1)
+ self.__pretty_printer.write_raw_str(self.__end_delimiter)
+
+
+class TypePrettyPrinter:
+ def __init__(self,
+ pretty_printer: "PrettyPrinter",
+ name: str,
+ start_delimiter: str = '(\n',
+ end_delimiter: str = ')',
+ key_value_separator: str = '=',
+ item_separator: str = ',\n'):
+ self.__pretty_printer = pretty_printer
+ self.__name = name
+ self.__start_delimiter = start_delimiter
+ self.__end_delimiter = end_delimiter
+ self.__key_value_separator = key_value_separator
+ self.__item_separator = item_separator
+
+ def __enter__(self):
+ self.__pretty_printer.write_raw_str(self.__name)
+ self.__pretty_printer.write_raw_str(self.__start_delimiter)
+ self.__pretty_printer.adjust_indent(1)
+ return self
+
+ def field(self, key: str, value: Any):
+ self.__pretty_printer.write_raw_str(key)
+ self.__pretty_printer.write_raw_str(self.__key_value_separator)
+ self.__pretty_printer.write(value)
+ self.__pretty_printer.write_raw_str(self.__item_separator)
+
+ def try_field(self, key: str, value: Callable[[], Any], exception: Type[Exception]):
+ self.__pretty_printer.write_raw_str(key)
+ self.__pretty_printer.write_raw_str(self.__key_value_separator)
+ self.__pretty_printer.try_write(value, exception)
+ self.__pretty_printer.write_raw_str(self.__item_separator)
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.__pretty_printer.adjust_indent(-1)
+ self.__pretty_printer.write_raw_str(self.__end_delimiter)
+
+
+# pprint isn't good enough, it doesn't allow customization for types
+class PrettyPrinter:
+ __PRETTY_PRINT_OVERRIDES: Dict[type,
+ Callable[["PrettyPrinter", Any], None]] = {}
+
+ def __init__(self):
+ self.__writer = StringIO()
+ self.__depth = 0
+ self.__at_line_start = True
+
+ def adjust_indent(self, amount: int):
+ self.__depth += amount
+
+ @contextmanager
+ def indent(self):
+ self.adjust_indent(1)
+ yield
+ self.adjust_indent(-1)
+
+ def write_raw_str(self, s: str):
+ for ch in s:
+ if ch == '\n':
+ self.__at_line_start = True
+ elif self.__at_line_start:
+ self.__at_line_start = False
+ self.__writer.write(' ' * (4 * self.__depth))
+ self.__writer.write(ch)
+
+ def write(self, obj: Any):
+ override = self.__PRETTY_PRINT_OVERRIDES.get(type(obj), None)
+ if override is not None:
+ override(self, obj)
+ else:
+ f = getattr(obj, "__pretty_print__", None)
+ if f is not None:
+ f(self)
+ else:
+ self.write_raw_str(repr(obj))
+
+ def try_write(self, f: Callable[[], Any], exception: Type[Exception]):
+ try:
+ v = f()
+ except exception:
+ self.write_raw_str(f"<failed with exception {exception.__name__}>")
return
+ self.write(v)
+
+ def get_str(self) -> str:
+ return self.__writer.getvalue()
+
+ @classmethod
+ def run(cls, obj: Any) -> str:
+ instance = cls()
+ instance.write(obj)
+ return instance.get_str()
+
+ @classmethod
+ def register_pretty_print_override(cls, ty: type, override: Callable[["PrettyPrinter", Any], None]):
+ cls.__PRETTY_PRINT_OVERRIDES[ty] = override
+
+ def type_pp(self, name: str, **kwargs) -> TypePrettyPrinter:
+ return TypePrettyPrinter(self, name, **kwargs)
+
+ def mapping_pp(self, **kwargs) -> MappingPrettyPrinter:
+ return MappingPrettyPrinter(self, **kwargs)
+
+ def sequence_pp(self, **kwargs) -> SequencePrettyPrinter:
+ return SequencePrettyPrinter(self, **kwargs)
+
+ def __write_list(self, obj: List[Any]):
+ with self.sequence_pp() as pp:
+ for i in obj:
+ pp.item(i)
+
+ __PRETTY_PRINT_OVERRIDES[list] = __write_list
+
+ def __write_tuple(self, obj: List[Any]):
+ with self.sequence_pp(start_delimiter='(\n',
+ end_delimiter=')',) as pp:
+ for i in obj:
+ pp.item(i)
+
+ __PRETTY_PRINT_OVERRIDES[tuple] = __write_tuple
+
+ def __write_ordered_set(self, obj: OrderedSet[Any]):
+ with self.sequence_pp(start_delimiter='OrderedSet([\n',
+ end_delimiter='])',) as pp:
+ for i in obj:
+ pp.item(i)
+
+ __PRETTY_PRINT_OVERRIDES[OrderedSet] = __write_ordered_set
+
+ def __write_dict(self, obj: Dict[Any, Any]):
+ with self.mapping_pp() as pp:
+ for k, v in obj.items():
+ pp.item(k, v)
+
+ __PRETTY_PRINT_OVERRIDES[dict] = __write_dict
+
+ def __write_ellipsis(self, obj: Any):
+ self.write_raw_str("...")
+
+ __PRETTY_PRINT_OVERRIDES[type(...)] = __write_ellipsis
+
+
+def pretty_print(obj: Any, **kwargs):
+ print(PrettyPrinter.run(obj), **kwargs)
--- /dev/null
+import csv
+from enum import Enum, auto
+from io import StringIO
+from typing import Any, Callable, Dict, List, Optional
+from budget_sync.budget_graph import BudgetGraph, Node, PayeeState, PaymentSummary
+from pathlib import Path
+from budget_sync.config import Milestone
+from budget_sync.money import Money
+from budget_sync.write_budget_markdown import markdown_escape
+
+
+def _budget_csv_row(budget_graph: BudgetGraph, milestone: Milestone, node: Optional[Node]) -> Dict[str, str]:
+ row_fns: Dict[str, Callable[[Node], Any]] = {
+ 'bug_id': lambda node: node.bug.id,
+ 'excl_subtasks': lambda node: node.budget_excluding_subtasks,
+ 'inc_subtasks': lambda node: node.budget_including_subtasks,
+ 'fixed_excl_subtasks': lambda node: node.fixed_budget_excluding_subtasks,
+ 'fixed_inc_subtasks': lambda node: node.fixed_budget_including_subtasks,
+ 'req_excl_subtasks': lambda node: node.submitted_excluding_subtasks,
+ 'paid_excl_subtasks': lambda node: node.paid_excluding_subtasks,
+ }
+ milestone_people = budget_graph.milestone_people[milestone]
+
+ def handle_person(person):
+ # need a nested function in order to create a new person variable
+ # for this iteration that can be bound to the lambdas
+ id = person.identifier
+ row_fns.update({
+ id + " (planned amt)": lambda node: node.payment_summaries[person].total,
+ id + " (req amt)": lambda node: node.payment_summaries[person].total_submitted,
+ id + " (req date)": lambda node: node.payment_summaries[person].submitted_date,
+ id + " (paid amt)": lambda node: node.payment_summaries[person].total_paid,
+ id + " (paid date)": lambda node: node.payment_summaries[person].paid_date,
+ })
+ for person in milestone_people:
+ handle_person(person)
+ row = {k: "" for k in row_fns.keys()}
+ if node is None:
+ return row
+ for k, fn in row_fns.items():
+ try:
+ v = fn(node)
+ except KeyError:
+ continue
+ if v is not None:
+ row[k] = str(v)
+ return row
+
+
+def _budget_csv_for_milestone(budget_graph: BudgetGraph, milestone: Milestone) -> str:
+ with StringIO() as string_io:
+ writer = csv.DictWriter(
+ string_io,
+ _budget_csv_row(budget_graph, milestone, None).keys(),
+ lineterminator="\n")
+ writer.writeheader()
+ for node in budget_graph.assigned_nodes_for_milestones[milestone]:
+ # skip uninteresting nodes
+ if len(node.payments) == 0 \
+ and node.budget_excluding_subtasks == 0 \
+ and node.budget_including_subtasks == 0:
+ continue
+ row = _budget_csv_row(budget_graph, milestone, node)
+ writer.writerow(row)
+ return string_io.getvalue()
+
+
+def write_budget_csv(budget_graph: BudgetGraph,
+ output_dir: Path):
+ output_dir.mkdir(parents=True, exist_ok=True)
+ milestones = budget_graph.config.milestones
+ csv_paths: Dict[Milestone, Path] = {}
+ for milestone in milestones.values():
+ csv_text = _budget_csv_for_milestone(budget_graph, milestone)
+ csv_paths[milestone] = output_dir.joinpath(
+ f"{milestone.identifier}.csv")
+ csv_paths[milestone].write_text(csv_text, encoding="utf-8")
+
+ markdown_text = "\n".join(f"# {markdown_escape(milestone.identifier)}\n"
+ "\n"
+ f"[[!table format=csv file=\"{path!s}\"]]"
+ for milestone, path in csv_paths.items())
+ output_dir.joinpath("csvs.mdwn").write_text(
+ markdown_text, encoding="utf-8")
+from collections import defaultdict
from pathlib import Path
from typing import Dict, List, Any, Optional
from io import StringIO
import enum
-from budget_sync.budget_graph import BudgetGraph, Node, Payment, PayeeState
-from budget_sync.config import Person, Milestone, Config
+from budget_sync.budget_graph import BudgetGraph, Node, Payment, PayeeState, PaymentSummary
+from budget_sync.config import Person, Milestone
+from budget_sync.money import Money
+from budget_sync.ordered_set import OrderedSet
from budget_sync.util import BugStatus
if headers == self.last_headers:
return
for i in range(len(headers)):
+ if not headers[i].startswith("\n" + "#" * (i + 1) + " "):
+ raise ValueError(
+ "invalid markdown header. if you're not trying to make a"
+ " markdown header, don't use write_headers!")
if i >= len(self.last_headers):
print(headers[i], file=self.buffer)
self.last_headers.append(headers[i])
self.last_headers, headers)
assert headers == self.last_headers
+ def write_node_header(self,
+ headers: List[str],
+ node: Optional[Node]):
+ self.write_headers(headers)
+ if node is None:
+ print("* None", file=self.buffer)
+ return
+ summary = markdown_escape(node.bug.summary)
+ print(f"* [Bug #{node.bug.id}]({node.bug_url}):\n {summary}",
+ file=self.buffer)
+
def write_node(self,
headers: List[str],
node: Node,
payment: Optional[Payment]):
- self.write_headers(headers)
- summary = markdown_escape(node.bug.summary)
- print(f"* [Bug #{node.bug.id}]({node.bug_url}): {summary}",
- file=self.buffer)
+ self.write_node_header(headers, node)
if payment is not None:
if node.fixed_budget_excluding_subtasks \
!= node.budget_excluding_subtasks:
f" which is €{node.budget_excluding_subtasks})")
else:
total = f"€{node.fixed_budget_excluding_subtasks}"
+ if payment.submitted:
+ print(f" * submitted on {payment.submitted}",
+ file=self.buffer)
+ if payment.paid:
+ print(f" * paid on {payment.paid}",
+ file=self.buffer)
if payment.amount != node.fixed_budget_excluding_subtasks \
or payment.amount != node.budget_excluding_subtasks:
print(f" * €{payment.amount} out of total of {total}",
else:
print(f" * €{payment.amount} which is the total amount",
file=self.buffer)
+ closest = node.closest_bug_in_mou
+ if closest is node:
+ print(f" * this task is a MoU Milestone",
+ file=self.buffer)
+ elif closest is not None:
+ print(f" * this task is part of MoU Milestone\n"
+ f" [Bug #{closest.bug.id}]({closest.bug_url})",
+ file=self.buffer)
+ elif payment is not None: # only report this if there's a payment
+ print(f" * neither this task nor any parent tasks are in "
+ f"the MoU",
+ file=self.buffer)
def _markdown_for_person(person: Person,
payments_dict: Dict[Milestone, List[Payment]],
- assigned_nodes: List[Node]) -> str:
+ assigned_nodes: List[Node],
+ nodes_subset: Optional[OrderedSet[Node]] = None,
+ ) -> str:
+ def node_included(node: Node) -> bool:
+ return nodes_subset is None or node in nodes_subset
writer = MarkdownWriter()
print(f"<!-- autogenerated by budget-sync -->", file=writer.buffer)
- writer.write_headers([f"# {person.identifier}"])
+ writer.write_headers([f"\n# {person.full_name} ({person.identifier})\n"])
print(file=writer.buffer)
- status_tracking_header = "# Status Tracking"
+ status_tracking_header = "\n# Status Tracking\n"
writer.write_headers([status_tracking_header])
displayed_nodes_dict: Dict[DisplayStatus, List[Node]]
displayed_nodes_dict = {i: [] for i in DisplayStatus}
displayed_nodes_dict[display_status].append(node)
def write_display_status_chunk(display_status: DisplayStatus):
- display_status_header = f"## {display_status.value}"
+ display_status_header = f"\n## {display_status.value}\n"
for node in displayed_nodes_dict[display_status]:
+ if not node_included(node):
+ continue
if display_status == DisplayStatus.Completed:
payment_found = False
for payment in node.payments.values():
write_display_status_chunk(display_status)
for payee_state in PayeeState:
+ # work out headers per status
if payee_state == PayeeState.NotYetSubmitted:
- display_status_header = f"## Completed but not yet paid"
+ display_status_header = "\n## Payment not yet submitted\n"
+ subtotals_msg = ("\nMoU Milestone subtotals for not "
+ "yet submitted payments\n")
elif payee_state == PayeeState.Submitted:
- display_status_header = f"## Submitted to NLNet but not yet paid"
+ display_status_header = ("\n## Submitted to NLNet but "
+ "not yet paid\n")
+ subtotals_msg = ("\nMoU Milestone subtotals for "
+ "submitted but not yet paid payments\n")
else:
assert payee_state == PayeeState.Paid
- display_status_header = f"## Paid by NLNet"
+ display_status_header = "\n## Paid by NLNet\n"
+ subtotals_msg = ("\nMoU Milestone subtotals for paid "
+ "payments\n")
+ # list all the payments grouped by Grant
for milestone, payments_list in payments_dict.items():
- milestone_header = f"### {milestone.identifier}"
+ milestone_header = f"\n### {milestone.identifier}\n"
+ mou_subtotals: Dict[Optional[Node], Money] = defaultdict(Money)
+ headers = [status_tracking_header,
+ display_status_header,
+ milestone_header]
+ # write out the payments and also compute the subtotals per
+ # mou milestone
for payment in payments_list:
- if payment.state == payee_state:
- writer.write_node(headers=[status_tracking_header,
- display_status_header,
- milestone_header],
+ node = payment.node
+ if payment.state == payee_state and node_included(node):
+ mou_subtotals[node.closest_bug_in_mou] += payment.amount
+ writer.write_node(headers=headers,
node=payment.node, payment=payment)
+ # now display the mou subtotals. really, this should be before
+ for node, subtotal in mou_subtotals.items():
+ writer.write_headers(headers)
+ print(subtotals_msg, file=writer.buffer)
+ writer.write_node_header(headers, node)
+ if node is None:
+ budget = ""
+ elif node.fixed_budget_including_subtasks \
+ != node.budget_including_subtasks:
+ budget = (" out of total including subtasks of "
+ f"€{node.fixed_budget_including_subtasks}"
+ " (budget is fixed from amount appearing in "
+ "bug report, which is "
+ f"€{node.budget_including_subtasks})")
+ else:
+ budget = (" out of total including subtasks of "
+ f"€{node.fixed_budget_including_subtasks}")
+ print(f" * subtotal €{subtotal}{budget}",
+ file=writer.buffer)
# write_display_status_chunk(DisplayStatus.NotYetStarted)
def write_budget_markdown(budget_graph: BudgetGraph,
- output_dir: Path):
+ output_dir: Path,
+ nodes_subset: Optional[OrderedSet[Node]] = None):
output_dir.mkdir(parents=True, exist_ok=True)
for person, payments_dict in budget_graph.payments.items():
markdown = _markdown_for_person(person,
payments_dict,
- budget_graph.assigned_nodes[person])
+ budget_graph.assigned_nodes[person],
+ nodes_subset)
output_file = output_dir.joinpath(person.output_markdown_file)
output_file.write_text(markdown, encoding="utf-8")
+
+
+def markdown_for_person(budget_graph: BudgetGraph, person: Person,
+ nodes_subset: Optional[OrderedSet[Node]] = None,
+ ) -> str:
+ return _markdown_for_person(person, budget_graph.payments[person],
+ budget_graph.assigned_nodes[person],
+ nodes_subset)
--- /dev/null
+#!/bin/bash
+set -e
+umask 077
+
+function fail()
+{
+ printf "error: $@" >&2
+ echo >&2
+ exit 1
+}
+
+any_warnings=0
+
+function warn()
+{
+ any_warnings=1
+ printf "warning: $@" >&2
+ echo >&2
+}
+
+function install_deps()
+{
+ if ! which wg >/dev/null; then
+ echo "wireguard not found, installing:"
+ sudo apt update
+ sudo apt install wireguard
+ fi
+}
+
+function check_cidr_addr()
+{
+ local addr="$1"
+ [[ "$addr" =~ ^(0|[1-9][0-9]*)'.'(0|[1-9][0-9]*)'.'(0|[1-9][0-9]*)'.'(0|[1-9][0-9]*)'/'(0|[1-9][0-9]*)$ ]] \
+ || fail "can't parse IPv4 CIDR address: %q" "$addr"
+ ((${BASH_REMATCH[1]} <= 255 &&
+ ${BASH_REMATCH[2]} <= 255 &&
+ ${BASH_REMATCH[3]} <= 255 &&
+ ${BASH_REMATCH[4]} <= 255 &&
+ ${BASH_REMATCH[5]} <= 32)) \
+ || fail "invalid IPv4 CIDR address: %s" "$addr"
+}
+
+function init_server()
+{
+ local server_iface_name="$1" listen_port="$2" server_iface_addr="$3"
+ check_cidr_addr "$server_iface_addr"
+ set -o noclobber
+ local private_key="$(wg genkey)"
+ local config_contents="[Interface]
+PrivateKey = $private_key
+ListenPort = $listen_port
+Address = $server_iface_addr
+SaveConfig = true
+"
+ echo "$config_contents" > "/etc/wireguard/$server_iface_name.conf"
+ echo "Created config for $server_iface_name"
+ echo "You need to allow UDP port $listen_port through your firewall"
+}
+
+function add_client()
+{
+ local server_iface_name="$1" server_public_addr="$2" client_config="$3" client_iface_addr="$4"
+ check_cidr_addr "$client_iface_addr"
+ if [[ " $(wg show interfaces) " =~ " $server_iface_name " ]]; then
+ fail "You need to shutdown interface %s first:\nwg-quick down %s\nor:\nsystemctl stop wg-quick@%s" \
+ "$server_iface_name" "$server_iface_name" "$server_iface_name"
+ fi
+ local server_iface_conf="/etc/wireguard/$server_iface_name.conf"
+ local lines line key eq_value value section="" server_private_key=""
+ local server_iface_addr="" server_listen_port=""
+ mapfile -t lines < "$server_iface_conf"
+ for line in "${lines[@]}"; do
+ line="${line%%#*}" # remove comments
+ [[ "$line" =~ ^([^=]*)('='(.*))?$ ]] || fail "regex failed -- not supposed to happen"
+ key="${BASH_REMATCH[1]}"
+ eq_value="${BASH_REMATCH[2]}"
+ value="${BASH_REMATCH[3]}"
+ key="${key#"${key%%[![:space:]]*}"}" # remove leading whitespace
+ key="${key%"${key##*[![:space:]]}"}" # remove trailing whitespace
+ value="${value#"${value%%[![:space:]]*}"}" # remove leading whitespace
+ value="${value%"${value##*[![:space:]]}"}" # remove trailing whitespace
+ [[ "$key" == "" && "$eq_value" == "" ]] && continue
+ if [[ "$key" =~ ^'['(.+)']'$ && "$eq_value" == "" ]]; then
+ section="${BASH_REMATCH[1]}"
+ case "$section" in
+ 'Interface'|'Peer')
+ ;;
+ *)
+ warn "unknown config section %s" "$key"
+ ;;
+ esac
+ elif [[ "$section" == "Interface" ]]; then
+ case "$key" in
+ 'PrivateKey')
+ [[ "$value" == "" ]] && fail "empty [Interface] PrivateKey value"
+ server_private_key="$value"
+ ;;
+ 'Address')
+ [[ "$value" == "" ]] && fail "empty [Interface] Address value"
+ [[ "$server_iface_addr" != ""
+ || "$value" =~ [[:space:],] ]] && \
+ fail "multiple [Interface] Address values not supported"
+ server_iface_addr="$value"
+ check_cidr_addr "$server_iface_addr"
+ ;;
+ 'ListenPort')
+ [[ "$value" =~ ^[1-9][0-9]* ]] || \
+ fail "invalid [Interface] ListenPort value: %s" "$value"
+ server_listen_port="$value"
+ ;;
+ 'SaveConfig'|'FwMark'|'DNS'|'MTU')
+ ;;
+ 'Table'|'PreUp'|'PreDown'|'PostUp'|'PostDown')
+ ;;
+ *)
+ warn "unknown config key [Interface] %s" "$key"
+ ;;
+ esac
+ elif [[ "$section" == "Peer" ]]; then
+ case "$key" in
+ 'AllowedIPs'|'PublicKey'|'PresharedKey'|'Endpoint'|'PersistentKeepalive')
+ ;;
+ *)
+ warn "unknown config key [Peer] %s" "$key"
+ ;;
+ esac
+ fi
+ done
+ [[ "$server_iface_addr" != "" ]] || fail "missing [Interface] Address config key"
+ [[ "$server_private_key" != "" ]] || fail "missing [Interface] PrivateKey config key"
+ [[ "$server_listen_port" != "" ]] || fail "missing [Interface] ListenPort config key"
+ if ((any_warnings)); then
+ echo -n "Warnings generated, do you want to continue? [y/N]: " >&2
+ local cont
+ read -r cont
+ if [[ "$cont" != "y" ]]; then
+ exit 1
+ fi
+ fi
+ local client_private_key="$(wg genkey)"
+ local server_public_key="$(wg pubkey <<<"$server_private_key")"
+ local client_public_key="$(wg pubkey <<<"$client_private_key")"
+ local preshared_key="$(wg genpsk)"
+ local client_config_contents="[Interface]
+PrivateKey = $client_private_key
+Address = $client_iface_addr
+SaveConfig = true
+
+[Peer]
+PublicKey = $server_public_key
+PresharedKey = $preshared_key
+AllowedIPs = $server_iface_addr
+Endpoint = $server_public_addr:$server_listen_port
+PersistentKeepalive = 25
+"
+ set -o noclobber
+ echo "$client_config_contents" > "$client_config"
+ set +o noclobber
+ local server_config_new_peer="
+[Peer]
+PublicKey = $client_public_key
+PresharedKey = $preshared_key
+AllowedIPs = $client_iface_addr
+PersistentKeepalive = 25
+"
+ echo "$server_config_new_peer" >> "$server_iface_conf"
+ cat <<EOF
+Client added.
+Move $client_config to /etc/wireguard/<client-interface>.conf on the client,
+making sure that it is owned by root and has mode 600. Make sure it is NOT
+left lying around since it contains the private key for the client, as well
+as the preshared key.
+
+Once you did that, run on the server:
+wg-quick up $server_iface_name
+or:
+systemctl start wg-quick@$server_iface_name
+and run on the client:
+wg-quick up <client-interface>
+or:
+systemctl start wg-quick@<client-interface>
+EOF
+}
+
+case "$1" in
+ init-server)
+ install_deps
+ init_server "${@:2}"
+ ;;
+ add-client)
+ install_deps
+ add_client "${@:2}"
+ ;;
+ *)
+ cat >&2 <<EOF
+Usage: $0 init-server <server-iface-name> <listen-port> <server-iface-addr>
+ or: $0 add-client <server-iface-name> <server-public-addr> <client-config.conf> <client-iface-addr>
+
+init-server: create a new wireguard config for the server, writes to
+ '/etc/wireguard/<server-iface-name>.conf'
+
+add-client: add a client to the wireguard config for the server at
+ '/etc/wireguard/<server-iface-name>.conf'
+ Writes the generated client config to <client-config.conf>.
+ The client will connect to the server through
+ public IP or DNS address <server-public-addr>.
+EOF
+ exit 1
+ ;;
+esac
projects / utils.git / commitdiff