bin/pick/core.py

   1 # Copyright © 2019-2020 Intel Corporation
   2
   3 # Permission is hereby granted, free of charge, to any person obtaining a copy
   4 # of this software and associated documentation files (the "Software"), to deal
   5 # in the Software without restriction, including without limitation the rights
   6 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   7 # copies of the Software, and to permit persons to whom the Software is
   8 # furnished to do so, subject to the following conditions:
   9
  10 # The above copyright notice and this permission notice shall be included in
  11 # all copies or substantial portions of the Software.
  12
  13 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  16 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  17 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  18 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  19 # SOFTWARE.
  20
  21 """Core data structures and routines for pick."""
  22
  23 import asyncio
  24 import enum
  25 import json
  26 import pathlib
  27 import re
  28 import subprocess
  29 import typing
  30
  31 import attr
  32
  33 if typing.TYPE_CHECKING:
  34     from .ui import UI
  35
  36     import typing_extensions
  37
  38     class CommitDict(typing_extensions.TypedDict):
  39
  40         sha: str
  41         description: str
  42         nominated: bool
  43         nomination_type: typing.Optional[int]
  44         resolution: typing.Optional[int]
  45         master_sha: typing.Optional[str]
  46         because_sha: typing.Optional[str]
  47
  48 IS_FIX = re.compile(r'^\s*fixes:\s*([a-f0-9]{6,40})', flags=re.MULTILINE | re.IGNORECASE)
  49 # FIXME: I dislike the duplication in this regex, but I couldn't get it to work otherwise
  50 IS_CC = re.compile(r'^\s*cc:\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*\<?mesa-stable',
  51                    flags=re.MULTILINE | re.IGNORECASE)
  52 IS_REVERT = re.compile(r'This reverts commit ([0-9a-f]{40})')
  53
  54 # XXX: hack
  55 SEM = asyncio.Semaphore(50)
  56
  57 COMMIT_LOCK = asyncio.Lock()
  58
  59 git_toplevel = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
  60                                        stderr=subprocess.DEVNULL).decode("ascii").strip()
  61 pick_status_json = pathlib.Path(git_toplevel) / '.pick_status.json'
  62
  63
  64 class PickUIException(Exception):
  65     pass
  66
  67
  68 @enum.unique
  69 class NominationType(enum.Enum):
  70
  71     CC = 0
  72     FIXES = 1
  73     REVERT = 2
  74
  75
  76 @enum.unique
  77 class Resolution(enum.Enum):
  78
  79     UNRESOLVED = 0
  80     MERGED = 1
  81     DENOMINATED = 2
  82     BACKPORTED = 3
  83     NOTNEEDED = 4
  84
  85
  86 async def commit_state(*, amend: bool = False, message: str = 'Update') -> bool:
  87     """Commit the .pick_status.json file."""
  88     async with COMMIT_LOCK:
  89         p = await asyncio.create_subprocess_exec(
  90             'git', 'add', pick_status_json.as_posix(),
  91             stdout=asyncio.subprocess.DEVNULL,
  92             stderr=asyncio.subprocess.DEVNULL,
  93         )
  94         v = await p.wait()
  95         if v != 0:
  96             return False
  97
  98         if amend:
  99             cmd = ['--amend', '--no-edit']
 100         else:
 101             cmd = ['--message', f'.pick_status.json: {message}']
 102         p = await asyncio.create_subprocess_exec(
 103             'git', 'commit', *cmd,
 104             stdout=asyncio.subprocess.DEVNULL,
 105             stderr=asyncio.subprocess.DEVNULL,
 106         )
 107         v = await p.wait()
 108         if v != 0:
 109             return False
 110     return True
 111
 112
 113 @attr.s(slots=True)
 114 class Commit:
 115
 116     sha: str = attr.ib()
 117     description: str = attr.ib()
 118     nominated: bool = attr.ib(False)
 119     nomination_type: typing.Optional[NominationType] = attr.ib(None)
 120     resolution: Resolution = attr.ib(Resolution.UNRESOLVED)
 121     master_sha: typing.Optional[str] = attr.ib(None)
 122     because_sha: typing.Optional[str] = attr.ib(None)
 123
 124     def to_json(self) -> 'CommitDict':
 125         d: typing.Dict[str, typing.Any] = attr.asdict(self)
 126         if self.nomination_type is not None:
 127             d['nomination_type'] = self.nomination_type.value
 128         if self.resolution is not None:
 129             d['resolution'] = self.resolution.value
 130         return typing.cast('CommitDict', d)
 131
 132     @classmethod
 133     def from_json(cls, data: 'CommitDict') -> 'Commit':
 134         c = cls(data['sha'], data['description'], data['nominated'], master_sha=data['master_sha'], because_sha=data['because_sha'])
 135         if data['nomination_type'] is not None:
 136             c.nomination_type = NominationType(data['nomination_type'])
 137         if data['resolution'] is not None:
 138             c.resolution = Resolution(data['resolution'])
 139         return c
 140
 141     async def apply(self, ui: 'UI') -> typing.Tuple[bool, str]:
 142         # FIXME: This isn't really enough if we fail to cherry-pick because the
 143         # git tree will still be dirty
 144         async with COMMIT_LOCK:
 145             p = await asyncio.create_subprocess_exec(
 146                 'git', 'cherry-pick', '-x', self.sha,
 147                 stdout=asyncio.subprocess.DEVNULL,
 148                 stderr=asyncio.subprocess.PIPE,
 149             )
 150             _, err = await p.communicate()
 151
 152         if p.returncode != 0:
 153             return (False, err.decode())
 154
 155         self.resolution = Resolution.MERGED
 156         await ui.feedback(f'{self.sha} ({self.description}) applied successfully')
 157
 158         # Append the changes to the .pickstatus.json file
 159         ui.save()
 160         v = await commit_state(amend=True)
 161         return (v, '')
 162
 163     async def abort_cherry(self, ui: 'UI', err: str) -> None:
 164         await ui.feedback(f'{self.sha} ({self.description}) failed to apply\n{err}')
 165         async with COMMIT_LOCK:
 166             p = await asyncio.create_subprocess_exec(
 167                 'git', 'cherry-pick', '--abort',
 168                 stdout=asyncio.subprocess.DEVNULL,
 169                 stderr=asyncio.subprocess.DEVNULL,
 170             )
 171             r = await p.wait()
 172         await ui.feedback(f'{"Successfully" if r == 0 else "Failed to"} abort cherry-pick.')
 173
 174     async def denominate(self, ui: 'UI') -> bool:
 175         self.resolution = Resolution.DENOMINATED
 176         ui.save()
 177         v = await commit_state(message=f'Mark {self.sha} as denominated')
 178         assert v
 179         await ui.feedback(f'{self.sha} ({self.description}) denominated successfully')
 180         return True
 181
 182     async def backport(self, ui: 'UI') -> bool:
 183         self.resolution = Resolution.BACKPORTED
 184         ui.save()
 185         v = await commit_state(message=f'Mark {self.sha} as backported')
 186         assert v
 187         await ui.feedback(f'{self.sha} ({self.description}) backported successfully')
 188         return True
 189
 190     async def resolve(self, ui: 'UI') -> None:
 191         self.resolution = Resolution.MERGED
 192         ui.save()
 193         v = await commit_state(amend=True)
 194         assert v
 195         await ui.feedback(f'{self.sha} ({self.description}) committed successfully')
 196
 197
 198 async def get_new_commits(sha: str) -> typing.List[typing.Tuple[str, str]]:
 199     # Try to get the authoritative upstream master
 200     p = await asyncio.create_subprocess_exec(
 201         'git', 'for-each-ref', '--format=%(upstream)', 'refs/heads/master',
 202         stdout=asyncio.subprocess.PIPE,
 203         stderr=asyncio.subprocess.DEVNULL)
 204     out, _ = await p.communicate()
 205     upstream = out.decode().strip()
 206
 207     p = await asyncio.create_subprocess_exec(
 208         'git', 'log', '--pretty=oneline', f'{sha}..{upstream}',
 209         stdout=asyncio.subprocess.PIPE,
 210         stderr=asyncio.subprocess.DEVNULL)
 211     out, _ = await p.communicate()
 212     assert p.returncode == 0, f"git log didn't work: {sha}"
 213     return list(split_commit_list(out.decode().strip()))
 214
 215
 216 def split_commit_list(commits: str) -> typing.Generator[typing.Tuple[str, str], None, None]:
 217     if not commits:
 218         return
 219     for line in commits.split('\n'):
 220         v = tuple(line.split(' ', 1))
 221         assert len(v) == 2, 'this is really just for mypy'
 222         yield typing.cast(typing.Tuple[str, str], v)
 223
 224
 225 async def is_commit_in_branch(sha: str) -> bool:
 226     async with SEM:
 227         p = await asyncio.create_subprocess_exec(
 228             'git', 'merge-base', '--is-ancestor', sha, 'HEAD',
 229             stdout=asyncio.subprocess.DEVNULL,
 230             stderr=asyncio.subprocess.DEVNULL,
 231         )
 232         await p.wait()
 233     return p.returncode == 0
 234
 235
 236 async def full_sha(sha: str) -> str:
 237     async with SEM:
 238         p = await asyncio.create_subprocess_exec(
 239             'git', 'rev-parse', sha,
 240             stdout=asyncio.subprocess.PIPE,
 241             stderr=asyncio.subprocess.DEVNULL,
 242         )
 243         out, _ = await p.communicate()
 244     if p.returncode:
 245         raise PickUIException(f'Invalid Sha {sha}')
 246     return out.decode().strip()
 247
 248
 249 async def resolve_nomination(commit: 'Commit', version: str) -> 'Commit':
 250     async with SEM:
 251         p = await asyncio.create_subprocess_exec(
 252             'git', 'log', '--format=%B', '-1', commit.sha,
 253             stdout=asyncio.subprocess.PIPE,
 254             stderr=asyncio.subprocess.DEVNULL,
 255         )
 256         _out, _ = await p.communicate()
 257         assert p.returncode == 0, f'git log for {commit.sha} failed'
 258     out = _out.decode()
 259
 260     # We give precedence to fixes and cc tags over revert tags.
 261     # XXX: not having the walrus operator available makes me sad :=
 262     m = IS_FIX.search(out)
 263     if m:
 264         # We set the nomination_type and because_sha here so that we can later
 265         # check to see if this fixes another staged commit.
 266         try:
 267             commit.because_sha = fixed = await full_sha(m.group(1))
 268         except PickUIException:
 269             pass
 270         else:
 271             commit.nomination_type = NominationType.FIXES
 272             if await is_commit_in_branch(fixed):
 273                 commit.nominated = True
 274                 return commit
 275
 276     m = IS_CC.search(out)
 277     if m:
 278         if m.groups() == (None, None) or version in m.groups():
 279             commit.nominated = True
 280             commit.nomination_type = NominationType.CC
 281             return commit
 282
 283     m = IS_REVERT.search(out)
 284     if m:
 285         # See comment for IS_FIX path
 286         try:
 287             commit.because_sha = reverted = await full_sha(m.group(1))
 288         except PickUIException:
 289             pass
 290         else:
 291             commit.nomination_type = NominationType.REVERT
 292             if await is_commit_in_branch(reverted):
 293                 commit.nominated = True
 294                 return commit
 295
 296     return commit
 297
 298
 299 async def resolve_fixes(commits: typing.List['Commit'], previous: typing.List['Commit']) -> None:
 300     """Determine if any of the undecided commits fix/revert a staged commit.
 301
 302     The are still needed if they apply to a commit that is staged for
 303     inclusion, but not yet included.
 304
 305     This must be done in order, because a commit 3 might fix commit 2 which
 306     fixes commit 1.
 307     """
 308     shas: typing.Set[str] = set(c.sha for c in previous if c.nominated)
 309     assert None not in shas, 'None in shas'
 310
 311     for commit in reversed(commits):
 312         if not commit.nominated and commit.nomination_type is NominationType.FIXES:
 313             commit.nominated = commit.because_sha in shas
 314
 315         if commit.nominated:
 316             shas.add(commit.sha)
 317
 318     for commit in commits:
 319         if (commit.nomination_type is NominationType.REVERT and
 320                 commit.because_sha in shas):
 321             for oldc in reversed(commits):
 322                 if oldc.sha == commit.because_sha:
 323                     # In this case a commit that hasn't yet been applied is
 324                     # reverted, we don't want to apply that commit at all
 325                     oldc.nominated = False
 326                     oldc.resolution = Resolution.DENOMINATED
 327                     commit.nominated = False
 328                     commit.resolution = Resolution.DENOMINATED
 329                     shas.remove(commit.because_sha)
 330                     break
 331
 332
 333 async def gather_commits(version: str, previous: typing.List['Commit'],
 334                          new: typing.List[typing.Tuple[str, str]], cb) -> typing.List['Commit']:
 335     # We create an array of the final size up front, then we pass that array
 336     # to the "inner" co-routine, which is turned into a list of tasks and
 337     # collected by asyncio.gather. We do this to allow the tasks to be
 338     # asynchronously gathered, but to also ensure that the commits list remains
 339     # in order.
 340     m_commits: typing.List[typing.Optional['Commit']] = [None] * len(new)
 341     tasks = []
 342
 343     async def inner(commit: 'Commit', version: str,
 344                     commits: typing.List[typing.Optional['Commit']],
 345                     index: int, cb) -> None:
 346         commits[index] = await resolve_nomination(commit, version)
 347         cb()
 348
 349     for i, (sha, desc) in enumerate(new):
 350         tasks.append(asyncio.ensure_future(
 351             inner(Commit(sha, desc), version, m_commits, i, cb)))
 352
 353     await asyncio.gather(*tasks)
 354     assert None not in m_commits
 355     commits = typing.cast(typing.List[Commit], m_commits)
 356
 357     await resolve_fixes(commits, previous)
 358
 359     for commit in commits:
 360         if commit.resolution is Resolution.UNRESOLVED and not commit.nominated:
 361             commit.resolution = Resolution.NOTNEEDED
 362
 363     return commits
 364
 365
 366 def load() -> typing.List['Commit']:
 367     if not pick_status_json.exists():
 368         return []
 369     with pick_status_json.open('r') as f:
 370         raw = json.load(f)
 371         return [Commit.from_json(c) for c in raw]
 372
 373
 374 def save(commits: typing.Iterable['Commit']) -> None:
 375     commits = list(commits)
 376     with pick_status_json.open('wt') as f:
 377         json.dump([c.to_json() for c in commits], f, indent=4)
 378
 379     asyncio.ensure_future(commit_state(message=f'Update to {commits[0].sha}'))