dab6028a4b0ba9282bc4209e299422550745f72f
[mesa.git] / bin / pick / core.py
1 # Copyright © 2019-2020 Intel Corporation
2
3 # Permission is hereby granted, free of charge, to any person obtaining a copy
4 # of this software and associated documentation files (the "Software"), to deal
5 # in the Software without restriction, including without limitation the rights
6 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 # copies of the Software, and to permit persons to whom the Software is
8 # furnished to do so, subject to the following conditions:
9
10 # The above copyright notice and this permission notice shall be included in
11 # all copies or substantial portions of the Software.
12
13 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19 # SOFTWARE.
20
21 """Core data structures and routines for pick."""
22
23 import asyncio
24 import enum
25 import json
26 import pathlib
27 import re
28 import typing
29
30 import attr
31
32 if typing.TYPE_CHECKING:
33 from .ui import UI
34
35 import typing_extensions
36
37 class CommitDict(typing_extensions.TypedDict):
38
39 sha: str
40 description: str
41 nominated: bool
42 nomination_type: typing.Optional[int]
43 resolution: typing.Optional[int]
44 master_sha: typing.Optional[str]
45 because_sha: typing.Optional[str]
46
47 IS_FIX = re.compile(r'^\s*fixes:\s*([a-f0-9]{6,40})', flags=re.MULTILINE | re.IGNORECASE)
48 # FIXME: I dislike the duplication in this regex, but I couldn't get it to work otherwise
49 IS_CC = re.compile(r'^\s*cc:\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*\<?mesa-stable',
50 flags=re.MULTILINE | re.IGNORECASE)
51 IS_REVERT = re.compile(r'This reverts commit ([0-9a-f]{40})')
52
53 # XXX: hack
54 SEM = asyncio.Semaphore(50)
55
56 COMMIT_LOCK = asyncio.Lock()
57
58
59 class PickUIException(Exception):
60 pass
61
62
63 @enum.unique
64 class NominationType(enum.Enum):
65
66 CC = 0
67 FIXES = 1
68 REVERT = 2
69
70
71 @enum.unique
72 class Resolution(enum.Enum):
73
74 UNRESOLVED = 0
75 MERGED = 1
76 DENOMINATED = 2
77 BACKPORTED = 3
78 NOTNEEDED = 4
79
80
81 async def commit_state(*, amend: bool = False, message: str = 'Update') -> bool:
82 """Commit the .pick_status.json file."""
83 f = pathlib.Path(__file__).parent.parent.parent / '.pick_status.json'
84 async with COMMIT_LOCK:
85 p = await asyncio.create_subprocess_exec(
86 'git', 'add', f.as_posix(),
87 stdout=asyncio.subprocess.DEVNULL,
88 stderr=asyncio.subprocess.DEVNULL,
89 )
90 v = await p.wait()
91 if v != 0:
92 return False
93
94 if amend:
95 cmd = ['--amend', '--no-edit']
96 else:
97 cmd = ['--message', f'.pick_status.json: {message}']
98 p = await asyncio.create_subprocess_exec(
99 'git', 'commit', *cmd,
100 stdout=asyncio.subprocess.DEVNULL,
101 stderr=asyncio.subprocess.DEVNULL,
102 )
103 v = await p.wait()
104 if v != 0:
105 return False
106 return True
107
108
109 @attr.s(slots=True)
110 class Commit:
111
112 sha: str = attr.ib()
113 description: str = attr.ib()
114 nominated: bool = attr.ib(False)
115 nomination_type: typing.Optional[NominationType] = attr.ib(None)
116 resolution: Resolution = attr.ib(Resolution.UNRESOLVED)
117 master_sha: typing.Optional[str] = attr.ib(None)
118 because_sha: typing.Optional[str] = attr.ib(None)
119
120 def to_json(self) -> 'CommitDict':
121 d: typing.Dict[str, typing.Any] = attr.asdict(self)
122 if self.nomination_type is not None:
123 d['nomination_type'] = self.nomination_type.value
124 if self.resolution is not None:
125 d['resolution'] = self.resolution.value
126 return typing.cast('CommitDict', d)
127
128 @classmethod
129 def from_json(cls, data: 'CommitDict') -> 'Commit':
130 c = cls(data['sha'], data['description'], data['nominated'], master_sha=data['master_sha'], because_sha=data['because_sha'])
131 if data['nomination_type'] is not None:
132 c.nomination_type = NominationType(data['nomination_type'])
133 if data['resolution'] is not None:
134 c.resolution = Resolution(data['resolution'])
135 return c
136
137 async def apply(self, ui: 'UI') -> typing.Tuple[bool, str]:
138 # FIXME: This isn't really enough if we fail to cherry-pick because the
139 # git tree will still be dirty
140 async with COMMIT_LOCK:
141 p = await asyncio.create_subprocess_exec(
142 'git', 'cherry-pick', '-x', self.sha,
143 stdout=asyncio.subprocess.DEVNULL,
144 stderr=asyncio.subprocess.PIPE,
145 )
146 _, err = await p.communicate()
147
148 if p.returncode != 0:
149 return (False, err.decode())
150
151 self.resolution = Resolution.MERGED
152 await ui.feedback(f'{self.sha} ({self.description}) applied successfully')
153
154 # Append the changes to the .pickstatus.json file
155 ui.save()
156 v = await commit_state(amend=True)
157 return (v, '')
158
159 async def abort_cherry(self, ui: 'UI', err: str) -> None:
160 await ui.feedback(f'{self.sha} ({self.description}) failed to apply\n{err}')
161 async with COMMIT_LOCK:
162 p = await asyncio.create_subprocess_exec(
163 'git', 'cherry-pick', '--abort',
164 stdout=asyncio.subprocess.DEVNULL,
165 stderr=asyncio.subprocess.DEVNULL,
166 )
167 r = await p.wait()
168 await ui.feedback(f'{"Successfully" if r == 0 else "Failed to"} abort cherry-pick.')
169
170 async def denominate(self, ui: 'UI') -> bool:
171 self.resolution = Resolution.DENOMINATED
172 ui.save()
173 v = await commit_state(message=f'Mark {self.sha} as denominated')
174 assert v
175 await ui.feedback(f'{self.sha} ({self.description}) denominated successfully')
176 return True
177
178 async def backport(self, ui: 'UI') -> bool:
179 self.resolution = Resolution.BACKPORTED
180 ui.save()
181 v = await commit_state(message=f'Mark {self.sha} as backported')
182 assert v
183 await ui.feedback(f'{self.sha} ({self.description}) backported successfully')
184 return True
185
186 async def resolve(self, ui: 'UI') -> None:
187 self.resolution = Resolution.MERGED
188 ui.save()
189 v = await commit_state(amend=True)
190 assert v
191 await ui.feedback(f'{self.sha} ({self.description}) committed successfully')
192
193
194 async def get_new_commits(sha: str) -> typing.List[typing.Tuple[str, str]]:
195 # Try to get the authoritative upstream master
196 p = await asyncio.create_subprocess_exec(
197 'git', 'for-each-ref', '--format=%(upstream)', 'refs/heads/master',
198 stdout=asyncio.subprocess.PIPE,
199 stderr=asyncio.subprocess.DEVNULL)
200 out, _ = await p.communicate()
201 upstream = out.decode().strip()
202
203 p = await asyncio.create_subprocess_exec(
204 'git', 'log', '--pretty=oneline', f'{sha}..{upstream}',
205 stdout=asyncio.subprocess.PIPE,
206 stderr=asyncio.subprocess.DEVNULL)
207 out, _ = await p.communicate()
208 assert p.returncode == 0, f"git log didn't work: {sha}"
209 return list(split_commit_list(out.decode().strip()))
210
211
212 def split_commit_list(commits: str) -> typing.Generator[typing.Tuple[str, str], None, None]:
213 if not commits:
214 return
215 for line in commits.split('\n'):
216 v = tuple(line.split(' ', 1))
217 assert len(v) == 2, 'this is really just for mypy'
218 yield typing.cast(typing.Tuple[str, str], v)
219
220
221 async def is_commit_in_branch(sha: str) -> bool:
222 async with SEM:
223 p = await asyncio.create_subprocess_exec(
224 'git', 'merge-base', '--is-ancestor', sha, 'HEAD',
225 stdout=asyncio.subprocess.DEVNULL,
226 stderr=asyncio.subprocess.DEVNULL,
227 )
228 await p.wait()
229 return p.returncode == 0
230
231
232 async def full_sha(sha: str) -> str:
233 async with SEM:
234 p = await asyncio.create_subprocess_exec(
235 'git', 'rev-parse', sha,
236 stdout=asyncio.subprocess.PIPE,
237 stderr=asyncio.subprocess.DEVNULL,
238 )
239 out, _ = await p.communicate()
240 if p.returncode:
241 raise PickUIException(f'Invalid Sha {sha}')
242 return out.decode().strip()
243
244
245 async def resolve_nomination(commit: 'Commit', version: str) -> 'Commit':
246 async with SEM:
247 p = await asyncio.create_subprocess_exec(
248 'git', 'log', '--format=%B', '-1', commit.sha,
249 stdout=asyncio.subprocess.PIPE,
250 stderr=asyncio.subprocess.DEVNULL,
251 )
252 _out, _ = await p.communicate()
253 assert p.returncode == 0, f'git log for {commit.sha} failed'
254 out = _out.decode()
255
256 # We give precedence to fixes and cc tags over revert tags.
257 # XXX: not having the walrus operator available makes me sad :=
258 m = IS_FIX.search(out)
259 if m:
260 # We set the nomination_type and because_sha here so that we can later
261 # check to see if this fixes another staged commit.
262 try:
263 commit.because_sha = fixed = await full_sha(m.group(1))
264 except PickUIException:
265 pass
266 else:
267 commit.nomination_type = NominationType.FIXES
268 if await is_commit_in_branch(fixed):
269 commit.nominated = True
270 return commit
271
272 m = IS_CC.search(out)
273 if m:
274 if m.groups() == (None, None) or version in m.groups():
275 commit.nominated = True
276 commit.nomination_type = NominationType.CC
277 return commit
278
279 m = IS_REVERT.search(out)
280 if m:
281 # See comment for IS_FIX path
282 try:
283 commit.because_sha = reverted = await full_sha(m.group(1))
284 except PickUIException:
285 pass
286 else:
287 commit.nomination_type = NominationType.REVERT
288 if await is_commit_in_branch(reverted):
289 commit.nominated = True
290 return commit
291
292 return commit
293
294
295 async def resolve_fixes(commits: typing.List['Commit'], previous: typing.List['Commit']) -> None:
296 """Determine if any of the undecided commits fix/revert a staged commit.
297
298 The are still needed if they apply to a commit that is staged for
299 inclusion, but not yet included.
300
301 This must be done in order, because a commit 3 might fix commit 2 which
302 fixes commit 1.
303 """
304 shas: typing.Set[str] = set(c.sha for c in previous if c.nominated)
305 assert None not in shas, 'None in shas'
306
307 for commit in reversed(commits):
308 if not commit.nominated and commit.nomination_type is NominationType.FIXES:
309 commit.nominated = commit.because_sha in shas
310
311 if commit.nominated:
312 shas.add(commit.sha)
313
314 for commit in commits:
315 if (commit.nomination_type is NominationType.REVERT and
316 commit.because_sha in shas):
317 for oldc in reversed(commits):
318 if oldc.sha == commit.because_sha:
319 # In this case a commit that hasn't yet been applied is
320 # reverted, we don't want to apply that commit at all
321 oldc.nominated = False
322 oldc.resolution = Resolution.DENOMINATED
323 commit.nominated = False
324 commit.resolution = Resolution.DENOMINATED
325 shas.remove(commit.because_sha)
326 break
327
328
329 async def gather_commits(version: str, previous: typing.List['Commit'],
330 new: typing.List[typing.Tuple[str, str]], cb) -> typing.List['Commit']:
331 # We create an array of the final size up front, then we pass that array
332 # to the "inner" co-routine, which is turned into a list of tasks and
333 # collected by asyncio.gather. We do this to allow the tasks to be
334 # asynchronously gathered, but to also ensure that the commits list remains
335 # in order.
336 m_commits: typing.List[typing.Optional['Commit']] = [None] * len(new)
337 tasks = []
338
339 async def inner(commit: 'Commit', version: str,
340 commits: typing.List[typing.Optional['Commit']],
341 index: int, cb) -> None:
342 commits[index] = await resolve_nomination(commit, version)
343 cb()
344
345 for i, (sha, desc) in enumerate(new):
346 tasks.append(asyncio.ensure_future(
347 inner(Commit(sha, desc), version, m_commits, i, cb)))
348
349 await asyncio.gather(*tasks)
350 assert None not in m_commits
351 commits = typing.cast(typing.List[Commit], m_commits)
352
353 await resolve_fixes(commits, previous)
354
355 for commit in commits:
356 if commit.resolution is Resolution.UNRESOLVED and not commit.nominated:
357 commit.resolution = Resolution.NOTNEEDED
358
359 return commits
360
361
362 def load() -> typing.List['Commit']:
363 p = pathlib.Path(__file__).parent.parent.parent / '.pick_status.json'
364 if not p.exists():
365 return []
366 with p.open('r') as f:
367 raw = json.load(f)
368 return [Commit.from_json(c) for c in raw]
369
370
371 def save(commits: typing.Iterable['Commit']) -> None:
372 p = pathlib.Path(__file__).parent.parent.parent / '.pick_status.json'
373 commits = list(commits)
374 with p.open('wt') as f:
375 json.dump([c.to_json() for c in commits], f, indent=4)
376
377 asyncio.ensure_future(commit_state(message=f'Update to {commits[0].sha}'))