codekingpro/portable-devtools
114k
1# Copyright (c) 2012 Google Inc. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import errno
6import filecmp
7import os.path
8import re
9import shlex
10import subprocess
11import sys
12import tempfile
13from collections.abc import MutableSet
14
15
16# A minimal memoizing decorator. It'll blow up if the args aren't immutable,
17# among other "problems".
18class memoize:
19 def __init__(self, func):
20 self.func = func
21 self.cache = {}
22
23 def __call__(self, *args):
24 try:
25 return self.cache[args]
26 except KeyError:
27 result = self.func(*args)
28 self.cache[args] = result
29 return result
30
31
32class GypError(Exception):
33 """Error class representing an error, which is to be presented
34 to the user. The main entry point will catch and display this.
35 """
36
37
38def ExceptionAppend(e, msg):
39 """Append a message to the given exception's message."""
40 if not e.args:
41 e.args = (msg,)
42 elif len(e.args) == 1:
43 e.args = (str(e.args[0]) + " " + msg,)
44 else:
45 e.args = (str(e.args[0]) + " " + msg,) + e.args[1:]
46
47
48def FindQualifiedTargets(target, qualified_list):
49 """
50 Given a list of qualified targets, return the qualified targets for the
51 specified |target|.
52 """
53 return [t for t in qualified_list if ParseQualifiedTarget(t)[1] == target]
54
55
56def ParseQualifiedTarget(target):
57 # Splits a qualified target into a build file, target name and toolset.
58
59 # NOTE: rsplit is used to disambiguate the Windows drive letter separator.
60 target_split = target.rsplit(":", 1)
61 if len(target_split) == 2:
62 [build_file, target] = target_split
63 else:
64 build_file = None
65
66 target_split = target.rsplit("#", 1)
67 if len(target_split) == 2:
68 [target, toolset] = target_split
69 else:
70 toolset = None
71
72 return [build_file, target, toolset]
73
74
75def ResolveTarget(build_file, target, toolset):
76 # This function resolves a target into a canonical form:
77 # - a fully defined build file, either absolute or relative to the current
78 # directory
79 # - a target name
80 # - a toolset
81 #
82 # build_file is the file relative to which 'target' is defined.
83 # target is the qualified target.
84 # toolset is the default toolset for that target.
85 [parsed_build_file, target, parsed_toolset] = ParseQualifiedTarget(target)
86
87 if parsed_build_file:
88 if build_file:
89 # If a relative path, parsed_build_file is relative to the directory
90 # containing build_file. If build_file is not in the current directory,
91 # parsed_build_file is not a usable path as-is. Resolve it by
92 # interpreting it as relative to build_file. If parsed_build_file is
93 # absolute, it is usable as a path regardless of the current directory,
94 # and os.path.join will return it as-is.
95 build_file = os.path.normpath(
96 os.path.join(os.path.dirname(build_file), parsed_build_file)
97 )
98 # Further (to handle cases like ../cwd), make it relative to cwd)
99 if not os.path.isabs(build_file):
100 build_file = RelativePath(build_file, ".")
101 else:
102 build_file = parsed_build_file
103
104 if parsed_toolset:
105 toolset = parsed_toolset
106
107 return [build_file, target, toolset]
108
109
110def BuildFile(fully_qualified_target):
111 # Extracts the build file from the fully qualified target.
112 return ParseQualifiedTarget(fully_qualified_target)[0]
113
114
115def GetEnvironFallback(var_list, default):
116 """Look up a key in the environment, with fallback to secondary keys
117 and finally falling back to a default value."""
118 for var in var_list:
119 if var in os.environ:
120 return os.environ[var]
121 return default
122
123
124def QualifiedTarget(build_file, target, toolset):
125 # "Qualified" means the file that a target was defined in and the target
126 # name, separated by a colon, suffixed by a # and the toolset name:
127 # /path/to/file.gyp:target_name#toolset
128 fully_qualified = build_file + ":" + target
129 if toolset:
130 fully_qualified = fully_qualified + "#" + toolset
131 return fully_qualified
132
133
134@memoize
135def RelativePath(path, relative_to, follow_path_symlink=True):
136 # Assuming both |path| and |relative_to| are relative to the current
137 # directory, returns a relative path that identifies path relative to
138 # relative_to.
139 # If |follow_symlink_path| is true (default) and |path| is a symlink, then
140 # this method returns a path to the real file represented by |path|. If it is
141 # false, this method returns a path to the symlink. If |path| is not a
142 # symlink, this option has no effect.
143
144 # Convert to normalized (and therefore absolute paths).
145 path = os.path.realpath(path) if follow_path_symlink else os.path.abspath(path)
146 relative_to = os.path.realpath(relative_to)
147
148 # On Windows, we can't create a relative path to a different drive, so just
149 # use the absolute path.
150 if sys.platform == "win32" and (
151 os.path.splitdrive(path)[0].lower()
152 != os.path.splitdrive(relative_to)[0].lower()
153 ):
154 return path
155
156 # Split the paths into components.
157 path_split = path.split(os.path.sep)
158 relative_to_split = relative_to.split(os.path.sep)
159
160 # Determine how much of the prefix the two paths share.
161 prefix_len = len(os.path.commonprefix([path_split, relative_to_split]))
162
163 # Put enough ".." components to back up out of relative_to to the common
164 # prefix, and then append the part of path_split after the common prefix.
165 relative_split = [os.path.pardir] * (
166 len(relative_to_split) - prefix_len
167 ) + path_split[prefix_len:]
168
169 if len(relative_split) == 0:
170 # The paths were the same.
171 return ""
172
173 # Turn it back into a string and we're done.
174 return os.path.join(*relative_split)
175
176
177@memoize
178def InvertRelativePath(path, toplevel_dir=None):
179 """Given a path like foo/bar that is relative to toplevel_dir, return
180 the inverse relative path back to the toplevel_dir.
181
182 E.g. os.path.normpath(os.path.join(path, InvertRelativePath(path)))
183 should always produce the empty string, unless the path contains symlinks.
184 """
185 if not path:
186 return path
187 toplevel_dir = "." if toplevel_dir is None else toplevel_dir
188 return RelativePath(toplevel_dir, os.path.join(toplevel_dir, path))
189
190
191def FixIfRelativePath(path, relative_to):
192 # Like RelativePath but returns |path| unchanged if it is absolute.
193 if os.path.isabs(path):
194 return path
195 return RelativePath(path, relative_to)
196
197
198def UnrelativePath(path, relative_to):
199 # Assuming that |relative_to| is relative to the current directory, and |path|
200 # is a path relative to the dirname of |relative_to|, returns a path that
201 # identifies |path| relative to the current directory.
202 rel_dir = os.path.dirname(relative_to)
203 return os.path.normpath(os.path.join(rel_dir, path))
204
205
206# re objects used by EncodePOSIXShellArgument. See IEEE 1003.1 XCU.2.2 at
207# http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_02
208# and the documentation for various shells.
209
210# _quote is a pattern that should match any argument that needs to be quoted
211# with double-quotes by EncodePOSIXShellArgument. It matches the following
212# characters appearing anywhere in an argument:
213# \t, \n, space parameter separators
214# # comments
215# $ expansions (quoted to always expand within one argument)
216# % called out by IEEE 1003.1 XCU.2.2
217# & job control
218# ' quoting
219# (, ) subshell execution
220# *, ?, [ pathname expansion
221# ; command delimiter
222# <, >, | redirection
223# = assignment
224# {, } brace expansion (bash)
225# ~ tilde expansion
226# It also matches the empty string, because "" (or '') is the only way to
227# represent an empty string literal argument to a POSIX shell.
228#
229# This does not match the characters in _escape, because those need to be
230# backslash-escaped regardless of whether they appear in a double-quoted
231# string.
232_quote = re.compile("[\t\n #$%&'()*;<=>?[{|}~]|^$")
233
234# _escape is a pattern that should match any character that needs to be
235# escaped with a backslash, whether or not the argument matched the _quote
236# pattern. _escape is used with re.sub to backslash anything in _escape's
237# first match group, hence the (parentheses) in the regular expression.
238#
239# _escape matches the following characters appearing anywhere in an argument:
240# " to prevent POSIX shells from interpreting this character for quoting
241# \ to prevent POSIX shells from interpreting this character for escaping
242# ` to prevent POSIX shells from interpreting this character for command
243# substitution
244# Missing from this list is $, because the desired behavior of
245# EncodePOSIXShellArgument is to permit parameter (variable) expansion.
246#
247# Also missing from this list is !, which bash will interpret as the history
248# expansion character when history is enabled. bash does not enable history
249# by default in non-interactive shells, so this is not thought to be a problem.
250# ! was omitted from this list because bash interprets "\!" as a literal string
251# including the backslash character (avoiding history expansion but retaining
252# the backslash), which would not be correct for argument encoding. Handling
253# this case properly would also be problematic because bash allows the history
254# character to be changed with the histchars shell variable. Fortunately,
255# as history is not enabled in non-interactive shells and
256# EncodePOSIXShellArgument is only expected to encode for non-interactive
257# shells, there is no room for error here by ignoring !.
258_escape = re.compile(r'(["\\`])')
259
260
261def EncodePOSIXShellArgument(argument):
262 """Encodes |argument| suitably for consumption by POSIX shells.
263
264 argument may be quoted and escaped as necessary to ensure that POSIX shells
265 treat the returned value as a literal representing the argument passed to
266 this function. Parameter (variable) expansions beginning with $ are allowed
267 to remain intact without escaping the $, to allow the argument to contain
268 references to variables to be expanded by the shell.
269 """
270
271 if not isinstance(argument, str):
272 argument = str(argument)
273
274 quote = '"' if _quote.search(argument) else ""
275
276 encoded = quote + re.sub(_escape, r"\\\1", argument) + quote
277
278 return encoded
279
280
281def EncodePOSIXShellList(list):
282 """Encodes |list| suitably for consumption by POSIX shells.
283
284 Returns EncodePOSIXShellArgument for each item in list, and joins them
285 together using the space character as an argument separator.
286 """
287
288 encoded_arguments = []
289 for argument in list:
290 encoded_arguments.append(EncodePOSIXShellArgument(argument))
291 return " ".join(encoded_arguments)
292
293
294def DeepDependencyTargets(target_dicts, roots):
295 """Returns the recursive list of target dependencies."""
296 dependencies = set()
297 pending = set(roots)
298 while pending:
299 # Pluck out one.
300 r = pending.pop()
301 # Skip if visited already.
302 if r in dependencies:
303 continue
304 # Add it.
305 dependencies.add(r)
306 # Add its children.
307 spec = target_dicts[r]
308 pending.update(set(spec.get("dependencies", [])))
309 pending.update(set(spec.get("dependencies_original", [])))
310 return list(dependencies - set(roots))
311
312
313def BuildFileTargets(target_list, build_file):
314 """From a target_list, returns the subset from the specified build_file."""
315 return [p for p in target_list if BuildFile(p) == build_file]
316
317
318def AllTargets(target_list, target_dicts, build_file):
319 """Returns all targets (direct and dependencies) for the specified build_file."""
320 bftargets = BuildFileTargets(target_list, build_file)
321 deptargets = DeepDependencyTargets(target_dicts, bftargets)
322 return bftargets + deptargets
323
324
325def WriteOnDiff(filename):
326 """Write to a file only if the new contents differ.
327
328 Arguments:
329 filename: name of the file to potentially write to.
330 Returns:
331 A file like object which will write to temporary file and only overwrite
332 the target if it differs (on close).
333 """
334
335 class Writer:
336 """Wrapper around file which only covers the target if it differs."""
337
338 def __init__(self):
339 # On Cygwin remove the "dir" argument
340 # `C:` prefixed paths are treated as relative,
341 # consequently ending up with current dir "/cygdrive/c/..."
342 # being prefixed to those, which was
343 # obviously a non-existent path,
344 # for example: "/cygdrive/c/<some folder>/C:\<my win style abs path>".
345 # For more details see:
346 # https://docs.python.org/2/library/tempfile.html#tempfile.mkstemp
347 base_temp_dir = "" if IsCygwin() else os.path.dirname(filename)
348 # Pick temporary file.
349 tmp_fd, self.tmp_path = tempfile.mkstemp(
350 suffix=".tmp",
351 prefix=os.path.split(filename)[1] + ".gyp.",
352 dir=base_temp_dir,
353 )
354 try:
355 self.tmp_file = os.fdopen(tmp_fd, "wb")
356 except Exception:
357 # Don't leave turds behind.
358 os.unlink(self.tmp_path)
359 raise
360
361 def __getattr__(self, attrname):
362 # Delegate everything else to self.tmp_file
363 return getattr(self.tmp_file, attrname)
364
365 def close(self):
366 try:
367 # Close tmp file.
368 self.tmp_file.close()
369 # Determine if different.
370 same = False
371 try:
372 same = filecmp.cmp(self.tmp_path, filename, False)
373 except OSError as e:
374 if e.errno != errno.ENOENT:
375 raise
376
377 if same:
378 # The new file is identical to the old one, just get rid of the new
379 # one.
380 os.unlink(self.tmp_path)
381 else:
382 # The new file is different from the old one,
383 # or there is no old one.
384 # Rename the new file to the permanent name.
385 #
386 # tempfile.mkstemp uses an overly restrictive mode, resulting in a
387 # file that can only be read by the owner, regardless of the umask.
388 # There's no reason to not respect the umask here,
389 # which means that an extra hoop is required
390 # to fetch it and reset the new file's mode.
391 #
392 # No way to get the umask without setting a new one? Set a safe one
393 # and then set it back to the old value.
394 umask = os.umask(0o77)
395 os.umask(umask)
396 os.chmod(self.tmp_path, 0o666 & ~umask)
397 if sys.platform == "win32" and os.path.exists(filename):
398 # NOTE: on windows (but not cygwin) rename will not replace an
399 # existing file, so it must be preceded with a remove.
400 # Sadly there is no way to make the switch atomic.
401 os.remove(filename)
402 os.rename(self.tmp_path, filename)
403 except Exception:
404 # Don't leave turds behind.
405 os.unlink(self.tmp_path)
406 raise
407
408 def write(self, s):
409 self.tmp_file.write(s.encode("utf-8"))
410
411 return Writer()
412
413
414def EnsureDirExists(path):
415 """Make sure the directory for |path| exists."""
416 try:
417 os.makedirs(os.path.dirname(path))
418 except OSError:
419 pass
420
421
422def GetCompilerPredefines(): # -> dict
423 cmd = []
424 defines = {}
425
426 # shlex.split() will eat '\' in posix mode, but
427 # setting posix=False will preserve extra '"' cause CreateProcess fail on Windows
428 # this makes '\' in %CC_target% and %CFLAGS% work
429 def replace_sep(s):
430 return s.replace(os.sep, "/") if os.sep != "/" else s
431
432 if CC := os.environ.get("CC_target") or os.environ.get("CC"):
433 cmd += shlex.split(replace_sep(CC))
434 if CFLAGS := os.environ.get("CFLAGS"):
435 cmd += shlex.split(replace_sep(CFLAGS))
436 elif CXX := os.environ.get("CXX_target") or os.environ.get("CXX"):
437 cmd += shlex.split(replace_sep(CXX))
438 if CXXFLAGS := os.environ.get("CXXFLAGS"):
439 cmd += shlex.split(replace_sep(CXXFLAGS))
440 else:
441 return defines
442
443 if sys.platform == "win32":
444 fd, input = tempfile.mkstemp(suffix=".c")
445 real_cmd = [*cmd, "-dM", "-E", "-x", "c", input]
446 try:
447 os.close(fd)
448 stdout = subprocess.run(
449 real_cmd, shell=True, capture_output=True, check=True
450 ).stdout
451 except subprocess.CalledProcessError as e:
452 print(
453 "Warning: failed to get compiler predefines\n"
454 "cmd: %s\n"
455 "status: %d" % (e.cmd, e.returncode),
456 file=sys.stderr,
457 )
458 return defines
459 finally:
460 os.unlink(input)
461 else:
462 input = "/dev/null"
463 real_cmd = [*cmd, "-dM", "-E", "-x", "c", input]
464 try:
465 stdout = subprocess.run(
466 real_cmd, shell=False, capture_output=True, check=True
467 ).stdout
468 except subprocess.CalledProcessError as e:
469 print(
470 "Warning: failed to get compiler predefines\n"
471 "cmd: %s\n"
472 "status: %d" % (e.cmd, e.returncode),
473 file=sys.stderr,
474 )
475 return defines
476
477 lines = stdout.decode("utf-8").replace("\r\n", "\n").split("\n")
478 for line in lines:
479 if (line or "").startswith("#define "):
480 _, key, *value = line.split(" ")
481 defines[key] = " ".join(value)
482 return defines
483
484
485def GetFlavorByPlatform():
486 """Returns |params.flavor| if it's set, the system's default flavor else."""
487 flavors = {
488 "cygwin": "win",
489 "win32": "win",
490 "darwin": "mac",
491 }
492
493 if sys.platform in flavors:
494 return flavors[sys.platform]
495 if sys.platform.startswith("sunos"):
496 return "solaris"
497 if sys.platform.startswith(("dragonfly", "freebsd")):
498 return "freebsd"
499 if sys.platform.startswith("openbsd"):
500 return "openbsd"
501 if sys.platform.startswith("netbsd"):
502 return "netbsd"
503 if sys.platform.startswith("aix"):
504 return "aix"
505 if sys.platform.startswith(("os390", "zos")):
506 return "zos"
507 if sys.platform == "os400":
508 return "os400"
509
510 return "linux"
511
512
513def GetFlavor(params):
514 if "flavor" in params:
515 return params["flavor"]
516
517 defines = GetCompilerPredefines()
518 if "__EMSCRIPTEN__" in defines:
519 return "emscripten"
520 if "__wasm__" in defines:
521 return "wasi" if "__wasi__" in defines else "wasm"
522
523 return GetFlavorByPlatform()
524
525
526def CopyTool(flavor, out_path, generator_flags={}):
527 """Finds (flock|mac|win)_tool.gyp in the gyp directory and copies it
528 to |out_path|."""
529 # aix and solaris just need flock emulation. mac and win use more complicated
530 # support scripts.
531 prefix = {
532 "aix": "flock",
533 "os400": "flock",
534 "solaris": "flock",
535 "mac": "mac",
536 "ios": "mac",
537 "win": "win",
538 }.get(flavor, None)
539 if not prefix:
540 return
541
542 # Slurp input file.
543 source_path = os.path.join(
544 os.path.dirname(os.path.abspath(__file__)), "%s_tool.py" % prefix
545 )
546 with open(source_path) as source_file:
547 source = source_file.readlines()
548
549 # Set custom header flags.
550 header = "# Generated by gyp. Do not edit.\n"
551 mac_toolchain_dir = generator_flags.get("mac_toolchain_dir", None)
552 if flavor == "mac" and mac_toolchain_dir:
553 header += "import os;\nos.environ['DEVELOPER_DIR']='%s'\n" % mac_toolchain_dir
554
555 # Add header and write it out.
556 tool_path = os.path.join(out_path, "gyp-%s-tool" % prefix)
557 with open(tool_path, "w") as tool_file:
558 tool_file.write("".join([source[0], header] + source[1:]))
559
560 # Make file executable.
561 os.chmod(tool_path, 0o755)
562
563
564# From Alex Martelli,
565# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52560
566# ASPN: Python Cookbook: Remove duplicates from a sequence
567# First comment, dated 2001/10/13.
568# (Also in the printed Python Cookbook.)
569
570
571def uniquer(seq, idfun=lambda x: x):
572 seen = {}
573 result = []
574 for item in seq:
575 marker = idfun(item)
576 if marker in seen:
577 continue
578 seen[marker] = 1
579 result.append(item)
580 return result
581
582
583# Based on http://code.activestate.com/recipes/576694/.
584class OrderedSet(MutableSet): # noqa: PLW1641
585 # TODO (cclauss): Fix eq-without-hash ruff rule PLW1641
586 def __init__(self, iterable=None):
587 self.end = end = []
588 end += [None, end, end] # sentinel node for doubly linked list
589 self.map = {} # key --> [key, prev, next]
590 if iterable is not None:
591 self |= iterable
592
593 def __len__(self):
594 return len(self.map)
595
596 def __contains__(self, key):
597 return key in self.map
598
599 def add(self, key):
600 if key not in self.map:
601 end = self.end
602 curr = end[1]
603 curr[2] = end[1] = self.map[key] = [key, curr, end]
604
605 def discard(self, key):
606 if key in self.map:
607 key, prev_item, next_item = self.map.pop(key)
608 prev_item[2] = next_item
609 next_item[1] = prev_item
610
611 def __iter__(self):
612 end = self.end
613 curr = end[2]
614 while curr is not end:
615 yield curr[0]
616 curr = curr[2]
617
618 def __reversed__(self):
619 end = self.end
620 curr = end[1]
621 while curr is not end:
622 yield curr[0]
623 curr = curr[1]
624
625 # The second argument is an addition that causes a pylint warning.
626 def pop(self, last=True): # pylint: disable=W0221
627 if not self:
628 raise KeyError("set is empty")
629 key = self.end[1][0] if last else self.end[2][0]
630 self.discard(key)
631 return key
632
633 def __repr__(self):
634 if not self:
635 return f"{self.__class__.__name__}()"
636 return f"{self.__class__.__name__}({list(self)!r})"
637
638 def __eq__(self, other):
639 if isinstance(other, OrderedSet):
640 return len(self) == len(other) and list(self) == list(other)
641 return set(self) == set(other)
642
643 # Extensions to the recipe.
644 def update(self, iterable):
645 for i in iterable:
646 if i not in self:
647 self.add(i)
648
649
650class CycleError(Exception):
651 """An exception raised when an unexpected cycle is detected."""
652
653 def __init__(self, nodes):
654 self.nodes = nodes
655
656 def __str__(self):
657 return "CycleError: cycle involving: " + str(self.nodes)
658
659
660def TopologicallySorted(graph, get_edges):
661 r"""Topologically sort based on a user provided edge definition.
662
663 Args:
664 graph: A list of node names.
665 get_edges: A function mapping from node name to a hashable collection
666 of node names which this node has outgoing edges to.
667 Returns:
668 A list containing all of the node in graph in topological order.
669 It is assumed that calling get_edges once for each node and caching is
670 cheaper than repeatedly calling get_edges.
671 Raises:
672 CycleError in the event of a cycle.
673 Example:
674 graph = {'a': '$(b) $(c)', 'b': 'hi', 'c': '$(b)'}
675 def GetEdges(node):
676 return re.findall(r'\$\(([^))]\)', graph[node])
677 print TopologicallySorted(graph.keys(), GetEdges)
678 ==>
679 ['a', 'c', b']
680 """
681 get_edges = memoize(get_edges)
682 visited = set()
683 visiting = set()
684 ordered_nodes = []
685
686 def Visit(node):
687 if node in visiting:
688 raise CycleError(visiting)
689 if node in visited:
690 return
691 visited.add(node)
692 visiting.add(node)
693 for neighbor in get_edges(node):
694 Visit(neighbor)
695 visiting.remove(node)
696 ordered_nodes.insert(0, node)
697
698 for node in sorted(graph):
699 Visit(node)
700 return ordered_nodes
701
702
703def CrossCompileRequested():
704 # TODO: figure out how to not build extra host objects in the
705 # non-cross-compile case when this is enabled, and enable unconditionally.
706 return (
707 os.environ.get("GYP_CROSSCOMPILE")
708 or os.environ.get("AR_host")
709 or os.environ.get("CC_host")
710 or os.environ.get("CXX_host")
711 or os.environ.get("AR_target")
712 or os.environ.get("CC_target")
713 or os.environ.get("CXX_target")
714 )
715
716
717def IsCygwin():
718 try:
719 out = subprocess.Popen(
720 "uname", stdout=subprocess.PIPE, stderr=subprocess.STDOUT
721 )
722 stdout = out.communicate()[0].decode("utf-8")
723 return "CYGWIN" in str(stdout)
724 except Exception:
725 return False
726