codekingpro/portable-devtools
114k
1# Copyright (c) 2012 Google Inc. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6import ast
7import multiprocessing
8import os.path
9import re
10import shlex
11import signal
12import subprocess
13import sys
14import threading
15import traceback
16
17from packaging.version import Version
18
19import gyp.common
20import gyp.simple_copy
21from gyp.common import GypError, OrderedSet
22
23# A list of types that are treated as linkable.
24linkable_types = [
25 "executable",
26 "shared_library",
27 "loadable_module",
28 "mac_kernel_extension",
29 "windows_driver",
30]
31
32# A list of sections that contain links to other targets.
33dependency_sections = ["dependencies", "export_dependent_settings"]
34
35# base_path_sections is a list of sections defined by GYP that contain
36# pathnames. The generators can provide more keys, the two lists are merged
37# into path_sections, but you should call IsPathSection instead of using either
38# list directly.
39base_path_sections = [
40 "destination",
41 "files",
42 "include_dirs",
43 "inputs",
44 "libraries",
45 "outputs",
46 "sources",
47]
48path_sections = set()
49
50# These per-process dictionaries are used to cache build file data when loading
51# in parallel mode.
52per_process_data = {}
53per_process_aux_data = {}
54
55
56def IsPathSection(section):
57 # If section ends in one of the '=+?!' characters, it's applied to a section
58 # without the trailing characters. '/' is notably absent from this list,
59 # because there's no way for a regular expression to be treated as a path.
60 while section and section[-1:] in "=+?!":
61 section = section[:-1]
62
63 if section in path_sections:
64 return True
65
66 # Sections matching the regexp '_(dir|file|path)s?$' are also
67 # considered PathSections. Using manual string matching since that
68 # is much faster than the regexp and this can be called hundreds of
69 # thousands of times so micro performance matters.
70 if "_" in section:
71 tail = section[-6:]
72 if tail[-1] == "s":
73 tail = tail[:-1]
74 if tail[-5:] in ("_file", "_path"):
75 return True
76 return tail[-4:] == "_dir"
77
78 return False
79
80
81# base_non_configuration_keys is a list of key names that belong in the target
82# itself and should not be propagated into its configurations. It is merged
83# with a list that can come from the generator to
84# create non_configuration_keys.
85base_non_configuration_keys = [
86 # Sections that must exist inside targets and not configurations.
87 "actions",
88 "configurations",
89 "copies",
90 "default_configuration",
91 "dependencies",
92 "dependencies_original",
93 "libraries",
94 "postbuilds",
95 "product_dir",
96 "product_extension",
97 "product_name",
98 "product_prefix",
99 "rules",
100 "run_as",
101 "sources",
102 "standalone_static_library",
103 "suppress_wildcard",
104 "target_name",
105 "toolset",
106 "toolsets",
107 "type",
108 # Sections that can be found inside targets or configurations, but that
109 # should not be propagated from targets into their configurations.
110 "variables",
111]
112non_configuration_keys = []
113
114# Keys that do not belong inside a configuration dictionary.
115invalid_configuration_keys = [
116 "actions",
117 "all_dependent_settings",
118 "configurations",
119 "dependencies",
120 "direct_dependent_settings",
121 "libraries",
122 "link_settings",
123 "sources",
124 "standalone_static_library",
125 "target_name",
126 "type",
127]
128
129# Controls whether or not the generator supports multiple toolsets.
130multiple_toolsets = False
131
132# Paths for converting filelist paths to output paths: {
133# toplevel,
134# qualified_output_dir,
135# }
136generator_filelist_paths = None
137
138
139def GetIncludedBuildFiles(build_file_path, aux_data, included=None):
140 """Return a list of all build files included into build_file_path.
141
142 The returned list will contain build_file_path as well as all other files
143 that it included, either directly or indirectly. Note that the list may
144 contain files that were included into a conditional section that evaluated
145 to false and was not merged into build_file_path's dict.
146
147 aux_data is a dict containing a key for each build file or included build
148 file. Those keys provide access to dicts whose "included" keys contain
149 lists of all other files included by the build file.
150
151 included should be left at its default None value by external callers. It
152 is used for recursion.
153
154 The returned list will not contain any duplicate entries. Each build file
155 in the list will be relative to the current directory.
156 """
157
158 if included is None:
159 included = []
160
161 if build_file_path in included:
162 return included
163
164 included.append(build_file_path)
165
166 for included_build_file in aux_data[build_file_path].get("included", []):
167 GetIncludedBuildFiles(included_build_file, aux_data, included)
168
169 return included
170
171
172def CheckedEval(file_contents):
173 """Return the eval of a gyp file.
174 The gyp file is restricted to dictionaries and lists only, and
175 repeated keys are not allowed.
176 Note that this is slower than eval() is.
177 """
178
179 syntax_tree = ast.parse(file_contents)
180 assert isinstance(syntax_tree, ast.Module)
181 c1 = syntax_tree.body
182 assert len(c1) == 1
183 c2 = c1[0]
184 assert isinstance(c2, ast.Expr)
185 return CheckNode(c2.value, [])
186
187
188def CheckNode(node, keypath):
189 if isinstance(node, ast.Dict):
190 dict = {}
191 for key, value in zip(node.keys, node.values):
192 assert isinstance(key, ast.Str)
193 key = key.s
194 if key in dict:
195 raise GypError(
196 "Key '"
197 + key
198 + "' repeated at level "
199 + repr(len(keypath) + 1)
200 + " with key path '"
201 + ".".join(keypath)
202 + "'"
203 )
204 kp = list(keypath) # Make a copy of the list for descending this node.
205 kp.append(key)
206 dict[key] = CheckNode(value, kp)
207 return dict
208 elif isinstance(node, ast.List):
209 children = []
210 for index, child in enumerate(node.elts):
211 kp = list(keypath) # Copy list.
212 kp.append(repr(index))
213 children.append(CheckNode(child, kp))
214 return children
215 elif isinstance(node, ast.Str):
216 return node.s
217 else:
218 raise TypeError(
219 "Unknown AST node at key path '" + ".".join(keypath) + "': " + repr(node)
220 )
221
222
223def LoadOneBuildFile(build_file_path, data, aux_data, includes, is_target, check):
224 if build_file_path in data:
225 return data[build_file_path]
226
227 if os.path.exists(build_file_path):
228 build_file_contents = open(build_file_path, encoding="utf-8").read()
229 else:
230 raise GypError(f"{build_file_path} not found (cwd: {os.getcwd()})")
231
232 build_file_data = None
233 try:
234 if check:
235 build_file_data = CheckedEval(build_file_contents)
236 else:
237 build_file_data = eval(build_file_contents, {"__builtins__": {}}, None)
238 except SyntaxError as e:
239 e.filename = build_file_path
240 raise
241 except Exception as e:
242 gyp.common.ExceptionAppend(e, "while reading " + build_file_path)
243 raise
244
245 if not isinstance(build_file_data, dict):
246 raise GypError("%s does not evaluate to a dictionary." % build_file_path)
247
248 data[build_file_path] = build_file_data
249 aux_data[build_file_path] = {}
250
251 # Scan for includes and merge them in.
252 if "skip_includes" not in build_file_data or not build_file_data["skip_includes"]:
253 try:
254 if is_target:
255 LoadBuildFileIncludesIntoDict(
256 build_file_data, build_file_path, data, aux_data, includes, check
257 )
258 else:
259 LoadBuildFileIncludesIntoDict(
260 build_file_data, build_file_path, data, aux_data, None, check
261 )
262 except Exception as e:
263 gyp.common.ExceptionAppend(
264 e, "while reading includes of " + build_file_path
265 )
266 raise
267
268 return build_file_data
269
270
271def LoadBuildFileIncludesIntoDict(
272 subdict, subdict_path, data, aux_data, includes, check
273):
274 includes_list = []
275 if includes is not None:
276 includes_list.extend(includes)
277 if "includes" in subdict:
278 for include in subdict["includes"]:
279 # "include" is specified relative to subdict_path, so compute the real
280 # path to include by appending the provided "include" to the directory
281 # in which subdict_path resides.
282 relative_include = os.path.normpath(
283 os.path.join(os.path.dirname(subdict_path), include)
284 )
285 includes_list.append(relative_include)
286 # Unhook the includes list, it's no longer needed.
287 del subdict["includes"]
288
289 # Merge in the included files.
290 for include in includes_list:
291 if "included" not in aux_data[subdict_path]:
292 aux_data[subdict_path]["included"] = []
293 aux_data[subdict_path]["included"].append(include)
294
295 gyp.DebugOutput(gyp.DEBUG_INCLUDES, "Loading Included File: '%s'", include)
296
297 MergeDicts(
298 subdict,
299 LoadOneBuildFile(include, data, aux_data, None, False, check),
300 subdict_path,
301 include,
302 )
303
304 # Recurse into subdictionaries.
305 for k, v in subdict.items():
306 if isinstance(v, dict):
307 LoadBuildFileIncludesIntoDict(v, subdict_path, data, aux_data, None, check)
308 elif isinstance(v, list):
309 LoadBuildFileIncludesIntoList(v, subdict_path, data, aux_data, check)
310
311
312# This recurses into lists so that it can look for dicts.
313def LoadBuildFileIncludesIntoList(sublist, sublist_path, data, aux_data, check):
314 for item in sublist:
315 if isinstance(item, dict):
316 LoadBuildFileIncludesIntoDict(
317 item, sublist_path, data, aux_data, None, check
318 )
319 elif isinstance(item, list):
320 LoadBuildFileIncludesIntoList(item, sublist_path, data, aux_data, check)
321
322
323# Processes toolsets in all the targets. This recurses into condition entries
324# since they can contain toolsets as well.
325def ProcessToolsetsInDict(data):
326 if "targets" in data:
327 target_list = data["targets"]
328 new_target_list = []
329 for target in target_list:
330 # If this target already has an explicit 'toolset', and no 'toolsets'
331 # list, don't modify it further.
332 if "toolset" in target and "toolsets" not in target:
333 new_target_list.append(target)
334 continue
335 if multiple_toolsets:
336 toolsets = target.get("toolsets", ["target"])
337 else:
338 toolsets = ["target"]
339 # Make sure this 'toolsets' definition is only processed once.
340 if "toolsets" in target:
341 del target["toolsets"]
342 if len(toolsets) > 0:
343 # Optimization: only do copies if more than one toolset is specified.
344 for build in toolsets[1:]:
345 new_target = gyp.simple_copy.deepcopy(target)
346 new_target["toolset"] = build
347 new_target_list.append(new_target)
348 target["toolset"] = toolsets[0]
349 new_target_list.append(target)
350 data["targets"] = new_target_list
351 if "conditions" in data:
352 for condition in data["conditions"]:
353 if isinstance(condition, list):
354 for condition_dict in condition[1:]:
355 if isinstance(condition_dict, dict):
356 ProcessToolsetsInDict(condition_dict)
357
358
359# TODO(mark): I don't love this name. It just means that it's going to load
360# a build file that contains targets and is expected to provide a targets dict
361# that contains the targets...
362def LoadTargetBuildFile(
363 build_file_path,
364 data,
365 aux_data,
366 variables,
367 includes,
368 depth,
369 check,
370 load_dependencies,
371):
372 # If depth is set, predefine the DEPTH variable to be a relative path from
373 # this build file's directory to the directory identified by depth.
374 if depth:
375 # TODO(dglazkov) The backslash/forward-slash replacement at the end is a
376 # temporary measure. This should really be addressed by keeping all paths
377 # in POSIX until actual project generation.
378 d = gyp.common.RelativePath(depth, os.path.dirname(build_file_path))
379 if d == "":
380 variables["DEPTH"] = "."
381 else:
382 variables["DEPTH"] = d.replace("\\", "/")
383
384 # The 'target_build_files' key is only set when loading target build files in
385 # the non-parallel code path, where LoadTargetBuildFile is called
386 # recursively. In the parallel code path, we don't need to check whether the
387 # |build_file_path| has already been loaded, because the 'scheduled' set in
388 # ParallelState guarantees that we never load the same |build_file_path|
389 # twice.
390 if "target_build_files" in data:
391 if build_file_path in data["target_build_files"]:
392 # Already loaded.
393 return False
394 data["target_build_files"].add(build_file_path)
395
396 gyp.DebugOutput(
397 gyp.DEBUG_INCLUDES, "Loading Target Build File '%s'", build_file_path
398 )
399
400 build_file_data = LoadOneBuildFile(
401 build_file_path, data, aux_data, includes, True, check
402 )
403
404 # Store DEPTH for later use in generators.
405 build_file_data["_DEPTH"] = depth
406
407 # Set up the included_files key indicating which .gyp files contributed to
408 # this target dict.
409 if "included_files" in build_file_data:
410 raise GypError(build_file_path + " must not contain included_files key")
411
412 included = GetIncludedBuildFiles(build_file_path, aux_data)
413 build_file_data["included_files"] = []
414 for included_file in included:
415 # included_file is relative to the current directory, but it needs to
416 # be made relative to build_file_path's directory.
417 included_relative = gyp.common.RelativePath(
418 included_file, os.path.dirname(build_file_path)
419 )
420 build_file_data["included_files"].append(included_relative)
421
422 # Do a first round of toolsets expansion so that conditions can be defined
423 # per toolset.
424 ProcessToolsetsInDict(build_file_data)
425
426 # Apply "pre"/"early" variable expansions and condition evaluations.
427 ProcessVariablesAndConditionsInDict(
428 build_file_data, PHASE_EARLY, variables, build_file_path
429 )
430
431 # Since some toolsets might have been defined conditionally, perform
432 # a second round of toolsets expansion now.
433 ProcessToolsetsInDict(build_file_data)
434
435 # Look at each project's target_defaults dict, and merge settings into
436 # targets.
437 if "target_defaults" in build_file_data:
438 if "targets" not in build_file_data:
439 raise GypError("Unable to find targets in build file %s" % build_file_path)
440
441 index = 0
442 while index < len(build_file_data["targets"]):
443 # This procedure needs to give the impression that target_defaults is
444 # used as defaults, and the individual targets inherit from that.
445 # The individual targets need to be merged into the defaults. Make
446 # a deep copy of the defaults for each target, merge the target dict
447 # as found in the input file into that copy, and then hook up the
448 # copy with the target-specific data merged into it as the replacement
449 # target dict.
450 old_target_dict = build_file_data["targets"][index]
451 new_target_dict = gyp.simple_copy.deepcopy(
452 build_file_data["target_defaults"]
453 )
454 MergeDicts(
455 new_target_dict, old_target_dict, build_file_path, build_file_path
456 )
457 build_file_data["targets"][index] = new_target_dict
458 index += 1
459
460 # No longer needed.
461 del build_file_data["target_defaults"]
462
463 # Look for dependencies. This means that dependency resolution occurs
464 # after "pre" conditionals and variable expansion, but before "post" -
465 # in other words, you can't put a "dependencies" section inside a "post"
466 # conditional within a target.
467
468 dependencies = []
469 if "targets" in build_file_data:
470 for target_dict in build_file_data["targets"]:
471 if "dependencies" not in target_dict:
472 continue
473 for dependency in target_dict["dependencies"]:
474 dependencies.append(
475 gyp.common.ResolveTarget(build_file_path, dependency, None)[0]
476 )
477
478 if load_dependencies:
479 for dependency in dependencies:
480 try:
481 LoadTargetBuildFile(
482 dependency,
483 data,
484 aux_data,
485 variables,
486 includes,
487 depth,
488 check,
489 load_dependencies,
490 )
491 except Exception as e:
492 gyp.common.ExceptionAppend(
493 e, "while loading dependencies of %s" % build_file_path
494 )
495 raise
496 else:
497 return (build_file_path, dependencies)
498
499
500def CallLoadTargetBuildFile(
501 global_flags,
502 build_file_path,
503 variables,
504 includes,
505 depth,
506 check,
507 generator_input_info,
508):
509 """Wrapper around LoadTargetBuildFile for parallel processing.
510
511 This wrapper is used when LoadTargetBuildFile is executed in
512 a worker process.
513 """
514
515 try:
516 signal.signal(signal.SIGINT, signal.SIG_IGN)
517
518 # Apply globals so that the worker process behaves the same.
519 for key, value in global_flags.items():
520 globals()[key] = value
521
522 SetGeneratorGlobals(generator_input_info)
523 result = LoadTargetBuildFile(
524 build_file_path,
525 per_process_data,
526 per_process_aux_data,
527 variables,
528 includes,
529 depth,
530 check,
531 False,
532 )
533 if not result:
534 return result
535
536 (build_file_path, dependencies) = result
537
538 # We can safely pop the build_file_data from per_process_data because it
539 # will never be referenced by this process again, so we don't need to keep
540 # it in the cache.
541 build_file_data = per_process_data.pop(build_file_path)
542
543 # This gets serialized and sent back to the main process via a pipe.
544 # It's handled in LoadTargetBuildFileCallback.
545 return (build_file_path, build_file_data, dependencies)
546 except GypError as e:
547 sys.stderr.write("gyp: %s\n" % e)
548 return None
549 except Exception as e:
550 print("Exception:", e, file=sys.stderr)
551 print(traceback.format_exc(), file=sys.stderr)
552 return None
553
554
555class ParallelProcessingError(Exception):
556 pass
557
558
559class ParallelState:
560 """Class to keep track of state when processing input files in parallel.
561
562 If build files are loaded in parallel, use this to keep track of
563 state during farming out and processing parallel jobs. It's stored
564 in a global so that the callback function can have access to it.
565 """
566
567 def __init__(self):
568 # The multiprocessing pool.
569 self.pool = None
570 # The condition variable used to protect this object and notify
571 # the main loop when there might be more data to process.
572 self.condition = None
573 # The "data" dict that was passed to LoadTargetBuildFileParallel
574 self.data = None
575 # The number of parallel calls outstanding; decremented when a response
576 # was received.
577 self.pending = 0
578 # The set of all build files that have been scheduled, so we don't
579 # schedule the same one twice.
580 self.scheduled = set()
581 # A list of dependency build file paths that haven't been scheduled yet.
582 self.dependencies = []
583 # Flag to indicate if there was an error in a child process.
584 self.error = False
585
586 def LoadTargetBuildFileCallback(self, result):
587 """Handle the results of running LoadTargetBuildFile in another process."""
588 self.condition.acquire()
589 if not result:
590 self.error = True
591 self.condition.notify()
592 self.condition.release()
593 return
594 (build_file_path0, build_file_data0, dependencies0) = result
595 self.data[build_file_path0] = build_file_data0
596 self.data["target_build_files"].add(build_file_path0)
597 for new_dependency in dependencies0:
598 if new_dependency not in self.scheduled:
599 self.scheduled.add(new_dependency)
600 self.dependencies.append(new_dependency)
601 self.pending -= 1
602 self.condition.notify()
603 self.condition.release()
604
605
606def LoadTargetBuildFilesParallel(
607 build_files, data, variables, includes, depth, check, generator_input_info
608):
609 parallel_state = ParallelState()
610 parallel_state.condition = threading.Condition()
611 # Make copies of the build_files argument that we can modify while working.
612 parallel_state.dependencies = list(build_files)
613 parallel_state.scheduled = set(build_files)
614 parallel_state.pending = 0
615 parallel_state.data = data
616
617 try:
618 parallel_state.condition.acquire()
619 while parallel_state.dependencies or parallel_state.pending:
620 if parallel_state.error:
621 break
622 if not parallel_state.dependencies:
623 parallel_state.condition.wait()
624 continue
625
626 dependency = parallel_state.dependencies.pop()
627
628 parallel_state.pending += 1
629 global_flags = {
630 "path_sections": globals()["path_sections"],
631 "non_configuration_keys": globals()["non_configuration_keys"],
632 "multiple_toolsets": globals()["multiple_toolsets"],
633 }
634
635 if not parallel_state.pool:
636 parallel_state.pool = multiprocessing.Pool(multiprocessing.cpu_count())
637 parallel_state.pool.apply_async(
638 CallLoadTargetBuildFile,
639 args=(
640 global_flags,
641 dependency,
642 variables,
643 includes,
644 depth,
645 check,
646 generator_input_info,
647 ),
648 callback=parallel_state.LoadTargetBuildFileCallback,
649 )
650 except KeyboardInterrupt as e:
651 parallel_state.pool.terminate()
652 raise e
653
654 parallel_state.condition.release()
655
656 parallel_state.pool.close()
657 parallel_state.pool.join()
658 parallel_state.pool = None
659
660 if parallel_state.error:
661 sys.exit(1)
662
663
664# Look for the bracket that matches the first bracket seen in a
665# string, and return the start and end as a tuple. For example, if
666# the input is something like "<(foo <(bar)) blah", then it would
667# return (1, 13), indicating the entire string except for the leading
668# "<" and trailing " blah".
669LBRACKETS = set("{[(")
670BRACKETS = {"}": "{", "]": "[", ")": "("}
671
672
673def FindEnclosingBracketGroup(input_str):
674 stack = []
675 start = -1
676 for index, char in enumerate(input_str):
677 if char in LBRACKETS:
678 stack.append(char)
679 if start == -1:
680 start = index
681 elif char in BRACKETS:
682 if not stack:
683 return (-1, -1)
684 if stack.pop() != BRACKETS[char]:
685 return (-1, -1)
686 if not stack:
687 return (start, index + 1)
688 return (-1, -1)
689
690
691def IsStrCanonicalInt(string):
692 """Returns True if |string| is in its canonical integer form.
693
694 The canonical form is such that str(int(string)) == string.
695 """
696 if isinstance(string, str):
697 # This function is called a lot so for maximum performance, avoid
698 # involving regexps which would otherwise make the code much
699 # shorter. Regexps would need twice the time of this function.
700 if string:
701 if string == "0":
702 return True
703 if string[0] == "-":
704 string = string[1:]
705 if not string:
706 return False
707 if "1" <= string[0] <= "9":
708 return string.isdigit()
709
710 return False
711
712
713# This matches things like "<(asdf)", "<!(cmd)", "<!@(cmd)", "<|(list)",
714# "<!interpreter(arguments)", "<([list])", and even "<([)" and "<(<())".
715# In the last case, the inner "<()" is captured in match['content'].
716early_variable_re = re.compile(
717 r"(?P<replace>(?P<type><(?:(?:!?@?)|\|)?)"
718 r"(?P<command_string>[-a-zA-Z0-9_.]+)?"
719 r"\((?P<is_array>\s*\[?)"
720 r"(?P<content>.*?)(\]?)\))"
721)
722
723# This matches the same as early_variable_re, but with '>' instead of '<'.
724late_variable_re = re.compile(
725 r"(?P<replace>(?P<type>>(?:(?:!?@?)|\|)?)"
726 r"(?P<command_string>[-a-zA-Z0-9_.]+)?"
727 r"\((?P<is_array>\s*\[?)"
728 r"(?P<content>.*?)(\]?)\))"
729)
730
731# This matches the same as early_variable_re, but with '^' instead of '<'.
732latelate_variable_re = re.compile(
733 r"(?P<replace>(?P<type>[\^](?:(?:!?@?)|\|)?)"
734 r"(?P<command_string>[-a-zA-Z0-9_.]+)?"
735 r"\((?P<is_array>\s*\[?)"
736 r"(?P<content>.*?)(\]?)\))"
737)
738
739# Global cache of results from running commands so they don't have to be run
740# more then once.
741cached_command_results = {}
742
743
744def FixupPlatformCommand(cmd):
745 if sys.platform == "win32":
746 if isinstance(cmd, list):
747 cmd = [re.sub("^cat ", "type ", cmd[0])] + cmd[1:]
748 else:
749 cmd = re.sub("^cat ", "type ", cmd)
750 return cmd
751
752
753PHASE_EARLY = 0
754PHASE_LATE = 1
755PHASE_LATELATE = 2
756
757
758def ExpandVariables(input, phase, variables, build_file):
759 # Look for the pattern that gets expanded into variables
760 if phase == PHASE_EARLY:
761 variable_re = early_variable_re
762 expansion_symbol = "<"
763 elif phase == PHASE_LATE:
764 variable_re = late_variable_re
765 expansion_symbol = ">"
766 elif phase == PHASE_LATELATE:
767 variable_re = latelate_variable_re
768 expansion_symbol = "^"
769 else:
770 assert False
771
772 input_str = str(input)
773 if IsStrCanonicalInt(input_str):
774 return int(input_str)
775
776 # Do a quick scan to determine if an expensive regex search is warranted.
777 if expansion_symbol not in input_str:
778 return input_str
779
780 # Get the entire list of matches as a list of MatchObject instances.
781 # (using findall here would return strings instead of MatchObjects).
782 matches = list(variable_re.finditer(input_str))
783 if not matches:
784 return input_str
785
786 output = input_str
787 # Reverse the list of matches so that replacements are done right-to-left.
788 # That ensures that earlier replacements won't mess up the string in a
789 # way that causes later calls to find the earlier substituted text instead
790 # of what's intended for replacement.
791 matches.reverse()
792 for match_group in matches:
793 match = match_group.groupdict()
794 gyp.DebugOutput(gyp.DEBUG_VARIABLES, "Matches: %r", match)
795 # match['replace'] is the substring to look for, match['type']
796 # is the character code for the replacement type (< > <! >! <| >| <@
797 # >@ <!@ >!@), match['is_array'] contains a '[' for command
798 # arrays, and match['content'] is the name of the variable (< >)
799 # or command to run (<! >!). match['command_string'] is an optional
800 # command string. Currently, only 'pymod_do_main' is supported.
801
802 # run_command is true if a ! variant is used.
803 run_command = "!" in match["type"]
804 command_string = match["command_string"]
805
806 # file_list is true if a | variant is used.
807 file_list = "|" in match["type"]
808
809 # Capture these now so we can adjust them later.
810 replace_start = match_group.start("replace")
811 replace_end = match_group.end("replace")
812
813 # Find the ending paren, and re-evaluate the contained string.
814 (c_start, c_end) = FindEnclosingBracketGroup(input_str[replace_start:])
815
816 # Adjust the replacement range to match the entire command
817 # found by FindEnclosingBracketGroup (since the variable_re
818 # probably doesn't match the entire command if it contained
819 # nested variables).
820 replace_end = replace_start + c_end
821
822 # Find the "real" replacement, matching the appropriate closing
823 # paren, and adjust the replacement start and end.
824 replacement = input_str[replace_start:replace_end]
825
826 # Figure out what the contents of the variable parens are.
827 contents_start = replace_start + c_start + 1
828 contents_end = replace_end - 1
829 contents = input_str[contents_start:contents_end]
830
831 # Do filter substitution now for <|().
832 # Admittedly, this is different than the evaluation order in other
833 # contexts. However, since filtration has no chance to run on <|(),
834 # this seems like the only obvious way to give them access to filters.
835 if file_list:
836 processed_variables = gyp.simple_copy.deepcopy(variables)
837 ProcessListFiltersInDict(contents, processed_variables)
838 # Recurse to expand variables in the contents
839 contents = ExpandVariables(contents, phase, processed_variables, build_file)
840 else:
841 # Recurse to expand variables in the contents
842 contents = ExpandVariables(contents, phase, variables, build_file)
843
844 # Strip off leading/trailing whitespace so that variable matches are
845 # simpler below (and because they are rarely needed).
846 contents = contents.strip()
847
848 # expand_to_list is true if an @ variant is used. In that case,
849 # the expansion should result in a list. Note that the caller
850 # is to be expecting a list in return, and not all callers do
851 # because not all are working in list context. Also, for list
852 # expansions, there can be no other text besides the variable
853 # expansion in the input string.
854 expand_to_list = "@" in match["type"] and input_str == replacement
855
856 if run_command or file_list:
857 # Find the build file's directory, so commands can be run or file lists
858 # generated relative to it.
859 build_file_dir = os.path.dirname(build_file)
860 if build_file_dir == "" and not file_list:
861 # If build_file is just a leaf filename indicating a file in the
862 # current directory, build_file_dir might be an empty string. Set
863 # it to None to signal to subprocess.Popen that it should run the
864 # command in the current directory.
865 build_file_dir = None
866
867 # Support <|(listfile.txt ...) which generates a file
868 # containing items from a gyp list, generated at gyp time.
869 # This works around actions/rules which have more inputs than will
870 # fit on the command line.
871 if file_list:
872 contents_list = (
873 contents if isinstance(contents, list) else contents.split(" ")
874 )
875 replacement = contents_list[0]
876 if os.path.isabs(replacement):
877 raise GypError('| cannot handle absolute paths, got "%s"' % replacement)
878
879 if not generator_filelist_paths:
880 path = os.path.join(build_file_dir, replacement)
881 else:
882 if os.path.isabs(build_file_dir):
883 toplevel = generator_filelist_paths["toplevel"]
884 rel_build_file_dir = gyp.common.RelativePath(
885 build_file_dir, toplevel
886 )
887 else:
888 rel_build_file_dir = build_file_dir
889 qualified_out_dir = generator_filelist_paths["qualified_out_dir"]
890 path = os.path.join(qualified_out_dir, rel_build_file_dir, replacement)
891 gyp.common.EnsureDirExists(path)
892
893 replacement = gyp.common.RelativePath(path, build_file_dir)
894 f = gyp.common.WriteOnDiff(path)
895 for i in contents_list[1:]:
896 f.write("%s\n" % i)
897 f.close()
898
899 elif run_command:
900 use_shell = True
901 if match["is_array"]:
902 contents = eval(contents)
903 use_shell = False
904
905 # Check for a cached value to avoid executing commands, or generating
906 # file lists more than once. The cache key contains the command to be
907 # run as well as the directory to run it from, to account for commands
908 # that depend on their current directory.
909 # TODO(http://code.google.com/p/gyp/issues/detail?id=111): In theory,
910 # someone could author a set of GYP files where each time the command
911 # is invoked it produces different output by design. When the need
912 # arises, the syntax should be extended to support no caching off a
913 # command's output so it is run every time.
914 cache_key = (str(contents), build_file_dir)
915 cached_value = cached_command_results.get(cache_key, None)
916 if cached_value is None:
917 gyp.DebugOutput(
918 gyp.DEBUG_VARIABLES,
919 "Executing command '%s' in directory '%s'",
920 contents,
921 build_file_dir,
922 )
923
924 replacement = ""
925
926 if command_string == "pymod_do_main":
927 # <!pymod_do_main(modulename param eters) loads |modulename| as a
928 # python module and then calls that module's DoMain() function,
929 # passing ["param", "eters"] as a single list argument. For modules
930 # that don't load quickly, this can be faster than
931 # <!(python modulename param eters). Do this in |build_file_dir|.
932 oldwd = os.getcwd() # Python doesn't like os.open('.'): no fchdir.
933 if build_file_dir: # build_file_dir may be None (see above).
934 os.chdir(build_file_dir)
935 sys.path.append(os.getcwd())
936 try:
937 parsed_contents = shlex.split(contents)
938 try:
939 py_module = __import__(parsed_contents[0])
940 except ImportError as e:
941 raise GypError(
942 "Error importing pymod_do_main"
943 "module (%s): %s" % (parsed_contents[0], e)
944 )
945 replacement = str(
946 py_module.DoMain(parsed_contents[1:])
947 ).rstrip()
948 finally:
949 sys.path.pop()
950 os.chdir(oldwd)
951 assert replacement is not None
952 elif command_string:
953 raise GypError(
954 "Unknown command string '%s' in '%s'."
955 % (command_string, contents)
956 )
957 else:
958 # Fix up command with platform specific workarounds.
959 contents = FixupPlatformCommand(contents)
960 try:
961 # stderr will be printed no matter what
962 result = subprocess.run(
963 contents,
964 stdout=subprocess.PIPE,
965 shell=use_shell,
966 cwd=build_file_dir,
967 check=False,
968 )
969 except Exception as e:
970 raise GypError(
971 "%s while executing command '%s' in %s"
972 % (e, contents, build_file)
973 )
974
975 if result.returncode > 0:
976 raise GypError(
977 "Call to '%s' returned exit status %d while in %s."
978 % (contents, result.returncode, build_file)
979 )
980 replacement = result.stdout.decode("utf-8").rstrip()
981
982 cached_command_results[cache_key] = replacement
983 else:
984 gyp.DebugOutput(
985 gyp.DEBUG_VARIABLES,
986 "Had cache value for command '%s' in directory '%s'",
987 contents,
988 build_file_dir,
989 )
990 replacement = cached_value
991
992 elif contents not in variables:
993 if contents[-1] in ["!", "/"]:
994 # In order to allow cross-compiles (nacl) to happen more naturally,
995 # we will allow references to >(sources/) etc. to resolve to
996 # and empty list if undefined. This allows actions to:
997 # 'action!': [
998 # '>@(_sources!)',
999 # ],
1000 # 'action/': [
1001 # '>@(_sources/)',
1002 # ],
1003 replacement = []
1004 else:
1005 raise GypError("Undefined variable " + contents + " in " + build_file)
1006 else:
1007 replacement = variables[contents]
1008
1009 if isinstance(replacement, bytes) and not isinstance(replacement, str):
1010 replacement = replacement.decode("utf-8") # done on Python 3 only
1011 if isinstance(replacement, list):
1012 for item in replacement:
1013 if isinstance(item, bytes) and not isinstance(item, str):
1014 item = item.decode("utf-8") # done on Python 3 only
1015 if not contents[-1] == "/" and type(item) not in (str, int):
1016 raise GypError(
1017 "Variable "
1018 + contents
1019 + " must expand to a string or list of strings; "
1020 + "list contains a "
1021 + item.__class__.__name__
1022 )
1023 # Run through the list and handle variable expansions in it. Since
1024 # the list is guaranteed not to contain dicts, this won't do anything
1025 # with conditions sections.
1026 ProcessVariablesAndConditionsInList(
1027 replacement, phase, variables, build_file
1028 )
1029 elif type(replacement) not in (str, int):
1030 raise GypError(
1031 "Variable "
1032 + contents
1033 + " must expand to a string or list of strings; "
1034 + "found a "
1035 + replacement.__class__.__name__
1036 )
1037
1038 if expand_to_list:
1039 # Expanding in list context. It's guaranteed that there's only one
1040 # replacement to do in |input_str| and that it's this replacement. See
1041 # above.
1042 if isinstance(replacement, list):
1043 # If it's already a list, make a copy.
1044 output = replacement[:]
1045 else:
1046 # Split it the same way sh would split arguments.
1047 output = shlex.split(str(replacement))
1048 else:
1049 # Expanding in string context.
1050 encoded_replacement = ""
1051 if isinstance(replacement, list):
1052 # When expanding a list into string context, turn the list items
1053 # into a string in a way that will work with a subprocess call.
1054 #
1055 # TODO(mark): This isn't completely correct. This should
1056 # call a generator-provided function that observes the
1057 # proper list-to-argument quoting rules on a specific
1058 # platform instead of just calling the POSIX encoding
1059 # routine.
1060 encoded_replacement = gyp.common.EncodePOSIXShellList(replacement)
1061 else:
1062 encoded_replacement = replacement
1063
1064 output = (
1065 output[:replace_start] + str(encoded_replacement) + output[replace_end:]
1066 )
1067 # Prepare for the next match iteration.
1068 input_str = output
1069
1070 if output == input:
1071 gyp.DebugOutput(
1072 gyp.DEBUG_VARIABLES,
1073 "Found only identity matches on %r, avoiding infinite recursion.",
1074 output,
1075 )
1076 else:
1077 # Look for more matches now that we've replaced some, to deal with
1078 # expanding local variables (variables defined in the same
1079 # variables block as this one).
1080 gyp.DebugOutput(gyp.DEBUG_VARIABLES, "Found output %r, recursing.", output)
1081 if isinstance(output, list):
1082 if output and isinstance(output[0], list):
1083 # Leave output alone if it's a list of lists.
1084 # We don't want such lists to be stringified.
1085 pass
1086 else:
1087 new_output = []
1088 for item in output:
1089 new_output.append(
1090 ExpandVariables(item, phase, variables, build_file)
1091 )
1092 output = new_output
1093 else:
1094 output = ExpandVariables(output, phase, variables, build_file)
1095
1096 # Convert all strings that are canonically-represented integers into integers.
1097 if isinstance(output, list):
1098 for index, outstr in enumerate(output):
1099 if IsStrCanonicalInt(outstr):
1100 output[index] = int(outstr)
1101 elif IsStrCanonicalInt(output):
1102 output = int(output)
1103
1104 return output
1105
1106
1107# The same condition is often evaluated over and over again so it
1108# makes sense to cache as much as possible between evaluations.
1109cached_conditions_asts = {}
1110
1111
1112def EvalCondition(condition, conditions_key, phase, variables, build_file):
1113 """Returns the dict that should be used or None if the result was
1114 that nothing should be used."""
1115 if not isinstance(condition, list):
1116 raise GypError(conditions_key + " must be a list")
1117 if len(condition) < 2:
1118 # It's possible that condition[0] won't work in which case this
1119 # attempt will raise its own IndexError. That's probably fine.
1120 raise GypError(
1121 conditions_key
1122 + " "
1123 + condition[0]
1124 + " must be at least length 2, not "
1125 + str(len(condition))
1126 )
1127
1128 i = 0
1129 result = None
1130 while i < len(condition):
1131 cond_expr = condition[i]
1132 true_dict = condition[i + 1]
1133 if not isinstance(true_dict, dict):
1134 raise GypError(
1135 f"{conditions_key} {cond_expr} must be followed by a dictionary, "
1136 f"not {type(true_dict)}"
1137 )
1138 if len(condition) > i + 2 and isinstance(condition[i + 2], dict):
1139 false_dict = condition[i + 2]
1140 i = i + 3
1141 if i != len(condition):
1142 raise GypError(
1143 f"{conditions_key} {cond_expr} has "
1144 f"{len(condition) - i} unexpected trailing items"
1145 )
1146 else:
1147 false_dict = None
1148 i = i + 2
1149 if result is None:
1150 result = EvalSingleCondition(
1151 cond_expr, true_dict, false_dict, phase, variables, build_file
1152 )
1153
1154 return result
1155
1156
1157def EvalSingleCondition(cond_expr, true_dict, false_dict, phase, variables, build_file):
1158 """Returns true_dict if cond_expr evaluates to true, and false_dict
1159 otherwise."""
1160 # Do expansions on the condition itself. Since the condition can naturally
1161 # contain variable references without needing to resort to GYP expansion
1162 # syntax, this is of dubious value for variables, but someone might want to
1163 # use a command expansion directly inside a condition.
1164 cond_expr_expanded = ExpandVariables(cond_expr, phase, variables, build_file)
1165 if type(cond_expr_expanded) not in (str, int):
1166 raise ValueError(
1167 "Variable expansion in this context permits str and int "
1168 + "only, found "
1169 + cond_expr_expanded.__class__.__name__
1170 )
1171
1172 try:
1173 if cond_expr_expanded in cached_conditions_asts:
1174 ast_code = cached_conditions_asts[cond_expr_expanded]
1175 else:
1176 ast_code = compile(cond_expr_expanded, "<string>", "eval")
1177 cached_conditions_asts[cond_expr_expanded] = ast_code
1178 env = {"__builtins__": {}, "v": Version}
1179 if eval(ast_code, env, variables):
1180 return true_dict
1181 return false_dict
1182 except SyntaxError as e:
1183 syntax_error = SyntaxError(
1184 "%s while evaluating condition '%s' in %s "
1185 "at character %d." % (str(e.args[0]), e.text, build_file, e.offset),
1186 e.filename,
1187 e.lineno,
1188 e.offset,
1189 e.text,
1190 )
1191 raise syntax_error
1192 except NameError as e:
1193 gyp.common.ExceptionAppend(
1194 e,
1195 f"while evaluating condition '{cond_expr_expanded}' in {build_file}",
1196 )
1197 raise GypError(e)
1198
1199
1200def ProcessConditionsInDict(the_dict, phase, variables, build_file):