codekingpro/portable-devtools
114k
1#!/usr/bin/env python3
2
3# Copyright (c) 2012 Google Inc. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Utility functions for Windows builds.
8
9These functions are executed via gyp-win-tool when using the ninja generator.
10"""
11
12import os
13import re
14import shutil
15import stat
16import string
17import subprocess
18import sys
19
20BASE_DIR = os.path.dirname(os.path.abspath(__file__))
21
22# A regex matching an argument corresponding to the output filename passed to
23# link.exe.
24_LINK_EXE_OUT_ARG = re.compile("/OUT:(?P<out>.+)$", re.IGNORECASE)
25
26
27def main(args):
28 executor = WinTool()
29 if (exit_code := executor.Dispatch(args)) is not None:
30 sys.exit(exit_code)
31
32
33class WinTool:
34 """This class performs all the Windows tooling steps. The methods can either
35 be executed directly, or dispatched from an argument list."""
36
37 def _UseSeparateMspdbsrv(self, env, args):
38 """Allows to use a unique instance of mspdbsrv.exe per linker instead of a
39 shared one."""
40 if len(args) < 1:
41 raise Exception("Not enough arguments")
42
43 if args[0] != "link.exe":
44 return
45
46 # Use the output filename passed to the linker to generate an endpoint name
47 # for mspdbsrv.exe.
48 endpoint_name = None
49 for arg in args:
50 m = _LINK_EXE_OUT_ARG.match(arg)
51 if m:
52 endpoint_name = re.sub(
53 r"\W+", "", "%s_%d" % (m.group("out"), os.getpid())
54 )
55 break
56
57 if endpoint_name is None:
58 return
59
60 # Adds the appropriate environment variable. This will be read by link.exe
61 # to know which instance of mspdbsrv.exe it should connect to (if it's
62 # not set then the default endpoint is used).
63 env["_MSPDBSRV_ENDPOINT_"] = endpoint_name
64
65 def Dispatch(self, args):
66 """Dispatches a string command to a method."""
67 if len(args) < 1:
68 raise Exception("Not enough arguments")
69
70 method = "Exec%s" % self._CommandifyName(args[0])
71 return getattr(self, method)(*args[1:])
72
73 def _CommandifyName(self, name_string):
74 """Transforms a tool name like recursive-mirror to RecursiveMirror."""
75 return name_string.title().replace("-", "")
76
77 def _GetEnv(self, arch):
78 """Gets the saved environment from a file for a given architecture."""
79 # The environment is saved as an "environment block" (see CreateProcess
80 # and msvs_emulation for details). We convert to a dict here.
81 # Drop last 2 NULs, one for list terminator, one for trailing vs. separator.
82 pairs = open(arch).read()[:-2].split("\0")
83 kvs = [item.split("=", 1) for item in pairs]
84 return dict(kvs)
85
86 def ExecStamp(self, path):
87 """Simple stamp command."""
88 open(path, "w").close()
89
90 def ExecRecursiveMirror(self, source, dest):
91 """Emulation of rm -rf out && cp -af in out."""
92 if os.path.exists(dest):
93 if os.path.isdir(dest):
94
95 def _on_error(fn, path, excinfo):
96 # The operation failed, possibly because the file is set to
97 # read-only. If that's why, make it writable and try the op again.
98 if not os.access(path, os.W_OK):
99 os.chmod(path, stat.S_IWRITE)
100 fn(path)
101
102 shutil.rmtree(dest, onerror=_on_error)
103 else:
104 if not os.access(dest, os.W_OK):
105 # Attempt to make the file writable before deleting it.
106 os.chmod(dest, stat.S_IWRITE)
107 os.unlink(dest)
108
109 if os.path.isdir(source):
110 shutil.copytree(source, dest)
111 else:
112 shutil.copy2(source, dest)
113
114 def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args):
115 """Filter diagnostic output from link that looks like:
116 ' Creating library ui.dll.lib and object ui.dll.exp'
117 This happens when there are exports from the dll or exe.
118 """
119 env = self._GetEnv(arch)
120 if use_separate_mspdbsrv == "True":
121 self._UseSeparateMspdbsrv(env, args)
122 if sys.platform == "win32":
123 args = list(args) # *args is a tuple by default, which is read-only.
124 args[0] = args[0].replace("/", "\\")
125 # https://docs.python.org/2/library/subprocess.html:
126 # "On Unix with shell=True [...] if args is a sequence, the first item
127 # specifies the command string, and any additional items will be treated as
128 # additional arguments to the shell itself. That is to say, Popen does the
129 # equivalent of:
130 # Popen(['/bin/sh', '-c', args[0], args[1], ...])"
131 # For that reason, since going through the shell doesn't seem necessary on
132 # non-Windows don't do that there.
133 link = subprocess.Popen(
134 args,
135 shell=sys.platform == "win32",
136 env=env,
137 stdout=subprocess.PIPE,
138 stderr=subprocess.STDOUT,
139 )
140 out = link.communicate()[0].decode("utf-8")
141 for line in out.splitlines():
142 if (
143 not line.startswith(" Creating library ")
144 and not line.startswith("Generating code")
145 and not line.startswith("Finished generating code")
146 ):
147 print(line)
148 return link.returncode
149
150 def ExecLinkWithManifests(
151 self,
152 arch,
153 embed_manifest,
154 out,
155 ldcmd,
156 resname,
157 mt,
158 rc,
159 intermediate_manifest,
160 *manifests,
161 ):
162 """A wrapper for handling creating a manifest resource and then executing
163 a link command."""
164 # The 'normal' way to do manifests is to have link generate a manifest
165 # based on gathering dependencies from the object files, then merge that
166 # manifest with other manifests supplied as sources, convert the merged
167 # manifest to a resource, and then *relink*, including the compiled
168 # version of the manifest resource. This breaks incremental linking, and
169 # is generally overly complicated. Instead, we merge all the manifests
170 # provided (along with one that includes what would normally be in the
171 # linker-generated one, see msvs_emulation.py), and include that into the
172 # first and only link. We still tell link to generate a manifest, but we
173 # only use that to assert that our simpler process did not miss anything.
174 variables = {
175 "python": sys.executable,
176 "arch": arch,
177 "out": out,
178 "ldcmd": ldcmd,
179 "resname": resname,
180 "mt": mt,
181 "rc": rc,
182 "intermediate_manifest": intermediate_manifest,
183 "manifests": " ".join(manifests),
184 }
185 add_to_ld = ""
186 if manifests:
187 subprocess.check_call(
188 "%(python)s gyp-win-tool manifest-wrapper %(arch)s %(mt)s -nologo "
189 "-manifest %(manifests)s -out:%(out)s.manifest" % variables
190 )
191 if embed_manifest == "True":
192 subprocess.check_call(
193 "%(python)s gyp-win-tool manifest-to-rc %(arch)s %(out)s.manifest"
194 " %(out)s.manifest.rc %(resname)s" % variables
195 )
196 subprocess.check_call(
197 "%(python)s gyp-win-tool rc-wrapper %(arch)s %(rc)s "
198 "%(out)s.manifest.rc" % variables
199 )
200 add_to_ld = " %(out)s.manifest.res" % variables
201 subprocess.check_call(ldcmd + add_to_ld)
202
203 # Run mt.exe on the theoretically complete manifest we generated, merging
204 # it with the one the linker generated to confirm that the linker
205 # generated one does not add anything. This is strictly unnecessary for
206 # correctness, it's only to verify that e.g. /MANIFESTDEPENDENCY was not
207 # used in a #pragma comment.
208 if manifests:
209 # Merge the intermediate one with ours to .assert.manifest, then check
210 # that .assert.manifest is identical to ours.
211 subprocess.check_call(
212 "%(python)s gyp-win-tool manifest-wrapper %(arch)s %(mt)s -nologo "
213 "-manifest %(out)s.manifest %(intermediate_manifest)s "
214 "-out:%(out)s.assert.manifest" % variables
215 )
216 assert_manifest = "%(out)s.assert.manifest" % variables
217 our_manifest = "%(out)s.manifest" % variables
218 # Load and normalize the manifests. mt.exe sometimes removes whitespace,
219 # and sometimes doesn't unfortunately.
220 with open(our_manifest) as our_f, open(assert_manifest) as assert_f:
221 translator = str.maketrans("", "", string.whitespace)
222 our_data = our_f.read().translate(translator)
223 assert_data = assert_f.read().translate(translator)
224 if our_data != assert_data:
225 os.unlink(out)
226
227 def dump(filename):
228 print(filename, file=sys.stderr)
229 print("-----", file=sys.stderr)
230 with open(filename) as f:
231 print(f.read(), file=sys.stderr)
232 print("-----", file=sys.stderr)
233
234 dump(intermediate_manifest)
235 dump(our_manifest)
236 dump(assert_manifest)
237 sys.stderr.write(
238 'Linker generated manifest "%s" added to final manifest "%s" '
239 '(result in "%s"). '
240 "Were /MANIFEST switches used in #pragma statements? "
241 % (intermediate_manifest, our_manifest, assert_manifest)
242 )
243 return 1
244
245 def ExecManifestWrapper(self, arch, *args):
246 """Run manifest tool with environment set. Strip out undesirable warning
247 (some XML blocks are recognized by the OS loader, but not the manifest
248 tool)."""
249 env = self._GetEnv(arch)
250 popen = subprocess.Popen(
251 args, shell=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
252 )
253 out = popen.communicate()[0].decode("utf-8")
254 for line in out.splitlines():
255 if line and "manifest authoring warning 81010002" not in line:
256 print(line)
257 return popen.returncode
258
259 def ExecManifestToRc(self, arch, *args):
260 """Creates a resource file pointing a SxS assembly manifest.
261 |args| is tuple containing path to resource file, path to manifest file
262 and resource name which can be "1" (for executables) or "2" (for DLLs)."""
263 manifest_path, resource_path, resource_name = args
264 with open(resource_path, "w") as output:
265 output.write(
266 '#include <windows.h>\n%s RT_MANIFEST "%s"'
267 % (resource_name, os.path.abspath(manifest_path).replace("\\", "/"))
268 )
269
270 def ExecMidlWrapper(self, arch, outdir, tlb, h, dlldata, iid, proxy, idl, *flags):
271 """Filter noisy filenames output from MIDL compile step that isn't
272 quietable via command line flags.
273 """
274 args = (
275 ["midl", "/nologo"]
276 + list(flags)
277 + [
278 "/out",
279 outdir,
280 "/tlb",
281 tlb,
282 "/h",
283 h,
284 "/dlldata",
285 dlldata,
286 "/iid",
287 iid,
288 "/proxy",
289 proxy,
290 idl,
291 ]
292 )
293 env = self._GetEnv(arch)
294 popen = subprocess.Popen(
295 args, shell=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
296 )
297 out = popen.communicate()[0].decode("utf-8")
298 # Filter junk out of stdout, and write filtered versions. Output we want
299 # to filter is pairs of lines that look like this:
300 # Processing C:\Program Files (x86)\Microsoft SDKs\...\include\objidl.idl
301 # objidl.idl
302 lines = out.splitlines()
303 prefixes = ("Processing ", "64 bit Processing ")
304 processing = {os.path.basename(x) for x in lines if x.startswith(prefixes)}
305 for line in lines:
306 if not line.startswith(prefixes) and line not in processing:
307 print(line)
308 return popen.returncode
309
310 def ExecAsmWrapper(self, arch, *args):
311 """Filter logo banner from invocations of asm.exe."""
312 env = self._GetEnv(arch)
313 popen = subprocess.Popen(
314 args, shell=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
315 )
316 out = popen.communicate()[0].decode("utf-8")
317 for line in out.splitlines():
318 if (
319 not line.startswith("Copyright (C) Microsoft Corporation")
320 and not line.startswith("Microsoft (R) Macro Assembler")
321 and not line.startswith(" Assembling: ")
322 and line
323 ):
324 print(line)
325 return popen.returncode
326
327 def ExecRcWrapper(self, arch, *args):
328 """Filter logo banner from invocations of rc.exe. Older versions of RC
329 don't support the /nologo flag."""
330 env = self._GetEnv(arch)
331 popen = subprocess.Popen(
332 args, shell=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
333 )
334 out = popen.communicate()[0].decode("utf-8")
335 for line in out.splitlines():
336 if (
337 not line.startswith("Microsoft (R) Windows (R) Resource Compiler")
338 and not line.startswith("Copyright (C) Microsoft Corporation")
339 and line
340 ):
341 print(line)
342 return popen.returncode
343
344 def ExecActionWrapper(self, arch, rspfile, *dir):
345 """Runs an action command line from a response file using the environment
346 for |arch|. If |dir| is supplied, use that as the working directory."""
347 env = self._GetEnv(arch)
348 # TODO(scottmg): This is a temporary hack to get some specific variables
349 # through to actions that are set after gyp-time. http://crbug.com/333738.
350 for k, v in os.environ.items():
351 if k not in env:
352 env[k] = v
353 args = open(rspfile).read()
354 dir = dir[0] if dir else None
355 return subprocess.call(args, shell=True, env=env, cwd=dir)
356
357 def ExecClCompile(self, project_dir, selected_files):
358 """Executed by msvs-ninja projects when the 'ClCompile' target is used to
359 build selected C/C++ files."""
360 project_dir = os.path.relpath(project_dir, BASE_DIR)
361 selected_files = selected_files.split(";")
362 ninja_targets = [
363 os.path.join(project_dir, filename) + "^^" for filename in selected_files
364 ]
365 cmd = ["ninja.exe"]
366 cmd.extend(ninja_targets)
367 return subprocess.call(cmd, shell=True, cwd=BASE_DIR)
368
369
370if __name__ == "__main__":
371 sys.exit(main(sys.argv[1:]))
372