resource: inline RequestSout()
[vlc.git] / extras / breakpad / symb_upload.py
1 #! /usr/bin/env python3
2 import os
3 import sys
4 import argparse
5 import subprocess
6 import logging
7 import requests
8 import io
9 import shutil
10 import typing
11
12 class Dumper:
13     def __init__(self, strip_path: str = None):
14         self.strip_path = strip_path
15
16     def can_process(self, fpath: str):
17         return False
18
19     def dump(self, fpath: str):
20         assert(False)
21
22     def _preparse_dump(self, source: str):
23         meta = {}
24         dest = io.StringIO()
25         if  not source.startswith("MODULE"):
26             logging.error("file doesn't starst with MODULE")
27             return None, None
28         for line in source.split("\n"):
29             if line.startswith("MODULE"):
30                 #MODULE <os> <arch> <buildid> <filename>
31                 line_split = line.split(" ")
32                 if len(line_split) != 5:
33                     logging.error("malformed MODULE entry")
34                     return None, None
35                 _, _os, cpu, buildid, filename  =  line_split
36                 if filename.endswith(".dbg"):
37                     filename = filename[:-4]
38                 meta["os"] = _os
39                 meta["cpu"] = cpu
40                 meta["debug_file"] = filename
41                 meta["code_file"] = filename
42                 #see CompactIdentifier in symbol_upload.cc
43                 meta["debug_identifier"] = buildid.replace("-", "")
44                 dest.write("MODULE {} {} {} {}".format(_os, cpu, buildid, filename))
45                 dest.write("\n")
46             elif line.startswith("FILE"):
47                 #FILE <LINE> <PATH>
48                 _, line, *path_split =  line.split(" ")
49                 path = " ".join(path_split)
50                 path = os.path.normpath(path)
51
52                 if self.strip_path and path.startswith(self.strip_path):
53                     path = os.path.relpath(path, self.strip_path)
54
55                 dest.write("FILE {} {}\n".format(line, path))
56             else:
57                 dest.write(line)
58                 dest.write("\n")
59         dest.seek(0)
60         return meta, dest
61
62 class WindowDumper(Dumper):
63     def __init__(self, *args, **kwargs):
64         super().__init__(*args, **kwargs)
65
66     def can_process(self, fpath: str):
67         return any(fpath.endswith(ext) for ext in ["dbg", "dll", "exe"])
68
69     def dump(self, fpath: str):
70         proc = subprocess.run(
71             ["dump_syms_win", "-r", fpath],
72             stdout=subprocess.PIPE,
73             stderr=subprocess.PIPE,
74         )
75         if  proc.returncode != 0:
76             logging.error("unable to extract symbols from {}".format(fpath))
77             logging.error(proc.stderr)
78             return None, None
79         return self._preparse_dump(proc.stdout.decode("utf8"))
80
81 class MacDumper(Dumper):
82     def __init__(self, *args, **kwargs):
83         super().__init__(*args, **kwargs)
84
85     # Helper to check Mach-O header
86     def is_mach_o(self, fpath: str):
87         file = open(fpath, "rb")
88         header = file.read(4)
89         file.close()
90         # MH_MAGIC
91         if b'\xFE\xED\xFA\xCE' == header:
92             return True
93         # MH_CIGAM
94         elif b'\xCE\xFA\xED\xFE' == header:
95             return True
96         # MH_MAGIC_64
97         elif b'\xFE\xED\xFA\xCF' == header:
98             return True
99         # MH_CIGAM_64
100         elif b'\xCF\xFA\xED\xFE' == header:
101             return True
102         return False
103
104     def can_process(self, fpath: str):
105         if fpath.endswith(".dylib") or os.access(fpath, os.X_OK):
106             return self.is_mach_o(fpath) and not os.path.islink(fpath)
107         return False
108
109     def dump(self, fpath: str):
110         dsymbundle = fpath + ".dSYM"
111         if os.path.exists(dsymbundle):
112             shutil.rmtree(dsymbundle)
113
114         #generate symbols file
115         proc = subprocess.run(
116             ["dsymutil", fpath],
117             stdout=subprocess.DEVNULL,
118             stderr=subprocess.PIPE,
119             check=True
120         )
121         if  proc.returncode != 0:
122             logging.error("unable to run dsymutil on {}:".format(fpath))
123             logging.error(proc.stderr)
124             return None, None
125         if not os.path.exists(dsymbundle):
126             logging.error("No symbols in {}".format(fpath))
127             return None, None
128
129         proc = subprocess.run(
130             ["dump_syms", "-r", "-g", dsymbundle, fpath],
131             stdout=subprocess.PIPE,
132             stderr=subprocess.PIPE,
133         )
134
135         # Cleanup dsymbundle file
136         shutil.rmtree(dsymbundle)
137
138         if  proc.returncode != 0:
139             logging.error("unable to extract symbols from {}:".format(fpath))
140             logging.error(proc.stderr)
141             return None, None
142
143         return self._preparse_dump(proc.stdout.decode("utf8"))
144
145
146 class OutputStore:
147     def store(self, dump: typing.io.TextIO, meta):
148         assert(False)
149
150 class HTTPOutputStore(OutputStore):
151     def __init__(self, url : str, version = None, prod = None):
152         super().__init__()
153         self.url = url
154         self.extra_args = {}
155         if version:
156             self.extra_args["ver"] = version
157         if prod:
158             self.extra_args["prod"] = prod
159
160     def store(self, dump: typing.io.TextIO, meta):
161         post_args = {**meta, **self.extra_args}
162         r = requests.post(self.url, post_args, files={"symfile": dump})
163         if not r.ok:
164             logging.error("Unable to perform request, ret {}".format(r.status_code))
165             r.raise_for_status()
166
167 class LocalDirOutputStore(OutputStore):
168     def __init__(self, rootdir: str):
169         super().__init__()
170         self.rootdir = rootdir
171
172     def store(self, dump: typing.io, meta):
173         basepath = os.path.join(self.rootdir, meta["debug_file"], meta["debug_identifier"])
174         if not os.path.exists(basepath):
175             os.makedirs(basepath)
176         with open(os.path.join(basepath, meta["debug_file"] + ".sym"), "w+") as fd:
177             shutil.copyfileobj(dump, fd)
178
179 def process_dir(sourcedir, dumper, store):
180     for root, dirnames, filenames, in os.walk(sourcedir):
181         for fname in filenames:
182             if not dumper.can_process(os.path.join(root, fname)):
183                 continue
184             logging.info("processing {}".format(fname))
185             meta, dump = dumper.dump(os.path.join(root, fname))
186             if meta is None or dump is None:
187                 logging.warning("unable to dump {}".format(fname))
188                 continue
189             store.store(dump, meta)
190
191
192 def main():
193     parser = argparse.ArgumentParser(description='extract symbols for breakpad and upload or store them')
194     parser.add_argument("sourcedir", help="source directory")
195     parser.add_argument("--upload-url", metavar="URL", dest="uploadurl", type=str, help="upload url")
196     parser.add_argument("--strip-path", metavar="PATH", dest="strippath", type=str, help="strip path prefix")
197     parser.add_argument("-p","--platform",metavar="OS", dest="platform",
198                         choices=["mac", "linux", "win"], required=True, help="symbol platform (mac, linux, win)")
199     parser.add_argument("--output-dir", metavar="DIRECTORY", dest="outdir", type=str, help="output directory")
200     parser.add_argument("--version", metavar="VERSION", dest="version", type=str, help="specify symbol version for uploading")
201     parser.add_argument("--prod", metavar="PRODUCT", dest="prod", type=str, help="specify product name for uploading")
202     parser.add_argument("--log", metavar="LOGLEVEL", dest="log", type=str, help="log level (INFO, WARNING, ERROR)")
203     args = parser.parse_args()
204
205     if args.log:
206         numeric_level = getattr(logging, args.log.upper(), None)
207         if not isinstance(numeric_level, int):
208             raise ValueError("Invalid log level: {}".format(loglevel))
209         logging.basicConfig(format='%(levelname)s: %(message)s', level=numeric_level)
210
211
212     if args.platform == "win":
213         dumper = WindowDumper(strip_path=args.strippath)
214     elif args.platform == "mac":
215         dumper = MacDumper(strip_path=args.strippath)
216     else:
217         logging.error("Dumper {} is not implemented yet".format(args.platform))
218         exit(1)
219
220     if args.uploadurl:
221         store=HTTPOutputStore(args.uploadurl, version=args.version, prod=args.prod)
222     elif args.outdir:
223         store=LocalDirOutputStore(args.outdir)
224     else:
225         logging.error("You must chose either --output-dir or --upload-url")
226         exit(1)
227
228     process_dir(args.sourcedir, dumper, store)
229
230
231 if __name__ == "__main__":
232     assert(sys.version_info >= (3,5))
233     main()