From 51b170b470ed587f99a7c2a773e4464687f0d07a Mon Sep 17 00:00:00 2001 From: crueter Date: Mon, 4 Aug 2025 04:50:14 +0200 Subject: [cmake] refactor: Use CPM over submodules (#143) Transfers the majority of submodules and large externals to CPM, using source archives rather than full Git clones. Not only does this save massive amounts of clone and configure time, but dependencies are grabbed on-demand rather than being required by default. Additionally, CPM will (generally) automatically search for system dependencies, though certain dependencies have options to control this. Testing shows gains ranging from 5x to 10x in terms of overall clone/configure time. Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/143 Reviewed-by: CamilleLaVey --- .../breakpad/src/tools/python/deps-to-manifest.py | 165 ----------------- externals/breakpad/src/tools/python/filter_syms.py | 202 --------------------- .../src/tools/python/tests/filter_syms_unittest.py | 136 -------------- 3 files changed, 503 deletions(-) delete mode 100755 externals/breakpad/src/tools/python/deps-to-manifest.py delete mode 100644 externals/breakpad/src/tools/python/filter_syms.py delete mode 100644 externals/breakpad/src/tools/python/tests/filter_syms_unittest.py (limited to 'externals/breakpad/src/tools/python') diff --git a/externals/breakpad/src/tools/python/deps-to-manifest.py b/externals/breakpad/src/tools/python/deps-to-manifest.py deleted file mode 100755 index b614f94fdd..0000000000 --- a/externals/breakpad/src/tools/python/deps-to-manifest.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2016 Google LLC -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Convert gclient's DEPS file to repo's manifest xml file.""" - -import argparse -import os -import sys - - -REMOTES = { - 'chromium': 'https://chromium.googlesource.com/', - 'github': 'https://github.com/', -} -REVIEWS = { - 'chromium': 'https://chromium-review.googlesource.com', -} - -MANIFEST_HEAD = """ - - - - -""" - -MANIFEST_REMOTE = """ - -""" - -MANIFEST_PROJECT = """ - -""" - -MANIFEST_TAIL = """ - -""" - - -def ConvertDepsToManifest(deps, manifest): - """Convert the |deps| file to the |manifest|.""" - # Load the DEPS file data. - ctx = {} - with open(deps, 'rb') as file: - exec(compile(file.read(), deps, 'exec'), ctx) - - new_contents = '' - - # Write out the common header. - data = { - 'prog': os.path.basename(__file__), - } - new_contents += MANIFEST_HEAD % data - - # Write out the sections. - for name, fetch in REMOTES.items(): - data = { - 'name': name, - 'fetch': fetch, - 'review': REVIEWS.get(name, ''), - } - new_contents += MANIFEST_REMOTE % data - - # Write out the main repo itself. - data = { - 'path': 'src', - 'name': 'breakpad/breakpad', - 'revision': 'refs/heads/main', - 'remote': 'chromium', - } - new_contents += MANIFEST_PROJECT % data - - # Write out the sections. - for path, url in ctx['deps'].items(): - for name, fetch in REMOTES.items(): - if url.startswith(fetch): - remote = name - break - else: - raise ValueError('Unknown DEPS remote: %s: %s' % (path, url)) - - # The DEPS url will look like: - # https://chromium.googlesource.com/external/gyp/@e8ab0833a42691cd2 - remote_path, rev = url.split('@') - remote_path = remote_path[len(fetch):] - - # If it's not a revision, assume it's a tag. Repo wants full ref names. - if len(rev) != 40: - rev = 'refs/tags/%s' % rev - - data = { - 'path': path, - 'name': remote_path, - 'revision': rev, - 'remote': remote, - } - new_contents += MANIFEST_PROJECT % data - - # Write out the common footer. - new_contents += MANIFEST_TAIL - - # See if the manifest has actually changed contents to avoid thrashing. - try: - old_contents = open(manifest).read() - except IOError: - # In case the file doesn't exist yet. - old_contents = '' - if old_contents != new_contents: - print('Updating %s due to changed %s' % (manifest, deps)) - with open(manifest, 'w') as fp: - fp.write(new_contents) - - -def GetParser(): - """Return a CLI parser.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('deps', - help='The DEPS file to convert') - parser.add_argument('manifest', - help='The manifest xml to generate') - return parser - - -def main(argv): - """The main func!""" - parser = GetParser() - opts = parser.parse_args(argv) - ConvertDepsToManifest(opts.deps, opts.manifest) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/externals/breakpad/src/tools/python/filter_syms.py b/externals/breakpad/src/tools/python/filter_syms.py deleted file mode 100644 index 8537769e90..0000000000 --- a/externals/breakpad/src/tools/python/filter_syms.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2012 Google LLC -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Normalizes and de-duplicates paths within Breakpad symbol files. - -When using DWARF for storing debug symbols, some file information will be -stored relative to the current working directory of the current compilation -unit, and may be further relativized based upon how the file was #included. - -This helper can be used to parse the Breakpad symbol file generated from such -DWARF files and normalize and de-duplicate the FILE records found within, -updating any references to the FILE records in the other record types. -""" - -import ntpath -import optparse -import os -import posixpath -import sys - -class BreakpadParseError(Exception): - """Unsupported Breakpad symbol record exception class.""" - pass - -class SymbolFileParser(object): - """Parser for Breakpad symbol files. - - The format of these files is documented at - https://chromium.googlesource.com/breakpad/breakpad/+/master/docs/symbol_files.md - """ - - def __init__(self, input_stream, output_stream, ignored_prefixes=None, - path_handler=os.path): - """Inits a SymbolFileParser to read symbol records from |input_stream| and - write the processed output to |output_stream|. - - |ignored_prefixes| contains a list of optional path prefixes that - should be stripped from the final, normalized path outputs. - - For example, if the Breakpad symbol file had all paths starting with a - common prefix, such as: - FILE 1 /b/build/src/foo.cc - FILE 2 /b/build/src/bar.cc - Then adding "/b/build/src" as an ignored prefix would result in an output - file that contained: - FILE 1 foo.cc - FILE 2 bar.cc - - Note that |ignored_prefixes| does not necessarily contain file system - paths, as the contents of the DWARF DW_AT_comp_dir attribute is dependent - upon the host system and compiler, and may contain additional information - such as hostname or compiler version. - """ - - self.unique_files = {} - self.duplicate_files = {} - self.input_stream = input_stream - self.output_stream = output_stream - self.ignored_prefixes = ignored_prefixes or [] - self.path_handler = path_handler - - def Process(self): - """Processes the Breakpad symbol file.""" - for line in self.input_stream: - parsed = self._ParseRecord(line.rstrip()) - if parsed: - self.output_stream.write(parsed + '\n') - - def _ParseRecord(self, record): - """Parses a single Breakpad symbol record - a single line from the symbol - file. - - Returns: - The modified string to write to the output file, or None if no line - should be written. - """ - record_type = record.partition(' ')[0] - if record_type == 'FILE': - return self._ParseFileRecord(record) - elif self._IsLineRecord(record_type): - return self._ParseLineRecord(record) - else: - # Simply pass the record through unaltered. - return record - - def _NormalizePath(self, path): - """Normalizes a file path to its canonical form. - - As this may not execute on the machine or file system originally - responsible for compilation, it may be necessary to further correct paths - for symlinks, junctions, or other such file system indirections. - - Returns: - A unique, canonical representation for the the file path. - """ - return self.path_handler.normpath(path) - - def _AdjustPath(self, path): - """Adjusts the supplied path after performing path de-duplication. - - This may be used to perform secondary adjustments, such as removing a - common prefix, such as "/D/build", or replacing the file system path with - information from the version control system. - - Returns: - The actual path to use when writing the FILE record. - """ - return path[len(next(filter(path.startswith, - self.ignored_prefixes + ['']))):] - - def _ParseFileRecord(self, file_record): - """Parses and corrects a FILE record.""" - file_info = file_record[5:].split(' ', 3) - if len(file_info) > 2: - raise BreakpadParseError('Unsupported FILE record: ' + file_record) - file_index = int(file_info[0]) - file_name = self._NormalizePath(file_info[1]) - existing_file_index = self.unique_files.get(file_name) - if existing_file_index is None: - self.unique_files[file_name] = file_index - file_info[1] = self._AdjustPath(file_name) - return 'FILE ' + ' '.join(file_info) - else: - self.duplicate_files[file_index] = existing_file_index - return None - - def _IsLineRecord(self, record_type): - """Determines if the current record type is a Line record""" - try: - line = int(record_type, 16) - except (ValueError, TypeError): - return False - return True - - def _ParseLineRecord(self, line_record): - """Parses and corrects a Line record.""" - line_info = line_record.split(' ', 5) - if len(line_info) > 4: - raise BreakpadParseError('Unsupported Line record: ' + line_record) - file_index = int(line_info[3]) - line_info[3] = str(self.duplicate_files.get(file_index, file_index)) - return ' '.join(line_info) - -def main(): - option_parser = optparse.OptionParser() - option_parser.add_option("-p", "--prefix", - action="append", dest="prefixes", type="string", - default=[], - help="A path prefix that should be removed from " - "all FILE lines. May be repeated to specify " - "multiple prefixes.") - option_parser.add_option("-t", "--path_type", - action="store", type="choice", dest="path_handler", - choices=['win32', 'posix'], - help="Indicates how file paths should be " - "interpreted. The default is to treat paths " - "the same as the OS running Python (eg: " - "os.path)") - options, args = option_parser.parse_args() - if args: - option_parser.error('Unknown argument: %s' % args) - - path_handler = { 'win32': ntpath, - 'posix': posixpath }.get(options.path_handler, os.path) - try: - symbol_parser = SymbolFileParser(sys.stdin, sys.stdout, options.prefixes, - path_handler) - symbol_parser.Process() - except BreakpadParseError as e: - print >> sys.stderr, 'Got an error while processing symbol file' - print >> sys.stderr, str(e) - return 1 - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/externals/breakpad/src/tools/python/tests/filter_syms_unittest.py b/externals/breakpad/src/tools/python/tests/filter_syms_unittest.py deleted file mode 100644 index f6364b8b32..0000000000 --- a/externals/breakpad/src/tools/python/tests/filter_syms_unittest.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2012 Google LLC -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Unit tests for filter_syms.py""" - -import io -import ntpath -import os -import sys -import unittest - -ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.join(ROOT_DIR, '..')) - -# In root -import filter_syms - -class FilterSysmsTest(unittest.TestCase): - def assertParsed(self, input_data, ignored_prefixes, expected): - input_io = io.StringIO(input_data) - output_io = io.StringIO() - parser = filter_syms.SymbolFileParser(input_io, output_io, - ignored_prefixes, ntpath) - parser.Process() - self.assertEqual(output_io.getvalue(), expected) - - def testDuplicateFiles(self): - """Tests that duplicate files in FILE records are correctly removed and - that Line records are updated.""" - - INPUT = \ -"""MODULE windows x86 111111111111111111111111111111111 module1.pdb -INFO CODE_ID FFFFFFFF module1.exe -FILE 1 foo/../file1_1.cc -FILE 2 bar/../file1_1.cc -FILE 3 baz/../file1_1.cc -FUNC 1000 c 0 Function1_1 -1000 8 45 2 -1008 4 46 3 -100c 4 44 1 -""" - EXPECTED_OUTPUT = \ -"""MODULE windows x86 111111111111111111111111111111111 module1.pdb -INFO CODE_ID FFFFFFFF module1.exe -FILE 1 file1_1.cc -FUNC 1000 c 0 Function1_1 -1000 8 45 1 -1008 4 46 1 -100c 4 44 1 -""" - self.assertParsed(INPUT, [], EXPECTED_OUTPUT) - - def testIgnoredPrefix(self): - """Tests that prefixes in FILE records are correctly removed.""" - - INPUT = \ -"""MODULE windows x86 111111111111111111111111111111111 module1.pdb -INFO CODE_ID FFFFFFFF module1.exe -FILE 1 /src/build/foo/../file1_1.cc -FILE 2 /src/build/bar/../file1_2.cc -FILE 3 /src/build/baz/../file1_2.cc -FUNC 1000 c 0 Function1_1 -1000 8 45 2 -1008 4 46 3 -100c 4 44 1 -""" - EXPECTED_OUTPUT = \ -"""MODULE windows x86 111111111111111111111111111111111 module1.pdb -INFO CODE_ID FFFFFFFF module1.exe -FILE 1 file1_1.cc -FILE 2 file1_2.cc -FUNC 1000 c 0 Function1_1 -1000 8 45 2 -1008 4 46 2 -100c 4 44 1 -""" - IGNORED_PREFIXES = ['\\src\\build\\'] - self.assertParsed(INPUT, IGNORED_PREFIXES, EXPECTED_OUTPUT) - - def testIgnoredPrefixesDuplicateFiles(self): - """Tests that de-duplication of FILE records happens BEFORE prefixes - are removed.""" - - INPUT = \ -"""MODULE windows x86 111111111111111111111111111111111 module1.pdb -INFO CODE_ID FFFFFFFF module1.exe -FILE 1 /src/build/foo/../file1_1.cc -FILE 2 /src/build/bar/../file1_2.cc -FILE 3 D:/src/build2/baz/../file1_2.cc -FUNC 1000 c 0 Function1_1 -1000 8 45 2 -1008 4 46 3 -100c 4 44 1 -""" - EXPECTED_OUTPUT = \ -"""MODULE windows x86 111111111111111111111111111111111 module1.pdb -INFO CODE_ID FFFFFFFF module1.exe -FILE 1 file1_1.cc -FILE 2 file1_2.cc -FILE 3 file1_2.cc -FUNC 1000 c 0 Function1_1 -1000 8 45 2 -1008 4 46 3 -100c 4 44 1 -""" - IGNORED_PREFIXES = ['\\src\\build\\', 'D:\\src\\build2\\'] - self.assertParsed(INPUT, IGNORED_PREFIXES, EXPECTED_OUTPUT) - -if __name__ == '__main__': - unittest.main() -- cgit v1.2.3