Improve binary symbolification

Summary:
Backtrace accepts a wide range of native debugging formats, e.g. PDB, dSYM, ELF files. Instead of converting to an intermediate format being breakpad sym files, package and upload the native formats directly.
Also reduce the list of debugging symbols to a reasonable minimum. Sample and test symbols are not required.

Reviewers: #gatekeepers, msutcliffe

Reviewed By: #gatekeepers, msutcliffe

Subscribers: msutcliffe

Maniphest Tasks: T588

Differential Revision: https://dev.swift-project.org/D102

installer/installbuilder/swift-pilotclient.xml

@@ -11,23 +11,55 @@ This installs the binaries to run all swift applications.</detailedDescription>
             <description>bin</description>
             <destination>${installdir}/bin</destination>
             <name>bin_linux</name>
-            <platforms>linux linux-x64 osx</platforms>
+            <platforms>linux linux-x64</platforms>
             <distributionFileList>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/bin/swiftcore*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
                 </distributionFile>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/bin/swiftdata*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
                 </distributionFile>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/bin/swiftguistd*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
                 </distributionFile>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/bin/swiftlauncher*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
+                </distributionFile>
+            </distributionFileList>
+        </folder>
+        <folder>
+            <description>bin</description>
+            <destination>${installdir}/bin</destination>
+            <name>bin_osx</name>
+            <platforms>osx</platforms>
+            <distributionFileList>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/bin/swiftcore*</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
+                </distributionFile>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/bin/swiftdata*</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
+                </distributionFile>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/bin/swiftguistd*</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
+                </distributionFile>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/bin/swiftlauncher*</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
                 </distributionFile>
             </distributionFileList>
         </folder>
@@ -90,31 +122,75 @@ This installs the binaries to run all swift applications.</detailedDescription>
             <description>lib</description>
             <destination>${installdir}/lib</destination>
             <name>lib_linux</name>
-            <platforms>linux linux-x64 osx</platforms>
+            <platforms>linux linux-x64</platforms>
             <distributionFileList>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/lib/libblackcore*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
                 </distributionFile>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/lib/libblackgui*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
                 </distributionFile>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/lib/libblackinput*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
                 </distributionFile>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/lib/libblackmisc*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
                 </distributionFile>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/lib/libblacksound*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
                 </distributionFile>
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/lib/libsimulatorplugincommon*</origin>
+                    <excludeFiles>*/*.debug</excludeFiles>
+                </distributionFile>
+            </distributionFileList>
+        </folder>
+        <folder>
+            <description>lib</description>
+            <destination>${installdir}/lib</destination>
+            <name>lib_osx</name>
+            <platforms>osx</platforms>
+            <distributionFileList>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/lib/libblackcore*.dylib</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
+                </distributionFile>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/lib/libblackgui*.dylib</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
+                </distributionFile>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/lib/libblackinput*.dylib</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
+                </distributionFile>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/lib/libblackmisc*.dylib</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
+                </distributionFile>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/lib/libblacksound*.dylib</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
+                </distributionFile>
+                <distributionFile>
+                    <allowWildcards>1</allowWildcards>
+                    <origin>../../dist/lib/libsimulatorplugincommon*.dylib</origin>
+                    <excludeFiles>*/*.dSYM</excludeFiles>
                 </distributionFile>
             </distributionFileList>
         </folder>

installer/installbuilder/swift-plugin-simulators.xml

@@ -119,6 +119,7 @@ This plugin acts as the interface between swift and X-Plane</detailedDescription
                         <distributionFile>
                             <allowWildcards>1</allowWildcards>
                             <origin>../../dist/bin/plugins/simulator/*simulatorxplane*</origin>
+                            <excludeFiles>*/*.pdb;*/*.debug;*/*.dSYM</excludeFiles>
                         </distributionFile>
                     </distributionFileList>
                 </folder>
@@ -142,6 +143,7 @@ This plugin acts as the interface between swift and FlightGear</detailedDescript
                         <distributionFile>
                             <allowWildcards>1</allowWildcards>
                             <origin>../../dist/bin/plugins/simulator/*simulatorflightgear*</origin>
+                            <excludeFiles>*/*.pdb;*/*.debug;*/*.dSYM</excludeFiles>
                         </distributionFile>
                     </distributionFileList>
                 </folder>
@@ -165,6 +167,7 @@ This plugin emulates a running simulator. Use this for testing swift only</detai
                         <distributionFile>
                             <allowWildcards>1</allowWildcards>
                             <origin>../../dist/bin/plugins/simulator/*simulatoremulated*</origin>
+                            <excludeFiles>*/*.pdb;*/*.debug;*/*.dSYM</excludeFiles>
                         </distributionFile>
                     </distributionFileList>
                 </folder>

installer/installbuilder/swift-plugin-weatherdata.xml

@@ -22,6 +22,7 @@
                         <distributionFile>
                             <allowWildcards>1</allowWildcards>
                             <origin>../../dist/bin/plugins/weatherdata/*weatherdatagfs*</origin>
+                            <excludeFiles>*/*.pdb;*/*.debug;*/*.dSYM</excludeFiles>
                         </distributionFile>
                     </distributionFileList>
                 </folder>

installer/installbuilder/swift-samples.xml

@@ -15,6 +15,7 @@
                 <distributionFile>
                     <allowWildcards>1</allowWildcards>
                     <origin>../../dist/bin/sample*</origin>
+                    <excludeFiles>*/*.pdb;*/*.debug;*/*.dSYM</excludeFiles>
                 </distributionFile>
             </distributionFileList>
         </folder>

mkspecs/features/common_pre.prf

@@ -77,6 +77,10 @@ swiftConfig(releaseWithDebugInfo) {
         QMAKE_CXXFLAGS_RELEASE *= /Z7
         QMAKE_LFLAGS_RELEASE *= /DEBUG /INCREMENTAL:NO /OPT:REF /OPT:ICF
     }
+    unix {
+        CONFIG += force_debug_info
+        CONFIG += separate_debug_info
+    }
 }
 swiftConfig(profileRelease) {
     msvc {

scripts/jenkins.py

@@ -19,7 +19,7 @@ import requests
 import subprocess
 import sys
 import datastore
-import symbolstore
+import tarfile
 from lib.util import get_vs_env
 
 if sys.version_info < (3, 0):
@@ -149,16 +149,46 @@ class Builder:
             build_path = self._get_swift_build_path()
             os.chdir(build_path)
             print('Creating symbols')
-            symbol_path = path.abspath(path.join(build_path, 'symbols'))
             binary_path = path.abspath(path.join(build_path, 'out'))
-            symbolstore.Dumper.global_init()
+
-            dumper = symbolstore.get_platform_specific_dumper(dump_syms=self.dump_syms, symbol_path=symbol_path)
-            dumper.process(binary_path)
-            dumper.finish()
             os_map = {'Linux': 'linux', 'Darwin': 'macos', 'Windows': 'windows'}
-            tar_filename = '-'.join(['swiftsymbols', os_map[platform.system()], self.word_size, self.version]) + '.tar.gz'
+            tar_filename = '-'.join(
+                ['swiftsymbols', os_map[platform.system()], self.word_size, self.version]) + '.tar.gz'
             tar_path = path.abspath(path.join(self._get_swift_source_path(), tar_filename))
-            dumper.pack(tar_path)
+            tar = tarfile.open(tar_path, "w:gz")
+
+            ignore_list = ['sample', 'test', 'win', 'liblin.so', 'libmac.dylib']
+            for root, dirs, files in os.walk(binary_path):
+                if platform.system() == 'Windows':
+                    for f in files:
+                        if f.endswith('.pdb') and not f.startswith(tuple(ignore_list)):
+                            symbol_path = path.abspath(path.join(root, f))
+                            print('Adding ' + symbol_path)
+                            tar.add(symbol_path, f)
+                            if self.word_size == '64':
+                                # Add also *.exe/*.dll with the same name if existing
+                                exe_path = symbol_path.replace('.pdb', '.exe')
+                                if os.path.isfile(exe_path):
+                                    print('Adding ' + exe_path)
+                                    tar.add(exe_path, f.replace('.pdb', '.exe'))
+                                dll_path = symbol_path.replace('.pdb', '.dll')
+                                if os.path.isfile(dll_path):
+                                    print('Adding ' + dll_path)
+                                    tar.add(dll_path, f.replace('.pdb', '.dll'))
+                elif platform.system() == 'Darwin':
+                    for d in dirs:
+                        if d.endswith('.dSYM') and not d.startswith(tuple(ignore_list)):
+                            symbol_path = path.abspath(path.join(root, d))
+                            print('Adding ' + symbol_path)
+                            tar.add(symbol_path, d)
+                elif platform.system() == 'Linux':
+                    for f in files:
+                        if f.endswith('.debug') and not f.startswith(tuple(ignore_list)):
+                            symbol_path = path.abspath(path.join(root, f))
+                            print('Adding ' + symbol_path)
+                            tar.add(symbol_path, f)
+            tar.close()
+
             if upload_symbols:
                 self.__upload_symbol_files(tar_path)