LLVM 17 support. (#103)

* Initial changes prior to running bootstrap on clang 17 * Basic support for clang 17 for BootstrapMacros. * More changed for llvm17. I got confused with the new arg parsing; didn't realize that they've introduced new overloads. * Bootstrapped macro file. * Bootstrapped LLVM17, using LLVM17. * Bootstrapped to LLVM17. * Try to integrate printing the attribute itself. * Use '~' as our way of handling builtins. * Fix bindings * Vendored LLVM and nanobind. * Disable mlir in vendored llvm * Try to fix the workflow * More tweaks to try to make CI / submodule configuration work. * Move quotes around * Making a big fatal error message to understand how ci sees things * Accidentally pointed cmake at the wrong directory. * I don't think cmake liked those backslashes * Fix up the execute process * Minor fix. * Made nanobind fetch recursive. Share the install directory between pasta and its vendored deps if pasta is being installed. * Update vendoring of LLVM/Clang to build with C++17. * Try to track the pasta branch of llvm-project * Make llvm-project track its pasta branch. * Updates pasta to track llvm17 branch. * Fix out-of-bounds access, triggered when including stop_token. XREF issue #92. * Fixes some issues in discovering the isysroot. * Update to latest llvm-project * Update to latest llvm-project * Update to latest llvm-project pasta branch. Has fixes for annotation attributtes. * Add some keywords to align tokens for regonitition. Fix a dumb typo in the llvm vendoring cmake * Fiddle with typeloc visitors in bounds. * Re-bootstrap with some nullable types. Fix some bugs in token alignment and bounds. * Disable debug printing * Workflow fix * Update LLVM to fix bug. Simplify pastaConfig.cmake.in. Add more nullable methods * Disable python bindings in ci.
trailofbits · Sep 25, 2023 · 4224ad9 · 4224ad9
1 parent dbc4695
commit 4224ad9
Show file tree

Hide file tree

Showing 857 changed files with 27,163 additions and 25,130 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -26,21 +26,12 @@ jobs:
       matrix:
         build_type: [
           'Release',
-          'Debug'
           ]
         image:
           - { name: 'ubuntu', tag: '22.04' }
-        cxx_common_version: [v0.5.0]
-        llvm_version: [16]
+        llvm_version: [17]
 
     runs-on: ubuntu-22.04
-    container:
-      # Image is built with https://github.com/lifting-bits/cxx-common/blob/master/docker/Dockerfile.ubuntu.vcpkg
-      # Look at that file for the full set of dependencies that may be required for building
-      image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }}
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
     - name: Get the 🍝
@@ -57,15 +48,14 @@ jobs:
     - name: Install system dependencies
       shell: bash
       run: |
-        apt update
-        apt install -y build-essential \
-                       ninja-build \
-                       curl \
-                       ccache \
-                       python3-dev \
-                       libncurses-dev \
-                       clang-15
-        python3 -m pip install nanobind nanobind[global]
+        sudo apt update
+        sudo apt install -y build-essential \
+                            ninja-build \
+                            curl \
+                            ccache \
+                            python3-dev \
+                            libncurses-dev \
+                            clang-15
 
     - name: Setup the build paths
       shell: bash
@@ -99,46 +89,21 @@ jobs:
         path: ${{ steps.build_paths.outputs.REL_CCACHE }}
 
         key: |
-          gitmodules_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.build_type }}_${{ matrix.llvm_version }}_${{ matrix.cxx_common_version }}_${{ github.sha }}
+          gitmodules_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.build_type }}_${{ matrix.llvm_version }}_${{ github.sha }}
 
         restore-keys: |
-          gitmodules_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.build_type }}_${{ matrix.llvm_version }}_${{ matrix.cxx_common_version }}
+          gitmodules_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.build_type }}_${{ matrix.llvm_version }}
 
     - name: Update the cache (downloads)
       uses: actions/cache@v3
       with:
         path: ${{ steps.build_paths.outputs.REL_DOWNLOADS }}
 
         key: |
-          gitmodules_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.build_type }}_${{ matrix.llvm_version }}_${{ matrix.cxx_common_version }}_${{ github.sha }}
+          gitmodules_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.build_type }}_${{ matrix.llvm_version }}_${{ github.sha }}
 
         restore-keys: |
-          gitmodules_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.build_type }}_${{ matrix.llvm_version }}_${{ matrix.cxx_common_version }}
-
-    - name: Acquire the cxx-common ${{ matrix.cxx_common_version }} package for LLVM ${{ matrix.llvm_version }}
-      shell: bash
-      id: cxx_common_installer
-      working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }}
-      run: |
-        folder_name="vcpkg_${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm-${{ matrix.llvm_version }}-pasta_amd64"
-        file_name="${folder_name}.tar.xz"
-        url="https://github.com/lifting-bits/cxx-common/releases/download/${{ matrix.cxx_common_version }}/${file_name}"
-
-        file_path="${{ steps.build_paths.outputs.DOWNLOADS }}/${file_name}"
-        if [[ ! -f "${file_path}" ]]; then
-          curl "${url}" -L -O
-
-          ls -t ${{ steps.build_paths.outputs.DOWNLOADS }}/*.tar.xz | tail -n +2 | while read archive_file ; do
-            rm "${archive_file}"
-          done
-        fi
-
-        tar xf "${file_path}" \
-            -C "${{ steps.build_paths.outputs.INSTALL }}"
-
-        echo ::set-output name=PATH::${{ steps.build_paths.outputs.INSTALL }}/${folder_name}
-
-        echo "${{ steps.build_paths.outputs.INSTALL }}/${folder_name}/installed/x64-linux-rel/bin" >> $GITHUB_PATH
+          gitmodules_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.build_type }}_${{ matrix.llvm_version }}
 
     - name: Configure the project
       working-directory: ${{ steps.build_paths.outputs.BUILD }}
@@ -149,17 +114,14 @@ jobs:
 
       run: |
         cmake -G "Ninja" \
-          -DCMAKE_TOOLCHAIN_FILE="${{ steps.cxx_common_installer.outputs.PATH }}/scripts/buildsystems/vcpkg.cmake" \
-          -DVCPKG_ROOT="${{ steps.cxx_common_installer.outputs.PATH }}" \
           -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
           -DCMAKE_C_COMPILER="$(which clang-15)" \
           -DCMAKE_CXX_COMPILER="$(which clang++-15)" \
           -DPASTA_BOOTSTRAP_MACROS=OFF \
           -DPASTA_BOOTSTRAP_TYPES=OFF \
           -DPASTA_ENABLE_TESTING=OFF \
-          -DPASTA_ENABLE_INSTALL=ON \
-          -DLLVM_DIR="${{ steps.cxx_common_installer.outputs.PATH }}/installed/x64-linux-rel/share/llvm" \
-          -DClang_DIR="${{ steps.cxx_common_installer.outputs.PATH }}/installed/x64-linux-rel/share/clang" \
+          -DPASTA_ENABLE_PY_BINDINGS=OFF \
+          -DPASTA_ENABLE_INSTALL=OFF \
           "${{ steps.build_paths.outputs.SOURCE }}"
 
     - name: Build the project

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,7 @@
+[submodule "vendor/llvm-project/src"]
+	path = vendor/llvm-project/src
+	url = https://github.com/trail-of-forks/llvm-project.git
+	branch = pasta
+[submodule "vendor/nanobind/src"]
+	path = vendor/nanobind/src
+	url = https://github.com/wjakob/nanobind.git
diff --git a/BootstrapConfig.h.in b/BootstrapConfig.h.in
@@ -13,7 +13,7 @@ static const char *kPastaIncludeSourcePath = "@PASTA_INCLUDE_SOURCE_PATH@";
 
 static const char *kPastaBinaryPath = "@PASTA_BINARY_PATH@";
 
-static const char *kVcpkgIncludePath = "@PASTA_VCPKG_INCLUDE_PATH@";
+static const char *kInstallIncludePath = "@CMAKE_INSTALL_PREFIX@/include";
 
 static const char *kClangMacroHeader = "@PASTA_BIN_BOOTSTRAP_TYPES_GENERATED_H_PATH@";
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -70,119 +70,28 @@ target_link_libraries(pasta_cxx_settings INTERFACE std::filesystem)
 # Clang/LLVM dependencies --------------------
 # --------------------------------------------
 
-find_package(zstd CONFIG REQUIRED)
-find_package(Z3 CONFIG REQUIRED)
+add_subdirectory(vendor)
+
 find_package(LLVM CONFIG REQUIRED)
 find_package(Clang CONFIG REQUIRED)
+
 add_library(pasta_thirdparty_llvm INTERFACE)
 if(PASTA_ENABLE_TESTING)
     list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
     include(AddLLVM)  # For LLVM lit test suite stuff.
 endif(PASTA_ENABLE_TESTING)
 
-if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-    set(_PASTA_PLATFORM_MACOS true)
-    add_library(pasta_thirdparty_llvm_force_load INTERFACE)
-else()
-    add_library(pasta_thirdparty_llvm_force_load ALIAS pasta_thirdparty_llvm)
-endif()
-
-# https://github.com/JonathanSalwan/Triton/issues/1082#issuecomment-1030826696
-if(LLVM_LINK_LLVM_DYLIB)
-  set(llvm_libs LLVM)
-else()
-  if (_PASTA_PLATFORM_MACOS)
-    llvm_map_components_to_libnames(llvm_libs_force_load
-      aarch64asmparser aarch64codegen aarch64desc
-      aarch64disassembler aarch64info aarch64utils
-      aggressiveinstcombine analysis armasmparser armcodegen
-      armdesc armdisassembler arminfo armutils asmparser
-      asmprinter binaryformat bitreader bitstreamreader
-      bitwriter cfguard codegen core coroutines coverage
-      debuginfocodeview debuginfodwarf debuginfogsym
-      debuginfologicalview debuginfomsf debuginfopdb demangle
-      dlltooldriver executionengine extensions frontendhlsl
-      frontendopenacc frontendopenmp globalisel instcombine
-      instrumentation interfacestub interpreter ipo irprinter
-      irreader jitlink libdriver linker mc mcdisassembler
-      mcjit mcparser mirparser native nativecodegen
-      nvptxcodegen nvptxdesc nvptxinfo objcarcopts object
-      objectyaml option orcjit orcshared orctargetprocess
-      passes profiledata remarks runtimedyld scalaropts
-      selectiondag sparcasmparser sparccodegen sparcdesc
-      sparcdisassembler sparcinfo support symbolize target
-      targetparser textapi transformutils vectorize
-      webassemblyasmparser webassemblycodegen webassemblydesc
-      webassemblydisassembler webassemblyinfo webassemblyutils
-      windowsdriver windowsmanifest x86asmparser x86codegen
-      x86desc x86disassembler x86info x86targetmca)
-  endif()
-  llvm_map_components_to_libnames(llvm_libs
-    support core
-    irreader
-    bitreader bitwriter
-    passes asmprinter
-    aarch64desc aarch64codegen aarch64asmparser
-    armdesc armcodegen armasmparser
-    interpreter mcjit
-    nvptxdesc nvptxcodegen
-    x86desc x86codegen x86asmparser
-    sparcdesc sparccodegen sparcasmparser
-    webassemblydesc webassemblycodegen webassemblyasmparser)
-endif()
-
 target_link_libraries(pasta_thirdparty_llvm INTERFACE
-    ${llvm_libs}
-
-    clang
-    clangFrontend
-    clangDriver
-    clangSerialization
-    clangParse
-    clangAnalysis
-    clangEdit
     clangAST
-    clangLex
+    clangASTMatchers
     clangBasic
+    clangCodeGen
+    clangDriver
+    clangFrontend
+    clangSerialization
     clangTooling
-    clangIndex
 )
 
-if (_PASTA_PLATFORM_MACOS)
-    target_link_libraries(pasta_thirdparty_llvm_force_load INTERFACE "clang")
-
-    set(libs
-        "clangFrontend"
-        "clangDriver"
-        "clangSerialization"
-        "clangParse"
-        "clangAnalysis"
-        "clangEdit"
-        "clangAST"
-        "clangLex"
-        "clangBasic"
-        "clangSema"
-        "clangASTMatchers"
-        "clangSupport"
-        "z3::libz3"
-        "zstd::libzstd_static"
-        "ZLIB::ZLIB")
-
-  foreach(lib IN LISTS libs llvm_libs_force_load)
-      target_link_options(pasta_thirdparty_llvm_force_load INTERFACE "SHELL:-force_load $<TARGET_FILE:${lib}>")
-  endforeach()
-
-  target_include_directories(pasta_thirdparty_llvm_force_load INTERFACE
-      "$<BUILD_INTERFACE:${LLVM_INCLUDE_DIRS};${LLVM_INCLUDE_DIR};${CLANG_INCLUDE_DIRS}>"
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-
-  target_compile_definitions(pasta_thirdparty_llvm_force_load INTERFACE
-      ${LLVM_DEFINITIONS}
-      ${Clang_DEFINITIONS}
-  )
-endif()
-
 target_include_directories(pasta_thirdparty_llvm INTERFACE
     "$<BUILD_INTERFACE:${LLVM_INCLUDE_DIRS};${LLVM_INCLUDE_DIR};${CLANG_INCLUDE_DIRS}>"
     $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
@@ -263,12 +172,12 @@ set(HOST_H_FILE "${CMAKE_CURRENT_BINARY_DIR}/lib/Compile/Host.h")
 get_compiler_paths(
     COMPILER "${CMAKE_C_COMPILER}"
     OUT_VAR CSTR_HOST_CC_VERSION_INFO
-    ARGS -x c -std=c18
+    ARGS -x c -std=c23
 )
 get_compiler_paths(
     COMPILER "${CMAKE_C_COMPILER}"
     OUT_VAR CSTR_HOST_CC_VERSION_INFO_FAKE_SYSROOT
-    ARGS -x c -std=c18 -isysroot "${CMAKE_CURRENT_BINARY_DIR}/xyz/" -Wno-missing-sysroot
+    ARGS -x c -std=c23 -isysroot "/xyz" -Wno-missing-sysroot
 )
 
 get_compiler_paths(
@@ -279,7 +188,7 @@ get_compiler_paths(
 get_compiler_paths(
     COMPILER "${CMAKE_CXX_COMPILER}"
     OUT_VAR CSTR_HOST_CXX_VERSION_INFO_FAKE_SYSROOT
-    ARGS -x c++ -std=c++20 -isysroot "${CMAKE_CURRENT_BINARY_DIR}/xyz" -Wno-missing-sysroot
+    ARGS -x c++ -std=c++20 -isysroot "/xyz" -Wno-missing-sysroot
 )
 
 configure_file("${HOST_H_IN_FILE}" "${HOST_H_FILE}" @ONLY)
@@ -381,7 +290,6 @@ add_library(pasta_compiler STATIC
     "lib/Compile/Job.h"
     "lib/Compile/PatchedMacroTracker.h"
     "lib/Compile/ParsedFileTracker.h"
-    "lib/Compile/Version.h"
 
     "lib/Compile/Builtins.cpp"
     "lib/Compile/Command.cpp"
@@ -419,14 +327,6 @@ target_link_libraries("pasta" INTERFACE
     pasta_compiler
 )
 
-add_library("pasta_force_load" INTERFACE)
-target_link_libraries("pasta_force_load" INTERFACE
-    pasta_cxx_settings
-    pasta_thirdparty_llvm_force_load
-    pasta_util
-    pasta_compiler
-)
-
 if(PASTA_BOOTSTRAP_MACROS OR PASTA_BOOTSTRAP_TYPES)
 
     # --------------------------------------------
@@ -463,7 +363,6 @@ endif()
 
 set(PASTA_INCLUDE_SOURCE_PATH "${PROJECT_SOURCE_DIR}/include")
 set(PASTA_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}")
-set(PASTA_VCPKG_INCLUDE_PATH "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include")
 set(PASTA_BIN_BOOTSTRAP_MACROS_MACRO_GENERATOR_CPP_PATH "${PROJECT_SOURCE_DIR}/bin/BootstrapMacros/MacroGenerator.cpp")
 set(PASTA_BIN_BOOTSTRAP_TYPES_GENERATED_H_PATH "${PROJECT_SOURCE_DIR}/bin/BootstrapTypes/Generated.h")
 set(PASTA_INCLUDE_AST_FORWARD_H_PATH "${PROJECT_SOURCE_DIR}/include/pasta/AST/Forward.h")
@@ -512,7 +411,6 @@ if(PASTA_ENABLE_INSTALL AND NOT PASTA_BOOTSTRAP_MACROS AND NOT PASTA_BOOTSTRAP_T
     TARGETS
       "pasta" "pasta_compiler" "pasta_util"
       "pasta_thirdparty_llvm" "pasta_cxx_settings"
-      "pasta_thirdparty_llvm_force_load" "pasta_force_load"
       ${PASTA_PY_BINDING_LIB_NAME}
     EXPORT
       "${PROJECT_NAME}Targets"