diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 09ac12c82b7e83..965bf3d97f3a4f 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -19,6 +19,8 @@ parameters: collectionName: '' targetOS: '' targetArch: '' + maxPartitionSizeMB: 25 + crossgen2Composite: '' jobs: - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -34,7 +36,7 @@ jobs: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true continueOnError: ${{ parameters.continueOnError }} - collectionType: $ {{ parameters.collectionType }} + collectionType: ${{ parameters.collectionType }} collectionName: ${{ parameters.collectionName }} ${{ if ne(parameters.displayName, '') }}: @@ -77,6 +79,15 @@ jobs: - name: TargetArchForCollection value: ${{ parameters.archType }} + # Cross-target collections can share collectionName/collectionType with a host-target + # collection on the same osGroup/archType/buildConfig, so published artifact names need + # the target to stay unique. + - name: CollectionTargetSuffix + ${{ if ne(parameters.targetArch, '') }}: + value: '_${{ parameters.targetOS }}_${{ parameters.targetArch }}' + ${{ else }}: + value: '' + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml parameters: osGroup: ${{ parameters.osGroup }} @@ -109,25 +120,20 @@ jobs: value: '$(Build.SourcesDirectory)/payload' - ${{ if eq(parameters.collectionName, 'libraries') }}: - - name: InputDirectory - value: '$(Core_Root_Dir)' - - ${{ if eq(parameters.collectionName, 'corelib') }}: - # For the corelib crossgen2 collection, InputDirectory holds the directory - # containing the single System.Private.CoreLib.dll that crossgen2 will compile. - # For wasm cross-target collections, source the wasm-built corelib (which has - # the correct TARGET_BROWSER / 32-bit conditional IL) instead of the host - # release Core_Root corelib. The wasm corelib build emits the managed assembly - # under an 'IL' subdirectory of the configuration output. + # Crossgen2 collections targeting browser/wasm compile the browser-wasm CoreCLR + # runtime pack rather than the host Core_Root. InputDirectory is the runtime pack + # RID directory; superpmi_collect_setup.py flattens lib//*.dll together with + # native/System.Private.CoreLib.dll into the assembly set to collect over. - ${{ if eq(parameters.targetArch, 'wasm') }}: - ${{ if eq(parameters.osGroup, 'windows') }}: - name: InputDirectory - value: '$(Build.SourcesDirectory)\artifacts\bin\coreclr\browser.wasm.Release\IL' + value: '$(Build.SourcesDirectory)\artifacts\bin\microsoft.netcore.app.runtime.browser-wasm\Release\runtimes\browser-wasm' - ${{ if ne(parameters.osGroup, 'windows') }}: - name: InputDirectory - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/browser.wasm.Release/IL' + value: '$(Build.SourcesDirectory)/artifacts/bin/microsoft.netcore.app.runtime.browser-wasm/Release/runtimes/browser-wasm' - ${{ if ne(parameters.targetArch, 'wasm') }}: - name: InputDirectory - value: '$(Release_Core_Root_Dir)' + value: '$(Core_Root_Dir)' - ${{ if eq(parameters.collectionName, 'benchmarks') }}: - name: InputDirectory value: '$(Core_Root_Dir)' @@ -167,7 +173,7 @@ jobs: displayName: Enable python venv condition: always() - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -release_core_root_directory $(Release_Core_Root_Dir) -arch $(archType) -platform $(osGroup) -target_os $(TargetOSForCollection) -target_arch $(TargetArchForCollection) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size 25 # size in MB + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -release_core_root_directory $(Release_Core_Root_Dir) -arch $(archType) -platform $(osGroup) -target_os $(TargetOSForCollection) -target_arch $(TargetArchForCollection) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size ${{ parameters.maxPartitionSizeMB }} # size in MB displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs @@ -186,7 +192,7 @@ jobs: - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml parameters: HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)/$(Architecture)' + HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)$(CollectionTargetSuffix)/$(Architecture)' HelixAccessToken: $(HelixApiAccessToken) HelixTargetQueues: $(Queue) HelixPreCommands: $(HelixPreCommand) @@ -206,6 +212,8 @@ jobs: InputArtifacts: '$(InputArtifacts)' CollectionType: '$(CollectionType)' CollectionName: '$(CollectionName)' + Crossgen2ReferenceDirName: '$(Crossgen2ReferenceDirName)' + Crossgen2Composite: '${{ parameters.crossgen2Composite }}' continueOnError: true # Run the future step i.e. merge-mch step even if this step fails. # Always run merge step even if collection of some partition fails so we can store collection @@ -223,7 +231,7 @@ jobs: archiveType: $(archiveType) tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) - artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)' + artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)$(CollectionTargetSuffix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)' displayName: ${{ format('Upload artifacts SuperPMI {0}-{1} collection', parameters.collectionName, parameters.collectionType) }} - ${{ if eq(variables['System.TeamProject'], 'internal') }}: @@ -260,7 +268,7 @@ jobs: displayName: Publish SuperPMI logs inputs: targetPath: $(SpmiLogsLocation) - artifactName: 'SuperPMI_Logs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)' + artifactName: 'SuperPMI_Logs_$(CollectionName)_$(CollectionType)$(CollectionTargetSuffix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)' condition: always() continueOnError: true @@ -268,6 +276,6 @@ jobs: displayName: Publish SuperPMI build logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'SuperPMI_BuildLogs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)' + artifactName: 'SuperPMI_BuildLogs_$(CollectionName)_$(CollectionType)$(CollectionTargetSuffix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)' condition: always() continueOnError: true diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml index 7045b07c0e95cc..2776c9816b299f 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml @@ -13,6 +13,8 @@ parameters: collectionName: '' targetOS: '' targetArch: '' + maxPartitionSizeMB: 25 + crossgen2Composite: '' ### SuperPMI job ### Each collection job depends on a corresponding build job with the same buildConfig and archType. @@ -22,7 +24,12 @@ jobs: parameters: # Compute job name from template parameters jobName: ${{ format('superpmi_collect_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('SuperPMI collect {4} {5} {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.collectionName, parameters.collectionType) }} + ${{ if eq(parameters.targetArch, '') }}: + displayName: ${{ format('SuperPMI collect {4} {5} {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.collectionName, parameters.collectionType) }} + ${{ else }}: + # Cross-target collections share the host osGroup/archType with a same-named host + # collection, so include the target in the display name to tell them apart. + displayName: ${{ format('SuperPMI collect {4} {5} {6} {7} {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.collectionName, parameters.collectionType, parameters.targetOS, parameters.targetArch) }} pool: ${{ parameters.pool }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -35,15 +42,17 @@ jobs: collectionName: ${{ parameters.collectionName }} targetOS: ${{ parameters.targetOS }} targetArch: ${{ parameters.targetArch }} + maxPartitionSizeMB: ${{ parameters.maxPartitionSizeMB }} + crossgen2Composite: ${{ parameters.crossgen2Composite }} # Test job depends on the corresponding build job dependsOn: - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}: - 'coreclr_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}' - # For the corelib crossgen2 collection targeting browser/wasm we also need a - # browser/wasm-built System.Private.CoreLib.dll, produced by the browser_wasm_win - # Release build. - - ${{ if and(eq(parameters.collectionName, 'corelib'), eq(parameters.targetArch, 'wasm')) }}: + # Crossgen2 collections targeting browser/wasm compile the browser/wasm managed + # assembly set, which comes from the browser-wasm CoreCLR runtime pack produced by + # the browser_wasm_win Release build. + - ${{ if eq(parameters.targetArch, 'wasm') }}: - 'build_browser_wasm_windows_Release_' variables: @@ -78,16 +87,16 @@ jobs: cleanUnpackFolder: false overwriteExistingFiles: true - # For the corelib crossgen2 collection targeting browser/wasm, also download the - # browser/wasm-built runtime artifacts. The wasm-built System.Private.CoreLib.dll - # under artifacts/bin/coreclr/browser.wasm.Release/ is what crossgen2 will compile. - - ${{ if and(eq(parameters.collectionName, 'corelib'), eq(parameters.targetArch, 'wasm')) }}: + # For crossgen2 collections targeting browser/wasm, also download the browser-wasm + # CoreCLR runtime pack. Its runtimes/browser-wasm/lib//*.dll and + # runtimes/browser-wasm/native/System.Private.CoreLib.dll are what crossgen2 compiles. + - ${{ if eq(parameters.targetArch, 'wasm') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: 'BuildArtifacts_browser_wasm_Release$(archiveExtension)' - artifactName: BuildArtifacts_browser_wasm_Release - displayName: 'browser/wasm Release runtime build artifacts' + artifactFileName: 'WasmRuntimePack_browser_wasm_Release$(archiveExtension)' + artifactName: WasmRuntimePack_browser_wasm_Release + displayName: 'browser/wasm Release runtime pack' cleanUnpackFolder: false overwriteExistingFiles: true diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index 05d65ac4ab07d1..d13466ef09c531 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -19,7 +19,7 @@ extends: testGroup: outerloop # Build clr.wasmjit on x64 hosts so the checked wasm cross-targeting JIT # (clrjit_universal_wasm_x64.dll) is present in the checked Core_Root for - # crossgen2-corelib-wasm SPMI collection. clr.alljits does not include it. + # crossgen2-libraries-wasm SPMI collection. clr.alljits does not include it. buildArgs: -s clr+libs+libs.tests+clr.wasmjit -rc $(_BuildConfig) -c Release /p:ArchiveTests=true timeoutInMinutes: 120 postBuildSteps: @@ -169,10 +169,12 @@ extends: testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline - # Release build of System.Private.CoreLib.dll targeting browser/wasm, hosted on - # Windows x64. Produces artifacts/bin/coreclr/browser.wasm.Release/System.Private.CoreLib.dll, - # which is the assembly the corelib_wasm SPMI collection runs crossgen2 over. - # We only need the managed CoreLib here, so build the minimal clr.corelib subset. + # Release build of the browser/wasm CoreCLR runtime pack, hosted on Windows x64. + # Produces artifacts/bin/microsoft.netcore.app.runtime.browser-wasm/Release, whose + # runtimes/browser-wasm/lib//*.dll and runtimes/browser-wasm/native/System.Private.CoreLib.dll + # are the managed assemblies the wasm SPMI collection runs crossgen2 over. + # The runtime pack layout comes from the packs.product subset, which harvests native + # runtime files, so the full clr+libs+packs set has to be built here. - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml @@ -180,17 +182,19 @@ extends: platforms: - browser_wasm_win jobParameters: - buildArgs: -s clr.corelib -c Release - timeoutInMinutes: 120 + buildArgs: -s clr+libs+packs -c Release + timeoutInMinutes: 180 postBuildSteps: + # Only the runtime pack is needed downstream, so avoid uploading the whole + # (much larger) artifacts/bin tree that clr+libs+packs produces. - template: /eng/pipelines/common/upload-artifact-step.yml parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/bin - includeRootFolder: false + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/microsoft.netcore.app.runtime.browser-wasm + includeRootFolder: true archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: WasmRuntimePack_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) disableComponentGovernance: true # No shipping artifacts produced by this pipeline - template: /eng/pipelines/common/platform-matrix.yml @@ -330,7 +334,8 @@ extends: collectionType: crossgen2 collectionName: libraries - # Crossgen2 of release System.Private.CoreLib.dll targeting browser/wasm. + # Crossgen2 of the release browser/wasm managed assembly set (the browser-wasm + # CoreCLR runtime pack: all shared framework libraries plus System.Private.CoreLib.dll). # Runs crossgen2 on a windows_x64 host with the wasm cross-targeting JIT # (clrjit_universal_wasm_x64.dll) under superpmi-shim-collector, producing an MCH # uploaded under /browser/x64/ (the wasm mch_arch override in @@ -350,9 +355,23 @@ extends: testGroup: outerloop liveLibrariesBuildConfig: Release collectionType: crossgen2 - collectionName: corelib + collectionName: libraries targetOS: browser targetArch: wasm + # Compile in composite mode so the whole framework is one version bubble and + # generic instantiations over types from other assemblies (most importantly + # CoreLib) get compiled instead of being skipped as cross-bubble. Measured on + # this collection: 82,870 -> 109,684 contexts (+32%). + crossgen2Composite: true + # Keep the collection partitioned. Each work item roots eligible methods from + # its partition's positional input assemblies. The rest of the framework is + # included as -u unrooted composite inputs in the same compilation and version + # bubble: not independently rooted, but able to contribute compiled methods + # and generic instantiations as dependencies. The redundant work between + # partitions is deduplicated by the merge step (177,230 contexts collected, + # 109,684 kept). That costs CPU rather than coverage, and keeps a failed + # partition from losing the whole collection. Work items ran 1-2 minutes each. + maxPartitionSizeMB: 5 - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml index 05ca4fcb7457a4..180e479f3038d1 100644 --- a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml +++ b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml @@ -24,6 +24,8 @@ parameters: InputArtifacts: '' CollectionType: '' CollectionName: '' + Crossgen2ReferenceDirName: '' # optional -- name of the crossgen2 reference assembly folder in the correlation payload (cross-target crossgen2 collections) + Crossgen2Composite: '' # optional -- 'true' to run crossgen2 in composite mode (whole reference set in the version bubble) ProjectFile: '' targetOS: '' # optional -- override target OS for cross-target collections (e.g. browser/wasm). Defaults to osGroup. targetArch: '' # optional -- override target architecture. Defaults to archType. @@ -51,6 +53,8 @@ steps: InputArtifacts: ${{ parameters.InputArtifacts }} CollectionType: ${{ parameters.CollectionType }} CollectionName: ${{ parameters.CollectionName }} + Crossgen2ReferenceDirName: ${{ parameters.Crossgen2ReferenceDirName }} + Crossgen2Composite: ${{ parameters.Crossgen2Composite }} HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 4efce6d0fd83c6..368ad85e9c8b4b 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* 406b139f-b910-4579-a9a8-91eb153ab024 */ - 0x406b139f, - 0xb910, - 0x4579, - {0xa9, 0xa8, 0x91, 0xeb, 0x15, 0x3a, 0xb0, 0x24} +constexpr GUID JITEEVersionIdentifier = { /* d9636771-eb8f-4e21-8a6f-d97f21e658f7 */ + 0xd9636771, + 0xeb8f, + 0x4e21, + {0x8a, 0x6f, 0xd9, 0x7f, 0x21, 0xe6, 0x58, 0xf7} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 83c18ae50fa53b..17c8eef5b85c03 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -159,6 +159,21 @@ -target_os $(TargetOS) -target_arch $(TargetArchitecture) + + + $(TargetArgs) -crossgen2_reference_directory $(SuperPMIDirectory)$(FileSeparatorChar)$(Crossgen2ReferenceDirName) + + + + $(TargetArgs) --crossgen2_composite + @@ -168,6 +183,9 @@ $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi.py collect --clean -log_level DEBUG --$(CollectionType) $(PmiArguments) $(InputKind) $(AssembliesDirectoryOnHelix) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory) $(TargetArgs) 2:00 + + 4:00 diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index e73e0f8700ec70..99f6b116e3b2c2 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -140,7 +140,7 @@ diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index cae2d49974dc22..b615104b8e5946 100644 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -20,6 +20,7 @@ import asyncio import csv import datetime +import glob import html import json import locale @@ -310,6 +311,8 @@ def add_core_root_arguments(parser, build_type_default, build_type_help): collect_parser.add_argument("--crossgen2", action="store_true", help="Run crossgen2 on a set of directories or assemblies.") collect_parser.add_argument("--nativeaot", action="store_true", help="Run nativeaot on a set of directories or 'ilc.rsps' files.") collect_parser.add_argument("-assemblies", dest="assemblies", nargs="+", default=[], help="A list of managed dlls or directories to recursively use while collecting with PMI or crossgen2. Required if --pmi or --crossgen2 is specified.") +collect_parser.add_argument("-crossgen2_reference_directory", help="Directory containing the assemblies crossgen2 should resolve references against. Optional; defaults to the Core_Root directory. Used for cross-target crossgen2 collections, where the assemblies being compiled are built for the target and not for the host.") +collect_parser.add_argument("--crossgen2_composite", action="store_true", help="Run crossgen2 in composite mode: compile all the assemblies in one invocation, with the whole reference set in the version bubble, so cross-assembly generic instantiations are compiled as well. Slower, but collects many more method contexts.") collect_parser.add_argument("-ilc_rsps", dest="ilc_rsps", nargs="+", default=[], help="For --nativeaot only. A list of 'ilc.rsp' files.") collect_parser.add_argument("-exclude", dest="exclude", nargs="+", default=[], help="A list of files or directories to exclude from the files and directories specified by `-assemblies`.") collect_parser.add_argument("-pmi_location", help="Path to pmi.dll to use during PMI run. Optional; pmi.dll will be downloaded from Azure Storage if necessary.") @@ -573,6 +576,85 @@ def decode_clrjit_build_string(clrjit_path): # Helper classes ################################################################################ +def get_assembly_simple_name(filename): + """ Get the assembly simple name crossgen2 will key a module by, i.e. the file name + without its directory or extension, lowercased for comparison. + + Args: + filename (str) : path to the assembly + + Returns: + The lowercased simple name. + """ + + return os.path.splitext(os.path.basename(filename))[0].lower() + + +def is_managed_assembly(filename): + """ Determine if a file is a managed assembly by looking for a non-empty CLI header + in its PE data directory. Used to keep native binaries out of the crossgen2 + composite input set, which crossgen2 refuses to load. + + Args: + filename (str) : path to the file to check + + Returns: + True if the file is a managed assembly, False otherwise (including if it + can't be read or isn't a PE file at all). + """ + + try: + with open(filename, "rb") as file_handle: + if file_handle.read(2) != b"MZ": + return False + + file_handle.seek(0x3C) + pe_header_offset_bytes = file_handle.read(4) + if len(pe_header_offset_bytes) != 4: + return False + pe_header_offset = int.from_bytes(pe_header_offset_bytes, "little") + + file_handle.seek(pe_header_offset) + if file_handle.read(4) != b"PE\0\0": + return False + + # Skip the COFF header (20 bytes) to reach the optional header. + optional_header_offset = pe_header_offset + 4 + 20 + file_handle.seek(optional_header_offset) + magic_bytes = file_handle.read(2) + if len(magic_bytes) != 2: + return False + magic = int.from_bytes(magic_bytes, "little") + if magic == 0x10B: # PE32 + data_directory_offset = optional_header_offset + 96 + elif magic == 0x20B: # PE32+ + data_directory_offset = optional_header_offset + 112 + else: + return False + + # The CLI header is data directory entry 14; each entry is 8 bytes. + file_handle.seek(data_directory_offset + 14 * 8) + cli_header_entry = file_handle.read(8) + # Require a complete entry with a nonzero CLI header RVA (first 4 bytes) and size (last 4 bytes). + return len(cli_header_entry) == 8 and \ + int.from_bytes(cli_header_entry[:4], "little") != 0 and \ + int.from_bytes(cli_header_entry[4:], "little") != 0 + except OSError: + return False + + +def get_crossgen2_reference_assemblies(reference_directory): + """ Get managed composite inputs, filtering out native binaries and rejecting an empty result. """ + + candidates = [] + for extension in [".dll", ".exe"]: + candidates += glob.glob(os.path.join(reference_directory, "*" + extension)) + assemblies = [path for path in sorted(candidates) if is_managed_assembly(path)] + if not assemblies: + raise RuntimeError("No managed assemblies found in crossgen2 reference directory: " + reference_directory) + return assemblies + + class AsyncSubprocessHelper: """ Class to help with async multiprocessing tasks. """ @@ -1048,11 +1130,22 @@ async def run_pmi(print_prefix, assembly, self): if self.coreclr_args.crossgen2 is True: logging.debug("Starting collection using crossgen2") - async def run_crossgen2(print_prefix, assembly, self): - """ Run crossgen2 over all dlls + async def run_crossgen2(print_prefix, input_assemblies, self): + """ Run crossgen2 over a set of assemblies. + + In the default (non-composite) mode `input_assemblies` holds a single + assembly and each assembly is compiled in its own version bubble. In + composite mode it holds the partition's positional input assemblies, + which root eligible methods. The rest of the framework is passed as + -u unrooted composite inputs in the same compilation and version bubble, + not independently rooted, but able to contribute compiled methods and + generic instantiations as dependencies. """ - root_crossgen2_output_filename = make_safe_filename("crossgen2_" + assembly) + ".out.dll" + is_composite = self.coreclr_args.crossgen2_composite + output_tag = "composite" if is_composite else input_assemblies[0] + + root_crossgen2_output_filename = make_safe_filename("crossgen2_" + output_tag) + ".out.dll" crossgen2_output_assembly_filename = os.path.join(self.temp_location, root_crossgen2_output_filename) try: if os.path.exists(crossgen2_output_assembly_filename): @@ -1065,19 +1158,28 @@ async def run_crossgen2(print_prefix, assembly, self): else: raise ose - root_output_filename = make_safe_filename("crossgen2_" + assembly + "_") + root_output_filename = make_safe_filename("crossgen2_" + output_tag + "_") # Create a temporary response file to put all the arguments to crossgen2 (otherwise the path length limit could be exceeded): # - # + # /// one line per input assembly # -o: - # -r:\System.*.dll - # -r:\Microsoft.*.dll - # -r:\System.Private.CoreLib.dll - # -r:\netstandard.dll + # --composite /// composite mode only + # -O /// composite mode only + # -u:"\.dll" /// composite mode only, for each unrooted input + # -r:\System.*.dll + # -r:\Microsoft.*.dll + # -r:\mscorlib.dll /// System.*.dll above includes CoreLib + # -r:\netstandard.dll # --jitpath: # --codegenopt: