From cacaa82e4f7b4fc07d75ec55c189941513a3af23 Mon Sep 17 00:00:00 2001 From: Petr Onderka Date: Thu, 29 Oct 2026 19:10:24 +0100 Subject: [PATCH 1/4] Fix native-library discovery and oneDAL test dependencies Use assembly-aware runtime loading on modern .NET, preserve the legacy fallback, and copy oneDAL/TBB dependencies using Native.proj's shared asset list. Add native-loading regression coverage and narrow the legacy wrapper's API. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0c982995-379f-4379-bcb7-dba5d1102bb4 --- Directory.Build.targets | 14 +++ src/Native/Native.proj | 60 ++-------- .../UnitTests/NativeLibraryTests.cs | 53 +++++++++ .../Utility/Kernel32.cs | 5 + .../Utility/LibraryLoader.cs | 4 + .../Utility/NativeLibrary.cs | 108 ++++-------------- .../Utility/PathResolver.cs | 7 +- 7 files changed, 118 insertions(+), 133 deletions(-) create mode 100644 test/Microsoft.ML.Core.Tests/UnitTests/NativeLibraryTests.cs diff --git a/Directory.Build.targets b/Directory.Build.targets index 56648dd5b7..14c8da9b80 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -38,6 +38,20 @@ + + + + + + + + $(NativeOutputPath)%(_OneDalNativeDependency.Identity) + + diff --git a/src/Native/Native.proj b/src/Native/Native.proj index 9e829fe250..c7ff95cb63 100644 --- a/src/Native/Native.proj +++ b/src/Native/Native.proj @@ -93,6 +93,15 @@ libtbbmalloc.2.dylib + + <_OneDalNativeDependency Include="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_core$(OneDalLibExtension)" /> + <_OneDalNativeDependency Include="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_thread$(OneDalLibExtension)" /> + <_OneDalNativeDependency Include="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbLibrary)" /> + <_OneDalNativeDependency Include="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbMallocLibrary)" /> + + + + @@ -151,35 +160,7 @@ - - - - - - - - - - - - - + - - - - - @@ -237,6 +199,8 @@ RelativePath="Microsoft.ML.Mkl.Redist\runtimes\$(PackageRid)\native" /> + diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/NativeLibraryTests.cs b/test/Microsoft.ML.Core.Tests/UnitTests/NativeLibraryTests.cs new file mode 100644 index 0000000000..b63d1d796c --- /dev/null +++ b/test/Microsoft.ML.Core.Tests/UnitTests/NativeLibraryTests.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.InteropServices; +using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFramework.Attributes; +using Xunit; +using Xunit.Abstractions; +using NativeLibrary = Microsoft.ML.TestFrameworkCommon.Utility.NativeLibrary; + +namespace Microsoft.ML.RunTests +{ + public sealed class NativeLibraryTests : BaseTestClass + { + private static bool IsOneDalSupported => + RuntimeInformation.ProcessArchitecture == Architecture.X64 && + // We don't test macOS x64 for OneDAL. + !RuntimeInformation.IsOSPlatform(OSPlatform.OSX); + + public NativeLibraryTests(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void SystemNativeLibraryCanBeLoaded() + { + string library = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? "kernel32" + : RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? "libc.so.6" : "libSystem.B.dylib"; + + Assert.True(NativeLibrary.NativeLibraryExists(library)); + } + + [Fact] + public void MissingNativeLibraryReturnsFalse() + { + Assert.False(NativeLibrary.NativeLibraryExists("MLNetNonexistentNativeLibrary")); + } + + [X64Fact("The LightGBM native package supports x64 only.")] + public void PackageNativeLibraryCanBeLoaded() + { + Assert.True(NativeLibrary.NativeLibraryExists("lib_lightgbm")); + } + + [ConditionalFact(nameof(IsOneDalSupported))] + public void OneDalNativeLibraryCanBeLoaded() + { + Assert.True(NativeLibrary.NativeLibraryExists("OneDalNative")); + } + } +} diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/Kernel32.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/Kernel32.cs index 2018677335..d02cacf4f4 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/Kernel32.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/Kernel32.cs @@ -6,9 +6,14 @@ namespace Microsoft.ML.TestFrameworkCommon.Utility { internal static class Kernel32 { + public const uint LoadWithAlteredSearchPath = 0x00000008; + [DllImport("kernel32")] public static extern IntPtr LoadLibrary(string fileName); + [DllImport("kernel32", CharSet = CharSet.Unicode)] + public static extern IntPtr LoadLibraryEx(string fileName, IntPtr file, uint flags); + [DllImport("kernel32")] public static extern IntPtr GetProcAddress(IntPtr module, string procName); diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/LibraryLoader.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/LibraryLoader.cs index 3fb89075ea..6e75271e36 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/LibraryLoader.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/LibraryLoader.cs @@ -194,6 +194,10 @@ protected override IntPtr CoreLoadFunctionPointer(IntPtr handle, string function protected override IntPtr CoreLoadNativeLibrary(string name) { + // Resolve dependencies beside the DLL, not just beside the test host. + if (Path.IsPathRooted(name)) + return Kernel32.LoadLibraryEx(Path.GetFullPath(name), IntPtr.Zero, Kernel32.LoadWithAlteredSearchPath); + return Kernel32.LoadLibrary(name); } } diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs index 92eb50743f..cc7e67c9e0 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs @@ -1,12 +1,14 @@ // Taken from https://github.com/mellinoe/nativelibraryloader/blob/586f9738ff12688df8f0662027da8c319aee3841/NativeLibraryLoader/NativeLibrary.cs using System; using System.Runtime.InteropServices; +#if NET5_0_OR_GREATER +using RuntimeNativeLibrary = System.Runtime.InteropServices.NativeLibrary; +#endif namespace Microsoft.ML.TestFrameworkCommon.Utility { /// /// Represents a native shared library opened by the operating system. - /// This type can be used to load native function pointers by name. /// public class NativeLibrary : IDisposable { @@ -15,6 +17,22 @@ public class NativeLibrary : IDisposable public static bool NativeLibraryExists(string name) { +#if NET5_0_OR_GREATER + // Use the runtime's assembly-aware loader to honor native NuGet asset probing. + // The legacy loader below can falsely report missing libraries on Linux: its "libdl" + // import may not resolve to libdl.so.2, and its distro-specific RID lookup can miss + // portable linux-x64 assets. Passing the owning assembly avoids that custom probing. + var assembly = typeof(NativeLibrary).Assembly; + if (!RuntimeNativeLibrary.TryLoad(name, assembly, null, out var handle) && + !RuntimeNativeLibrary.TryLoad("lib" + name, assembly, null, out handle) && + !RuntimeNativeLibrary.TryLoad("lib_" + name, assembly, null, out handle)) + { + return false; + } + + RuntimeNativeLibrary.Free(handle); + return true; +#else NativeLibrary nativeLibrary = default; string extension = default; string prefix = "lib"; @@ -60,102 +78,26 @@ public static bool NativeLibraryExists(string name) { nativeLibrary?.Dispose(); } +#endif } /// /// The operating system handle of the loaded library. /// - public IntPtr Handle { get; } + private IntPtr Handle { get; } /// /// Constructs a new NativeLibrary using the platform's default library loader. /// /// The name of the library to load. - public NativeLibrary(string name) : this(name, _platformDefaultLoader, PathResolver.Default) - { - } - - /// - /// Constructs a new NativeLibrary using the platform's default library loader. - /// - /// An ordered list of names to attempt to load. - public NativeLibrary(string[] names) : this(names, _platformDefaultLoader, PathResolver.Default) - { - } - - /// - /// Constructs a new NativeLibrary using the specified library loader. - /// - /// The name of the library to load. - /// The loader used to open and close the library, and to load function pointers. - public NativeLibrary(string name, LibraryLoader loader) : this(name, loader, PathResolver.Default) - { - } - - /// - /// Constructs a new NativeLibrary using the specified library loader. - /// - /// An ordered list of names to attempt to load. - /// The loader used to open and close the library, and to load function pointers. - public NativeLibrary(string[] names, LibraryLoader loader) : this(names, loader, PathResolver.Default) - { - } - - /// - /// Constructs a new NativeLibrary using the specified library loader. - /// - /// The name of the library to load. - /// The loader used to open and close the library, and to load function pointers. - /// The path resolver, used to identify possible load targets for the library. - public NativeLibrary(string name, LibraryLoader loader, PathResolver pathResolver) - { - _loader = loader; - Handle = _loader.LoadNativeLibrary(name, pathResolver); - } - - /// - /// Constructs a new NativeLibrary using the specified library loader. - /// - /// An ordered list of names to attempt to load. - /// The loader used to open and close the library, and to load function pointers. - /// The path resolver, used to identify possible load targets for the library. - public NativeLibrary(string[] names, LibraryLoader loader, PathResolver pathResolver) - { - _loader = loader; - Handle = _loader.LoadNativeLibrary(names, pathResolver); - } - - /// - /// Loads a function whose signature matches the given delegate type's signature. - /// - /// The type of delegate to return. - /// The name of the native export. - /// A delegate wrapping the native function. - /// Thrown when no function with the given name - /// is exported from the native library. - public T LoadFunction(string name) - { - IntPtr functionPtr = _loader.LoadFunctionPointer(Handle, name); - if (functionPtr == IntPtr.Zero) - { - throw new InvalidOperationException($"No function was found with the name {name}."); - } - - return Marshal.GetDelegateForFunctionPointer(functionPtr); - } - - /// - /// Loads a function pointer with the given name. - /// - /// The name of the native export. - /// A function pointer for the given name, or 0 if no function with that name exists. - public IntPtr LoadFunction(string name) + private NativeLibrary(string name) { - return _loader.LoadFunctionPointer(Handle, name); + _loader = _platformDefaultLoader; + Handle = _loader.LoadNativeLibrary(name, PathResolver.Default); } /// - /// Frees the native library. Function pointers retrieved from this library will be void. + /// Frees the native library. /// public void Dispose() { diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs index d56ab680e8..805fa73e55 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs @@ -30,7 +30,7 @@ public abstract class PathResolver /// Enumerates possible library load targets. This default implementation returns the following load targets: /// First: The library contained in the applications base folder. /// Second: The simple name, unchanged. - /// Third: The library as resolved via the default DependencyContext, in the default nuget package cache folder. + /// Third: The library as resolved via the default DependencyContext, in the configured NuGet package cache folder. /// public class DefaultPathResolver : PathResolver { @@ -147,7 +147,10 @@ private bool AddFallbacks(List fallbacks, string rid, IReadOnlyList Date: Thu, 29 Oct 2026 19:10:49 +0100 Subject: [PATCH 2/4] Clarify native-dependent test skip messages Describe supported platforms and loadable native dependencies rather than reporting every native-loading failure as an architecture restriction. Leave the availability conditions unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0c982995-379f-4379-bcb7-dba5d1102bb4 --- .../Attributes/LightGBMFactAttribute.cs | 2 +- .../Attributes/LightGBMTheoryAttribute.cs | 2 +- .../Attributes/OnnxTheoryAttribute.cs | 2 +- .../Attributes/TensorflowFactAttribute.cs | 2 +- .../Attributes/TensorflowTheoryAttribute.cs | 2 +- .../Attributes/FieldAwareFactorizationMachineFactAttribute.cs | 2 +- .../Attributes/NativeDependencyFactAttribute.cs | 2 +- .../Attributes/NativeDependencyTheoryAttribute.cs | 2 +- .../Attributes/OnnxFactAttribute.cs | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs index 634b8a5251..89d0bec9fa 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs @@ -13,7 +13,7 @@ namespace Microsoft.ML.TestFramework.Attributes /// public sealed class LightGBMFactAttribute : EnvironmentSpecificFactAttribute { - public LightGBMFactAttribute() : base("LightGBM is 64-bit only") + public LightGBMFactAttribute() : base("LightGBM requires a 64-bit process and a loadable native library.") { } diff --git a/test/Microsoft.ML.TestFramework/Attributes/LightGBMTheoryAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/LightGBMTheoryAttribute.cs index 829f51eb69..de1335b1f1 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/LightGBMTheoryAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/LightGBMTheoryAttribute.cs @@ -13,7 +13,7 @@ namespace Microsoft.ML.TestFramework.Attributes /// public sealed class LightGBMTheoryAttribute : EnvironmentSpecificTheoryAttribute { - public LightGBMTheoryAttribute() : base("LightGBM is 64-bit only") + public LightGBMTheoryAttribute() : base("LightGBM requires a 64-bit process and a loadable native library.") { } diff --git a/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs index bd46e402b5..dcbb17c3cf 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs @@ -11,7 +11,7 @@ namespace Microsoft.ML.TestFramework.Attributes /// public sealed class OnnxTheoryAttribute : EnvironmentSpecificTheoryAttribute { - public OnnxTheoryAttribute() : base("Onnx is 64-bit only and is not supported on Linux with libc < v2.23") + public OnnxTheoryAttribute() : base("ONNX Runtime requires a supported platform and a loadable native library.") { } diff --git a/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs index 2455d61381..5dd4e8a45c 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs @@ -13,7 +13,7 @@ namespace Microsoft.ML.TestFramework.Attributes /// public sealed class TensorFlowFactAttribute : EnvironmentSpecificFactAttribute { - public TensorFlowFactAttribute() : base("TensorFlow is 64-bit only and is not supported on Linux with libc < v2.23") + public TensorFlowFactAttribute() : base("TensorFlow requires a 64-bit process, a supported platform, and a loadable native library.") { } diff --git a/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs index 9c6de5fd4c..89f9d310e4 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs @@ -13,7 +13,7 @@ namespace Microsoft.ML.TestFramework.Attributes /// public sealed class TensorFlowTheoryAttribute : EnvironmentSpecificTheoryAttribute { - public TensorFlowTheoryAttribute() : base("TensorFlow is 64-bit only and is not supported on Linux with libc < v2.23") + public TensorFlowTheoryAttribute() : base("TensorFlow requires a 64-bit process, a supported platform, and a loadable native library.") { } diff --git a/test/Microsoft.ML.TestFrameworkCommon/Attributes/FieldAwareFactorizationMachineFactAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/FieldAwareFactorizationMachineFactAttribute.cs index f9636f112d..2ecc3247bc 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Attributes/FieldAwareFactorizationMachineFactAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/FieldAwareFactorizationMachineFactAttribute.cs @@ -9,7 +9,7 @@ namespace Microsoft.ML.TestFramework.Attributes { public class FieldAwareFactorizationMachineFactAttribute : EnvironmentSpecificFactAttribute { - private const string SkipMessage = "FieldAwareFactorizationMachine doesn't currently support non x86/x64. https://github.com/dotnet/machinelearning/issues/5871"; + private const string SkipMessage = "FieldAwareFactorizationMachine requires a loadable CpuMathNative library."; public FieldAwareFactorizationMachineFactAttribute() : base(SkipMessage) { diff --git a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs index f40f8e1be2..acfdb6ec56 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs @@ -11,7 +11,7 @@ public sealed class NativeDependencyFactAttribute : EnvironmentSpecificFactAttri { private readonly string _library; - public NativeDependencyFactAttribute(string library) : base($"This test requires a native library {library} that wasn't found.") + public NativeDependencyFactAttribute(string library) : base($"This test requires a supported platform and a loadable native library {library}.") { _library = library; } diff --git a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs index 1f4e1e6b1a..97a46f2943 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs @@ -11,7 +11,7 @@ public sealed class NativeDependencyTheoryAttribute : EnvironmentSpecificTheoryA { private readonly string _library; - public NativeDependencyTheoryAttribute(string library) : base($"This test requires a native library {library} that wasn't found.") + public NativeDependencyTheoryAttribute(string library) : base($"This test requires a supported platform and a loadable native library {library}.") { _library = library; } diff --git a/test/Microsoft.ML.TestFrameworkCommon/Attributes/OnnxFactAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/OnnxFactAttribute.cs index 5fe6cf9ce4..168aef1e7a 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Attributes/OnnxFactAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/OnnxFactAttribute.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.TestFrameworkCommon.Attributes /// public sealed class OnnxFactAttribute : EnvironmentSpecificFactAttribute { - public OnnxFactAttribute() : base("Onnx is 64-bit only and is not supported on Linux with libc < v2.23") + public OnnxFactAttribute() : base("ONNX Runtime requires a supported platform and a loadable native library.") { } From e38ff6cadb96750ae8361f873dcda51fbbb9cd7f Mon Sep 17 00:00:00 2001 From: Petr Onderka Date: Mon, 2 Nov 2026 17:48:36 +0100 Subject: [PATCH 3/4] Quarantine SymSGD estimator tests on Linux ARM32 Skip the two affected estimator tests on Linux ARM32 for active issue #7837 while preserving native dependency checks elsewhere. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0c982995-379f-4379-bcb7-dba5d1102bb4 --- .../TrainerEstimators/SymSgdClassificationTests.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/SymSgdClassificationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/SymSgdClassificationTests.cs index b1597469d6..7670e4d0a7 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/SymSgdClassificationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/SymSgdClassificationTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System.Linq; +using System.Runtime.InteropServices; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework.Attributes; @@ -13,7 +14,14 @@ namespace Microsoft.ML.Tests.TrainerEstimators { public partial class TrainerEstimators { - [NativeDependencyFact("SymSgdNative")] + private static bool IsLinuxArm32 => + RuntimeInformation.IsOSPlatform(OSPlatform.Linux) && RuntimeInformation.ProcessArchitecture == Architecture.Arm; + + // Skip Linux ARM32 due to active issue https://github.com/dotnet/machinelearning/issues/7837. + private static bool CanRunSymSgdEstimatorTests => + !IsLinuxArm32 && new NativeDependencyFactAttribute("SymSgdNative").Skip == null; + + [ConditionalFact(nameof(CanRunSymSgdEstimatorTests))] public void TestEstimatorSymSgdClassificationTrainer() { (var pipe, var dataView) = GetBinaryClassificationPipeline(); @@ -27,7 +35,7 @@ public void TestEstimatorSymSgdClassificationTrainer() Done(); } - [NativeDependencyFact("SymSgdNative")] + [ConditionalFact(nameof(CanRunSymSgdEstimatorTests))] public void TestEstimatorSymSgdInitPredictor() { (var pipe, var dataView) = GetBinaryClassificationPipeline(); From 05a003032736b425427dedbf25d1c4ed758b69d7 Mon Sep 17 00:00:00 2001 From: Petr Onderka Date: Tue, 3 Nov 2026 18:35:15 +0100 Subject: [PATCH 4/4] Isolate oneDAL estimator test in a child process Select oneDAL before MLContext's static initialization without leaking backend selection to other tests. Use RemoteExecutor's package targets for Framework binding redirects and align Arcade's xUnit configuration for publishing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0c982995-379f-4379-bcb7-dba5d1102bb4 --- test/Directory.Build.props | 2 + .../Microsoft.ML.Tests.csproj | 1 + .../TrainerEstimators/OneDalEstimators.cs | 53 +++++++++++-------- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/test/Directory.Build.props b/test/Directory.Build.props index 10ff4aad95..1115738c99 100644 --- a/test/Directory.Build.props +++ b/test/Directory.Build.props @@ -24,6 +24,8 @@ trx $(OutputPath) + + $(MSBuildThisFileDirectory)Microsoft.ML.TestFrameworkCommon\xunit.runner.json true diff --git a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj index d9b79b93b5..41931d448b 100644 --- a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj +++ b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj @@ -47,6 +47,7 @@ + diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/OneDalEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/OneDalEstimators.cs index 50edbd48ef..ac9130a42c 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/OneDalEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/OneDalEstimators.cs @@ -3,24 +3,12 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.DotNet.PlatformAbstractions; -using Microsoft.Extensions.DependencyModel; -using Microsoft.ML.Calibrators; +using System.IO; +using Microsoft.DotNet.RemoteExecutor; using Microsoft.ML.Data; -using Microsoft.ML.Internal.Utilities; -using Microsoft.ML.Model; using Microsoft.ML.RunTests; -using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework.Attributes; -using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; -using Microsoft.ML.Trainers.LightGbm; -using Microsoft.ML.Transforms; using Xunit; namespace Microsoft.ML.Tests.TrainerEstimators @@ -28,15 +16,36 @@ namespace Microsoft.ML.Tests.TrainerEstimators public partial class TrainerEstimators : TestDataPipeBase { - // [Fact] [NativeDependencyFact("OneDalNative")] public void OneDalFastTreeBinaryEstimator() { - Environment.SetEnvironmentVariable("MLNET_BACKEND", "ONEDAL"); + var originalBackend = Environment.GetEnvironmentVariable("MLNET_BACKEND"); var trainDataPath = GetDataPath("binary_synth_data_train.csv"); var testDataPath = GetDataPath("binary_synth_data_test.csv"); - var loader = ML.Data.CreateTextLoader(columns: new[] { + // oneDAL has to be selected before static initialization of MLContext runs, which means using RemoteExecutor here. + var options = new RemoteInvokeOptions(); + options.StartInfo.EnvironmentVariables["MLNET_BACKEND"] = "ONEDAL"; + // Enable verbose output for oneDAL, so that the output check below can verify that oneDAL is being used. + options.StartInfo.EnvironmentVariables["MLNET_BACKEND_VERBOSE"] = "1"; + // oneDAL loads its threading runtime dynamically; test dependencies are beside the test assembly. + options.StartInfo.EnvironmentVariables["PATH"] = + Path.GetDirectoryName(typeof(TrainerEstimators).Assembly.Location) + Path.PathSeparator + Environment.GetEnvironmentVariable("PATH"); + options.StartInfo.RedirectStandardOutput = true; + var remote = RemoteExecutor.Invoke(TrainOneDalFastForest, trainDataPath, testDataPath, options); + var output = remote.Process.StandardOutput.ReadToEndAsync(); + remote.Dispose(); + + Assert.Contains("Decision Forest Classification parameters:", output.GetAwaiter().GetResult()); + Assert.Equal(originalBackend, Environment.GetEnvironmentVariable("MLNET_BACKEND")); + } + + private static void TrainOneDalFastForest(string trainDataPath, string testDataPath) + { + var ml = new MLContext(seed: 1); + Assert.True(MLContext.OneDalDispatchingEnabled); + + var loader = ml.Data.CreateTextLoader(columns: new[] { new TextLoader.Column("f0", DataKind.Single, 0), new TextLoader.Column("f1", DataKind.Single, 1), new TextLoader.Column("f2", DataKind.Single, 2), @@ -53,13 +62,11 @@ public void OneDalFastTreeBinaryEstimator() var trainingData = loader.Load(trainDataPath); var testingData = loader.Load(testDataPath); - var preprocessingPipeline = ML.Transforms.Concatenate("Features", new string[] { "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7" }); + var preprocessingPipeline = ml.Transforms.Concatenate("Features", new string[] { "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7" }); var preprocessedTrainingData = preprocessingPipeline.Fit(trainingData).Transform(trainingData); var preprocessedTestingData = preprocessingPipeline.Fit(trainingData).Transform(testingData); - // Output.WriteLine($"**** After preprocessing the data got {preprocessedTrainingData.Schema.Count} columns."); - FastForestBinaryTrainer.Options options = new FastForestBinaryTrainer.Options(); options.LabelColumnName = "target"; options.FeatureColumnName = "Features"; @@ -68,12 +75,12 @@ public void OneDalFastTreeBinaryEstimator() options.MinimumExampleCountPerLeaf = 5; options.FeatureFraction = 1.0; - var trainer = ML.BinaryClassification.Trainers.FastForest(options); + var trainer = ml.BinaryClassification.Trainers.FastForest(options); var model = trainer.Fit(preprocessedTrainingData); var trainingPredictions = model.Transform(preprocessedTrainingData); - var trainingMetrics = ML.BinaryClassification.EvaluateNonCalibrated(trainingPredictions, labelColumnName: "target"); + var trainingMetrics = ml.BinaryClassification.EvaluateNonCalibrated(trainingPredictions, labelColumnName: "target"); var testingPredictions = model.Transform(preprocessedTestingData); - var testingMetrics = ML.BinaryClassification.EvaluateNonCalibrated(testingPredictions, labelColumnName: "target"); + var testingMetrics = ml.BinaryClassification.EvaluateNonCalibrated(testingPredictions, labelColumnName: "target"); Assert.True(testingMetrics.Accuracy > 0.8); }