Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions Directory.Build.targets
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,20 @@
<NativeAssemblyReference Remove="MklProxyNative"/>
<NativeAssemblyReference Remove="libiomp5md"/>
</ItemGroup>

<MSBuild Condition="'@(NativeAssemblyReference->WithMetadataValue('Identity', 'OneDalNative'))' != ''"
Projects="$(RepoRoot)src\Native\Native.proj"
Targets="GetOneDalNativeDependencies"
Properties="Configuration=$(Configuration);TargetArchitecture=$(TargetArchitecture)"
RemoveProperties="TargetFramework;RuntimeIdentifier">
<Output TaskParameter="TargetOutputs" ItemName="_OneDalNativeDependency" />
</MSBuild>

<ItemGroup>
<NativeAssemblyReference Include="@(_OneDalNativeDependency)">
<FullAssemblyPath>$(NativeOutputPath)%(_OneDalNativeDependency.Identity)</FullAssemblyPath>
</NativeAssemblyReference>
</ItemGroup>
</Target>

<Target Name="CopyNativeAssembliesPublish" AfterTargets="Publish" DependsOnTargets="SetCopyProperties">
Expand Down
60 changes: 12 additions & 48 deletions src/Native/Native.proj
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,15 @@
<TbbMallocLibrary Condition="$([MSBuild]::IsOSPlatform('osx'))">libtbbmalloc.2.dylib</TbbMallocLibrary>
</PropertyGroup>

<ItemGroup Condition="'$(TargetArchitecture)' == 'x64'">
<_OneDalNativeDependency Include="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_core$(OneDalLibExtension)" />
<_OneDalNativeDependency Include="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_thread$(OneDalLibExtension)" />
<_OneDalNativeDependency Include="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbLibrary)" />
<_OneDalNativeDependency Include="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbMallocLibrary)" />
</ItemGroup>

<Target Name="GetOneDalNativeDependencies" Returns="@(_OneDalNativeDependency->'%(Filename)%(Extension)')" />

<Target Name="BuildNativeUnix"
Condition="'$(OS)' != 'Windows_NT'"
DependsOnTargets="GenerateNativeVersionFile">
Expand Down Expand Up @@ -151,35 +160,7 @@
<Copy Condition="'$(NonArmOnWindows)' == 'true'" SourceFiles="$(NuGetPackageRoot)mlnetmkldeps\$(MlNetMklDepsVersion)\runtimes\$(PackageRid)\native\libiomp5md$(NativeLibExtension)"
DestinationFolder="$(NativeAssetsBuiltPath)" />

<!-- Copy oneDAL -->
<Copy Condition="'$(TargetArchitecture)' == 'x64'"
SourceFiles="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_core$(OneDalLibExtension)"
DestinationFolder="$(NativeAssetsBuiltPath)" />
<Copy Condition="'$(TargetArchitecture)' == 'x64'"
SourceFiles="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_core$(OneDalLibExtension)"
DestinationFolder="$(PackageAssetsPath)\Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />

<Copy Condition="'$(TargetArchitecture)' == 'x64'"
SourceFiles="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_thread$(OneDalLibExtension)"
DestinationFolder="$(NativeAssetsBuiltPath)" />
<Copy Condition="'$(TargetArchitecture)' == 'x64'"
SourceFiles="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_thread$(OneDalLibExtension)"
DestinationFolder="$(PackageAssetsPath)\Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />

<!-- Copy oneTBB (dependency of oneDAL) -->
<Copy Condition="'$(TargetArchitecture)' == 'x64'"
SourceFiles="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbLibrary)"
DestinationFolder="$(NativeAssetsBuiltPath)" />
<Copy Condition="'$(TargetArchitecture)' == 'x64'"
SourceFiles="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbLibrary)"
DestinationFolder="$(PackageAssetsPath)\Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />

<Copy Condition="'$(TargetArchitecture)' == 'x64'"
SourceFiles="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbMallocLibrary)"
DestinationFolder="$(NativeAssetsBuiltPath)" />
<Copy Condition="'$(TargetArchitecture)' == 'x64'"
SourceFiles="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbMallocLibrary)"
DestinationFolder="$(PackageAssetsPath)\Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />
<Copy SourceFiles="@(_OneDalNativeDependency)" DestinationFolder="$(NativeAssetsBuiltPath)" />

<ItemGroup>
<NativePackageAsset Condition="'$(TargetArchitecture)' != 'arm64' And '$(TargetArchitecture)' != 'arm'"
Expand All @@ -201,25 +182,6 @@
Include="$(NativeAssetsBuiltPath)\$(NativeLibPrefix)FastTreeNative$(NativeLibExtension)"
RelativePath="Microsoft.ML.FastTree\runtimes\$(PackageRid)\native" />

<!-- Copy oneDAL -->
<!--
<NativePackageAsset Condition="'$(TargetArchitecture)' == 'x64'"
Include="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_core$(OneDalLibExtension)"
RelativePath="Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />
<NativePackageAsset Condition="'$(TargetArchitecture)' == 'x64'"
Include="$(NuGetPackageRoot)inteldal.redist.$(PackageRid)\$(OneDalPkgVersion)\build\native\daal\latest\$(OneDalLibDir)\$(NativeLibPrefix)onedal_thread$(OneDalLibExtension)"
RelativePath="Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />
-->
<!-- Copy oneTBB (dependency of oneDAL) -->
<!--
<NativePackageAsset Condition="'$(TargetArchitecture)' == 'x64'"
Include="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbLibrary)"
RelativePath="Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />
<NativePackageAsset Condition="'$(TargetArchitecture)' == 'x64'"
Include="$(NuGetPackageRoot)inteltbb.$(TbbPkgType).$(TbbSystem)\$(OneTbbPkgVersion)\runtimes\$(PackageRid)\native\$(TbbMallocLibrary)"
RelativePath="Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />
-->

</ItemGroup>

<ItemGroup>
Expand All @@ -237,6 +199,8 @@
RelativePath="Microsoft.ML.Mkl.Redist\runtimes\$(PackageRid)\native" />
<NativePackageAsset Condition="'$(NonArmOnWindows)' == 'true'" Include="$(NativeAssetsBuiltPath)\libiomp5md$(NativeLibExtension)"
RelativePath="Microsoft.ML.Mkl.Redist\runtimes\$(PackageRid)\native" />
<NativePackageAsset Include="@(_OneDalNativeDependency)"
RelativePath="Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />
<NativePackageAsset Condition="'$(TargetArchitecture)' == 'x64'"
Include="$(NativeAssetsBuiltPath)\$(NativeLibPrefix)OneDalNative$(NativeLibExtension)"
RelativePath="Microsoft.ML.OneDal\runtimes\$(PackageRid)\native" />
Expand Down
2 changes: 2 additions & 0 deletions test/Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
<PropertyGroup>
<VSTestLogger>trx</VSTestLogger>
<VSTestResultsDirectory>$(OutputPath)</VSTestResultsDirectory>
<!-- Cleanly override the arcade default xunit.runner.json. -->
<XUnitDesktopSettingsFile>$(MSBuildThisFileDirectory)Microsoft.ML.TestFrameworkCommon\xunit.runner.json</XUnitDesktopSettingsFile>
<CopyLocalLockFileAssemblies Condition="'$(Coverage)' != 'true'">true</CopyLocalLockFileAssemblies>
</PropertyGroup>

Expand Down
53 changes: 53 additions & 0 deletions test/Microsoft.ML.Core.Tests/UnitTests/NativeLibraryTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Runtime.InteropServices;
using Microsoft.ML.TestFramework;
using Microsoft.ML.TestFramework.Attributes;
using Xunit;
using Xunit.Abstractions;
using NativeLibrary = Microsoft.ML.TestFrameworkCommon.Utility.NativeLibrary;

namespace Microsoft.ML.RunTests
{
public sealed class NativeLibraryTests : BaseTestClass
{
private static bool IsOneDalSupported =>
RuntimeInformation.ProcessArchitecture == Architecture.X64 &&
// We don't test macOS x64 for OneDAL.
!RuntimeInformation.IsOSPlatform(OSPlatform.OSX);

public NativeLibraryTests(ITestOutputHelper output) : base(output)
{
}

[Fact]
public void SystemNativeLibraryCanBeLoaded()
{
string library = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? "kernel32"
: RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? "libc.so.6" : "libSystem.B.dylib";

Assert.True(NativeLibrary.NativeLibraryExists(library));
}

[Fact]
public void MissingNativeLibraryReturnsFalse()
{
Assert.False(NativeLibrary.NativeLibraryExists("MLNetNonexistentNativeLibrary"));
}

[X64Fact("The LightGBM native package supports x64 only.")]
public void PackageNativeLibraryCanBeLoaded()
{
Assert.True(NativeLibrary.NativeLibraryExists("lib_lightgbm"));
}

[ConditionalFact(nameof(IsOneDalSupported))]
public void OneDalNativeLibraryCanBeLoaded()
{
Assert.True(NativeLibrary.NativeLibraryExists("OneDalNative"));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace Microsoft.ML.TestFramework.Attributes
/// </summary>
public sealed class LightGBMFactAttribute : EnvironmentSpecificFactAttribute
{
public LightGBMFactAttribute() : base("LightGBM is 64-bit only")
public LightGBMFactAttribute() : base("LightGBM requires a 64-bit process and a loadable native library.")
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace Microsoft.ML.TestFramework.Attributes
/// </summary>
public sealed class LightGBMTheoryAttribute : EnvironmentSpecificTheoryAttribute
{
public LightGBMTheoryAttribute() : base("LightGBM is 64-bit only")
public LightGBMTheoryAttribute() : base("LightGBM requires a 64-bit process and a loadable native library.")
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ namespace Microsoft.ML.TestFramework.Attributes
/// </summary>
public sealed class OnnxTheoryAttribute : EnvironmentSpecificTheoryAttribute
{
public OnnxTheoryAttribute() : base("Onnx is 64-bit only and is not supported on Linux with libc < v2.23")
public OnnxTheoryAttribute() : base("ONNX Runtime requires a supported platform and a loadable native library.")
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace Microsoft.ML.TestFramework.Attributes
/// </summary>
public sealed class TensorFlowFactAttribute : EnvironmentSpecificFactAttribute
{
public TensorFlowFactAttribute() : base("TensorFlow is 64-bit only and is not supported on Linux with libc < v2.23")
public TensorFlowFactAttribute() : base("TensorFlow requires a 64-bit process, a supported platform, and a loadable native library.")
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace Microsoft.ML.TestFramework.Attributes
/// </summary>
public sealed class TensorFlowTheoryAttribute : EnvironmentSpecificTheoryAttribute
{
public TensorFlowTheoryAttribute() : base("TensorFlow is 64-bit only and is not supported on Linux with libc < v2.23")
public TensorFlowTheoryAttribute() : base("TensorFlow requires a 64-bit process, a supported platform, and a loadable native library.")
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace Microsoft.ML.TestFramework.Attributes
{
public class FieldAwareFactorizationMachineFactAttribute : EnvironmentSpecificFactAttribute
{
private const string SkipMessage = "FieldAwareFactorizationMachine doesn't currently support non x86/x64. https://github.com/dotnet/machinelearning/issues/5871";
private const string SkipMessage = "FieldAwareFactorizationMachine requires a loadable CpuMathNative library.";

public FieldAwareFactorizationMachineFactAttribute() : base(SkipMessage)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ public sealed class NativeDependencyFactAttribute : EnvironmentSpecificFactAttri
{
private readonly string _library;

public NativeDependencyFactAttribute(string library) : base($"This test requires a native library {library} that wasn't found.")
public NativeDependencyFactAttribute(string library) : base($"This test requires a supported platform and a loadable native library {library}.")
{
_library = library;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ public sealed class NativeDependencyTheoryAttribute : EnvironmentSpecificTheoryA
{
private readonly string _library;

public NativeDependencyTheoryAttribute(string library) : base($"This test requires a native library {library} that wasn't found.")
public NativeDependencyTheoryAttribute(string library) : base($"This test requires a supported platform and a loadable native library {library}.")
{
_library = library;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace Microsoft.ML.TestFrameworkCommon.Attributes
/// </summary>
public sealed class OnnxFactAttribute : EnvironmentSpecificFactAttribute
{
public OnnxFactAttribute() : base("Onnx is 64-bit only and is not supported on Linux with libc < v2.23")
public OnnxFactAttribute() : base("ONNX Runtime requires a supported platform and a loadable native library.")
{
}

Expand Down
5 changes: 5 additions & 0 deletions test/Microsoft.ML.TestFrameworkCommon/Utility/Kernel32.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,14 @@ namespace Microsoft.ML.TestFrameworkCommon.Utility
{
internal static class Kernel32
{
public const uint LoadWithAlteredSearchPath = 0x00000008;

[DllImport("kernel32")]
public static extern IntPtr LoadLibrary(string fileName);

[DllImport("kernel32", CharSet = CharSet.Unicode)]
public static extern IntPtr LoadLibraryEx(string fileName, IntPtr file, uint flags);

[DllImport("kernel32")]
public static extern IntPtr GetProcAddress(IntPtr module, string procName);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,10 @@ protected override IntPtr CoreLoadFunctionPointer(IntPtr handle, string function

protected override IntPtr CoreLoadNativeLibrary(string name)
{
// Resolve dependencies beside the DLL, not just beside the test host.
if (Path.IsPathRooted(name))
return Kernel32.LoadLibraryEx(Path.GetFullPath(name), IntPtr.Zero, Kernel32.LoadWithAlteredSearchPath);

return Kernel32.LoadLibrary(name);
}
}
Expand Down
Loading
Loading