diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs index dfe6b3631f709e..a5805bacead6f8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.Wasm; using System.Runtime.Intrinsics.X86; namespace System.Buffers @@ -33,11 +34,13 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAny(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAnyExcept(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs index e6aa0c870486d8..92f516faf20505 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.Wasm; using System.Runtime.Intrinsics.X86; namespace System.Buffers @@ -23,11 +24,13 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAny(ReadOnlySpan span) => PackedSpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAnyExcept(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs index 7b789febead068..7a3602dd1cbd99 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs @@ -40,11 +40,13 @@ internal override bool ContainsCore(char value) [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAny(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAnyExcept(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs index d951b8e2375e0f..3088a1fc9604ba 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.Wasm; using System.Runtime.Intrinsics.X86; namespace System.Buffers @@ -23,11 +24,13 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAny(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAnyExcept(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs index cebb695aa3fb79..9a5f24bfc880b2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.Wasm; using System.Runtime.Intrinsics.X86; namespace System.Buffers @@ -23,11 +24,13 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAny(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal override int IndexOfAnyExcept(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesPackedThreeChars.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesPackedThreeChars.cs index 5c1d64bf9b0dfb..8425ce899fdc45 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesPackedThreeChars.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesPackedThreeChars.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; using System.Runtime.Intrinsics.X86; using static System.Buffers.StringSearchValuesHelper; @@ -36,20 +37,20 @@ internal sealed class SingleStringSearchValuesPackedThreeChars - Sse2.IsSupported && + (Sse2.IsSupported || PackedSimd.IsSupported) && typeof(TValueLength) == typeof(ValueLengthLessThan4) && (typeof(TCaseSensitivity) == typeof(CaseSensitive) || typeof(TCaseSensitivity) == typeof(CaseInsensitiveAsciiLetters)); } public SingleStringSearchValuesPackedThreeChars(HashSet? uniqueValues, string value, int ch2Offset, int ch3Offset) : base(uniqueValues) { - Debug.Assert(Sse2.IsSupported || AdvSimd.Arm64.IsSupported); + Debug.Assert(Sse2.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported); // We could have more than one entry in 'uniqueValues' if this value is an exact prefix of all the others. Debug.Assert(value.Length > 1); @@ -182,7 +183,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) goto LoopFooter; } } - else if ((Sse2.IsSupported || AdvSimd.Arm64.IsSupported) && searchSpaceMinusValueTailLength - Vector128.Count >= 0) + else if ((Sse2.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported) && searchSpaceMinusValueTailLength - Vector128.Count >= 0) { Vector128 ch1 = Vector128.Create(_ch1); Vector128 ch2 = Vector128.Create(_ch2); @@ -250,6 +251,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + [CompExactlyDependsOn(typeof(PackedSimd))] private static Vector128 GetComparisonResult(ref char searchSpace, nuint ch2ByteOffset, nuint ch3ByteOffset, Vector128 ch1, Vector128 ch2, Vector128 ch3) { // Load 3 vectors from the input. @@ -390,14 +392,30 @@ internal override string[] GetValues() => HasUniqueValues [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + [CompExactlyDependsOn(typeof(PackedSimd))] private static Vector128 LoadPacked128(ref char searchSpace, nuint byteOffset) { Vector128 input0 = Vector128.LoadUnsafe(ref Unsafe.AddByteOffset(ref searchSpace, byteOffset)); Vector128 input1 = Vector128.LoadUnsafe(ref Unsafe.AddByteOffset(ref searchSpace, byteOffset + (uint)Vector128.Count)); - return Sse2.IsSupported - ? Sse2.PackUnsignedSaturate(input0.AsInt16(), input1.AsInt16()) - : AdvSimd.Arm64.UnzipEven(input0.AsByte(), input1.AsByte()); + if (Sse2.IsSupported) + { + return Sse2.PackUnsignedSaturate(input0.AsInt16(), input1.AsInt16()); + } + else if (AdvSimd.Arm64.IsSupported) + { + return AdvSimd.Arm64.UnzipEven(input0.AsByte(), input1.AsByte()); + } + else if (PackedSimd.IsSupported) + { + return PackedSimd.ConvertNarrowingSaturateUnsigned(input0.AsInt16(), input1.AsInt16()); + } + else + { + // We explicitly recheck each IsSupported query to ensure that the trimmer can see which paths are live/dead + ThrowHelper.ThrowUnreachableException(); + return default; + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index dcfb8e66275b83..e0a8b21d52085c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -444,8 +444,8 @@ private static SearchValues CreateSingleValuesThreeChars(uniqueValues, value, ch2Offset, ch3Offset); - // Unlike with PackedSpanHelpers (Sse2 only), we are also using this approach on ARM64. - // We use PackUnsignedSaturate on X86 and UnzipEven on ARM, so the set of allowed characters differs slightly (we can't use it for \0 and \xFF on X86). + // Unlike with PackedSpanHelpers (Sse2 and PackedSimd only), we are also using this approach on ARM64. + // We use saturating packing on X86 and WASM and UnzipEven on ARM, so the set of allowed characters differs slightly (we can't use it for \0 and \xFF on X86 and WASM). static bool CanUsePackedImpl(char c) => PackedSpanHelpers.PackedIndexOfIsSupported ? PackedSpanHelpers.CanUsePackedIndexOf(c) : (AdvSimd.Arm64.IsSupported && c <= byte.MaxValue); diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index be05f8dca2bd00..bef79d5fca3a5a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -5,6 +5,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Wasm; using System.Runtime.Intrinsics.X86; namespace System @@ -13,9 +14,9 @@ namespace System // included in this file which are specific to the packed implementation. internal static partial class PackedSpanHelpers { - // We only do this optimization if we have support for X86 intrinsics (Sse2) as the packing is noticeably cheaper compared to ARM (AdvSimd). + // We only do this optimization if we have support for X86 intrinsics (Sse2) or WASM SIMD (PackedSimd) as the packing is noticeably cheaper compared to ARM (AdvSimd). // While the impact on the worst-case (match at the start) is minimal on X86, it's prohibitively large on ARM. - public static bool PackedIndexOfIsSupported => Sse2.IsSupported; + public static bool PackedIndexOfIsSupported => Sse2.IsSupported || PackedSimd.IsSupported; // Not all values can benefit from packing the searchSpace. See comments in PackSources below. [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -30,36 +31,43 @@ public static unsafe bool CanUsePackedIndexOf(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOf(ref char searchSpace, char value, int length) => IndexOf, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAnyExcept(ref char searchSpace, char value, int length) => IndexOf, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAny(ref char searchSpace, char value0, char value1, int length) => IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAny(ref char searchSpace, char value0, char value1, char value2, int length) => IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, char value2, int length) => IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value, int length) { Debug.Assert((value | 0x20) == value); @@ -69,6 +77,7 @@ public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value, int len [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value, int length) { Debug.Assert((value | 0x20) == value); @@ -78,6 +87,7 @@ public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value, i [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value0, char value1, int length) { Debug.Assert((value0 | 0x20) == value0); @@ -88,6 +98,7 @@ public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value0, char v [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value0, char value1, int length) { Debug.Assert((value0 | 0x20) == value0); @@ -98,15 +109,18 @@ public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value0, [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => IndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static int IndexOfAnyExceptInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => IndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] public static bool Contains(ref short searchSpace, short value, int length) { Debug.Assert(CanUsePackedIndexOf(value)); @@ -309,6 +323,7 @@ public static bool Contains(ref short searchSpace, short value, int length) } [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] private static int IndexOf(ref short searchSpace, short value, int length) where TNegator : struct, SpanHelpers.INegator where TTransform : struct, ITransform @@ -512,6 +527,7 @@ private static int IndexOf(ref short searchSpace, short va } [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] private static int IndexOfAny(ref short searchSpace, short value0, short value1, int length) where TNegator : struct, SpanHelpers.INegator where TTransform : struct, ITransform @@ -726,6 +742,7 @@ private static int IndexOfAny(ref short searchSpace, short } [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] private static int IndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) where TNegator : struct, SpanHelpers.INegator { @@ -944,6 +961,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho } [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] private static int IndexOfAnyInRange(ref short searchSpace, short lowInclusive, short rangeInclusive, int length) where TNegator : struct, SpanHelpers.INegator { @@ -1164,14 +1182,27 @@ internal static Vector256 PackSources(Vector256 source0, Vector256< [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(PackedSimd))] internal static Vector128 PackSources(Vector128 source0, Vector128 source1) { - Debug.Assert(Sse2.IsSupported); // Pack two vectors of characters into bytes. While the type is Vector128, these are really UInt16 characters. - // X86: Downcast every character using saturation. + // X86 and WASM: Downcast every character using saturation. // - Values <= 32767 result in min(value, 255). // - Values > 32767 result in 0. Because of this we can't accept needles that contain 0. - return Sse2.PackUnsignedSaturate(source0, source1).AsByte(); + if (Sse2.IsSupported) + { + return Sse2.PackUnsignedSaturate(source0, source1).AsByte(); + } + else if (PackedSimd.IsSupported) + { + return PackedSimd.ConvertNarrowingSaturateUnsigned(source0, source1); + } + else + { + // We explicitly recheck each IsSupported query to ensure that the trimmer can see which paths are live/dead + ThrowHelper.ThrowUnreachableException(); + return default; + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index f3a2c16a910579..2638ad3a2da291 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2322,7 +2322,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r Vector128 v2 = Vector128.Create((ushort)c2); Vector128 v3 = Vector128.Create((ushort)c3); - if (Sse2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) + if (PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) { // Process in double chunks & check if either chunk is likely to contain matches at once. // If we get multiple matches in a single chunk, we assume they're likely to be close &