Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics.Wasm;
using System.Runtime.Intrinsics.X86;

namespace System.Buffers
Expand Down Expand Up @@ -33,11 +34,13 @@ internal override bool ContainsCore(char value) =>

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAny(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics.Wasm;
using System.Runtime.Intrinsics.X86;

namespace System.Buffers
Expand All @@ -23,11 +24,13 @@ internal override bool ContainsCore(char value) =>

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAny(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,13 @@ internal override bool ContainsCore(char value)

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAny(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics.Wasm;
using System.Runtime.Intrinsics.X86;

namespace System.Buffers
Expand All @@ -23,11 +24,13 @@ internal override bool ContainsCore(char value) =>

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAny(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics.Wasm;
using System.Runtime.Intrinsics.X86;

namespace System.Buffers
Expand All @@ -23,11 +24,13 @@ internal override bool ContainsCore(char value) =>

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAny(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(PackedSimd))]
internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.Wasm;
using System.Runtime.Intrinsics.X86;
using static System.Buffers.StringSearchValuesHelper;

Expand Down Expand Up @@ -36,20 +37,20 @@ internal sealed class SingleStringSearchValuesPackedThreeChars<TValueLength, TCa
// With case-sensitive comparisons, we've therefore already confirmed the match, so we can skip doing so here.
// With case-insensitive comparisons, we applied a mask to the input, so while the anchors likely matched, we can't be sure.
// If the value is composed of only ASCII letters, masking the input can't produce false positives, so we can also skip the verification step.
// We only do this when running on X86 and not ARM64, as the latter uses UnzipEven when packing inputs, which may produce false positive anchor matches.
// We only do this when running on X86 or WASM and not ARM64, as the latter uses UnzipEven when packing inputs, which may produce false positive anchor matches.
// We use that instead of ExtractNarrowingSaturate because it allows for higher searching throughput.
private static bool CanSkipAnchorMatchVerification
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get =>
Sse2.IsSupported &&
(Sse2.IsSupported || PackedSimd.IsSupported) &&
typeof(TValueLength) == typeof(ValueLengthLessThan4) &&
(typeof(TCaseSensitivity) == typeof(CaseSensitive) || typeof(TCaseSensitivity) == typeof(CaseInsensitiveAsciiLetters));
}

public SingleStringSearchValuesPackedThreeChars(HashSet<string>? uniqueValues, string value, int ch2Offset, int ch3Offset) : base(uniqueValues)
{
Debug.Assert(Sse2.IsSupported || AdvSimd.Arm64.IsSupported);
Debug.Assert(Sse2.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported);

// We could have more than one entry in 'uniqueValues' if this value is an exact prefix of all the others.
Debug.Assert(value.Length > 1);
Expand Down Expand Up @@ -182,7 +183,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength)
goto LoopFooter;
}
}
else if ((Sse2.IsSupported || AdvSimd.Arm64.IsSupported) && searchSpaceMinusValueTailLength - Vector128<byte>.Count >= 0)
else if ((Sse2.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported) && searchSpaceMinusValueTailLength - Vector128<byte>.Count >= 0)
{
Vector128<byte> ch1 = Vector128.Create(_ch1);
Vector128<byte> ch2 = Vector128.Create(_ch2);
Expand Down Expand Up @@ -250,6 +251,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(AdvSimd.Arm64))]
[CompExactlyDependsOn(typeof(PackedSimd))]
private static Vector128<byte> GetComparisonResult(ref char searchSpace, nuint ch2ByteOffset, nuint ch3ByteOffset, Vector128<byte> ch1, Vector128<byte> ch2, Vector128<byte> ch3)
{
// Load 3 vectors from the input.
Expand Down Expand Up @@ -390,14 +392,30 @@ internal override string[] GetValues() => HasUniqueValues
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Sse2))]
[CompExactlyDependsOn(typeof(AdvSimd.Arm64))]
[CompExactlyDependsOn(typeof(PackedSimd))]
private static Vector128<byte> LoadPacked128(ref char searchSpace, nuint byteOffset)
{
Vector128<ushort> input0 = Vector128.LoadUnsafe(ref Unsafe.AddByteOffset(ref searchSpace, byteOffset));
Vector128<ushort> input1 = Vector128.LoadUnsafe(ref Unsafe.AddByteOffset(ref searchSpace, byteOffset + (uint)Vector128<byte>.Count));

return Sse2.IsSupported
? Sse2.PackUnsignedSaturate(input0.AsInt16(), input1.AsInt16())
: AdvSimd.Arm64.UnzipEven(input0.AsByte(), input1.AsByte());
if (Sse2.IsSupported)
{
return Sse2.PackUnsignedSaturate(input0.AsInt16(), input1.AsInt16());
}
else if (AdvSimd.Arm64.IsSupported)
{
return AdvSimd.Arm64.UnzipEven(input0.AsByte(), input1.AsByte());
}
else if (PackedSimd.IsSupported)
{
return PackedSimd.ConvertNarrowingSaturateUnsigned(input0.AsInt16(), input1.AsInt16());
}
else
{
// We explicitly recheck each IsSupported query to ensure that the trimmer can see which paths are live/dead
ThrowHelper.ThrowUnreachableException();
return default;
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -444,8 +444,8 @@ private static SearchValues<string> CreateSingleValuesThreeChars<TValueLength, T

return new SingleStringSearchValuesThreeChars<TValueLength, TCaseSensitivity>(uniqueValues, value, ch2Offset, ch3Offset);

// Unlike with PackedSpanHelpers (Sse2 only), we are also using this approach on ARM64.
// We use PackUnsignedSaturate on X86 and UnzipEven on ARM, so the set of allowed characters differs slightly (we can't use it for \0 and \xFF on X86).
// Unlike with PackedSpanHelpers (Sse2 and PackedSimd only), we are also using this approach on ARM64.
// We use saturating packing on X86 and WASM and UnzipEven on ARM, so the set of allowed characters differs slightly (we can't use it for \0 and \xFF on X86 and WASM).
static bool CanUsePackedImpl(char c) =>
PackedSpanHelpers.PackedIndexOfIsSupported ? PackedSpanHelpers.CanUsePackedIndexOf(c) :
(AdvSimd.Arm64.IsSupported && c <= byte.MaxValue);
Expand Down
Loading
Loading