From f742387533c7408931b4f7e427e228cd3ba7b276 Mon Sep 17 00:00:00 2001 From: Ennio Visconti Date: Sat, 11 Mar 2017 12:10:53 +0100 Subject: [PATCH 1/5] Refactoring to reduce technical debt --- WowPacketParser/Loading/SniffFile.cs | 488 ++++++++++++---------- WowPacketParser/Misc/IOutputBuilder.cs | 9 + WowPacketParser/Misc/Packet.cs | 37 +- WowPacketParser/Misc/Settings.cs | 2 + WowPacketParser/Misc/TextOutputBuilder.cs | 28 ++ WowPacketParser/Misc/TextOutputFormat.cs | 14 + WowPacketParser/WowPacketParser.csproj | 3 + 7 files changed, 336 insertions(+), 245 deletions(-) create mode 100644 WowPacketParser/Misc/IOutputBuilder.cs create mode 100644 WowPacketParser/Misc/TextOutputBuilder.cs create mode 100644 WowPacketParser/Misc/TextOutputFormat.cs diff --git a/WowPacketParser/Loading/SniffFile.cs b/WowPacketParser/Loading/SniffFile.cs index 5f6ae5f474..638ff298c4 100644 --- a/WowPacketParser/Loading/SniffFile.cs +++ b/WowPacketParser/Loading/SniffFile.cs @@ -28,6 +28,7 @@ public class SniffFile private readonly Statistics _stats; private readonly DumpFormatType _dumpFormat; private readonly string _logPrefix; + private static long _lastPercent; private readonly List _withErrorHeaders = new List(); private readonly List _skippedHeaders = new List(); @@ -113,22 +114,7 @@ private void ProcessFileImpl() if (packets.Count == 0) break; - var firstPacket = packets.First(); - var lastPacket = packets.Last(); - - // CSV format - // ReSharper disable once UseStringInterpolation - Trace.WriteLine(string.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}", - FileName, // - sniff file name - firstPacket.Time, // - time of first packet - lastPacket.Time, // - time of last packet - (lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds) - packets.Count, // - packet count - packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes) - packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes) - packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes) - packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes) - + statsAnalysis(packets); break; } case DumpFormatType.SniffDataOnly: @@ -146,131 +132,7 @@ private void ProcessFileImpl() break; } - Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; - File.Delete(outFileName); - - _stats.SetStartTime(DateTime.Now); - - var threadCount = Settings.Threads; - if (threadCount == 0) - threadCount = Environment.ProcessorCount; - - ThreadPool.SetMinThreads(threadCount + 2, 4); - - var written = false; - using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) - { - var firstRead = true; - var firstWrite = true; - - var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); - - var pwp = new ParallelWorkProcessor(() => // read - { - if (!reader.PacketReader.CanRead()) - return Tuple.Create(null, true); - - Packet packet; - var b = reader.TryRead(out packet); - - if (firstRead) - { - Trace.WriteLine( - $"{_logPrefix}: Parsing {Utilities.BytesToString(reader.PacketReader.GetTotalSize())} of packets. Detected version {ClientVersion.VersionString}"); - - firstRead = false; - } - - return Tuple.Create(packet, b); - }, packet => // parse - { - // Parse the packet, adding text to Writer and stuff to the stores - if (packet.Direction == Direction.BNClientToServer || - packet.Direction == Direction.BNServerToClient) - BattlenetHandler.ParseBattlenet(packet); - else - Handler.Parse(packet); - - // Update statistics - _stats.AddByStatus(packet.Status); - return packet; - }, - packet => // write - { - ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); - - if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile) - { - packet.ClosePacket(); - return; - } - - written = true; - - if (firstWrite) - { - // ReSharper disable AccessToDisposedClosure - writer?.WriteLine(GetHeader(FileName)); - // ReSharper restore AccessToDisposedClosure - - firstWrite = false; - } - - // get packet header if necessary - if (Settings.LogPacketErrors) - { - switch (packet.Status) - { - case ParsedStatus.WithErrors: - _withErrorHeaders.Add(packet.GetHeader()); - break; - case ParsedStatus.NotParsed: - _skippedHeaders.Add(packet.GetHeader()); - break; - case ParsedStatus.NoStructure: - _noStructureHeaders.Add(packet.GetHeader()); - break; - } - } - -// ReSharper disable AccessToDisposedClosure - if (writer != null) - { - // Write to file - writer.WriteLine(packet.Writer); - writer.Flush(); - } -// ReSharper restore AccessToDisposedClosure - - // Close Writer, Stream - Dispose - packet.ClosePacket(); - }, threadCount); - - pwp.WaitForFinished(Timeout.Infinite); - - reader.PacketReader.Dispose(); - - _stats.SetEndTime(DateTime.Now); - } - - if (written) - Trace.WriteLine($"{_logPrefix}: Saved file to '{outFileName}'"); - else - { - Trace.WriteLine($"{_logPrefix}: No file produced"); - File.Delete(outFileName); - } - - Trace.WriteLine($"{_logPrefix}: {_stats}"); - - if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog()) - WriteSQLs(); - - if (Settings.LogPacketErrors) - WritePacketErrors(); - - GC.Collect(); // Force a GC collect after parsing a file. It seems to help. - + launchProcessing(outFileName); break; } case DumpFormatType.Pkt: @@ -279,29 +141,7 @@ private void ProcessFileImpl() if (packets.Count == 0) break; - if (Settings.FilterPacketsNum < 0) - { - var packetsPerSplit = Math.Abs(Settings.FilterPacketsNum); - var totalPackets = packets.Count; - - var numberOfSplits = (int)Math.Ceiling((double)totalPackets/packetsPerSplit); - - for (var i = 0; i < numberOfSplits; ++i) - { - var fileNamePart = FileName + "_part_" + (i + 1) + ".pkt"; - - var packetsPart = packets.Take(packetsPerSplit).ToList(); - packets.RemoveRange(0, packetsPart.Count); - - BinaryDump(fileNamePart, packetsPart); - } - } - else - { - var fileNameExcerpt = Path.ChangeExtension(FileName, null) + "_excerpt.pkt"; - BinaryDump(fileNameExcerpt, packets); - } - + processPacket(packets); break; } case DumpFormatType.PktSplit: @@ -344,88 +184,296 @@ private void ProcessFileImpl() } case DumpFormatType.SniffVersionSplit: { - var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); + processVersion(); + break; + } + case DumpFormatType.ConnectionIndexes: + { + var packets = ReadPackets(); + if (packets.Count == 0) + break; - if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead()) + processIndexes(packets); + break; + } + case DumpFormatType.Fusion: + { + var packets = ReadPackets(); + if (packets.Count == 0) + break; + + FusionDump(packets); + break; + } + default: + { + Trace.WriteLine($"{_logPrefix}: Dump format is none, nothing will be processed."); + break; + } + } + } + + private void launchProcessing(string outFileName) + { + var threadCount = setupThreads(outFileName); + + var written = false; + using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) + { + var firstRead = true; + var firstWrite = true; + + var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); + + var pwp = new ParallelWorkProcessor(() => // read + { + var tuple = readingProcedure(reader); + + if (firstRead) { - Packet packet; - reader.TryRead(out packet); - packet.ClosePacket(); + Trace.WriteLine( + $"{_logPrefix}: Parsing {Utilities.BytesToString(reader.PacketReader.GetTotalSize())} of packets. Detected version {ClientVersion.VersionString}"); + + firstRead = false; } - reader.PacketReader.Dispose(); + return tuple; + }, packet => // process + { + processingProcedure(packet); + + return packet; + }, + packet => // write + { + writingProcedure(packet, reader, writer, written, firstWrite); - var version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString; + }, threadCount); - var realFileName = GetCompressedFileName(); + pwp.WaitForFinished(Timeout.Infinite); - var destPath = Path.Combine(Path.GetDirectoryName(realFileName), version, - Path.GetFileName(realFileName)); + reader.PacketReader.Dispose(); - var destDir = Path.GetDirectoryName(destPath); - if (!Directory.Exists(destDir)) - Directory.CreateDirectory(destDir); + _stats.SetEndTime(DateTime.Now); + } - File.Move(realFileName, destPath); + if (written) + Trace.WriteLine($"{_logPrefix}: Saved file to '{outFileName}'"); + else + { + Trace.WriteLine($"{_logPrefix}: No file produced"); + File.Delete(outFileName); + } - Trace.WriteLine("Moved " + realFileName + " to " + destPath); + Trace.WriteLine($"{_logPrefix}: {_stats}"); - break; - } - case DumpFormatType.ConnectionIndexes: + if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog()) + WriteSQLs(); + + if (Settings.LogPacketErrors) + WritePacketErrors(); + + GC.Collect(); // Force a GC collect after parsing a file. It seems to help. + } + + private int setupThreads(string outFileName) + { + Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; + File.Delete(outFileName); + + _stats.SetStartTime(DateTime.Now); + + var threadCount = Settings.Threads; + if (threadCount == 0) + threadCount = Environment.ProcessorCount; + + ThreadPool.SetMinThreads(threadCount + 2, 4); + + return threadCount; + } + + private Tuple readingProcedure(Reader reader) + { + if (!reader.PacketReader.CanRead()) + return Tuple.Create(null, true); + + Packet packet; + var b = reader.TryRead(out packet); + + return Tuple.Create(packet, b); + } + + private void processingProcedure(Packet packet) + { + // Parse the packet, adding text to Writer and stuff to the stores + if (packet.Direction == Direction.BNClientToServer || + packet.Direction == Direction.BNServerToClient) + BattlenetHandler.ParseBattlenet(packet); + else + Handler.Parse(packet); + + // Update statistics + _stats.AddByStatus(packet.Status); + } + + private void writingProcedure(Packet packet, Reader reader, StreamWriter writer, bool written, bool firstWrite) + { + ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); + + if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile) + { + packet.ClosePacket(); + return; + } + + written = true; + + if (firstWrite) + { + // ReSharper disable AccessToDisposedClosure + writer?.WriteLine(GetHeader(FileName)); + // ReSharper restore AccessToDisposedClosure + firstWrite = false; + } + + // get packet header if necessary + if (Settings.LogPacketErrors) + { + switch (packet.Status) { - var packets = ReadPackets(); - if (packets.Count == 0) + case ParsedStatus.WithErrors: + _withErrorHeaders.Add(packet.GetHeader()); + break; + case ParsedStatus.NotParsed: + _skippedHeaders.Add(packet.GetHeader()); break; + case ParsedStatus.NoStructure: + _noStructureHeaders.Add(packet.GetHeader()); + break; + } + } - using (var writer = new StreamWriter(Path.ChangeExtension(FileName, null) + "_connidx.txt")) - { - if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342) - writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values."); + // ReSharper disable AccessToDisposedClosure + if (writer != null) + { + // Write to file + writer.WriteLine(packet.Writer); + writer.Flush(); + } + // ReSharper restore AccessToDisposedClosure + // Close Writer, Stream - Dispose + packet.ClosePacket(); + } - IEnumerable, Packet>> groupsOpcode = packets - .GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction)) - .OrderBy(grouping => grouping.Key.Item2); + private void statsAnalysis(List packets) + { - foreach (var groupOpcode in groupsOpcode) - { - var groups = groupOpcode - .GroupBy(packet => packet.ConnectionIndex) - .OrderBy(grouping => grouping.Key) - .ToList(); + var firstPacket = packets.First(); + var lastPacket = packets.Last(); + + // CSV format + // ReSharper disable once UseStringInterpolation + Trace.WriteLine(string.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}", + FileName, // - sniff file name + firstPacket.Time, // - time of first packet + lastPacket.Time, // - time of last packet + (lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds) + packets.Count, // - packet count + packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes) + packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes) + packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes) + packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes) - writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2)); + } - for (var i = 0; i < groups.Count; i++) - { - var idx = groups[i].Key; - writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : ""); + private void processIndexes(List packets) + { + using (var writer = new StreamWriter(Path.ChangeExtension(FileName, null) + "_connidx.txt")) + { + if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342) + writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values."); - if (i != groups.Count - 1) - writer.Write(", "); - } + IEnumerable, Packet>> groupsOpcode = packets + .GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction)) + .OrderBy(grouping => grouping.Key.Item2); + + foreach (var groupOpcode in groupsOpcode) + { + var groups = groupOpcode + .GroupBy(packet => packet.ConnectionIndex) + .OrderBy(grouping => grouping.Key) + .ToList(); + + writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2)); + + for (var i = 0; i < groups.Count; i++) + { + var idx = groups[i].Key; + writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : ""); - writer.WriteLine(); - } + if (i != groups.Count - 1) + writer.Write(", "); } - break; + writer.WriteLine(); } - case DumpFormatType.Fusion: - { - var packets = ReadPackets(); - if (packets.Count == 0) - break; + } + } - FusionDump(packets); - break; - } - default: + private void processVersion() + { + var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); + + if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead()) + { + Packet packet; + reader.TryRead(out packet); + packet.ClosePacket(); + } + + reader.PacketReader.Dispose(); + + var version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString; + + var realFileName = GetCompressedFileName(); + + var destPath = Path.Combine(Path.GetDirectoryName(realFileName), version, + Path.GetFileName(realFileName)); + + var destDir = Path.GetDirectoryName(destPath); + if (!Directory.Exists(destDir)) + Directory.CreateDirectory(destDir); + + File.Move(realFileName, destPath); + + Trace.WriteLine("Moved " + realFileName + " to " + destPath); + + } + + private void processPacket(List packets) + { + if (Settings.FilterPacketsNum < 0) + { + var packetsPerSplit = Math.Abs(Settings.FilterPacketsNum); + var totalPackets = packets.Count; + + var numberOfSplits = (int)Math.Ceiling((double)totalPackets / packetsPerSplit); + + for (var i = 0; i < numberOfSplits; ++i) { - Trace.WriteLine($"{_logPrefix}: Dump format is none, nothing will be processed."); - break; + var fileNamePart = FileName + "_part_" + (i + 1) + ".pkt"; + + var packetsPart = packets.Take(packetsPerSplit).ToList(); + packets.RemoveRange(0, packetsPart.Count); + + BinaryDump(fileNamePart, packetsPart); } } + else + { + var fileNameExcerpt = Path.ChangeExtension(FileName, null) + "_excerpt.pkt"; + BinaryDump(fileNameExcerpt, packets); + } } public static string GetHeader(string fileName) @@ -438,7 +486,6 @@ public static string GetHeader(string fileName) "# Parsing date: " + DateTime.Now.ToString(CultureInfo.InvariantCulture) + Environment.NewLine; } - private static long _lastPercent; static void ShowPercentProgress(string message, long curr, long total) { var percent = (100 * curr) / total; @@ -606,5 +653,6 @@ private string GetCompressedFileName() { return FileName + _compression.GetExtension(); } + } } diff --git a/WowPacketParser/Misc/IOutputBuilder.cs b/WowPacketParser/Misc/IOutputBuilder.cs new file mode 100644 index 0000000000..725d1845ca --- /dev/null +++ b/WowPacketParser/Misc/IOutputBuilder.cs @@ -0,0 +1,9 @@ +namespace WowPacketParser.Misc +{ + public interface IOutputBuilder + { + void Append(string value); + + void Clear(); + } +} \ No newline at end of file diff --git a/WowPacketParser/Misc/Packet.cs b/WowPacketParser/Misc/Packet.cs index 06dcb8f955..e89a20f371 100644 --- a/WowPacketParser/Misc/Packet.cs +++ b/WowPacketParser/Misc/Packet.cs @@ -20,7 +20,7 @@ public sealed partial class Packet : BinaryReader private static DateTime _firstPacketTime; [SuppressMessage("Microsoft.Reliability", "CA2000", Justification = "MemoryStream is disposed in ClosePacket().")] - public Packet(byte[] input, int opcode, DateTime time, Direction direction, int number, StringBuilder writer, string fileName) + public Packet(byte[] input, int opcode, DateTime time, Direction direction, int number, IOutputBuilder writer, string fileName) : base(new MemoryStream(input, 0, input.Length), Encoding.UTF8) { Opcode = opcode; @@ -40,21 +40,8 @@ public Packet(byte[] input, int opcode, DateTime time, Direction direction, int [SuppressMessage("Microsoft.Reliability", "CA2000", Justification = "MemoryStream is disposed in ClosePacket().")] public Packet(byte[] input, int opcode, DateTime time, Direction direction, int number, string fileName) - : base(new MemoryStream(input, 0, input.Length), Encoding.UTF8) + : this(input, opcode, time, direction, number, null, fileName) { - Opcode = opcode; - Time = time; - Direction = direction; - Number = number; - Writer = null; - FileName = fileName; - Status = ParsedStatus.None; - WriteToFile = true; - - if (number == 0) - _firstPacketTime = Time; - - TimeSpan = Time - _firstPacketTime; } public int Opcode { get; set; } // setter can't be private because it's used in multiple_packets @@ -62,7 +49,7 @@ public Packet(byte[] input, int opcode, DateTime time, Direction direction, int public TimeSpan TimeSpan { get; } public Direction Direction { get; } public int Number { get; } - public StringBuilder Writer { get; private set; } + public IOutputBuilder Writer { get; private set; } public string FileName { get; } public ParsedStatus Status { get; set; } public bool WriteToFile { get; private set; } @@ -254,7 +241,7 @@ public void Write(string value) return; if (Writer == null) - Writer = new StringBuilder(); + Writer = new TextOutputBuilder(); Writer.Append(value); } @@ -265,9 +252,9 @@ public void Write(string format, params object[] args) return; if (Writer == null) - Writer = new StringBuilder(); + Writer = new TextOutputBuilder(); - Writer.AppendFormat(format, args); + Writer.Append(string.Format(format, args)); } public void WriteLine() @@ -276,9 +263,9 @@ public void WriteLine() return; if (Writer == null) - Writer = new StringBuilder(); + Writer = new TextOutputBuilder(); - Writer.AppendLine(); + Writer.Append(Environment.NewLine); } public void WriteLine(string value) @@ -287,9 +274,9 @@ public void WriteLine(string value) return; if (Writer == null) - Writer = new StringBuilder(); + Writer = new TextOutputBuilder(); - Writer.AppendLine(value); + Writer.Append(value + Environment.NewLine); } public void WriteLine(string format, params object[] args) @@ -298,9 +285,9 @@ public void WriteLine(string format, params object[] args) return; if (Writer == null) - Writer = new StringBuilder(); + Writer = new TextOutputBuilder(); - Writer.AppendLine(string.Format(format, args)); + Writer.Append(string.Format(format, args) + Environment.NewLine); } public void ClosePacket(bool clearWriter = true) diff --git a/WowPacketParser/Misc/Settings.cs b/WowPacketParser/Misc/Settings.cs index 439ec8535d..6d9b9ebabf 100644 --- a/WowPacketParser/Misc/Settings.cs +++ b/WowPacketParser/Misc/Settings.cs @@ -29,6 +29,8 @@ public static class Settings public static readonly bool DevMode = Conf.GetBoolean("DevMode", false); public static readonly int Threads = Conf.GetInt("Threads", 8); + public static readonly TextOutputFormat DumpTextFormat = TextOutputFormat.Txt; + public static readonly bool SSHEnabled = Conf.GetBoolean("SSHEnabled", false); public static readonly string SSHHost = Conf.GetString("SSHHost", "localhost"); public static readonly string SSHUsername = Conf.GetString("SSHUsername", string.Empty); diff --git a/WowPacketParser/Misc/TextOutputBuilder.cs b/WowPacketParser/Misc/TextOutputBuilder.cs new file mode 100644 index 0000000000..df34e1c5f6 --- /dev/null +++ b/WowPacketParser/Misc/TextOutputBuilder.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace WowPacketParser.Misc +{ + class TextOutputBuilder : IOutputBuilder + { + private StringBuilder _builder; + + public TextOutputBuilder() + { + _builder = new StringBuilder(); + } + + public void Append(string value) + { + _builder.Append(value); + } + + public void Clear() + { + _builder.Clear(); + } + } +} diff --git a/WowPacketParser/Misc/TextOutputFormat.cs b/WowPacketParser/Misc/TextOutputFormat.cs new file mode 100644 index 0000000000..e91eaba3cc --- /dev/null +++ b/WowPacketParser/Misc/TextOutputFormat.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace WowPacketParser.Misc +{ + public enum TextOutputFormat + { + Txt, + Xml + } +} diff --git a/WowPacketParser/WowPacketParser.csproj b/WowPacketParser/WowPacketParser.csproj index ac78f4218e..e2278f1112 100644 --- a/WowPacketParser/WowPacketParser.csproj +++ b/WowPacketParser/WowPacketParser.csproj @@ -413,6 +413,9 @@ + + + From ecb23a8dabe8b80ee6d98aa2d1791593cfe98c69 Mon Sep 17 00:00:00 2001 From: Ennio Visconti Date: Sat, 11 Mar 2017 12:42:15 +0100 Subject: [PATCH 2/5] Fixed logical error --- WowPacketParser/Loading/SniffFile.cs | 41 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/WowPacketParser/Loading/SniffFile.cs b/WowPacketParser/Loading/SniffFile.cs index 638ff298c4..55fed3309c 100644 --- a/WowPacketParser/Loading/SniffFile.cs +++ b/WowPacketParser/Loading/SniffFile.cs @@ -80,7 +80,6 @@ private string FileName public void ProcessFile() { - try { ProcessFileImpl(); @@ -246,7 +245,25 @@ private void launchProcessing(string outFileName) }, packet => // write { - writingProcedure(packet, reader, writer, written, firstWrite); + ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); + + if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile) + { + packet.ClosePacket(); + return; + } + + written = true; + + if (firstWrite) + { + // ReSharper disable AccessToDisposedClosure + writer?.WriteLine(GetHeader(FileName)); + // ReSharper restore AccessToDisposedClosure + firstWrite = false; + } + + writingProcedure(packet, writer); }, threadCount); @@ -316,26 +333,8 @@ private void processingProcedure(Packet packet) _stats.AddByStatus(packet.Status); } - private void writingProcedure(Packet packet, Reader reader, StreamWriter writer, bool written, bool firstWrite) + private void writingProcedure(Packet packet, StreamWriter writer) { - ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); - - if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile) - { - packet.ClosePacket(); - return; - } - - written = true; - - if (firstWrite) - { - // ReSharper disable AccessToDisposedClosure - writer?.WriteLine(GetHeader(FileName)); - // ReSharper restore AccessToDisposedClosure - firstWrite = false; - } - // get packet header if necessary if (Settings.LogPacketErrors) { From faae36cebac1e355f85b6a4328f93005f55b9a0b Mon Sep 17 00:00:00 2001 From: Ennio Visconti Date: Sat, 11 Mar 2017 17:33:51 +0100 Subject: [PATCH 3/5] Implemented basic XML output file -not really useful- --- WowPacketParser/Loading/IWritingStrategy.cs | 11 +++++ WowPacketParser/Loading/SniffFile.cs | 31 ++++++++++-- WowPacketParser/Loading/TextDumpWriter.cs | 30 ++++++++++++ WowPacketParser/Loading/XmlDumpWriter.cs | 52 +++++++++++++++++++++ WowPacketParser/Misc/TextOutputBuilder.cs | 5 ++ WowPacketParser/WowPacketParser.csproj | 4 ++ 6 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 WowPacketParser/Loading/IWritingStrategy.cs create mode 100644 WowPacketParser/Loading/TextDumpWriter.cs create mode 100644 WowPacketParser/Loading/XmlDumpWriter.cs diff --git a/WowPacketParser/Loading/IWritingStrategy.cs b/WowPacketParser/Loading/IWritingStrategy.cs new file mode 100644 index 0000000000..f8ef461402 --- /dev/null +++ b/WowPacketParser/Loading/IWritingStrategy.cs @@ -0,0 +1,11 @@ +using System; + +namespace WowPacketParser.Loading +{ + public interface IWritingStrategy : IDisposable + { + void Write(object input); + + void Flush(); + } +} \ No newline at end of file diff --git a/WowPacketParser/Loading/SniffFile.cs b/WowPacketParser/Loading/SniffFile.cs index 55fed3309c..154b4d01d3 100644 --- a/WowPacketParser/Loading/SniffFile.cs +++ b/WowPacketParser/Loading/SniffFile.cs @@ -94,7 +94,7 @@ public void ProcessFile() { if (_tempName != null) { - File.Delete(_tempName); + //File.Delete(_tempName); Trace.WriteLine(_logPrefix + " Deleted temporary file " + Path.GetFileName(_tempName)); } } @@ -217,7 +217,7 @@ private void launchProcessing(string outFileName) var threadCount = setupThreads(outFileName); var written = false; - using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) + using (var writer = (Settings.DumpFormatWithText() ? GetWriter(outFileName) : null)) { var firstRead = true; var firstWrite = true; @@ -258,7 +258,7 @@ private void launchProcessing(string outFileName) if (firstWrite) { // ReSharper disable AccessToDisposedClosure - writer?.WriteLine(GetHeader(FileName)); + writer?.Write(GetHeader(FileName)); // ReSharper restore AccessToDisposedClosure firstWrite = false; } @@ -293,6 +293,24 @@ private void launchProcessing(string outFileName) GC.Collect(); // Force a GC collect after parsing a file. It seems to help. } + private static IWritingStrategy GetWriter(string outFileName) + { + IWritingStrategy writingStrategy = null; + + switch (Settings.DumpTextFormat) + { + case TextOutputFormat.Txt: + writingStrategy = new TextDumpWriter(outFileName); + break; + case TextOutputFormat.Xml: + writingStrategy = new XmlDumpWriter(outFileName); + break; + default: + break; + } + return writingStrategy; + } + private int setupThreads(string outFileName) { Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; @@ -333,7 +351,7 @@ private void processingProcedure(Packet packet) _stats.AddByStatus(packet.Status); } - private void writingProcedure(Packet packet, StreamWriter writer) + private void writingProcedure(Packet packet, IWritingStrategy writer) { // get packet header if necessary if (Settings.LogPacketErrors) @@ -356,8 +374,11 @@ private void writingProcedure(Packet packet, StreamWriter writer) if (writer != null) { // Write to file - writer.WriteLine(packet.Writer); + writer.Write(packet.Writer); writer.Flush(); + //Console.Write("\r\n Continue..."); + //Console.ReadKey(); + //Console.Write("\r"); } // ReSharper restore AccessToDisposedClosure // Close Writer, Stream - Dispose diff --git a/WowPacketParser/Loading/TextDumpWriter.cs b/WowPacketParser/Loading/TextDumpWriter.cs new file mode 100644 index 0000000000..0f2cd7c3b4 --- /dev/null +++ b/WowPacketParser/Loading/TextDumpWriter.cs @@ -0,0 +1,30 @@ +using System; +using System.IO; + +namespace WowPacketParser.Loading +{ + public class TextDumpWriter : IWritingStrategy + { + private StreamWriter _writer; + + public TextDumpWriter(string outFileName) + { + _writer = new StreamWriter(outFileName, true); + } + + public void Write(object input) + { + _writer.WriteLine(input.ToString()); + } + + public void Flush() + { + _writer.Flush(); + } + + public void Dispose() + { + _writer.Dispose(); + } + } +} \ No newline at end of file diff --git a/WowPacketParser/Loading/XmlDumpWriter.cs b/WowPacketParser/Loading/XmlDumpWriter.cs new file mode 100644 index 0000000000..10be50446b --- /dev/null +++ b/WowPacketParser/Loading/XmlDumpWriter.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Xml; + +namespace WowPacketParser.Loading +{ + public class XmlDumpWriter : IWritingStrategy + { + private StreamWriter _writer; + private XmlDocument _document; + + public XmlDumpWriter(string outFileName) + { + _document = new XmlDocument(); + + if(!File.Exists(outFileName)) + { + XmlTextWriter textWritter = new XmlTextWriter(outFileName, null); + textWritter.WriteStartDocument(); + textWritter.WriteStartElement("packets"); + textWritter.WriteEndElement(); + + textWritter.Close(); + } + + _document.Load(outFileName); + + + //_writer = new StreamWriter(Console.OpenStandardOutput()); + _writer = new StreamWriter(outFileName, true); + } + + public void Write(object input) + { + XmlElement elem = _document.CreateElement("line"); + elem.InnerText = input.ToString(); + _document.GetElementsByTagName("packets")[0].AppendChild(elem); + } + + public void Flush() + { + _writer.Flush(); + } + + public void Dispose() + { + _document.Save(_writer); + _writer.Dispose(); + //_reader.Dispose(); + } + } +} \ No newline at end of file diff --git a/WowPacketParser/Misc/TextOutputBuilder.cs b/WowPacketParser/Misc/TextOutputBuilder.cs index df34e1c5f6..217356fb41 100644 --- a/WowPacketParser/Misc/TextOutputBuilder.cs +++ b/WowPacketParser/Misc/TextOutputBuilder.cs @@ -20,6 +20,11 @@ public void Append(string value) _builder.Append(value); } + public override string ToString() + { + return _builder.ToString(); + } + public void Clear() { _builder.Clear(); diff --git a/WowPacketParser/WowPacketParser.csproj b/WowPacketParser/WowPacketParser.csproj index e2278f1112..0447d97356 100644 --- a/WowPacketParser/WowPacketParser.csproj +++ b/WowPacketParser/WowPacketParser.csproj @@ -114,6 +114,7 @@ + False ..\packages\Tamir.SharpSSH.1.1.1.13\lib\Tamir.SharpSSH.dll @@ -400,7 +401,10 @@ + + + From cfc82778b69bb31fe22373cadd7c1f7191ae6cac Mon Sep 17 00:00:00 2001 From: Ennio Visconti Date: Sat, 11 Mar 2017 17:51:41 +0100 Subject: [PATCH 4/5] Added missing newline at end of file --- WowPacketParser/Loading/IWritingStrategy.cs | 2 +- WowPacketParser/Loading/SniffFile.cs | 2 +- WowPacketParser/Loading/TextDumpWriter.cs | 2 +- WowPacketParser/Loading/XmlDumpWriter.cs | 3 +-- WowPacketParser/Misc/IOutputBuilder.cs | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/WowPacketParser/Loading/IWritingStrategy.cs b/WowPacketParser/Loading/IWritingStrategy.cs index f8ef461402..df228e32eb 100644 --- a/WowPacketParser/Loading/IWritingStrategy.cs +++ b/WowPacketParser/Loading/IWritingStrategy.cs @@ -8,4 +8,4 @@ public interface IWritingStrategy : IDisposable void Flush(); } -} \ No newline at end of file +} diff --git a/WowPacketParser/Loading/SniffFile.cs b/WowPacketParser/Loading/SniffFile.cs index 154b4d01d3..89ac7902f3 100644 --- a/WowPacketParser/Loading/SniffFile.cs +++ b/WowPacketParser/Loading/SniffFile.cs @@ -94,7 +94,7 @@ public void ProcessFile() { if (_tempName != null) { - //File.Delete(_tempName); + File.Delete(_tempName); Trace.WriteLine(_logPrefix + " Deleted temporary file " + Path.GetFileName(_tempName)); } } diff --git a/WowPacketParser/Loading/TextDumpWriter.cs b/WowPacketParser/Loading/TextDumpWriter.cs index 0f2cd7c3b4..1c98bba604 100644 --- a/WowPacketParser/Loading/TextDumpWriter.cs +++ b/WowPacketParser/Loading/TextDumpWriter.cs @@ -27,4 +27,4 @@ public void Dispose() _writer.Dispose(); } } -} \ No newline at end of file +} diff --git a/WowPacketParser/Loading/XmlDumpWriter.cs b/WowPacketParser/Loading/XmlDumpWriter.cs index 10be50446b..143498f28e 100644 --- a/WowPacketParser/Loading/XmlDumpWriter.cs +++ b/WowPacketParser/Loading/XmlDumpWriter.cs @@ -46,7 +46,6 @@ public void Dispose() { _document.Save(_writer); _writer.Dispose(); - //_reader.Dispose(); } } -} \ No newline at end of file +} diff --git a/WowPacketParser/Misc/IOutputBuilder.cs b/WowPacketParser/Misc/IOutputBuilder.cs index 725d1845ca..12f15e0fe8 100644 --- a/WowPacketParser/Misc/IOutputBuilder.cs +++ b/WowPacketParser/Misc/IOutputBuilder.cs @@ -6,4 +6,4 @@ public interface IOutputBuilder void Clear(); } -} \ No newline at end of file +} From 0b0f15afa75cb49689a3bddeffc30d6844fb63c9 Mon Sep 17 00:00:00 2001 From: Ennio Visconti Date: Sat, 11 Mar 2017 20:33:21 +0100 Subject: [PATCH 5/5] Update to comply to coding standards. --- WowPacketParser/Loading/SniffFile.cs | 46 +++++++++---------- ...{TextDumpWriter.cs => TextOutputWriter.cs} | 4 +- .../{XmlDumpWriter.cs => XmlOutputWriter.cs} | 4 +- .../{TextOutputFormat.cs => OutputFormat.cs} | 2 +- WowPacketParser/Misc/Settings.cs | 2 +- WowPacketParser/WowPacketParser.csproj | 6 +-- 6 files changed, 32 insertions(+), 32 deletions(-) rename WowPacketParser/Loading/{TextDumpWriter.cs => TextOutputWriter.cs} (81%) rename WowPacketParser/Loading/{XmlDumpWriter.cs => XmlOutputWriter.cs} (92%) rename WowPacketParser/Misc/{TextOutputFormat.cs => OutputFormat.cs} (85%) diff --git a/WowPacketParser/Loading/SniffFile.cs b/WowPacketParser/Loading/SniffFile.cs index 89ac7902f3..ea67b4a863 100644 --- a/WowPacketParser/Loading/SniffFile.cs +++ b/WowPacketParser/Loading/SniffFile.cs @@ -113,7 +113,7 @@ private void ProcessFileImpl() if (packets.Count == 0) break; - statsAnalysis(packets); + StatsAnalysis(packets); break; } case DumpFormatType.SniffDataOnly: @@ -131,7 +131,7 @@ private void ProcessFileImpl() break; } - launchProcessing(outFileName); + LaunchProcessing(outFileName); break; } case DumpFormatType.Pkt: @@ -140,7 +140,7 @@ private void ProcessFileImpl() if (packets.Count == 0) break; - processPacket(packets); + ProcessPacket(packets); break; } case DumpFormatType.PktSplit: @@ -183,7 +183,7 @@ private void ProcessFileImpl() } case DumpFormatType.SniffVersionSplit: { - processVersion(); + ProcessVersion(); break; } case DumpFormatType.ConnectionIndexes: @@ -192,7 +192,7 @@ private void ProcessFileImpl() if (packets.Count == 0) break; - processIndexes(packets); + ProcessIndexes(packets); break; } case DumpFormatType.Fusion: @@ -212,9 +212,9 @@ private void ProcessFileImpl() } } - private void launchProcessing(string outFileName) + private void LaunchProcessing(string outFileName) { - var threadCount = setupThreads(outFileName); + var threadCount = SetupThreads(outFileName); var written = false; using (var writer = (Settings.DumpFormatWithText() ? GetWriter(outFileName) : null)) @@ -226,7 +226,7 @@ private void launchProcessing(string outFileName) var pwp = new ParallelWorkProcessor(() => // read { - var tuple = readingProcedure(reader); + var tuple = ReadingProcedure(reader); if (firstRead) { @@ -239,7 +239,7 @@ private void launchProcessing(string outFileName) return tuple; }, packet => // process { - processingProcedure(packet); + ProcessingProcedure(packet); return packet; }, @@ -263,7 +263,7 @@ private void launchProcessing(string outFileName) firstWrite = false; } - writingProcedure(packet, writer); + WritingProcedure(packet, writer); }, threadCount); @@ -297,13 +297,13 @@ private static IWritingStrategy GetWriter(string outFileName) { IWritingStrategy writingStrategy = null; - switch (Settings.DumpTextFormat) + switch (Settings.OutputTextFormat) { - case TextOutputFormat.Txt: - writingStrategy = new TextDumpWriter(outFileName); + case OutputFormat.Txt: + writingStrategy = new TextOutputWriter(outFileName); break; - case TextOutputFormat.Xml: - writingStrategy = new XmlDumpWriter(outFileName); + case OutputFormat.Xml: + writingStrategy = new XmlOutputWriter(outFileName); break; default: break; @@ -311,7 +311,7 @@ private static IWritingStrategy GetWriter(string outFileName) return writingStrategy; } - private int setupThreads(string outFileName) + private int SetupThreads(string outFileName) { Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; File.Delete(outFileName); @@ -327,7 +327,7 @@ private int setupThreads(string outFileName) return threadCount; } - private Tuple readingProcedure(Reader reader) + private Tuple ReadingProcedure(Reader reader) { if (!reader.PacketReader.CanRead()) return Tuple.Create(null, true); @@ -338,7 +338,7 @@ private Tuple readingProcedure(Reader reader) return Tuple.Create(packet, b); } - private void processingProcedure(Packet packet) + private void ProcessingProcedure(Packet packet) { // Parse the packet, adding text to Writer and stuff to the stores if (packet.Direction == Direction.BNClientToServer || @@ -351,7 +351,7 @@ private void processingProcedure(Packet packet) _stats.AddByStatus(packet.Status); } - private void writingProcedure(Packet packet, IWritingStrategy writer) + private void WritingProcedure(Packet packet, IWritingStrategy writer) { // get packet header if necessary if (Settings.LogPacketErrors) @@ -385,7 +385,7 @@ private void writingProcedure(Packet packet, IWritingStrategy writer) packet.ClosePacket(); } - private void statsAnalysis(List packets) + private void StatsAnalysis(List packets) { var firstPacket = packets.First(); @@ -406,7 +406,7 @@ private void statsAnalysis(List packets) } - private void processIndexes(List packets) + private void ProcessIndexes(List packets) { using (var writer = new StreamWriter(Path.ChangeExtension(FileName, null) + "_connidx.txt")) { @@ -440,7 +440,7 @@ private void processIndexes(List packets) } } - private void processVersion() + private void ProcessVersion() { var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); @@ -470,7 +470,7 @@ private void processVersion() } - private void processPacket(List packets) + private void ProcessPacket(List packets) { if (Settings.FilterPacketsNum < 0) { diff --git a/WowPacketParser/Loading/TextDumpWriter.cs b/WowPacketParser/Loading/TextOutputWriter.cs similarity index 81% rename from WowPacketParser/Loading/TextDumpWriter.cs rename to WowPacketParser/Loading/TextOutputWriter.cs index 1c98bba604..115b2488d8 100644 --- a/WowPacketParser/Loading/TextDumpWriter.cs +++ b/WowPacketParser/Loading/TextOutputWriter.cs @@ -3,11 +3,11 @@ namespace WowPacketParser.Loading { - public class TextDumpWriter : IWritingStrategy + public class TextOutputWriter : IWritingStrategy { private StreamWriter _writer; - public TextDumpWriter(string outFileName) + public TextOutputWriter(string outFileName) { _writer = new StreamWriter(outFileName, true); } diff --git a/WowPacketParser/Loading/XmlDumpWriter.cs b/WowPacketParser/Loading/XmlOutputWriter.cs similarity index 92% rename from WowPacketParser/Loading/XmlDumpWriter.cs rename to WowPacketParser/Loading/XmlOutputWriter.cs index 143498f28e..d77cfb8c52 100644 --- a/WowPacketParser/Loading/XmlDumpWriter.cs +++ b/WowPacketParser/Loading/XmlOutputWriter.cs @@ -4,12 +4,12 @@ namespace WowPacketParser.Loading { - public class XmlDumpWriter : IWritingStrategy + public class XmlOutputWriter : IWritingStrategy { private StreamWriter _writer; private XmlDocument _document; - public XmlDumpWriter(string outFileName) + public XmlOutputWriter(string outFileName) { _document = new XmlDocument(); diff --git a/WowPacketParser/Misc/TextOutputFormat.cs b/WowPacketParser/Misc/OutputFormat.cs similarity index 85% rename from WowPacketParser/Misc/TextOutputFormat.cs rename to WowPacketParser/Misc/OutputFormat.cs index e91eaba3cc..65bf68b676 100644 --- a/WowPacketParser/Misc/TextOutputFormat.cs +++ b/WowPacketParser/Misc/OutputFormat.cs @@ -6,7 +6,7 @@ namespace WowPacketParser.Misc { - public enum TextOutputFormat + public enum OutputFormat { Txt, Xml diff --git a/WowPacketParser/Misc/Settings.cs b/WowPacketParser/Misc/Settings.cs index 6d9b9ebabf..b9e731ff8f 100644 --- a/WowPacketParser/Misc/Settings.cs +++ b/WowPacketParser/Misc/Settings.cs @@ -29,7 +29,7 @@ public static class Settings public static readonly bool DevMode = Conf.GetBoolean("DevMode", false); public static readonly int Threads = Conf.GetInt("Threads", 8); - public static readonly TextOutputFormat DumpTextFormat = TextOutputFormat.Txt; + public static readonly OutputFormat OutputTextFormat = OutputFormat.Txt; public static readonly bool SSHEnabled = Conf.GetBoolean("SSHEnabled", false); public static readonly string SSHHost = Conf.GetString("SSHHost", "localhost"); diff --git a/WowPacketParser/WowPacketParser.csproj b/WowPacketParser/WowPacketParser.csproj index 0447d97356..af03e4c32a 100644 --- a/WowPacketParser/WowPacketParser.csproj +++ b/WowPacketParser/WowPacketParser.csproj @@ -403,8 +403,8 @@ - - + + @@ -419,7 +419,7 @@ - +