diff --git a/SpssLib/Compression/DecompressedDataStream.cs b/SpssLib/Compression/DecompressedDataStream.cs index f94b5a1..6eac2cd 100644 --- a/SpssLib/Compression/DecompressedDataStream.cs +++ b/SpssLib/Compression/DecompressedDataStream.cs @@ -3,6 +3,8 @@ using System.Text; using System.IO; +using SpssLib.FileParser; + namespace SpssLib.Compression { class DecompressedDataStream: Stream @@ -24,15 +26,15 @@ class DecompressedDataStream: Stream private byte[] _systemMissingBytes; private byte[] _spacesBytes; - private BinaryReader _reader; + private DualBinaryReader _reader; - public DecompressedDataStream(Stream compressedDataStream, double bias, double systemMissing) + public DecompressedDataStream(Stream compressedDataStream, double bias, double systemMissing, bool isLittleEndian) { CompressedDataStream = compressedDataStream; Bias = bias; SystemMissing = systemMissing; - _reader = new BinaryReader(compressedDataStream, Encoding.ASCII); - + _reader = new DualBinaryReader(compressedDataStream, Encoding.ASCII); + _reader.IsLittleEndian = isLittleEndian; _spacesBytes = Encoding.ASCII.GetBytes(SpaceString); _systemMissingBytes = BitConverter.GetBytes(SystemMissing); } @@ -163,11 +165,19 @@ private bool ParseNextInstructionSet() { } - else if (instruction > 0 && instruction < 252) // compressed value + else if (instruction > 0 && instruction < 252) // compressed value (small 1-byte integers) { // compute actual value: double value = instruction - Bias; - _elementBuffer[bufferPosition++] = BitConverter.GetBytes(value); + byte[] element = BitConverter.GetBytes(value); + if(_reader.IsLittleEndian) + _elementBuffer[bufferPosition++] = BitConverter.GetBytes(value); + else + { + byte[] revElement = new byte[InstructionSetByteSize]; + Helper.ReverseArray(element, revElement); + _elementBuffer[bufferPosition++] = revElement; + } } else if (instruction == 252) // end of file { diff --git a/SpssLib/DataReader/SpssReader.cs b/SpssLib/DataReader/SpssReader.cs index c122a41..c8ec030 100644 --- a/SpssLib/DataReader/SpssReader.cs +++ b/SpssLib/DataReader/SpssReader.cs @@ -17,7 +17,7 @@ public class SpssReader : IDisposable // TODO add needed metadata info (use SpssOptions?) /// - /// A collection of variables read from teh file + /// A collection of variables read from the file /// public ICollection Variables { get; private set; } /// diff --git a/SpssLib/FileParser/DualBinaryReader.cs b/SpssLib/FileParser/DualBinaryReader.cs new file mode 100644 index 0000000..de33ae8 --- /dev/null +++ b/SpssLib/FileParser/DualBinaryReader.cs @@ -0,0 +1,113 @@ +using System; +using System.IO; +using System.Text; + +namespace SpssLib.FileParser +{ + public static class Helper + { + public static void ReverseArray(byte[] source, byte[] target) + { + int len = source.Length; + for (int i = 0; i < len; i++) + target[i] = source[len - 1 - i]; + } + } + + // Binary reader that supports both little-endian and big-endian byte orders. It assumes that the current environment is little-indian. + public class DualBinaryReader : BinaryReader + { + public DualBinaryReader(Stream stream, Encoding encoding) : base(stream, encoding) + { + + } + + public bool IsLittleEndian { get; set; } = true; + + public override Int16 ReadInt16() + { + Int16 i; + + if (this.IsLittleEndian) + i = base.ReadInt16(); + else + { + byte b1 = this.ReadByte(); + byte b0 = this.ReadByte(); + i = (short)(b1 << 8 + b0); + } + + return i; + } + + public override UInt16 ReadUInt16() + { + UInt16 i; + + if (this.IsLittleEndian) + i = base.ReadUInt16(); + else + { + byte b1 = this.ReadByte(); + byte b0 = this.ReadByte(); + i = (UInt16)(b1 << 8 + b0); + } + + return i; + } + + public override int ReadInt32() + { + int i; + + if (this.IsLittleEndian) + i = base.ReadInt32(); + else + { + i = this.ReadByte(); + + for (int k = 0; k < 3; k++) + i = (i << 8) | this.ReadByte(); + } + + return i; + } + + public override uint ReadUInt32() + { + uint i; + + if (this.IsLittleEndian) + i = base.ReadUInt32(); + else + { + i = this.ReadByte(); + + for (int k = 0; k < 3; k++) + i = (i << 8) | this.ReadByte(); + } + + return i; + } + + public override double ReadDouble() + { + double f; + + if (this.IsLittleEndian) + f = base.ReadDouble(); + else + { + UInt64 l = this.ReadByte(); + + for(int k = 0; k < 7; k++) + l = (l << 8) | this.ReadByte(); + + f = BitConverter.ToDouble( BitConverter.GetBytes(l), 0); + } + + return f; + } + + } +} diff --git a/SpssLib/FileParser/MetaData.cs b/SpssLib/FileParser/MetaData.cs index a6253c1..b7e5bb3 100644 --- a/SpssLib/FileParser/MetaData.cs +++ b/SpssLib/FileParser/MetaData.cs @@ -69,6 +69,19 @@ internal set public IList InfoRecords { get; private set; } + public bool IsLittleEndian + { + get + { + if (this.MachineIntegerInfo.Endianness == 2) + return true; + else if (this.MachineIntegerInfo.Endianness == 1) + return false; + else + throw new SpssFileFormatException("Failed to detect endianness."); + } + } + public double SystemMissingValue { get; private set; } // Count number of variables (the number of variable-records with a name, // the rest is part of a long string variable), this includes the variables diff --git a/SpssLib/FileParser/Records/BaseInfoRecord.cs b/SpssLib/FileParser/Records/BaseInfoRecord.cs index 6539ed7..ce3facb 100644 --- a/SpssLib/FileParser/Records/BaseInfoRecord.cs +++ b/SpssLib/FileParser/Records/BaseInfoRecord.cs @@ -21,7 +21,7 @@ public void WriteRecord(BinaryWriter writer) WriteInfo(writer); } - public void FillRecord(BinaryReader reader) + public void FillRecord(DualBinaryReader reader) { ItemSize = reader.ReadInt32(); ItemCount = reader.ReadInt32(); @@ -49,7 +49,7 @@ protected void CheckInfoHeader(int itemSize = -1, int itemCount = -1) } protected abstract void WriteInfo(BinaryWriter writer); - protected abstract void FillInfo(BinaryReader reader); + protected abstract void FillInfo(DualBinaryReader reader); } public class UnknownInfoRecord : BaseInfoRecord @@ -78,7 +78,7 @@ protected override void WriteInfo(BinaryWriter writer) writer.Write(Data); } - protected override void FillInfo(BinaryReader reader) + protected override void FillInfo(DualBinaryReader reader) { Data = reader.ReadBytes(ItemCount * ItemSize); } diff --git a/SpssLib/FileParser/Records/CharacterEncodingRecord.cs b/SpssLib/FileParser/Records/CharacterEncodingRecord.cs index 6302dfb..b2043ee 100644 --- a/SpssLib/FileParser/Records/CharacterEncodingRecord.cs +++ b/SpssLib/FileParser/Records/CharacterEncodingRecord.cs @@ -39,7 +39,7 @@ protected override void WriteInfo(BinaryWriter writer) writer.Write(bytes); } - protected override void FillInfo(BinaryReader reader) + protected override void FillInfo(DualBinaryReader reader) { CheckInfoHeader(1); // items must be of size 1 (byte) diff --git a/SpssLib/FileParser/Records/DictionaryTerminationRecord.cs b/SpssLib/FileParser/Records/DictionaryTerminationRecord.cs index 1d1caf6..81def12 100644 --- a/SpssLib/FileParser/Records/DictionaryTerminationRecord.cs +++ b/SpssLib/FileParser/Records/DictionaryTerminationRecord.cs @@ -17,7 +17,7 @@ public void WriteRecord(BinaryWriter writer) writer.Write(0); } - public void FillRecord(BinaryReader reader) + public void FillRecord(DualBinaryReader reader) { // skip filler reader.ReadInt32(); diff --git a/SpssLib/FileParser/Records/DocumentRecord.cs b/SpssLib/FileParser/Records/DocumentRecord.cs index 42da13a..b005ed6 100644 --- a/SpssLib/FileParser/Records/DocumentRecord.cs +++ b/SpssLib/FileParser/Records/DocumentRecord.cs @@ -20,7 +20,7 @@ public DocumentRecord(IList lines) LineCount = lines.Count; } - public void FillRecord(BinaryReader reader) + public void FillRecord(DualBinaryReader reader) { LineCount = reader.ReadInt32(); LineCollection = new List(); diff --git a/SpssLib/FileParser/Records/EbcdicHeaderRecord.cs b/SpssLib/FileParser/Records/EbcdicHeaderRecord.cs index 6a08282..f198c83 100644 --- a/SpssLib/FileParser/Records/EbcdicHeaderRecord.cs +++ b/SpssLib/FileParser/Records/EbcdicHeaderRecord.cs @@ -5,7 +5,7 @@ namespace SpssLib.FileParser.Records { internal class EbcdicHeaderRecord : IRecord { - public NotSupportedException Exception => new NotSupportedException("EBCDIC???? Who uses that? Honestly!!"); + public NotSupportedException Exception => new NotSupportedException("EBCDIC records not supported."); public RecordType RecordType => RecordType.EbcdicHeaderRecord; @@ -14,7 +14,7 @@ public void WriteRecord(BinaryWriter writer) throw Exception; } - public void FillRecord(BinaryReader reader) + public void FillRecord(DualBinaryReader reader) { throw Exception; } diff --git a/SpssLib/FileParser/Records/HeaderRecord.cs b/SpssLib/FileParser/Records/HeaderRecord.cs index 484cb7e..632b257 100644 --- a/SpssLib/FileParser/Records/HeaderRecord.cs +++ b/SpssLib/FileParser/Records/HeaderRecord.cs @@ -58,10 +58,24 @@ public void WriteRecord(BinaryWriter writer) writer.Write(new byte[3]); } - public void FillRecord(BinaryReader reader) + public void FillRecord(DualBinaryReader reader) { ProductName = new string(reader.ReadChars(60)); LayoutCode = reader.ReadInt32(); + switch(LayoutCode) + { + case 2: + case 3: + reader.IsLittleEndian = true; + break; + case 0x02000000: + case 0x03000000: + reader.IsLittleEndian = false; + break; + default: + throw new SpssFileFormatException("Failed to detect endianness."); + } + NominalCaseSize = reader.ReadInt32(); Compressed = reader.ReadInt32() == 1; WeightIndex = reader.ReadInt32(); diff --git a/SpssLib/FileParser/Records/IRecord.cs b/SpssLib/FileParser/Records/IRecord.cs index 9299265..173b7f3 100644 --- a/SpssLib/FileParser/Records/IRecord.cs +++ b/SpssLib/FileParser/Records/IRecord.cs @@ -6,7 +6,7 @@ internal interface IRecord { RecordType RecordType { get; } void WriteRecord(BinaryWriter writer); // TODO: split to internal interface - void FillRecord(BinaryReader reader); + void FillRecord(DualBinaryReader reader); void RegisterMetadata(MetaData metaData); } diff --git a/SpssLib/FileParser/Records/IRecordParser.cs b/SpssLib/FileParser/Records/IRecordParser.cs index 131d919..4644f76 100644 --- a/SpssLib/FileParser/Records/IRecordParser.cs +++ b/SpssLib/FileParser/Records/IRecordParser.cs @@ -6,7 +6,7 @@ namespace SpssLib.FileParser.Records internal interface IRecordParser { RecordType Accepts { get; } - IRecord ParseRecord(BinaryReader reader); + IRecord ParseRecord(DualBinaryReader reader); } internal class GeneralRecordParser : IRecordParser where TRecord : IRecord @@ -18,7 +18,7 @@ public GeneralRecordParser(RecordType accepts) Accepts = accepts; } - public IRecord ParseRecord(BinaryReader reader) + public IRecord ParseRecord(DualBinaryReader reader) { TRecord record = CreateRecord(); record.FillRecord(reader); diff --git a/SpssLib/FileParser/Records/InfoRecordParser.cs b/SpssLib/FileParser/Records/InfoRecordParser.cs index dad406c..cd71bbf 100644 --- a/SpssLib/FileParser/Records/InfoRecordParser.cs +++ b/SpssLib/FileParser/Records/InfoRecordParser.cs @@ -15,7 +15,7 @@ public InfoRecordParser(IDictionary infoRecordsTypes) _infoRecordsTypes = infoRecordsTypes; } - public IRecord ParseRecord(BinaryReader reader) + public IRecord ParseRecord(DualBinaryReader reader) { IRecord record = CreateRecord(reader); record.FillRecord(reader); diff --git a/SpssLib/FileParser/Records/MachineFloatingPointInfoRecord.cs b/SpssLib/FileParser/Records/MachineFloatingPointInfoRecord.cs index 0c622e0..3245c50 100644 --- a/SpssLib/FileParser/Records/MachineFloatingPointInfoRecord.cs +++ b/SpssLib/FileParser/Records/MachineFloatingPointInfoRecord.cs @@ -35,7 +35,7 @@ protected override void WriteInfo(BinaryWriter writer) writer.Write(MissingLowestValue); } - protected override void FillInfo(BinaryReader reader) + protected override void FillInfo(DualBinaryReader reader) { CheckInfoHeader(8, 3); diff --git a/SpssLib/FileParser/Records/MachineIntegerInfoRecord.cs b/SpssLib/FileParser/Records/MachineIntegerInfoRecord.cs index a036b10..ebb125b 100644 --- a/SpssLib/FileParser/Records/MachineIntegerInfoRecord.cs +++ b/SpssLib/FileParser/Records/MachineIntegerInfoRecord.cs @@ -91,7 +91,7 @@ protected override void WriteInfo(BinaryWriter writer) writer.Write(CharacterCode); } - protected override void FillInfo(BinaryReader reader) + protected override void FillInfo(DualBinaryReader reader) { // Must have 8 int of 32 bits CheckInfoHeader(itemSize:4, itemCount:8); diff --git a/SpssLib/FileParser/Records/ValueLabelRecord.cs b/SpssLib/FileParser/Records/ValueLabelRecord.cs index 0ecb3d1..1060842 100644 --- a/SpssLib/FileParser/Records/ValueLabelRecord.cs +++ b/SpssLib/FileParser/Records/ValueLabelRecord.cs @@ -59,7 +59,7 @@ public void WriteRecord(BinaryWriter writer) } } - public void FillRecord(BinaryReader reader) + public void FillRecord(DualBinaryReader reader) { LabelCount = reader.ReadInt32(); _labelsRaw = new Dictionary>(); diff --git a/SpssLib/FileParser/Records/VariableDataInfoRecord.cs b/SpssLib/FileParser/Records/VariableDataInfoRecord.cs index 8620335..47a7d40 100644 --- a/SpssLib/FileParser/Records/VariableDataInfoRecord.cs +++ b/SpssLib/FileParser/Records/VariableDataInfoRecord.cs @@ -96,7 +96,7 @@ protected void BuildDataArray() Data = buffer; } - protected override void FillInfo(BinaryReader reader) + protected override void FillInfo(DualBinaryReader reader) { CheckInfoHeader(1); Data = reader.ReadBytes(ItemCount); diff --git a/SpssLib/FileParser/Records/VariableDisplayParameterRecord.cs b/SpssLib/FileParser/Records/VariableDisplayParameterRecord.cs index 2ab1810..00b31df 100644 --- a/SpssLib/FileParser/Records/VariableDisplayParameterRecord.cs +++ b/SpssLib/FileParser/Records/VariableDisplayParameterRecord.cs @@ -101,7 +101,7 @@ protected override void WriteInfo(BinaryWriter writer) } } - protected override void FillInfo(BinaryReader reader) + protected override void FillInfo(DualBinaryReader reader) { CheckInfoHeader(4); diff --git a/SpssLib/FileParser/Records/VariableRecord.cs b/SpssLib/FileParser/Records/VariableRecord.cs index 519525e..9bda65d 100644 --- a/SpssLib/FileParser/Records/VariableRecord.cs +++ b/SpssLib/FileParser/Records/VariableRecord.cs @@ -472,7 +472,7 @@ public void WriteRecord(BinaryWriter writer) } - public void FillRecord(BinaryReader reader) + public void FillRecord(DualBinaryReader reader) { Type = reader.ReadInt32(); HasVariableLabel = reader.ReadInt32() == 1; diff --git a/SpssLib/FileParser/SavFileParser.cs b/SpssLib/FileParser/SavFileParser.cs index 22c4045..4bc645a 100644 --- a/SpssLib/FileParser/SavFileParser.cs +++ b/SpssLib/FileParser/SavFileParser.cs @@ -18,7 +18,7 @@ public class SavFileParser: IDisposable public MetaData MetaData { get; private set; } public double SysmisValue { get; set; } - private BinaryReader _reader; + private DualBinaryReader _reader; private Stream _dataRecordStream; private long _dataStartPosition; @@ -29,7 +29,7 @@ public SavFileParser(Stream fileStream) public void ParseMetaData() { - _reader = new BinaryReader(Stream, Encoding.ASCII); + _reader = new DualBinaryReader(Stream, Encoding.ASCII); var parsers = new ParserProvider(); IList records = new List(1000); @@ -86,9 +86,13 @@ record = recordParser.ParseRecord(_reader); private void SetDataRecordStream() { _dataRecordStream = MetaData.HeaderRecord.Compressed ? - new DecompressedDataStream(Stream, MetaData.HeaderRecord.Bias, MetaData.SystemMissingValue) + new DecompressedDataStream(Stream, MetaData.HeaderRecord.Bias, MetaData.SystemMissingValue, MetaData.IsLittleEndian) : Stream; - _reader = new BinaryReader(_dataRecordStream, Encoding.ASCII); + + _reader = new DualBinaryReader(_dataRecordStream, Encoding.ASCII); + + if (MetaData.MachineIntegerInfo.Endianness == 1) // Machine endianness. 1 indicates big-endian, 2 indicates little-endian. + _reader.IsLittleEndian = false; } public IEnumerable DataRecords @@ -301,7 +305,17 @@ private static byte[] MoveNext(byte[][] record, IList variableRe /// private object ParseDoubleValue(byte[] element) { - var value = BitConverter.ToDouble(element, 0); + double value; + + if (MetaData.IsLittleEndian) + value = BitConverter.ToDouble(element, 0); + else + { + // Big-endian: reverse bytes in element before conversion to double. + byte[] revBytes = new byte[8]; + Helper.ReverseArray(element, revBytes); + value = BitConverter.ToDouble(revBytes, 0); + } // ReSharper disable CompareOfFloatsByEqualityOperator SysMiss is an exact value if (value == MetaData.SystemMissingValue) // ReSharper restore CompareOfFloatsByEqualityOperator @@ -406,7 +420,11 @@ private Variable GetVariable(int variableIndex, int dictionaryIndex, MetaData me { foreach (var label in valueLabelRecord.Labels) { - var key = BitConverter.ToDouble(label.Key, 0); + object nullableKey = ParseDoubleValue(label.Key); + if (!(nullableKey is double)) + throw new SpssFileFormatException(); + double key = (double)nullableKey; + var value = label.Value.Replace("\0", string.Empty).Trim(); if (variable.ValueLabels.ContainsKey(key)) diff --git a/SpssLib/SpssDataset/Variable.cs b/SpssLib/SpssDataset/Variable.cs index 5d75e85..63e809b 100644 --- a/SpssLib/SpssDataset/Variable.cs +++ b/SpssLib/SpssDataset/Variable.cs @@ -155,7 +155,7 @@ public bool IsDate() } - private DateTime AsDate(object value) + public DateTime AsDate(object value) { var dVal = (double)value; return _epoc.AddSeconds(dVal); diff --git a/SpssLib/SpssLib.csproj b/SpssLib/SpssLib.csproj index 82950bd..91f1721 100644 --- a/SpssLib/SpssLib.csproj +++ b/SpssLib/SpssLib.csproj @@ -49,6 +49,7 @@ + diff --git a/Test.SpssLib/Test.SpssLib.csproj b/Test.SpssLib/Test.SpssLib.csproj index 2c4adf8..28acf33 100644 --- a/Test.SpssLib/Test.SpssLib.csproj +++ b/Test.SpssLib/Test.SpssLib.csproj @@ -56,6 +56,9 @@ + + PreserveNewest + PreserveNewest diff --git a/Test.SpssLib/TestFiles/BigEndian.sav b/Test.SpssLib/TestFiles/BigEndian.sav new file mode 100644 index 0000000..b5aa69a Binary files /dev/null and b/Test.SpssLib/TestFiles/BigEndian.sav differ diff --git a/Test.SpssLib/TestSpssReader.cs b/Test.SpssLib/TestSpssReader.cs index d4283e5..87ccb55 100644 --- a/Test.SpssLib/TestSpssReader.cs +++ b/Test.SpssLib/TestSpssReader.cs @@ -126,6 +126,76 @@ public void TestReadMissingValuesAsNull() } } + [TestMethod] + [DeploymentItem(@"TestFiles\BigEndian.sav")] + public void TestReadFileWithBigEndianNumbers() + { + FileStream fileStream = new FileStream("BigEndian.sav", FileMode.Open, FileAccess.Read, + FileShare.Read, 2048 * 10, FileOptions.SequentialScan); + + int varCount; + int rowCount; + try + { + ReadData(fileStream, out varCount, out rowCount, + new Dictionary> + { + {0, (i, variable) => + { + Assert.AreEqual("ID", variable.Name, "Name mismatch"); + Assert.AreEqual(DataType.Numeric, variable.Type, "First file variable should be a Number"); + }}, + {1, (i, variable) => + { + Assert.AreEqual("SEX", variable.Name, "Name mismatch"); + Assert.AreEqual(DataType.Numeric, variable.Type, "Second file variable should be a Number"); + CollectionAssert.AreEqual(new Dictionary(){ { 1, "MALE" }, {2, "FEMALE" } }, (Dictionary)variable.ValueLabels, "Value labels mismatch"); + }}, + {2, (i, variable) => + { + Assert.AreEqual("GROUP", variable.Name, "Name mismatch"); + Assert.AreEqual(DataType.Numeric, variable.Type, "Third file variable should be a Number"); + CollectionAssert.AreEqual(new Dictionary(){ { 0, "Control" }, {1, "Treatment" } }, (Dictionary)variable.ValueLabels, "Value labels mismatch"); + }} + }, + new Dictionary> + { + {0, (r, c, variable, value) => + { // ID column contains 1-based row numbers + Assert.IsInstanceOfType(value, typeof(double), "First row variable should be a Number"); + double v = (double) value; + Assert.AreEqual(r + 1, v, "First row variable should contain number of the current row"); + }}, + {1, (r, c, variable, value) => + { // SEX column contains numbers 1 or 2 + Assert.IsInstanceOfType(value, typeof(double), "Second row variable should be a Number"); + double v = (double) value; + Assert.IsTrue((v == 1 || v == 2), "Second row variable should contain either 1 or 2"); + }}, + {5, (r, c, variable, value) => + { + // POSTTEST columns contains numbers from 47.5110642482932 to 111.011279434702 + Assert.IsInstanceOfType(value, typeof(double), "Sixth row variable should be a Number"); + double v = (double) value; + Assert.IsTrue((v > 47 && v < 112), "Sixth row variable should contain numbers from 47 to 112 but it contains {0} in row {1}", v, r); + + // Check number in the first row + if (r == 0) + { + Assert.AreEqual(53.321541454892696, v, "Numeric value in the sixth variable and the first row is different"); + } + }} + }); + } + finally + { + fileStream.Close(); + } + + Assert.AreEqual(varCount, 6, "Variable count does not match"); + Assert.AreEqual(rowCount, 49, "Rows count does not match"); + } + internal static void ReadData(Stream fileStream, out int varCount, out int rowCount, IDictionary> variableValidators = null, IDictionary> valueValidators = null) {