Unified output path resolution

Author: caetera

MainClass.cs

@@ -768,6 +768,9 @@ private static void RegularParametersParsing(string[] args)
                     if (parseInput.OutputFormat == OutputFormat.IndexMzML) parseInput.OutputFormat = OutputFormat.MzML;
                 }
 
+                // Switch off gzip compression for Parquet
+                if (parseInput.OutputFormat == OutputFormat.Parquet) parseInput.Gzip = false;
+
                 parseInput.MaxLevel = parseInput.MsLevel.Max();
 
                 if (parseInput.S3Url != null && parseInput.S3AccessKeyId != null &&

ThermoRawFileParserTest/WriterTests.cs

@@ -281,5 +281,30 @@ public void TestMzML_MS2()
 
             Assert.That(testMzMl.run.chromatogramList.chromatogram[0].defaultArrayLength, Is.EqualTo(95));
         }
+
+        [Test]
+        public void TestParquet()
+        {
+            // Get temp path for writing the test mzML
+            var tempFilePath = Path.GetTempPath();
+
+            var testRawFile = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"Data/small.RAW");
+            var parseInput = new ParseInput(testRawFile, null, tempFilePath, OutputFormat.Parquet);
+
+            RawFileParser.Parse(parseInput);
+
+            // Actual test
+            //var xmlSerializer = new XmlSerializer(typeof(mzMLType));
+            //var testMzMl = (mzMLType)xmlSerializer.Deserialize(new FileStream(
+            //    Path.Combine(tempFilePath, "small.mzML"), FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
+
+            //Assert.That(testMzMl.run.spectrumList.count, Is.EqualTo("48"));
+            //Assert.That(testMzMl.run.spectrumList.spectrum.Length, Is.EqualTo(48));
+
+            //Assert.That(testMzMl.run.chromatogramList.count, Is.EqualTo("1"));
+            //Assert.That(testMzMl.run.chromatogramList.chromatogram.Length, Is.EqualTo(1));
+
+            //Assert.That(testMzMl.run.chromatogramList.chromatogram[0].defaultArrayLength, Is.EqualTo(48));
+        }
     }
 }

Writer/ParquetSpectrumWriter.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Reflection;
 using log4net;
 using Parquet.Serialization;
@@ -43,17 +42,18 @@ public override void Write(IRawDataPlus raw, int firstScanNumber, int lastScanNu
                 throw new RawFileParserException("No MS data in RAW file, no output will be produced");
             }
 
+            //TODO: Correct iterator based on MS-level filter
             var enumerator = raw.GetFilteredScanEnumerator(" ");
 
-            // NB: replace with more robust strategy?
-            var output = ParseInput.OutputDirectory + "//" + Path.GetFileNameWithoutExtension(ParseInput.RawFilePath) + ".mzparquet";
-
+            ConfigureWriter(".mzparquet");
+            
             ParquetSerializerOptions opts = new ParquetSerializerOptions();
             opts.CompressionLevel = System.IO.Compression.CompressionLevel.Fastest;
             opts.CompressionMethod = Parquet.CompressionMethod.Zstd;
 
             var data = new List<MzParquet>();
 
+            //TODO Precursor tree
             // map last (msOrder - 1) -> scan number (e.g. mapping precursors)
             // note, this assumes time dependence of MS1 -> MS2 -> MSN
             var last_scans = new Dictionary<int, uint>();
@@ -63,6 +63,7 @@ public override void Write(IRawDataPlus raw, int firstScanNumber, int lastScanNu
             {
                 var scanFilter = raw.GetFilterForScanNumber(scanNumber);
 
+                //TODO Centroiding if centroidStream is not available
                 CentroidStream centroidStream = new CentroidStream();
 
                 // Pull out m/z and intensity values
@@ -170,7 +171,7 @@ public override void Write(IRawDataPlus raw, int firstScanNumber, int lastScanNu
                 //   present in the same row group (critical property of mzparquet)
                 if (data.Count >= 1_048_576)
                 {
-                    var task = ParquetSerializer.SerializeAsync(data, output, opts);
+                    var task = ParquetSerializer.SerializeAsync(data, Writer.BaseStream, opts);
                     task.Wait();
                     opts.Append = true;
                     data.Clear();
@@ -182,7 +183,7 @@ public override void Write(IRawDataPlus raw, int firstScanNumber, int lastScanNu
             // serialize any remaining ions into the final row group
             if (data.Count > 0)
             {
-                var task = ParquetSerializer.SerializeAsync(data, output, opts);
+                var task = ParquetSerializer.SerializeAsync(data, Writer.BaseStream, opts);
                 task.Wait();
                 Log.Info("writing row group");
             }

Writer/SpectrumWriter.cs

@@ -68,42 +68,27 @@ protected void ConfigureWriter(string extension)
                 return;
             }
 
-            if (ParseInput.OutputFile == null)
+            var fileName = NormalizeFileName(ParseInput.OutputFile, extension, ParseInput.Gzip);
+            if (ParseInput.OutputFormat == OutputFormat.Parquet)
             {
-                var fullExtension = ParseInput.Gzip ? extension + ".gz" : extension;
-                if (!ParseInput.Gzip || ParseInput.OutputFormat == OutputFormat.IndexMzML)
-                {
-                    Writer = File.CreateText(ParseInput.OutputDirectory + "//" +
-                                             ParseInput.RawFileNameWithoutExtension +
-                                             extension);
-                }
-                else
-                {
-                    var fileStream = File.Create(ParseInput.OutputDirectory + "//" +
-                                                 ParseInput.RawFileNameWithoutExtension + fullExtension);
-                    var compress = new GZipStream(fileStream, CompressionMode.Compress);
-                    Writer = new StreamWriter(compress);
-                }
+                Writer = new StreamWriter(File.Create(fileName));
+            }
+            else if (!ParseInput.Gzip || ParseInput.OutputFormat == OutputFormat.IndexMzML)
+            {
+                Writer = File.CreateText(fileName);
             }
             else
             {
-                var fileName = NormalizeFileName(ParseInput.OutputFile, extension, ParseInput.Gzip);
-                if (!ParseInput.Gzip || ParseInput.OutputFormat == OutputFormat.IndexMzML)
-                {
-                    Writer = File.CreateText(fileName);
-                }
-                else
-                {
-                    var fileStream = File.Create(fileName);
-                    var compress = new GZipStream(fileStream, CompressionMode.Compress);
-                    Writer = new StreamWriter(compress);
-                }
+                var fileStream = File.Create(fileName);
+                var compress = new GZipStream(fileStream, CompressionMode.Compress);
+                Writer = new StreamWriter(compress);
             }
+            
         }
 
         private string NormalizeFileName(string outputFile, string extension, bool gzip)
         {
-            string result = outputFile;
+            string result = outputFile == null ? Path.Combine(ParseInput.OutputDirectory, ParseInput.RawFileNameWithoutExtension) : outputFile;
             string tail = "";
 
             string[] extensions;