content: /*************************************************************************
  *                                                                        *
  *  This file is part of the 20n/act project.                             *
  *  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
  *  Copyright (C) 2017 20n Labs, Inc.                                     *
  *                                                                        *
  *  Please direct all queries to act@20n.com.                             *
  *                                                                        *
  *  This program is free software: you can redistribute it and/or modify  *
  *  it under the terms of the GNU General Public License as published by  *
  *  the Free Software Foundation, either version 3 of the License, or     *
  *  (at your option) any later version.                                   *
  *                                                                        *
  *  This program is distributed in the hope that it will be useful,       *
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  *  GNU General Public License for more details.                          *
  *                                                                        *
  *  You should have received a copy of the GNU General Public License     *
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  *                                                                        *
  *************************************************************************/
  
  package com.act.lcms;
  
  import org.apache.commons.io.input.ReaderInputStream;
  import org.apache.commons.lang3.tuple.Pair;
  import org.w3c.dom.Document;
  
  import javax.xml.parsers.DocumentBuilder;
  import javax.xml.parsers.DocumentBuilderFactory;
  import javax.xml.parsers.ParserConfigurationException;
  import javax.xml.stream.XMLEventReader;
  import javax.xml.stream.XMLEventWriter;
  import javax.xml.stream.XMLInputFactory;
  import javax.xml.stream.XMLOutputFactory;
  import javax.xml.stream.XMLStreamException;
  import javax.xml.stream.events.XMLEvent;
  import javax.xml.xpath.XPath;
  import javax.xml.xpath.XPathConstants;
  import javax.xml.xpath.XPathException;
  import javax.xml.xpath.XPathFactory;
  import java.io.FileInputStream;
  import java.io.StringReader;
  import java.io.StringWriter;
  import java.nio.ByteBuffer;
  import java.nio.ByteOrder;
  import java.util.ArrayList;
  import java.util.Base64;
  import java.util.Iterator;
  import java.util.List;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  import java.io.IOException;
  
  
  public abstract class MzMLParser<S> {
    public static final String SPECTRUM_OBJECT_TAG = "spectrum";
    public static final String XML_PREAMBLE = "<?xml version=\"1.0\" encoding=\"utf-8\"?>";
  
    // Paths for data extraction.
    public static final String SPECTRUM_PATH_INDEX = "/spectrum/@index";
    public static final String SPECTRUM_PATH_ID = "/spectrum/@id";
    public static final String SPECTRUM_PATH_BASE_PEAK_MZ = "/spectrum/cvParam[@name='base peak m/z']/@value";
    public static final String SPECTRUM_PATH_BASE_PEAK_INTENSITY = "/spectrum/cvParam[@name='base peak intensity']/@value";
    public static final String SPECTRUM_PATH_SCAN_START_TIME =
        "/spectrum/scanList/scan/cvParam[@name='scan start time']/@value";
    public static final String SPECTRUM_PATH_SCAN_START_TIME_UNIT =
        "/spectrum/scanList/scan/cvParam[@name='scan start time']/@unitName";
    public static final String SPECTRUM_PATH_MZ_BINARY_DATA =
        "/spectrum/binaryDataArrayList/binaryDataArray[./cvParam/@name='m/z array']/binary/text()";
    public static final String SPECTRUM_PATH_INTENSITY_BINARY_DATA =
        "/spectrum/binaryDataArrayList/binaryDataArray[./cvParam/@name='intensity array']/binary/text()";
  
    public static final Pattern SPECTRUM_EXTRACTION_REGEX =
        Pattern.compile("function=(\\d+) *process=(\\d+) scan=(\\d+)");
  
  
    // XPathFactory is known to be non-thread-safe.
    protected static final ThreadLocal<XPathFactory> XPATH_FACTORY = new ThreadLocal<XPathFactory>() {
      @Override
      protected XPathFactory initialValue() {
        return XPathFactory.newInstance();
      }
    };
  
    /**
     * Helper function: builds an XML DocumentBuilderFactory that can be used repeatedly in this class.
     * <p>
     * TODO: move this to an XML utility class, as I'm sure we'll use it again some day.
     *
     * @return An XML DocumentBuilderFactory.
     * @throws ParserConfigurationException
     */
    public static DocumentBuilderFactory mkDocBuilderFactory() throws ParserConfigurationException {
      /* This factory must be configured within the context of a method call for exception handling.
       * TODO: can we work around this w/ dependency injection? */
      // from http://stackoverflow.com/questions/155101/make-documentbuilder-parse-ignore-dtd-references
      DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
      docFactory.setValidating(false);
      docFactory.setNamespaceAware(true);
      docFactory.setFeature("http://xml.org/sax/features/namespaces", false);
      docFactory.setFeature("http://xml.org/sax/features/validation", false);
      docFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false);
      docFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
      return docFactory;
    }
  
    protected static List<Double> base64ToDoubleList(String b64) {
      byte[] decodedBytes = Base64.getDecoder().decode(b64);
      ByteBuffer buf = ByteBuffer.wrap(decodedBytes).order(ByteOrder.LITTLE_ENDIAN);
      List<Double> values = new ArrayList<>(decodedBytes.length / 8);
      while (buf.hasRemaining()) {
        values.add(buf.getDouble());
      }
      return values;
    }
  
    // TODO: isn't there some library method for this?  The Interwebs seem to say there isn't...
    protected static <K, V> List<Pair<K, V>> zipLists(List<K> keys, List<V> vals) {
      if (keys.size() != vals.size()) {
        throw new RuntimeException(String.format("Mismatched list sizes: %d vs %d", keys.size(), vals.size()));
      }
      List<Pair<K, V>> res = new ArrayList<>(keys.size());
      Iterator<K> ki = keys.listIterator();
      Iterator<V> vi = vals.listIterator();
      while (ki.hasNext() && vi.hasNext()) { // Length check should ensure these are exhausted simultaneously.
        K k = ki.next();
        V v = vi.next();
        res.add(Pair.of(k, v));
      }
      return res;
    }
  
    public MzMLParser() {
    }
  
    protected XPathFactory getXPathFactory() {
      return XPATH_FACTORY.get();
    }
  
    public Iterator<S> getIterator(String inputFile)
        throws ParserConfigurationException, IOException, XMLStreamException {
      DocumentBuilderFactory docFactory = mkDocBuilderFactory();
      DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
  
      final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
      final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
  
      return new Iterator<S>() {
        boolean inEntry = false;
  
        XMLEventReader xr = xmlInputFactory.createXMLEventReader(new FileInputStream(inputFile), "utf-8");
        // TODO: is the use of the XML version/encoding tag definitely necessary?
        StringWriter w = new StringWriter().append(XML_PREAMBLE).append("\n");
        XMLEventWriter xw = xmlOutputFactory.createXMLEventWriter(w);
  
        S next = null;
  
        /* Because we're handling the XML as a stream, we can only determine whether we have another Spectrum to return
         * by attempting to parse the next one.  `this.next()` reads
         */
        private S getNextSpectrum() {
          S spectrum = null;
          if (xr == null || !xr.hasNext()) {
            return null;
          }
  
          try {
            while (xr.hasNext()) {
              XMLEvent e = xr.nextEvent();
              if (!inEntry && e.isStartElement() &&
                  e.asStartElement().getName().getLocalPart().equals((SPECTRUM_OBJECT_TAG))) {
                xw.add(e);
                inEntry = true;
              } else if (e.isEndElement() && e.asEndElement().getName().getLocalPart().equals(SPECTRUM_OBJECT_TAG)) {
                xw.add(e);
                xw.flush();
                /* TODO: the XMLOutputFactory docs don't make it clear if/how events can be written directly into a new
                 * document structure, so we incur the cost of extracting each spectrum entry, serializing it, and
                 * re-reading it into its own document so it can be handled by XPath.  Master this strange corner of the
                 * Java ecosystem and get rid of <></>his doc -> string -> doc conversion. */
                Document doc = docBuilder.parse(new ReaderInputStream(new StringReader(w.toString())));
                spectrum = handleSpectrumEntry(doc);
                xw.close();
                /* Note: this can also be accomplished with `w.getBuffer().setLength(0);`, but using a new event writer
                 * seems safer. */
                w = new StringWriter();
                w.append(XML_PREAMBLE).append("\n");
                xw = xmlOutputFactory.createXMLEventWriter(w);
                inEntry = false;
                // Don't stop parsing if handleSpectrumEntry didn't like this spectrum document.
                if (spectrum != null) {
                  break;
                }
              } else if (inEntry) {
                // Add this element if we're in an entry
                xw.add(e);
              }
            }
  
            // We've reached the end of the document; close the reader to show that we're done.
            if (!xr.hasNext()) {
              xr.close();
              xr = null;
            }
          } catch (Exception e) {
            // TODO: do better.  We seem to run into this sort of thing with Iterators a lot...
            throw new RuntimeException(e);
          }
  
          return spectrum;
        }
  
        private S tryParseNext() {
          // Fail the attempt if the reader is closed.
          if (xr == null || !xr.hasNext()) {
            return null;
          }
  
          // No checks on whether we already have a spectrum stored: we expect the callers to do that.
          return getNextSpectrum();
        }
  
        @Override
        public boolean hasNext() {
          // Prime the pump if the iterator doesn't have a value stored yet.
          if (this.next == null) {
            this.next = tryParseNext();
          }
  
          // If we have an entry waiting, return true; otherwise read the next entry and return true if successful.
          return this.next != null;
        }
  
        @Override
        public S next() {
          // Prime the pump like we do in hasNext().
          if (this.next == null) {
            this.next = tryParseNext();
          }
  
          // Take available spectrum and return it.
          S res = this.next;
          /* Advance to the next element immediately, making next() do the heavy lifting most of the time.  Otherwise,
           * the parsing will resume on hasNext(), which seems like it ought to be a light-weight operation. */
          this.next = tryParseNext();
  
          return res;
        }
  
      };
    }
  
    public List<S> parse(String inputFile)
        throws ParserConfigurationException, IOException, XMLStreamException {
      List<S> spectra = new ArrayList<>();
      Iterator<S> iter = this.getIterator(inputFile);
      while (iter.hasNext()) {
        spectra.add(iter.next());
      }
  
      return spectra;
    }
  
    protected abstract S handleSpectrumEntry(Document doc) throws XPathException;
  }
  