content: /*
   *
   * Copyright 2008 The New York Times Company
   *
   * Licensed under the Apache License, Version 2.0 (the "License"); you may not
   * use this file except in compliance with the License. You may obtain a copy of
   * the License at
   *
   * http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   * License for the specific language governing permissions and limitations under
   * the License.
   *
   * This copy is modified from the original. Per the license,
   * this notice serves to indicate it is a changed file.
   */
  package de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.uima.custom.readers.util.nyt;
  
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.w3c.dom.Document;
  import org.w3c.dom.NamedNodeMap;
  import org.w3c.dom.Node;
  import org.w3c.dom.NodeList;
  import org.xml.sax.SAXException;
  
  import javax.xml.parsers.DocumentBuilder;
  import javax.xml.parsers.DocumentBuilderFactory;
  import javax.xml.parsers.ParserConfigurationException;
  import java.io.*;
  import java.net.MalformedURLException;
  import java.net.URL;
  import java.text.DateFormat;
  import java.text.ParseException;
  import java.text.SimpleDateFormat;
  import java.util.ArrayList;
  import java.util.Date;
  import java.util.List;
  
  /**
   * NYTCorpusDocumentParser <BR>
   * Created: Jun 17, 2008 <BR>
   * Author: Evan Sandhaus (sandhes@nytimes.com)<BR>
   * <P>
   * Class for parsing New York Times articles from NITF files.
   * <P>
   *
   * The original version contained a possible memory leak:
   * the BufferedReader object was not closed.
   *
   * Additionally, an API to read from an {@link InputStream} was
   * added.
   *
   * @author Evan Sandhaus
   *
   */
  public class NYTCorpusDocumentParser {
  
  	private Logger logger = LoggerFactory.getLogger(NYTCorpusDocumentParser.class);
  
  	/** NITF Constant */
  	private static final String CORRECTION_TEXT = "correction_text";
  
  	/** NITF Constant */
  	private static final String SERIES_NAME_TAG = "series.name";
  
  	/** NITF Constant */
  	private static final DateFormat format = new SimpleDateFormat(
  			"yyyyMMdd'T'HHmmss");
  
  	/** NITF Constant */
  	private static final String TAGLINE_TAG = "tagline";
  
  	/** NITF Constant */
  	private static final String CLASS_ATTRIBUTE = "class";
  
  	/** NITF Constant */
  	private static final String CLASSIFIER_TAG = "classifier";
  
  	/** NITF Constant */
  	private static final String HL2_TAG = "hl2";
  
  	/** NITF Constant */
  	private static final String BLOCK_TAG = "block";
  
  	/** NITF Constant */
  	private static final String ABSTRACT_TAG = "abstract";
  
  	/** NITF Constant */
  	private static final String DATELINE_TAG = "dateline";
  
  	/** NITF Constant */
  	private static final String BYLINE_TAG = "byline";
  
  	/** NITF Constant */
  	private static final String HEDLINE_TAG = "hedline";
  
  	/** NITF Constant */
  	private static final String BODY_END_TAG = "body.end";
  
  	/** NITF Constant */
  	private static final String BODY_CONTENT_TAG = "body.content";
  
  	/** NITF Constant */
  	private static final String BODY_HEAD_TAG = "body.head";
  
  	/** NITF Constant */
  	private static final String TYPE_ATTRIBUTE = "type";
  
  	/** NITF Constant */
  	private static final String NAME_ATTRIBUTE = "name";
  
  	/** NITF Constant */
  	private static final String ITEM_LENGTH_ATTRIBUTE = "item-length";
  
  	/** NITF Constant */
  	private static final String EX_REF_ATTRIBUTE = "ex-ref";
  
  	/** NITF Constant */
  	private static final String SLUG_ATTRIBUTE = "slug";
  
  	/** NITF Constant */
  	private static final String PRINT_SECTION_ATTRIBUTE = "print_section";
  
  	/** NITF Constant */
  	private static final String PRINT_PAGE_NUMBER_ATTRIBUTE = "print_page_number";
  
  	/** NITF Constant */
  	private static final String DSK_ATTRIBUTE = "dsk";
  
  	/** NITF Constant */
  	private static final String HL1_TAG = "hl1";
  
  	/** NITF Constant */
  	private static final String CONTENT_ATTRIBUTE = "content";
  
  	/** NITF Constant */
  	private static final String DOC_ID_TAG = "doc-id";
  
  	/** NITF Constant */
  	private static final String IDENTIFIED_CONTENT_TAG = "identified-content";
  
  	/** NITF Constant */
  	private static final String ID_STRING_ATTRIBUTE = "id-string";
  
  	/** NITF Constant */
  	private static final String LOCATION_TAG = "location";
  
  	/** NITF Constant */
  	private static final String OBJECT_TITLE_TAG = "object.title";
  
  	/** NITF Constant */
  	private static final String PERSON_TAG = "person";
  
  	/** NITF Constant */
  	private static final String PUBDATA_TAG = "pubdata";
  
  	/** NITF Constant */
  	private static final String DOCDATA_TAG = "docdata";
  
  	/** NITF Constant */
  	private static final String META_TAG = "meta";
  
  	/** NITF Constant */
  	private static final String BODY_TAG = "body";
  
  	/** NITF Constant */
  	private static final String HEAD_TAG = "head";
  
  	/** NITF Constant */
  	private static final String NITF_TAG = "nitf";
  
  	/** NITF Constant */
  	private static final String ALTERNATE_URL_ATTRIBUTE = "alternate_url";
  
  	/** NITF Constant */
  	private static final String AUTHOR_INFO_ATTRIBUTE = "author_info";
  
  	/** NITF Constant */
  	private static final String DESCRIPTOR_ATTRIBUTE = "descriptor";
  
  	/** NITF Constant */
  	private static final String FULL_TEXT_ATTRIBUTE = "full_text";
  
  	/** NITF Constant */
  	private static final String INDEXING_SERVICE_ATTRIBUTE = "indexing_service";
  
  	/** NITF Constant */
  	private static final String LEAD_PARAGRAPH_ATTRIBUTE = "lead_paragraph";
  
  	/** NITF Constant */
  	private static final String NORMALIZED_BYLINE_ATTRIBUTE = "normalized_byline";
  
  	/** NITF Constant */
  	private static final String ONLINE_HEADLINE_ATTRIBUTE = "online_headline";
  
  	/** NITF Constant */
  	private static final String ONLINE_LEAD_PARAGRAPH_ATTRIBUTE = "online_lead_paragraph";
  
  	/** NITF Constant */
  	private static final String ONLINE_PRODUCER_ATTRIBUTE = "online_producer";
  
  	/** NITF Constant */
  	private static final String ONLINE_SECTIONS_ATTRIBUTE = "online_sections";
  
  	/** NITF Constant */
  	private static final String ORGANIZATION_TAG = "org";
  
  	/** NITF Constant */
  	private static final String P_TAG = "p";
  
  	/** NITF Constant */
  	private static final String PRINT_BYLINE_ATTRIBUTE = "print_byline";
  
  	/** NITF Constant */
  	private static final String PRINT_COLUMN_ATTRIBUTE = "print_column";
  
  	/** NITF Constant */
  	private static final String PUBLICATION_DAY_OF_MONTH_ATTRIBUTE = "publication_day_of_month";
  
  	/** NITF Constant */
  	private static final String PUBLICATION_MONTH_ATTRIBUTE = "publication_month";
  
  	/** NITF Constant */
  	private static final String PUBLICATION_YEAR_ATTRIBUTE = "publication_year";
  
  	/** NITF Constant */
  	private static final String PULICATION_DAY_OF_WEEK_ATTRIBUTE = "publication_day_of_week";
  
  	/** NITF Constant */
  	private static final String SERIES_NAME_ATTRIBUTE = "series_name";
  
  	/** NITF Constant */
  	private static final String SERIES_TAG = "series";
  
  	/** NITF Constant */
  	private static final String TAXONOMIC_CLASSIFIER_ATTRIBUTE = "taxonomic_classifier";
  
  	/** NITF Constant */
  	private static final String BANNER_ATTRIBUTE = "banner";
  
  	/** NITF Constant */
  	private static final String CORRECTION_DATE_ATTRIBUTE = "correction_date";
  
  	/** NITF Constant */
  	private static final String FEATURE_PAGE_ATTRIBUTE = "feature_page";
  
  	/** NITF Constant */
  	private static final String COLUMN_NAME_ATTRIBUTE = "column_name";
  
  	/** NITF Constant */
  	private static final String TYPES_OF_MATERIAL_ATTRIBUTE = "types_of_material";
  
  	/** NITF Constant */
  	private static final String NAMES_ATTRIBUTE = "names";
  
  	/** NITF Constant */
  	private static final String BIOGRAPHICAL_CATEGORIES_ATTRIBUTE = "biographical_categories";
  
  	/** NITF Constant */
  	public static final String DATE_PUBLICATION_ATTRIBUTE = "date.publication";
  
  	/** NITF Constant */
  	private static final String GENERAL_DESCRIPTOR_ATTRIBUTE = "general_descriptor";
  
  	public NYTCorpusDocument fromByteArray(byte[] bytes, boolean validating) throws ParserConfigurationException, SAXException, ParseException, IOException {
  		return this.parseNYTCorpusDocumentFromFile(new ByteArrayInputStream(bytes), validating);
  	}
  
  	/**
  	 * Parse an New York Times Document from a file.
  	 *
  	 * @param file
  	 *            The file from which to parse the document.
  	 * @param validating
  	 *            True if the file is to be validated against the nitf DTD and
  	 *            false if it is not. It is recommended that validation be
  	 *            disabled, as all documents in the corpus have previously been
  	 *            validated against the NITF DTD.
  	 * @return The parsed document, or null if an error occurs.
  	 */
  	public NYTCorpusDocument parseNYTCorpusDocumentFromFile(File file,
  																			   boolean validating) throws IOException, ParseException, ParserConfigurationException, SAXException {
  
  		Document document = null;
  		if (validating) {
  			document = loadValidating(file);
  		} else {
  			document = loadNonValidating(file);
  		}
  		return parseNYTCorpusDocumentFromDOMDocument(file, document);
  	}
  
  	public NYTCorpusDocument parseNYTCorpusDocumentFromDOMDocument(
  			File file, Document document) throws MalformedURLException, ParseException {
  		NYTCorpusDocument ldcDocument = new NYTCorpusDocument();
  		ldcDocument.setSourceFile(file);
  		NodeList children = document.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(NITF_TAG)) {
  				handleNITFNode(child, ldcDocument);
  			}
  		}
  
  		return ldcDocument;
  	}
  
  	/**
  	 * Parse an New York Times Document from an {@link InputStream}.
  	 *
  	 * @param is
  	 *            The {@link InputStream} from which to parse the document.
  	 * @param validating
  	 *            True if the file is to be validated against the nitf DTD and
  	 *            false if it is not. It is recommended that validation be
  	 *            disabled, as all documents in the corpus have previously been
  	 *            validated against the NITF DTD.
  	 * @return The parsed document, or null if an error occurs.
  	 */
  	public NYTCorpusDocument parseNYTCorpusDocumentFromFile(InputStream is, boolean validating) throws ParserConfigurationException, SAXException, IOException, ParseException {
  
  		Document document = null;
  		if (validating) {
  			document = loadValidating(is);
  		} else {
  			document = loadNonValidating(is);
  		}
  		return parseNYTCorpusDocumentFromDOMDocument(is, document);
  	}
  
  	public NYTCorpusDocument parseNYTCorpusDocumentFromDOMDocument(InputStream is, Document document) throws MalformedURLException, ParseException {
  		NYTCorpusDocument ldcDocument = new NYTCorpusDocument();
  		ldcDocument.setSourceFile(null);
  		NodeList children = document.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(NITF_TAG)) {
  				handleNITFNode(child, ldcDocument);
  			}
  		}
  
  		return ldcDocument;
  	}
  
  	/**
  	 * Parse the specified file into a DOM Document.
  	 *
  	 * @param
  	 * @return The parsed DOM Document or null if an error occurs.
  	 */
  	private Document loadValidating(InputStream is) throws SAXException, IOException, ParserConfigurationException {
  		try {
  			return getDOMObject(is, true);
  		} catch (SAXException | IOException | ParserConfigurationException e) {
  			logger.error("Error parsing digital document from nitf inputstream.");
  			throw e;
  		}
  	}
  
  	/**
  	 * Load a document without validating it. Since instructing the java.xml
  	 * libraries to do this does not actually disable validation, this method
  	 * disables validation by removing the doctype declaration from the XML
  	 * document before it is parsed.
  	 *
  	 * @param
  	 * @return The parsed document or null if an error occurs.
  	 */
  	private Document loadNonValidating(InputStream is) throws IOException, ParserConfigurationException, SAXException {
  		Document document;
  		StringBuffer sb = new StringBuffer();
  		BufferedReader in = null;
  		try {
  			in = new BufferedReader(new InputStreamReader(is, "UTF8"));
  			String line = null;
  			while ((line = in.readLine()) != null) {
  				sb.append(line + "\n");
  			}
  			String xmlData = sb.toString();
  			xmlData = xmlData.replace("<!DOCTYPE nitf " + "SYSTEM \"http://www.nitf.org/" + "IPTC/NITF/3.3/specification/dtd/nitf-3-3.dtd\">", "");
  			document = parseStringToDOM(xmlData, "UTF-8");
  			return document;
  		} catch (UnsupportedEncodingException e) {
  			logger.error("Error loading inputstream.");
  			throw e;
  		} catch (IOException e) {
  			logger.error("Error loading file inputstream.");
  			throw e;
  		} finally {
  			if (in != null) {
  				in.close();
  			}
  
  		}
  	}
  
  	/**
  	 * Parse a string to a DOM document.
  	 *
  	 * @param s
  	 *            A string containing an XML document.
  	 *
  	 * @return The DOM document if it can be parsed, or null otherwise.
  	 */
  	private Document parseStringToDOM(String s, String encoding) throws IOException, ParserConfigurationException, SAXException {
  		try {
  			DocumentBuilderFactory factory = DocumentBuilderFactory
  					.newInstance();
  			factory.setValidating(false);
  			InputStream is = new ByteArrayInputStream(s.getBytes(encoding));
  			Document doc = factory.newDocumentBuilder().parse(is);
  			return doc;
  		} catch (SAXException | ParserConfigurationException | IOException e) {
  			logger.error("Exception processing string '{}'."+s);
  			throw e;
  		}
  	}
  
  	/**
  	 * Parse an {@link InputStream} containing an XML document, into a DOM object.
  	 *
  	 * @param is
  	 *            An {@link InputStream} representing an xml file.
  	 * @param validating
  	 *            True iff validating should be turned on.
  	 * @return A DOM Object containing a parsed XML document or a null value if
  	 *         there is an error in parsing.
  	 * @throws ParserConfigurationException
  	 * @throws IOException
  	 * @throws SAXException
  	 */
  	private Document getDOMObject(InputStream is, boolean validating)
  			throws SAXException, IOException, ParserConfigurationException {
  
  		// Create a builder factory
  
  		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
  		if (!validating) {
  			factory.setValidating(validating);
  			factory.setSchema(null);
  			factory.setNamespaceAware(false);
  		}
  
  		DocumentBuilder builder = factory.newDocumentBuilder();
  		// Create the builder and parse the file
  		Document doc = builder.parse(is);
  		return doc;
  	}
  
  	private void handleNITFNode(Node node, NYTCorpusDocument ldcDocument) {
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(HEAD_TAG)) {
  				handleHeadNode(child, ldcDocument);
  			} else if (name.equals(BODY_TAG)) {
  				handleBodyNode(child, ldcDocument);
  			}
  		}
  	}
  
  	private void handleBodyNode(Node node, NYTCorpusDocument ldcDocument) {
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(BODY_HEAD_TAG)) {
  				handleBodyHead(child, ldcDocument);
  			} else if (name.equals(BODY_CONTENT_TAG)) {
  				handleBodyContent(child, ldcDocument);
  			} else if (name.equals(BODY_END_TAG)) {
  				handleBodyEnd(child, ldcDocument);
  			}
  		}
  	}
  
  	private void handleBodyHead(Node node, NYTCorpusDocument ldcDocument) {
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(DATELINE_TAG)) {
  				handleDatelineNode(ldcDocument, child);
  			} else if (name.equals(ABSTRACT_TAG)) {
  				handleAbstractNode(child, ldcDocument);
  			} else if (name.equals(BYLINE_TAG)) {
  				handleBylineNode(child, ldcDocument);
  			} else if (name.equals(HEDLINE_TAG)) {
  				handleHeadlineNode(child, ldcDocument);
  			}
  		}
  	}
  
  	private void handleDatelineNode(NYTCorpusDocument ldcDocument, Node child) {
  		String datelineString = getAllText(child);
  		ldcDocument.setDateline(datelineString.trim());
  	}
  
  	private void handleAbstractNode(Node node, NYTCorpusDocument ldcDocument) {
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(P_TAG)) {
  				String abstractText = getAllText(child).trim();
  				ldcDocument.setArticleAbstract(abstractText);
  			}
  		}
  	}
  
  	private void handleBylineNode(Node node, NYTCorpusDocument ldcDocument) {
  		String classAttribute = getAttributeValue(node, CLASS_ATTRIBUTE);
  		if (classAttribute != null) {
  			String text = getAllText(node).trim();
  			if (classAttribute.equals(NORMALIZED_BYLINE_ATTRIBUTE)) {
  				ldcDocument.setNormalizedByline(text);
  			} else if (classAttribute.equals(PRINT_BYLINE_ATTRIBUTE)) {
  				ldcDocument.setByline(text);
  			}
  		}
  	}
  
  	private void handleHeadlineNode(Node node, NYTCorpusDocument ldcDocument) {
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			String text = getAllText(child).trim();
  			if (name.equals(HL1_TAG)) {
  				ldcDocument.setHeadline(text);
  			} else if (name.equals(HL2_TAG)) {
  				String classAttribute = getAttributeValue(child,
  						CLASS_ATTRIBUTE);
  				if (classAttribute != null
  						&& classAttribute.equals(ONLINE_HEADLINE_ATTRIBUTE)) {
  					ldcDocument.setOnlineHeadline(text);
  				}
  			}
  		}
  	}
  
  	private void handleBodyContent(Node node, NYTCorpusDocument ldcDocument) {
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(BLOCK_TAG)) {
  				handleBlockNode(child, ldcDocument);
  			}
  		}
  	}
  
  	private void handleBlockNode(Node node, NYTCorpusDocument ldcDocument) {
  		String classAttribute = getAttributeValue(node, CLASS_ATTRIBUTE);
  		if (classAttribute != null) {
  			if (classAttribute.equals(ONLINE_LEAD_PARAGRAPH_ATTRIBUTE)) {
  				ldcDocument.setOnlineLeadParagraph(parseBlock(node));
  			} else if (classAttribute.equals(LEAD_PARAGRAPH_ATTRIBUTE)) {
  				ldcDocument.setLeadParagraph(parseBlock(node));
  			} else if (classAttribute.equals(FULL_TEXT_ATTRIBUTE)) {
  				ldcDocument.setBody(parseBlock(node));
  			} else if (classAttribute.equals(CORRECTION_TEXT)) {
  				ldcDocument.setCorrectionText(parseBlock(node));
  			}
  		}
  	}
  
  	private void handleBodyEnd(Node node, NYTCorpusDocument ldcDocument) {
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(TAGLINE_TAG)) {
  				String classAttribute = getAttributeValue(child,
  						CLASS_ATTRIBUTE);
  				if (classAttribute != null
  						&& classAttribute.equals(AUTHOR_INFO_ATTRIBUTE)) {
  					String text = getAllText(child);
  					ldcDocument.setAuthorBiography(text);
  				}
  			}
  		}
  	}
  
  	private void handleHeadNode(Node node, NYTCorpusDocument ldcDocument)  {
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(META_TAG)) {
  				handleMetaNode(child, ldcDocument);
  			} else if (name.equals(DOCDATA_TAG)) {
  				handleDocdataNode(child, ldcDocument);
  			} else if (name.equals(PUBDATA_TAG)) {
  				handlePubdata(child, ldcDocument);
  			}
  		}
  	}
  
  	private void handleDocdataNode(Node node, NYTCorpusDocument ldcDocument) {
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(DOC_ID_TAG)) {
  				handleDocumentIdNode(ldcDocument, child);
  			} else if (name.equals(SERIES_TAG)) {
  				ldcDocument
  						.setKicker(getAttributeValue(child, SERIES_NAME_TAG));
  			} else if (name.equals(IDENTIFIED_CONTENT_TAG)) {
  				handleIdentifiedContent(child, ldcDocument);
  			}
  		}
  	}
  
  	private void handlePubdata(Node node, NYTCorpusDocument ldcDocument) {
  		String publicationDateString = getAttributeValue(node, DATE_PUBLICATION_ATTRIBUTE);
  		if (publicationDateString != null) {
  			try {
  				Date date = format.parse(publicationDateString);
  				ldcDocument.setPublicationDate(date);
  			} catch (ParseException | NumberFormatException e) {
  				logger.warn("Error parsing date from string '{}' in file '{}'.", publicationDateString, ldcDocument.getSourceFile());
  				//Don't throw the exception back because this will prevent creating the file
  				//throw new ParseException("Error parsing date from string '"+ publicationDateString+"' in file '"+ldcDocument.getSourceFile()+"'. Error Message: "+e.getMessage(), e.getErrorOffset());
  			}
  
  		}
  
  		String urlString = getAttributeValue(node, EX_REF_ATTRIBUTE);
  		if (urlString != null) {
  			try {
  				URL url = new URL(urlString);
  				ldcDocument.setUrl(url);
  			} catch (MalformedURLException e) {
  				logger.error("Error parsing url from string '{}' in file '{}'.", urlString, ldcDocument.getSourceFile());
  				//Also don't throw exception here, just catch and log, its not so important if this is not properly parsed.
  				//throw new MalformedURLException("Error parsing url from string '"+urlString+"' in file '"+ldcDocument.getSourceFile()+"'. Error Message: "+e.getMessage());
  			}
  		}
  
  		String wordCountString = getAttributeValue(node, ITEM_LENGTH_ATTRIBUTE);
  		if (wordCountString != null) {
  			try {
  				Integer wordCount = Integer.parseInt(wordCountString);
  				ldcDocument.setWordCount(wordCount);
  			} catch (NumberFormatException e) {
  				logger.error("Error parsing integer from string '{}' in file '{}'.", wordCountString, ldcDocument.getSourceFile() );
  				//Don't throw the exception back because this will prevent creating the file
  				//throw new NumberFormatException("Error parsing integer from string '"+wordCountString+"' in file '"+ldcDocument.getSourceFile()+"'. Error Message: "+e.getMessage());
  			}
  		}
  
  		String creatorString = getAttributeValue(node, NAME_ATTRIBUTE);
  		if (creatorString != null) {
  			ldcDocument.setCredit(creatorString);
  		}
  	}
  
  	private void handleIdentifiedContent(Node node,
  										 NYTCorpusDocument ldcDocument) {
  
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			String value = getAllText(child).trim();
  			String classAttribute = getAttributeValue(child, CLASS_ATTRIBUTE);
  
  			if (name.equals(CLASSIFIER_TAG)) {
  				String typeAttribute = getAttributeValue(child, TYPE_ATTRIBUTE);
  				if (classAttribute.equals(INDEXING_SERVICE_ATTRIBUTE)) {
  					if (typeAttribute.equals(DESCRIPTOR_ATTRIBUTE)) {
  						ldcDocument.getDescriptors().add(value);
  					} else if (typeAttribute
  							.equals(BIOGRAPHICAL_CATEGORIES_ATTRIBUTE)) {
  						ldcDocument.getBiographicalCategories().add(value);
  					} else if (typeAttribute.equals(NAMES_ATTRIBUTE)) {
  						ldcDocument.getNames().add(value);
  					}
  				} else if (classAttribute.equals(ONLINE_PRODUCER_ATTRIBUTE)) {
  					if (typeAttribute.equals(DESCRIPTOR_ATTRIBUTE)) {
  						ldcDocument.getOnlineDescriptors().add(value);
  					} else if (typeAttribute
  							.equals(GENERAL_DESCRIPTOR_ATTRIBUTE)) {
  						ldcDocument.getGeneralOnlineDescriptors().add(value);
  					} else if (typeAttribute
  							.equals(TAXONOMIC_CLASSIFIER_ATTRIBUTE)) {
  						ldcDocument.getTaxonomicClassifiers().add(value);
  					} else if (typeAttribute
  							.equals(TYPES_OF_MATERIAL_ATTRIBUTE)) {
  						ldcDocument.getTypesOfMaterial().add(value);
  					}
  				}
  			} else if (name.equals(LOCATION_TAG)) {
  				if (classAttribute.equals(INDEXING_SERVICE_ATTRIBUTE)) {
  					ldcDocument.getLocations().add(value);
  				} else if (classAttribute.equals(ONLINE_PRODUCER_ATTRIBUTE)) {
  					ldcDocument.getOnlineLocations().add(value);
  				}
  			} else if (name.equals(OBJECT_TITLE_TAG)) {
  				if (classAttribute.equals(INDEXING_SERVICE_ATTRIBUTE)) {
  					ldcDocument.getTitles().add(value);
  				} else if (classAttribute.equals(ONLINE_PRODUCER_ATTRIBUTE)) {
  					ldcDocument.getOnlineTitles().add(value);
  				}
  			} else if (name.equals(ORGANIZATION_TAG)) {
  				if (classAttribute.equals(INDEXING_SERVICE_ATTRIBUTE)) {
  					ldcDocument.getOrganizations().add(value);
  				} else if (classAttribute.equals(ONLINE_PRODUCER_ATTRIBUTE)) {
  					ldcDocument.getOnlineOrganizations().add(value);
  				}
  			} else if (name.equals(PERSON_TAG)) {
  				if (classAttribute.equals(INDEXING_SERVICE_ATTRIBUTE)) {
  					ldcDocument.getPeople().add(value);
  				} else if (classAttribute.equals(ONLINE_PRODUCER_ATTRIBUTE)) {
  					ldcDocument.getOnlinePeople().add(value);
  				}
  			}
  		}
  	}
  
  	private void handleDocumentIdNode(NYTCorpusDocument ldcDocument, Node child) throws NumberFormatException{
  		String docIdString = getAttributeValue(child, ID_STRING_ATTRIBUTE);
  		if (docIdString != null) {
  			try {
  				ldcDocument.setGuid(Integer.parseInt(docIdString));
  			} catch (NumberFormatException e) {
  				logger.error("Error parsing long from string '{}' in file '{}'.", docIdString, ldcDocument.getSourceFile());
  				throw e;
  			}
  		}
  	}
  
  	private void handleMetaNode(Node node, NYTCorpusDocument ldcDocument) {
  		NamedNodeMap attributes = node.getAttributes();
  
  		if (attributes.getNamedItem(NAME_ATTRIBUTE) == null
  				|| attributes.getNamedItem(CONTENT_ATTRIBUTE) == null) {
  			return;
  		}
  
  		String name = attributes.getNamedItem(NAME_ATTRIBUTE).getNodeValue();
  		String content = attributes.getNamedItem(CONTENT_ATTRIBUTE)
  				.getNodeValue();
  		try {
  			if (name.equals(DSK_ATTRIBUTE)) {
  				ldcDocument.setNewsDesk(content);
  			} else if (name.equals(ALTERNATE_URL_ATTRIBUTE)) {
  				ldcDocument.setAlternateURL((new URL(content)));
  			} else if (name.equals(ONLINE_SECTIONS_ATTRIBUTE)) {
  				ldcDocument.setOnlineSection(content);
  			} else if (name.equals(PRINT_PAGE_NUMBER_ATTRIBUTE)) {
  				ldcDocument.setPage(Integer.parseInt(content));
  			} else if (name.equals(PRINT_SECTION_ATTRIBUTE)) {
  				ldcDocument.setSection(content);
  			} else if (name.equals(SLUG_ATTRIBUTE)) {
  				ldcDocument.setSlug(content);
  			} else if (name.equals(PRINT_COLUMN_ATTRIBUTE)) {
  				ldcDocument.setColumnNumber(Integer.parseInt(content.trim()));
  			} else if (name.equals(BANNER_ATTRIBUTE)) {
  				ldcDocument.setBanner(content);
  			} else if (name.equals(CORRECTION_DATE_ATTRIBUTE)) {
  				ldcDocument.setCorrectionDate(format.parse(content));
  			} else if (name.equals(FEATURE_PAGE_ATTRIBUTE)) {
  				ldcDocument.setFeaturePage(content);
  			} else if (name.equals(COLUMN_NAME_ATTRIBUTE)) {
  				ldcDocument.setColumnName(content);
  			} else if (name.equals(SERIES_NAME_ATTRIBUTE)) {
  				ldcDocument.setSeriesName(content);
  			} else if (name.equals(PUBLICATION_DAY_OF_MONTH_ATTRIBUTE)) {
  				ldcDocument.setPublicationDayOfMonth(Integer.parseInt(content));
  			} else if (name.equals(PUBLICATION_MONTH_ATTRIBUTE)) {
  				ldcDocument.setPublicationMonth(Integer.parseInt(content));
  			} else if (name.equals(PUBLICATION_YEAR_ATTRIBUTE)) {
  				ldcDocument.setPublicationYear(Integer.parseInt(content));
  			} else if (name.equals(PULICATION_DAY_OF_WEEK_ATTRIBUTE)) {
  				ldcDocument.setDayOfWeek(content);
  			}
  
  			//Don't throw the exceptions, just catch them and log. Its not so critical to have all parsed for our task
  		} catch (MalformedURLException e) {
  			logger.error("Error parsing url from string '{}' in file '{}'.", content, ldcDocument.getSourceFile());
  			//throw e;
  		} catch (NumberFormatException e) {
  			logger.error("Error parsing integer from string '{}' in file '{}'.", content,ldcDocument.getSourceFile());
  			//throw e;
  		} catch (ParseException e) {
  			logger.error("Error parsing date from string '{}' in file '{}'.", content, ldcDocument.getSourceFile());
  			//throw e;
  		}
  	}
  
  	/**
  	 * Load a document without validating it. Since instructing the java.xml
  	 * libraries to do this does not actually disable validation, this method
  	 * disables validation by removing the doctype declaration from the XML
  	 * document before it is parsed.
  	 *
  	 * @param file
  	 *            The file to parse.
  	 * @return The parsed document or null if an error occurs.
  	 */
  	private Document loadNonValidating(File file) throws IOException, ParserConfigurationException, SAXException {
  		Document document;
  		StringBuffer sb = new StringBuffer();
  		try (BufferedReader in = new BufferedReader(new InputStreamReader(
  				new FileInputStream(file), "UTF8"));){
  			String line = null;
  			while ((line = in.readLine()) != null) {
  				sb.append(line + "\n");
  			}
  			String xmlData = sb.toString();
  			xmlData = xmlData.replace("<!DOCTYPE nitf "
  					+ "SYSTEM \"http://www.nitf.org/"
  					+ "IPTC/NITF/3.3/specification/dtd/nitf-3-3.dtd\">", "");
  			document = parseStringToDOM(xmlData, "UTF-8", file);
  			return document;
  		} catch (IOException | ParserConfigurationException | SAXException e) {
  			logger.error("Error loading file '{}'.", file);
  			throw e;
  		}
  	}
  
  	/**
  	 * Parse the specified file into a DOM Document.
  	 *
  	 * @param file
  	 *            The file to parse.
  	 * @return The parsed DOM Document or null if an error occurs.
  	 */
  	private Document loadValidating(File file) throws ParserConfigurationException, IOException, SAXException {
  		try {
  			return getDOMObject(file.getAbsolutePath(), true);
  		} catch (SAXException | IOException | ParserConfigurationException e) {
  			logger.error("Error parsing digital document from nitf file '{}'.", file);
  			throw e;
  		}
  	}
  
  	/**
  	 * Parse a string to a DOM document.
  	 *
  	 * @param s
  	 *            A string containing an XML document.
  	 *
  	 * @return The DOM document if it can be parsed, or null otherwise.
  	 */
  	private Document parseStringToDOM(String s, String encoding, File file) throws IOException, ParserConfigurationException, SAXException {
  		try {
  			DocumentBuilderFactory factory = DocumentBuilderFactory
  					.newInstance();
  			factory.setValidating(false);
  			InputStream is = new ByteArrayInputStream(s.getBytes(encoding));
  			Document doc = factory.newDocumentBuilder().parse(is);
  			return doc;
  		} catch (SAXException | ParserConfigurationException | IOException e) {
  			logger.error("Exception processing file '{}'.", file);
  			throw e;
  		}
  	}
  
  	/**
  	 * Parse a file containing an XML document, into a DOM object.
  	 *
  	 * @param filename
  	 *            A path to a valid file.
  	 * @param validating
  	 *            True iff validating should be turned on.
  	 * @return A DOM Object containing a parsed XML document or a null value if
  	 *         there is an error in parsing.
  	 * @throws ParserConfigurationException
  	 * @throws IOException
  	 * @throws SAXException
  	 */
  	private Document getDOMObject(String filename, boolean validating)
  			throws SAXException, IOException, ParserConfigurationException {
  
  		// Create a builder factory
  
  		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
  		if (!validating) {
  			factory.setValidating(validating);
  			factory.setSchema(null);
  			factory.setNamespaceAware(false);
  		}
  
  		DocumentBuilder builder = factory.newDocumentBuilder();
  		// Create the builder and parse the file
  		Document doc = builder.parse(new File(filename));
  		return doc;
  	}
  
  	private String parseBlock(Node node) {
  		StringBuffer sb = new StringBuffer();
  		NodeList children = node.getChildNodes();
  		for (int i = 0; i < children.getLength(); i++) {
  			Node child = children.item(i);
  			String name = child.getNodeName();
  			if (name.equals(P_TAG)) {
  				sb.append(getAllText(child).trim() + "\n");
  			}
  		}
  
  		if (sb.length() > 0) {
  			sb.setLength(sb.length() - 1);
  			String returnVal = sb.toString();
  			return returnVal.length() > 0 ? returnVal : null;
  		}
  		return null;
  	}
  
  	private String getAttributeValue(Node node, String attributeName) {
  		NamedNodeMap attributes = node.getAttributes();
  		if (attributes != null) {
  			Node attribute = attributes.getNamedItem(attributeName);
  			if (attribute != null) {
  				return attribute.getNodeValue();
  			}
  		}
  		return null;
  	}
  
  	private String getAllText(Node node) {
  		List<Node> textNodes = getNodesByTagName(node, "#text");
  		StringBuffer sb = new StringBuffer();
  		for (Node textNode : textNodes) {
  			sb.append(textNode.getNodeValue().trim() + " ");
  		}
  		return sb.toString().trim();
  	}
  
  	private List<Node> getNodesByTagName(Node node, String tagName) {
  		List<Node> matches = new ArrayList<Node>();
  		recursiveGetNodesByTagName(node, tagName.toLowerCase(), matches);
  		return matches;
  	}
  
  	private void recursiveGetNodesByTagName(Node node, String tagName,
  											List<Node> matches) {
  		String name = node.getNodeName();
  		if (name != null && name.toLowerCase().equals(tagName)) {
  			matches.add(node);
  		}
  		if (node.getChildNodes() != null
  				&& node.getChildNodes().getLength() > 0) {
  			for (int i = 0; i < node.getChildNodes().getLength(); i++) {
  				recursiveGetNodesByTagName(node.getChildNodes().item(i),
  						tagName, matches);
  			}
  		}
  	}
  
  }
  
  
  