content: package de.mpg.mpi_inf.ambiversenlu.nlu.ner.configuration.resourcechecker;
  
  import com.google.gson.Gson;
  import com.google.gson.reflect.TypeToken;
  import de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.uima.custom.readers.OrderType;
  import de.mpg.mpi_inf.ambiversenlu.nlu.ner.configuration.CorpusConfiguration;
  import de.mpg.mpi_inf.ambiversenlu.nlu.ner.configuration.KnowNERLanguageConfiguratorException;
  import de.mpg.mpi_inf.ambiversenlu.nlu.ner.wikigeneration.WikiCorpusGenerator;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  import java.io.FileReader;
  import java.io.IOException;
  import java.nio.file.Files;
  import java.nio.file.Path;
  import java.nio.file.Paths;
  import java.util.HashMap;
  import java.util.Map;
  
  /**
   * Checks if the default corpus and the properties for the given language configuration exist.
   * If they partially exist (either the corpus file of the properties), returns error result,
   * otherwise if the corpus file exists and is in sync with the properties, returns success result.
   *
   * If nothing exists, auto-generates the corpus and the properties and returns success result.
    */
  public class KnowNERCorpusChecker extends KnowNERLocalResourceChecker {
  
  	public static final Logger logger = LoggerFactory.getLogger(KnowNERCorpusChecker.class);
  	public static final OrderType DEFAULT_ORDER_TYPE = OrderType.WORD_POSITION_TYPE;
  	private final int autogeneratedCorpusSize;
  
  	private enum CorpusProportions {
  		TRAIN(0.7),
  		TESTA(0.2),
  		TESTB(0.1);
  
  		private final double value;
  
  		CorpusProportions(double value) {
  			this.value = value;
  		}
  	}
  
  	public KnowNERCorpusChecker(String mainDir, String language) {
  		super(mainDir, language);
  		this.autogeneratedCorpusSize = CorpusConfiguration.DEFAULT_AUTOGENERATED_CORPUS_SIZE;
  	}
  
  	public KnowNERCorpusChecker(String mainDir, String language, int autogeneratedCorpusSize) {
  		super(mainDir, language);
  		this.autogeneratedCorpusSize = autogeneratedCorpusSize;
  //		should be greater than 6, otherwise the dataset ranges are wrong
  		if (autogeneratedCorpusSize < 6) {
  			throw new IllegalArgumentException("Autogenerated corpus size should be greater than 6");
  		}
  	}
  
  	@Override
  	public KnowNERResourceResult check() throws KnowNERLanguageConfiguratorException {
  		try {
  			Path corpusPath = Paths.get(languageDir, CorpusConfiguration.CORPUS_DIRECTORY_NAME,
  					CorpusConfiguration.DEFAULT_CORPUS_NAME, CorpusConfiguration.DEFAULT_FILE_NAME);
  			Path configPath = Paths.get(languageDir, CorpusConfiguration.CORPUS_DIRECTORY_NAME,
  					CorpusConfiguration.DEFAULT_CORPUS_NAME, CorpusConfiguration.DEFAULT_CORPUS_CONFIG_NAME);
  			if (Files.exists(corpusPath)) {
  				if (!Files.exists(configPath)) {
  					return new KnowNERResourceResult("Corpus file exists, but configuration file is missing");
  				}
  			} else {
  				if (Files.exists(configPath)) {
  					return new KnowNERResourceResult("Corpus file does not exist, but configuration file is present");
  				}
  				Files.createDirectories(Paths.get(languageDir, CorpusConfiguration.CORPUS_DIRECTORY_NAME, CorpusConfiguration.DEFAULT_CORPUS_NAME));
  				autogenerateCorpusFiles(corpusPath, configPath);
  			}
  			return checkConfiguration(corpusPath, configPath);
  		} catch (Throwable e) {
  			throw new KnowNERLanguageConfiguratorException(e);
  		}
  	}
  
  	private KnowNERResourceResult checkConfiguration(Path corpusPath, Path configPath) throws IOException {
  		CorpusConfiguration configuration = new Gson().fromJson(new FileReader(configPath.toFile()), new TypeToken<CorpusConfiguration>() {
  		}.getType());
  		KnowNERResourceResult result;
  		if (Files.exists(corpusPath)) {
  			Map<CorpusConfiguration.Range, int[]> rangeMap = configuration.getRangeMap();
  			if (configuration.getCorpusFormat() != null &&
  					rangeMap.keySet().stream()
  						.allMatch(
  								k-> rangeMap.get(k)[0] <= rangeMap.get(k)[1]) &&
  					rangeMap.get(CorpusConfiguration.Range.TRAIN)[0] == 0 &&
  					rangeMap.get(CorpusConfiguration.Range.TRAIN)[1] < rangeMap.get(CorpusConfiguration.Range.TESTA)[0] &&
  					rangeMap.get(CorpusConfiguration.Range.TESTA)[1] < rangeMap.get(CorpusConfiguration.Range.TESTB)[0] &&
  					rangeMap.get(CorpusConfiguration.Range.TRAINA)[0] == rangeMap.get(CorpusConfiguration.Range.TRAIN)[0] &&
  					rangeMap.get(CorpusConfiguration.Range.TRAINA)[1] == rangeMap.get(CorpusConfiguration.Range.TESTA)[1]
  					) {
  				if (Files.readAllLines(corpusPath)
  						.stream()
  						.filter(s-> s.startsWith("-DOCSTART-"))
  						.count() == rangeMap.get(CorpusConfiguration.Range.TESTB)[1] + 1) {
  					result = new KnowNERResourceResult();
  				} else {
  					result = new KnowNERResourceResult("Corpus " + corpusPath + " does not contain the required number of documents");
  				}
  			} else {
  				result = new KnowNERResourceResult("Configuration file " + configPath + " does not contain all required fields");
  			}
  		} else {
  			result = new KnowNERResourceResult("Corpus file " + corpusPath + " does not exist");
  		}
  
  		return result;
  	}
  
  	private void autogenerateCorpusFiles(Path corpusPath, Path configPath) throws Throwable {
  		new WikiCorpusGenerator(new String[] {
  				"-p", corpusPath.toString(),
  				"-l", language,
  				"-s", "ENTITY_RANK",
  				"-m", Integer.toString(autogeneratedCorpusSize)
  		}).run();
  		int trainEnd = (int)(autogeneratedCorpusSize * CorpusProportions.TRAIN.value) - 1;
  		int testABegin = trainEnd + 1;
  		int testAEnd = testABegin + (int)(autogeneratedCorpusSize * CorpusProportions.TESTA.value) - 1;
  		int testBBegin = testAEnd + 1;
  		int testBEnd = autogeneratedCorpusSize - 1;
  
  		Map<CorpusConfiguration.Range, int[]> rangeMap = new HashMap<>();
  		rangeMap.put(CorpusConfiguration.Range.TRAIN, new int[]{0, trainEnd});
  		rangeMap.put(CorpusConfiguration.Range.TESTA, new int[]{testABegin, testAEnd});
  		rangeMap.put(CorpusConfiguration.Range.TESTB, new int[]{testBBegin, testBEnd});
  		rangeMap.put(CorpusConfiguration.Range.TRAINA, new int[]{0, testAEnd});
  
  		CorpusConfiguration configuration = new CorpusConfiguration(DEFAULT_ORDER_TYPE, rangeMap);
  
  		String jsonString = new Gson().toJson(configuration);
  		Files.write(configPath, jsonString.getBytes());
  	}
  }
  