content: package de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.similarity;
  
  import de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.graph.similarity.MentionEntitySimilarity;
  import de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.graph.similarity.MentionEntitySimilarityPackage;
  import de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.graph.similarity.importance.EntityImportance;
  import de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.graph.similarity.util.SimilaritySettings;
  import de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.model.Entities;
  import de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.model.ExternalEntitiesContext;
  import de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.trace.NullTracer;
  import de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.trace.Tracer;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import weka.classifiers.functions.Logistic;
  import weka.classifiers.functions.SMO;
  import weka.core.Attribute;
  import weka.core.DenseInstance;
  import weka.core.FastVector;
  import weka.core.Instances;
  import weka.filters.Filter;
  import weka.filters.unsupervised.attribute.Normalize;
  
  import java.io.*;
  import java.lang.reflect.InvocationTargetException;
  import java.util.*;
  import java.util.stream.Collectors;
  
  /**
   * Generates new SimilaritySettings with weights estimated
   * by the WEKA machine learning framework with the given method.
   * It will combine the score files generated by 
   * {@link GenerateScores GenerateScores}
   * and runs the WEKA machine learning on this data.
   */
  public class EstimateParameterWeightsFromScores {
  
    private static final Logger logger = LoggerFactory.getLogger(EstimateParameterWeightsFromScores.class);
  
    // Strings that are used in the Instances for the attribute names.
    public static final String NO_PRIOR_POSTFIX = "+NoPrior";
  
    public static final String WITH_PRIOR_POSTFIX = "+WithPrior";
  
    public static final String SWITCHED_PRIOR_LABEL = "switchedPrior";
  
    public static final String CORRECT_LABEL = "correct";
  
    /**
     * The settings file which is used to determine the combinatoin of '.score' files
     * as well as the information if its switched prior or not.
     */
    private SimilaritySettings settings;
  
    /** The directory where the '.score' files are located */
    private File inputDir;
  
    /** The right combination of the '.score' files */
    private Instances data;
  
    /**
     * @param settings The settings file which is used to determine the combinatoin of '.score' files 
     *                 as well as the information if its switched prior or not.
     * @param inputDir The directory where the '.score' files are located
     * @throws NoSuchMethodException
     * @throws IOException
     * @throws InstantiationException
     * @throws IllegalAccessException
     * @throws InvocationTargetException
     * @throws ClassNotFoundException
     */
    EstimateParameterWeightsFromScores(SimilaritySettings settings, File inputDir)
        throws NoSuchMethodException, IOException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException {
      this.settings = settings;
      this.inputDir = inputDir;
      data = null;
    }
  
    /**
     * Loads the data from the '.score' files and combines them.
     *
     * @return This object
     * @throws IOException
     * @throws InvocationTargetException
     * @throws NoSuchMethodException
     * @throws ClassNotFoundException
     * @throws InstantiationException
     * @throws IllegalAccessException
     */
    public EstimateParameterWeightsFromScores loadData()
        throws IOException, InvocationTargetException, NoSuchMethodException, ClassNotFoundException, InstantiationException, IllegalAccessException {
      Map<String, Boolean> gt = new HashMap<>();
      Map<String, Integer> method2idx = new HashMap<>();
      Map<String, double[]> scores = new HashMap<>();
  
      File gtFile = new File(inputDir, TrueFalseCalculator.FILE_NAME);
  
      logger.info("Reading Ground Truth");
  
      BufferedReader reader = new BufferedReader(new FileReader(gtFile));
  
      for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        String[] data = line.split(ScoreCalculator.VALUE_SEPARATOR);
  
        boolean tf = data[2].equals(TrueFalseCalculator.TRUE_STRING);
  
        gt.put(buildMentionEntityKey(data[0], data[1]), tf);
      }
  
      reader.close();
  
      // get scores
      Set<String> filesNeeded = getFilesNeeded();
      if (settingNeedsPrior(settings)) {
        filesNeeded.add(PriorCalculator.getFileName(settings.getIdentifier()));
        filesNeeded.add(PriorCalculator.getFileNameBest(settings.getIdentifier()));
      }
      File[] scoreFiles = inputDir.listFiles((dir, name) -> filesNeeded.contains(name));
  
      Arrays.sort(scoreFiles);
  
      int current = 0;
  
      for (File file : scoreFiles) {
        reader = new BufferedReader(new FileReader(file));
  
        logger.info("Reading " + file);
  
        String attribute = file.getName().substring(0, file.getName().lastIndexOf(".")).replace("-", ":");
        method2idx.put(attribute, current);
  
        int misses = 0;
  
        for (String line = reader.readLine(); line != null; line = reader.readLine()) {
          String[] data = line.split(ScoreCalculator.VALUE_SEPARATOR);
  
          if (data.length != 3) {
            logger.info("Skipping line in " + file);
            continue;
          }
  
          double[] score = scores.get(buildMentionEntityKey(data[0], data[1]));
  
          if (score == null) {
            score = new double[scoreFiles.length];
            Arrays.fill(score, 0.0);
            scores.put(buildMentionEntityKey(data[0], data[1]), score);
            misses++;
          }
  
          score[current] = Double.parseDouble(data[2]);
        }
  
        logger.info("Found " + misses + " new occurrences");
  
        current++;
      }
  
      data = createInstances(method2idx, scores, gt);
  
      return this;
    }
  
    /*
    * Writes the loaded scores into a Instances object
    */
    private Instances createInstances(Map<String, Integer> method2idx, Map<String, double[]> scores, Map<String, Boolean> gt)
        throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException, ClassNotFoundException {
  
      boolean switchedPrior = settings.getPriorThreshold() > 0.0;
      boolean includePrior = settingNeedsPrior(settings);
  
      String priorLabel = PriorCalculator.getLabel(settings.getIdentifier());
      String priorLabelBest = PriorCalculator.getLabelBest(settings.getIdentifier());
  
      List<String> attributes = method2idx.keySet().stream().filter(s -> !s.equals(priorLabelBest) && (includePrior || !s.equals(priorLabel)))
          .collect(Collectors.toList());
  
      List<String> attributesWithPrior = null;
      List<String> attributesNoPrior = null;
  
      // add only the needed attributes
      FastVector attributesVector = new FastVector(attributes.size());
      if (!switchedPrior) {
        // no switched prior => add all
        attributes.stream().map(Attribute::new).forEach(attributesVector::addElement);
      } else {
        // switched prior =>
        // add all without prior
        MentionEntitySimilarityPackage sims = settings.getMentionEntitySimilarities(new Entities(), new ExternalEntitiesContext(), null);
        attributesNoPrior = sims.getMentionEntitySimilarityNoPrior().stream().map(MentionEntitySimilarity::getIdentifier).filter(attributes::contains)
            .collect(Collectors.toList());
        attributesNoPrior.stream().map(s -> s.concat(NO_PRIOR_POSTFIX)).map(Attribute::new).forEach(attributesVector::addElement);
  
        // add all with prior
        attributesWithPrior = sims.getMentionEntitySimilarityWithPrior().stream().map(MentionEntitySimilarity::getIdentifier)
            .filter(attributes::contains).collect(Collectors.toList());
        attributesWithPrior.stream().map(s -> s.concat(WITH_PRIOR_POSTFIX)).map(Attribute::new).forEach(attributesVector::addElement);
  
        // add switched prior
        attributesVector.addElement(new Attribute(SWITCHED_PRIOR_LABEL));
      }
  
      // add the correct column
      // TRUE => 0.0, FALSE => 1.0
      FastVector correctColValues = new FastVector(2);
      correctColValues.addElement(TrueFalseCalculator.TRUE_STRING);   // 0.0
      correctColValues.addElement(TrueFalseCalculator.FALSE_STRING);  // 1.0
      attributesVector.addElement(new Attribute(CORRECT_LABEL, correctColValues));
  
      Instances instances = new Instances("entity-disambiguation", attributesVector, scores.size());
  
      // write data
      for (String occurrence : scores.keySet()) {
        // only write when GT is present
        if (!gt.containsKey(occurrence)) {
          //        System.out.println("Skipping " + occurrence + ", not present in GT");
          continue;
        }
  
        double[] instanceScores = scores.get(occurrence);
  
        double[] attValues = new double[instances.numAttributes()];
        if (!switchedPrior) {
          for (int i = 0; i < attValues.length - 1; i++) {
            attValues[i] = instanceScores[method2idx.get(attributes.get(i))];
          }
        } else {
          int count = 0;
          if (instanceScores[method2idx.get(priorLabelBest)] >= settings.getPriorThreshold()) {
  
            // skipping no prior attributes
            for (int i = 0; i < attributesNoPrior.size(); i++) {
              attValues[count++] = 0d;
            }
  
            for (String attribute : attributesWithPrior) {
              attValues[count++] = instanceScores[method2idx.get(attribute)];
            }
  
            attValues[count] = instanceScores[method2idx.get(priorLabel)];
          } else {
  
            for (String attribute : attributesNoPrior) {
              attValues[count++] = instanceScores[method2idx.get(attribute)];
            }
  
            // skipping with prior attributes
            for (int i = 0; i < attributesWithPrior.size(); i++) {
              attValues[count++] = 0d;
            }
  
            // don't use prior
            attValues[count] = 0d;
          }
        }
  
        // add the ground truth
        // TRUE => 0.0, FALSE => 1.0
        attValues[attValues.length - 1] = gt.get(occurrence) ? 0d : 1d;
  
        instances.add(new DenseInstance(1.0, attValues));
      }
  
      return instances;
    }
  
    /**
     * Estimates the prior for each attribute with a given method and 
     * writes them into the given properties file.
     *
     * @param outputFile The Properties file the result should be written to.
     * @param method The method that should be used.
     * @return This object
     * @throws Exception
     */
    public EstimateParameterWeightsFromScores estimateWeight(File outputFile, String method) throws Exception {
      if (data == null) throw new NullPointerException("No data loaded");
  
      if (data.classIndex() == -1) {
        data.setClassIndex(data.numAttributes() - 1);
      }
  
      double[] weights = null;
  
      if (method.equals("SMO")) {
        SMO lr = new SMO();
        lr.setC(2.0);
        //      lr.setOptions(new String[] {"C","2.0"} );
        //    lr.setOptions(new String[] {"D","model","libsvm.model"});
        lr.buildClassifier(data);
        //    System.out.println(lr);
  
        weights = normalizeSMOCoefficientsToWeights(data, lr.sparseWeights());
  
      } else if (method.equals("Logistic")) {
        Logistic lr = new Logistic();
  
        Normalize norm = new Normalize();
        norm.setInputFormat(data);
        Instances normData = Filter.useFilter(data, norm);
  
        lr.buildClassifier(normData);
  
        weights = normalizeLogisticCoefficientsToWeights(normData, lr.coefficients());
      }
  
      List<String> meListNoPrior = new ArrayList<>();
      List<String> meListWithPrior = new ArrayList<>();
      List<String> eiListNoPrior = new ArrayList<>();
      List<String> eiListWithPrior = new ArrayList<>();
  
      double prior = 0.0;
  
      for (int i = 0; i < data.numAttributes(); i++) {
        String name = data.attribute(i).name();
  
        if (i < weights.length) {
          logger.info(name + " (norm): " + weights[i]);
        }
  
        if (name.equals(PriorCalculator.getLabel(settings.getIdentifier()))) {
          prior = weights[i];
        } else if (name.equals(SWITCHED_PRIOR_LABEL)) {
          prior = weights[i];
        } else if ((name.split(":").length == 2) && (name.endsWith(NO_PRIOR_POSTFIX))) {
  //        String tmp = name.replace(NO_PRIOR_POSTFIX, "");
  //        String[] split = tmp.split(":");
  //        String simClass = "de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.graph.similarity.measure." + split[0];
  //        String contextClass = "de.mpg.mpi_inf.ambiversenlu.nlu.entitylinking.graph.similarity.context." + split[1];
  //        
  //        meListNoPrior.add(simClass + ":" + contextClass + ":" + weights[i]);
          meListNoPrior.add(name.replace(NO_PRIOR_POSTFIX, "") + ":" + weights[i]);
        } else if ((name.split(":").length == 2) && (name.endsWith(WITH_PRIOR_POSTFIX))) {
          meListWithPrior.add(name.replace(WITH_PRIOR_POSTFIX, "") + ":" + weights[i]);
        } else if (name.split(":").length == 2) {
          meListWithPrior.add(name + ":" + weights[i]);
        } else if (name.equals(CORRECT_LABEL)) {
          continue;
        } else if (name.startsWith("Coli")) {
          meListWithPrior.add(name + ":EmptyEntitiesContext:" + weights[i]);
        } else if (name.endsWith(NO_PRIOR_POSTFIX)) {
          eiListNoPrior.add(name.replace(NO_PRIOR_POSTFIX, "") + ":" + weights[i]);
        } else if (name.endsWith(WITH_PRIOR_POSTFIX)) {
          eiListWithPrior.add(name.replace(WITH_PRIOR_POSTFIX, "") + ":" + weights[i]);
        } else {
          eiListWithPrior.add(name + ":" + weights[i]);
        }
      }
  
      Properties properties = new Properties();
      properties.load(new FileInputStream(new File(settings.getFullPath())));
  
      if (!meListNoPrior.isEmpty()) properties.put("mentionEntitySimilaritiesNoPrior", String.join(" ", meListNoPrior));
      if (!meListWithPrior.isEmpty()) properties.put("mentionEntitySimilaritiesWithPrior", String.join(" ", meListWithPrior));
      if (!eiListNoPrior.isEmpty()) properties.put("entityImportanceWeightsNoPrior", String.join(" ", eiListNoPrior));
      if (!eiListWithPrior.isEmpty()) properties.put("entityImportanceWeightsWithPrior", String.join(" ", eiListWithPrior));
      properties.put("priorWeight", String.valueOf(prior));
  
      properties.store(new FileWriter(outputFile), settings.getIdentifier() + " - Settings");
  
      return this;
    }
  
    private double[] normalizeSMOCoefficientsToWeights(Instances data, double[][][] coefficients) {
      double totalNoPrior = 0;
      double totalWithPrior = 0;
  
      for (int i = 0; i < coefficients[0][1].length; i++) {
        String attrName = data.attribute(i).name();
        if (attrName.endsWith(NO_PRIOR_POSTFIX)) {
          totalNoPrior += Math.abs(coefficients[0][1][i]);
        } else { // WithPrior and shwitchedPrior
          totalWithPrior += Math.abs(coefficients[0][1][i]);
        }
      }
  
      double[] weights = new double[coefficients[0][1].length];
      for (int i = 0; i < coefficients[0][1].length; i++) {
        String attrName = data.attribute(i).name();
        if (attrName.endsWith(NO_PRIOR_POSTFIX)) {
          weights[i] = Math.abs(coefficients[0][1][i]) / totalNoPrior;
        } else { // WithPrior and shwitchedPrior
          weights[i] = Math.abs(coefficients[0][1][i]) / totalWithPrior;
        }
      }
  
      return weights;
    }
  
    private double[] normalizeLogisticCoefficientsToWeights(Instances data, double[][] coefficients) {
      double totalNoPrior = 0;
      double totalWithPrior = 0;
  
      for (int i = 1; i < coefficients.length; i++) {
        String attrName = data.attribute(i).name();
        if (attrName.endsWith(NO_PRIOR_POSTFIX)) {
          totalNoPrior += Math.abs(coefficients[i][0]);
        } else { // WithPrior and shwitchedPrior
          totalWithPrior += Math.abs(coefficients[i][0]);
        }
      }
  
      double[] weights = new double[coefficients.length - 1];
      for (int i = 1; i < coefficients.length; i++) {
        String attrName = data.attribute(i).name();
        if (attrName.endsWith(NO_PRIOR_POSTFIX)) {
          weights[i] = Math.abs(coefficients[i][0]) / totalNoPrior;
        } else { // WithPrior and shwitchedPrior
          weights[i] = Math.abs(coefficients[i][0]) / totalWithPrior;
        }
      }
  
      return weights;
    }
  
    // returns files needed build from the MentionEntitySimilarities
    // and EntityImportances from the settings
    private Set<String> getFilesNeeded()
        throws IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException, ClassNotFoundException {
      Set<String> result = new HashSet<>();
      Tracer tracer = new NullTracer();
      Entities entities = new Entities();
      ExternalEntitiesContext externalEntitiesContext = new ExternalEntitiesContext();
      for (MentionEntitySimilarity mentionEntitySimilarity : settings.getAllMentionEntitySimilarities(entities, externalEntitiesContext, tracer)) {
        result.add(MentionEntitySimilarityCalculator.mesId2FileName(mentionEntitySimilarity.getFileIdentifier()));
      }
      for (EntityImportance entityImportance : settings.getAllEntityImportances(new Entities())) {
        result.add(EntityImportanceCalculator.getFileName(entityImportance.getIdentifier()));
      }
      return result;
    }
  
    private String buildMentionEntityKey(String mention, String entity) {
      return transformId(mention) + ScoreCalculator.VALUE_SEPARATOR + entity;
    }
  
    private String transformId(String id) {
      return id;
    }
  
    protected static boolean settingNeedsPrior(SimilaritySettings settings) {
      return settings.getPriorWeight() > 0d || settings.getPriorThreshold() > 0d;
    }
  
    /**
     * Returns the loaded data in ARFF format.
     *
     * @see weka.core.Instances#toString()
     */
    public String getDataInArffFormat() {
      if (data == null) throw new NullPointerException("No data loaded");
      return data.toString();
    }
  }
  