content: package net.agkn.hll;
  
  /*
   * Copyright 2013 Aggregate Knowledge, Inc.
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  import java.io.FileWriter;
  import java.io.IOException;
  import java.util.Random;
  
  import net.agkn.hll.HLL;
  import net.agkn.hll.HLLType;
  import net.agkn.hll.serialization.ISchemaVersion;
  import net.agkn.hll.serialization.SerializationUtil;
  import net.agkn.hll.util.NumberUtil;
  import static net.agkn.hll.ProbabilisticTestUtil.constructHLLValue;
  
  /**
   * Generates test files for testing other implementations of HLL
   * serialization/deserialization, namely the PostgreSQL implementation.
   *
   * @author timon
   */
  public class IntegrationTestGenerator {
      // ************************************************************************
      // directory to output the generated tests
      private static final String OUTPUT_DIRECTORY = "/tmp/hll_test/";
      // seed to make results reproducible
      private static final long SEED = 1L;
  
      // ------------------------------------------------------------------------
      // configurations for HLLs, should mirror settings in PostgreSQL impl. tests
      private static final int REGWIDTH = 5;
      private static final int LOG2M = 11;
      // NOTE:  This differs from the PostgreSQL impl. parameter 'expthresh'. This
      //        is a literal threshold to use in the promotion hierarchy, implying
      //        that both EXPLICIT representation should be used and it should
      //        NOT be automatically computed. This is done to ensure that the
      //        parameters of the test are very explicitly defined.
      private static final int EXPLICIT_THRESHOLD = 256;
      // NOTE:  This is not the PostgreSQL impl. parameter 'sparseon'. 'sparseon'
      //        is assumed to be true and this is a literal register-count threshold
      //        to use in the promotion hierarchy. This is done to ensure that the
      //        parameters of the test are very explicitly defined.
      private static final int SPARSE_THRESHOLD = 850;
  
      // ------------------------------------------------------------------------
      // computed constants
      private static final int REGISTER_COUNT = (1 << LOG2M);
      private static final int REGISTER_MAX_VALUE = (1 << REGWIDTH) - 1;
  
      // ========================================================================
      // Tests
      /**
       * Cumulatively adds random values to a FULL HLL through the small range
       * correction, uncorrected range, and large range correction of the HLL's
       * cardinality estimator.
       *
       * Format: cumulative add
       * Tests:
       * - FULL cardinality computation
       */
      private static void fullCardinalityCorrectionTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "cardinality_correction", TestType.ADD);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.FULL);
          initLineAdd(output, hll, schemaVersion);
  
          // run through some values in the small range correction
          for(int i=0; i<((1 << LOG2M) - 1); i++) {
              final long rawValue = constructHLLValue(LOG2M, i, 1);
              cumulativeAddLine(output, hll, rawValue, schemaVersion);
          }
  
          // run up past some values in the uncorrected range
          for(int i=0; i<(1 << LOG2M); i++) {
              final long rawValue = constructHLLValue(LOG2M, i, 7);
              cumulativeAddLine(output, hll, rawValue, schemaVersion);
          }
  
          // run through some values in the large range correction
          for(int i=0; i<(1 << LOG2M); i++) {
              final long rawValue = constructHLLValue(LOG2M, i, 30);
              cumulativeAddLine(output, hll, rawValue, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Cumulatively adds random values to an EMPTY HLL.
       *
       * Format: cumulative add
       * Tests:
       * - EMPTY, EXPLICIT, SPARSE, PROBABILSTIC addition
       * - EMPTY to EXPLICIT promotion
       * - EXPLICIT to SPARSE promotion
       * - SPARSE to FULL promotion
       */
      private static void globalStepTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "comprehensive_promotion", TestType.ADD);
  
          final Random random = new Random(SEED);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.EMPTY);
          initLineAdd(output, hll, schemaVersion);
  
          for(int i=0; i<10000/*arbitrary*/; i++) {
              cumulativeAddLine(output, hll, random.nextLong(), schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Cumulatively unions "underpopulated" FULL HLLs into the
       * accumulator to verify the correct behavior from the PostgreSQL implementation.
       * The PostgreSQL implementation's representations of probabilistic HLLs should
       * depend exclusively on the chosen SPARSE-to-FULL cutoff.
       *
       * Format: cumulative union
       * Tests:
       * - EMPTY U "underpopulated" FULL => SPARSE
       * - SPARSE U "underpopulated" FULL => SPARSE
       * - SPARSE U "barely underpopulated" FULL => FULL
       */
      private static void sparseFullRepresentationTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "sparse_full_representation", TestType.UNION);
  
          final HLL emptyHLL1 = newHLL(HLLType.EMPTY);
          final HLL emptyHLL2 = newHLL(HLLType.EMPTY);
  
          cumulativeUnionLine(output, emptyHLL1, emptyHLL2, schemaVersion);
  
          // NOTE:  In this test the sparseReference will be the "expected" value
          //        from the C representation, since it doesn't choose representation
          //        based on original encoding, but rather on the promotion rules
          //        and the declared type of the "receiving" field.
          //        It is the manually-constructed union result.
  
          // "underpopulated" FULL U EMPTY => SPARSE
          final HLL fullHLL = newHLL(HLLType.FULL);
          fullHLL.addRaw(constructHLLValue(LOG2M, 0/*ix*/, 1/*val*/));
  
          final HLL sparseHLL = newHLL(HLLType.SPARSE);
          sparseHLL.addRaw(constructHLLValue(LOG2M, 0/*ix*/, 1/*val*/));
  
          output.write(stringCardinality(fullHLL) + "," + toByteA(fullHLL, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
          output.flush();
  
          // "underpopulated" FULL (small) U SPARSE (small) => SPARSE
          final HLL fullHLL2 = newHLL(HLLType.FULL);
          fullHLL2.addRaw(constructHLLValue(LOG2M, 1/*ix*/, 1/*val*/));
  
          sparseHLL.addRaw(constructHLLValue(LOG2M, 1/*ix*/, 1/*val*/));
  
          output.write(stringCardinality(fullHLL2) + "," + toByteA(fullHLL2, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
          output.flush();
  
          // "underpopulated" FULL (just on edge) U SPARSE (small) => FULL
          final HLL fullHLL3 = newHLL(HLLType.FULL);
          for(int i=2; i<(SPARSE_THRESHOLD + 1); i++) {
              fullHLL3.addRaw(constructHLLValue(LOG2M, i/*ix*/, 1/*val*/));
              sparseHLL.addRaw(constructHLLValue(LOG2M, i/*ix*/, 1/*val*/));
          }
  
          output.write(stringCardinality(fullHLL3) + "," + toByteA(fullHLL3, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
          output.flush();
      }
  
      /**
       * Cumulatively sets successive registers to:
       *
       *     <code>(registerIndex % REGISTER_MAX_VALUE) + 1</code>
       *
       * by adding specifically constructed values to a SPARSE HLL.
       * Does not induce promotion.
       *
       * Format: cumulative add
       * Tests:
       * - SPARSE addition (predictable)
       */
      private static void sparseStepTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "sparse_step", TestType.ADD);
  
          // the accumulator, starts empty sparse probabilistic
          final HLL hll = newHLL(HLLType.SPARSE);
          initLineAdd(output, hll, schemaVersion);
  
          for(int i=0; i<SPARSE_THRESHOLD; i++) {
              final long rawValue = constructHLLValue(LOG2M, i, ((i % REGISTER_MAX_VALUE) + 1));
              cumulativeAddLine(output, hll, rawValue, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Cumulatively sets random registers of a SPARSE HLL to
       * random values by adding random values. Does not induce promotion.
       *
       * Format: cumulative add
       * Tests:
       * - SPARSE addition (random)
       */
      private static void sparseRandomTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "sparse_random", TestType.ADD);
  
          final Random random = new Random(SEED);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.SPARSE);
          initLineAdd(output, hll, schemaVersion);
  
          for(int i=0; i<SPARSE_THRESHOLD; i++) {
              final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
              final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
              final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
  
              cumulativeAddLine(output, hll, rawValue, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Cumulatively sets the first register (index 0) to value 2, the last
       * register (index m-1) to value 2, and then sets registers with indices in
       * the range 2 to (sparseCutoff + 2) to value 1 to trigger promotion.
       *
       * This tests for register alignment in the promotion from SPARSE
       * to FULL.
       *
       * Format: cumulative add
       * Tests:
       * - SPARSE addition
       * - SPARSE to FULL promotion
       */
      private static void sparseEdgeTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "sparse_edge", TestType.ADD);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.SPARSE);
          initLineAdd(output, hll, schemaVersion);
  
          final long firstValue = constructHLLValue(LOG2M, 0, 2);
          cumulativeAddLine(output, hll, firstValue, schemaVersion);
  
          final long lastValue = constructHLLValue(LOG2M, (1 << LOG2M) - 1, 2);
          cumulativeAddLine(output, hll, lastValue, schemaVersion);
  
          for(int i=2; i<(SPARSE_THRESHOLD + 2); i++) {
              final long middleValue = constructHLLValue(LOG2M, i, 1);
  
              cumulativeAddLine(output, hll, middleValue, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Unions an EMPTY accumulator with EXPLICIT HLLs, each containing a
       * single random value.
       *
       * Format: cumulative union
       * Tests:
       * - EMPTY U EXPLICIT
       * - EXPLICIT U EXPLICIT
       * - EXPLICIT to SPARSE promotion
       * - SPARSE U EXPLICIT
       */
      private static void explicitPromotionTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "explicit_promotion", TestType.UNION);
  
          final Random random = new Random(SEED);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.EMPTY);
          final HLL emptyHLL = newHLL(HLLType.EMPTY);
          cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);
  
          for(int i=0; i<(EXPLICIT_THRESHOLD+500)/*should be greater than promotion cutoff*/; i++) {
              // make an EXPLICIT set and populate with cardinality 1
              final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
              explicitHLL.addRaw(random.nextLong());
  
              cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Unions an EMPTY accumulator with SPARSE HLLs, each
       * having one register set.
       *
       * Format: cumulative union
       * Tests:
       * - EMPTY U SPARSE
       * - SPARSE U SPARSE
       * - SPARSE promotion
       * - SPARSE U FULL
       */
      private static void sparseProbabilisticPromotionTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "sparse_promotion", TestType.UNION);
  
          final Random random = new Random(SEED);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.EMPTY);
          final HLL emptyHLL = newHLL(HLLType.EMPTY);
          cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);
  
  
          for(int i=0; i<(SPARSE_THRESHOLD + 1000)/*should be greater than promotion cutoff*/; i++) {
              // make a SPARSE set and populate with cardinality 1
              final HLL sparseHLL = newHLL(HLLType.SPARSE);
  
              final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
              final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
              final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
              sparseHLL.addRaw(rawValue);
  
              cumulativeUnionLine(output, hll, sparseHLL, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Unions an EMPTY accumulator with EXPLICIT HLLs, each having a single
       * random value, twice in a row to verify that the set properties are
       * satisfied.
       *
       * Format: cumulative union
       * Tests:
       * - EMPTY U EXPLICIT
       * - EXPLICIT U EXPLICIT
       */
      private static void explicitOverlapTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "explicit_explicit", TestType.UNION);
  
          final Random random = new Random(SEED);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.EMPTY);
          final HLL emptyHLL = newHLL(HLLType.EMPTY);
  
          cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);
  
          for(int i=0; i<EXPLICIT_THRESHOLD; i++) {
              // make an EXPLICIT set and populate with cardinality 1
              final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
              explicitHLL.addRaw(random.nextLong());
  
              // union it into the accumulator twice, to test overlap (cardinality should not change)
              cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
              cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Unions an EMPTY accumulator with SPARSE HLLs, each
       * having a single register set, twice in a row to verify that the set
       * properties are satisfied.
       *
       * Format: cumulative union
       * Tests:
       * - EMPTY U SPARSE
       * - SPARSE U SPARSE
       */
      private static void sparseProbabilisticOverlapTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "sparse_sparse", TestType.UNION);
  
          final Random random = new Random(SEED);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.EMPTY);
          final HLL emptyHLL = newHLL(HLLType.EMPTY);
  
          cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);
  
          for(int i=0; i<SPARSE_THRESHOLD; i++) {
              // make a SPARSE set and populate with cardinality 1
              final HLL sparseHLL = newHLL(HLLType.SPARSE);
              final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
              final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
              final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
              sparseHLL.addRaw(rawValue);
  
              cumulativeUnionLine(output, hll, sparseHLL, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Unions an EMPTY accumulator with FULL HLLs, each having
       * many registers set, twice in a row to verify that the set properties are
       * satisfied.
       *
       * Format: cumulative union
       * Tests:
       * - EMPTY U FULL
       * - FULL U FULL
       */
      private static void probabilisticUnionTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "probabilistic_probabilistic", TestType.UNION);
  
          final Random random = new Random(SEED);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.EMPTY);
          final HLL emptyHLL = newHLL(HLLType.EMPTY);
          cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);
  
          for(int i=0; i<1000/*number of rows to generate*/; i++) {
              // make a FULL set and populate with
              final HLL fullHLL = newHLL(HLLType.FULL);
              final int elementCount = random.nextInt(10000/*arbitrary maximum cardinality*/);
              for(int j=0;j<elementCount;j++) {
                  fullHLL.addRaw(random.nextLong());
              }
  
              cumulativeUnionLine(output, hll, fullHLL, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      /**
       * Unions an EMPTY accumulator with random HLLs.
       *
       * Format: cumulative union
       * Tests:
       * - hopefully all union possibilities
       */
      private static void globalUnionTest(final ISchemaVersion schemaVersion) throws IOException {
          final FileWriter output = openOutput(schemaVersion, "comprehensive", TestType.UNION);
  
          final Random random = new Random(SEED);
  
          // the accumulator, starts empty
          final HLL hll = newHLL(HLLType.EMPTY);
          final HLL emptyHLL = newHLL(HLLType.EMPTY);
  
          cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);
  
          for(int i=0; i<1000/*number of rows to generate*/; i++) {
              final HLL randomHLL = generateRandomHLL(random);
              cumulativeUnionLine(output, hll, randomHLL, schemaVersion);
          }
  
          output.flush();
          output.close();
      }
  
      // ========================================================================
      // Main
      public static void fullSuite(final ISchemaVersion schemaVersion) throws IOException {
          fullCardinalityCorrectionTest(schemaVersion);
          globalUnionTest(schemaVersion);
          globalStepTest(schemaVersion);
          probabilisticUnionTest(schemaVersion);
          explicitPromotionTest(schemaVersion);
          explicitOverlapTest(schemaVersion);
          sparseFullRepresentationTest(schemaVersion);
          sparseStepTest(schemaVersion);
          sparseRandomTest(schemaVersion);
          sparseEdgeTest(schemaVersion);
          sparseProbabilisticPromotionTest(schemaVersion);
          sparseProbabilisticOverlapTest(schemaVersion);
      }
  
      public static void main(String[] args) throws IOException {
          fullSuite(SerializationUtil.VERSION_ONE);
      }
  
      // ************************************************************************
      // Helpers
      /**
       * Shortcut for testing constructor, which uses the constants defined at
       * the top of the file as default parameters.
       *
       * @return a new {@link HLL} of specified type, which uses the parameters
       *         ({@link #LOG2M}, {@link #REGWIDTH}, {@link #EXPLICIT_THRESHOLD},
       *         and {@link #SPARSE_THRESHOLD}) specified above.
       */
      private static HLL newHLL(final HLLType type) {
          return newHLL(type);
      }
  
      /**
       * Returns the algorithm-specific cardinality of the specified {@link HLL}
       * as a {@link String} appropriate for comparison with the algorithm-specific
       * cardinality provided by the PostgreSQL implementation.
       *
       * @param  hll the HLL whose algorithm-specific cardinality is to be printed.
       *         This cannot be <code>null</code>.
       * @return the algorithm-specific cardinality of the instance as a PostgreSQL-
       *         compatible String. This will never be <code>null</code>
       */
      private static String stringCardinality(final HLL hll) {
          switch(hll.getType()) {
              case EMPTY:
                  return "0";
              case EXPLICIT:/*promotion has not yet occurred*/
                  return Long.toString(hll.cardinality());
              case SPARSE:
                  return Double.toString(hll.sparseProbabilisticAlgorithmCardinality());
              case FULL:
                  return Double.toString(hll.fullProbabilisticAlgorithmCardinality());
              default:
                  throw new RuntimeException("Unknown HLL type " + hll.getType());
          }
      }
  
      /**
       * Generates a random HLL and populates it with random values.
       *
       * @param  random the {@link Random random number generator} used to populate
       *         the HLL. This cannot be <code>null</code>.
       * @return the populated HLL. This will never be <code>null</code>.
       */
      public static HLL generateRandomHLL(final Random random) {
          final int randomTypeInt = random.nextInt(HLLType.values().length);
          final HLLType type;
          switch(randomTypeInt) {
              case 0:
                  type = HLLType.EMPTY;
                  break;
              case 1:
                  type = HLLType.EXPLICIT;
                  break;
              case 2:
                  type = HLLType.FULL;
                  break;
              case 3:
                  type = HLLType.EMPTY;
                  break;
              case 4:
                  type = HLLType.SPARSE;
                  break;
              default:
                  throw new RuntimeException("Unassigned type int " + randomTypeInt);
          }
  
          final int cardinalityCap;
          final int cardinalityBaseline;
  
          switch(type) {
              case EMPTY:
                  return newHLL(HLLType.EMPTY);
              case EXPLICIT:
                  cardinalityCap = EXPLICIT_THRESHOLD;
                  cardinalityBaseline = 1;
                  break;
              case SPARSE:
                  cardinalityCap = SPARSE_THRESHOLD;
                  cardinalityBaseline = (EXPLICIT_THRESHOLD + 1);
                  break;
              case FULL:
                  cardinalityCap = 100000;
                  cardinalityBaseline = (SPARSE_THRESHOLD*10);
                  break;
              default:
                  throw new RuntimeException("We should never be here.");
          }
  
          final HLL hll = newHLL(HLLType.EMPTY);
          for(int i=0; i<cardinalityBaseline; i++) {
              hll.addRaw(random.nextLong());
          }
          for(int i=0; i<random.nextInt(cardinalityCap - cardinalityBaseline); i++) {
              hll.addRaw(random.nextLong());
          }
  
          return hll;
      }
  
      /**
       * Opens a {@link FileWriter} and writes out an appropriate CSV header.
       *
       * @param  schemaVersion Schema version of the output. This cannot be
       *         <code>null</code>.
       * @param  description Description string used to build the filename.
       *         This cannot be <code>null</code>.
       * @param  type {@link TestType type} of the test file to be written.
       *         This cannot be <code>null</code>.
       * @return The opened {@link FileWriter writer}. This will never be <code>null</code>.
       */
      private static FileWriter openOutput(final ISchemaVersion schemaVersion, final String description, final TestType type) throws IOException {
          final String schemaVersionPrefix = "v"+ schemaVersion.schemaVersionNumber() + "_";
          final String header;
          final String filename;
          switch(type) {
              case ADD:
                  header = "cardinality,raw_value,HLL\n";
                  filename = schemaVersionPrefix + "cumulative_add_" + description + ".csv";
                  break;
              case UNION:
                  header = "cardinality,HLL,union_cardinality,union_HLL\n";
                  filename = schemaVersionPrefix + "cumulative_union_" + description + ".csv";
                  break;
              default:
                  throw new RuntimeException("Unknown test type " + type);
          }
  
          final FileWriter output = new FileWriter(OUTPUT_DIRECTORY + filename);
          output.write(header);
          output.flush();
          return output;
      }
  
      /**
       * Writes out a {@link TestType#ADD}-formatted test line.
       *
       * @param  output The output {@link FileWriter writer}. This cannot be <code>null</code>.
       * @param  hll The "accumulator" HLL instance. This cannot be <code>null</code>.
       * @param  rawValue The raw value added to the HLL.
       * @param  schemaVersion the schema with which to serialize the HLLs. This cannot
       *         be <code>null</code>.
       */
      private static void cumulativeAddLine(final FileWriter output, final HLL hll, final long rawValue, final ISchemaVersion schemaVersion) throws IOException {
          hll.addRaw(rawValue);
          final String accumulatorCardinality = stringCardinality(hll);
  
          output.write(accumulatorCardinality + "," + rawValue + "," + toByteA(hll, schemaVersion) + "\n");
          output.flush();
      }
  
      /**
       * Writes an initial line for a {@link TestType#ADD}-formatted test.
       *
       * @param  output The output {@link FileWriter writer}. This cannot be <code>null</code>.
       * @param  hll The "accumulator" HLL instance. This cannot be <code>null</code>.
       * @param  rawValue The raw value added to the HLL.
       * @param  schemaVersion the schema with which to serialize the HLLs. This cannot
       *         be <code>null</code>.
       */
      private static void initLineAdd(final FileWriter output, final HLL hll, final ISchemaVersion schemaVersion) throws IOException {
          output.write(0 + "," + 0 + "," + toByteA(hll, schemaVersion) + "\n");
          output.flush();
      }
  
      /**
       * Writes out a {@link TestType#UNION}-formatted test line.
       *
       * @param  output The output {@link FileWriter writer}. This cannot be <code>null</code>.
       * @param  hll The "accumulator" HLL instance. This cannot be <code>null</code>.
       * @param  increment The "increment" HLL instance which will be unioned into
       *         the accumulator. This cannot be <code>null</code>.
       * @param  schemaVersion the schema with which to serialize the HLLs. This cannot
       *         be <code>null</code>.
       */
      private static void cumulativeUnionLine(final FileWriter output, final HLL hll, final HLL increment, final ISchemaVersion schemaVersion) throws IOException {
          hll.union(increment);
  
          final String incrementCardinality = stringCardinality(increment);
          final String accumulatorCardinality = stringCardinality(hll);
          output.write(incrementCardinality + "," + toByteA(increment, schemaVersion) + "," + accumulatorCardinality + "," + toByteA(hll, schemaVersion) + "\n");
          output.flush();
      }
  
      /**
       * Serializes a HLL to Postgres 9 'bytea' hex-format, for CSV ingest.
       *
       * @param  hll the HLL to serialize. This cannot be <code>null</code>.
       * @param  schemaVersion the schema with which to serialize the HLLs. This cannot
       *         be <code>null</code>.
       * @return a PostgreSQL 'bytea' string representing the HLL.
       */
      private static String toByteA(final HLL hll, final ISchemaVersion schemaVersion) {
          final byte[] bytes = hll.toBytes(schemaVersion);
          return ("\\x" + NumberUtil.toHex(bytes, 0, bytes.length));
      }
  
      /**
       * Indicates what kind of test output a test will generate.
       */
      private static enum TestType {
          /**
           * This type of test is characterized by values being added to an
           * accumulator HLL whose serialized representation (after the value is added)
           * is printed to each line along with the cardinality and added value.
           */
          ADD,
          /**
           * This type of test is characterized by HLLs being unioned into an
           * accumulator HLL whose serialized representation (after the HLL is
           * union'd) is printed to each line along with the cardinalities and the
           * serialized representation of the HLL union'd in.
           */
          UNION;
      }
  }
  