content: /*
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   *
   *    http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package org.apache.celeborn.service.deploy.worker.storage.memory;
  
  import static org.mockito.Mockito.when;
  
  import java.io.IOException;
  import java.nio.ByteBuffer;
  import java.util.HashMap;
  import java.util.Map;
  import java.util.Random;
  
  import io.netty.buffer.ByteBufAllocator;
  import io.netty.buffer.CompositeByteBuf;
  import org.junit.Assert;
  import org.junit.Test;
  import org.mockito.Mockito;
  
  import org.apache.celeborn.common.CelebornConf;
  import org.apache.celeborn.common.identity.UserIdentifier;
  import org.apache.celeborn.common.meta.FileInfo;
  import org.apache.celeborn.common.meta.MemoryFileInfo;
  import org.apache.celeborn.common.meta.ReduceFileMeta;
  import org.apache.celeborn.common.metrics.source.AbstractSource;
  import org.apache.celeborn.common.network.util.NettyUtils;
  import org.apache.celeborn.common.unsafe.Platform;
  import org.apache.celeborn.service.deploy.worker.WorkerSource;
  import org.apache.celeborn.service.deploy.worker.memory.MemoryManager;
  import org.apache.celeborn.service.deploy.worker.storage.PartitionDataWriter;
  import org.apache.celeborn.service.deploy.worker.storage.PartitionFilesSorter;
  import org.apache.celeborn.service.deploy.worker.storage.StorageManager;
  
  public class MemoryPartitionFilesSorterSuiteJ {
  
    private final Random random = new Random();
    private MemoryFileInfo fileInfo;
    private PartitionDataWriter partitionDataWriter;
    private final UserIdentifier userIdentifier = new UserIdentifier("mock-tenantId", "mock-name");
  
    private static final int MAX_MAP_ID = 50;
  
    public long[] prepare(int mapCount) {
      long[] partitionSize = new long[MAX_MAP_ID];
      byte[] batchHeader = new byte[16];
      fileInfo = new MemoryFileInfo(userIdentifier, true, new ReduceFileMeta(8 * 1024 * 1024));
  
      AbstractSource source = Mockito.mock(AbstractSource.class);
      ByteBufAllocator allocator =
          NettyUtils.getSharedByteBufAllocator(new CelebornConf(), source, false);
      CompositeByteBuf buffer = allocator.compositeBuffer();
      fileInfo.setBuffer(buffer);
  
      Map<Integer, Integer> batchIds = new HashMap<>();
  
      for (int i = 0; i < mapCount; i++) {
        int mapId = random.nextInt(MAX_MAP_ID);
        int currentAttemptId = 0;
        int batchId =
            batchIds.compute(
                mapId,
                (k, v) -> {
                  if (v == null) {
                    v = 0;
                  } else {
                    v++;
                  }
                  return v;
                });
        // [63.9k, 192k + 63.9k]
        int dataSize = random.nextInt(192 * 1024) + 65525;
        byte[] mockedData = new byte[dataSize];
        Platform.putInt(batchHeader, Platform.BYTE_ARRAY_OFFSET, mapId);
        Platform.putInt(batchHeader, Platform.BYTE_ARRAY_OFFSET + 4, currentAttemptId);
        Platform.putInt(batchHeader, Platform.BYTE_ARRAY_OFFSET + 8, batchId);
        Platform.putInt(batchHeader, Platform.BYTE_ARRAY_OFFSET + 12, dataSize);
        ByteBuffer buf1 = ByteBuffer.wrap(batchHeader);
        while (buf1.hasRemaining()) {
          buffer.writeBytes(buf1);
        }
        random.nextBytes(mockedData);
        ByteBuffer buf2 = ByteBuffer.wrap(mockedData);
        while (buf2.hasRemaining()) {
          buffer.writeBytes(buf2);
        }
        partitionSize[mapId] = partitionSize[mapId] + batchHeader.length + mockedData.length;
      }
      long originFileLen = buffer.readableBytes();
      fileInfo.getReduceFileMeta().addChunkOffset(originFileLen);
      fileInfo.updateBytesFlushed((int) originFileLen);
  
      CelebornConf conf = new CelebornConf();
      conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key(), "0.8");
      conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), "0.9");
      conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_RESUME().key(), "0.5");
      conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_READ_BUFFER().key(), "0.1");
      conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_MEMORY_FILE_STORAGE().key(), "0.1");
      conf.set(CelebornConf.WORKER_DIRECT_MEMORY_CHECK_INTERVAL().key(), "10");
      conf.set(CelebornConf.WORKER_DIRECT_MEMORY_REPORT_INTERVAL().key(), "10");
      conf.set(CelebornConf.WORKER_READBUFFER_ALLOCATIONWAIT().key(), "10ms");
  
      StorageManager storageManager = Mockito.mock(StorageManager.class);
      Mockito.when(storageManager.storageBufferAllocator()).thenReturn(allocator);
      MemoryManager.reset();
      MemoryManager.initialize(conf, storageManager, null);
      partitionDataWriter = Mockito.mock(PartitionDataWriter.class);
      when(partitionDataWriter.getMemoryFileInfo()).thenAnswer(i -> fileInfo);
  
      return partitionSize;
    }
  
    private void check(int mapCount, int startMapIndex, int endMapIndex) throws IOException {
      long[] partitionSize = prepare(mapCount);
      CelebornConf conf = new CelebornConf();
      conf.set(CelebornConf.SHUFFLE_CHUNK_SIZE().key(), "8m");
      PartitionFilesSorter partitionFilesSorter =
          new PartitionFilesSorter(MemoryManager.instance(), conf, new WorkerSource(conf));
      FileInfo info =
          partitionFilesSorter.getSortedFileInfo(
              "application-1",
              "",
              partitionDataWriter.getMemoryFileInfo(),
              startMapIndex,
              endMapIndex);
      long totalSizeToFetch = 0;
      for (int i = startMapIndex; i < endMapIndex; i++) {
        totalSizeToFetch += partitionSize[i];
      }
      long numChunks = totalSizeToFetch / conf.shuffleChunkSize() + 1;
      Assert.assertTrue(
          0 < ((ReduceFileMeta) info.getFileMeta()).getNumChunks()
              && ((ReduceFileMeta) info.getFileMeta()).getNumChunks() <= numChunks);
      long actualTotalChunkSize =
          ((ReduceFileMeta) info.getFileMeta()).getLastChunkOffset()
              - ((ReduceFileMeta) info.getFileMeta()).getChunkOffsets().get(0);
      Assert.assertEquals(totalSizeToFetch, actualTotalChunkSize);
      fileInfo.getBuffer().release();
    }
  
    @Test
    public void testSortMemoryShuffleFile() throws IOException {
      int startMapIndex = random.nextInt(5);
      int endMapIndex = startMapIndex + random.nextInt(5) + 5;
      check(1000, startMapIndex, endMapIndex);
    }
  }
  