content: /*
   * Copyright (C) 2010-2101 Alibaba Group Holding Limited.
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package com.alibaba.otter.node.etl.common.pipe.impl.memory;
  
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileNotFoundException;
  import java.io.FileOutputStream;
  import java.io.InputStream;
  import java.text.MessageFormat;
  import java.text.SimpleDateFormat;
  import java.util.Date;
  import java.util.List;
  
  import org.apache.commons.io.FilenameUtils;
  import org.apache.commons.io.IOUtils;
  import org.apache.commons.lang.ClassUtils;
  import org.apache.commons.lang.StringUtils;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.springframework.util.Assert;
  
  import com.alibaba.otter.node.etl.common.pipe.PipeDataType;
  import com.alibaba.otter.node.etl.common.pipe.exception.PipeException;
  import com.alibaba.otter.node.etl.load.loader.db.FileloadDumper;
  import com.alibaba.otter.shared.common.utils.NioUtils;
  import com.alibaba.otter.shared.etl.model.DbBatch;
  import com.alibaba.otter.shared.etl.model.FileBatch;
  import com.alibaba.otter.shared.etl.model.FileData;
  import com.alibaba.otter.shared.etl.model.Identity;
  
  public class RowDataMemoryPipe extends AbstractMemoryPipe<DbBatch, MemoryPipeKey> {
  
      private static final Logger logger      = LoggerFactory.getLogger(RowDataMemoryPipe.class);
      private static final String DATE_FORMAT = "yyyy-MM-dd-HH-mm-ss";
      private int                 retry       = 3;
      private String              downloadDir;
  
      public MemoryPipeKey put(DbBatch data) {
          MemoryPipeKey key = new MemoryPipeKey();
          key.setIdentity(data.getRowBatch().getIdentity());
          // if (data.getRoot() == null && data.getFileBatch() != null
          // && !CollectionUtils.isEmpty(data.getFileBatch().getFiles())) {
          // logger.warn("Identity[{}] memory pipe exist fileBatch!",
          // key.getIdentity());
          // // data.setRoot(prepareFile(data.getFileBatch()));
          // }
          key.setDataType(PipeDataType.DB_BATCH);
          cache.put(key, data);
          return key;
      }
  
      public DbBatch get(MemoryPipeKey key) {
          return cache.remove(key);
      }
  
      // 处理对应的附件
      @SuppressWarnings("unused")
      private File prepareFile(FileBatch fileBatch) {
          // 处理构造对应的文件url
          String dirname = buildFileName(fileBatch.getIdentity(), ClassUtils.getShortClassName(fileBatch.getClass()));
          File dir = new File(downloadDir, dirname);
          NioUtils.create(dir, false, 3);// 创建父目录
          // 压缩对应的文件数据
          List<FileData> fileDatas = fileBatch.getFiles();
  
          for (FileData fileData : fileDatas) {
              String namespace = fileData.getNameSpace();
              String path = fileData.getPath();
              boolean isLocal = StringUtils.isBlank(namespace);
              String entryName = null;
              if (true == isLocal) {
                  entryName = FilenameUtils.getPath(path) + FilenameUtils.getName(path);
              } else {
                  entryName = namespace + File.separator + path;
              }
  
              InputStream input = retrive(fileBatch.getIdentity(), fileData);
              if (input == null) {
                  continue;
              }
              File entry = new File(dir, entryName);
              NioUtils.create(entry.getParentFile(), false, retry);// 尝试创建父路径
              FileOutputStream output = null;
              try {
                  output = new FileOutputStream(entry);
                  NioUtils.copy(input, output);// 输出到压缩流中
              } catch (Exception e) {
                  throw new PipeException("prepareFile error for file[" + entry.getPath() + "]");
              } finally {
                  IOUtils.closeQuietly(output);
              }
          }
  
          return dir;
      }
  
      private InputStream retrive(Identity identity, FileData fileData) {
          boolean miss = false;
          try {
              if (StringUtils.isNotEmpty(fileData.getNameSpace())) {
                  throw new RuntimeException(fileData + " is not support!");
              } else {
                  try {
                      File source = new File(fileData.getPath());
                      if (source.exists() && source.isFile()) {
                          return new FileInputStream(source);
                      } else {
                          miss = true;
                          return null;
                      }
                  } catch (FileNotFoundException ex) {
                      miss = true;
                      return null;
                  }
              }
          } finally {
              if (miss) {
                  logger.error(FileloadDumper.dumpMissFileDatas(identity, fileData));
              }
          }
      }
  
      // 构造文件名
      private String buildFileName(Identity identity, String prefix) {
          Date now = new Date();
          String time = new SimpleDateFormat(DATE_FORMAT).format(now);
          return MessageFormat.format("{0}-{1}-{2}-{3}-{4}", prefix, time, String.valueOf(identity.getChannelId()),
                                      String.valueOf(identity.getPipelineId()), String.valueOf(identity.getProcessId()));
      }
  
      public void afterPropertiesSet() throws Exception {
          super.afterPropertiesSet();
          Assert.notNull(downloadDir);
          NioUtils.create(new File(downloadDir), false, 3);
      }
  
      public void setDownloadDir(String downloadDir) {
          this.downloadDir = downloadDir;
      }
  
      public void setRetry(int retry) {
          this.retry = retry;
      }
  
  }
  