content: /**
   * Copyright (C) 2024 AIDC-AI
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package com.alibaba.langengine.core.tokenizers;
  
  import com.alibaba.langengine.core.jtokkit.api.EncodingType;
  import lombok.Data;
  
  import java.util.List;
  
  /**
   * Tokenizer基类
   *
   * @author xiaoxuan.lp
   */
  @Data
  public abstract class Tokenizer {
  
      /**
       * 编码
       *
       * @param text
       * @return
       */
      public List<Integer> encode(String text) {
          return TikTokensUtil.encode(getEncoding(), text);
      }
  
      /**
       * 解码
       *
       * @param encodes
       * @return
       */
      public String decode(List<Integer> encodes) {
          return TikTokensUtil.decode(getEncoding(), encodes);
      }
  
      /**
       * 统计token个数
       *
       * @param text
       * @return
       */
      public int getTokenCount(String text) {
          return TikTokensUtil.tokens(getEncoding(), text);
      }
  
      /**
       * 获取编码器
       *
       * @return
       */
      public abstract EncodingType getEncoding();
  }
  