content: /**
   * Copyright (C) 2024 AIDC-AI
   * <p>
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   * <p>
   * http://www.apache.org/licenses/LICENSE-2.0
   * <p>
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package com.alibaba.langengine.deepsearch.loader.crawler;
  
  import java.util.*;
  import java.util.concurrent.atomic.AtomicInteger;
  import java.util.regex.Pattern;
  
  import com.alibaba.langengine.core.indexes.Document;
  import com.alibaba.langengine.core.textsplitter.RecursiveCharacterTextSplitter;
  import com.alibaba.langengine.deepsearch.DeepSearcher;
  import org.apache.http.Header;
  
  import edu.uci.ics.crawler4j.crawler.Page;
  import edu.uci.ics.crawler4j.crawler.WebCrawler;
  import edu.uci.ics.crawler4j.parser.HtmlParseData;
  import edu.uci.ics.crawler4j.url.WebURL;
  
  public class BasicCrawler extends WebCrawler {
  
      private static final Pattern IMAGE_EXTENSIONS = Pattern.compile(".*\\.(bmp|gif|jpg|png)$");
  
      private final AtomicInteger numSeenImages;
  
      private DeepSearcher deepSearcher;
  
      /**
       * Creates a new crawler instance.
       *
       * @param numSeenImages This is just an example to demonstrate how you can pass objects to crawlers. In this
       * example, we pass an AtomicInteger to all crawlers and they increment it whenever they see a url which points
       * to an image.
       */
      public BasicCrawler(AtomicInteger numSeenImages, DeepSearcher deepSearcher) {
          this.numSeenImages = numSeenImages;
          this.deepSearcher = deepSearcher;
      }
  
      /**
       * You should implement this function to specify whether the given url
       * should be crawled or not (based on your crawling logic).
       */
      @Override
      public boolean shouldVisit(Page referringPage, WebURL url) {
          String href = url.getURL().toLowerCase();
          // Ignore the url if it has an extension that matches our defined set of image extensions.
          if (IMAGE_EXTENSIONS.matcher(href).matches()) {
              numSeenImages.incrementAndGet();
              return false;
          }
  
          // Only accept the url if it is in the "www.ics.uci.edu" domain and protocol is "http".
  //        return href.startsWith("https://www.ics.uci.edu/");
          return true;
      }
  
      /**
       * This function is called when a page is fetched and ready to be processed
       * by your program.
       */
      @Override
      public void visit(Page page) {
          int docid = page.getWebURL().getDocid();
          String url = page.getWebURL().getURL();
          String domain = page.getWebURL().getDomain();
          String path = page.getWebURL().getPath();
          String subDomain = page.getWebURL().getSubDomain();
          String parentUrl = page.getWebURL().getParentUrl();
          String anchor = page.getWebURL().getAnchor();
  
          logger.debug("Docid: {}", docid);
          logger.info("URL: {}", url);
          logger.debug("Domain: '{}'", domain);
          logger.debug("Sub-domain: '{}'", subDomain);
          logger.debug("Path: '{}'", path);
          logger.debug("Parent page: {}", parentUrl);
          logger.debug("Anchor text: {}", anchor);
  
          if (page.getParseData() instanceof HtmlParseData) {
              HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
              String text = htmlParseData.getText();
              String html = htmlParseData.getHtml();
              Set<WebURL> links = htmlParseData.getOutgoingUrls();
  
              logger.debug("Text length: {}", text.length());
              logger.debug("Html length: {}", html.length());
              logger.debug("Number of outgoing links: {}", links.size());
  
              if(deepSearcher != null) {
                  List<Document> documentList = new ArrayList<>();
                  Document document = new Document();
                  document.setPageContent(html);
                  Map<String, Object> metadata = new HashMap<>();
                  metadata.put("source", url);
                  document.setMetadata(metadata);
                  documentList.add(document);
  
                  RecursiveCharacterTextSplitter textSplitter = new RecursiveCharacterTextSplitter();
                  textSplitter.setMaxChunkSize(1500);
                  textSplitter.setMaxChunkOverlap(100);
                  List<Document> chunks = textSplitter.splitDocuments(documentList);
                  deepSearcher.getVectorStore().addDocuments(chunks);
              }
          }
  
          Header[] responseHeaders = page.getFetchResponseHeaders();
          if (responseHeaders != null) {
              logger.debug("Response headers:");
              for (Header header : responseHeaders) {
                  logger.debug("\t{}: {}", header.getName(), header.getValue());
              }
          }
  
          logger.debug("=============");
      }
  }
  