content: /*
   * Copyright (c) 2009 Andrejs Jermakovics.
   * 
   * All rights reserved. This program and the accompanying materials
   * are made available under the terms of the Eclipse Public License v1.0
   * which accompanies this distribution, and is available at
   * http://www.eclipse.org/legal/epl-v10.html
   *
   * Contributors:
   *     Andrejs Jermakovics - initial implementation
   */
  package it.unibz.instasearch.indexing;
  
  import it.unibz.instasearch.InstaSearchPlugin;
  import it.unibz.instasearch.prefs.PreferenceConstants;
  
  import java.io.File;
  import java.io.IOException;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collections;
  import java.util.List;
  import java.util.Locale;
  import java.util.regex.Pattern;
  
  import org.apache.lucene.index.IndexReader;
  import org.apache.lucene.index.IndexWriter;
  import org.apache.lucene.index.Term;
  import org.apache.lucene.search.IndexSearcher;
  import org.apache.lucene.search.PrefixQuery;
  import org.apache.lucene.search.ScoreDoc;
  import org.apache.lucene.search.TopDocs;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.store.FSDirectory;
  import org.eclipse.core.resources.IContainer;
  import org.eclipse.core.resources.IFile;
  import org.eclipse.core.resources.IFolder;
  import org.eclipse.core.resources.IProject;
  import org.eclipse.core.resources.IResource;
  import org.eclipse.core.resources.IStorage;
  import org.eclipse.core.resources.IWorkspaceRoot;
  import org.eclipse.core.runtime.CoreException;
  import org.eclipse.core.runtime.IProgressMonitor;
  import org.eclipse.core.runtime.Platform;
  import org.eclipse.core.runtime.content.IContentType;
  import org.eclipse.core.runtime.content.IContentTypeManager;
  import org.eclipse.core.runtime.jobs.ISchedulingRule;
  import org.eclipse.jface.util.IPropertyChangeListener;
  import org.eclipse.jface.util.PropertyChangeEvent;
  import org.eclipse.ui.IEditorInput;
  import org.eclipse.ui.ide.IDE;
  import org.eclipse.ui.part.FileEditorInput;
  
  /**
   * WorkspaceIndexer
   * Indexes the Eclipse workspace
   */
  public class WorkspaceIndexer extends StorageIndexer implements ISchedulingRule, IPropertyChangeListener {
  		
  	private IContentType TEXT_CONTENT_TYPE;
  	
  	private static final ResourceCollector resourceCollector = new ResourceCollector();
  	
  	// Prefs
  	private String fileExtensions[] = getIndexableFileExtensions();
  	private List<Pattern> excludedDirRegExes = getExcludedDirsRegExes();
  	private boolean indexEmptyExtension = InstaSearchPlugin.getBoolPref(PreferenceConstants.P_INDEX_EMPTY_EXTENSION);
  	
  	/**
  	 * @throws Exception
  	 */
  	public WorkspaceIndexer() throws Exception  
  	{
      	super();
      	
      	if( Platform.getContentTypeManager() != null )
      		TEXT_CONTENT_TYPE = Platform.getContentTypeManager().getContentType(IContentTypeManager.CT_TEXT);
  	}
  	
  	@Override
  	public Directory getIndexDir() throws IOException 
  	{
  		return FSDirectory.open(getIndexDirLocation()); // FSDirectory.getDirectory(getIndexDirLocation(), false); 
  	}
  	
  	/**
  	 * @param monitor
  	 * @throws Exception
  	 */
  	public void createIndex(IWorkspaceRoot root, IProgressMonitor monitor) throws Exception {
  		
  		getIndexChangeListener().onIndexReset();
  		
  		deleteIndex();
  		Directory indexDirectory = FSDirectory.open(getIndexDirLocation());
  		
  		IndexWriter indexWriter = createIndexWriter(true);
  		
  		indexContainers(indexWriter, root, monitor);
  		
  		monitor.setTaskName("Optimizing Index");
  		indexWriter.optimize();
  		
  		indexWriter.close();
  		indexDirectory.close();
  		
  		getIndexChangeListener().onIndexUpdate();
  		
  		monitor.done();
  	}
  	
  	
  	/**
  	 * Index all containers in the workspace
  	 * 
  	 * @param indexWriter
  	 * @param workspaceRoot
  	 * @param monitor
  	 * @throws Exception
  	 */
  	protected void indexContainers(IndexWriter indexWriter, IWorkspaceRoot workspaceRoot, IProgressMonitor monitor) throws Exception {
  		
  		resourceCollector.clear();
  		resourceCollector.setExcludedDirRegExes( excludedDirRegExes );
  		workspaceRoot.accept(resourceCollector); // use visitor to collect containers
  		
  		monitor.beginTask("File Indexing", resourceCollector.getContainers().size());
  		
  		for(IContainer container: resourceCollector.getContainers()) 
  		{	
  			if( monitor.isCanceled() ) 
  				break;
  			
  			monitor.setTaskName("Indexing: " + container.getProject().getName() + " - " + container.getName());
  			indexContainer(indexWriter, container, monitor);
  			monitor.worked(1);
  		}
  		
  		monitor.done();
  	}
  	
  	private File getIndexDirLocation()
  	{
  		File location = InstaSearchPlugin.getIndexDirLocation();
  		
  		if( ! location.exists() )
  			location.mkdirs();
  		
  		return location;
  	}
  	
  	@Override
  	public boolean isIndexed() throws IOException 
  	{
  		return IndexReader.indexExists(getIndexDir()) && super.isIndexed();
  	}
  	
  	private static List<Pattern> getExcludedDirsRegExes() 
  	{
  		String excludeDirList = InstaSearchPlugin.getDefault().getPreferenceStore().getString(PreferenceConstants.P_EXCLUDE_DIRS);
  		
  		if( excludeDirList == null || "".equals(excludeDirList) ) 
  			return Collections.emptyList();
  		
  		List<Pattern> excludeDirSet = new ArrayList<Pattern>();
  		String[] excludedDirArr = excludeDirList.split(File.pathSeparator);
  		
  		for (String wildCardPattern : excludedDirArr)
  		{
  			Pattern pattern = null;
  			if( wildCardPattern.startsWith("/") ) wildCardPattern = wildCardPattern.substring(1);
  			
  			try {
  				pattern = wildcardToRegex(wildCardPattern);
  				excludeDirSet.add(pattern);
  			} catch(Throwable t) {
  				InstaSearchPlugin.debug(t);
  			}
  		}
  		
  		return excludeDirSet;
  	}
  	
  	private static String[] getIndexableFileExtensions() {
  		
  		String extensionList = InstaSearchPlugin.getDefault().getPreferenceStore().getString(PreferenceConstants.P_INDEXABLE_EXTENSIONS);
  		if( extensionList == null || "".equals(extensionList) || "*".equals(extensionList) ) return null;
  		
  		extensionList = extensionList.replace(" ", "");
  		
  		String[] extensions = extensionList.split("[,|;:]");
  		if( extensions.length == 0 ) return null;
  		
  		for(int i = 0; i<extensions.length; i++) {
  			String ext = extensions[i].toLowerCase(Locale.ENGLISH).trim();
  			if( ext.startsWith("*") ) ext = ext.substring(1);
  			if( ext.startsWith(".") ) ext = ext.substring(1);
  			extensions[i] = ext;
  		}
  		
  		Arrays.sort(extensions);
  		
  		return extensions;
  	}
  	
  	/**
  	 * @param file
  	 * @return isIndexable
  	 * @throws CoreException 
  	 */
  	public boolean isIndexable(IFile file) throws CoreException {
  		
  		String ext = file.getFileExtension();
  		
  		return isIndexableExtension(ext) || isTextFile(file);
  	}
  
  	/**
  	 * @param ext
  	 * @return isIndexableExtension
  	 */
  	public boolean isIndexableExtension(String ext) {
  		
  		if( fileExtensions == null || fileExtensions.length == 0 )
  			return true; // all files indexable
  		
  		if( ext == null || "".equals(ext) ) 
  			return indexEmptyExtension;
  		
  		if( Arrays.binarySearch(fileExtensions, ext.toLowerCase(Locale.ENGLISH)) >= 0 )
  			return true;
  		
  		return false;
  	}
  	
  	private void indexFile(IndexWriter indexWriter, IFile file) throws Exception {
  		
  		if( ! file.isAccessible() )
  			return;
  		
  		if( file.isDerived(IResource.CHECK_ANCESTORS) )
  			return;
  		
  		if( ! file.isSynchronized(IResource.DEPTH_ZERO) )
  			return;
  		
  		if( file.getRawLocation() == null ) // unknown location
  			return;
  		
  		File f = file.getRawLocation().toFile();
  		if( f == null || ! f.canRead() ) 
  			return;
  		
  		indexStorageWithRetry(indexWriter, file, file.getProject().getName(), file.getLocalTimeStamp(), null);
  	}
  	
  	/**
  	 * @param file
  	 * @throws Exception
  	 */
  	public void updateFile(IFile file) throws Exception {
  		
  		if( !isIndexed() )
  			return;
  		
  		deleteStorage(file);
  		
  		if( !isIndexable(file) )
  			return;
  		
  		if( file.isAccessible() && !file.isDerived(IResource.CHECK_ANCESTORS) ) 
  		{
  			IndexWriter w = createIndexWriter(false);
  			indexFile(w, file);
  			w.close();
  		}
  
  	}
  	
  	/**
  	 * Deletes and re-indexes files in a folder
  	 * 
  	 * @param folder
  	 * @param monitor 
  	 * @throws Exception 
  	 */
  	public void updateFolder(IFolder folder, IProgressMonitor monitor) throws Exception {
  		
  		if( !isIndexed() )
  			return;
  		
  		IndexReader reader = IndexReader.open(getIndexDir(), false);
  		deleteFolder(reader, folder);
  		reader.close();
  		
  		if( !folder.isAccessible() )	
  			return;
  		
  		resourceCollector.clear();
  		resourceCollector.setExcludedDirRegExes(excludedDirRegExes);
  		folder.accept(resourceCollector); // get also subfolders
  		
  		IndexWriter w = createIndexWriter(false);
  		
  		for(IContainer container: resourceCollector.getContainers())
  		{
  			if( isExcluded(container) ) continue;
  			indexContainer(w, container, monitor);
  		}
  		
  		w.close();
  	}
  
  	/**
  	 * @param container
  	 * @return
  	 */
  	private boolean isExcluded(IContainer container) {
  		
  		if( container == null || excludedDirRegExes == null || excludedDirRegExes.isEmpty() )
  			return false;
  		
  		return ResourceCollector.isResourceExcluded(container, excludedDirRegExes) 
  				|| isExcluded(container.getParent());
  	}
  
  	/**
  	 * @param project
  	 * @param monitor
  	 * @throws Exception
  	 */
  	public void updateProject(IProject project, IProgressMonitor monitor) throws Exception {
  		
  		if( !isIndexed() )
  			return;
  		
  		deleteProject(project);
  		
  		if( project.exists() && project.isAccessible() && project.isOpen() ) {
  			IndexWriter w = createIndexWriter(false);
  			
  			resourceCollector.clear();
  			resourceCollector.setExcludedDirRegExes(excludedDirRegExes);
  			project.accept(resourceCollector);
  			
  			for(IContainer container: resourceCollector.getContainers())
  			{
  				indexContainer(w, container, monitor);
  			}
  			
  			w.close();
  		}
  		
  	}
  	
  	/**
  	 * @param w
  	 * @param container
  	 * @param monitor 
  	 * @throws Exception 
  	 */
  	private void indexContainer(IndexWriter w, IContainer container, IProgressMonitor monitor) throws Exception
  	{
  		if( !container.isAccessible() || container.isDerived(IResource.CHECK_ANCESTORS) )
  			return;
  		
  		IResource[] members = container.members(false);
  		
  		for(IResource member: members) {
  			
  			if( monitor.isCanceled() )
  				return;
  			
  			if( member.getType() != IResource.FILE || !member.isAccessible() || member.isDerived() ) 
  				continue;
  			
  			IFile file = (IFile) member;
  			
  			if( isIndexable(file) )
  				indexFile(w, file);
  		}
  	}
  	
  	/**
  	 * @param file
  	 * @return
  	 * @throws CoreException 
  	 */
  	private boolean isTextFile(IFile file) throws CoreException {
  		
  		if( !file.isSynchronized(IResource.DEPTH_ZERO) )
  			return false;
  		
  		IContentType contentType = IDE.getContentType(file);
  		if( contentType == null ) contentType = IDE.guessContentType(file);
  		if( contentType == null ) return false;
  		
  		if( TEXT_CONTENT_TYPE != null && contentType.isKindOf(TEXT_CONTENT_TYPE) )
  			return true;
  		
  		return false;
  	}
  
  	/**
  	 * @throws IOException 
  	 * 
  	 */
  	private void deleteFolder(IndexReader reader, IContainer container) throws Exception {
  		
  		IndexSearcher searcher = new IndexSearcher(reader);
  		String path = container.getFullPath().addTrailingSeparator().toString();
  		TopDocs topDocs = searcher.search(new PrefixQuery(Field.FILE.createTerm(path)), reader.numDocs());
  		
  		for(ScoreDoc doc: topDocs.scoreDocs)
  		{
  			int docNum = doc.doc;
  			reader.deleteDocument(docNum);
  		}
  		
  		searcher.close();
  	}
  	
  	/**
  	 * @param project
  	 * @return deletedCount
  	 * @throws Exception
  	 */
  	public int deleteProject(IProject project) throws Exception {
  		IndexReader reader = IndexReader.open(getIndexDir(), false);
  		String filePath = project.getFullPath().toString();
  		
  		Term term = Field.PROJ.createTerm(filePath);
  		int deletedCount = reader.deleteDocuments(term);
  		
  		reader.close();
  		
  		return deletedCount;
  	}
  	
  	public boolean isConflicting(ISchedulingRule rule)
  	{
  		return rule == this;  // prevent concurrent writing of the index from jobs
  	}
  
  	public boolean contains(ISchedulingRule rule)
  	{
  		return rule == this;
  	}
  
  	/**
  	 * @param doc
  	 * @return IEditorInput
  	 * @throws Exception 
  	 * @throws IOException 
  	 */
  	public IEditorInput getEditorInput(SearchResultDoc doc) throws Exception {
  		
  		return new FileEditorInput(doc.getFile());
  	}
  
  	/**
  	 * @param doc
  	 * @return IStorage
  	 * @throws Exception 
  	 * @throws IOException 
  	 */
  	public IStorage getStorage(SearchResultDoc doc) throws Exception {
  		return doc.getFile(); 
  	}
  	
  	public void propertyChange(PropertyChangeEvent event) {
  		String prop = event.getProperty();
  		
  		if( PreferenceConstants.P_INDEXABLE_EXTENSIONS.equals(prop) )
  			fileExtensions = getIndexableFileExtensions();
  		else if( PreferenceConstants.P_EXCLUDE_DIRS.equals(prop) )
  			excludedDirRegExes = getExcludedDirsRegExes();	
  		else if( PreferenceConstants.P_INDEX_EMPTY_EXTENSION.equals(prop) )
  			indexEmptyExtension = InstaSearchPlugin.getBoolPref(PreferenceConstants.P_INDEX_EMPTY_EXTENSION);
  		
  	}
  	
  	/**
  	 * Convert path matching wildcard pattern to regular expression.
  	 * 
  	 * 
  	 * @param pathWildcardPattern
  	 * @return regex pattern
  	 */
  	public static Pattern wildcardToRegex(String pathWildcardPattern)
  	{
  		String regex = pathWildcardPattern;
  		
  		regex = regex.replaceAll("\\*\\*", "<double-star>"); // escape initially
  		regex = regex.replaceAll("\\.", "\\."); // escape .
  		regex = regex.replaceAll("\\*", "[^/]*");
  		regex = regex.replaceAll("\\?", ".");
  		regex = regex.replaceAll("<double-star>", ".*");
  		
  		Pattern pattern = Pattern.compile(regex);
  		
  		return pattern;
  	}
  }
  