content: 
  //A script to headlessly create FIDB files based on all programs within the given project. Based on "CreateMultipleLibraries.java" by the Ghidra team, inspired by Threatrack's work (https://blog.threatrack.de/2019/09/20/ghidra-fid-generator/)
  //@author Max 'Libra' Kersten for Trellix
  //@category FunctionID
  //@keybinding
  //@menupath
  //@toolbar
  import java.io.BufferedReader;
  import java.io.File;
  import java.io.FileReader;
  import java.io.IOException;
  import java.nio.file.Files;
  import java.nio.file.Path;
  import java.nio.file.StandardOpenOption;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.LinkedList;
  import java.util.List;
  import java.util.Map;
  import java.util.Map.Entry;
  import java.util.Set;
  import java.util.TreeSet;
  
  import ghidra.app.script.GhidraScript;
  import ghidra.feature.fid.db.FidDB;
  import ghidra.feature.fid.db.FidFile;
  import ghidra.feature.fid.db.FidFileManager;
  import ghidra.feature.fid.db.LibraryRecord;
  import ghidra.feature.fid.service.FidPopulateResult;
  import ghidra.feature.fid.service.FidPopulateResult.Disposition;
  import ghidra.feature.fid.service.FidPopulateResultReporter;
  import ghidra.feature.fid.service.FidService;
  import ghidra.feature.fid.service.Location;
  import ghidra.framework.model.DomainFile;
  import ghidra.framework.model.DomainFolder;
  import ghidra.program.database.ProgramContentHandler;
  import ghidra.program.model.lang.LanguageID;
  import ghidra.program.model.mem.MemoryAccessException;
  import ghidra.util.Msg;
  import ghidra.util.exception.CancelledException;
  import ghidra.util.exception.VersionException;
  import ghidra.util.task.TaskMonitor;
  
  public class AutomaticallyCreateFunctionIdDatabases extends GhidraScript {
  
  	/**
  	 * The Function ID service to be used
  	 */
  	private FidService fidService;
  
  	/**
  	 * The path to the log file where all logs are written (appended) to
  	 */
  	private Path logFile;
  
  	/**
  	 * Logs a message to Ghidra's console and to the log file in the specified
  	 * output folder
  	 * 
  	 * @param message the message to print and write
  	 */
  	private void log(String message, boolean isError) {
  		/*
  		 * Write the (error) message to Ghidra's console, depending on the given boolean
  		 */
  		if (isError) {
  			printerr(message);
  		} else {
  			println(message);
  		}
  
  		/*
  		 * Try to append data to the log file
  		 */
  		try {
  			// Append the error tag if the message is an error
  			if (isError) {
  				message = "ERROR: " + message;
  			}
  			// Append a newline to the message
  			message += "\n";
  			// Write the data to the log file
  			Files.write(logFile, message.getBytes(), StandardOpenOption.APPEND);
  		} catch (IOException e) {
  			// Ignore exceptions, the console log already contains the logged information
  		}
  	}
  
  	/**
  	 * The reporter class, which implements the required Ghidra report class
  	 */
  	class FidReporter implements FidPopulateResultReporter {
  		@Override
  		public void report(FidPopulateResult result) {
  			// Check if a result is given, if not, simply return
  			if (result == null) {
  				return;
  			}
  
  			// Get the library record from the result
  			LibraryRecord libraryRecord = result.getLibraryRecord();
  
  			// Get the library family name, version, and variant
  			String libraryFamilyName = libraryRecord.getLibraryFamilyName();
  			String libraryVersion = libraryRecord.getLibraryVersion();
  			String libraryVariant = libraryRecord.getLibraryVariant();
  
  			// Log the obtained information
  			log(libraryFamilyName + ':' + libraryVersion + ':' + libraryVariant, false);
  
  			// Log the attempted, added, and excluded count for the generated FIDB
  			log(result.getTotalAttempted() + " total functions visited", false);
  			log(result.getTotalAdded() + " total functions added", false);
  			log(result.getTotalExcluded() + " total functions excluded", false);
  
  			// Log all exclusions
  			log("Breakdown of exclusions:", false);
  			for (Entry<Disposition, Integer> entry : result.getFailures().entrySet()) {
  				if (entry.getKey() != Disposition.INCLUDED) {
  					log("    " + entry.getKey() + ": " + entry.getValue(), false);
  				}
  			}
  			// Log all unresolved symbols
  			log("List of unresolved symbols:", false);
  			TreeSet<String> symbols = new TreeSet<>();
  			for (Location location : result.getUnresolvedSymbols()) {
  				symbols.add(location.getFunctionName());
  			}
  			for (String symbol : symbols) {
  				log("    " + symbol, false);
  			}
  		}
  	}
  
  	/**
  	 * Parses the symbols within the given file, reading it line-by-line where each
  	 * line is assumed to be a symbol
  	 * 
  	 * @param commonSymbolsFile the file to read
  	 * @return a list of symbols to exclude when making FunctionID signatures
  	 * @throws IOException
  	 * @throws CancelledException
  	 */
  	private List<String> parseSymbols(File commonSymbolsFile) throws IOException, CancelledException {
  		// Check if the common symbols file exists
  		if (commonSymbolsFile.exists() == false) {
  			return null;
  		}
  
  		// If it exists, read the file
  		BufferedReader reader = new BufferedReader(new FileReader(commonSymbolsFile));
  
  		// Create a new linked list to store the common symbols in
  		List<String> commonSymbols = new LinkedList<>();
  
  		// Read a single line from the file
  		String line = reader.readLine();
  
  		// Iterate over all lines
  		while (line != null) {
  			/*
  			 * If the monitor isn't cancelled, continue, else throw an exception which
  			 * causes the whole script to halt
  			 */
  			monitor.checkCancelled();
  
  			// If the line of the length is not zero
  			if (line.length() != 0) {
  				// Add it to the list
  				commonSymbols.add(line);
  			}
  			// Read the next line
  			line = reader.readLine();
  		}
  
  		// Once the entire file has been read, close the file access
  		reader.close();
  
  		// Return all common symbols
  		return commonSymbols;
  	}
  
  	/**
  	 * Recursively generate a mapping for all programs within the given folder and
  	 * its sub-folders
  	 * 
  	 * @param folder              the folder to start in
  	 * @param languageIdMapping   the LanguageID mapping which is populated within
  	 *                            each call
  	 * @param rawLanguageIds      the raw LanguageIDs
  	 * @param languageIds         the LanguageID
  	 * @param compilerSpecMapping the compiler spec mapping
  	 * @throws CancelledException
  	 */
  	private void generateLanguageIdProgramMapping(Map<String, List<DomainFile>> languageIdMapping, DomainFolder folder,
  			Set<String> rawLanguageIds, Set<String> languageIds, Map<String, List<DomainFile>> compilerSpecMapping)
  			throws CancelledException {
  		// Iterate over all domain files within the given folder
  		for (DomainFile domainFile : folder.getFiles()) {
  			/*
  			 * As within any longer loop, a cancel check is present to allow a user to
  			 * cancel the script in a timely manner
  			 */
  			monitor.checkCancelled();
  
  			/*
  			 * Only if the file is of the "PROGRAM" type, it is to be used
  			 */
  			if (domainFile.getContentType().equals(ProgramContentHandler.PROGRAM_CONTENT_TYPE)) {
  				Map<String, String> metadata = domainFile.getMetadata();
  
  				String languageId = metadata.get("Language ID");
  				if (languageId != null) {
  					rawLanguageIds.add(languageId);
  
  					/*
  					 * An example of a raw language ID is "x86:LE:32:default (3.0)". Splitting by
  					 * the space removes the version number, giving the language ID that is required
  					 * for later use.
  					 */
  					String filteredLanguageId = languageId.split(" ")[0];
  
  					languageIds.add(filteredLanguageId);
  
  					/*
  					 * Attempt to get the value from the mapping for the given key. If the result is
  					 * null, no such key (nor value) is present in the mapping.
  					 */
  					List<DomainFile> domainFiles = languageIdMapping.get(filteredLanguageId);
  
  					/*
  					 * If the result is null, there is no key in the mapping, meaning it is new. A
  					 * new list is created, and the key and value are added to the mapping.
  					 */
  					if (domainFiles == null) {
  						domainFiles = new ArrayList<>();
  						domainFiles.add(domainFile);
  						languageIdMapping.put(filteredLanguageId, domainFiles);
  					} else {
  						/*
  						 * Since the list from the mapping is not null, the key exists with a value. Add
  						 * the current file to the list, and overwrite the mapping's entry with the new
  						 * value for the given key.
  						 */
  						domainFiles.add(domainFile);
  						languageIdMapping.put(filteredLanguageId, domainFiles);
  					}
  
  				}
  			}
  		}
  		// Iterate over all folders within the given folder
  		for (DomainFolder domainFolder : folder.getFolders()) {
  			monitor.checkCancelled();
  			generateLanguageIdProgramMapping(languageIdMapping, domainFolder, rawLanguageIds, languageIds,
  					compilerSpecMapping);
  		}
  	}
  
  	/**
  	 * Populates an FIDB file based on the given list of programs for the given
  	 * language ID, excluding the given common symbols
  	 * 
  	 * @param fidDb          the database to populate
  	 * @param languageID     the LanguageID to match with
  	 * @param programs       the programs to parse
  	 * @param commonSymbols  the common symbols to exclude
  	 * @param reporter       the report to use when returning results
  	 * @param projectName    the project name to use within the database
  	 * @param projectVersion the version to use within the database
  	 * @throws CancelledException
  	 */
  	private void createLibraryFromPrograms(FidDB fidDb, LanguageID languageID, List<DomainFile> programs,
  			List<String> commonSymbols, FidReporter reporter, String projectName, String projectVersion)
  			throws CancelledException {
  		/*
  		 * Set the library name, version, and variant
  		 */
  		String libraryName = projectName;
  		String libraryVersion = projectVersion;
  		String libraryVariant = languageID.getIdAsString();
  
  		// Create the message
  		String message = "[" + libraryVariant + "] Generating \"" + libraryName + " (" + libraryVersion
  				+ ")\" based on " + programs.size() + " files";
  		// Log the message
  		log(message, false);
  		// Set the monitor message to inform the analyst when running via a GUI
  		monitor.setMessage(message);
  
  		try {
  			// Call the FID service to create populate the new library
  			FidPopulateResult result = fidService.createNewLibraryFromPrograms(fidDb, libraryName, libraryVersion,
  					libraryVariant, programs, null, languageID, null, commonSymbols, TaskMonitor.DUMMY);
  			// report the results via the reporter
  			reporter.report(result);
  		} catch (MemoryAccessException e) {
  			Msg.showError(this, null, "Unexpected memory access exception", "Please notify the Ghidra team:", e);
  		} catch (VersionException e) {
  			Msg.showError(this, null, "Version Exception",
  					"One of the programs in your domain folder cannot be upgraded: " + e.getMessage());
  		} catch (IllegalStateException e) {
  			Msg.showError(this, null, "Illegal State Exception", "Unknown error: " + e.getMessage());
  		} catch (IOException e) {
  			Msg.showError(this, null, "FidDb IOException", "Please notify the Ghidra team:", e);
  		}
  	}
  
  	/**
  	 * Creates a new FunctionID database file
  	 * 
  	 * @param fidbFile    the file to be created
  	 * @param fidFileName the file name to use
  	 * @return the newly created FIDB file with write permissions
  	 * @throws IOException
  	 * @throws VersionException
  	 */
  	private FidDB createFidDB(File fidbFile, String fidFileName) throws IOException, VersionException {
  		// Declare the object
  		FidFile fidFile = null;
  
  		// Get the FID file manager
  		FidFileManager fidFileManager = FidFileManager.getInstance();
  
  		// Create a new database
  		fidFileManager.createNewFidDatabase(fidbFile);
  
  		// Get all user added FIDB files
  		List<FidFile> fids = fidFileManager.getUserAddedFiles();
  
  		// Iterate over all FIDB files
  		for (FidFile fid : fids) {
  			// If the name equals the given file name
  			if (fid.getName().equalsIgnoreCase(fidFileName)) {
  				// If the match is there, set the return variable and break the loop
  				fidFile = fid;
  				break;
  			}
  		}
  
  		// If no matching file was found, something went wrong and null is returned
  		if (fidFile == null) {
  			return null;
  		}
  
  		// Return the FIDB file with write permissions
  		return fidFile.getFidDB(true);
  	}
  
  	@Override
  	protected void run() throws Exception {
  		/*
  		 * A note about the askSomething functions. This script is intended to be
  		 * executed by Ghidra in its headless mode, since the GUI serves no purpose in
  		 * this script. Any askSomething function will take an argument from the
  		 * command-line interface when running headless. As such, it allows one to use
  		 * the command-line interface in an easy manner, or with the help of a
  		 * properties file. In this case, all askSomething functions are listed in the
  		 * start of the run-method, since the command-line interface argument order
  		 * matters. This is a convenient way for users to understand what arguments are
  		 * to be provided when running this script headless, without going over all code
  		 * within this script.
  		 */
  
  		/*
  		 * Ask for the project name, which is to be used in the to-be created FIDBs
  		 */
  		String projectName = askString("Project name", "What is the project name?");
  
  		/*
  		 * Ask for the project version, which is to be used in the to-be created FIDBs
  		 */
  		String projectVersion = askString("Project version", "What is the project version?");
  
  		/*
  		 * Ask for the output folder. This will be the location where the FIDB files
  		 * along with the log file will be written to.
  		 */
  		String outputFolder = askString("Desired output folder", "What is the desired output folder?");
  
  		// Create a file object for the output folder
  		File outputFolderFile = new File(outputFolder);
  
  		// If the output folder does not exist, create all required folders
  		if (outputFolderFile.exists() == false) {
  			outputFolderFile.mkdirs();
  		}
  
  		/*
  		 * Get the absolute path for the output folder, excluding a trailing folder
  		 * separator
  		 */
  		outputFolder = outputFolderFile.getAbsolutePath();
  
  		/*
  		 * Ask if a common symbols file is to be used
  		 */
  		boolean useCommonSymbolsFile = askYesNo("Common symbols file usage",
  				"Do you want to use a common symbols file?");
  
  		/*
  		 * The path to the symbols file, if desired
  		 */
  		String symbolsFilePath = null;
  
  		/*
  		 * The file object which refers to the provided path
  		 */
  		File commonSymbolsFile = null;
  
  		/*
  		 * The list of symbols
  		 */
  		List<String> symbols = null;
  
  		/*
  		 * If a common symbols file is desired, the path to said file is requested,
  		 * after which a file object pointing to said location is instantiated. Lastly,
  		 * the symbols are parsed, and stored in the symbols list. If the common symbols
  		 * file cannot be found, the script returns early with a related error message
  		 */
  		if (useCommonSymbolsFile) {
  			symbolsFilePath = askString("Common symbols file location",
  					"What is the location of the common symbols file?");
  			log("Received the following location for the common symbols file: " + symbolsFilePath, false);
  			commonSymbolsFile = new File(symbolsFilePath);
  			if (commonSymbolsFile.exists() == false) {
  				log("Cannot find the common symbols file:" + symbolsFilePath, true);
  				return;
  			}
  			symbols = parseSymbols(commonSymbolsFile);
  		}
  
  		/*
  		 * Initialise the FunctionID service
  		 */
  		fidService = new FidService();
  
  		/*
  		 * Initialises the log file object
  		 */
  		File log = new File(outputFolder + File.separator + "log.txt");
  
  		/*
  		 * Check if the log file exists. If it does, no action is required, as data can
  		 * be appended to it. If it does not, the file itself is created
  		 */
  		if (log.exists() == false) {
  			log.createNewFile();
  		}
  
  		/*
  		 * Set the log file, which is used by the log function
  		 */
  		logFile = log.toPath();
  
  		/*
  		 * Logging is possible from this point
  		 */
  		log("Received project name: " + projectName, false);
  		log("Received project version: " + projectVersion, false);
  		log("Received output folder path: " + outputFolder, false);
  		log("Received choice with regards to the usage of a common symbols file: " + useCommonSymbolsFile, false);		
  
  		/*
  		 * Initialise a new report instance, to later report on the FIDB generation
  		 * results
  		 */
  		FidReporter reporter = new FidReporter();
  
  		/*
  		 * Gets the root folder of the current project, allowing the recursive calls
  		 * later on to start at the root node
  		 */
  		DomainFolder rootFolder = state.getProject().getProjectData().getRootFolder();
  
  		/*
  		 * Contains a mapping of the original compiler spec IDs for the given list of
  		 * files
  		 */
  		Map<String, List<DomainFile>> compilerSpecMapping = new HashMap<>();
  
  		/*
  		 * Contains all language IDs without version information
  		 */
  		Set<String> languageIds = new HashSet<>();
  
  		/*
  		 * Contains all language IDs with version information
  		 */
  		Set<String> rawLanguageIds = new HashSet<>();
  
  		/*
  		 * Initialises the mapping of language IDs and their respective domain files
  		 */
  		Map<String, List<DomainFile>> languageIdProgramMapping = new HashMap<>();
  
  		/*
  		 * Generate a mapping, stored in a global variable, which contains unique
  		 * language ID strings as a key, and a list of programs associated with said
  		 * language ID as value. This mapping is used later on to make FIDBs for each
  		 * language ID, with the respective files.
  		 */
  		generateLanguageIdProgramMapping(languageIdProgramMapping, rootFolder, rawLanguageIds, languageIds,
  				compilerSpecMapping);
  
  		/*
  		 * Create a log message with all unique language IDs, including their version
  		 */
  		String collectLanguageIds = "Collected language IDs and their versions:\n";
  		for (String id : rawLanguageIds) {
  			collectLanguageIds += "\t" + id + "\n";
  		}
  
  		/*
  		 * Write the message to Ghidra's console and the log file
  		 */
  		log(collectLanguageIds, false);
  
  		/*
  		 * Create a string which is to be logged, which contains each unique language ID
  		 * and the number of files that are to be processed for said language ID
  		 */
  		String languageIdFileMessage = "";
  		for (Map.Entry<String, List<DomainFile>> entry : languageIdProgramMapping.entrySet()) {
  			String languageID = entry.getKey();
  			List<DomainFile> domainFiles = entry.getValue();
  
  			languageIdFileMessage += languageID + " contains " + domainFiles.size() + " files\n";
  		}
  
  		// Log the language ID and file summary
  		log(languageIdFileMessage, false);
  
  		/*
  		 * Iterate over the language IDs without version information
  		 */
  		for (String id : languageIds) {
  			/*
  			 * Replace the colon with a dot to avoid invalid file name errors. The replace
  			 * all function uses regular expressions for both arguments, meaning that the
  			 * dot has to be escaped
  			 */
  			String fileName = id.replaceAll(":", "\\.") + ".fidb";
  
  			/*
  			 * The FIDB file name is the file name that the file will have on disk, based on
  			 * the project name, project version, and the language ID
  			 */
  			String fidFileName = projectName.replaceAll(" ", "\\.") + "_" + projectVersion.replaceAll(" ", "\\.") + "_"
  					+ fileName;
  
  			// Create a file object for the FIDB file
  			File fidbFile = new File(outputFolder + File.separator + fidFileName);
  
  			/*
  			 * The createNewFidDatabase function requires the file to NOT exist, meaning it
  			 * needs to be deleted if it exists
  			 */
  			if (fidbFile.exists()) {
  				String error = "The FIDB file already exists: " + fidbFile.getAbsolutePath();
  				log(error, true);
  				return;
  			}
  
  			/*
  			 * Create the FIDB file
  			 */
  			FidDB fidDb = createFidDB(fidbFile, fidFileName);
  
  			/*
  			 * If createFidDB returns null, the file could not be found, and the execution
  			 * needs to exit
  			 */
  			if (fidDb == null) {
  				log("Cannot find any FIDB file named \"" + fidFileName + "\"", true);
  				return;
  			}
  
  			/*
  			 * Create a language ID object, which is used and required during the generation
  			 * of the FIDB
  			 */
  			LanguageID languageID = new LanguageID(id);
  
  			/*
  			 * A list of all programs which fall under the given language ID, within the
  			 * given Ghidra project
  			 */
  			List<DomainFile> programs = languageIdProgramMapping.get(id);
  
  			/*
  			 * Generate the hashes for functions for each program in the list, for the given
  			 * language ID
  			 */
  			createLibraryFromPrograms(fidDb, languageID, programs, symbols, reporter, projectName, projectVersion);
  
  			// Save the database, otherwise all changes will be lost
  			fidDb.saveDatabase("Saving", monitor);
  
  			// Close the FIDB
  			fidDb.close();
  		}
  	}
  }