content: {-# LANGUAGE OverloadedStrings #-}
  {-# LANGUAGE RecordWildCards #-}
  {-# LANGUAGE ScopedTypeVariables #-}
  {-# LANGUAGE LambdaCase #-}
  {-# LANGUAGE DeriveGeneric #-}
  {-# LANGUAGE DerivingStrategies #-}
  {-# LANGUAGE DeriveAnyClass #-}
  {-# LANGUAGE TypeApplications #-}
  
  -- |
  -- Module      : Clod.FileSystem.Checksums
  -- Description : Checksums-based file tracking for Clod
  -- Copyright   : (c) Fuzz Leonard, 2025
  -- License     : MIT
  -- Maintainer  : fuzz@fuzz.ink
  -- Stability   : experimental
  --
  -- This module provides functions for tracking file changes using checksums.
  -- It calculates XXH3 (64-bit) hashes of file content and maintains a database of files
  -- that have been processed, allowing us to detect new, modified, deleted, and renamed files.
  --
  -- The file checksum database is stored as a Dhall configuration file with the following structure:
  --
  -- @
  -- { files =
  --     { "path/to/file1.txt" =
  --         { path = "path/to/file1.txt"
  --         , checksum = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
  --         , lastModified = "2025-01-01T12:00:00Z"
  --         , optimizedName = "path-to-file1.txt"
  --         }
  --     , "path/to/file2.md" =
  --         { path = "path/to/file2.md"
  --         , checksum = "7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730"
  --         , lastModified = "2025-01-02T14:30:00Z"
  --         , optimizedName = "path-to-file2.md"
  --         }
  --     }
  -- , checksums =
  --     { "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" = "path/to/file1.txt"
  --     , "7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730" = "path/to/file2.md"
  --     }
  -- , lastStagingDir = Some "./staging/20250101-120000"
  -- , lastRunTime = "2025-01-01T12:00:00Z"
  -- }
  -- @
  --
  -- This database allows efficient lookup of files by path or checksum,
  -- detection of renamed files (same content with different paths),
  -- and tracking of previous staging directories.
  
  module Clod.FileSystem.Checksums
    ( -- * Checksum operations
      calculateChecksum
    , checksumFile
      
      -- * Database operations
    , initializeDatabase
    , loadDatabase
    , saveDatabase
    , updateDatabase
    
      -- * Change detection
    , detectFileChanges
    , findChangedFiles
    , findRenamedFiles
    , getFileStatus
    , FileStatus(..)
    
      -- * Database management
    , cleanupStagingDirectories
    , flushMissingEntries
    ) where
  
  import Control.Exception (try, IOException, SomeException)
  import Control.Monad (forM, when)
  import qualified Data.ByteString as BS
  import qualified Data.Map.Strict as Map
  import Data.Maybe (mapMaybe, catMaybes)
  import Data.Time (UTCTime, getCurrentTime)
  import System.Directory (doesFileExist, doesDirectoryExist, getModificationTime, 
                           removeDirectoryRecursive, createDirectoryIfMissing, renameFile)
  import System.FilePath ((</>), takeDirectory)
  import GHC.Generics (Generic)
  import Numeric (showHex)
  import Clod.Types ((^.), (.~), (%~), (&), verbose, flushMode, dbFiles, dbChecksums,
                    entryChecksum, entryPath, previousStaging, 
                    ClodDatabase(..), FileEntry(..), 
                    Checksum(..), ClodM, FileReadCap(..), ClodError(..), OptimizedName(..),
                    DatabaseErrorType(..), -- Add the DatabaseErrorType
                    toSerializable, fromSerializable, ask, liftIO, throwError)
  import Clod.FileSystem.Detection (safeFileExists, safeIsTextFile)
  import Clod.FileSystem.Operations (safeReadFile)
  import Clod.Output (whenVerbose)
  
  import qualified Data.Text.IO as TextIO
  import qualified Dhall
  import qualified Dhall.Core
  import qualified Data.Digest.XXHash.FFI as XXH
  import Data.Hashable (hash)
  
  -- User error helper for IOErrors
  createError :: String -> IOError
  createError = Prelude.userError
  
  -- | Data type for tracking file status
  data FileStatus
    = Unchanged     -- ^ File has not changed
    | New           -- ^ New file
    | Modified      -- ^ Existing file with modified content
    | Deleted       -- ^ File no longer exists
    | Renamed FilePath  -- ^ File was renamed (new path)
    deriving (Show, Eq, Generic)
  
  -- | Calculate XXH3 checksum (64-bit) of a ByteString
  -- XXH3 is a fast non-cryptographic hash function with excellent performance
  calculateChecksum :: BS.ByteString -> Checksum
  calculateChecksum content =
    let -- Use the newer XXH3 implementation (faster than xxh64)
        hashVal = hash (XXH.XXH3 content)
        -- Convert the hash to an absolute value to handle negative hash values
        absHash = abs hashVal
        -- Convert the 64-bit integer to a hex string for consistent representation
        hexStr = showHex absHash ""
    in Checksum hexStr
  
  -- | Calculate the checksum of a file
  -- Only text files are allowed to be checksummed
  checksumFile :: FileReadCap -> FilePath -> ClodM Checksum
  checksumFile readCap path = do
    -- Check if file exists
    fileExists <- safeFileExists readCap path
    if not fileExists
      then throwError $ FileSystemError path (createError "File does not exist")
      else do
        -- Check if it's a text file
        isText <- safeIsTextFile readCap path
        if not isText
          then throwError $ ChecksumError path "Cannot checksum binary or ineligible file"
          else do
            -- Read file content and calculate checksum
            content <- safeReadFile readCap path
            return $ calculateChecksum content
  
  -- | Initialize a new, empty database
  initializeDatabase :: ClodM ClodDatabase
  initializeDatabase = do
    currentTime <- liftIO getCurrentTime
    return $ ClodDatabase
      { _dbFiles = Map.empty
      , _dbChecksums = Map.empty
      , _dbLastStagingDir = Nothing
      , _dbLastRunTime = currentTime
      }
  
  -- | Load the database from disk using Dhall
  loadDatabase :: FilePath -> ClodM ClodDatabase
  loadDatabase dbPath = do
    -- Check if the database file exists
    fileExists <- liftIO $ doesFileExist dbPath
    if not fileExists
      then do
        -- If it doesn't exist, create a new database
        db <- initializeDatabase
        -- Ensure the directory exists and save
        liftIO $ createDirectoryIfMissing True (takeDirectory dbPath)
        saveDatabase dbPath db
        return db
      else do
        -- Try to parse the database file using Dhall
        eitherResult <- liftIO $ try @SomeException $ do
          -- Use Dhall.inputFile to correctly parse the database file
          sdb <- Dhall.inputFile Dhall.auto dbPath
          -- Convert to ClodDatabase and return
          return $ fromSerializable sdb
        
        case eitherResult of
          Right db -> return db
          Left err -> do
            -- If parsing fails, log the error in verbose mode
            config <- ask
            when (config ^. verbose) $ do
              liftIO $ putStrLn $ "Warning: Failed to parse database: " ++ show err
              -- Log the enhanced error format for better diagnostics
              liftIO $ putStrLn $ "Database error details: " ++ 
                show (DatabaseError dbPath (DBCorrupted (show err)))
            
            -- Create a new database
            whenVerbose $ liftIO $ putStrLn "Creating a new empty database"
            db <- initializeDatabase
            -- Save it right away to ensure it's in the right format for next time
            saveDatabase dbPath db
            return db
  
  -- | Save the database to disk using Dhall serialization
  saveDatabase :: FilePath -> ClodDatabase -> ClodM ()
  saveDatabase dbPath db = do
    -- Ensure the directory exists
    liftIO $ createDirectoryIfMissing True (takeDirectory dbPath)
    
    -- Convert to serializable form 
    let serializedDb = toSerializable db
    
    -- Write to temporary file first to avoid locking issues
    let tempPath = dbPath ++ ".new"
    
    -- Use proper Dhall encoding
    eitherResult <- liftIO $ try @IOException $ do
      -- Use Dhall's encoding to create a properly formatted Dhall expression
      let dhallExpr = Dhall.embed Dhall.inject serializedDb
      let dhallText = Dhall.Core.pretty dhallExpr
      
      -- Write to the temp file
      TextIO.writeFile tempPath dhallText
      -- Then rename to actual path (atomic operation on most filesystems)
      renameFile tempPath dbPath
    
    case eitherResult of
      Left err -> throwError $ DatabaseError dbPath (DBOtherError $ "Failed to save: " ++ show err)
      Right _ -> whenVerbose $ liftIO $ putStrLn $ "Successfully saved database to: " ++ dbPath
        
  
  
  -- | Update the database with a new file entry
  updateDatabase :: ClodDatabase -> FilePath -> Checksum -> UTCTime -> OptimizedName -> ClodDatabase
  updateDatabase db path checksum modTime optName =
    let 
      -- Create new file entry
      newEntry = FileEntry
        { _entryPath = path
        , _entryChecksum = checksum
        , _entryLastModified = modTime
        , _entryOptimizedName = optName
        }
      
      -- Update maps using lenses
      db1 = db & dbFiles %~ Map.insert path newEntry
      db2 = db1 & dbChecksums %~ Map.insert (unChecksum checksum) path
    in
      db2
  
  -- | Detect file status by comparing against database
  getFileStatus :: ClodDatabase -> FilePath -> Checksum -> ClodM FileStatus
  getFileStatus db path checksum = do
    let 
      files = db ^. dbFiles
      checksums = db ^. dbChecksums
      checksumStr = unChecksum checksum
  
    -- Check if file exists in database
    case Map.lookup path files of
      -- File doesn't exist in database
      Nothing -> 
        -- Check if file with same checksum exists (renamed file)
        case Map.lookup checksumStr checksums of
          Just oldPath -> 
            -- Only consider it renamed if the old path is different
            if oldPath /= path 
              then return $ Renamed oldPath
              else return New
          Nothing -> return New
  
      -- File exists in database
      Just entry ->
        -- Check if checksum matches
        if entry ^. entryChecksum == checksum
          then return Unchanged
          else return Modified
  
  -- | Find files that need processing (new, modified, renamed)
  findChangedFiles :: ClodDatabase -> [(FilePath, Checksum, UTCTime)] -> ClodM [(FilePath, FileStatus)]
  findChangedFiles db fileInfos = do
    whenVerbose $ liftIO $ putStrLn $ "Processing " ++ show (length fileInfos) ++ " files for change detection"
    
    -- Process each file
    forM fileInfos $ \(path, checksum, _) -> do
      status <- getFileStatus db path checksum
      whenVerbose $ liftIO $ putStrLn $ "File status for " ++ path ++ ": " ++ show status
      return (path, status)
  
  -- | Find files that have been renamed
  findRenamedFiles :: ClodDatabase -> [(FilePath, FileStatus)] -> [(FilePath, FilePath)]
  findRenamedFiles _ fileStatuses =
    mapMaybe extractRename fileStatuses
    where
      extractRename (newPath, Renamed oldPath) = Just (newPath, oldPath)
      extractRename _ = Nothing
  
  -- | Detect changes by comparing current files with database
  detectFileChanges :: FileReadCap -> ClodDatabase -> [FilePath] -> FilePath -> ClodM ([(FilePath, FileStatus)], [(FilePath, FilePath)])
  detectFileChanges readCap db filePaths projectRoot = do
    whenVerbose $ liftIO $ putStrLn $ "Detecting changes for " ++ show (length filePaths) ++ " files"
    whenVerbose $ liftIO $ putStrLn $ "Database has " ++ show (Map.size (db ^. dbFiles)) ++ " entries"
    
    -- For each file, calculate checksum and get modification time
    fileInfos <- catMaybes <$> forM filePaths (\path -> do
        let fullPath = projectRoot </> path
        
        -- Check if file exists
        fileExists <- safeFileExists readCap fullPath
        if not fileExists
          then do
            whenVerbose $ liftIO $ putStrLn $ "File does not exist: " ++ fullPath
            return Nothing
          else do
            -- Check if it's a text file
            isText <- safeIsTextFile readCap fullPath
            if not isText
              then do
                whenVerbose $ liftIO $ putStrLn $ "Not a text file: " ++ fullPath
                return Nothing
              else do
                -- Calculate checksum
                checksum <- checksumFile readCap fullPath
                -- Get modification time
                modTime <- liftIO $ getModificationTime fullPath
                whenVerbose $ liftIO $ putStrLn $ "Processed file: " ++ path ++ " with checksum: " ++ unChecksum checksum
                return $ Just (path, checksum, modTime)
      )
    
    -- Detect file statuses
    whenVerbose $ liftIO $ putStrLn $ "Got " ++ show (length fileInfos) ++ " files to check"
    changedFiles <- findChangedFiles db fileInfos
    
    -- Find renamed files
    let renamedFiles = findRenamedFiles db changedFiles
    whenVerbose $ liftIO $ putStrLn $ "Found " ++ show (length renamedFiles) ++ " renamed files"
    
    return (changedFiles, renamedFiles)
    
  
  
  -- | Clean up old staging directories
  cleanupStagingDirectories :: ClodM ()
  cleanupStagingDirectories = do
    config <- ask
    
    -- Check if there's a previous staging directory to clean up
    case config ^. previousStaging of
      Nothing -> return ()
      Just oldDir -> do
        -- Check if directory exists
        dirExists <- liftIO $ doesDirectoryExist oldDir
        when dirExists $ do
          -- Remove the directory if it exists
          whenVerbose $ liftIO $ putStrLn $ "Cleaning up previous staging directory: " ++ oldDir
          result <- liftIO $ try $ removeDirectoryRecursive oldDir :: ClodM (Either IOException ())
          case result of
            Left err -> whenVerbose $ liftIO $ putStrLn $ "Warning: Failed to remove old staging directory: " ++ show err
            Right _ -> return ()
  
  
  -- | Find and remove missing files from the database
  flushMissingEntries :: FileReadCap -> ClodDatabase -> FilePath -> ClodM ClodDatabase
  flushMissingEntries readCap db projectRoot = do
    config <- ask
    
    -- Don't proceed unless in flush mode
    if not (config ^. flushMode)
      then return db
      else do
        whenVerbose $ liftIO $ putStrLn "Checking for missing files to flush from database..."
        
        -- Check each file in database to see if it still exists
        let files = Map.toList (db ^. dbFiles)
        existingEntries <- forM files $ \(path, entry) -> do
          let fullPath = projectRoot </> path
          fileExists <- safeFileExists readCap fullPath
          
          if fileExists
            then return (path, Just entry)
            else do
              whenVerbose $ liftIO $ putStrLn $ "File no longer exists: " ++ path
              return (path, Nothing)
        
        -- Filter out missing files
        let newFiles = Map.fromList [(path, entry) | (path, Just entry) <- existingEntries]
            missingCount = Map.size (db ^. dbFiles) - Map.size newFiles
            
        -- Rebuild checksums map
        let newChecksums = Map.fromList 
                         $ map (\entry -> (unChecksum (entry ^. entryChecksum), entry ^. entryPath)) 
                         $ Map.elems newFiles
        
        -- Report results
        whenVerbose $ liftIO $ putStrLn $ "Removed " ++ show missingCount ++ " missing files from database"
        
        -- Return updated database
        let db1 = db & dbFiles .~ newFiles
            db2 = db1 & dbChecksums .~ newChecksums
        return db2
  