content: {-# OPTIONS_GHC -fno-warn-tabs #-}
  {-# LANGUAGE
  	OverloadedStrings,
  	ScopedTypeVariables,
  	RecordWildCards,
  	ViewPatterns,
  	LambdaCase,
  	TupleSections,
  	NamedFieldPuns,
  	FlexibleInstances,
  	FlexibleContexts,
  	RankNTypes,
  	MultiParamTypeClasses,
  	FunctionalDependencies,
  	UndecidableInstances,
  	RecursiveDo #-}
  
  module Load14882 (parseIndex, load14882) where
  
  import qualified LaTeXParser as Parser
  import qualified Data.IntMap as IntMap
  import qualified Data.List as List
  import Data.IntMap (IntMap)
  import LaTeXBase
  	( LaTeXUnit(..), TeXArg, ArgKind(..), lookForCommand
  	, mapTeX, mapTeXRaw, concatRaws, texStripInfix, allUnits)
  import Data.Text (Text, replace, isPrefixOf)
  import Data.Text.IO (readFile)
  import Text.Regex (mkRegex, matchRegexAll)
  import qualified Data.Text as Text
  import Control.Monad (forM, when)
  import Prelude hiding (take, (.), takeWhile, (++), lookup, readFile)
  import Data.Char (isAlpha)
  import Control.Arrow (first)
  import Data.Map (Map)
  import Data.Maybe (isJust, fromJust)
  import qualified Data.Map as Map
  import Data.List (unfoldr, (\\), takeWhile)
  import System.Process (readProcess)
  import System.IO.Unsafe (unsafePerformIO)
  import Control.Monad.Fix (MonadFix)
  import Control.Monad.State (MonadState, evalState, get, put, liftM2, modify)
  import Util ((.), (++), mapLast, stripInfix, measure, textStripInfix)
  import RawDocument
  import Sentences (splitIntoSentences, isActualSentence, breakSentence)
  import Document
  
  getCommitUrl :: IO Text
  getCommitUrl = do
  	url <- gitGetRemoteUrl
  	commit <- gitGetCommitRef
  	return $
  		( Text.replace "git@github.com:" "http://github.com/"
  		$ Text.replace ".git" "/commit/" url)
  		++ commit
  
  gitGetRemoteUrl :: IO Text
  gitGetRemoteUrl = do
  	x <- readProcess "git" ["ls-remote", "--get-url"] ""
  	return $ Text.strip $ Text.pack x
  
  gitGetCommitRef :: IO Text
  gitGetCommitRef = do
  	x <- readProcess "git" ["rev-parse", "HEAD"] ""
  	return $ Text.strip $ Text.pack $ x
  
  -- In the LaTeX sources, \definition is often preceded by corresponding \indexdefns.
  -- Since we treat definitions like sections (and generate pages for them), we need
  -- to move the \indexdefns inside (after) the \definition, so that the index entries
  -- don't link to the page for the preceding section.
  
  moveIndexEntriesIntoDefs :: [Text] -> [Text]
  moveIndexEntriesIntoDefs [] = []
  moveIndexEntriesIntoDefs (x:xs)
  	| "\\indexdefn{" `isPrefixOf` x = case moveIndexEntriesIntoDefs xs of
  		[] -> [x]
  		y:ys
  			| "\\definition{" `isPrefixOf` y -> y : x : ys
  			| otherwise -> x : y : ys
  	| otherwise = x : moveIndexEntriesIntoDefs xs
  
  moveIndexEntriesIntoSecs :: [Text] -> [Text]
  moveIndexEntriesIntoSecs = go []
  	where
  		go x [] = x
  		go x (h:t)
  		    | "\\indextext{" `isPrefixOf` h = go (h : x) t
  		    | "\\rSec" `isPrefixOf` h = h : reverse x ++ go [] t
  		    | otherwise = reverse x ++ [h] ++ go [] t
  
  {- The document has a ton of:
  
    \indexlibraryglobal{bla}%
    \begin{itemdecl}
    void bla();
    \end{itemdecl}
  
  To highlight the whole itemdecl, indexItemDecls converts this to:
  
    \begin{indexeditemdecl}{
    \indexlibraryglobal{bla}%
    }
    void bla();
    \end{indexeditemdecl}
  -}
  
  indexCodeEnvs :: [Text] -> [Text] -> [Text]
  indexCodeEnvs envs = go []
  	where
  		go collected [] = collected
  		go collected (x:xs)
  			| "\\index" `isPrefixOf` x = go (collected ++ [x]) xs
  			| [e] <- [e | e <- envs, ("\\begin{" ++ e ++ "}") `isPrefixOf` x] =
  				let (code, _ : rest) = span (not . (("\\end{" ++ e ++ "}") `isPrefixOf`)) xs
  				in (if null collected then
  						["\\begin{" ++ e ++ "}"]
  						++ code
  						++ ["\\end{" ++ e ++ "}"]
  					else
  						["\\begin{indexed" ++ e ++ "}{"] ++ collected ++ ["}"]
  						++ code
  						++ ["\\end{indexed" ++ e ++ "}"])
  				 	++ go [] rest
  			| otherwise = collected ++ (x : go [] xs)
  
  data Numbers = Numbers
  	{ tableNr, figureNr, footnoteRefNr, footnoteNr, itemDeclNr
  	, nextIndexEntryNr, noteNr, exampleNr, nextSentenceNr, formulaNr :: Int }
  
  class AssignNumbers a b | a -> b where
  	assignNumbers :: forall m . (Functor m, MonadFix m, MonadState Numbers m) => Section -> a -> m b
  
  instance AssignNumbers TeXArg TeXArg where
  	assignNumbers s (y, x) = (y, ) . assignNumbers s x
  
  instance AssignNumbers LaTeXUnit LaTeXUnit where
  	assignNumbers s (TeXEnv "itemdecl" [] x) = do
  		n <- get
  		put n{itemDeclNr = itemDeclNr n + 1}
  		TeXEnv "itemdecl" [(FixArg, [TeXRaw $ Text.pack $ show $ itemDeclNr n])] . assignNumbers s x
  	assignNumbers s (TeXEnv "indexeditemdecl" indices x) = do
  		n <- get
  		put n{itemDeclNr = itemDeclNr n + 1}
  		liftM2 (TeXEnv "indexeditemdecl") (assignNumbers s indices) (assignNumbers s x)
  	assignNumbers s (TeXEnv x y z) = liftM2 (TeXEnv x) (assignNumbers s y) (assignNumbers s z)
  	assignNumbers _ (TeXComm "index" ws args) = do
  		n <- get
  		put n{nextIndexEntryNr = nextIndexEntryNr n + 1}
  		return $ TeXComm "index" ws $ (FixArg, [TeXRaw $ Text.pack $ show $ nextIndexEntryNr n]) : args
  	assignNumbers _ (TeXComm "defnx" ws args) = do
  		n <- get
  		put n{nextIndexEntryNr = nextIndexEntryNr n + 1}
  		return $ TeXComm "defnx" ws $ (FixArg, [TeXRaw $ Text.pack $ show $ nextIndexEntryNr n]) : args
  	assignNumbers _ (TeXComm "footnoteref" ws []) = do
  		Numbers{..} <- get
  		put Numbers{footnoteRefNr = footnoteRefNr+1, ..}
  		return $ TeXComm "footnoteref" ws [(FixArg, [TeXRaw $ Text.pack $ show footnoteRefNr])]
  	assignNumbers s (TeXComm x ws args) = TeXComm x ws . assignNumbers s args
  	assignNumbers _ x = return x
  
  instance AssignNumbers a b => AssignNumbers (Cell a) (Cell b) where
  	assignNumbers s x@Cell{..} = do
  		n <- get
  		put n{nextSentenceNr=1}
  		content' <- assignNumbers s content
  		modify $ \m -> m{nextSentenceNr = nextSentenceNr n}
  		return x{content=content'}
  
  instance AssignNumbers a b => AssignNumbers (Row a) (Row b) where
  	assignNumbers s x@Row{..} = do
  		cells' <- assignNumbers s cells
  		return x{cells=cells'}
  
  instance AssignNumbers RawTexPara TeXPara where
  	assignNumbers s (RawTexPara (splitIntoSentences -> x)) = TeXPara . f x
  		where
  			f [] = return []
  			f (h:t) = do
  				h' <- assignNumbers s h
  				let actual = isActualSentence h
  				n <- get
  				put n{nextSentenceNr = nextSentenceNr n + (if actual then 1 else 0)}
  				(Sentence (if actual then Just (nextSentenceNr n) else Nothing) h' :) . f t
  
  assignNonInlineItem :: (MonadState Numbers m, MonadFix m) => Section -> RawItem -> m Item
  assignNonInlineItem s (RawItem label content) = do
  	n <- get
  	put n{nextSentenceNr = 1}
  	Item Nothing (if null label then Nothing else Just label) [] . assignNumbers s content
  
  breakFirstSentence :: [TeXPara] -> (Sentence, [TeXPara])
  breakFirstSentence (TeXPara [x] : z) = (x, z)
  breakFirstSentence (TeXPara (x:y) : z) = (x, TeXPara y : z)
  breakFirstSentence x = error $ "breakFirstSentence: " ++ show x
  
  assignInlineItem :: (MonadState Numbers m, MonadFix m) => Section -> RawItem -> m Item
  assignInlineItem s (RawItem label content) = do
  	n <- get
  	put n{nextSentenceNr = 1}
  	content' <- assignNumbers s content
  	let (Sentence _ x, y) = breakFirstSentence content'
  	return $ Item Nothing (if null label then Nothing else Just label) x y
  
  endsWithFullStop :: [RawElement] -> Bool
  endsWithFullStop = isJust . breakSentence
  
  instance AssignNumbers RawElement Element where
  	assignNumbers section RawFigure{..} = do
  		Numbers{..} <- get
  		put Numbers{figureNr = figureNr+1, ..}
  		return $ FigureElement Figure
  			{ figureNumber  = figureNr
  			, figureName    = rawFigureName
  			, figureAbbr    = "fig:" ++ rawFigureAbbr
  			, figureSvg     = rawFigureSvg
  			, figureSection = section }
  	assignNumbers section RawFormula{..} = do
  	    Numbers{..} <- get
  	    put Numbers{formulaNr = formulaNr + 1, ..}
  	    return $ FormulaElement Formula
  	        { formulaNumber = formulaNr
  	        , formulaAbbr = "eq:" ++ rawFormulaAbbr
  	        , formulaContent = rawFormulaContent
  	        , formulaSection = section }
  	assignNumbers s RawTable{..} = do
  		Numbers{..} <- get
  		put Numbers{tableNr = tableNr+1, ..}
  		tableCaption <- assignNumbers s rawTableCaption
  		tableBody <- assignNumbers s rawTableBody
  		return $ TableElement Table
  			{ tableNumber  = tableNr
  			, columnSpec   = rawColumnSpec
  			, tableAbbr   = rawTableAbbr
  			, tableCaption = tableCaption
  			, tableSection = s
  			, .. }
  	assignNumbers s (RawEnumerated x p) = do
  		origNum <- nextSentenceNr . get
  		let c = length (filter (any (endsWithFullStop . rawTexParaElems) . rawItemContent) p)
  		r <- mapM (if c > 1 then assignNonInlineItem s else assignInlineItem s) p
  		modify $ \y -> y{nextSentenceNr = origNum}
  		return $ Enumerated x r
  	assignNumbers s (RawLatexElement x) = LatexElement . assignNumbers s x
  	assignNumbers s (RawBnf x y) = Bnf x . assignNumbers s y
  	assignNumbers _ (RawTabbing x) = return $ Tabbing x
  	assignNumbers s (RawCodeblock x) = Codeblock . assignNumbers s x
  	assignNumbers s (RawItemdescr x) = Itemdescr . assignNumbers s x
  	assignNumbers s (RawNote label x) = do
  		Numbers{..} <- get
  		put Numbers{noteNr = noteNr+1, ..}
  		x' <- assignNumbers s x
  		return $ NoteElement $ Note noteNr label x'
  	assignNumbers s (RawExample x) = do
  		Numbers{..} <- get
  		put Numbers{exampleNr = exampleNr+1, ..}
  		x' <- assignNumbers s x
  		return $ ExampleElement $ Example exampleNr x'
  
  instance AssignNumbers RawFootnote Footnote where
  	assignNumbers s (RawFootnote t) = do
  		Numbers{..} <- get
  		put Numbers{footnoteNr = footnoteNr+1, nextSentenceNr = 1, ..}
  		t' <- assignNumbers s t
  		return $ Footnote{footnoteNumber=footnoteNr,footnoteContent=t'}
  
  lsectionLevel :: LinearSection -> Int
  lsectionLevel (lsectionKind -> NormalSection l) = l
  lsectionLevel (lsectionKind -> DefinitionSection l) = l
  lsectionLevel _ = 0
  
  paraNumbers :: [Bool] -> [Maybe Int]
  paraNumbers = f 1
  	where
  		f _ [] = []
  		f i (True : x) = Just i : f (i + 1) x
  		f i (False : x) = Nothing : f i x
  
  treeizeChapters :: forall m . (Functor m, MonadFix m, MonadState Numbers m) =>
  	Bool -> Int -> [LinearSection] -> m [Section]
  treeizeChapters _ _ [] = return []
  treeizeChapters annexes secNumber (LinearSection{..} : more) = mdo
  		nums <- get
  		put nums{formulaNr = 1}
  		sectionFootnotes <- assignNumbers newSec lsectionFootnotes
  		let
  			ie = rawIndexEntriesForSec newSec
  			newSec = Section{sectionKind=lsectionKind, secIndexEntries=ie, secIndexEntriesByPath=reverseIndexEntryMap ie, ..}
  		let pn = paraNumbers $ paraNumbered . lsectionParagraphs
  		paragraphs <- forM (zip pn lsectionParagraphs) $ assignNumbers newSec
  		subsections <- treeizeSections 1 chapter [newSec] lsubsections
  		(newSec :) . treeizeChapters annexes' (sectionNumber + 1) more'
  	where
  		sectionNumber = if annexes' /= annexes then 0 else secNumber
  		annexes' = chapter /= NormalChapter
  		parents = []
  		chapter
  			| lsectionKind == InformativeAnnexSection = InformativeAnnex
  			| lsectionKind == NormativeAnnexSection = NormativeAnnex
  			| otherwise = NormalChapter
  		abbreviation = lsectionAbbreviation
  		sectionName = lsectionName
  		(lsubsections, more') = span ((> 0) . lsectionLevel) more
  
  rawIndexEntriesForSec :: Section -> IntMap IndexEntry
  rawIndexEntriesForSec s = IntMap.fromList
  	[(n, e) | e@IndexEntry{indexEntryNr=Just n} <- sectionIndexEntries s]
  
  reverseIndexEntryMap :: IntMap IndexEntry -> Map IndexPath [(Int, IndexEntry)]
  reverseIndexEntryMap m = Map.fromListWith (++) [(indexPath x, [(i, x)])  | (i, x) <- IntMap.assocs m]
  
  assignItemNumbers :: Paragraph -> Paragraph
  assignItemNumbers p
  	| Just n <- paraNumber p = p{ paraElems = fst $ goParas [n, 1] $ paraElems p }
  	| otherwise = p
  	where
  
  		goParas :: [Int] -> [TeXPara] -> ([TeXPara], [Int])
  		goParas nn [] = ([], nn)
  		goParas nn (TeXPara e : pp) = first (TeXPara e' :) (goParas nn' pp)
  			where (e', nn') = goSentences nn e
  
  		goSentences :: [Int] -> [Sentence] -> ([Sentence], [Int])
  		goSentences nn [] = ([], nn)
  		goSentences nn (Sentence m e : ss) = first (Sentence m e' :) (goSentences nn' ss)
  			where (e', nn') = goElems nn e
  
  		goElems :: [Int] -> [Element] -> ([Element], [Int])
  		goElems nn [] = ([], nn)
  		goElems nn (e:ee) = first (e' :) (goElems nn' ee)
  			where (e', nn') = goElem nn e
  
  		goElem :: [Int] -> Element -> (Element, [Int])
  		goElem nn Enumerated{..} = (Enumerated enumCmd items', mapLast (+ length enumItems) nn)
  			where
  				items' = map (\(i, Item{..}) ->
  					Item
  						(Just (map show $ mapLast (+i) nn))
  						itemLabel
  						(fst $ goElems (mapLast (+i) nn ++ [1]) itemInlineContent)
  						(fst $ goParas (mapLast (+i) nn ++ [1]) itemBlockContent)
  					) (zip [0..] enumItems)
  		goElem nn (NoteElement (Note nr label paras)) = (NoteElement (Note nr label paras'), nn')
  			where (paras', nn') = goParas nn paras
  		goElem nn (ExampleElement (Example nr paras)) = (ExampleElement (Example nr paras'), nn')
  			where (paras', nn') = goParas nn paras
  		goElem nn x = (x, nn)
  
  instance AssignNumbers (Maybe Int, RawParagraph) Paragraph where
  	assignNumbers paraSection (paraNumber, RawParagraph{..}) = do
  		nums <- get
  		put nums{nextSentenceNr=if paraNumbered then 1 else nextSentenceNr nums}
  		paraElems <- assignNumbers paraSection rawParaElems
  		when paraNumbered $ modify $ \newnums -> newnums{nextSentenceNr = nextSentenceNr nums}
  		return $ assignItemNumbers Paragraph
  		  { paraInItemdescr = rawParaInItemdescr
  		  , paraSourceLoc = rawParaSourceLoc
  		  , allParaElems = allElements paraElems
  		  , .. }
  
  treeizeSections :: forall m . (Functor m, MonadFix m, MonadState Numbers m) =>
  	Int -> Chapter -> [Section] -> [LinearSection] -> m [Section]
  treeizeSections _ _ _ [] = return []
  treeizeSections sectionNumber chapter parents
  	(s@LinearSection{..} : (span ((> lsectionLevel s) . lsectionLevel) -> (lsubsections, more'))) = mdo
  		let
  		  ie = rawIndexEntriesForSec newSec
  		  newSec = Section
  			{ sectionKind = lsectionKind
  			, secIndexEntries = ie
  			, secIndexEntriesByPath = reverseIndexEntryMap ie
  			, sectionName = lsectionName
  			, abbreviation = lsectionAbbreviation
  			, .. }
  		let pn = paraNumbers $ paraNumbered . lsectionParagraphs
  		nums <- get
  		put nums{noteNr=1, exampleNr=1, itemDeclNr=1}
  		sectionFootnotes <- assignNumbers newSec lsectionFootnotes
  		modify $ \n -> n{nextSentenceNr=1}
  		paragraphs <- forM (zip pn lsectionParagraphs) $ assignNumbers newSec
  		subsections <- treeizeSections 1 chapter (newSec : parents) lsubsections
  		(newSec :) . treeizeSections (sectionNumber + 1) chapter parents more'
  
  instance AssignNumbers a b => AssignNumbers [a] [b] where
  	assignNumbers s = mapM (assignNumbers s)
  
  resolveGrammarterms :: Parser.Macros -> [Text] -> LinearSection -> LinearSection
  resolveGrammarterms macros links LinearSection{..} =
  	LinearSection{lsectionParagraphs = map resolve lsectionParagraphs, ..}
  	where
  		resolveTexPara :: RawTexPara -> RawTexPara
  		resolveTexPara RawTexPara{..} = RawTexPara{rawTexParaElems = map resolveRawElem rawTexParaElems, ..}
  		resolveRawElem :: RawElement -> RawElement
  		resolveRawElem (RawBnf s tex) = RawBnf s (bnfGrammarterms macros links tex)
  		resolveRawElem (RawEnumerated s items) = RawEnumerated s (map resolveItem items)
  		resolveRawElem y = y
  		resolveItem :: RawItem -> RawItem
  		resolveItem (RawItem label content) = RawItem label (map resolveTexPara content)
  		resolve :: RawParagraph -> RawParagraph
  		resolve RawParagraph{..} = RawParagraph{rawParaElems = map resolveTexPara rawParaElems, ..}
  
  bnfGrammarterms :: Parser.Macros -> [Text] -> LaTeX -> LaTeX
  bnfGrammarterms macros links = mapTeX go . mapTeX wordify
  	where
  		wordify :: LaTeXUnit -> Maybe LaTeX
  		wordify (TeXRaw stuff) = Just $ map TeXRaw $ unfoldr f stuff
  			where
  				f s | Text.null s = Nothing
  				f s | isName $ Text.head s = Just $ Text.span isName s
  				f s = Just $ Text.break isName s
  
  				isName c = isAlpha c || c `elem` ['-', '_']
  		wordify _ = Nothing
  
  		go :: LaTeXUnit -> Maybe LaTeX
  		go d@(TeXComm cmd _ _) | cmd `elem` ["tcode", "index", "textnormal", "indexlink", "hiddenindexlink", "indexedspan", "terminal", "literalterminal", "noncxxterminal"] = Just [d]
  		go (TeXRaw name)
  			| name `elem` links = Just $ fst $ RawDocument.doParse macros $ "\\grammarterm{" ++ name ++ "}"
  		go _ = Nothing
  
  parseIndex :: LaTeX -> (IndexPath, Maybe IndexKind)
  parseIndex = go . mapTeXRaw unescapeIndexPath . concatRaws
  	where
  		go (texStripInfix "|seealso" -> Just (x, [TeXBraces y])) = (parseIndexPath x, Just $ See True y)
  		go (texStripInfix "|see " -> Just (x, [TeXBraces y])) = (parseIndexPath x, Just $ See False y)
  		go (texStripInfix "|see" -> Just (x, [TeXBraces y])) = (parseIndexPath x, Just $ See False y)
  		go (texStripInfix "|(" -> Just (t, _)) = (parseIndexPath t, Just IndexOpen)
  		go (texStripInfix "|)" -> Just (t, _)) = (parseIndexPath t, Just IndexClose)
  		go (texStripInfix "|idxbfpage" -> Just (t, _)) = (parseIndexPath t, Just DefinitionIndexEntry)
  		go t = (parseIndexPath t, Nothing)
  
  		unescapeIndexPath :: Text -> LaTeXUnit
  		unescapeIndexPath = TeXRaw
  			. replace "\5" "\""
  
  			. replace "\2" "!"
  			. replace "!" "\1"
  			. replace "\"!" "\2"
  
  			. replace "\4" "@"
  			. replace "@" "\3"
  			. replace "\"@" "\4"
  
  			. replace "\"|" "|"
  			. replace "\"\"" "\5"
  
  			. (!! 10) . iterate (replace "  " " ")
  			. replace "\n" " "
  
  		parseIndexPath :: LaTeX -> IndexPath
  		parseIndexPath (texStripInfix "\1" -> Just (x, y)) = parseIndexPath x ++ parseIndexPath y
  		parseIndexPath (texStripInfix "\3" -> Just (x, y)) = [IndexComponent x y]
  		parseIndexPath t = [IndexComponent [] t]
  
  sectionTexParas :: Section -> [TeXPara]
  sectionTexParas s = (paragraphs s >>= paraElems) ++ (sectionFootnotes s >>= footnoteContent)
  
  sectionTex :: Section -> LaTeX
  sectionTex s = sectionTexParas s >>= texParaTex
  
  sectionIndexEntries :: Section -> [IndexEntry]
  sectionIndexEntries s =
  	[ IndexEntry{indexEntrySection=abbreviation sec, ..}
  	| sec <- sections s
  	, [ (FixArg, [TeXRaw (Text.unpack -> read -> Just -> indexEntryNr)])
  	  , (OptArg, [TeXRaw indexCategory]), (FixArg, (parseIndex -> (indexPath, indexEntryKind)))
  	  ] <- lookForCommand "index" (sectionTex sec)]
  
  sectionLabels :: Section -> [(Text, Section)]
  sectionLabels s =
  	[ (label, sec) | sec <- sections s
  	, [ (FixArg, [TeXRaw label]) ] <- lookForCommand "label" (sectionTex sec)]
  
  toIndex :: IndexEntry -> Index
  toIndex IndexEntry{..} = Map.singleton indexCategory $ go indexPath
  	where
  		go :: [IndexComponent] -> IndexTree
  		go [c] = Map.singleton c (IndexNode [IndexEntry indexEntrySection indexEntryKind indexPath indexEntryNr indexCategory] Map.empty)
  		go (c:cs) = Map.singleton c $ IndexNode [] $ go cs
  		go _ = error "toIndex"
  
  trackPnums :: FilePath -> Text -> Text
  	-- Replaces \pnum with \pnum{file}{line}
  trackPnums file = Text.pack . unlines . map (uncurry f) . zip [1..] . lines . Text.unpack
  	where
  		f :: Integer -> String -> String
  		f lineNr line
  			| Just (pre, post) <- stripInfix "\\pnum" line
  				= pre ++ "\\pnum{" ++ file ++ "}{" ++ show lineNr ++ "}" ++ (if null post then "%" else post)
  			| otherwise = line
  
  getFileList :: IO [FilePath]
  getFileList =
  	(\\ ["front", "back"]) .
  	map (Text.unpack . Text.dropEnd 1 . Text.drop (Text.length pre)) .
  	filter (pre `isPrefixOf`) .
  	Text.lines . readFile "std.tex"
    where pre = "\\include{"
  
  grabBnf :: [String] -> [String]
  grabBnf [] = []
  grabBnf (line : rest)
      | "\\begin{bnf}" `List.isPrefixOf` line =
          let (x, end : more) = break ("\\end{bnf}" `List.isPrefixOf`) rest
          in ["", line] ++ x ++ [end] ++ grabBnf more
      | "\\gramSec" `List.isPrefixOf` line = ["", line] ++ grabBnf rest
      | otherwise = grabBnf rest
  
  generateStdGramExt :: [FilePath] -> IO Text
  generateStdGramExt files =
      Text.pack . unlines . grabBnf . lines . Text.unpack .
      Text.concat . mapM readFile ((++ ".tex") . files)
  
  importExampleFile :: FilePath -> IO Text
  importExampleFile =
      (Text.strip .
       Text.unlines .
       takeWhile (/= "\\end{document}") .
       tail .
       dropWhile (/= "\\begin{document}") .
       Text.lines .) .
      readFile
  
  importExamples :: Text -> Text
  importExamples x = case matchRegexAll r (Text.unpack x) of
          Nothing -> x
          Just (before, _match, after, subs) ->
              Text.pack before ++
              unsafePerformIO (importExampleFile $ "assets/" ++ (subs !! 1) ++ ".tex") ++
              importExamples (Text.pack after)
      where r = mkRegex "\\\\importexample(\\[[0-9a-zA-Z.-]*\\])?{([a-zA-Z0-9_-]*)}"
  
  parseFiles :: Parser.Macros -> IO ([LinearSection], Parser.Macros)
  parseFiles m = do
  	files <- getFileList
  	stdGramExt <- generateStdGramExt files
  	let
  		go [] macros = return ([], macros)
  		go (c:cc) macros = do
  			let p = c ++ ".tex"
  
  			stuff <-
  				importExamples .
  				replace "multicolfloattable" "floattable" .
  				replace "\\indeximpldef{" "\\index[impldefindex]{" .
  				Text.unlines .
  					indexCodeEnvs ["codeblock", "itemdecl"] .
  					moveIndexEntriesIntoSecs .
  					moveIndexEntriesIntoDefs .
  				Text.lines .
  				trackPnums p .
  				replace "\\nodiffref\n\\change" "\n\\pnum\\textbf{Change:}\\space" .
  				replace "\n\\diffref" "\n\\pnum\\nopnumdiffref" .
  					-- Done here because (1) the real \nodiffref is defined with \def in a way
  					-- we don't support yet, and (2) this way a source link is generated for the pnum.
  				readFile p
  
  			let extra = if c /= "grammar" then "" else replace "\\gramSec" "\\rSec1" stdGramExt
  			let (r, macros') = parseFile macros (stuff ++ extra)
  			if length r == 0 then undefined else
  				first (r ++) . go cc (macros ++ macros')
  	
  	bib <- fst . parseFile m .
  	       fst . fromJust .
  	       textStripInfix "\\clearpage" .
  	       ("\\rSec0[bibliography]{Bibliography}\n" ++) .
  	       readFile "back.tex"
  
  	first (++ bib) . go files m
  
  load14882 :: Text -> IO Draft
  load14882 extraMacros = do
  
  	commitUrl <- getCommitUrl
  
  	(macros@Parser.Macros{..}, took) <- measure (loadMacros extraMacros)
  	putStrLn $ "Loaded macros in " ++ show (took * 1000) ++ "ms."
  
  	(secs :: [LinearSection], took2) <- measure $ fst . parseFiles macros
  	putStrLn $ "Parsed LaTeX in " ++ show (took2 * 1000) ++ "ms."
  
  	xrefDelta <- loadXrefDelta
  
  	(r, took3) <- measure $ if length (show secs) == 0 then undefined else do
  		-- force eval before we leave the dir
  		let
  			grammarNames = [n |
  				TeXComm "index" _ [
  					(OptArg, [TeXRaw "grammarindex"]) ,
  					(FixArg, [TeXRaw _
  					 ,TeXComm "textcolor" "" [(FixArg,[TeXRaw "grammar-gray"]),(FixArg,[TeXComm "textsf" _ [(FixArg,[TeXComm "textit" "" [(FixArg,[TeXRaw n])]])]])]
  					 ,TeXRaw "|idxbfpage"]
  					)] <- allUnits secs]
  
  			secs' = map (resolveGrammarterms macros grammarNames) secs
  			chapters = evalState (treeizeChapters False 1 secs') (Numbers 1 1 1 1 0 0 1 1 1 1)
  			allEntries :: [IndexEntry]
  			allEntries = chapters >>= sectionIndexEntries
  			index = mergeIndices $ map toIndex allEntries
  			indexEntryMap = IntMap.fromList [(n, e) | e@IndexEntry{indexEntryNr=Just n} <- allEntries]
  			indexEntriesByPath = reverseIndexEntryMap indexEntryMap
  			labels = Map.fromList $ chapters >>= sectionLabels
  	
  			abbrMap = makeAbbrMap dr
  			dr = Draft{..}
  		return dr
  	
  	putStrLn $ "Processed in " ++ show (took3 * 1000) ++ "ms."
  	return r
  