<commit_before>module Ho.Library(
    loadLibraries,
    createLibrary
    ) where

import Control.Monad(when,foldM)
import Data.List(sort)
import Data.Monoid
import qualified Data.Map as Map
import System.IO
import Char

import GenUtil
import Ho.Build
import Ho.LibraryMap
import Ho.Type
import HsSyn
import Options
import PackedString
import qualified CharIO
import qualified FlagDump as FD
import Util.SHA1(sha1file)
import Version(versionString)



data Library = Library {
    libraryDesc  :: [(PackedString,PackedString)],
    libraryHo    :: Ho,
    libraryFP    :: FilePath,
    librarySHA1  :: CheckSum
    }
type LMap = Map.Map LibraryName Library

-- Load a library in a recursive fashion

libraryDeps :: Library -> [(LibraryName, CheckSum)]
libraryDeps = Map.toList . hoLibraries . libraryHo

loadP :: Maybe CheckSum -> LMap -> LibraryName -> IO LMap
loadP mbcs got name = do
    (n',fp) <- libraryMapFind name
    case Map.lookup n' got of
      Nothing -> do
        pkg <- readLibraryFile n' fp mbcs
        let got' = Map.insert n' pkg got
        foldM (\gm (pn,cs) -> loadP (Just cs) gm pn) got' $ libraryDeps pkg
      Just pkg | mbcs == Nothing                -> return got
               | mbcs == Just (librarySHA1 pkg) -> return got
               | otherwise                      -> fail ("Checksum mismatch for library "++name)

-- load libraries



loadLibraries :: IO Ho
loadLibraries = do
    wdump FD.Progress $ putErrLn $ "Loading libraries: " ++ show (optHls options)
    ps <- foldM (loadP Nothing) Map.empty (optHls options)
    return $ fixupHo $ mconcat (initialHo : map libraryHo (Map.elems ps))
    --return $ mconcat (initialHo : map libraryHo (Map.elems ps))

-- Write a library and mutilate it to fit the description



createLibrary ::
    FilePath
    -> ([Module] -> IO Ho)
    -> IO ()
createLibrary fp wtd = do
    putVerboseLn $ "Creating library from description file: " ++ show fp
    desc <- readDescFile fp
    when verbose2 $ mapM_ print desc
    let field x = lookup x desc
    let jfield x = maybe (error "createLibrary: description lacks required field "++show x) id $ field x
    let mfield x = maybe [] (words . map (\c -> if c == ',' then ' ' else c)) $ field x
    let name  = jfield "name"
        vers  = jfield "version"
        hmods = mfield "hidden-modules"
        emods = mfield "exposed-modules"
    let allmods  = sort $ map Module (emods ++ hmods)
    ho <- wtd (map Module emods)<commit_msg>clean up library code. check to see if we pulled in any odd modules that wern't expected and warn about them<commit_after>module Ho.Library(
    loadLibraries,
    createLibrary
    ) where

import Char
import Control.Monad(when,foldM)
import Data.List(sort)
import Data.Monoid
import System.IO
import qualified Data.Map as Map

import GenUtil
import Ho.Build
import Ho.LibraryMap
import Ho.Type
import HsSyn
import Options
import PackedString
import Util.SHA1(sha1file)
import Util.SetLike
import Version(versionString)
import qualified CharIO
import qualified FlagDump as FD



data Library = Library {
    libraryDesc  :: [(PackedString,PackedString)],
    libraryHo    :: Ho,
    libraryFP    :: FilePath,
    librarySHA1  :: CheckSum
    }
type LMap = Map.Map LibraryName Library

-- Load a library in a recursive fashion

libraryDeps :: Library -> [(LibraryName, CheckSum)]
libraryDeps = Map.toList . hoLibraries . libraryHo

loadP :: Maybe CheckSum -> LMap -> LibraryName -> IO LMap
loadP mbcs got name = do
    (n',fp) <- libraryMapFind name
    case Map.lookup n' got of
      Nothing -> do
        pkg <- readLibraryFile n' fp mbcs
        let got' = Map.insert n' pkg got
        foldM (\gm (pn,cs) -> loadP (Just cs) gm pn) got' $ libraryDeps pkg
      Just pkg | mbcs == Nothing                -> return got
               | mbcs == Just (librarySHA1 pkg) -> return got
               | otherwise                      -> fail ("Checksum mismatch for library "++name)

-- load libraries



loadLibraries :: IO Ho
loadLibraries = do
    wdump FD.Progress $ putErrLn $ "Loading libraries: " ++ show (optHls options)
    ps <- foldM (loadP Nothing) Map.empty (optHls options)
    return $ fixupHo $ mconcat (initialHo : map libraryHo (Map.elems ps))
    --return $ mconcat (initialHo : map libraryHo (Map.elems ps))

-- Write a library and mutilate it to fit the description



createLibrary ::
    FilePath
    -> ([Module] -> IO Ho)
    -> IO ()
createLibrary fp wtd = do
    putVerboseLn $ "Creating library from description file: " ++ show fp
    desc <- readDescFile fp
    when verbose2 $ mapM_ print desc
    let field x = lookup x desc
    let jfield x = maybe (error "createLibrary: description lacks required field "++show x) id $ field x
    let mfield x = maybe [] (words . map (\c -> if c == ',' then ' ' else c)) $ field x
    let name  = jfield "name"
        vers  = jfield "version"
        hmods = snub $ mfield "hidden-modules"
        emods = snub $ mfield "exposed-modules"
    let allmods  = sort $ map Module (emods ++ hmods)
    ho <- wtd (map Module emods)
    let unknownMods = [ m | m <- allmods , m `mnotMember` (hoExports ho)]
    mapM_ ((putStrLn . ("*** Module included in library that is not in export list: " ++)) . show) unknownMods
{'fraction_non_alphanumeric': 0.09702062643239114, 'fraction_numerical': 0.0019098548510313217, 'mean_word_length': 2.7649173256649893, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '859480', 'n_tokens_mistral': 1649, 'n_tokens_neox': 1559, 'n_words': 592}